+ * Threshold that defines how many sessions of a user are require to deem
+ * the user frequent. Note, that only sessions whose lengths is in range if
+ * {@link #minLength} and {@link #maxLength} are counted.
+ *
+ * Sets the URL of the server from which this log was generated. Often
+ * required for replay generation
+ *
+ *
+ * @param url
+ * URL of the server
+ */
+ public void setUrl(String url) {
+ this.url = url;
+ }
+
+ /**
+ *
+ * Sets the threshold for frequent users.
+ *
+ *
+ * @param threshold
+ * threshold value; if the value is <1, the sessions of the
+ * frequent users will not be determined
+ */
+ public void setFrequentUserThreshold(int threshold) {
+ this.frequentUsersThreshold = threshold;
+ }
+
+ /**
+ *
+ * Returns the IDs of all frequent users.
+ *
+ *
+ * @return IDs of the frequent users
+ */
+ public List getFrequentUsers() {
+ return frequentUsers;
+ }
+
+ /**
+ *
+ * Returns the sequences of all frequent users.
+ *
+ *
+ *
+ * @return list of the sequences of all frequent users
+ */
+ public List>> getFrequentUserSequences() {
+ return sequencesFrequentUsers;
+ }
+
+ /**
+ *
+ * Parses a web log file.
+ *
+ *
+ * @param filename
+ * name and path of the log file
+ * @throws IOException
+ * thrown if there is a problem with reading the log file
+ * @throws FileNotFoundException
+ * thrown if the log file is not found
+ * @throws ParseException
+ * thrown the date format is invalid
+ */
+ public void parseFile(String filename) throws IOException,
+ FileNotFoundException, ParseException {
+ String[] lines = FileTools.getLinesFromFile(filename);
+
+ Map> cookieSessionMap = new HashMap>();
+ Map cookieTimestampMap = new HashMap();
+
+ int lastId = -1;
+
+ SimpleDateFormat dateFormat = new SimpleDateFormat(
+ "yyyy-MM-dd HH:mm:ss");
+ loadRobotRegex();
+
+ sequences = new ArrayList>();
+ users = new ArrayList();
+
+ int lineCounter = 0;
+ for (String line : lines) {
+ lineCounter++;
+ String[] values = line.substring(1, line.length() - 1).split(
+ "\" \"");
+
+ // use cookie as session identifier
+ int cookieStart = values[0].lastIndexOf('.');
+ String cookie = values[0].substring(cookieStart + 1);
+ String dateString = values[1];
+ long timestamp = dateFormat.parse(dateString).getTime();
+ String uriString = values[2];
+ // String ref = values[3]; // referer is not yet used!
+ String agent;
+ if (values.length > 4) {
+ agent = values[4];
+ } else {
+ agent = "noagent";
+ }
+
+ List postedVars = new ArrayList();
+ if (values.length == 6) { // post vars found
+ for (String postVar : values[5].trim().split(" ")) {
+ if (!isBrokenVariable(postVar)) {
+ postedVars.add(postVar);
+ }
+ }
+ }
+ if (!isRobot(agent)) {
+ try {
+ URI uri = new URI(uriString);
+ String path = uri.getPath();
+ List getVars = extractGetVarsFromUri(uri);
+
+ IEventType type = new PHPEventType(path, postedVars, getVars);
+ IEventTarget target = new PHPEventTarget(path);
+ Event event = new Event(type, target);
+ event.addReplayable(new WebRequest(url, path, postedVars, getVars));
+
+ // find session and add event
+ List sessionIds = cookieSessionMap.get(cookie);
+ if (sessionIds == null) {
+ sessionIds = new ArrayList();
+ // start new session
+ sessionIds.add(++lastId);
+ cookieSessionMap.put(cookie, sessionIds);
+ sequences.add(new LinkedList());
+ users.add(cookie);
+ }
+ Integer lastSessionIndex = sessionIds
+ .get(sessionIds.size() - 1);
+ List lastSession = sequences
+ .get(lastSessionIndex);
+ long lastEventTime = timestamp;
+ if (!lastSession.isEmpty()) {
+ lastEventTime = cookieTimestampMap.get(cookie);
+ }
+ if (timestamp - lastEventTime > timeout) {
+ sessionIds.add(++lastId);
+ List newSession = new LinkedList();
+ newSession.add(event);
+ sequences.add(newSession);
+ users.add(cookie);
+ } else {
+ lastSession.add(event);
+ }
+ cookieTimestampMap.put(cookie, timestamp);
+ } catch (URISyntaxException e) {
+ Console.traceln(Level.FINE, "Ignored line " + lineCounter + ": "
+ + e.getMessage());
+ }
+ }
+ }
+ Console.traceln(Level.INFO, "" + sequences.size() + " user sequences found");
+ pruneSequences();
+ Console.traceln(Level.INFO, "" + sequences.size()
+ + " remaining after pruning of sequences shorter than "
+ + minLength);
+ Set uniqueUsers = new HashSet(users);
+ Console.traceln(Level.INFO, "" + uniqueUsers.size() + " unique users");
+ if (frequentUsersThreshold > 0) {
+ generateFrequentUserSequences(uniqueUsers);
+ }
+ }
+
+ /**
+ *
+ * Generates the frequent user sequences, according to the threshold
+ * {@link #frequentUsersThreshold}.
+ *
+ *
+ * @param uniqueUsers
+ * set with all user IDs
+ */
+ private void generateFrequentUserSequences(Set uniqueUsers) {
+ frequentUsers = new ArrayList();
+ sequencesFrequentUsers = new ArrayList>>();
+ for (String user : uniqueUsers) {
+ List tmp = new ArrayList();
+ tmp.add(user);
+ List usersCopy = new LinkedList(users);
+ usersCopy.retainAll(tmp);
+ int size = usersCopy.size();
+ if (size >= frequentUsersThreshold) {
+ frequentUsers.add(user);
+ Collection> sequencesUser = new ArrayList>();
+ for (int i = 0; i < sequences.size(); i++) {
+ if (users.get(i).equals(user)) {
+ sequencesUser.add(sequences.get(i));
+ }
+ }
+ sequencesFrequentUsers.add(sequencesUser);
+
+ }
+ }
+ Console.traceln(Level.INFO, "" + frequentUsers.size() + " users with more than "
+ + frequentUsersThreshold + " sequences");
+ }
+
+ /**
+ *
+ * Prunes sequences shorter than {@link #minLength} and longer than
+ * {@link #maxLength}.
+ *
+ * Reads {@link #ROBOTFILTERFILE} and creates a regular expression that
+ * matches all the robots defined in the file. The regular expression is
+ * stored in the field {@link #robotRegex}.
+ *
+ *
+ * @throws IOException
+ * thrown if there is a problem reading the robot filter
+ * @throws FileNotFoundException
+ * thrown if the robot filter is not found
+ */
+ private void loadRobotRegex() throws IOException, FileNotFoundException {
+ String[] lines = FileTools.getLinesFromFile(ROBOTFILTERFILE);
+ StringBuilder regex = new StringBuilder();
+ for (int i = 0; i < lines.length; i++) {
+ regex.append("(.*" + lines[i] + ".*)");
+ if (i != lines.length - 1) {
+ regex.append('|');
+ }
+ }
+ robotRegex = regex.toString();
+ }
+
+ /**
+ *
+ * Checks whether an agent is a robot.
+ *
+ *
+ * @param agent
+ * agent that is checked
+ * @return true, if the agent is a robot; false otherwise
+ */
+ private boolean isRobot(String agent) {
+ return agent.matches(robotRegex);
+ }
+
+ /**
+ *
+ * Parses the URI and extracts the GET variables that have been passed.
+ *
+ *
+ * @param uri
+ * URI that is parsed
+ * @return a list with all GET variables
+ * @throws URISyntaxException
+ * thrown if one of the variables seems to indicate that the
+ * request is a malicious attack on the web application
+ */
+ private List extractGetVarsFromUri(URI uri)
+ throws URISyntaxException {
+ List getVars = new ArrayList();
+ String query = uri.getQuery();
+ if (query != null) {
+ String[] paramPairs = query.split("&");
+ for (String paramPair : paramPairs) {
+ String[] paramSplit = paramPair.split("=");
+ if (!isBrokenVariable(paramSplit[0])) {
+ for (int i = 1; i < paramSplit.length; i++) {
+ checkForAttack(paramSplit[i]);
+ }
+ getVars.add(paramSplit[0]);
+ }
+ }
+ }
+ return getVars;
+ }
+
+ /**
+ *
+ * Checks if a variable is broken.Currently, the check rather imprecise and
+ * checks only if the term "and" is part of the variable name.
+ *
+ *
+ * @param var
+ * variable that is checked
+ * @return true if the variable is broken, false otherwise
+ */
+ private boolean isBrokenVariable(String var) {
+ return var.contains("and");
+ }
+
+ /**
+ *
+ * Checks if the variable name send with a request seems like an attack on the server.
+ *