[54] | 1 | package de.ugoe.cs.eventbench.web;
|
---|
| 2 |
|
---|
| 3 | import java.io.FileNotFoundException;
|
---|
| 4 | import java.io.IOException;
|
---|
| 5 | import java.text.ParseException;
|
---|
| 6 | import java.text.SimpleDateFormat;
|
---|
| 7 | import java.util.ArrayList;
|
---|
| 8 | import java.util.HashMap;
|
---|
| 9 | import java.util.LinkedList;
|
---|
| 10 | import java.util.List;
|
---|
| 11 | import java.util.Map;
|
---|
| 12 |
|
---|
| 13 | import de.ugoe.cs.eventbench.web.data.WebEvent;
|
---|
[74] | 14 | import de.ugoe.cs.util.FileTools;
|
---|
[68] | 15 | import de.ugoe.cs.util.console.Console;
|
---|
[54] | 16 |
|
---|
| 17 | public class WeblogParser {
|
---|
| 18 |
|
---|
| 19 | private long timeout;
|
---|
| 20 |
|
---|
[68] | 21 | private int minLength = 2;
|
---|
| 22 |
|
---|
[54] | 23 | private List<List<WebEvent>> sequences;
|
---|
| 24 |
|
---|
[72] | 25 | private static final String ROBOTFILTERFILE = "misc/robotfilter.txt";
|
---|
| 26 |
|
---|
| 27 | private String robotRegex = ".*";
|
---|
| 28 |
|
---|
[54] | 29 | public WeblogParser() {
|
---|
| 30 | timeout = 3600000; // 1 hour session-timeout as default
|
---|
| 31 | }
|
---|
| 32 |
|
---|
| 33 | public WeblogParser(long timeout) {
|
---|
| 34 | this.timeout = timeout;
|
---|
| 35 | }
|
---|
| 36 |
|
---|
| 37 | public List<List<WebEvent>> getSequences() {
|
---|
| 38 | return sequences;
|
---|
| 39 | }
|
---|
| 40 |
|
---|
[68] | 41 | public void setTimeout(long timeout) {
|
---|
| 42 | this.timeout = timeout;
|
---|
| 43 | }
|
---|
| 44 |
|
---|
| 45 | public void setMinLength(int minLength) {
|
---|
| 46 | this.minLength = minLength;
|
---|
| 47 | }
|
---|
| 48 |
|
---|
[54] | 49 | public void parseFile(String filename) throws IOException, FileNotFoundException, ParseException {
|
---|
[74] | 50 | String[] lines = FileTools.getLinesFromFile(filename);
|
---|
[54] | 51 |
|
---|
| 52 | Map<String, List<Integer>> cookieSessionMap = new HashMap<String, List<Integer>>();
|
---|
| 53 | int lastId = -1;
|
---|
| 54 |
|
---|
| 55 | SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
---|
[72] | 56 | loadRobotRegex();
|
---|
[54] | 57 |
|
---|
| 58 | sequences = new ArrayList<List<WebEvent>>();
|
---|
| 59 |
|
---|
| 60 | for( String line : lines ) {
|
---|
[72] | 61 | String[] values = line.substring(1, line.length()-1).split("\" \"");
|
---|
[54] | 62 |
|
---|
| 63 | // use cookie as session identifier
|
---|
| 64 | int cookieStart = values[0].lastIndexOf('.');
|
---|
| 65 | String cookie = values[0].substring(cookieStart+1);
|
---|
[72] | 66 | String dateString = values[1];
|
---|
[54] | 67 | long timestamp = dateFormat.parse(dateString).getTime();
|
---|
[72] | 68 | String uri = values[2];
|
---|
| 69 | // String ref = values[3]; // referer is not yet used!
|
---|
[78] | 70 | String agent;
|
---|
| 71 | if( values.length>4 ) {
|
---|
| 72 | agent = values[4];
|
---|
| 73 | } else {
|
---|
| 74 | agent = "noagent";
|
---|
| 75 | }
|
---|
| 76 |
|
---|
[54] | 77 | List<String> postedVars = new ArrayList<String>();
|
---|
[72] | 78 | if( values.length==6 ) { // post vars found
|
---|
| 79 | for( String postVar : values[5].split(" ") ) {
|
---|
| 80 | postedVars.add(postVar);
|
---|
| 81 | }
|
---|
[54] | 82 | }
|
---|
[72] | 83 | if( !isRobot(agent) ) {
|
---|
| 84 | WebEvent event = new WebEvent(uri, timestamp, postedVars);
|
---|
[54] | 85 |
|
---|
[72] | 86 | // find session and add event
|
---|
| 87 | List<Integer> sessionIds = cookieSessionMap.get(cookie);
|
---|
| 88 | if( sessionIds==null ) {
|
---|
| 89 | sessionIds = new ArrayList<Integer>();
|
---|
| 90 | // start new session
|
---|
| 91 | sessionIds.add(++lastId);
|
---|
| 92 | cookieSessionMap.put(cookie, sessionIds);
|
---|
| 93 | sequences.add(new LinkedList<WebEvent>());
|
---|
| 94 | }
|
---|
| 95 | Integer lastSessionIndex = sessionIds.get(sessionIds.size()-1);
|
---|
| 96 | List<WebEvent> lastSession = sequences.get(lastSessionIndex);
|
---|
| 97 | long lastEventTime = timestamp;
|
---|
| 98 | if( !lastSession.isEmpty() ) {
|
---|
| 99 | lastEventTime = lastSession.get(lastSession.size()-1).getTimestamp();
|
---|
| 100 | }
|
---|
| 101 | if( timestamp-lastEventTime>timeout ) {
|
---|
| 102 | sessionIds.add(++lastId);
|
---|
| 103 | List<WebEvent> newSession = new LinkedList<WebEvent>();
|
---|
| 104 | newSession.add(event);
|
---|
| 105 | sequences.add(newSession);
|
---|
| 106 | } else {
|
---|
| 107 | lastSession.add(event);
|
---|
| 108 | }
|
---|
[54] | 109 | }
|
---|
| 110 | }
|
---|
[74] | 111 | pruneShortSequences();
|
---|
| 112 | }
|
---|
| 113 |
|
---|
| 114 | private void pruneShortSequences() {
|
---|
[68] | 115 | Console.traceln(""+sequences.size()+ " user sequences found");
|
---|
| 116 | // prune sequences shorter than min-length
|
---|
[72] | 117 | int i=0;
|
---|
| 118 | while( i<sequences.size() ) {
|
---|
[68] | 119 | if( sequences.get(i).size()<minLength ) {
|
---|
| 120 | sequences.remove(i);
|
---|
[72] | 121 | } else {
|
---|
| 122 | i++;
|
---|
[68] | 123 | }
|
---|
| 124 | }
|
---|
| 125 | Console.traceln(""+sequences.size()+ " remaining after pruning of sequences shorter than " + minLength);
|
---|
[54] | 126 | }
|
---|
[72] | 127 |
|
---|
| 128 | private void loadRobotRegex() throws IOException, FileNotFoundException {
|
---|
[74] | 129 | String[] lines = FileTools.getLinesFromFile(ROBOTFILTERFILE);
|
---|
[72] | 130 | StringBuilder regex = new StringBuilder();
|
---|
| 131 | for( int i=0; i<lines.length; i++ ) {
|
---|
| 132 | regex.append("(.*"+lines[i]+".*)");
|
---|
| 133 | if( i!=lines.length-1 ) {
|
---|
| 134 | regex.append("|");
|
---|
| 135 | }
|
---|
| 136 | }
|
---|
| 137 | robotRegex = regex.toString();
|
---|
| 138 | }
|
---|
| 139 |
|
---|
| 140 | private boolean isRobot(String agent) {
|
---|
| 141 | return agent.matches(robotRegex);
|
---|
| 142 | }
|
---|
[54] | 143 | }
|
---|