[54] | 1 | package de.ugoe.cs.eventbench.web;
|
---|
| 2 |
|
---|
| 3 | import java.io.FileNotFoundException;
|
---|
| 4 | import java.io.IOException;
|
---|
[111] | 5 | import java.net.URI;
|
---|
| 6 | import java.net.URISyntaxException;
|
---|
[54] | 7 | import java.text.ParseException;
|
---|
| 8 | import java.text.SimpleDateFormat;
|
---|
| 9 | import java.util.ArrayList;
|
---|
| 10 | import java.util.HashMap;
|
---|
| 11 | import java.util.LinkedList;
|
---|
| 12 | import java.util.List;
|
---|
| 13 | import java.util.Map;
|
---|
| 14 |
|
---|
| 15 | import de.ugoe.cs.eventbench.web.data.WebEvent;
|
---|
[74] | 16 | import de.ugoe.cs.util.FileTools;
|
---|
[68] | 17 | import de.ugoe.cs.util.console.Console;
|
---|
[54] | 18 |
|
---|
[171] | 19 | /**
|
---|
| 20 | * <p>
|
---|
| 21 | * Provides functionality to parse log files with web request.
|
---|
| 22 | * </p>
|
---|
| 23 | *
|
---|
| 24 | * @author Steffen Herbold
|
---|
| 25 | * @version 1.0
|
---|
| 26 | */
|
---|
[54] | 27 | public class WeblogParser {
|
---|
[171] | 28 |
|
---|
| 29 | /**
|
---|
| 30 | * <p>
|
---|
| 31 | * Timeout between two sessions in milliseconds.
|
---|
| 32 | * </p>
|
---|
| 33 | */
|
---|
[54] | 34 | private long timeout;
|
---|
[171] | 35 |
|
---|
| 36 | /**
|
---|
| 37 | * <p>
|
---|
| 38 | * Minimal length of a session. All shorter sessions will be pruned.
|
---|
| 39 | * Default: 2
|
---|
| 40 | * </p>
|
---|
| 41 | */
|
---|
[68] | 42 | private int minLength = 2;
|
---|
[171] | 43 |
|
---|
| 44 | /**
|
---|
| 45 | * <p>
|
---|
| 46 | * Collection of generated sequences.
|
---|
| 47 | * </p>
|
---|
| 48 | */
|
---|
[54] | 49 | private List<List<WebEvent>> sequences;
|
---|
[171] | 50 |
|
---|
| 51 | /**
|
---|
| 52 | * <p>
|
---|
| 53 | * Name and path of the robot filter.
|
---|
| 54 | * </p>
|
---|
| 55 | */
|
---|
[72] | 56 | private static final String ROBOTFILTERFILE = "misc/robotfilter.txt";
|
---|
[171] | 57 |
|
---|
| 58 | /**
|
---|
| 59 | * <p>
|
---|
| 60 | * Field that contains a regular expression that matches all robots
|
---|
| 61 | * contained in {@link #ROBOTFILTERFILE}.
|
---|
| 62 | * </p>
|
---|
| 63 | */
|
---|
| 64 | private String robotRegex = null;
|
---|
| 65 |
|
---|
| 66 | /**
|
---|
| 67 | * <p>
|
---|
| 68 | * Constructor. Creates a new WeblogParser with a default timeout of
|
---|
| 69 | * 3,600,000 milliseconds (1 hour).
|
---|
| 70 | * </p>
|
---|
| 71 | */
|
---|
[54] | 72 | public WeblogParser() {
|
---|
[171] | 73 | this(3600000);
|
---|
[54] | 74 | }
|
---|
[171] | 75 |
|
---|
| 76 | /**
|
---|
| 77 | * <p>
|
---|
| 78 | * Constructor. Creates a new WeblogParser.
|
---|
| 79 | * </p>
|
---|
| 80 | *
|
---|
| 81 | * @param timeout
|
---|
| 82 | * session timeout
|
---|
| 83 | */
|
---|
[54] | 84 | public WeblogParser(long timeout) {
|
---|
| 85 | this.timeout = timeout;
|
---|
| 86 | }
|
---|
[171] | 87 |
|
---|
| 88 | /**
|
---|
| 89 | * <p>
|
---|
| 90 | * Returns the generated event sequences.
|
---|
| 91 | * </p>
|
---|
| 92 | *
|
---|
| 93 | * @return generated event sequences
|
---|
| 94 | */
|
---|
[54] | 95 | public List<List<WebEvent>> getSequences() {
|
---|
[171] | 96 | return sequences;
|
---|
[54] | 97 | }
|
---|
[171] | 98 |
|
---|
| 99 | /**
|
---|
| 100 | * <p>
|
---|
| 101 | * Sets the session timeout.
|
---|
| 102 | * </p>
|
---|
| 103 | *
|
---|
| 104 | * @param timeout
|
---|
| 105 | * new session timeout
|
---|
| 106 | */
|
---|
[68] | 107 | public void setTimeout(long timeout) {
|
---|
| 108 | this.timeout = timeout;
|
---|
| 109 | }
|
---|
[171] | 110 |
|
---|
| 111 | /**
|
---|
| 112 | * <p>
|
---|
| 113 | * Sets the minimal length of a session. All sessions that contain less
|
---|
| 114 | * events will be pruned.
|
---|
| 115 | * </p>
|
---|
| 116 | *
|
---|
| 117 | * @param minLength
|
---|
| 118 | * new minimal length
|
---|
| 119 | */
|
---|
[68] | 120 | public void setMinLength(int minLength) {
|
---|
| 121 | this.minLength = minLength;
|
---|
| 122 | }
|
---|
[171] | 123 |
|
---|
| 124 | /**
|
---|
| 125 | * <p>
|
---|
| 126 | * Parses a web log file.
|
---|
| 127 | * </p>
|
---|
| 128 | *
|
---|
| 129 | * @param filename
|
---|
| 130 | * name and path of the log file
|
---|
| 131 | * @throws IOException
|
---|
| 132 | * thrown if there is a problem with reading the log file
|
---|
| 133 | * @throws FileNotFoundException
|
---|
| 134 | * thrown if the log file is not found
|
---|
| 135 | * @throws ParseException
|
---|
| 136 | * thrown the date format is invalid
|
---|
| 137 | * @throws URISyntaxException
|
---|
| 138 | * thrown if the URI is invalid
|
---|
| 139 | */
|
---|
| 140 | public void parseFile(String filename) throws IOException,
|
---|
| 141 | FileNotFoundException, ParseException, URISyntaxException {
|
---|
[74] | 142 | String[] lines = FileTools.getLinesFromFile(filename);
|
---|
[171] | 143 |
|
---|
[54] | 144 | Map<String, List<Integer>> cookieSessionMap = new HashMap<String, List<Integer>>();
|
---|
| 145 | int lastId = -1;
|
---|
[171] | 146 |
|
---|
| 147 | SimpleDateFormat dateFormat = new SimpleDateFormat(
|
---|
| 148 | "yyyy-MM-dd HH:mm:ss");
|
---|
[72] | 149 | loadRobotRegex();
|
---|
[171] | 150 |
|
---|
[54] | 151 | sequences = new ArrayList<List<WebEvent>>();
|
---|
[171] | 152 |
|
---|
| 153 | for (String line : lines) {
|
---|
| 154 | String[] values = line.substring(1, line.length() - 1).split(
|
---|
| 155 | "\" \"");
|
---|
| 156 |
|
---|
[54] | 157 | // use cookie as session identifier
|
---|
| 158 | int cookieStart = values[0].lastIndexOf('.');
|
---|
[171] | 159 | String cookie = values[0].substring(cookieStart + 1);
|
---|
[72] | 160 | String dateString = values[1];
|
---|
[54] | 161 | long timestamp = dateFormat.parse(dateString).getTime();
|
---|
[111] | 162 | String uriString = values[2];
|
---|
[72] | 163 | // String ref = values[3]; // referer is not yet used!
|
---|
[78] | 164 | String agent;
|
---|
[171] | 165 | if (values.length > 4) {
|
---|
[78] | 166 | agent = values[4];
|
---|
| 167 | } else {
|
---|
| 168 | agent = "noagent";
|
---|
| 169 | }
|
---|
[171] | 170 |
|
---|
[54] | 171 | List<String> postedVars = new ArrayList<String>();
|
---|
[171] | 172 | if (values.length == 6) { // post vars found
|
---|
| 173 | for (String postVar : values[5].trim().split(" ")) {
|
---|
[72] | 174 | postedVars.add(postVar);
|
---|
| 175 | }
|
---|
[54] | 176 | }
|
---|
[171] | 177 | if (!isRobot(agent)) {
|
---|
[111] | 178 | URI uri = new URI(uriString);
|
---|
[171] | 179 |
|
---|
| 180 | String path = uri.getPath();
|
---|
[111] | 181 | List<String> getVars = extractGetVarsFromUri(uri);
|
---|
[171] | 182 |
|
---|
| 183 | WebEvent event = new WebEvent(path, timestamp, postedVars,
|
---|
| 184 | getVars);
|
---|
| 185 |
|
---|
[72] | 186 | // find session and add event
|
---|
| 187 | List<Integer> sessionIds = cookieSessionMap.get(cookie);
|
---|
[171] | 188 | if (sessionIds == null) {
|
---|
[72] | 189 | sessionIds = new ArrayList<Integer>();
|
---|
| 190 | // start new session
|
---|
| 191 | sessionIds.add(++lastId);
|
---|
| 192 | cookieSessionMap.put(cookie, sessionIds);
|
---|
| 193 | sequences.add(new LinkedList<WebEvent>());
|
---|
[171] | 194 | }
|
---|
| 195 | Integer lastSessionIndex = sessionIds
|
---|
| 196 | .get(sessionIds.size() - 1);
|
---|
[72] | 197 | List<WebEvent> lastSession = sequences.get(lastSessionIndex);
|
---|
| 198 | long lastEventTime = timestamp;
|
---|
[171] | 199 | if (!lastSession.isEmpty()) {
|
---|
| 200 | lastEventTime = lastSession.get(lastSession.size() - 1)
|
---|
| 201 | .getTimestamp();
|
---|
[72] | 202 | }
|
---|
[171] | 203 | if (timestamp - lastEventTime > timeout) {
|
---|
[72] | 204 | sessionIds.add(++lastId);
|
---|
| 205 | List<WebEvent> newSession = new LinkedList<WebEvent>();
|
---|
| 206 | newSession.add(event);
|
---|
| 207 | sequences.add(newSession);
|
---|
| 208 | } else {
|
---|
| 209 | lastSession.add(event);
|
---|
| 210 | }
|
---|
[54] | 211 | }
|
---|
| 212 | }
|
---|
[74] | 213 | pruneShortSequences();
|
---|
| 214 | }
|
---|
| 215 |
|
---|
[171] | 216 | /**
|
---|
| 217 | * <p>
|
---|
| 218 | * Prunes sequences shorter than {@link #minLength}.
|
---|
| 219 | * </p>
|
---|
| 220 | */
|
---|
[74] | 221 | private void pruneShortSequences() {
|
---|
[171] | 222 | Console.traceln("" + sequences.size() + " user sequences found");
|
---|
[68] | 223 | // prune sequences shorter than min-length
|
---|
[171] | 224 | int i = 0;
|
---|
| 225 | while (i < sequences.size()) {
|
---|
| 226 | if (sequences.get(i).size() < minLength) {
|
---|
[68] | 227 | sequences.remove(i);
|
---|
[72] | 228 | } else {
|
---|
| 229 | i++;
|
---|
[68] | 230 | }
|
---|
| 231 | }
|
---|
[171] | 232 | Console.traceln("" + sequences.size()
|
---|
| 233 | + " remaining after pruning of sequences shorter than "
|
---|
| 234 | + minLength);
|
---|
[54] | 235 | }
|
---|
[171] | 236 |
|
---|
| 237 | /**
|
---|
| 238 | * <p>
|
---|
| 239 | * Reads {@link #ROBOTFILTERFILE} and creates a regular expression that
|
---|
| 240 | * matches all the robots defined in the file. The regular expression is
|
---|
| 241 | * stored in the field {@link #robotRegex}.
|
---|
| 242 | * </p>
|
---|
| 243 | *
|
---|
| 244 | * @throws IOException
|
---|
| 245 | * thrown if there is a problem reading the robot filter
|
---|
| 246 | * @throws FileNotFoundException
|
---|
| 247 | * thrown if the robot filter is not found
|
---|
| 248 | */
|
---|
[72] | 249 | private void loadRobotRegex() throws IOException, FileNotFoundException {
|
---|
[74] | 250 | String[] lines = FileTools.getLinesFromFile(ROBOTFILTERFILE);
|
---|
[72] | 251 | StringBuilder regex = new StringBuilder();
|
---|
[171] | 252 | for (int i = 0; i < lines.length; i++) {
|
---|
| 253 | regex.append("(.*" + lines[i] + ".*)");
|
---|
| 254 | if (i != lines.length - 1) {
|
---|
[176] | 255 | regex.append('|');
|
---|
[72] | 256 | }
|
---|
| 257 | }
|
---|
| 258 | robotRegex = regex.toString();
|
---|
| 259 | }
|
---|
[171] | 260 |
|
---|
| 261 | /**
|
---|
| 262 | * <p>
|
---|
| 263 | * Checks whether an agent is a robot.
|
---|
| 264 | * </p>
|
---|
| 265 | *
|
---|
| 266 | * @param agent
|
---|
| 267 | * agent that is checked
|
---|
| 268 | * @return true, if the agent is a robot; false otherwise
|
---|
| 269 | */
|
---|
[72] | 270 | private boolean isRobot(String agent) {
|
---|
| 271 | return agent.matches(robotRegex);
|
---|
| 272 | }
|
---|
[171] | 273 |
|
---|
| 274 | /**
|
---|
| 275 | * <p>
|
---|
| 276 | * Parses the URI and extracts the GET variables that have been passed.
|
---|
| 277 | * </p>
|
---|
| 278 | *
|
---|
| 279 | * @param uri
|
---|
| 280 | * URI that is parsed
|
---|
| 281 | * @return a list with all GET variables
|
---|
| 282 | */
|
---|
[111] | 283 | private List<String> extractGetVarsFromUri(URI uri) {
|
---|
| 284 | List<String> getVars = new ArrayList<String>();
|
---|
| 285 | String query = uri.getQuery();
|
---|
[171] | 286 | if (query != null) {
|
---|
[111] | 287 | String[] paramPairs = query.split("&");
|
---|
[171] | 288 | for (String paramPair : paramPairs) {
|
---|
[111] | 289 | String[] paramSplit = paramPair.split("=");
|
---|
| 290 | getVars.add(paramSplit[0]);
|
---|
| 291 | }
|
---|
| 292 | }
|
---|
| 293 | return getVars;
|
---|
| 294 | }
|
---|
[54] | 295 | }
|
---|