1 | // Copyright 2012 Georg-August-Universität Göttingen, Germany
|
---|
2 | //
|
---|
3 | // Licensed under the Apache License, Version 2.0 (the "License");
|
---|
4 | // you may not use this file except in compliance with the License.
|
---|
5 | // You may obtain a copy of the License at
|
---|
6 | //
|
---|
7 | // http://www.apache.org/licenses/LICENSE-2.0
|
---|
8 | //
|
---|
9 | // Unless required by applicable law or agreed to in writing, software
|
---|
10 | // distributed under the License is distributed on an "AS IS" BASIS,
|
---|
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
---|
12 | // See the License for the specific language governing permissions and
|
---|
13 | // limitations under the License.
|
---|
14 |
|
---|
15 | package de.ugoe.cs.autoquest.plugin.php;
|
---|
16 |
|
---|
17 | import java.io.FileNotFoundException;
|
---|
18 | import java.io.IOException;
|
---|
19 | import java.net.URI;
|
---|
20 | import java.net.URISyntaxException;
|
---|
21 | import java.text.ParseException;
|
---|
22 | import java.text.SimpleDateFormat;
|
---|
23 | import java.util.ArrayList;
|
---|
24 | import java.util.Collection;
|
---|
25 | import java.util.HashMap;
|
---|
26 | import java.util.HashSet;
|
---|
27 | import java.util.LinkedList;
|
---|
28 | import java.util.List;
|
---|
29 | import java.util.Map;
|
---|
30 | import java.util.Set;
|
---|
31 | import java.util.logging.Level;
|
---|
32 |
|
---|
33 | import de.ugoe.cs.autoquest.eventcore.Event;
|
---|
34 | import de.ugoe.cs.autoquest.eventcore.IEventTarget;
|
---|
35 | import de.ugoe.cs.autoquest.eventcore.IEventType;
|
---|
36 | import de.ugoe.cs.autoquest.plugin.php.eventcore.PHPEventTarget;
|
---|
37 | import de.ugoe.cs.autoquest.plugin.php.eventcore.PHPEventType;
|
---|
38 | import de.ugoe.cs.autoquest.plugin.php.eventcore.WebRequest;
|
---|
39 | import de.ugoe.cs.util.FileTools;
|
---|
40 | import de.ugoe.cs.util.console.Console;
|
---|
41 |
|
---|
42 | /**
|
---|
43 | * <p>
|
---|
44 | * Provides functionality to parse log files with web request.
|
---|
45 | * </p>
|
---|
46 | *
|
---|
47 | * @author Steffen Herbold
|
---|
48 | * @version 1.0
|
---|
49 | */
|
---|
50 | public class WeblogParser {
|
---|
51 |
|
---|
52 | /**
|
---|
53 | * <p>
|
---|
54 | * Timeout between two sessions in milliseconds.
|
---|
55 | * </p>
|
---|
56 | */
|
---|
57 | private long timeout;
|
---|
58 |
|
---|
59 | /**
|
---|
60 | * <p>
|
---|
61 | * Minimal length of a session. All shorter sessions will be pruned.<br>
|
---|
62 | * Default: 2
|
---|
63 | * </p>
|
---|
64 | */
|
---|
65 | private int minLength = 2;
|
---|
66 |
|
---|
67 | /**
|
---|
68 | * <p>
|
---|
69 | * Maximal length of a session. All longer sessions will be pruned.<br>
|
---|
70 | * Default: 100
|
---|
71 | * </p>
|
---|
72 | */
|
---|
73 | private int maxLength = 100;
|
---|
74 |
|
---|
75 | /**
|
---|
76 | * <p>
|
---|
77 | * URL of the server that generated the log that is currently parser; null
|
---|
78 | * of URL is not available.<br>
|
---|
79 | * Default: null
|
---|
80 | * </p>
|
---|
81 | */
|
---|
82 | private String url = null;
|
---|
83 |
|
---|
84 | /**
|
---|
85 | * <p>
|
---|
86 | * Collection of generated sequences.
|
---|
87 | * </p>
|
---|
88 | */
|
---|
89 | private List<List<Event>> sequences;
|
---|
90 |
|
---|
91 | /**
|
---|
92 | * <p>
|
---|
93 | * List that stores the users (identified through their cookie id) to each
|
---|
94 | * sequence.
|
---|
95 | * </p>
|
---|
96 | */
|
---|
97 | private List<String> users;
|
---|
98 |
|
---|
99 | /**
|
---|
100 | * <p>
|
---|
101 | * List that stores the frequent users (identified through their cookie id)
|
---|
102 | * to each sequence.
|
---|
103 | * </p>
|
---|
104 | */
|
---|
105 | private List<String> frequentUsers;
|
---|
106 |
|
---|
107 | /**
|
---|
108 | * <p>
|
---|
109 | * Sequences for all frequent users.
|
---|
110 | * </p>
|
---|
111 | */
|
---|
112 | private List<Collection<List<Event>>> sequencesFrequentUsers;
|
---|
113 |
|
---|
114 | /**
|
---|
115 | * <p>
|
---|
116 | * Threshold that defines how many sessions of a user are require to deem
|
---|
117 | * the user frequent. Note, that only sessions whose lengths is in range if
|
---|
118 | * {@link #minLength} and {@link #maxLength} are counted.
|
---|
119 | * </p>
|
---|
120 | */
|
---|
121 | private int frequentUsersThreshold = -1;
|
---|
122 |
|
---|
123 | /**
|
---|
124 | * <p>
|
---|
125 | * Name and path of the robot filter.
|
---|
126 | * </p>
|
---|
127 | */
|
---|
128 | private static final String ROBOTFILTERFILE = "misc/robotfilter.txt";
|
---|
129 |
|
---|
130 | /**
|
---|
131 | * <p>
|
---|
132 | * Field that contains a regular expression that matches all robots
|
---|
133 | * contained in {@link #ROBOTFILTERFILE}.
|
---|
134 | * </p>
|
---|
135 | */
|
---|
136 | private String robotRegex = null;
|
---|
137 |
|
---|
138 | /**
|
---|
139 | * <p>
|
---|
140 | * Constructor. Creates a new WeblogParser with a default timeout of
|
---|
141 | * 3,600,000 milliseconds (1 hour).
|
---|
142 | * </p>
|
---|
143 | */
|
---|
144 | public WeblogParser() {
|
---|
145 | this(3600000);
|
---|
146 | }
|
---|
147 |
|
---|
148 | /**
|
---|
149 | * <p>
|
---|
150 | * Constructor. Creates a new WeblogParser.
|
---|
151 | * </p>
|
---|
152 | *
|
---|
153 | * @param timeout
|
---|
154 | * session timeout
|
---|
155 | */
|
---|
156 | public WeblogParser(long timeout) {
|
---|
157 | this.timeout = timeout;
|
---|
158 | }
|
---|
159 |
|
---|
160 | /**
|
---|
161 | * <p>
|
---|
162 | * Returns the generated event sequences.
|
---|
163 | * </p>
|
---|
164 | *
|
---|
165 | * @return generated event sequences
|
---|
166 | */
|
---|
167 | public Collection<List<Event>> getSequences() {
|
---|
168 | return sequences;
|
---|
169 | }
|
---|
170 |
|
---|
171 | /**
|
---|
172 | * <p>
|
---|
173 | * Sets the session timeout.
|
---|
174 | * </p>
|
---|
175 | *
|
---|
176 | * @param timeout
|
---|
177 | * new session timeout
|
---|
178 | */
|
---|
179 | public void setTimeout(long timeout) {
|
---|
180 | this.timeout = timeout;
|
---|
181 | }
|
---|
182 |
|
---|
183 | /**
|
---|
184 | * <p>
|
---|
185 | * Sets the minimal length of a session. All sessions that contain less
|
---|
186 | * events will be pruned.
|
---|
187 | * </p>
|
---|
188 | *
|
---|
189 | * @param minLength
|
---|
190 | * new minimal length
|
---|
191 | */
|
---|
192 | public void setMinLength(int minLength) {
|
---|
193 | this.minLength = minLength;
|
---|
194 | }
|
---|
195 |
|
---|
196 | /**
|
---|
197 | * <p>
|
---|
198 | * Sets the maximal length of a session. All sessions that contain more
|
---|
199 | * events will be pruned.
|
---|
200 | * </p>
|
---|
201 | *
|
---|
202 | * @param maxLength
|
---|
203 | * new maximal length
|
---|
204 | */
|
---|
205 | public void setMaxLength(int maxLength) {
|
---|
206 | this.maxLength = maxLength;
|
---|
207 | }
|
---|
208 |
|
---|
209 | /**
|
---|
210 | * <p>
|
---|
211 | * Sets the URL of the server from which this log was generated. Often
|
---|
212 | * required for replay generation
|
---|
213 | * </p>
|
---|
214 | *
|
---|
215 | * @param url
|
---|
216 | * URL of the server
|
---|
217 | */
|
---|
218 | public void setUrl(String url) {
|
---|
219 | this.url = url;
|
---|
220 | }
|
---|
221 |
|
---|
222 | /**
|
---|
223 | * <p>
|
---|
224 | * Sets the threshold for frequent users.
|
---|
225 | * </p>
|
---|
226 | *
|
---|
227 | * @param threshold
|
---|
228 | * threshold value; if the value is <1, the sessions of the
|
---|
229 | * frequent users will not be determined
|
---|
230 | */
|
---|
231 | public void setFrequentUserThreshold(int threshold) {
|
---|
232 | this.frequentUsersThreshold = threshold;
|
---|
233 | }
|
---|
234 |
|
---|
235 | /**
|
---|
236 | * <p>
|
---|
237 | * Returns the IDs of all frequent users.
|
---|
238 | * </p>
|
---|
239 | *
|
---|
240 | * @return IDs of the frequent users
|
---|
241 | */
|
---|
242 | public List<String> getFrequentUsers() {
|
---|
243 | return frequentUsers;
|
---|
244 | }
|
---|
245 |
|
---|
246 | /**
|
---|
247 | * <p>
|
---|
248 | * Returns the sequences of all frequent users.
|
---|
249 | * </p>
|
---|
250 | * </p>
|
---|
251 | *
|
---|
252 | * @return list of the sequences of all frequent users
|
---|
253 | */
|
---|
254 | public List<Collection<List<Event>>> getFrequentUserSequences() {
|
---|
255 | return sequencesFrequentUsers;
|
---|
256 | }
|
---|
257 |
|
---|
258 | /**
|
---|
259 | * <p>
|
---|
260 | * Parses a web log file.
|
---|
261 | * </p>
|
---|
262 | *
|
---|
263 | * @param filename
|
---|
264 | * name and path of the log file
|
---|
265 | * @throws IOException
|
---|
266 | * thrown if there is a problem with reading the log file
|
---|
267 | * @throws FileNotFoundException
|
---|
268 | * thrown if the log file is not found
|
---|
269 | * @throws ParseException
|
---|
270 | * thrown the date format is invalid
|
---|
271 | */
|
---|
272 | public void parseFile(String filename) throws IOException,
|
---|
273 | FileNotFoundException, ParseException {
|
---|
274 | String[] lines = FileTools.getLinesFromFile(filename);
|
---|
275 |
|
---|
276 | Map<String, List<Integer>> cookieSessionMap = new HashMap<String, List<Integer>>();
|
---|
277 | Map<String, Long> cookieTimestampMap = new HashMap<String, Long>();
|
---|
278 |
|
---|
279 | int lastId = -1;
|
---|
280 |
|
---|
281 | SimpleDateFormat dateFormat = new SimpleDateFormat(
|
---|
282 | "yyyy-MM-dd HH:mm:ss");
|
---|
283 | loadRobotRegex();
|
---|
284 |
|
---|
285 | sequences = new ArrayList<List<Event>>();
|
---|
286 | users = new ArrayList<String>();
|
---|
287 |
|
---|
288 | int lineCounter = 0;
|
---|
289 | for (String line : lines) {
|
---|
290 | lineCounter++;
|
---|
291 | String[] values = line.substring(1, line.length() - 1).split(
|
---|
292 | "\" \"");
|
---|
293 |
|
---|
294 | // use cookie as session identifier
|
---|
295 | int cookieStart = values[0].lastIndexOf('.');
|
---|
296 | String cookie = values[0].substring(cookieStart + 1);
|
---|
297 | String dateString = values[1];
|
---|
298 | long timestamp = dateFormat.parse(dateString).getTime();
|
---|
299 | String uriString = values[2];
|
---|
300 | // String ref = values[3]; // referer is not yet used!
|
---|
301 | String agent;
|
---|
302 | if (values.length > 4) {
|
---|
303 | agent = values[4];
|
---|
304 | } else {
|
---|
305 | agent = "noagent";
|
---|
306 | }
|
---|
307 |
|
---|
308 | List<String> postedVars = new ArrayList<String>();
|
---|
309 | if (values.length == 6) { // post vars found
|
---|
310 | for (String postVar : values[5].trim().split(" ")) {
|
---|
311 | if (!isBrokenVariable(postVar)) {
|
---|
312 | postedVars.add(postVar);
|
---|
313 | }
|
---|
314 | }
|
---|
315 | }
|
---|
316 | if (!isRobot(agent)) {
|
---|
317 | try {
|
---|
318 | URI uri = new URI(uriString);
|
---|
319 | String path = uri.getPath();
|
---|
320 | List<String> getVars = extractGetVarsFromUri(uri);
|
---|
321 |
|
---|
322 | IEventType type = new PHPEventType(path, postedVars, getVars);
|
---|
323 | IEventTarget target = new PHPEventTarget(path);
|
---|
324 | Event event = new Event(type, target);
|
---|
325 | event.addReplayable(new WebRequest(url, path, postedVars, getVars));
|
---|
326 |
|
---|
327 | // find session and add event
|
---|
328 | List<Integer> sessionIds = cookieSessionMap.get(cookie);
|
---|
329 | if (sessionIds == null) {
|
---|
330 | sessionIds = new ArrayList<Integer>();
|
---|
331 | // start new session
|
---|
332 | sessionIds.add(++lastId);
|
---|
333 | cookieSessionMap.put(cookie, sessionIds);
|
---|
334 | sequences.add(new LinkedList<Event>());
|
---|
335 | users.add(cookie);
|
---|
336 | }
|
---|
337 | Integer lastSessionIndex = sessionIds
|
---|
338 | .get(sessionIds.size() - 1);
|
---|
339 | List<Event> lastSession = sequences
|
---|
340 | .get(lastSessionIndex);
|
---|
341 | long lastEventTime = timestamp;
|
---|
342 | if (!lastSession.isEmpty()) {
|
---|
343 | lastEventTime = cookieTimestampMap.get(cookie);
|
---|
344 | }
|
---|
345 | if (timestamp - lastEventTime > timeout) {
|
---|
346 | sessionIds.add(++lastId);
|
---|
347 | List<Event> newSession = new LinkedList<Event>();
|
---|
348 | newSession.add(event);
|
---|
349 | sequences.add(newSession);
|
---|
350 | users.add(cookie);
|
---|
351 | } else {
|
---|
352 | lastSession.add(event);
|
---|
353 | }
|
---|
354 | cookieTimestampMap.put(cookie, timestamp);
|
---|
355 | } catch (URISyntaxException e) {
|
---|
356 | Console.traceln(Level.FINE, "Ignored line " + lineCounter + ": "
|
---|
357 | + e.getMessage());
|
---|
358 | }
|
---|
359 | }
|
---|
360 | }
|
---|
361 | Console.traceln(Level.INFO, "" + sequences.size() + " user sequences found");
|
---|
362 | pruneSequences();
|
---|
363 | Console.traceln(Level.INFO, "" + sequences.size()
|
---|
364 | + " remaining after pruning of sequences shorter than "
|
---|
365 | + minLength);
|
---|
366 | Set<String> uniqueUsers = new HashSet<String>(users);
|
---|
367 | Console.traceln(Level.INFO, "" + uniqueUsers.size() + " unique users");
|
---|
368 | if (frequentUsersThreshold > 0) {
|
---|
369 | generateFrequentUserSequences(uniqueUsers);
|
---|
370 | }
|
---|
371 | }
|
---|
372 |
|
---|
373 | /**
|
---|
374 | * <p>
|
---|
375 | * Generates the frequent user sequences, according to the threshold
|
---|
376 | * {@link #frequentUsersThreshold}.
|
---|
377 | * </p>
|
---|
378 | *
|
---|
379 | * @param uniqueUsers
|
---|
380 | * set with all user IDs
|
---|
381 | */
|
---|
382 | private void generateFrequentUserSequences(Set<String> uniqueUsers) {
|
---|
383 | frequentUsers = new ArrayList<String>();
|
---|
384 | sequencesFrequentUsers = new ArrayList<Collection<List<Event>>>();
|
---|
385 | for (String user : uniqueUsers) {
|
---|
386 | List<String> tmp = new ArrayList<String>();
|
---|
387 | tmp.add(user);
|
---|
388 | List<String> usersCopy = new LinkedList<String>(users);
|
---|
389 | usersCopy.retainAll(tmp);
|
---|
390 | int size = usersCopy.size();
|
---|
391 | if (size >= frequentUsersThreshold) {
|
---|
392 | frequentUsers.add(user);
|
---|
393 | Collection<List<Event>> sequencesUser = new ArrayList<List<Event>>();
|
---|
394 | for (int i = 0; i < sequences.size(); i++) {
|
---|
395 | if (users.get(i).equals(user)) {
|
---|
396 | sequencesUser.add(sequences.get(i));
|
---|
397 | }
|
---|
398 | }
|
---|
399 | sequencesFrequentUsers.add(sequencesUser);
|
---|
400 |
|
---|
401 | }
|
---|
402 | }
|
---|
403 | Console.traceln(Level.INFO, "" + frequentUsers.size() + " users with more than "
|
---|
404 | + frequentUsersThreshold + " sequences");
|
---|
405 | }
|
---|
406 |
|
---|
407 | /**
|
---|
408 | * <p>
|
---|
409 | * Prunes sequences shorter than {@link #minLength} and longer than
|
---|
410 | * {@link #maxLength}.
|
---|
411 | * </p>
|
---|
412 | */
|
---|
413 | private void pruneSequences() {
|
---|
414 | int i = 0;
|
---|
415 | while (i < sequences.size()) {
|
---|
416 | if ((sequences.get(i).size() < minLength)
|
---|
417 | || sequences.get(i).size() > maxLength) {
|
---|
418 | sequences.remove(i);
|
---|
419 | users.remove(i);
|
---|
420 | } else {
|
---|
421 | i++;
|
---|
422 | }
|
---|
423 | }
|
---|
424 |
|
---|
425 | }
|
---|
426 |
|
---|
427 | /**
|
---|
428 | * <p>
|
---|
429 | * Reads {@link #ROBOTFILTERFILE} and creates a regular expression that
|
---|
430 | * matches all the robots defined in the file. The regular expression is
|
---|
431 | * stored in the field {@link #robotRegex}.
|
---|
432 | * </p>
|
---|
433 | *
|
---|
434 | * @throws IOException
|
---|
435 | * thrown if there is a problem reading the robot filter
|
---|
436 | * @throws FileNotFoundException
|
---|
437 | * thrown if the robot filter is not found
|
---|
438 | */
|
---|
439 | private void loadRobotRegex() throws IOException, FileNotFoundException {
|
---|
440 | String[] lines = FileTools.getLinesFromFile(ROBOTFILTERFILE);
|
---|
441 | StringBuilder regex = new StringBuilder();
|
---|
442 | for (int i = 0; i < lines.length; i++) {
|
---|
443 | regex.append("(.*" + lines[i] + ".*)");
|
---|
444 | if (i != lines.length - 1) {
|
---|
445 | regex.append('|');
|
---|
446 | }
|
---|
447 | }
|
---|
448 | robotRegex = regex.toString();
|
---|
449 | }
|
---|
450 |
|
---|
451 | /**
|
---|
452 | * <p>
|
---|
453 | * Checks whether an agent is a robot.
|
---|
454 | * </p>
|
---|
455 | *
|
---|
456 | * @param agent
|
---|
457 | * agent that is checked
|
---|
458 | * @return true, if the agent is a robot; false otherwise
|
---|
459 | */
|
---|
460 | private boolean isRobot(String agent) {
|
---|
461 | return agent.matches(robotRegex);
|
---|
462 | }
|
---|
463 |
|
---|
464 | /**
|
---|
465 | * <p>
|
---|
466 | * Parses the URI and extracts the GET variables that have been passed.
|
---|
467 | * </p>
|
---|
468 | *
|
---|
469 | * @param uri
|
---|
470 | * URI that is parsed
|
---|
471 | * @return a list with all GET variables
|
---|
472 | * @throws URISyntaxException
|
---|
473 | * thrown if one of the variables seems to indicate that the
|
---|
474 | * request is a malicious attack on the web application
|
---|
475 | */
|
---|
476 | private List<String> extractGetVarsFromUri(URI uri)
|
---|
477 | throws URISyntaxException {
|
---|
478 | List<String> getVars = new ArrayList<String>();
|
---|
479 | String query = uri.getQuery();
|
---|
480 | if (query != null) {
|
---|
481 | String[] paramPairs = query.split("&");
|
---|
482 | for (String paramPair : paramPairs) {
|
---|
483 | String[] paramSplit = paramPair.split("=");
|
---|
484 | if (!isBrokenVariable(paramSplit[0])) {
|
---|
485 | for (int i = 1; i < paramSplit.length; i++) {
|
---|
486 | checkForAttack(paramSplit[i]);
|
---|
487 | }
|
---|
488 | getVars.add(paramSplit[0]);
|
---|
489 | }
|
---|
490 | }
|
---|
491 | }
|
---|
492 | return getVars;
|
---|
493 | }
|
---|
494 |
|
---|
495 | /**
|
---|
496 | * <p>
|
---|
497 | * Checks if a variable is broken.Currently, the check rather imprecise and
|
---|
498 | * checks only if the term "and" is part of the variable name.
|
---|
499 | * </p>
|
---|
500 | *
|
---|
501 | * @param var
|
---|
502 | * variable that is checked
|
---|
503 | * @return true if the variable is broken, false otherwise
|
---|
504 | */
|
---|
505 | private boolean isBrokenVariable(String var) {
|
---|
506 | return var.contains("and");
|
---|
507 | }
|
---|
508 |
|
---|
509 | /**
|
---|
510 | * <p>
|
---|
511 | * Checks if the variable name send with a request seems like an attack on the server.
|
---|
512 | * </p>
|
---|
513 | * @param value
|
---|
514 | * @throws URISyntaxException
|
---|
515 | */
|
---|
516 | private void checkForAttack(String value) throws URISyntaxException {
|
---|
517 | if (value.contains("UNION+") || value.contains("SELECT+")) {
|
---|
518 | throw new URISyntaxException(value, "possible injection attack");
|
---|
519 | }
|
---|
520 | }
|
---|
521 | }
|
---|