Changeset 1019 for trunk/autoquest-htmlmonitor/src/main/java/de/ugoe/cs/autoquest/htmlmonitor/HtmlMonitorServlet.java

Timestamp:

12/14/12 15:10:12 (12 years ago)

Author:

pharms

Message:

• changed logfile format to XML
• included logging of GUI structure
• moved robot detection into monitor

File:

• trunk/autoquest-htmlmonitor/src/main/java/de/ugoe/cs/autoquest/htmlmonitor/HtmlMonitorServlet.java (modified) (13 diffs)

trunk/autoquest-htmlmonitor/src/main/java/de/ugoe/cs/autoquest/htmlmonitor/HtmlMonitorServlet.java

@@ -16 +16 @@
 
 import java.io.BufferedReader;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
@@ -37 +38 @@
 import org.mortbay.jetty.servlet.DefaultServlet;
 
+import de.ugoe.cs.util.FileTools;
 import de.ugoe.cs.util.console.Console;
 
@@ -59 +61 @@
     
     /**
+     * <p>
+     * Name and path of the robot filter.
+     * </p>
+     */
+    private static final String ROBOTFILTERFILE = "data/robots/robotfilter.txt";
+
+    /**
+     * <p>
+     * Field that contains a regular expression that matches all robots
+     * contained in {@link #ROBOTFILTERFILE}.
+     * </p>
+     */
+    private String robotRegex = null;
+
+    /**
      * the message listener to forward received messages to.
      */
@@ -72 +89 @@
     HtmlMonitorServlet(HtmlMonitorMessageListener messageListener) {
         this.messageListener = messageListener;
+        try {
+            loadRobotRegex();
+        }
+        catch (Exception e) {
+            Console.println
+                ("robot filtering disabled: could not parse robot filter file " + ROBOTFILTERFILE);
+        }
     }
 
@@ -89 +113 @@
                 InputStream script = this.getClass().getClassLoader().getResourceAsStream
                      ("autoquest-htmlmonitor.js");
-            
+                
                 if (script == null) {
                     Console.printerrln("could not read autoquest-htmlmonitor.js from classpath");
@@ -126 +150 @@
         Object value = null;
         try {
+            //InputStream requestInputStream = dumpStreamContent(request.getInputStream());
             InputStream requestInputStream = request.getInputStream();
-            
+
             value = JSONValue.parseWithException
                 (new InputStreamReader(requestInputStream, "UTF-8"));
@@ -169 +194 @@
                     ("incoming message does not contain valid client infos --> discarding it");
             }
-            else {
+            else if (isRobot(clientInfos.getUserAgent())) {
+                Console.printerrln
+                    ("ignoring robot " + clientInfos.getUserAgent());
+            }
+            else {
+                HtmlPageElement guiStructure = extractHtmlPageElements(message, clientInfos);
                 HtmlEvent[] events = extractHtmlEvents(message, clientInfos);
+                
                 if (events == null) {
                     Console.printerrln
@@ -176 +207 @@
                 }
                 else {
-                    messageListener.handleMessage(clientInfos, events);
+                    messageListener.handleMessage(clientInfos, guiStructure, events);
                 }
             }
@@ -244 +275 @@
             events = new ArrayList<HtmlEvent>();
             
+            HtmlPageElement server = getServerElement(clientInfos);
+            HtmlPageElement page = getPageElementRepresentingWebPage(clientInfos, server);
+
             for (int i = 0; i < eventArray.size(); i++) {
                 Object eventObj = eventArray.get(i);
@@ -277 +311 @@
                                 (eventType, coordinates, key, scrollPosition, selectedValue))
                     {
-                        events.add(new HtmlEvent(clientInfos, time, path, eventType, coordinates,
-                                                 key, scrollPosition, selectedValue));
+                        path = page.getPath() + path;
+                        events.add(new HtmlEvent(clientInfos, time, path, eventType,
+                                                 coordinates, key, scrollPosition, selectedValue));
                     }
                     else {
@@ -294 +329 @@
             return null;
         }
+    }
+
+    /**
+     * <p>
+     * TODO: comment
+     * </p>
+     *
+     * @param message
+     * @param clientInfos
+     * @return
+     */
+    private HtmlPageElement extractHtmlPageElements(JSONObject      object,
+                                                    HtmlClientInfos clientInfos)
+    {
+        HtmlPageElement server = getServerElement(clientInfos);
+        HtmlPageElement page = getPageElementRepresentingWebPage(clientInfos, server);
+
+        JSONObject jsonPageElement = assertValue(object, "guiModel", JSONObject.class);
+        page.addChild(convert(jsonPageElement, page.getPath()));
+        
+        return server;
+    }
+
+    /**
+     * <p>
+     * TODO: comment
+     * </p>
+     *
+     * @param clientInfos
+     * @return
+     */
+    private HtmlPageElement getServerElement(HtmlClientInfos clientInfos) {
+        String id = clientInfos.getUrl().getHost();
+        if (clientInfos.getUrl().getPort() > -1) {
+            id += ":" + clientInfos.getUrl().getPort();
+        }
+        
+        return new HtmlPageElement(null, "server", id, 0);
+    }
+
+    /**
+     * <p>
+     * TODO: comment
+     * </p>
+     *
+     * @param clientInfos
+     */
+    private HtmlPageElement getPageElementRepresentingWebPage(HtmlClientInfos clientInfos,
+                                                              HtmlPageElement server)
+    {
+        String id = clientInfos.getUrl().getPath();
+        
+        if (clientInfos.getUrl().getQuery() != null) {
+            id += "?" + clientInfos.getUrl().getQuery();
+        }
+        
+        HtmlPageElement page =
+            new HtmlPageElement(server.getPath(), "document", id, clientInfos.getTitle(), 0);
+        
+        server.addChild(page);
+        
+        return page;
+    }
+
+    /**
+     * <p>
+     * TODO: comment
+     * </p>
+     *
+     * @param jsonPageElement
+     * @return
+     */
+    private HtmlPageElement convert(JSONObject jsonPageElement, String parentPath) {
+        HtmlPageElement result = null;
+
+        if (jsonPageElement != null) {
+            String tagName = assertValue(jsonPageElement, "tagName", String.class);
+            String id = assertValue(jsonPageElement, "id", String.class);
+            Integer index = assertValue(jsonPageElement, "index", Integer.class);
+
+            result = new HtmlPageElement(parentPath, tagName, id, index);
+
+            JSONArray childElements = assertValue(jsonPageElement, "children", JSONArray.class);
+            
+            if (childElements != null) {
+                Object jsonChild;
+
+                for (int i = 0; i < childElements.size(); i++) {
+                    jsonChild = childElements.get(i);
+                    if (!(jsonChild instanceof JSONObject)) {
+                        Console.printerrln("child " + (i + 1) + " of HTML page element " + tagName +
+                                           " is no valid HTML page element");
+                    }
+                    else {
+                        result.addChild(convert((JSONObject) jsonChild, result.getPath()));
+                    }
+                }
+            }
+            
+        }
+        
+        return result;    
     }
 
@@ -495 +632 @@
     /**
      * <p>
+     * Checks whether an agent is a robot.
+     * </p>
+     * 
+     * @param agent
+     *            agent that is checked
+     * @return true, if the agent is a robot; false otherwise
+     */
+    private boolean isRobot(String agent) {
+        return agent.matches(robotRegex);
+    }
+
+    /**
+     * <p>
+     * Reads {@link #ROBOTFILTERFILE} and creates a regular expression that
+     * matches all the robots defined in the file. The regular expression is
+     * stored in the field {@link #robotRegex}.
+     * </p>
+     * 
+     * @throws IOException
+     *             thrown if there is a problem reading the robot filter
+     * @throws FileNotFoundException
+     *             thrown if the robot filter is not found
+     */
+    private void loadRobotRegex() throws IOException, FileNotFoundException {
+        String[] lines = FileTools.getLinesFromFile(ROBOTFILTERFILE);
+        StringBuilder regex = new StringBuilder();
+        for (int i = 0; i < lines.length; i++) {
+            regex.append("(.*" + lines[i] + ".*)");
+            if (i != lines.length - 1) {
+                regex.append('|');
+            }
+        }
+        robotRegex = regex.toString();
+    }
+
+    /**
+     * <p>
      * convenience method for dumping the content of a stream and returning a new stream
      * containing the same data.
@@ -504 +678 @@
      * @throws IOException if the stream can not be read 
      */
-    /*private InputStream dumpStreamContent(ServletInputStream inputStream) throws IOException {
+/*    private InputStream dumpStreamContent(ServletInputStream inputStream) throws IOException {
         List<Byte> bytes = new ArrayList<Byte>();
         int buf;