Changeset 1019 for trunk

Timestamp:

12/14/12 15:10:12 (12 years ago)

Author:

pharms

Message:

• changed logfile format to XML
• included logging of GUI structure
• moved robot detection into monitor

Location:

trunk/autoquest-htmlmonitor

Files:

• data (added)
• data/robots (added)
• data/robots/robotfilter.txt (added)
• src/main/assembly/bin.xml (modified) (1 diff)
• src/main/java/de/ugoe/cs/autoquest/htmlmonitor/HtmlClientInfos.java (modified) (1 diff)
• src/main/java/de/ugoe/cs/autoquest/htmlmonitor/HtmlMonitorLogManager.java (modified) (3 diffs)
• src/main/java/de/ugoe/cs/autoquest/htmlmonitor/HtmlMonitorMessageListener.java (modified) (1 diff)
• src/main/java/de/ugoe/cs/autoquest/htmlmonitor/HtmlMonitorOutputWriter.java (modified) (6 diffs)
• src/main/java/de/ugoe/cs/autoquest/htmlmonitor/HtmlMonitorServlet.java (modified) (13 diffs)
• src/main/java/de/ugoe/cs/autoquest/htmlmonitor/HtmlPageElement.java (added)

trunk/autoquest-htmlmonitor/src/main/assembly/bin.xml

@@ -34 +34 @@
       </includes>
     </fileSet>
+    <fileSet>
+      <directory>data</directory>
+      <outputDirectory>data</outputDirectory>
+      <fileMode>664</fileMode>
+      <includes>
+        <include>**/*</include>
+      </includes>
+    </fileSet>
   </fileSets>
 </assembly>

trunk/autoquest-htmlmonitor/src/main/java/de/ugoe/cs/autoquest/htmlmonitor/HtmlClientInfos.java

@@ -57 +57 @@
      * @param title     title of the web site shown by the browser of the client
      */
-    public HtmlClientInfos(String clientId, String userAgent, URL url, String title) {
+    HtmlClientInfos(String clientId, String userAgent, URL url, String title) {
         this.clientId = clientId;
         this.userAgent = userAgent;

trunk/autoquest-htmlmonitor/src/main/java/de/ugoe/cs/autoquest/htmlmonitor/HtmlMonitorLogManager.java

@@ -115 +115 @@
      */
     @Override
-    public void handleMessage(HtmlClientInfos clientInfos, HtmlEvent[] events) {
+    public void handleMessage(HtmlClientInfos clientInfos,
+                              HtmlPageElement guiStructure,
+                              HtmlEvent[]     events)
+    {
         HtmlMonitorOutputWriter writer = writers.get(clientInfos.getClientId());
         
@@ -132 +135 @@
             }
 
-            writer.handleMessage(clientInfos, events);
+            writer.handleMessage(clientInfos, guiStructure, events);
         }
         catch (Exception e) {
@@ -144 +147 @@
             if (writer != null) {
                 try {
-                    writer.handleMessage(clientInfos, events);
+                    writer.handleMessage(clientInfos, guiStructure, events);
                 }
                 catch (Exception e1) {

trunk/autoquest-htmlmonitor/src/main/java/de/ugoe/cs/autoquest/htmlmonitor/HtmlMonitorMessageListener.java

@@ -32 +32 @@
      *
      * @param clientInfos infos about the client that send the events
+     * @param TODO comment
      * @param events      the received events
      */
-    void handleMessage(HtmlClientInfos clientInfos, HtmlEvent[] events);
+    void handleMessage(HtmlClientInfos clientInfos,
+                       HtmlPageElement guiStructure,
+                       HtmlEvent[]     events);
 
 }

trunk/autoquest-htmlmonitor/src/main/java/de/ugoe/cs/autoquest/htmlmonitor/HtmlMonitorOutputWriter.java

@@ -22 +22 @@
 import java.text.DecimalFormat;
 
+import de.ugoe.cs.util.StringTools;
 import de.ugoe.cs.util.console.Console;
 
@@ -177 +178 @@
      */
     @Override
-    public void handleMessage(HtmlClientInfos clientInfos, HtmlEvent[] events) {
+    public void handleMessage(HtmlClientInfos clientInfos,
+                              HtmlPageElement guiStructure,
+                              HtmlEvent[]     events)
+    {
         if (outputWriter == null) {
             throw new IllegalStateException("not initialized. Call init() first");
         }
         
+        if (guiStructure != null) {
+            dumpGuiStructure(guiStructure);
+        }
+        
         for (HtmlEvent event : events) {
             dumpEvent(event);
@@ -200 +208 @@
     /**
      * <p>
+     * TODO: comment
+     * </p>
+     *
+     * @param guiStructure
+     */
+    private void dumpGuiStructure(HtmlPageElement guiStructure) {
+        outputWriter.print("<component path=\"");
+        outputWriter.print(guiStructure.getPath());
+        outputWriter.println("\">");
+        
+        dumpParam("class", guiStructure.getTagName());
+        dumpParam("htmlId", guiStructure.getId());
+        dumpParam("title", guiStructure.getTitle());
+        dumpParam("index", guiStructure.getIndex());
+        dumpParam("parent", guiStructure.getParentPath());
+        
+        outputWriter.println("</component>");
+        
+        if (guiStructure.getChildren() != null) {
+            for (HtmlPageElement child : guiStructure.getChildren()) {
+                dumpGuiStructure(child);
+            }
+        }
+    }
+
+    /**
+     * <p>
      * formats a received event and writes it to the log file. One event results in one line
      * in the log file containing all infos of the event.
@@ -207 +242 @@
      */
     private void dumpEvent(HtmlEvent event) {
-        dumpString(event.getClientInfos().getClientId());
-        outputWriter.print(' ');
-        dumpString(event.getTime().toString());
-        outputWriter.print(' ');
-        dumpString(event.getClientInfos().getTitle());
-        outputWriter.print(' ');
-        dumpString(event.getClientInfos().getUrl().toString());
-        outputWriter.print(' ');
-        dumpString(event.getClientInfos().getUserAgent());
-        outputWriter.print(' ');
-        dumpString(event.getEventType());
-        outputWriter.print(' ');
-        dumpString(event.getPath());
-
+        outputWriter.print("<event type=\"");
+        outputWriter.print(event.getEventType());
+        outputWriter.println("\">");
+        
         if (event.getCoordinates() != null) {
-            outputWriter.print(' ');
-            
-            StringBuffer value = new StringBuffer();
-            for (int i = 0; i < event.getCoordinates().length; i++) {
-                if (i > 0) {
-                    value.append(',');
-                }
-                value.append(event.getCoordinates()[i]);
-            }
-            
-            dumpString(value.toString());
-        }
-
-        if (event.getKey() != null) {
-            outputWriter.print(' ');
-            dumpString(event.getKey().toString());
-        }
+            dumpParam("X", event.getCoordinates()[0]);
+            dumpParam("Y", event.getCoordinates()[1]);
+        }
+
+        dumpParam("key", event.getKey());
             
         if (event.getScrollPosition() != null) {
-            outputWriter.print(' ');
-            
-            StringBuffer value = new StringBuffer();
-            for (int i = 0; i < event.getScrollPosition().length; i++) {
-                if (i > 0) {
-                    value.append(',');
-                }
-                value.append(event.getScrollPosition()[i]);
-            }
-            
-            dumpString(value.toString());
-        }
-
-        if (event.getSelectedValue() != null) {
-            outputWriter.print(' ');
-            dumpString(event.getSelectedValue());
-        }
-            
-        outputWriter.println();
-    }
-
-    /**
-     * <p>
-     * convenience method to dump a string with trailing and leading " as well as replaced
-     * backslashes, ", and newlines
-     * </p>
-     *
-     * @param clientId2
-     */
-    private void dumpString(String str) {
-        String value = str;
-        value = value.replaceAll("\\\\", "\\\\\\\\");
-        value = value.replaceAll("\\\"", "\\\\\\\"");
-        value = value.replaceAll("\n", "\\\\n");
-        
-        outputWriter.print('"');
-        outputWriter.print(value);
-        outputWriter.print('"');
+            dumpParam("scrollX", event.getScrollPosition()[0]);
+            dumpParam("scrollY", event.getScrollPosition()[1]);
+        }
+
+        dumpParam("selectedValue", event.getSelectedValue());
+        dumpParam("target", event.getPath());
+        dumpParam("timestamp", event.getTime());
+        
+        outputWriter.println("</event>");
+    }
+
+    /**
+     * <p>
+     * TODO: comment
+     * </p>
+     *
+     * @param string
+     * @param integer
+     */
+    private void dumpParam(String name, Object value) {
+        if (value == null) {
+            return;
+        }
+        
+        String val;
+        
+        if (value instanceof String) {
+            val = (String) value;
+        }
+        else {
+            val = value.toString();
+        }
+        
+        outputWriter.print(" <param name=\"");
+        outputWriter.print(name);
+        outputWriter.print("\" value=\"");
+        outputWriter.print(StringTools.xmlEntityReplacement(val));
+        outputWriter.println("\"/>");
     }
 
@@ -332 +345 @@
         FileOutputStream fis = new FileOutputStream(logFile);
         outputWriter = new PrintWriter(new OutputStreamWriter(fis, "UTF-8"));
+        outputWriter.println("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
+        outputWriter.println("<session>");
     }
 
@@ -341 +356 @@
     private void closeLogWriter() {
         if (outputWriter != null) {
+            outputWriter.println("</session>");
             outputWriter.flush();
             outputWriter.close();

trunk/autoquest-htmlmonitor/src/main/java/de/ugoe/cs/autoquest/htmlmonitor/HtmlMonitorServlet.java

@@ -16 +16 @@
 
 import java.io.BufferedReader;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
@@ -37 +38 @@
 import org.mortbay.jetty.servlet.DefaultServlet;
 
+import de.ugoe.cs.util.FileTools;
 import de.ugoe.cs.util.console.Console;
 
@@ -59 +61 @@
     
     /**
+     * <p>
+     * Name and path of the robot filter.
+     * </p>
+     */
+    private static final String ROBOTFILTERFILE = "data/robots/robotfilter.txt";
+
+    /**
+     * <p>
+     * Field that contains a regular expression that matches all robots
+     * contained in {@link #ROBOTFILTERFILE}.
+     * </p>
+     */
+    private String robotRegex = null;
+
+    /**
      * the message listener to forward received messages to.
      */
@@ -72 +89 @@
     HtmlMonitorServlet(HtmlMonitorMessageListener messageListener) {
         this.messageListener = messageListener;
+        try {
+            loadRobotRegex();
+        }
+        catch (Exception e) {
+            Console.println
+                ("robot filtering disabled: could not parse robot filter file " + ROBOTFILTERFILE);
+        }
     }
 
@@ -89 +113 @@
                 InputStream script = this.getClass().getClassLoader().getResourceAsStream
                      ("autoquest-htmlmonitor.js");
-            
+                
                 if (script == null) {
                     Console.printerrln("could not read autoquest-htmlmonitor.js from classpath");
@@ -126 +150 @@
         Object value = null;
         try {
+            //InputStream requestInputStream = dumpStreamContent(request.getInputStream());
             InputStream requestInputStream = request.getInputStream();
-            
+
             value = JSONValue.parseWithException
                 (new InputStreamReader(requestInputStream, "UTF-8"));
@@ -169 +194 @@
                     ("incoming message does not contain valid client infos --> discarding it");
             }
-            else {
+            else if (isRobot(clientInfos.getUserAgent())) {
+                Console.printerrln
+                    ("ignoring robot " + clientInfos.getUserAgent());
+            }
+            else {
+                HtmlPageElement guiStructure = extractHtmlPageElements(message, clientInfos);
                 HtmlEvent[] events = extractHtmlEvents(message, clientInfos);
+                
                 if (events == null) {
                     Console.printerrln
@@ -176 +207 @@
                 }
                 else {
-                    messageListener.handleMessage(clientInfos, events);
+                    messageListener.handleMessage(clientInfos, guiStructure, events);
                 }
             }
@@ -244 +275 @@
             events = new ArrayList<HtmlEvent>();
             
+            HtmlPageElement server = getServerElement(clientInfos);
+            HtmlPageElement page = getPageElementRepresentingWebPage(clientInfos, server);
+
             for (int i = 0; i < eventArray.size(); i++) {
                 Object eventObj = eventArray.get(i);
@@ -277 +311 @@
                                 (eventType, coordinates, key, scrollPosition, selectedValue))
                     {
-                        events.add(new HtmlEvent(clientInfos, time, path, eventType, coordinates,
-                                                 key, scrollPosition, selectedValue));
+                        path = page.getPath() + path;
+                        events.add(new HtmlEvent(clientInfos, time, path, eventType,
+                                                 coordinates, key, scrollPosition, selectedValue));
                     }
                     else {
@@ -294 +329 @@
             return null;
         }
+    }
+
+    /**
+     * <p>
+     * TODO: comment
+     * </p>
+     *
+     * @param message
+     * @param clientInfos
+     * @return
+     */
+    private HtmlPageElement extractHtmlPageElements(JSONObject      object,
+                                                    HtmlClientInfos clientInfos)
+    {
+        HtmlPageElement server = getServerElement(clientInfos);
+        HtmlPageElement page = getPageElementRepresentingWebPage(clientInfos, server);
+
+        JSONObject jsonPageElement = assertValue(object, "guiModel", JSONObject.class);
+        page.addChild(convert(jsonPageElement, page.getPath()));
+        
+        return server;
+    }
+
+    /**
+     * <p>
+     * TODO: comment
+     * </p>
+     *
+     * @param clientInfos
+     * @return
+     */
+    private HtmlPageElement getServerElement(HtmlClientInfos clientInfos) {
+        String id = clientInfos.getUrl().getHost();
+        if (clientInfos.getUrl().getPort() > -1) {
+            id += ":" + clientInfos.getUrl().getPort();
+        }
+        
+        return new HtmlPageElement(null, "server", id, 0);
+    }
+
+    /**
+     * <p>
+     * TODO: comment
+     * </p>
+     *
+     * @param clientInfos
+     */
+    private HtmlPageElement getPageElementRepresentingWebPage(HtmlClientInfos clientInfos,
+                                                              HtmlPageElement server)
+    {
+        String id = clientInfos.getUrl().getPath();
+        
+        if (clientInfos.getUrl().getQuery() != null) {
+            id += "?" + clientInfos.getUrl().getQuery();
+        }
+        
+        HtmlPageElement page =
+            new HtmlPageElement(server.getPath(), "document", id, clientInfos.getTitle(), 0);
+        
+        server.addChild(page);
+        
+        return page;
+    }
+
+    /**
+     * <p>
+     * TODO: comment
+     * </p>
+     *
+     * @param jsonPageElement
+     * @return
+     */
+    private HtmlPageElement convert(JSONObject jsonPageElement, String parentPath) {
+        HtmlPageElement result = null;
+
+        if (jsonPageElement != null) {
+            String tagName = assertValue(jsonPageElement, "tagName", String.class);
+            String id = assertValue(jsonPageElement, "id", String.class);
+            Integer index = assertValue(jsonPageElement, "index", Integer.class);
+
+            result = new HtmlPageElement(parentPath, tagName, id, index);
+
+            JSONArray childElements = assertValue(jsonPageElement, "children", JSONArray.class);
+            
+            if (childElements != null) {
+                Object jsonChild;
+
+                for (int i = 0; i < childElements.size(); i++) {
+                    jsonChild = childElements.get(i);
+                    if (!(jsonChild instanceof JSONObject)) {
+                        Console.printerrln("child " + (i + 1) + " of HTML page element " + tagName +
+                                           " is no valid HTML page element");
+                    }
+                    else {
+                        result.addChild(convert((JSONObject) jsonChild, result.getPath()));
+                    }
+                }
+            }
+            
+        }
+        
+        return result;    
     }
 
@@ -495 +632 @@
     /**
      * <p>
+     * Checks whether an agent is a robot.
+     * </p>
+     * 
+     * @param agent
+     *            agent that is checked
+     * @return true, if the agent is a robot; false otherwise
+     */
+    private boolean isRobot(String agent) {
+        return agent.matches(robotRegex);
+    }
+
+    /**
+     * <p>
+     * Reads {@link #ROBOTFILTERFILE} and creates a regular expression that
+     * matches all the robots defined in the file. The regular expression is
+     * stored in the field {@link #robotRegex}.
+     * </p>
+     * 
+     * @throws IOException
+     *             thrown if there is a problem reading the robot filter
+     * @throws FileNotFoundException
+     *             thrown if the robot filter is not found
+     */
+    private void loadRobotRegex() throws IOException, FileNotFoundException {
+        String[] lines = FileTools.getLinesFromFile(ROBOTFILTERFILE);
+        StringBuilder regex = new StringBuilder();
+        for (int i = 0; i < lines.length; i++) {
+            regex.append("(.*" + lines[i] + ".*)");
+            if (i != lines.length - 1) {
+                regex.append('|');
+            }
+        }
+        robotRegex = regex.toString();
+    }
+
+    /**
+     * <p>
      * convenience method for dumping the content of a stream and returning a new stream
      * containing the same data.
@@ -504 +678 @@
      * @throws IOException if the stream can not be read 
      */
-    /*private InputStream dumpStreamContent(ServletInputStream inputStream) throws IOException {
+/*    private InputStream dumpStreamContent(ServletInputStream inputStream) throws IOException {
         List<Byte> bytes = new ArrayList<Byte>();
         int buf;