// Copyright 2012 Georg-August-Universität Göttingen, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package de.ugoe.cs.autoquest.plugin.jfc.commands; import java.io.BufferedWriter; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStreamWriter; import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Calendar; import java.util.Collection; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Stack; import java.util.UUID; import java.util.logging.Level; import javax.swing.UIManager; import de.ugoe.cs.autoquest.CommandHelpers; import de.ugoe.cs.autoquest.SequenceInstanceOf; import de.ugoe.cs.util.console.Command; import de.ugoe.cs.autoquest.eventcore.Event; import de.ugoe.cs.autoquest.eventcore.IEventTarget; import de.ugoe.cs.autoquest.eventcore.gui.*; import de.ugoe.cs.autoquest.eventcore.guimodel.GUIModel; import de.ugoe.cs.autoquest.eventcore.guimodel.IGUIElement; import de.ugoe.cs.autoquest.keyboardmaps.VirtualKey; import de.ugoe.cs.autoquest.plugin.jfc.guimodel.JFCGUIElement; import de.ugoe.cs.autoquest.plugin.jfc.guimodel.JFCMenu; import de.ugoe.cs.autoquest.plugin.jfc.guimodel.JFCMenuButton; import de.ugoe.cs.util.console.Console; import de.ugoe.cs.util.console.GlobalDataContainer; // helper class for the tree like structure part within a Jacareto file class StructureNode { public static int nextRef = 0; public String content; public ArrayList children; public StructureNode(String type) { setContent(type); children = new ArrayList(); } public StructureNode() { content = ""; children = new ArrayList(); } public void setContent(String type) { content = ""; } public StructureNode add(String type) { StructureNode node = new StructureNode(type); children.add(node); return node; } public void addRecordable() { children.add(new StructureNode()); } @Override public String toString() { String separator = System.getProperty("line.separator"); String result = content + separator; for (StructureNode child : children) { result += child.toString(); } if (content.endsWith("/>")) { return result; } return result + "" + separator; } } /** *

* Command to create a Jacareto xml replay file from stored sessions. *

* * @author Daniel May * @version 1.0 */ public class CMDgenerateJacaretoReplay implements Command { private static final int EVENT_DURATION = 150; private static final int DOUBLE_CLICK_DURATION = 50; private static final int STARTUP_DELAY = 10000; private JFCGUIElement currentFocus; private StructureNode structure; private StructureNode lastKeySequenceEvent; private StructureNode lastKeyTypedEvent; private int currentKeyModifiers; private HashMap modifiers; private StructureNode lastMouseClickEvent; private StructureNode lastFocusChangeEvent; private StructureNode lastItemActionEvent; private IEventTarget lastMouseDownTarget; private HashMap menuElements; private List menuList; /* * (non-Javadoc) * * @see de.ugoe.cs.util.console.Command#help() */ @Override public String help() { return "generateJacaretoReplay {}"; } /* * (non-Javadoc) * * @see de.ugoe.cs.util.console.Command#run(java.util.List) */ @SuppressWarnings("unchecked") @Override public void run(List parameters) { String filename; String sequencesName; String classpath; String basepath; String classpathext; try { filename = (String) parameters.get(0); sequencesName = (String) parameters.get(1); classpath = (String) parameters.get(2); basepath = (String) parameters.get(3); classpathext = (String) parameters.get(4); } catch (Exception e) { throw new IllegalArgumentException(); } if (parameters.size() > 5) { try { menuList = Files.readAllLines(Paths.get((String) parameters.get(5)), Charset.defaultCharset()); } catch (IOException e) { Console.printerrln("Unable to open menu file"); Console.logException(e); } } Collection> sequences = null; Object dataObject = GlobalDataContainer.getInstance().getData(sequencesName); if (dataObject == null) { CommandHelpers.objectNotFoundMessage(sequencesName); return; } if (!SequenceInstanceOf.isCollectionOfSequences(dataObject)) { CommandHelpers.objectNotType(sequencesName, "Collection>>"); return; } sequences = (Collection>) dataObject; menuElements = new HashMap<>(); // map which maps VirtualKeys back to awt key modifier codes modifiers = new HashMap<>(); modifiers.put(VirtualKey.SHIFT, 1); modifiers.put(VirtualKey.CONTROL, 2); modifiers.put(VirtualKey.ALT, 8); modifiers.put(VirtualKey.ALT_GRAPH, 32); currentKeyModifiers = 0; StructureNode.nextRef = 0; writeJacaretoXML(sequences, filename, classpath, basepath, classpathext); } private void writeLine(BufferedWriter writer, String line) throws IOException { writer.write(line); writer.newLine(); } private void writeJacaretoHead(BufferedWriter writer, String classname, String basepath, String classpathext) throws IOException { Calendar now = Calendar.getInstance(); writeLine(writer, ""); writeLine(writer, ""); writeLine(writer, ""); //@formatter:off writeLine(writer, "" ); writeLine(writer, ""); writeLine(writer, ""); writeLine(writer, ""); writeLine(writer, "" ); //@formatter:on } private void writeJacaretoEvents(BufferedWriter writer, Collection> sequences) throws IOException { structure = new StructureNode("RootElement"); // reference the elements that we included in the header structure.addRecordable(); // Calendar structure.addRecordable(); // SystemInfo structure.addRecordable(); // KeyboardState structure.addRecordable(); // ComponentMode structure.addRecordable(); // ApplicationStarter for (List sequence : sequences) { for (Iterator eventIter = sequence.iterator(); eventIter.hasNext();) { Event event = eventIter.next(); if (event.getType() instanceof MouseButtonDown) { handleMouseDown(writer, event, "MouseClick"); } else if (event.getType() instanceof MouseButtonUp) { handleMouseUp(writer, event); } else if (event.getType() instanceof MouseDoubleClick) { StructureNode multiClick = structure.add("MultipleMouseClick"); // first click lastMouseClickEvent = multiClick.add("MouseClick"); writeMouseClickEvent(writer, event, (JFCGUIElement) event.getTarget(), DOUBLE_CLICK_DURATION, 501); writeMouseClickEvent(writer, event, (JFCGUIElement) event.getTarget(), DOUBLE_CLICK_DURATION, 502); writeMouseClickEvent(writer, event, (JFCGUIElement) event.getTarget(), DOUBLE_CLICK_DURATION, 500); // second click lastMouseClickEvent = multiClick.add("MouseClick"); writeMouseClickEvent(writer, event, (JFCGUIElement) event.getTarget(), DOUBLE_CLICK_DURATION, 501); writeMouseClickEvent(writer, event, (JFCGUIElement) event.getTarget(), DOUBLE_CLICK_DURATION, 502); writeMouseClickEvent(writer, event, (JFCGUIElement) event.getTarget(), DOUBLE_CLICK_DURATION, 500); lastMouseClickEvent = null; } else if (event.getType() instanceof MouseClick) { if (event.getTarget() instanceof JFCMenuButton) { // if a menu file was provided, use the improved event // generation if (menuList != null) { if (menuElements.isEmpty()) { // parse the menu structure GUIModel model = ((IGUIElement) event.getTarget()).getGUIModel(); getMenuElements(model.getRootElements(), model); } Stack hierarchy = findMenuItemHierarchy((JFCGUIElement) event.getTarget()); while (!hierarchy.empty()) { generateFullClick(writer, event, hierarchy.pop()); } continue; } } lastKeySequenceEvent = null; if (lastMouseClickEvent != null) { if (lastMouseDownTarget == event.getTarget()) { // this is the standard case: // mouseDown, mouseUp and mouseClick sequence // was triggered on this target writeMouseClickEvent(writer, event, (JFCGUIElement) event.getTarget(), EVENT_DURATION, 500); writeItemActionEvent(writer, event); if (lastFocusChangeEvent == null) { // write structure sequentially structure.children.add(lastMouseClickEvent); structure.children.add(lastItemActionEvent); } else { // with nested structure structure.children.add(lastItemActionEvent); lastItemActionEvent.children.add(0, lastFocusChangeEvent); lastFocusChangeEvent.children.add(0, lastMouseClickEvent); lastFocusChangeEvent = null; lastMouseClickEvent = null; } } else { // target of mouseDown and mouseClick are different // -> this is, for example, a click on a menu item // within a condensed sequence commitFocusEvent(); // finish the last click on the old target writeMouseClickEvent(writer, event, (JFCGUIElement) event.getTarget(), EVENT_DURATION, 500); structure.children.add(lastMouseClickEvent); // and generate a new one generateFullClick(writer, event, (JFCGUIElement) event.getTarget()); } } else { // a target was clicked repeatedly: // the condensed sequence contains no mouseDowns or // mouseUps anymore // -> just generate another full click generateFullClick(writer, event, (JFCGUIElement) event.getTarget()); } } else if (event.getType() instanceof KeyboardFocusChange) { lastKeySequenceEvent = null; writeFocusChangeEvent(writer, event); } else if (event.getType() instanceof MouseDragAndDrop) { handleMouseDragAndDrop(writer, event); } else if (event.getType() instanceof KeyPressed) { handleKeyPressed(writer, event); } else if (event.getType() instanceof KeyReleased) { handleKeyReleased(writer, event); } else if (event.getType() instanceof TextInput) { handleTextInput(writer, event); } else { Console.traceln(Level.WARNING, "No handler for event \"" + event + "\". Skipped."); } } } } private void handleMouseDown(BufferedWriter writer, Event event, String structureName) throws IOException { commitFocusEvent(); lastKeySequenceEvent = null; lastMouseClickEvent = new StructureNode(structureName); lastMouseDownTarget = event.getTarget(); writeMouseClickEvent(writer, event, (JFCGUIElement) event.getTarget(), EVENT_DURATION, 501); } private void handleMouseUp(BufferedWriter writer, Event event) throws IOException { lastKeySequenceEvent = null; writeMouseClickEvent(writer, event, (JFCGUIElement) event.getTarget(), EVENT_DURATION, 502); } private void handleMouseDragAndDrop(BufferedWriter writer, Event event) throws IOException { commitFocusEvent(); MouseDragAndDrop dragEvent = (MouseDragAndDrop) event.getType(); lastMouseClickEvent = new StructureNode("MouseDrag"); lastMouseDownTarget = null; writeMouseClickEvent(writer, event, (JFCGUIElement) event.getTarget(), EVENT_DURATION, dragEvent.getXStart(), dragEvent.getYStart(), 501); writeMouseClickEvent(writer, event, (JFCGUIElement) event.getTarget(), EVENT_DURATION, dragEvent.getX(), dragEvent.getY(), 506); structure.children.add(lastMouseClickEvent); } private void handleKeyPressed(BufferedWriter writer, Event event) throws IOException { commitFocusEvent(); if (lastKeySequenceEvent == null) { lastKeySequenceEvent = structure.add("KeySequence"); } lastKeyTypedEvent = lastKeySequenceEvent.add("KeyTyped"); writeKeyEvent(writer, event, 401); } private void handleKeyReleased(BufferedWriter writer, Event event) throws IOException { commitFocusEvent(); writeKeyEvent(writer, event, 402); } private void handleTextInput(BufferedWriter writer, Event event) throws IOException { List textEvents = ((TextInput) event.getType()).getTextInputEvents(); // just split the text event into its key events again for (Event textEvent : textEvents) { if (textEvent.getType() instanceof KeyPressed) { handleKeyPressed(writer, textEvent); } else if (textEvent.getType() instanceof KeyReleased) { handleKeyReleased(writer, textEvent); } } } private void getMenuElements(List elements, GUIModel model) { for (IGUIElement child : elements) { if (child instanceof JFCMenuButton || child instanceof JFCMenu) { menuElements.put(((JFCGUIElement) child).getName().replaceAll("^\"|\"$", ""), (JFCGUIElement) child); } getMenuElements(model.getChildren(child), model); } } private Stack findMenuItemHierarchy(JFCGUIElement item) { Stack elements = new Stack<>(); // find line that contains this menu item name int lineOfItem = -1; for (int i = 0; i < menuList.size(); i++) { String name = "\"" + menuList.get(i).trim().toLowerCase() + "\""; if (name.equals(item.getName().trim().toLowerCase())) { lineOfItem = i; } } // now go backwards until the toplevel menu is found int oldIndent = Integer.MAX_VALUE; for (int j = lineOfItem; j >= 0; j--) { String stripped = menuList.get(j).replaceFirst("^ *", ""); int indent = menuList.get(j).length() - stripped.length(); if (indent < oldIndent) { // this is a parent submenu elements.push(menuElements.get(stripped)); oldIndent = indent; } } return elements; } private void commitFocusEvent() { if (lastFocusChangeEvent != null) { structure.children.add(lastFocusChangeEvent); lastFocusChangeEvent = null; } } private void generateFullClick(BufferedWriter writer, Event event, JFCGUIElement target) throws IOException { lastMouseClickEvent = new StructureNode("MouseClick"); lastMouseDownTarget = event.getTarget(); writeMouseClickEvent(writer, event, target, EVENT_DURATION, 501); writeMouseClickEvent(writer, event, target, EVENT_DURATION, 502); writeMouseClickEvent(writer, event, target, EVENT_DURATION, 500); writeItemActionEvent(writer, event); structure.children.add(lastMouseClickEvent); structure.children.add(lastItemActionEvent); lastMouseDownTarget = null; } private void writeJacaretoTail(BufferedWriter writer) throws IOException { writeLine(writer, ""); // write the recording's structure writeLine(writer, ""); writer.write(structure.toString()); // close root element writeLine(writer, ""); } private void writeJacaretoXML(Collection> sequences, String filename, String classpath, String basepath, String classpathext) { BufferedWriter writer = new BufferedWriter(openReplayFile(filename + ".xml")); try { writeJacaretoHead(writer, classpath, basepath, classpathext); writeJacaretoEvents(writer, sequences); writeJacaretoTail(writer); writeLine(writer, ""); writer.flush(); writer.close(); } catch (IOException e) { Console.printerrln("Unable to write Jacareto replay file " + filename); } } /** *

* Helper function that opens the replay file for writing. *

* * @param filename * name and path of the replay file * @param encoding * file encoding, empty string for platform default * @return {@link OutputStreamWriter} that writes to the replay file */ private OutputStreamWriter openReplayFile(String filename) { File file = new File(filename); boolean fileCreated; try { fileCreated = file.createNewFile(); if (!fileCreated) { Console.traceln(Level.INFO, "Created logfile " + filename); } else { Console.traceln(Level.INFO, "Overwrote existing logfile " + filename); } } catch (IOException e) { Console.printerrln("Unable to create file " + filename); Console.logException(e); } OutputStreamWriter writer = null; try { writer = new OutputStreamWriter(new FileOutputStream(file)); } catch (IOException e) { Console.printerrln("Unable to open file for writing (read-only file):" + filename); Console.logException(e); } return writer; } private void writeItemActionEvent(BufferedWriter writer, Event event) throws IOException { JFCGUIElement target = (JFCGUIElement) event.getTarget(); MouseButtonInteraction info = (MouseButtonInteraction) event.getType(); //@formatter:off writeLine(writer, "" ); writeLine(writer, "" ); //@formatter:on lastItemActionEvent = new StructureNode("ItemStateChange"); lastItemActionEvent.addRecordable(); lastItemActionEvent.addRecordable(); } private void writeFocusChangeEvent(BufferedWriter writer, Event event) throws IOException { KeyboardFocusChange info = (KeyboardFocusChange) event.getType(); JFCGUIElement target = (JFCGUIElement) event.getTarget(); if (currentFocus != null) { lastFocusChangeEvent = new StructureNode("FocusChange"); // focus lost on old target writeFocusEvent(writer, info, currentFocus, 1005); // focus gained on new target writeFocusEvent(writer, info, target, 1004); } else { // TODO: it seems like Jacareto wants a window activation before // the first focus event but that is not the case in autoquest, // skip for now } currentFocus = target; } private void writeFocusEvent(BufferedWriter writer, KeyboardFocusChange info, JFCGUIElement target, int jacId) throws IOException { //@formatter:off writeLine(writer, "" ); //@formatter:on lastFocusChangeEvent.addRecordable(); } private void writeMouseClickEvent(BufferedWriter writer, Event event, JFCGUIElement target, int duration, int jacId) throws IOException { MouseButtonInteraction info = (MouseButtonInteraction) event.getType(); writeMouseClickEvent(writer, event, target, duration, info.getX(), info.getY(), jacId); } private void writeMouseClickEvent(BufferedWriter writer, Event event, JFCGUIElement target, int duration, int x, int y, int jacId) throws IOException { MouseButtonInteraction info = (MouseButtonInteraction) event.getType(); int clickCount = event.getType() instanceof MouseDoubleClick ? 2 : 1; //@formatter:off writeLine(writer, "" ); writeLine(writer, "" ); writeLine(writer, ""); //@formatter:on lastMouseClickEvent.addRecordable(); } private int getButtonModifier(MouseButtonInteraction info) { switch (info.getButton()) { case LEFT: return 16; case MIDDLE: return 8; case RIGHT: return 4; default: // TODO: handle unknown mouse button return -1; } } private void writeKeyEvent(BufferedWriter writer, Event event, int jacId) throws IOException { KeyInteraction info = (KeyInteraction) event.getType(); JFCGUIElement target = (JFCGUIElement) event.getTarget(); int keyCode = info.getKey().getVirtualKeyCode(); applyKeyModifier(info.getKey(), jacId == 401); //@formatter:off writeLine(writer, "" ); writeLine(writer, "" ); writeLine(writer, ""); lastKeyTypedEvent.addRecordable(); } private String getKeyChar (int keyCode) { if (keyCode >= 32 && keyCode < 127) { return String.valueOf((char)keyCode); } return "_NO_LEGAL_XML_CHAR"; } private void applyKeyModifier (VirtualKey key, boolean set) { Integer modifier = modifiers.get(key); if (modifier != null) { if (set) { currentKeyModifiers |= modifier; } else { currentKeyModifiers &= ~modifier; } } } }