Changeset 1730

Timestamp:

09/05/14 17:32:57 (10 years ago)

Author:

rkrimmel

Message:

Implemented parallel Match replacer

File:

• branches/autoquest-core-tasktrees-alignment/src/main/java/de/ugoe/cs/autoquest/tasktrees/temporalrelation/SequenceForTaskDetectionRuleAlignment.java (modified) (14 diffs)

branches/autoquest-core-tasktrees-alignment/src/main/java/de/ugoe/cs/autoquest/tasktrees/temporalrelation/SequenceForTaskDetectionRuleAlignment.java

@@ -232 +232 @@
                         detectAndReplaceIterations(appData);
                         appData.getStopWatch().start("task replacement");
-                        appData.updateSubstitutionMatrix();
+                        //appData.updateSubstitutionMatrix();
                         detectAndReplaceTasks(appData); //
                         appData.getStopWatch().stop("task replacement");
@@ -670 +670 @@
                 // Replace matches in the sessions
                 Console.traceln(Level.INFO, "Replacing matches in sessions");
+                replaceMatches(appData);
                 appData.getStopWatch().start("replacing tasks");
-                HashMap<Integer, List<MatchOccurence>> replacedOccurences = new HashMap<Integer, List<MatchOccurence>>();
-                for (int i = 0; i < appData.getMatchseqs().size(); i++) {
-
-                        // Every pattern consists of 2 sequences, therefore the minimum
-                        // occurrences here is 2.
-                        // We just need the sequences also occurring in other sequences as
-                        // well
-                        if (appData.getMatchseqs().get(i).occurenceCount() > 2) {
-                                appData.detectedAndReplacedTasks = true;
-                                ISequence task = matchAsSequence(appData, appData
-                                                .getMatchseqs().get(i));
-                                invalidOccurence: for (Iterator<MatchOccurence> it = appData
-                                                .getMatchseqs().get(i).getOccurences().iterator(); it
-                                                .hasNext();) {
-                                        MatchOccurence oc = it.next();
-
-                                        // Check if nothing has been replaced in the sequence we
-                                        // want to replace
-                                        if (replacedOccurences.get(oc.getSequenceId()) == null) {
-                                                replacedOccurences.put(oc.getSequenceId(),
-                                                                new LinkedList<MatchOccurence>());
-                                        } else {
-                                                // check if we have any replaced occurence with indexes
-                                                // smaller than ours. If so, we need to adjust our start
-                                                // and endpoints
-                                                // of the replacement.
-                                                // Also do a check if we have replaced this specific
-                                                // MatchOccurence in this sequence already. Jump to the
-                                                // next occurence if this is the case.
-                                                // This is no more neccessary once the matches are
-                                                // harmonized.
-                                                for (Iterator<MatchOccurence> jt = replacedOccurences
-                                                                .get(oc.getSequenceId()).iterator(); jt
-                                                                .hasNext();) {
-                                                        MatchOccurence tmpOC = jt.next();
-
-                                                        if (oc.getStartindex() >= tmpOC.getStartindex()
-                                                                        && oc.getStartindex() <= tmpOC
-                                                                                        .getEndindex()) {
-                                                                continue invalidOccurence;
-                                                        }
-                                                        if (oc.getEndindex() >= tmpOC.getStartindex()) {
-                                                                continue invalidOccurence;
-
-                                                        } else if (oc.getStartindex() > tmpOC.getEndindex()) {
-                                                                int diff = tmpOC.getEndindex()
-                                                                                - tmpOC.getStartindex();
-                                                                // Just to be sure.
-                                                                if (diff > 0) {
-                                                                        oc.setStartindex(oc.getStartindex() - diff
-                                                                                        + 1);
-                                                                        oc.setEndindex(oc.getEndindex() - diff + 1);
-                                                                } else {
-                                                                        Console.traceln(Level.WARNING,
-                                                                                        "End index of a Match before start. This should never happen");
-                                                                }
-                                                        }
-                                                }
-                                        }
-                                        System.out.println("Replacing in sequence"
-                                                        + oc.getSequenceId());
-                                        ISequenceInstance sequenceInstances = RuleUtils
-                                                        .createNewSubSequenceInRange(appData.getSessions()
-                                                                        .get(oc.getSequenceId()), oc
-                                                                        .getStartindex(), oc.getEndindex(), task,
-                                                                        taskFactory, taskBuilder);
-                                        // Adjust the length of the match regarding to the length of
-                                        // instance. (OptionalInstances may be shorter)
-                                        oc.setEndindex(oc.getStartindex()
-                                                        + sequenceInstances.size()
-                                                        - RuleUtils.missedOptionals);
-                                        replacedOccurences.get(oc.getSequenceId()).add(oc);
-                                }
-                        }
-                }
+                
                 // appData.setMatchseqs(null);
                 appData.getStopWatch().stop("replacing tasks");
         }
 
-        private void replaceMatches() {
-
-        }
-
-        private void searchMatchesInAllSessions(RuleApplicationData appData) {
-                Console.traceln(Level.INFO,
-                                "searching for patterns occuring most with " + nThreads
-                                                + " threads");
-                // Prepare parallel search of matchseqs
-                ExecutorService executor = Executors.newFixedThreadPool(nThreads);
+        private void replaceMatches(RuleApplicationData appData) {
+                appData.replacedOccurences = new HashMap<Integer, List<MatchOccurence>>();
+                
                 int matchSeqSize = appData.getMatchseqs().size();
-                int interval = matchSeqSize / nThreads;
-                int rest = matchSeqSize % nThreads;
-
+                int newThreads = nThreads;
+                if (matchSeqSize < nThreads) {
+                        newThreads = matchSeqSize; 
+                }
+                ExecutorService executor = Executors.newFixedThreadPool(newThreads);
+                int interval = matchSeqSize / newThreads;
+                int rest = matchSeqSize % newThreads;
+                
                 for (int i = 0; i < matchSeqSize - interval; i += interval) {
                         int offset = 0;
@@ -772 +697 @@
                         int from = i;
                         int to = i + interval + offset;
-                        System.out.println("Creating thread with matches from " + from
+                        System.out.println("Replacement: Creating thread with matches from " + from
                                         + " to " + to);
                         // search each match in every other sequence
-                        ParallelMatchOcurrencesFinder finder = new ParallelMatchOcurrencesFinder(
+                        ParallelMatchReplacer replacer = new ParallelMatchReplacer(
                                         appData, from, to);
-                        executor.execute(finder);
+                        executor.execute(replacer);
                 }
                 executor.shutdown();
@@ -786 +711 @@
                         e.printStackTrace();
                 }
+        }
+
+        private void searchMatchesInAllSessions(RuleApplicationData appData) {
+                Console.traceln(Level.INFO,
+                                "searching for patterns occuring most with " + nThreads
+                                                + " threads");
+                // Prepare parallel search of matchseqs
+                
+                int matchSeqSize = appData.getMatchseqs().size();
+                int newThreads = nThreads;
+                if (matchSeqSize < nThreads) {
+                        newThreads = matchSeqSize; 
+                }
+                int interval = matchSeqSize / newThreads;
+                int rest = matchSeqSize % newThreads;
+                ExecutorService executor = Executors.newFixedThreadPool(nThreads);
+                
+                for (int i = 0; i < matchSeqSize - interval; i += interval) {
+                        int offset = 0;
+                        if (rest != 0) {
+                                offset = 1;
+                                rest--;
+                        }
+                        int from = i;
+                        int to = i + interval + offset;
+                        System.out.println("Match finding: Creating thread with matches from " + from
+                                        + " to " + to);
+                        // search each match in every other sequence
+                        ParallelMatchOcurrencesFinder finder = new ParallelMatchOcurrencesFinder(
+                                        appData, from, to);
+                        executor.execute(finder);
+                }
+                executor.shutdown();
+                try {
+                        executor.awaitTermination(2, TimeUnit.HOURS);
+                } catch (InterruptedException e) {
+                        // TODO Auto-generated catch block
+                        e.printStackTrace();
+                }
 
         }
 
         // Print out the progress
-        private static void printProgressPercentage(int count, int size) {
+        private static void printProgressPercentage(String message,int count, int size) {
                 if (size > 100) {
                         if ((count % (size / 100) == 0)) {
@@ -796 +760 @@
                                 // Thread.currentThread().getName() + ": " + Math.round((float)
                                 // count/size*100))+ "%");
-                                System.out.println("Thread" + Thread.currentThread().getName()
+                                System.out.println(message + " in thread" + Thread.currentThread().getName()
                                                 + ": " + Math.round((float) count / size * 100) + "%");
                         }
@@ -803 +767 @@
                         // Thread.currentThread().getName() + ": " +Math.round((float)
                         // count/size*100))+ "%");
-                        System.out.println("Thread" + Thread.currentThread().getName()
+                        System.out.println(message + " in thread" + Thread.currentThread().getName()
                                         + ": " + Math.round((float) count / size * 100) + "%");
 
@@ -811 +775 @@
         private class ParallelMatchReplacer implements Runnable {
 
+                private final RuleApplicationData appData;
+                private final int from;
+                private final int to;
+
+                ParallelMatchReplacer(RuleApplicationData appData, int from,
+                                int to) {
+                        this.appData = appData;
+                        this.from = from;
+                        this.to = to;
+                }
                 @Override
                 public void run() {
-
-                }
-
-        }
+                        int count = 0;
+                        int size = to - from;
+                        for (int i = from; i < to; i++) {
+                        count++;
+                        // Every pattern consists of 2 sequences, therefore the minimum
+                        // occurrences here is 2.
+                        // We just need the sequences also occurring in other sequences as well
+                        if (appData.getMatchseqs().get(i).occurenceCount() > 2) {
+                                
+                                ISequence task = matchAsSequence(appData, appData.getMatchseqs().get(i));
+                                                        invalidOccurence: for (Iterator<MatchOccurence> it = appData
+                                                                        .getMatchseqs().get(i).getOccurences().iterator(); it
+                                                                        .hasNext();) {
+                                                                MatchOccurence oc = it.next();
+
+                                                                // Check if nothing has been replaced in the sequence we 
+                                                                // want to replace now
+                                                                
+                                                                synchronized (appData.getReplacedOccurences()) {
+                                                                if (appData.getReplacedOccurences().get(oc.getSequenceId()) == null) {
+                                                                        appData.getReplacedOccurences().put(oc.getSequenceId(),
+                                                                                        new LinkedList<MatchOccurence>());
+                                                                } else {
+                                                                        // check if we have any replaced occurence with indexes
+                                                                        // smaller than ours. If so, we need to adjust our start
+                                                                        // and endpoints
+                                                                        // of the replacement.
+                                                                        // Also do a check if we have replaced this specific
+                                                                        // MatchOccurence in this sequence already. Jump to the
+                                                                        // next occurence if this is the case.
+                                                                        // This is no more neccessary once the matches are
+                                                                        // harmonized.
+                                                                        for (Iterator<MatchOccurence> jt = appData.getReplacedOccurences()
+                                                                                        .get(oc.getSequenceId()).iterator(); jt
+                                                                                        .hasNext();) {
+                                                                                MatchOccurence tmpOC = jt.next();
+
+                                                                                if (oc.getStartindex() >= tmpOC.getStartindex()
+                                                                                                && oc.getStartindex() <= tmpOC
+                                                                                                                .getEndindex()) {
+                                                                                        continue invalidOccurence;
+                                                                                }
+                                                                                if (oc.getEndindex() >= tmpOC.getStartindex()) {
+                                                                                        continue invalidOccurence;
+
+                                                                                } else if (oc.getStartindex() > tmpOC.getEndindex()) {
+                                                                                        int diff = tmpOC.getEndindex()
+                                                                                                        - tmpOC.getStartindex();
+                                                                                        // Just to be sure.
+                                                                                        if (diff > 0) {
+                                                                                                oc.setStartindex(oc.getStartindex() - diff
+                                                                                                                + 1);
+                                                                                                oc.setEndindex(oc.getEndindex() - diff + 1);
+                                                                                        } else {
+                                                                                                Console.traceln(Level.WARNING,
+                                                                                                                "End index of a Match before start. This should never happen");
+                                                                                        }
+                                                                                }
+                                                                        }
+                                                                }
+                                                                System.out.println("Replacing in sequence"
+                                                                                + oc.getSequenceId());
+                                                                synchronized(appData) {
+                                                                        appData.detectedAndReplacedTasks = true;
+                                                                }
+                                                                synchronized (appData.getSessions().get(oc.getSequenceId())) {
+                                                                        ISequenceInstance sequenceInstances = RuleUtils
+                                                                                        .createNewSubSequenceInRange(appData.getSessions()
+                                                                                                        .get(oc.getSequenceId()), oc
+                                                                                                        .getStartindex(), oc.getEndindex(), task,
+                                                                                                        taskFactory, taskBuilder);
+                                                                        oc.setEndindex(oc.getStartindex()
+                                                                                        + sequenceInstances.size()
+                                                                                        - RuleUtils.missedOptionals);
+                                                                        }
+                                                                }
+                                                                // Adjust the length of the match regarding to the length of
+                                                                // instance. (OptionalInstances may be shorter)
+                                                                
+                                                                synchronized(appData.getReplacedOccurences().get(oc.getSequenceId())) {
+                                                                        appData.getReplacedOccurences().get(oc.getSequenceId()).add(oc);
+                                                                }
+                                                        }
+                                                }
+                                        }
+                                }       
+                        }
+                
+        
 
         private class ParallelMatchOcurrencesFinder implements Runnable {
@@ -838 +897 @@
                                 Match pattern = appData.getMatchseqs().get(i);
                                 count++;
-                                printProgressPercentage(count, size);
+                                printProgressPercentage("Match finding progress",count, size);
                                 // Skip sequences with more 0 events (scrolls) than other
                                 // events.
@@ -885 +944 @@
                                 NumberSequence ns1 = appData.getNumberSequences().get(i);
                                 count++;
-                                printProgressPercentage(count, size);
+                                printProgressPercentage("Aligning Progress",count, size);
                                 for (int j = 0; j < appData.getNumberSequences().size(); j++) {
                                         NumberSequence ns2 = appData.getNumberSequences().get(j);
@@ -908 +967 @@
                 Console.traceln(Level.INFO, "generating pairwise alignments from "
                                 + numberSeqSize + " sessions with " + nThreads + " threads");
-                ExecutorService executor = Executors.newFixedThreadPool(nThreads);
-                int interval = numberSeqSize / nThreads;
-                int rest = numberSeqSize % nThreads;
+                
+                int newThreads = nThreads;
+                if (numberSeqSize < nThreads) {
+                        newThreads = numberSeqSize; 
+                }
+                
+                ExecutorService executor = Executors.newFixedThreadPool(newThreads);
+                int interval = numberSeqSize / newThreads;
+                int rest = numberSeqSize % newThreads;
 
                 for (int i = 0; i < numberSeqSize - interval; i += interval) {
@@ -933 +998 @@
                         e.printStackTrace();
                 }
-
         }
 
@@ -952 +1016 @@
                 private HashSet<ITask> uniqueTasks;
 
-                ObjectDistanceSubstitionMatrix submat;
+                private ObjectDistanceSubstitionMatrix submat;
                 
+                public HashMap<Integer, List<MatchOccurence>> replacedOccurences;
                 
-
-                LinkedList<Match> matchseqs;
+        
+                public LinkedList<Match> matchseqs;
 
                 private ArrayList<NumberSequence> numberseqs;
@@ -1005 +1070 @@
                 }
 
-                private void resetNewlyCreatedTasks() {
+                synchronized private void resetNewlyCreatedTasks() {
                         uniqueTasks.addAll(newTasks);
                         newTasks.clear();
@@ -1021 +1086 @@
                         return sessions;
                 }
+                
+
+                public HashMap<Integer, List<MatchOccurence>> getReplacedOccurences() {
+                        return replacedOccurences;
+                }
+
+                public void setReplacedOccurences(
+                                HashMap<Integer, List<MatchOccurence>> replacedOccurences) {
+                        this.replacedOccurences = replacedOccurences;
+                }
+
 
                 private HashSet<ITask> getUniqueTasks() {