Changeset 1723 for branches/autoquest-core-tasktrees-alignment/src/main

Timestamp:

09/04/14 17:08:05 (10 years ago)

Author:

rkrimmel

Message:

More parallelism

File:

• branches/autoquest-core-tasktrees-alignment/src/main/java/de/ugoe/cs/autoquest/tasktrees/temporalrelation/SequenceForTaskDetectionRuleAlignment.java (modified) (10 diffs)

branches/autoquest-core-tasktrees-alignment/src/main/java/de/ugoe/cs/autoquest/tasktrees/temporalrelation/SequenceForTaskDetectionRuleAlignment.java

@@ -23 +23 @@
 import java.io.Serializable;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
@@ -33 +32 @@
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -656 +654 @@
                 
                 // Generate pairwise alignments
-                int numberSeqSize = appData.getNumberSequences().size();
-                Console.traceln(Level.INFO, "generating pairwise alignments from " + numberSeqSize + " sessions");
-                int count = 0;
-                
-                //Checking if i have an already calculated file result of this action in the working directory 
-                File aligned = new File("aligned" + iteration + ".dat");
-                if(!(aligned.exists() && !aligned.isDirectory())) {
-                        appData.matchseqs = new LinkedList<Match>();
-                        for (int i = 0; i < appData.getNumberSequences().size(); i++) {
-                                NumberSequence ns1 = appData.getNumberSequences().get(i);
-                                count++;
-                                printProgressPercentage(count,numberSeqSize);
-                                for (int j = 0; j < appData.getNumberSequences().size(); j++) {
-                                        NumberSequence ns2 = appData.getNumberSequences().get(j);
-                                        if (i != j) {
-                                                //Console.traceln(Level.FINEST,"Aligning sequence " + i + " with sequence " + j);
-                                                AlignmentAlgorithm aa = AlignmentAlgorithmFactory.create();
-                                                aa.align(ns1, ns2, appData.getSubmat(),9);
-                                                appData.getMatchseqs().addAll(aa.getMatches());
-                                        }
-                                }
-                        }
-                        GlobalDataContainer.getInstance().addData("aligned" + iteration, appData);
-                        saveAppData("aligned" + iteration);
-                }
-                else {
-                        Console.traceln(Level.INFO,"loading matches from file");
-                        appData = loadAppData("aligned"+iteration);
-                }
-
-
-                Console.traceln(Level.INFO, "searching for patterns occuring most with " + nThreads + " threads");
-                
-                //Prepare parallel search of matchseqs
-                ExecutorService executor = Executors.newFixedThreadPool(nThreads);
-                int matchSeqSize=appData.getMatchseqs().size();
-                int interval = Math.round(matchSeqSize/nThreads);
-                int rest = matchSeqSize % nThreads;
-                
-                for(int i =0;i<matchSeqSize-interval;i+=interval) {
-                        int offset =0;
-                        if(rest!=0) {
-                                offset=1;
-                                rest--;
-                        }
-                        int from = i;
-                        int to = i+interval+offset;
-                        System.out.println("Creating thread with matches from " + from + " to " + to);
-                        // search each match in every other sequence
-                        ParallelMatchOcurrencesFinder finder = new ParallelMatchOcurrencesFinder(appData,from,to);
-                        executor.execute(finder);
-                }
-                executor.shutdown();
-                try {
-                        executor.awaitTermination(2, TimeUnit.HOURS);
-                } catch (InterruptedException e) {
-                        // TODO Auto-generated catch block
-                        e.printStackTrace();
-                }
-                
-                
-                Console.traceln(Level.INFO, "sorting results");
+                appData.setMatchseqs(generatePairwiseAlignments(appData));
+
+                //Searching each match in all other sessions, counting its occurences
+                searchMatchesInAllSessions(appData);
+                
                 // Sort results to get the most occurring results
+                Console.traceln(Level.INFO, "sorting " + appData.getMatchseqs().size() + " results");
                 Comparator<Match> comparator = new Comparator<Match>() {
                         public int compare(Match m1, Match m2) {
@@ -725 +667 @@
                         }
                 };
+                
                 Collections.sort(appData.getMatchseqs(), comparator);
                 appData.getStopWatch().stop("detecting tasks");
 
+                // Replace matches in the sessions
+                Console.traceln(Level.INFO, "Replacing matches in sessions");
                 appData.getStopWatch().start("replacing tasks");
                 HashMap<Integer, List<MatchOccurence>> replacedOccurences = new HashMap<Integer, List<MatchOccurence>>();
-                // Replace matches in the sessions
                 for (int i = 0; i < appData.getMatchseqs().size(); i++) {
                 
@@ -744 +688 @@
                                         MatchOccurence oc = it.next();
                                         
-                                        /*
-                                        if (iteration > 0) {
-                                         
-                                        System.out.println("Trying to replace sequence: ");
-                                        appData.getMatchseqs().get(i).getFirstSequence().printSequence();
-                                        appData.getMatchseqs().get(i).getSecondSequence().printSequence();
-                                        System.out.println(" in session number: "
-                                                        + (oc.getSequenceId() + 1) + " at position "
-                                                        + (oc.getStartindex()) + "-" + oc.getEndindex());
-                                        System.out.println();
-
-                                        System.out.println("Printing session: ");
-                                        for (int j = 0; j < appData.getSessions()
-                                                        .get(oc.getSequenceId()).size(); j++) {
-                                                System.out.println(j
-                                                                + ": "
-                                                                + appData.getSessions().get(oc.getSequenceId())
-                                                                                .get(j));
-                                        }
-                                        }*/
                                         
                                         // Check if nothing has been replaced in the sequence we
@@ -808 +732 @@
                                                 }
                                         }
+                                        System.out.println("Replacing");
                                         ISequenceInstance sequenceInstances = RuleUtils
                                                         .createNewSubSequenceInRange(appData.getSessions()
@@ -827 +752 @@
         
         
+        
+
+        private void searchMatchesInAllSessions(RuleApplicationData appData) {
+                Console.traceln(Level.INFO, "searching for patterns occuring most with " + nThreads + " threads");
+                //Prepare parallel search of matchseqs
+                ExecutorService executor = Executors.newFixedThreadPool(nThreads);
+                int matchSeqSize=appData.getMatchseqs().size();
+                int interval = Math.round(matchSeqSize/nThreads);
+                int rest = matchSeqSize % nThreads;
+                
+                for(int i =0;i<matchSeqSize-interval;i+=interval) {
+                        int offset =0;
+                        if(rest!=0) {
+                                offset=1;
+                                rest--;
+                        }
+                        int from = i;
+                        int to = i+interval+offset;
+                        System.out.println("Creating thread with matches from " + from + " to " + to);
+                        // search each match in every other sequence
+                        ParallelMatchOcurrencesFinder finder = new ParallelMatchOcurrencesFinder(appData,from,to);
+                        executor.execute(finder);
+                }
+                executor.shutdown();
+                try {
+                        executor.awaitTermination(2, TimeUnit.HOURS);
+                } catch (InterruptedException e) {
+                        // TODO Auto-generated catch block
+                        e.printStackTrace();
+                }
+                
+        }
+
         //Print out the progress
          private static void printProgressPercentage(int count, int size) {
-                
                 if(size>100) {
                         if((count%(size/100)==0)) {     
-                                Console.traceln(Level.INFO,("Thread" + Thread.currentThread().getName() + ": " + Math.round((float) count/size*100))+ "%");
+                                //Console.traceln(Level.INFO,("Thread" + Thread.currentThread().getName() + ": " + Math.round((float) count/size*100))+ "%");
+                                System.out.println("Thread" + Thread.currentThread().getName() + ": " + Math.round((float) count/size*100)+ "%");
                         }
                 }
                 else {
-                        Console.traceln(Level.INFO,("Thread" + Thread.currentThread().getName() + ": " +Math.round((float) count/size*100))+ "%");
+                        //Console.traceln(Level.INFO,("Thread" + Thread.currentThread().getName() + ": " +Math.round((float) count/size*100))+ "%");
+                        System.out.println("Thread" + Thread.currentThread().getName() + ": " +Math.round((float) count/size*100)+ "%");
+                        
                 }
         }
@@ -857 +817 @@
                 int count = 0;
                 int size=to-from;
-                Console.traceln(Level.INFO, "Pattern count is " + size);
-                for (int i=from; i<to;i++) { {
+                
+                for (int i=from; i<to;i++) { 
                         Match pattern = appData.getMatchseqs().get(i);
                         count++;
@@ -880 +840 @@
                                                                 + pattern.size(), j));
                                         }
-
                                 }
                         }
                 }  
+        }
+        }
+        
+        
+        private class ParallelPairwiseAligner implements Runnable {
+        private final RuleApplicationData appData;
+        private final int from;
+        private final int to;
+            ParallelPairwiseAligner(RuleApplicationData appData, int from, int to) {
+            this.appData = appData;
+            this.from = from;
+            this.to = to;
+        }
+        
+        @Override
+        public void run() {
+                int count = 0;
+                int size=to-from;
+                
+                for (int i=from; i<to;i++) {
+                        NumberSequence ns1 = appData.getNumberSequences().get(i);
+                                count++;
+                                printProgressPercentage(count,size);
+                                for (int j = 0; j < appData.getNumberSequences().size(); j++) {
+                                        NumberSequence ns2 = appData.getNumberSequences().get(j);
+                                        if (i != j) {
+                                                AlignmentAlgorithm aa = AlignmentAlgorithmFactory.create();
+                                                aa.align(ns1, ns2, appData.getSubmat(),9);
+                                                synchronized(appData.getMatchseqs()) {
+                                                        appData.getMatchseqs().addAll(aa.getMatches());
+                                                }
+                                        }
+                                }
+                }  
         }
-        }
-        }
-        
-        
-
+        }
+
+        private LinkedList<Match> generatePairwiseAlignments(RuleApplicationData appData) {
+                int numberSeqSize = appData.getNumberSequences().size();
+                LinkedList<Match> result;
+                //Checking if i have an already calculated file result of this action in the working directory 
+                File aligned = new File("aligned" + iteration + ".dat");
+                if(!(aligned.exists() && !aligned.isDirectory())) {
+                        Console.traceln(Level.INFO, "generating pairwise alignments from " + numberSeqSize + " sessions");
+                        result = new LinkedList<Match>();
+                        
+                        ExecutorService executor = Executors.newFixedThreadPool(nThreads);
+                        int interval = Math.round(numberSeqSize/nThreads);
+                        int rest = numberSeqSize % nThreads;
+                        
+                        for (int i = 0; i < numberSeqSize-interval; i+=interval) {
+                                int offset = 0;
+                                if(rest!=0) {
+                                        offset=1;
+                                        rest--;
+                                }
+                                int from = i;
+                                int to = i+interval+offset;
+                                System.out.println("Creating thread with matches from " + from + " to " + to);
+                                ParallelPairwiseAligner aligner = new ParallelPairwiseAligner(appData,from,to);
+                                executor.execute(aligner);
+                        }
+                        executor.shutdown();
+                        try {
+                                executor.awaitTermination(10, TimeUnit.MINUTES);
+                        } catch (InterruptedException e) {
+                                // TODO Auto-generated catch block
+                                e.printStackTrace();
+                        }
+        
+                        GlobalDataContainer.getInstance().addData("aligned" + iteration, appData);
+                        saveAppData("aligned" + iteration);
+                        return result;
+                }
+                else {
+                        Console.traceln(Level.INFO,"loading matches from file");
+                        appData = loadAppData("aligned"+iteration);
+                        return appData.getMatchseqs();
+                }
+        }
+        
 
         /**
@@ -947 +981 @@
                         newTasks = new LinkedList<ITask>();
                         
-                        //this.detectedAndReplacedTasks = true;
+                        this.detectedAndReplacedTasks = true;
                 }