Changeset 1729
- Timestamp:
- 09/05/14 14:17:47 (10 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
branches/autoquest-core-tasktrees-alignment/src/main/java/de/ugoe/cs/autoquest/tasktrees/temporalrelation/SequenceForTaskDetectionRuleAlignment.java
r1724 r1729 79 79 */ 80 80 public class SequenceForTaskDetectionRuleAlignment implements ISessionScopeRule { 81 82 public static int nThreads = Runtime.getRuntime().availableProcessors()-1; 83 84 81 82 public static int nThreads = Runtime.getRuntime().availableProcessors() - 1; 83 85 84 private int iteration = 0; 86 85 /** … … 98 97 */ 99 98 private ITaskBuilder taskBuilder; 100 101 99 102 100 /** … … 161 159 } 162 160 } 163 161 164 162 public RuleApplicationData loadAppData(String name) { 165 163 String objectName = name; 166 164 String filename = name + ".dat"; 167 165 168 166 Object data = null; 169 167 FileInputStream fis = null; … … 182 180 CommandHelpers.dataOverwritten(objectName); 183 181 } 184 return (RuleApplicationData) GlobalDataContainer.getInstance().getData(name); 185 } 186 182 return (RuleApplicationData) GlobalDataContainer.getInstance().getData( 183 name); 184 } 185 187 186 /* 188 187 * (non-Javadoc) … … 195 194 public RuleApplicationResult apply(List<IUserSession> sessions) { 196 195 RuleApplicationData appData = new RuleApplicationData(sessions); 197 // File harmonized = new File("harmonized.dat");198 // if(harmonized.exists() && !harmonized.isDirectory()) {199 // 200 // 201 // }202 // else {203 //appData.getStopWatch().start("harmonization");204 205 //appData.getStopWatch().stop("harmonization");206 207 //Saving intermediate results to file208 Console.traceln(Level.INFO,"saving substitution matrix to file");209 //saveAppData("harmonized");210 // }211 212 // File substitution = new File("substitution.dat");213 // if(!(substitution.exists() && !substitution.isDirectory())) {214 Console.traceln(Level.INFO, "generating substitution matrix from " + appData.getUniqueTasks().size() + " unique tasks");215 appData.getStopWatch().start("substitution matrix");216 appData.getSubmat().generate(appData.getUniqueTasks());217 appData.getStopWatch().stop("substitution matrix");218 // GlobalDataContainer.getInstance().addData("substitution", appData);219 // saveAppData("substitution");220 // }221 // else {222 // Console.traceln(Level.INFO,"loading substitution matrix from file");223 // appData = loadAppData("substitution");224 // }225 226 196 // File harmonized = new File("harmonized.dat"); 197 // if(harmonized.exists() && !harmonized.isDirectory()) { 198 // Console.traceln(Level.INFO,"loading harmonized sessions from file"); 199 // appData = loadAppData("harmonized"); 200 // } 201 // else { 202 // appData.getStopWatch().start("harmonization"); 203 harmonizeEventTaskInstancesModel(appData); 204 // appData.getStopWatch().stop("harmonization"); 205 GlobalDataContainer.getInstance().addData("harmonized", appData); 206 // Saving intermediate results to file 207 Console.traceln(Level.INFO, "saving substitution matrix to file"); 208 // saveAppData("harmonized"); 209 // } 210 211 // File substitution = new File("substitution.dat"); 212 // if(!(substitution.exists() && !substitution.isDirectory())) { 213 Console.traceln(Level.INFO, "generating substitution matrix from " 214 + appData.getUniqueTasks().size() + " unique tasks"); 215 appData.getStopWatch().start("substitution matrix"); 216 appData.getSubmat().generate(appData.getUniqueTasks()); 217 appData.getStopWatch().stop("substitution matrix"); 218 // GlobalDataContainer.getInstance().addData("substitution", appData); 219 // saveAppData("substitution"); 220 // } 221 // else { 222 // Console.traceln(Level.INFO,"loading substitution matrix from file"); 223 // appData = loadAppData("substitution"); 224 // } 225 227 226 Console.traceln(Level.INFO, "Starting main loop"); 228 227 do { … … 256 255 return appData.getResult(); 257 256 } 258 259 257 260 258 private ArrayList<NumberSequence> createNumberSequences( … … 531 529 * @return 532 530 */ 533 public ISequence matchAsSequence(RuleApplicationData appData, Match m) { 531 public ISequence matchAsSequence(RuleApplicationData appData, Match m) { 534 532 ISequence sequence = taskFactory.createNewSequence(); 535 533 appData.newTaskCreated(sequence); … … 598 596 taskBuilder.addChild(selection, subsequence1); 599 597 taskBuilder.addChild(selection, subsequence2); 600 taskBuilder.addChild(sequence, selection);598 taskBuilder.addChild(sequence, selection); 601 599 while (i < first.length - 1 && selectionfound) { 602 600 selectionfound = false; … … 608 606 && second[i + 1] != -1) { 609 607 selectionfound = true; 610 } 611 else{ 608 } else { 612 609 continue; 613 610 } 614 611 i++; 615 612 } 616 if (i==first.length-1 && selectionfound) {613 if (i == first.length - 1 && selectionfound) { 617 614 taskBuilder.addChild(subsequence1, appData 618 615 .getNumber2Task().get(first[i])); … … 621 618 } 622 619 } 623 } 624 else { 620 } else { 625 621 if ((first[i] != second[i])) { 626 622 … … 651 647 // Create NumberSequences 652 648 appData.setNumberSequences(this.createNumberSequences(appData)); 653 649 654 650 // Generate pairwise alignments 655 // appData.setMatchseqs(generatePairwiseAlignments(appData));651 // appData.setMatchseqs(generatePairwiseAlignments(appData)); 656 652 generatePairwiseAlignments(appData); 657 653 658 // Searching each match in all other sessions, counting its occurences654 // Searching each match in all other sessions, counting its occurences 659 655 searchMatchesInAllSessions(appData); 660 656 661 657 // Sort results to get the most occurring results 662 Console.traceln(Level.INFO, "sorting " + appData.getMatchseqs().size() + " results"); 658 Console.traceln(Level.INFO, "sorting " + appData.getMatchseqs().size() 659 + " results"); 663 660 Comparator<Match> comparator = new Comparator<Match>() { 664 661 public int compare(Match m1, Match m2) { … … 667 664 } 668 665 }; 669 666 670 667 Collections.sort(appData.getMatchseqs(), comparator); 671 668 appData.getStopWatch().stop("detecting tasks"); … … 676 673 HashMap<Integer, List<MatchOccurence>> replacedOccurences = new HashMap<Integer, List<MatchOccurence>>(); 677 674 for (int i = 0; i < appData.getMatchseqs().size(); i++) { 678 675 679 676 // Every pattern consists of 2 sequences, therefore the minimum 680 677 // occurrences here is 2. … … 683 680 if (appData.getMatchseqs().get(i).occurenceCount() > 2) { 684 681 appData.detectedAndReplacedTasks = true; 685 ISequence task = matchAsSequence(appData, appData.getMatchseqs().get(i)); 686 invalidOccurence: for (Iterator<MatchOccurence> it = appData.getMatchseqs() 687 .get(i).getOccurences().iterator(); it.hasNext();) { 682 ISequence task = matchAsSequence(appData, appData 683 .getMatchseqs().get(i)); 684 invalidOccurence: for (Iterator<MatchOccurence> it = appData 685 .getMatchseqs().get(i).getOccurences().iterator(); it 686 .hasNext();) { 688 687 MatchOccurence oc = it.next(); 689 688 690 689 // Check if nothing has been replaced in the sequence we 691 690 // want to replace … … 731 730 } 732 731 } 733 System.out.println("Replacing in sequence" + oc.getSequenceId()); 732 System.out.println("Replacing in sequence" 733 + oc.getSequenceId()); 734 734 ISequenceInstance sequenceInstances = RuleUtils 735 735 .createNewSubSequenceInRange(appData.getSessions() … … 746 746 } 747 747 } 748 // appData.setMatchseqs(null);748 // appData.setMatchseqs(null); 749 749 appData.getStopWatch().stop("replacing tasks"); 750 750 } 751 752 753 751 752 private void replaceMatches() { 753 754 } 754 755 755 756 private void searchMatchesInAllSessions(RuleApplicationData appData) { 756 Console.traceln(Level.INFO, "searching for patterns occuring most with " + nThreads + " threads"); 757 //Prepare parallel search of matchseqs 757 Console.traceln(Level.INFO, 758 "searching for patterns occuring most with " + nThreads 759 + " threads"); 760 // Prepare parallel search of matchseqs 758 761 ExecutorService executor = Executors.newFixedThreadPool(nThreads); 759 int matchSeqSize =appData.getMatchseqs().size();760 int interval = Math.round(matchSeqSize/nThreads);762 int matchSeqSize = appData.getMatchseqs().size(); 763 int interval = matchSeqSize / nThreads; 761 764 int rest = matchSeqSize % nThreads; 762 763 for (int i =0;i<matchSeqSize-interval;i+=interval) {764 int offset = 0;765 if (rest!=0) {766 offset =1;765 766 for (int i = 0; i < matchSeqSize - interval; i += interval) { 767 int offset = 0; 768 if (rest != 0) { 769 offset = 1; 767 770 rest--; 768 771 } 769 772 int from = i; 770 int to = i+interval+offset; 771 System.out.println("Creating thread with matches from " + from + " to " + to); 773 int to = i + interval + offset; 774 System.out.println("Creating thread with matches from " + from 775 + " to " + to); 772 776 // search each match in every other sequence 773 ParallelMatchOcurrencesFinder finder = new ParallelMatchOcurrencesFinder(appData,from,to); 777 ParallelMatchOcurrencesFinder finder = new ParallelMatchOcurrencesFinder( 778 appData, from, to); 774 779 executor.execute(finder); 775 780 } … … 781 786 e.printStackTrace(); 782 787 } 783 784 } 785 786 //Print out the progress 787 private static void printProgressPercentage(int count, int size) { 788 if(size>100) { 789 if((count%(size/100)==0)) { 790 //Console.traceln(Level.INFO,("Thread" + Thread.currentThread().getName() + ": " + Math.round((float) count/size*100))+ "%"); 791 System.out.println("Thread" + Thread.currentThread().getName() + ": " + Math.round((float) count/size*100)+ "%"); 792 } 793 } 794 else { 795 //Console.traceln(Level.INFO,("Thread" + Thread.currentThread().getName() + ": " +Math.round((float) count/size*100))+ "%"); 796 System.out.println("Thread" + Thread.currentThread().getName() + ": " +Math.round((float) count/size*100)+ "%"); 797 798 } 799 } 800 801 802 803 788 789 } 790 791 // Print out the progress 792 private static void printProgressPercentage(int count, int size) { 793 if (size > 100) { 794 if ((count % (size / 100) == 0)) { 795 // Console.traceln(Level.INFO,("Thread" + 796 // Thread.currentThread().getName() + ": " + Math.round((float) 797 // count/size*100))+ "%"); 798 System.out.println("Thread" + Thread.currentThread().getName() 799 + ": " + Math.round((float) count / size * 100) + "%"); 800 } 801 } else { 802 // Console.traceln(Level.INFO,("Thread" + 803 // Thread.currentThread().getName() + ": " +Math.round((float) 804 // count/size*100))+ "%"); 805 System.out.println("Thread" + Thread.currentThread().getName() 806 + ": " + Math.round((float) count / size * 100) + "%"); 807 808 } 809 } 810 811 private class ParallelMatchReplacer implements Runnable { 812 813 @Override 814 public void run() { 815 816 } 817 818 } 819 804 820 private class ParallelMatchOcurrencesFinder implements Runnable { 805 private final RuleApplicationData appData; 806 private final int from; 807 private final int to; 808 ParallelMatchOcurrencesFinder(RuleApplicationData appData, int from, int to) { 809 this.appData = appData; 810 this.from = from; 811 this.to = to; 812 } 813 814 @Override 815 public void run() { 816 int count = 0; 817 int size=to-from; 818 819 for (int i=from; i<to;i++) { 820 Match pattern = appData.getMatchseqs().get(i); 821 count++; 822 printProgressPercentage(count,size); 823 // Skip sequences with more 0 events (scrolls) than other events. 824 // Both of the pattern sequences are equally long, so the zero 825 // counts just need to be smaller than the length of one sequence 826 if (pattern.getFirstSequence().eventCount(0) 827 + pattern.getSecondSequence().eventCount(0) + 1 > pattern 828 .getFirstSequence().size()) 829 continue; 830 831 for (int j = 0; j < appData.getNumberSequences().size(); j++) { 832 LinkedList<Integer> startpositions = appData 833 .getNumberSequences().get(j).containsPattern(pattern); 834 if (startpositions.size() > 0) { 835 for (Iterator<Integer> jt = startpositions.iterator(); jt 836 .hasNext();) { 837 int start = jt.next(); 838 pattern.addOccurence(new MatchOccurence(start, start 839 + pattern.size(), j)); 840 } 841 } 842 } 843 } 844 } 845 } 846 847 821 private final RuleApplicationData appData; 822 private final int from; 823 private final int to; 824 825 ParallelMatchOcurrencesFinder(RuleApplicationData appData, int from, 826 int to) { 827 this.appData = appData; 828 this.from = from; 829 this.to = to; 830 } 831 832 @Override 833 public void run() { 834 int count = 0; 835 int size = to - from; 836 837 for (int i = from; i < to; i++) { 838 Match pattern = appData.getMatchseqs().get(i); 839 count++; 840 printProgressPercentage(count, size); 841 // Skip sequences with more 0 events (scrolls) than other 842 // events. 843 // Both of the pattern sequences are equally long, so the zero 844 // counts just need to be smaller than the length of one 845 // sequence 846 if (pattern.getFirstSequence().eventCount(0) 847 + pattern.getSecondSequence().eventCount(0) + 1 > pattern 848 .getFirstSequence().size()) 849 continue; 850 851 for (int j = 0; j < appData.getNumberSequences().size(); j++) { 852 LinkedList<Integer> startpositions = appData 853 .getNumberSequences().get(j) 854 .containsPattern(pattern); 855 if (startpositions.size() > 0) { 856 for (Iterator<Integer> jt = startpositions.iterator(); jt 857 .hasNext();) { 858 int start = jt.next(); 859 pattern.addOccurence(new MatchOccurence(start, 860 start + pattern.size(), j)); 861 } 862 } 863 } 864 } 865 } 866 } 867 848 868 private class ParallelPairwiseAligner implements Runnable { 849 private final RuleApplicationData appData; 850 private final int from; 851 private final int to; 852 ParallelPairwiseAligner(RuleApplicationData appData, int from, int to) { 853 this.appData = appData; 854 this.from = from; 855 this.to = to; 856 } 857 858 @Override 859 public void run() { 860 int count = 0; 861 int size=to-from; 862 863 for (int i=from; i<to;i++) { 864 NumberSequence ns1 = appData.getNumberSequences().get(i); 869 private final RuleApplicationData appData; 870 private final int from; 871 private final int to; 872 873 ParallelPairwiseAligner(RuleApplicationData appData, int from, int to) { 874 this.appData = appData; 875 this.from = from; 876 this.to = to; 877 } 878 879 @Override 880 public void run() { 881 int count = 0; 882 int size = to - from; 883 884 for (int i = from; i < to; i++) { 885 NumberSequence ns1 = appData.getNumberSequences().get(i); 865 886 count++; 866 printProgressPercentage(count, size);887 printProgressPercentage(count, size); 867 888 for (int j = 0; j < appData.getNumberSequences().size(); j++) { 868 889 NumberSequence ns2 = appData.getNumberSequences().get(j); 869 890 if (i != j) { 870 AlignmentAlgorithm aa = AlignmentAlgorithmFactory.create(); 871 aa.align(ns1, ns2, appData.getSubmat(),9); 872 synchronized(appData.getMatchseqs()) { 891 AlignmentAlgorithm aa = AlignmentAlgorithmFactory 892 .create(); 893 aa.align(ns1, ns2, appData.getSubmat(), 9); 894 synchronized (appData.getMatchseqs()) { 873 895 appData.getMatchseqs().addAll(aa.getMatches()); 874 896 } 875 897 } 876 898 } 877 } 878 } 879 } 880 881 //private LinkedList<Match> generatePairwiseAlignments(RuleApplicationData appData) { 899 } 900 } 901 } 902 903 // private LinkedList<Match> generatePairwiseAlignments(RuleApplicationData 904 // appData) { 882 905 private void generatePairwiseAlignments(RuleApplicationData appData) { 883 906 int numberSeqSize = appData.getNumberSequences().size(); 884 907 appData.matchseqs = new LinkedList<Match>(); 885 //Checking if i have an already calculated file result of this action in the working directory 886 //File aligned = new File("aligned" + iteration + ".dat"); 887 //if(!(aligned.exists() && !aligned.isDirectory())) { 888 Console.traceln(Level.INFO, "generating pairwise alignments from " + numberSeqSize + " sessions with " + nThreads + " threads"); 889 ExecutorService executor = Executors.newFixedThreadPool(nThreads); 890 int interval = Math.round(numberSeqSize/nThreads); 891 int rest = numberSeqSize % nThreads; 892 893 for (int i = 0; i < numberSeqSize-interval; i+=interval) { 894 int offset = 0; 895 if(rest!=0) { 896 offset=1; 897 rest--; 898 } 899 int from = i; 900 int to = i+interval+offset; 901 System.out.println("Creating thread for sessions " + from + " till " + to); 902 ParallelPairwiseAligner aligner = new ParallelPairwiseAligner(appData,from,to); 903 executor.execute(aligner); 904 } 905 executor.shutdown(); 906 try { 907 executor.awaitTermination(10, TimeUnit.MINUTES); 908 } catch (InterruptedException e) { 909 // TODO Auto-generated catch block 910 e.printStackTrace(); 911 } 912 913 //GlobalDataContainer.getInstance().addData("aligned" + iteration, appData); 914 //saveAppData("aligned" + iteration); 915 // return result; 916 } 917 //else { 918 // Console.traceln(Level.INFO,"loading matches from file"); 919 // appData = loadAppData("aligned"+iteration); 920 // return appData.getMatchseqs(); 921 //} 922 908 Console.traceln(Level.INFO, "generating pairwise alignments from " 909 + numberSeqSize + " sessions with " + nThreads + " threads"); 910 ExecutorService executor = Executors.newFixedThreadPool(nThreads); 911 int interval = numberSeqSize / nThreads; 912 int rest = numberSeqSize % nThreads; 913 914 for (int i = 0; i < numberSeqSize - interval; i += interval) { 915 int offset = 0; 916 if (rest != 0) { 917 offset = 1; 918 rest--; 919 } 920 int from = i; 921 int to = i + interval + offset; 922 System.out.println("Creating thread for sessions " + from 923 + " till " + to); 924 ParallelPairwiseAligner aligner = new ParallelPairwiseAligner( 925 appData, from, to); 926 executor.execute(aligner); 927 } 928 executor.shutdown(); 929 try { 930 executor.awaitTermination(2, TimeUnit.HOURS); 931 } catch (InterruptedException e) { 932 // TODO Auto-generated catch block 933 e.printStackTrace(); 934 } 935 936 } 923 937 924 938 /** … … 934 948 private HashMap<Integer, ITask> number2task; 935 949 936 // TODO: We Actually just need number2task here, this structure can be removed in the future. 950 // TODO: We Actually just need number2task here, this structure can be 951 // removed in the future. 937 952 private HashSet<ITask> uniqueTasks; 953 954 ObjectDistanceSubstitionMatrix submat; 938 955 939 ObjectDistanceSubstitionMatrix submat;940 956 957 941 958 LinkedList<Match> matchseqs; 942 959 943 960 private ArrayList<NumberSequence> numberseqs; 944 961 945 962 private LinkedList<ITask> newTasks; 946 963 947 964 /** 948 965 * … … 975 992 stopWatch = new StopWatch(); 976 993 result = new RuleApplicationResult(); 977 submat = new ObjectDistanceSubstitionMatrix(6,-3,false);994 submat = new ObjectDistanceSubstitionMatrix(6, -3, false); 978 995 newTasks = new LinkedList<ITask>(); 979 996 this.detectedAndReplacedTasks = true; … … 984 1001 } 985 1002 986 987 1003 private ObjectDistanceSubstitionMatrix getSubmat() { 988 1004 return submat; 989 1005 } 990 1006 991 1007 private void resetNewlyCreatedTasks() { 992 1008 uniqueTasks.addAll(newTasks); 993 1009 newTasks.clear(); 994 1010 } 995 1011 996 1012 private void newTaskCreated(ITask task) { 997 1013 number2task.put(task.getId(), task); … … 1023 1039 } 1024 1040 1025 1026 1027 1041 /** 1028 1042 * … … 1042 1056 return newTasks; 1043 1057 } 1044 1058 1045 1059 /** 1046 1060 * @return the stopWatch
Note: See TracChangeset
for help on using the changeset viewer.
