From 8df7df2e721d7015e685a803608be634b7a8bfe6 Mon Sep 17 00:00:00 2001 From: adamopolous Date: Tue, 17 Sep 2019 07:57:57 -0400 Subject: [PATCH] removed timing considerations from test sorting --- gradle/root/test.gradle | 37 +++--- gradle/support/app_config_breakout.txt | 1 + gradle/support/testUtils.gradle | 162 ++++++++++++++++++------- 3 files changed, 140 insertions(+), 60 deletions(-) diff --git a/gradle/root/test.gradle b/gradle/root/test.gradle index 479abddd31..3000852453 100644 --- a/gradle/root/test.gradle +++ b/gradle/root/test.gradle @@ -367,6 +367,15 @@ def createTestTask(Project subproject, String testType, String bucketName, int t group "test" testClassesDirs = files subproject.sourceSets["$testType"].output.classesDirs classpath = subproject.sourceSets["$testType"].runtimeClasspath + + logger.info("********TEST CREATION") + logger.info(" " + subproject) + logger.info(" " + testType) + logger.info(" " + bucketName) + logger.info(" " + taskNameCounter) + logger.info(" " + classesList) + logger.info(" " + classesListPosition) + logger.info(" " + numMaxParallelForks) maxParallelForks = numMaxParallelForks @@ -419,9 +428,9 @@ configure(subprojects.findAll {parallelMode == true}) { subproject -> if (!shouldSkipTestTaskCreation(subproject)) { logger.info("parallelCombinedTestReport: Creating 'test' tasks for " + subproject.name + " subproject.") - Map testMap = getTestsForSubProject(subproject.sourceSets.test.java) + Map testMap = getTestsForSubProject(subproject.sourceSets.test.java) - for (Map.Entry classMap : testMap.entrySet()) { + for (Map.Entry classMap : testMap.entrySet()) { String bucketName = classMap.getKey(); @@ -429,12 +438,10 @@ configure(subprojects.findAll {parallelMode == true}) { subproject -> int taskNameCounter = 1 // task suffix int numMaxParallelForks = 40 // unit tests are fast; 40 seems to be a reasonable number - Map tests = classMap.getValue(); - def sorted = tests.sort { a, b -> b.value <=> a.value }; - List classesList = new ArrayList(sorted.keySet()); + List tests = classMap.getValue(); - while (classesListPosition < classesList.size()) { - createTestTask(subproject, "test", bucketName, taskNameCounter, classesList, classesListPosition, numMaxParallelForks) + while (classesListPosition < tests.size()) { + createTestTask(subproject, "test", bucketName, taskNameCounter, tests, classesListPosition, numMaxParallelForks) classesListPosition+=numMaxParallelForks taskNameCounter+=1; // "test_1_appConfig", "test_2_appConfig, etc. } @@ -444,10 +451,12 @@ configure(subprojects.findAll {parallelMode == true}) { subproject -> if (!shouldSkipIntegrationTestTaskCreation(subproject)) { logger.info("parallelCombinedTestReport: Creating 'integrationTest' tasks for " + subproject.name + " subproject.") - Map testMap = getTestsForSubProject(subproject.sourceSets.integrationTest.java) + Map testMap = getTestsForSubProject(subproject.sourceSets.integrationTest.java) + logger.info("====CREATING TASKS FOR: " + subproject) - for (Map.Entry classMap : testMap.entrySet()) { + for (Map.Entry classMap : testMap.entrySet()) { + logger.info(" bucket: " + classMap.getKey()) String bucketName = classMap.getKey(); int classesListPosition = 0 // current position in classesList @@ -458,12 +467,12 @@ configure(subprojects.findAll {parallelMode == true}) { subproject -> // 20 seems like a good balance of throughput vs resource usage for ghidratest server. int numMaxParallelForks = 20 - Map tests = classMap.getValue(); - def sorted = tests.sort { a, b -> b.value <=> a.value }; - List classesList = new ArrayList(sorted.keySet()); + List tests = classMap.getValue(); - while (classesListPosition < classesList.size()) { - createTestTask(subproject, "integrationTest", bucketName, taskNameCounter, classesList, classesListPosition, numMaxParallelForks) + logger.info(" tests: " + tests) + + while (classesListPosition < tests.size()) { + createTestTask(subproject, "integrationTest", bucketName, taskNameCounter, tests, classesListPosition, numMaxParallelForks) classesListPosition+=numMaxParallelForks taskNameCounter+=1; // "integrationTest_1_appConfig", "integrationTest_2_appConfig, etc. } diff --git a/gradle/support/app_config_breakout.txt b/gradle/support/app_config_breakout.txt index 1199881418..ad13021655 100644 --- a/gradle/support/app_config_breakout.txt +++ b/gradle/support/app_config_breakout.txt @@ -82,6 +82,7 @@ AbstractToolSavingTest AbstractVersionControlActionTest AbstractVTCorrelatorTest AbstractVTMarkupItemTest +CallTreePluginTest DiffTestAdapter DWARFTestBase AbstractSelfSimilarCorrelatorTest diff --git a/gradle/support/testUtils.gradle b/gradle/support/testUtils.gradle index 45dad3f85f..c2698084da 100644 --- a/gradle/support/testUtils.gradle +++ b/gradle/support/testUtils.gradle @@ -8,6 +8,11 @@ import java.io.*; // to tests (test name, duration) ext.testReport = null; +ext.integrationConfigs = new ArrayList<>(); +ext.dockingConfigs = new ArrayList<>(); +ext.appConfigs = new ArrayList<>(); +ext.ghidraConfigs = new ArrayList<>(); + /* * Checks if html test report for an individual test class has a valid name. */ @@ -67,6 +72,7 @@ long getDurationFromTestReportClass(String fileContents, String fileName) { * * eg: GhidraAppConfiguration -> DiffTestTypeAdapter, 0.135s */ + def Map> getTestReport() { // If we have already created the test report, do not waste time creating @@ -79,10 +85,6 @@ def Map> getTestReport() { testReport = new HashMap(); - List integrationConfigs = new ArrayList<>(); - List dockingConfigs = new ArrayList<>(); - List appConfigs = new ArrayList<>(); - List ghidraConfigs = new ArrayList<>(); parseApplicationConfigs(dockingConfigs, integrationConfigs, appConfigs, ghidraConfigs); File classesReportDir = new File(testTimeParserInputDir) @@ -118,7 +120,6 @@ def Map> getTestReport() { String shortName = fqNameFromTestReport.substring(nameIndex+1); long durationInMillis = getDurationFromTestReportClass(fileContents, file.name) - File rootDir = project.rootDir.getParentFile(); File foundFile; fileTree(rootDir.getAbsolutePath()).visit { FileVisitDetails details -> @@ -172,6 +173,12 @@ def Map> getTestReport() { testReport.put("ghidra", ghidraBucket); testReport.put("unknown", unknownBucket); + logger.debug("integration bucket: " + integrationBucket) + logger.debug("docking bucket: " + dockingBucket) + logger.debug("app bucket: " + appBucket) + logger.debug("ghidra bucket: " + ghidraBucket) + logger.debug("unknown bucket: " + unknownBucket) + logger.debug("getTestReport: Added to testReport: class name = '" + fqNameFromTestReport + "' and durationInMillis = '"+ durationInMillis +"' from " + file.name) @@ -304,7 +311,7 @@ String constructFullyQualifiedClassName(String fileContents, String fileName) { * Then traverses a test sourceSet for a subproject for a test to include and assigns a duration value. * Returns a sorted list of test classes for the sourceSet parameter. */ -def Map getTestsForSubProject(SourceDirectorySet sourceDirectorySet) { +def Map getTestsForSubProject(SourceDirectorySet sourceDirectorySet) { def testsForSubProject = new HashMap(); @@ -317,9 +324,18 @@ def Map getTestsForSubProject(SourceDirectorySet sourceDirectorySet logger.debug("getTestsForSubProject: Found " + sourceDirectorySet.files.size() + " file(s) in source set to process.") - Map testReports = getTestReport(); - assert (testReports != null) : "getTestsForSubProject: testReport should not be null" + //Map testReports = getTestReport(); + + parseApplicationConfigs(dockingConfigs, integrationConfigs, appConfigs, ghidraConfigs); + + //assert (testReports != null) : "getTestsForSubProject: testReport should not be null" + List dockingBucket = new ArrayList(); + List integrationBucket = new ArrayList(); + List appBucket = new ArrayList(); + List ghidraBucket = new ArrayList(); + List unknownBucket = new ArrayList(); + for (File file : sourceDirectorySet.getFiles()) { logger.debug("getTestsForSubProject: Found file in sourceSet = " + file.name) @@ -331,7 +347,7 @@ def Map getTestsForSubProject(SourceDirectorySet sourceDirectorySet } String fileContents = file.text - + // Must not have a Category annotation if (hasCategoryExcludes(fileContents)) { logger.debug("getTestsForSubProject: Found category exclude for '" @@ -339,38 +355,100 @@ def Map getTestsForSubProject(SourceDirectorySet sourceDirectorySet excludedClassFilesCategory++ continue } - - String fqName = constructFullyQualifiedClassName( fileContents, file.name) - boolean foundTest = false; - for (Map.Entry entry : testReports.entrySet()) { - String configName = entry.getKey(); - Map tests = entry.getValue(); + // Get any extending class so we can see what bucket it belongs to + // Match the word right after "extends", if there is one + Pattern p = Pattern.compile("extends\\W+(\\w+)"); + Matcher m = p.matcher(fileContents); + String extendsClass = ""; + while (m.find()) { + extendsClass = m.group(1); + break; + } + + + //String absFilename = file.getAbsolutePath(); + + // Get full package name of the class - this is what needs to go in the bucket + Pattern p2 = Pattern.compile("package\\s+([a-zA_Z_][\\.\\w]*);"); + Matcher m2 = p2.matcher(fileContents); + String packageName = ""; + while (m2.find()) { + packageName = m2.group(1); + break; + } + + String className = packageName + "." + file.name + className = className.replace(".java", "") - if (tests.containsKey(fqName)) { - foundTest = true; - if (!testsForSubProject.containsKey(configName)) { - Map configToTestMap = new LinkedHashMap<>(); - testsForSubProject.put(configName, configToTestMap); - } - - Map subTests = testsForSubProject.get(configName); - - long duration = tests.get(fqName); - - if (duration > 0) { - subTests.put(fqName,duration); - logger.debug("getTestsForSubProject: Adding '" + fqName + "'") - includedClassFilesInTestReport++ - } - else { - logger.debug("getTestsForSubProject: Excluding '" + fqName - + "' because duration from test report is " + duration - + "ms. Probably because all test methods are @Ignore'd." ) - excludedClassAllTestsIgnored++ - } + if (extendsClass.isEmpty()) { + unknownBucket.add(className); + } + else { + if (integrationConfigs.contains(extendsClass)) { + integrationBucket.add(className); + } + else if (dockingConfigs.contains(extendsClass)) { + dockingBucket.add(className); + } + else if (appConfigs.contains(extendsClass)) { + appBucket.add(className); + } + else if (ghidraConfigs.contains(extendsClass)) { + ghidraBucket.add(className); + } + else { + unknownBucket.add(className); } } + } + + testReport = new HashMap(); + testReport.put("docking", dockingBucket) + testReport.put("integration", integrationBucket) + testReport.put("app", appBucket) + testReport.put("ghidra", ghidraBucket) + testReport.put("unknown", unknownBucket) + + logger.debug("integration bucket: " + integrationBucket) + logger.debug("docking bucket: " + dockingBucket) + logger.debug("app bucket: " + appBucket) + logger.debug("ghidra bucket: " + ghidraBucket) + logger.debug("unknown bucket: " + unknownBucket) + + return testReport; + + /**String fqName = constructFullyQualifiedClassName( fileContents, file.name) + + boolean foundTest = false; + for (Map.Entry entry : testReport.entrySet()) { + String configName = entry.getKey(); + List tests = entry.getValue(); + + if (tests.contains(fqName)) { + foundTest = true; + if (!testsForSubProject.containsKey(configName)) { + Map configToTestMap = new LinkedHashMap<>(); + testsForSubProject.put(configName, configToTestMap); + } + + Map subTests = testsForSubProject.get(configName); + + long duration = tests.get(fqName); + + if (duration > 0) { + subTests.put(fqName,duration); + logger.debug("getTestsForSubProject: Adding '" + fqName + "'") + includedClassFilesInTestReport++ + } + else { + logger.debug("getTestsForSubProject: Excluding '" + fqName + + "' because duration from test report is " + duration + + "ms. Probably because all test methods are @Ignore'd." ) + excludedClassAllTestsIgnored++ + } + } + } if (!foundTest) { // Don't know what this test is so put it in the "unknown" bucket if (!testsForSubProject.containsKey("unknown")) { @@ -393,21 +471,13 @@ def Map getTestsForSubProject(SourceDirectorySet sourceDirectorySet testMap.sort { a, b -> b.value <=> a.value } } -/* logger.info ("getTestsForSubProject:\n" - + "\tIncluding " + includedClassFilesInTestReport + " test classes for this sourceSet because they are in the test report.\n" - + "\tIncluding/bumping " + includedClassFilesNotInTestReport + " not in test report.\n" - + "\tExcluding "+ excludedClassFilesBadName +" based on name not ending in 'Test' or contains 'Abstract' or 'Suite', " + excludedClassFilesCategory - + " based on '@Category, " + excludedClassAllTestsIgnored + " because duration = 0ms.\n" - + "\tReturning sorted list of size "+ sorted.size() + " out of " + sourceDirectorySet.files.size() - + " total files found in sourceSet.") -*/ - int filesProcessed = includedClassFilesNotInTestReport + includedClassFilesInTestReport + excludedClassFilesBadName + excludedClassFilesCategory + excludedClassAllTestsIgnored assert sourceDirectorySet.files.size() == filesProcessed : "getTestsForSubProject did not process every file in sourceSet" return testsForSubProject; + */ } /*********************************************************************************