diff options
| -rw-r--r-- | tests/hostlib/src/com/android/testutils/host/DeflakeHostTestBase.kt | 99 |
1 files changed, 88 insertions, 11 deletions
diff --git a/tests/hostlib/src/com/android/testutils/host/DeflakeHostTestBase.kt b/tests/hostlib/src/com/android/testutils/host/DeflakeHostTestBase.kt index 987cf23..e4cc2ff 100644 --- a/tests/hostlib/src/com/android/testutils/host/DeflakeHostTestBase.kt +++ b/tests/hostlib/src/com/android/testutils/host/DeflakeHostTestBase.kt @@ -17,15 +17,35 @@ package com.android.testutils.host import com.android.tests.util.ModuleTestUtils +import com.android.tradefed.config.Option import com.android.tradefed.testtype.DeviceJUnit4ClassRunner import com.android.tradefed.testtype.junit4.BaseHostJUnit4Test +import com.android.tradefed.testtype.junit4.DeviceTestRunOptions import com.android.tradefed.util.AaptParser import org.junit.Before import org.junit.Test import org.junit.runner.RunWith -import kotlin.test.assertTrue import kotlin.test.fail +private data class TestFailure(val description: String, val stacktrace: String) + +/** + * Base class for host-driven tests to deflake a test package. + * + * <p>Classes implementing this base class must define a test APK to be run, and default run + * count, timeout and test classes. In manual runs, the run count, timeout and test classes can be + * overridden via command-line parameters, such as: + * + * <pre> + * atest TestName -- \ + * --test-arg com.android.tradefed.testtype.HostTest:set-option:deflake_run_count:10 \ + * --test-arg com.android.tradefed.testtype.HostTest:set-option:deflake_single_run_timeout:10s \ + * --test-arg \ + * com.android.tradefed.testtype.HostTest:set-option:deflake_test:one.test.Class \ + * --test-arg \ + * com.android.tradefed.testtype.HostTest:set-option:deflake_test:another.test.Class + * </pre> + */ @RunWith(DeviceJUnit4ClassRunner::class) abstract class DeflakeHostTestBase : BaseHostJUnit4Test() { @@ -34,6 +54,11 @@ abstract class DeflakeHostTestBase : BaseHostJUnit4Test() { */ protected abstract val runCount: Int + @Option(name = "deflake_run_count", + description = "How many times to run each test case.", + importance = Option.Importance.ALWAYS) + private var mRunCountOption: Int? = null + /** * Filename of the APK to run as part of the test. * @@ -48,11 +73,24 @@ abstract class DeflakeHostTestBase : BaseHostJUnit4Test() { */ protected open val singleRunTimeoutMs = 5 * 60_000L + @Option(name = "deflake_single_run_timeout", + description = "Timeout for each single run.", + importance = Option.Importance.ALWAYS, + isTimeVal = true) + private var mSingleRunTimeoutMsOption: Long? = null + /** * List of classes to run in the test package. If empty, all classes in the package will be run. */ protected open val testClasses: List<String> = emptyList() + // TODO: also support single methods, not just whole classes + @Option(name = "deflake_test", + description = "Test class to deflake. Can be repeated. " + + "Default classes configured for the test are run if omitted.", + importance = Option.Importance.ALWAYS) + private var mTestClassesOption: ArrayList<String?> = ArrayList() + @Before fun setUp() { // APK will be auto-cleaned @@ -62,16 +100,55 @@ abstract class DeflakeHostTestBase : BaseHostJUnit4Test() { @Test fun testDeflake() { val apkFile = ModuleTestUtils(this).getTestFile(testApkFilename) - val pkgName = AaptParser.parse(apkFile)?.packageName ?: - fail("Could not parse test package name") - // null class name runs all classes in the package - val tc = if (testClasses.isEmpty()) listOf(null) else testClasses - - repeat(runCount) { - // TODO: improve reporting by always running all tests and counting flakes - tc.forEach { - assertTrue(runDeviceTests(pkgName, it, singleRunTimeoutMs)) + val pkgName = AaptParser.parse(apkFile)?.packageName + ?: fail("Could not parse test package name") + + val classes = mTestClassesOption.filterNotNull().ifEmpty { testClasses } + .ifEmpty { listOf(null) } // null class name runs all classes in the package + val runOptions = DeviceTestRunOptions(pkgName) + .setDevice(device) + .setTestTimeoutMs(mSingleRunTimeoutMsOption ?: singleRunTimeoutMs) + .setCheckResults(false) + // Pair is (test identifier, last stacktrace) + val failures = ArrayList<TestFailure>() + val count = mRunCountOption ?: runCount + repeat(count) { + classes.forEach { testClass -> + runDeviceTests(runOptions.setTestClassName(testClass)) + failures += getLastRunFailures() + } + } + if (failures.isEmpty()) return + val failuresByTest = failures.groupBy(TestFailure::description) + val failMessage = failuresByTest.toList().fold("") { msg, (testDescription, failures) -> + val stacktraces = formatStacktraces(failures) + msg + "\n$testDescription: ${failures.count()}/$count failures. " + + "Stacktraces:\n$stacktraces" + } + fail("Some tests failed:$failMessage") + } + + private fun getLastRunFailures(): List<TestFailure> { + with(lastDeviceRunResults) { + if (isRunFailure) { + return listOf(TestFailure("All tests in run", runFailureMessage)) } + + return failedTests.map { + val stackTrace = testResults[it]?.stackTrace + ?: fail("Missing stacktrace for failed test $it") + TestFailure(it.toString(), stackTrace) + } + } + } + + private fun formatStacktraces(failures: List<TestFailure>): String { + // Calculate list of (stacktrace, frequency) pairs ordered from most to least frequent + val frequencies = failures.groupingBy(TestFailure::stacktrace).eachCount().toList() + .sortedByDescending { it.second } + // Print each stacktrace with its frequency + return frequencies.fold("") { msg, (stacktrace, numFailures) -> + "$msg\n$numFailures failures:\n$stacktrace" } } -}
\ No newline at end of file +} |