Add a test runner to ignore tests on old SDKs

Generally DevSdkIgnoreRule should be used for that purpose (using rules
is preferable over replacing the test runner), however JUnit runners
inspect all methods in the test class before processing test rules. This
may cause issues if the test methods are referencing classes that do not
exist on the SDK of the device the test is run on.

Test: tests in change Iddd00e1c85abe767b1a41a1761d3266ba322dba6
Bug: 150918852
Merged-In: Ia8df394ed65fad3a3333da32c868c4623975ff79
(cherry picked from commit 1d2d5731c2ac4a23904f6093ac0f2a1bf0c11985, aosp/1281921)

Change-Id: Iedc8d1f93254146baef61b34b965a89fee1b4c57

3 files changed, 94 insertions, 10 deletions

diff --git a/tests/lib/Android.bp b/tests/lib/Android.bp
index d43243f..47b950d 100644
--- a/tests/lib/Android.bp
+++ b/tests/lib/Android.bp
@@ -39,6 +39,7 @@ java_library {
         "androidx.annotation_annotation",
     ],
     static_libs: [
+        "androidx.test.ext.junit",
         "net-tests-utils-multivariant",
     ],
 }
diff --git a/tests/lib/src/com/android/testutils/DevSdkIgnoreRule.kt b/tests/lib/src/com/android/testutils/DevSdkIgnoreRule.kt
index d30138d..4a83f6f 100644
--- a/tests/lib/src/com/android/testutils/DevSdkIgnoreRule.kt
+++ b/tests/lib/src/com/android/testutils/DevSdkIgnoreRule.kt
@@ -23,6 +23,23 @@ import org.junit.runner.Description
 import org.junit.runners.model.Statement
 
 /**
+ * Returns true if the development SDK version of the device is in the provided range.
+ *
+ * If the device is not using a release SDK, the development SDK is considered to be higher than
+ * [Build.VERSION.SDK_INT].
+ */
+fun isDevSdkInRange(minExclusive: Int?, maxInclusive: Int?): Boolean {
+    // In-development API n+1 will have SDK_INT == n and CODENAME != REL.
+    // Stable API n has SDK_INT == n and CODENAME == REL.
+    val release = "REL" == Build.VERSION.CODENAME
+    val sdkInt = Build.VERSION.SDK_INT
+    val devApiLevel = sdkInt + if (release) 0 else 1
+
+    return (minExclusive == null || devApiLevel > minExclusive) &&
+            (maxInclusive == null || devApiLevel <= maxInclusive)
+}
+
+/**
  * A test rule to ignore tests based on the development SDK level.
  *
  * If the device is not using a release SDK, the development SDK is considered to be higher than
@@ -63,16 +80,10 @@ class DevSdkIgnoreRule @JvmOverloads constructor(
             val ignoreAfter = description.getAnnotation(IgnoreAfter::class.java)
             val ignoreUpTo = description.getAnnotation(IgnoreUpTo::class.java)
 
-            // In-development API n+1 will have SDK_INT == n and CODENAME != REL.
-            // Stable API n has SDK_INT == n and CODENAME == REL.
-            val release = "REL" == Build.VERSION.CODENAME
-            val sdkInt = Build.VERSION.SDK_INT
-            val devApiLevel = sdkInt + if (release) 0 else 1
-            val message = "Skipping test for ${if (!release) "non-" else ""}release SDK $sdkInt"
-            assumeTrue(message, ignoreClassAfter == null || devApiLevel <= ignoreClassAfter)
-            assumeTrue(message, ignoreClassUpTo == null || devApiLevel > ignoreClassUpTo)
-            assumeTrue(message, ignoreAfter == null || devApiLevel <= ignoreAfter.value)
-            assumeTrue(message, ignoreUpTo == null || devApiLevel > ignoreUpTo.value)
+            val message = "Skipping test for build ${Build.VERSION.CODENAME} " +
+                    "with SDK ${Build.VERSION.SDK_INT}"
+            assumeTrue(message, isDevSdkInRange(ignoreClassUpTo, ignoreClassAfter))
+            assumeTrue(message, isDevSdkInRange(ignoreUpTo?.value, ignoreAfter?.value))
             base.evaluate()
         }
     }
diff --git a/tests/lib/src/com/android/testutils/DevSdkIgnoreRunner.kt b/tests/lib/src/com/android/testutils/DevSdkIgnoreRunner.kt
new file mode 100644
index 0000000..73b2843
--- /dev/null
+++ b/tests/lib/src/com/android/testutils/DevSdkIgnoreRunner.kt
@@ -0,0 +1,72 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.testutils
+
+import androidx.test.ext.junit.runners.AndroidJUnit4
+import com.android.testutils.DevSdkIgnoreRule.IgnoreAfter
+import com.android.testutils.DevSdkIgnoreRule.IgnoreUpTo
+import org.junit.runner.Description
+import org.junit.runner.Runner
+import org.junit.runner.notification.RunNotifier
+
+/**
+ * A runner that can skip tests based on the development SDK as defined in [DevSdkIgnoreRule].
+ *
+ * Generally [DevSdkIgnoreRule] should be used for that purpose (using rules is preferable over
+ * replacing the test runner), however JUnit runners inspect all methods in the test class before
+ * processing test rules. This may cause issues if the test methods are referencing classes that do
+ * not exist on the SDK of the device the test is run on.
+ *
+ * This runner inspects [IgnoreAfter] and [IgnoreUpTo] annotations on the test class, and will skip
+ * the whole class if they do not match the development SDK as defined in [DevSdkIgnoreRule].
+ * Otherwise, it will delegate to [AndroidJUnit4] to run the test as usual.
+ *
+ * Example usage:
+ *
+ *     @RunWith(DevSdkIgnoreRunner::class)
+ *     @IgnoreUpTo(Build.VERSION_CODES.Q)
+ *     class MyTestClass { ... }
+ */
+class DevSdkIgnoreRunner(private val klass: Class<*>) : Runner() {
+    private val baseRunner = klass.let {
+        val ignoreAfter = it.getAnnotation(IgnoreAfter::class.java)
+        val ignoreUpTo = it.getAnnotation(IgnoreUpTo::class.java)
+
+        if (isDevSdkInRange(ignoreUpTo?.value, ignoreAfter?.value)) AndroidJUnit4(klass) else null
+    }
+
+    override fun run(notifier: RunNotifier) {
+        if (baseRunner != null) {
+            baseRunner.run(notifier)
+            return
+        }
+
+        // Report a single, skipped placeholder test for this class, so that the class is still
+        // visible as skipped in test results.
+        notifier.fireTestIgnored(
+                Description.createTestDescription(klass, "skippedClassForDevSdkMismatch"))
+    }
+
+    override fun getDescription(): Description {
+        return baseRunner?.description ?: Description.createSuiteDescription(klass)
+    }
+
+    override fun testCount(): Int {
+        // When ignoring the tests, a skipped placeholder test is reported, so test count is 1.
+        return baseRunner?.testCount() ?: 1
+    }
+}
\ No newline at end of file