Minor refactorings on Autofill tests:

- Move common code to AbstractAutofillTestCase.
- Copied existing codes to LoginTest, which does not use @Parameters.
- Deprecated AutofillPerfTest (it will be removed once LoginTest has some runs).
- Added some (temporary) no-op tests to help us investigate why some tests are
  missing from the dashboard.

Test: mmma -j ./frameworks/base/apct-tests/perftests/core/ &&  \
  adb install -r $OUT/data/app/CorePerfTests/CorePerfTests.apk && \
  adb shell am instrument -w -e class android.view.autofill.AutofillPerfTest \
  com.android.perftests.core/android.support.test.runner.AndroidJUnitRunner

Bug: 38345816

Change-Id: Iab703205f1e01a1e5413b10c3008aac8efcc5dc2

4 files changed, 396 insertions, 57 deletions

diff --git a/apct-tests/perftests/core/src/android/view/autofill/AbstractAutofillPerfTestCase.java b/apct-tests/perftests/core/src/android/view/autofill/AbstractAutofillPerfTestCase.java
new file mode 100644
index 000000000000..90d71d522836
--- /dev/null
+++ b/apct-tests/perftests/core/src/android/view/autofill/AbstractAutofillPerfTestCase.java
@@ -0,0 +1,116 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+
+package android.view.autofill;
+
+import android.os.Looper;
+import android.perftests.utils.PerfStatusReporter;
+import android.perftests.utils.SettingsHelper;
+import android.perftests.utils.SettingsStateKeeperRule;
+import android.perftests.utils.ShellHelper;
+import android.view.View;
+import android.perftests.utils.StubActivity;
+import android.provider.Settings;
+import android.support.test.filters.LargeTest;
+import android.support.test.rule.ActivityTestRule;
+import android.support.test.InstrumentationRegistry;
+
+import com.android.perftests.core.R;
+
+import java.util.Locale;
+import java.util.Collection;
+import java.util.Arrays;
+
+import org.junit.Test;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+import org.junit.runner.RunWith;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Base class for all autofill tests.
+ */
+@LargeTest
+public abstract class AbstractAutofillPerfTestCase {
+
+    @ClassRule
+    public static final SettingsStateKeeperRule mServiceSettingsKeeper =
+            new SettingsStateKeeperRule(InstrumentationRegistry.getTargetContext(),
+                    Settings.Secure.AUTOFILL_SERVICE);
+
+    @Rule
+    public ActivityTestRule<StubActivity> mActivityRule =
+            new ActivityTestRule<StubActivity>(StubActivity.class);
+
+    @Rule
+    public PerfStatusReporter mPerfStatusReporter = new PerfStatusReporter();
+
+    private final int mLayoutId;
+
+    protected AbstractAutofillPerfTestCase(int layoutId) {
+        mLayoutId = layoutId;
+    }
+
+    /**
+     * Prepares the activity so that by the time the test is run it has reference to its fields.
+     */
+    @Before
+    public void prepareActivity() throws Throwable {
+        mActivityRule.runOnUiThread(() -> {
+            assertTrue("We should be running on the main thread",
+                    Looper.getMainLooper().getThread() == Thread.currentThread());
+            assertTrue("We should be running on the main thread",
+                    Looper.myLooper() == Looper.getMainLooper());
+            StubActivity activity = mActivityRule.getActivity();
+            activity.setContentView(mLayoutId);
+            onCreate(activity);
+        });
+    }
+
+    @Before
+    public void resetStaticState() {
+        MyAutofillService.resetStaticState();
+    }
+
+    /**
+     * Initializes the {@link StubActivity} after it was launched.
+     */
+    protected abstract void onCreate(StubActivity activity);
+
+    /**
+     * Uses the {@code settings} binary to set the autofill service.
+     */
+    protected void setService() {
+        SettingsHelper.syncSet(InstrumentationRegistry.getTargetContext(),
+                SettingsHelper.NAMESPACE_SECURE,
+                Settings.Secure.AUTOFILL_SERVICE,
+                MyAutofillService.COMPONENT_NAME);
+    }
+
+    /**
+     * Uses the {@code settings} binary to reset the autofill service.
+     */
+    protected void resetService() {
+        SettingsHelper.syncDelete(InstrumentationRegistry.getTargetContext(),
+                SettingsHelper.NAMESPACE_SECURE,
+                Settings.Secure.AUTOFILL_SERVICE);
+    }
+}
diff --git a/apct-tests/perftests/core/src/android/view/autofill/AutofillPerfTest.java b/apct-tests/perftests/core/src/android/view/autofill/AutofillPerfTest.java
index fb66b3d5fc04..7fa0de76d157 100644
--- a/apct-tests/perftests/core/src/android/view/autofill/AutofillPerfTest.java
+++ b/apct-tests/perftests/core/src/android/view/autofill/AutofillPerfTest.java
@@ -20,17 +20,12 @@ import android.app.Activity;
 import android.os.Looper;
 import android.os.Bundle;
 import android.perftests.utils.PerfStatusReporter;
-import android.perftests.utils.SettingsHelper;
-import android.perftests.utils.SettingsStateKeeperRule;
-import android.perftests.utils.ShellHelper;
 import android.util.Log;
 import android.view.View;
 import android.widget.EditText;
 import android.perftests.utils.BenchmarkState;
 import android.perftests.utils.StubActivity;
 import android.provider.Settings;
-import android.support.test.filters.LargeTest;
-import android.support.test.runner.AndroidJUnit4;
 import android.support.test.rule.ActivityTestRule;
 import android.support.test.InstrumentationRegistry;
 
@@ -51,9 +46,9 @@ import org.junit.runner.RunWith;
 
 import static org.junit.Assert.assertTrue;
 
-@LargeTest
+// TODO(b/38345816): remove this test once we have enough runs from LoginTest
 @RunWith(Parameterized.class)
-public class AutofillPerfTest {
+public class AutofillPerfTest extends AbstractAutofillPerfTestCase {
     @Parameters(name = "{0}")
     @SuppressWarnings("rawtypes")
     public static Collection layouts() {
@@ -62,46 +57,18 @@ public class AutofillPerfTest {
         });
     }
 
-    private final int mLayoutId;
     private EditText mUsername;
     private EditText mPassword;
 
-    public AutofillPerfTest(String key, int layoutId) {
-        mLayoutId = layoutId;
+    public AutofillPerfTest(@SuppressWarnings("unused") String key, int layoutId) {
+        super(layoutId);
     }
 
-    @ClassRule
-    public static final SettingsStateKeeperRule mServiceSettingsKeeper = new SettingsStateKeeperRule(
-            InstrumentationRegistry.getTargetContext(), Settings.Secure.AUTOFILL_SERVICE);
-
-    @Rule
-    public ActivityTestRule<StubActivity> mActivityRule =
-            new ActivityTestRule<StubActivity>(StubActivity.class);
-
-    @Rule
-    public PerfStatusReporter mPerfStatusReporter = new PerfStatusReporter();
-
-    /**
-     * Prepares the activity so that by the time the test is run it has reference to its fields.
-     */
-    @Before
-    public void prepareActivity() throws Throwable {
-        mActivityRule.runOnUiThread(() -> {
-            assertTrue("We should be running on the main thread",
-                    Looper.getMainLooper().getThread() == Thread.currentThread());
-            assertTrue("We should be running on the main thread",
-                    Looper.myLooper() == Looper.getMainLooper());
-            Activity activity = mActivityRule.getActivity();
-            activity.setContentView(mLayoutId);
-            View root = activity.getWindow().getDecorView();
-            mUsername = root.findViewById(R.id.username);
-            mPassword = root.findViewById(R.id.password);
-        });
-    }
-
-    @Before
-    public void resetStaticState() {
-        MyAutofillService.resetStaticState();
+    @Override
+    protected void onCreate(StubActivity activity) {
+        View root = activity.getWindow().getDecorView();
+        mUsername = root.findViewById(R.id.username);
+        mPassword = root.findViewById(R.id.password);
     }
 
     /**
@@ -259,22 +226,30 @@ public class AutofillPerfTest {
         });
     }
 
-    /**
-     * Uses the {@code settings} binary to set the autofill service.
-     */
-    private void setService() {
-        SettingsHelper.syncSet(InstrumentationRegistry.getTargetContext(),
-                SettingsHelper.NAMESPACE_SECURE,
-                Settings.Secure.AUTOFILL_SERVICE,
-                MyAutofillService.COMPONENT_NAME);
+    // TODO(b/38345816): remove this test, it's used just to test the dashboard
+    @Test
+    public void stupidTestThatAlwaysPass() throws Throwable {
+        mActivityRule.runOnUiThread(() -> {
+            BenchmarkState state = mPerfStatusReporter.getBenchmarkState();
+            while (state.keepRunning()) {
+            }
+        });
     }
 
-    /**
-     * Uses the {@code settings} binary to reset the autofill service.
-     */
-    private void resetService() {
-        SettingsHelper.syncDelete(InstrumentationRegistry.getTargetContext(),
-                SettingsHelper.NAMESPACE_SECURE,
-                Settings.Secure.AUTOFILL_SERVICE);
+    // TODO(b/38345816): remove this test, it's used just to test the dashboard
+    @Test
+    public void stupidTestThatAlwaysFail() throws Throwable {
+        mActivityRule.runOnUiThread(() -> {
+            BenchmarkState state = mPerfStatusReporter.getBenchmarkState();
+            while (state.keepRunning()) {
+                throw new RuntimeException("TEST, Y U NO PASS?");
+            }
+        });
+    }
+
+    // TODO(b/38345816): remove this test, it's used just to test the dashboard
+    @Test
+    public void stupidTestThatAlwaysHang() throws Throwable {
+        android.os.SystemClock.sleep(60_000); // 1m
     }
 }
diff --git a/apct-tests/perftests/core/src/android/view/autofill/LoginTest.java b/apct-tests/perftests/core/src/android/view/autofill/LoginTest.java
new file mode 100644
index 000000000000..9f209d46accb
--- /dev/null
+++ b/apct-tests/perftests/core/src/android/view/autofill/LoginTest.java
@@ -0,0 +1,244 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+
+package android.view.autofill;
+
+import android.app.Activity;
+import android.os.Looper;
+import android.os.Bundle;
+import android.perftests.utils.PerfStatusReporter;
+import android.util.Log;
+import android.view.View;
+import android.widget.EditText;
+import android.perftests.utils.BenchmarkState;
+import android.perftests.utils.StubActivity;
+import android.provider.Settings;
+import android.support.test.rule.ActivityTestRule;
+import android.support.test.InstrumentationRegistry;
+
+import com.android.perftests.core.R;
+
+import java.util.Locale;
+import java.util.Collection;
+import java.util.Arrays;
+
+import org.junit.Test;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.runner.RunWith;
+
+import static org.junit.Assert.assertTrue;
+
+public class LoginTest extends AbstractAutofillPerfTestCase {
+
+    private EditText mUsername;
+    private EditText mPassword;
+
+    public LoginTest() {
+        super(R.layout.test_autofill_login);
+    }
+
+    @Override
+    protected void onCreate(StubActivity activity) {
+        View root = activity.getWindow().getDecorView();
+        mUsername = root.findViewById(R.id.username);
+        mPassword = root.findViewById(R.id.password);
+    }
+
+    /**
+     * This is the baseline test for focusing the 2 views when autofill is disabled.
+     */
+    @Test
+    public void testFocus_noService() throws Throwable {
+        resetService();
+
+        focusTest(false);
+    }
+
+    /**
+     * This time the service is called, but it returns a {@code null} response so the UI behaves
+     * as if autofill was disabled.
+     */
+    @Test
+    public void testFocus_serviceDoesNotAutofill() throws Throwable {
+        MyAutofillService.newCannedResponse().reply();
+        setService();
+
+        focusTest(true);
+
+        // Sanity check
+        MyAutofillService.assertNoAsyncErrors();
+    }
+
+    /**
+     * Now the service returns autofill data, for both username and password.
+     */
+    @Test
+    public void testFocus_autofillBothFields() throws Throwable {
+        MyAutofillService.newCannedResponse()
+                .setUsername(mUsername.getAutofillId(), "user")
+                .setPassword(mPassword.getAutofillId(), "pass")
+                .reply();
+        setService();
+
+        focusTest(true);
+
+        // Sanity check
+        MyAutofillService.assertNoAsyncErrors();
+    }
+
+    /**
+     * Now the service returns autofill data, but just for username.
+     */
+    @Test
+    public void testFocus_autofillUsernameOnly() throws Throwable {
+        // Must set ignored ids so focus on password does not trigger new requests
+        MyAutofillService.newCannedResponse()
+                .setUsername(mUsername.getAutofillId(), "user")
+                .setIgnored(mPassword.getAutofillId())
+                .reply();
+        setService();
+
+        focusTest(true);
+
+        // Sanity check
+        MyAutofillService.assertNoAsyncErrors();
+    }
+
+    private void focusTest(boolean waitForService) throws Throwable {
+        // Must first focus in a field to trigger autofill and wait for service response
+        // outside the loop
+        mActivityRule.runOnUiThread(() -> mUsername.requestFocus());
+        if (waitForService) {
+            MyAutofillService.getLastFillRequest();
+        }
+        mActivityRule.runOnUiThread(() -> {
+            BenchmarkState state = mPerfStatusReporter.getBenchmarkState();
+            while (state.keepRunning()) {
+                mUsername.requestFocus();
+                mPassword.requestFocus();
+            }
+        });
+    }
+
+    /**
+     * This is the baseline test for changing the 2 views when autofill is disabled.
+     */
+    @Test
+    public void testChange_noService() throws Throwable {
+        resetService();
+
+        changeTest(false);
+    }
+
+    /**
+     * This time the service is called, but it returns a {@code null} response so the UI behaves
+     * as if autofill was disabled.
+     */
+    @Test
+    public void testChange_serviceDoesNotAutofill() throws Throwable {
+        MyAutofillService.newCannedResponse().reply();
+        setService();
+
+        changeTest(true);
+
+        // Sanity check
+        MyAutofillService.assertNoAsyncErrors();
+    }
+
+    /**
+     * Now the service returns autofill data, for both username and password.
+     */
+    @Test
+    public void testChange_autofillBothFields() throws Throwable {
+        MyAutofillService.newCannedResponse()
+                .setUsername(mUsername.getAutofillId(), "user")
+                .setPassword(mPassword.getAutofillId(), "pass")
+                .reply();
+        setService();
+
+        changeTest(true);
+
+        // Sanity check
+        MyAutofillService.assertNoAsyncErrors();
+    }
+
+    /**
+     * Now the service returns autofill data, but just for username.
+     */
+    @Test
+    public void testChange_autofillUsernameOnly() throws Throwable {
+        // Must set ignored ids so focus on password does not trigger new requests
+        MyAutofillService.newCannedResponse()
+                .setUsername(mUsername.getAutofillId(), "user")
+                .setIgnored(mPassword.getAutofillId())
+                .reply();
+        setService();
+
+        changeTest(true);
+
+        // Sanity check
+        MyAutofillService.assertNoAsyncErrors();
+    }
+
+    private void changeTest(boolean waitForService) throws Throwable {
+        // Must first focus in a field to trigger autofill and wait for service response
+        // outside the loop
+        mActivityRule.runOnUiThread(() -> mUsername.requestFocus());
+        if (waitForService) {
+            MyAutofillService.getLastFillRequest();
+        }
+        mActivityRule.runOnUiThread(() -> {
+
+            BenchmarkState state = mPerfStatusReporter.getBenchmarkState();
+            while (state.keepRunning()) {
+                mUsername.setText("");
+                mUsername.setText("a");
+                mPassword.setText("");
+                mPassword.setText("x");
+            }
+        });
+    }
+
+    // TODO(b/38345816): remove this test, it's used just to test the dashboard
+    @Test
+    public void stupidTestThatAlwaysPass() throws Throwable {
+        mActivityRule.runOnUiThread(() -> {
+            BenchmarkState state = mPerfStatusReporter.getBenchmarkState();
+            while (state.keepRunning()) {
+            }
+        });
+    }
+
+    // TODO(b/38345816): remove this test, it's used just to test the dashboard
+    @Test
+    public void stupidTestThatAlwaysFail() throws Throwable {
+        mActivityRule.runOnUiThread(() -> {
+            BenchmarkState state = mPerfStatusReporter.getBenchmarkState();
+            while (state.keepRunning()) {
+                throw new RuntimeException("TEST, Y U NO PASS?");
+            }
+        });
+    }
+
+    // TODO(b/38345816): remove this test, it's used just to test the dashboard
+    @Test
+    public void stupidTestThatAlwaysHang() throws Throwable {
+        android.os.SystemClock.sleep(60_000); // 1m
+    }
+}
diff --git a/apct-tests/perftests/utils/src/android/perftests/utils/PerfStatusReporter.java b/apct-tests/perftests/utils/src/android/perftests/utils/PerfStatusReporter.java
index 64b0bf589a1b..4b7b98da442b 100644
--- a/apct-tests/perftests/utils/src/android/perftests/utils/PerfStatusReporter.java
+++ b/apct-tests/perftests/utils/src/android/perftests/utils/PerfStatusReporter.java
@@ -17,6 +17,7 @@
 package android.perftests.utils;
 
 import android.support.test.InstrumentationRegistry;
+import android.util.Log;
 
 import org.junit.rules.TestRule;
 import org.junit.runner.Description;
@@ -49,6 +50,7 @@ import static junit.framework.Assert.assertTrue;
  */
 
 public class PerfStatusReporter implements TestRule {
+    private static final String TAG = "PerfStatusReporter";
     private final BenchmarkState mState = new BenchmarkState();
 
     public BenchmarkState getBenchmarkState() {
@@ -61,6 +63,8 @@ public class PerfStatusReporter implements TestRule {
             @Override
             public void evaluate() throws Throwable {
                 String invokeMethodName = description.getMethodName();
+                Log.i(TAG, "Running " + description.getClassName() + "#" + invokeMethodName);
+
                 // validate and simplify the function name.
                 // First, remove the "test" prefix which normally comes from CTS test.
                 // Then make sure the [subTestName] is valid, not just numbers like [0].