diff options
5 files changed, 299 insertions, 41 deletions
diff --git a/apct-tests/perftests/multiuser/Android.mk b/apct-tests/perftests/multiuser/Android.mk index f67004358ee6..e3f7775383bd 100644 --- a/apct-tests/perftests/multiuser/Android.mk +++ b/apct-tests/perftests/multiuser/Android.mk @@ -20,8 +20,7 @@ LOCAL_MODULE_TAGS := tests LOCAL_SRC_FILES := $(call all-java-files-under, src) LOCAL_STATIC_JAVA_LIBRARIES := \ - android-support-test \ - apct-perftests-utils + android-support-test LOCAL_PACKAGE_NAME := MultiUserPerfTests diff --git a/apct-tests/perftests/multiuser/src/android/multiuser/BenchmarkResults.java b/apct-tests/perftests/multiuser/src/android/multiuser/BenchmarkResults.java new file mode 100644 index 000000000000..0d764ce29c74 --- /dev/null +++ b/apct-tests/perftests/multiuser/src/android/multiuser/BenchmarkResults.java @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.multiuser; + +import android.os.Bundle; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.concurrent.TimeUnit; + +public class BenchmarkResults { + private final ArrayList<Long> mResults = new ArrayList<>(); + + public void addDuration(long duration) { + mResults.add(TimeUnit.NANOSECONDS.toMillis(duration)); + } + + public Bundle getStats() { + final Bundle stats = new Bundle(); + stats.putDouble("Mean (ms)", mean()); + stats.putDouble("Median (ms)", median()); + stats.putDouble("Sigma (ms)", standardDeviation()); + return stats; + } + + public ArrayList<Long> getAllDurations() { + return mResults; + } + + private double mean() { + final int size = mResults.size(); + long sum = 0; + for (int i = 0; i < size; ++i) { + sum += mResults.get(i); + } + return (double) sum / size; + } + + private double median() { + final int size = mResults.size(); + if (size == 0) { + return 0f; + } + Collections.sort(mResults); + final int idx = size / 2; + return size % 2 == 0 + ? (double) (mResults.get(idx) + mResults.get(idx - 1)) / 2 + : mResults.get(idx); + } + + private double standardDeviation() { + final int size = mResults.size(); + if (size == 0) { + return 0f; + } + final double mean = mean(); + double sd = 0; + for (int i = 0; i < size; ++i) { + double diff = mResults.get(i) - mean; + sd += diff * diff; + } + return Math.sqrt(sd / size); + } +} diff --git a/apct-tests/perftests/multiuser/src/android/multiuser/BenchmarkResultsReporter.java b/apct-tests/perftests/multiuser/src/android/multiuser/BenchmarkResultsReporter.java new file mode 100644 index 000000000000..7472865e9a5a --- /dev/null +++ b/apct-tests/perftests/multiuser/src/android/multiuser/BenchmarkResultsReporter.java @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.multiuser; + +import android.app.Activity; +import android.app.Instrumentation; +import android.os.Bundle; +import android.support.test.InstrumentationRegistry; +import android.util.Log; + +import org.junit.rules.TestRule; +import org.junit.runner.Description; +import org.junit.runners.model.Statement; + +import java.util.ArrayList; + +public class BenchmarkResultsReporter implements TestRule { + private final BenchmarkRunner mRunner; + + public BenchmarkResultsReporter(BenchmarkRunner benchmarkRunner) { + mRunner = benchmarkRunner; + } + + @Override + public Statement apply(final Statement base, final Description description) { + return new Statement() { + @Override + public void evaluate() throws Throwable { + base.evaluate(); + final Bundle stats = mRunner.getStats(); + final String summary = getSummaryString(description.getMethodName(), stats); + logSummary(description.getTestClass().getSimpleName(), summary, mRunner.getAllDurations()); + stats.putString(Instrumentation.REPORT_KEY_STREAMRESULT, summary); + InstrumentationRegistry.getInstrumentation().sendStatus( + Activity.RESULT_OK, stats); + } + }; + } + + private void logSummary(String tag, String summary, ArrayList<Long> durations) { + final StringBuilder sb = new StringBuilder(summary); + final int size = durations.size(); + for (int i = 0; i < size; ++i) { + sb.append("\n").append(i).append("->").append(durations.get(i)); + } + Log.d(tag, sb.toString()); + } + + private String getSummaryString(String testName, Bundle stats) { + final StringBuilder sb = new StringBuilder(); + sb.append("\n\n").append(getKey(testName)); + for (String key : stats.keySet()) { + sb.append("\n").append(key).append(": ").append(stats.get(key)); + } + return sb.toString(); + } + + private String getKey(String testName) { + return testName.replaceAll("Perf$", ""); + } +} diff --git a/apct-tests/perftests/multiuser/src/android/multiuser/BenchmarkRunner.java b/apct-tests/perftests/multiuser/src/android/multiuser/BenchmarkRunner.java new file mode 100644 index 000000000000..ccadc9a8f6a9 --- /dev/null +++ b/apct-tests/perftests/multiuser/src/android/multiuser/BenchmarkRunner.java @@ -0,0 +1,103 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.multiuser; + +import android.os.Bundle; +import android.os.SystemClock; + +import java.util.ArrayList; + +// Based on //platform/frameworks/base/apct-tests/perftests/utils/BenchmarkState.java +public class BenchmarkRunner { + + private static long COOL_OFF_PERIOD_MS = 2000; + + private static final int NUM_ITERATIONS = 4; + + private static final int NOT_STARTED = 0; // The benchmark has not started yet. + private static final int RUNNING = 1; // The benchmark is running. + private static final int PAUSED = 2; // The benchmark is paused + private static final int FINISHED = 3; // The benchmark has stopped. + + private final BenchmarkResults mResults = new BenchmarkResults(); + private int mState = NOT_STARTED; // Current benchmark state. + private int mIteration; + + public long mStartTimeNs; + public long mPausedDurationNs; + public long mPausedTimeNs; + + public boolean keepRunning() { + switch (mState) { + case NOT_STARTED: + mState = RUNNING; + prepareForNextRun(); + return true; + case RUNNING: + mIteration++; + return startNextTestRun(); + case PAUSED: + throw new IllegalStateException("Benchmarking is in paused state"); + case FINISHED: + throw new IllegalStateException("Benchmarking is finished"); + default: + throw new IllegalStateException("BenchmarkRunner is in unknown state"); + } + } + + private boolean startNextTestRun() { + mResults.addDuration(System.nanoTime() - mStartTimeNs - mPausedDurationNs); + if (mIteration == NUM_ITERATIONS) { + mState = FINISHED; + return false; + } else { + prepareForNextRun(); + return true; + } + } + + private void prepareForNextRun() { + // TODO: Once http://b/63115387 is fixed, look into using "am wait-for-broadcast-idle" + // command instead of waiting for a fixed amount of time. + SystemClock.sleep(COOL_OFF_PERIOD_MS); + mStartTimeNs = System.nanoTime(); + mPausedDurationNs = 0; + } + + public void pauseTiming() { + if (mState != RUNNING) { + throw new IllegalStateException("Unable to pause the runner: not running currently"); + } + mPausedTimeNs = System.nanoTime(); + mState = PAUSED; + } + + public void resumeTiming() { + if (mState != PAUSED) { + throw new IllegalStateException("Unable to resume the runner: already running"); + } + mPausedDurationNs += System.nanoTime() - mPausedTimeNs; + mState = RUNNING; + } + + public Bundle getStats() { + return mResults.getStats(); + } + + public ArrayList<Long> getAllDurations() { + return mResults.getAllDurations(); + } +}
\ No newline at end of file diff --git a/apct-tests/perftests/multiuser/src/android/multiuser/UserLifecycleTest.java b/apct-tests/perftests/multiuser/src/android/multiuser/UserLifecycleTest.java index e89157b52108..f114ef47007b 100644 --- a/apct-tests/perftests/multiuser/src/android/multiuser/UserLifecycleTest.java +++ b/apct-tests/perftests/multiuser/src/android/multiuser/UserLifecycleTest.java @@ -27,8 +27,6 @@ import android.content.pm.UserInfo; import android.os.RemoteException; import android.os.UserHandle; import android.os.UserManager; -import android.perftests.utils.BenchmarkState; -import android.perftests.utils.PerfStatusReporter; import android.support.test.InstrumentationRegistry; import android.support.test.filters.LargeTest; import android.support.test.runner.AndroidJUnit4; @@ -47,26 +45,34 @@ import java.util.concurrent.TimeUnit; * Perf tests for user life cycle events. * * Running the tests: + * * make MultiUserPerfTests && * adb install -r \ * ${ANDROID_PRODUCT_OUT}/data/app/MultiUserPerfTests/MultiUserPerfTests.apk && * adb shell am instrument -e class android.multiuser.UserLifecycleTest \ * -w com.android.perftests.multiuser/android.support.test.runner.AndroidJUnitRunner + * + * or + * + * bit MultiUserPerfTests:android.multiuser.UserLifecycleTest + * + * Note: If you use bit for running the tests, benchmark results won't be printed on the host side. + * But in either case, results can be checked on the device side 'adb logcat -s UserLifecycleTest' */ @LargeTest @RunWith(AndroidJUnit4.class) public class UserLifecycleTest { - private final int TIMEOUT_IN_SECOND = 10; + private final int TIMEOUT_IN_SECOND = 30; private final int CHECK_USER_REMOVED_INTERVAL_MS = 200; private UserManager mUm; private ActivityManager mAm; private IActivityManager mIam; - private BenchmarkState mState; private ArrayList<Integer> mUsersToRemove; + private final BenchmarkRunner mRunner = new BenchmarkRunner(); @Rule - public PerfStatusReporter mPerfStatusReporter = new PerfStatusReporter(); + public BenchmarkResultsReporter mReporter = new BenchmarkResultsReporter(mRunner); @Before public void setUp() { @@ -74,7 +80,6 @@ public class UserLifecycleTest { mUm = UserManager.get(context); mAm = context.getSystemService(ActivityManager.class); mIam = ActivityManager.getService(); - mState = mPerfStatusReporter.getBenchmarkState(); mUsersToRemove = new ArrayList<>(); } @@ -91,7 +96,7 @@ public class UserLifecycleTest { @Test public void createAndStartUserPerf() throws Exception { - while (mState.keepRunning()) { + while (mRunner.keepRunning()) { final UserInfo userInfo = mUm.createUser("TestUser", 0); final CountDownLatch latch = new CountDownLatch(1); @@ -99,91 +104,91 @@ public class UserLifecycleTest { mIam.startUserInBackground(userInfo.id); latch.await(TIMEOUT_IN_SECOND, TimeUnit.SECONDS); - mState.pauseTiming(); + mRunner.pauseTiming(); removeUser(userInfo.id); - mState.resumeTiming(); + mRunner.resumeTiming(); } } @Test public void switchUserPerf() throws Exception { - while (mState.keepRunning()) { - mState.pauseTiming(); + while (mRunner.keepRunning()) { + mRunner.pauseTiming(); final int startUser = mAm.getCurrentUser(); final UserInfo userInfo = mUm.createUser("TestUser", 0); - mState.resumeTiming(); + mRunner.resumeTiming(); switchUser(userInfo.id); - mState.pauseTiming(); + mRunner.pauseTiming(); switchUser(startUser); removeUser(userInfo.id); - mState.resumeTiming(); + mRunner.resumeTiming(); } } @Test public void stopUserPerf() throws Exception { - while (mState.keepRunning()) { - mState.pauseTiming(); + while (mRunner.keepRunning()) { + mRunner.pauseTiming(); final UserInfo userInfo = mUm.createUser("TestUser", 0); final CountDownLatch latch = new CountDownLatch(1); registerBroadcastReceiver(Intent.ACTION_USER_STARTED, latch, userInfo.id); mIam.startUserInBackground(userInfo.id); latch.await(TIMEOUT_IN_SECOND, TimeUnit.SECONDS); - mState.resumeTiming(); + mRunner.resumeTiming(); stopUser(userInfo.id, false); - mState.pauseTiming(); + mRunner.pauseTiming(); removeUser(userInfo.id); - mState.resumeTiming(); + mRunner.resumeTiming(); } } @Test public void lockedBootCompletedPerf() throws Exception { - while (mState.keepRunning()) { - mState.pauseTiming(); + while (mRunner.keepRunning()) { + mRunner.pauseTiming(); final int startUser = mAm.getCurrentUser(); final UserInfo userInfo = mUm.createUser("TestUser", 0); final CountDownLatch latch = new CountDownLatch(1); registerUserSwitchObserver(null, latch, userInfo.id); - mState.resumeTiming(); + mRunner.resumeTiming(); mAm.switchUser(userInfo.id); latch.await(TIMEOUT_IN_SECOND, TimeUnit.SECONDS); - mState.pauseTiming(); + mRunner.pauseTiming(); switchUser(startUser); removeUser(userInfo.id); - mState.resumeTiming(); + mRunner.resumeTiming(); } } @Test public void managedProfileUnlockPerf() throws Exception { - while (mState.keepRunning()) { - mState.pauseTiming(); + while (mRunner.keepRunning()) { + mRunner.pauseTiming(); final UserInfo userInfo = mUm.createProfileForUser("TestUser", UserInfo.FLAG_MANAGED_PROFILE, mAm.getCurrentUser()); final CountDownLatch latch = new CountDownLatch(1); registerBroadcastReceiver(Intent.ACTION_USER_UNLOCKED, latch, userInfo.id); - mState.resumeTiming(); + mRunner.resumeTiming(); mIam.startUserInBackground(userInfo.id); latch.await(TIMEOUT_IN_SECOND, TimeUnit.SECONDS); - mState.pauseTiming(); + mRunner.pauseTiming(); removeUser(userInfo.id); - mState.resumeTiming(); + mRunner.resumeTiming(); } } @Test public void ephemeralUserStoppedPerf() throws Exception { - while (mState.keepRunning()) { - mState.pauseTiming(); + while (mRunner.keepRunning()) { + mRunner.pauseTiming(); final int startUser = mAm.getCurrentUser(); final UserInfo userInfo = mUm.createUser("TestUser", UserInfo.FLAG_EPHEMERAL | UserInfo.FLAG_DEMO); @@ -200,35 +205,35 @@ public class UserLifecycleTest { }, new IntentFilter(Intent.ACTION_USER_STOPPED)); final CountDownLatch switchLatch = new CountDownLatch(1); registerUserSwitchObserver(switchLatch, null, startUser); - mState.resumeTiming(); + mRunner.resumeTiming(); mAm.switchUser(startUser); latch.await(TIMEOUT_IN_SECOND, TimeUnit.SECONDS); - mState.pauseTiming(); + mRunner.pauseTiming(); switchLatch.await(TIMEOUT_IN_SECOND, TimeUnit.SECONDS); removeUser(userInfo.id); - mState.resumeTiming(); + mRunner.resumeTiming(); } } @Test public void managedProfileStoppedPerf() throws Exception { - while (mState.keepRunning()) { - mState.pauseTiming(); + while (mRunner.keepRunning()) { + mRunner.pauseTiming(); final UserInfo userInfo = mUm.createProfileForUser("TestUser", UserInfo.FLAG_MANAGED_PROFILE, mAm.getCurrentUser()); final CountDownLatch latch = new CountDownLatch(1); registerBroadcastReceiver(Intent.ACTION_USER_UNLOCKED, latch, userInfo.id); mIam.startUserInBackground(userInfo.id); latch.await(TIMEOUT_IN_SECOND, TimeUnit.SECONDS); - mState.resumeTiming(); + mRunner.resumeTiming(); stopUser(userInfo.id, true); - mState.pauseTiming(); + mRunner.pauseTiming(); removeUser(userInfo.id); - mState.resumeTiming(); + mRunner.resumeTiming(); } } |