Initial setup of a SystemMemoryTest.

Set up a memory test to experiment with actionable memory metrics and
detect regressions in system server memory use.

The initial CUJ is to launch an instrumentation that does nothing for
a few seconds.

The initial metric is to read showmap for system_server and report
VSS, RSS, and PSS.

The CUJ and metrics will be made more interesting once the basic
infrastructure for continuously running the test is set up.

Bug: 111830582
Test: tradefed.sh run commandAndExit template/local_min --template:map test=system-memory-test

Change-Id: I8793adb66de4adab254173585e2c8afc754d4a3a

10 files changed, 463 insertions, 0 deletions

diff --git a/tests/SystemMemoryTest/Android.mk b/tests/SystemMemoryTest/Android.mk
new file mode 100644
index 000000000000..09a1618c6ad1
--- /dev/null
+++ b/tests/SystemMemoryTest/Android.mk
@@ -0,0 +1,17 @@
+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+LOCAL_PATH:= $(call my-dir)
+
+include $(call all-subdir-makefiles)
diff --git a/tests/SystemMemoryTest/README.txt b/tests/SystemMemoryTest/README.txt
new file mode 100644
index 000000000000..de5042cf3732
--- /dev/null
+++ b/tests/SystemMemoryTest/README.txt
@@ -0,0 +1,21 @@
+This directory contains a test for system server memory use.
+
+Directory structure
+===================
+device
+  - those parts of the test that run on device.
+
+host
+  - those parts of the test that run on host.
+
+Running the test
+================
+
+You can manually run the test as follows:
+
+  make tradefed-all system-memory-test SystemMemoryTestDevice
+  tradefed.sh run commandAndExit template/local_min --template:map test=system-memory-test
+
+This installs and runs the test on device. You can see the metrics in the
+tradefed output.
+
diff --git a/tests/SystemMemoryTest/device/Android.mk b/tests/SystemMemoryTest/device/Android.mk
new file mode 100644
index 000000000000..75408df4b295
--- /dev/null
+++ b/tests/SystemMemoryTest/device/Android.mk
@@ -0,0 +1,24 @@
+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+LOCAL_PATH := $(call my-dir)
+include $(CLEAR_VARS)
+
+LOCAL_MODULE_TAGS := optional
+LOCAL_MODULE_PATH := $(TARGET_OUT_DATA_APPS)
+LOCAL_PACKAGE_NAME := SystemMemoryTestDevice
+LOCAL_SDK_VERSION := current
+LOCAL_SRC_FILES := $(call all-java-files-under, src)
+LOCAL_COMPATIBILITY_SUITE := general-tests
+include $(BUILD_PACKAGE)
diff --git a/tests/SystemMemoryTest/device/AndroidManifest.xml b/tests/SystemMemoryTest/device/AndroidManifest.xml
new file mode 100644
index 000000000000..e5e7844f9660
--- /dev/null
+++ b/tests/SystemMemoryTest/device/AndroidManifest.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright (C) 2018 The Android Open Source Project
+
+     Licensed under the Apache License, Version 2.0 (the "License");
+     you may not use this file except in compliance with the License.
+     You may obtain a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+     Unless required by applicable law or agreed to in writing, software
+     distributed under the License is distributed on an "AS IS" BASIS,
+     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+     See the License for the specific language governing permissions and
+     limitations under the License.
+-->
+
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    package="com.android.tests.sysmem.device">
+
+    <uses-sdk android:minSdkVersion="19" />
+
+    <instrumentation
+            android:name="com.android.tests.sysmem.device.Cujs"
+            android:targetPackage="com.android.tests.sysmem.device" />
+
+    <application
+            android:allowBackup="false"
+            android:label="System Memory Test">
+    </application>
+</manifest>
diff --git a/tests/SystemMemoryTest/device/src/com/android/tests/sysmem/device/Cujs.java b/tests/SystemMemoryTest/device/src/com/android/tests/sysmem/device/Cujs.java
new file mode 100644
index 000000000000..6c0c5931ed8c
--- /dev/null
+++ b/tests/SystemMemoryTest/device/src/com/android/tests/sysmem/device/Cujs.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.tests.sysmem.device;
+
+import android.app.Activity;
+import android.app.Instrumentation;
+import android.os.Bundle;
+import android.util.Log;
+
+/**
+ * Critical user journeys used to exercise the system for test, driven from
+ * the device.
+ */
+public class Cujs extends Instrumentation {
+
+    private static final String TAG = "SystemMemoryTest";
+
+    @Override
+    public void onCreate(Bundle arguments) {
+        start();
+    }
+
+    @Override
+    public void onStart() {
+        // TODO: Exercise the system in more interesting ways.
+        // Mostly what matters is that whatever we do here is sustainable: it
+        // can be repeated indefinitely on the system without causing
+        // problems.  For example, launching activities is fine. Installing
+        // applications is only fine if we also uninstall them as part of the
+        // test.
+        try {
+            Log.i(TAG, "Sleeping for 10 seconds...");
+            Thread.sleep(10 * 1000);
+        } catch (InterruptedException ignored) {
+        }
+
+        finish(Activity.RESULT_OK, null);
+    }
+}
diff --git a/tests/SystemMemoryTest/host/Android.mk b/tests/SystemMemoryTest/host/Android.mk
new file mode 100644
index 000000000000..a516e38adfec
--- /dev/null
+++ b/tests/SystemMemoryTest/host/Android.mk
@@ -0,0 +1,23 @@
+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+LOCAL_PATH := $(call my-dir)
+
+include $(CLEAR_VARS)
+LOCAL_SRC_FILES := $(call all-java-files-under, src)
+LOCAL_MODULE := system-memory-test
+LOCAL_MODULE_TAGS := optional
+LOCAL_JAVA_LIBRARIES := tradefed
+LOCAL_COMPATIBILITY_SUITE := general-tests
+include $(BUILD_HOST_JAVA_LIBRARY)
diff --git a/tests/SystemMemoryTest/host/AndroidTest.xml b/tests/SystemMemoryTest/host/AndroidTest.xml
new file mode 100644
index 000000000000..6d2c95f3094a
--- /dev/null
+++ b/tests/SystemMemoryTest/host/AndroidTest.xml
@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright (C) 2018 The Android Open Source Project
+
+     Licensed under the Apache License, Version 2.0 (the "License");
+     you may not use this file except in compliance with the License.
+     You may obtain a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+     Unless required by applicable law or agreed to in writing, software
+     distributed under the License is distributed on an "AS IS" BASIS,
+     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+     See the License for the specific language governing permissions and
+     limitations under the License.
+-->
+<configuration description="Runs the system memory tests">
+    <option name="test-suite-tag" value="system-memory-test" />
+    <target_preparer class="com.android.tradefed.targetprep.suite.SuiteApkInstaller">
+        <option name="cleanup-apks" value="true" />
+        <option name="test-file-name" value="SystemMemoryTestDevice.apk" />
+    </target_preparer>
+    <test class="com.android.tradefed.testtype.HostTest" >
+        <option name="class" value="com.android.tests.sysmem.host.MemoryTest" />
+    </test>
+</configuration>
diff --git a/tests/SystemMemoryTest/host/src/com/android/tests/sysmem/host/Cujs.java b/tests/SystemMemoryTest/host/src/com/android/tests/sysmem/host/Cujs.java
new file mode 100644
index 000000000000..579d9723977f
--- /dev/null
+++ b/tests/SystemMemoryTest/host/src/com/android/tests/sysmem/host/Cujs.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+
+package com.android.tests.sysmem.host;
+
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.device.ITestDevice;
+
+/**
+ * Critical user journeys with which to exercise the system, driven from the
+ * host.
+ */
+public class Cujs {
+    private ITestDevice device;
+
+    public Cujs(ITestDevice device) {
+        this.device = device;
+    }
+
+    /**
+     * Runs the critical user journeys.
+     */
+    public void run() throws DeviceNotAvailableException {
+        // Invoke the Device Cujs instrumentation to run the cujs.
+        // TODO: Consider exercising the system in other interesting ways as
+        // well.
+        String command = "am instrument -w com.android.tests.sysmem.device/.Cujs";
+        device.executeShellCommand(command);
+    }
+}
diff --git a/tests/SystemMemoryTest/host/src/com/android/tests/sysmem/host/MemoryTest.java b/tests/SystemMemoryTest/host/src/com/android/tests/sysmem/host/MemoryTest.java
new file mode 100644
index 000000000000..bbec0654b7ea
--- /dev/null
+++ b/tests/SystemMemoryTest/host/src/com/android/tests/sysmem/host/MemoryTest.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+
+package com.android.tests.sysmem.host;
+
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.device.ITestDevice;
+import com.android.tradefed.testtype.DeviceJUnit4ClassRunner;
+import com.android.tradefed.testtype.DeviceJUnit4ClassRunner.TestLogData;
+import com.android.tradefed.testtype.DeviceJUnit4ClassRunner.TestMetrics;
+import com.android.tradefed.testtype.IDeviceTest;
+import java.io.IOException;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+@RunWith(DeviceJUnit4ClassRunner.class)
+public class MemoryTest implements IDeviceTest {
+
+    @Rule public TestMetrics testMetrics = new TestMetrics();
+    @Rule public TestLogData testLogs = new TestLogData();
+
+    private ITestDevice testDevice;
+    private int iterations = 0;     // Number of times cujs have been run.
+    private Metrics metrics;
+    private Cujs cujs;
+
+    @Override
+    public void setDevice(ITestDevice device) {
+        testDevice = device;
+        metrics = new Metrics(device, testMetrics, testLogs);
+        cujs = new Cujs(device);
+    }
+
+    @Override
+    public ITestDevice getDevice() {
+        return testDevice;
+    }
+
+    // Invoke a single iteration of running the cujs.
+    private void runCujs() throws DeviceNotAvailableException {
+        cujs.run();
+        iterations++;
+    }
+
+    // Sample desired memory.
+    private void sample()
+            throws DeviceNotAvailableException, IOException, Metrics.MetricsException {
+        metrics.sample(String.format("%03d", iterations));
+    }
+
+    @Test
+    public void run() throws Exception {
+        sample();   // Sample before running cujs
+        runCujs();
+        sample();   // Sample after first iteration of cujs
+
+        // Run cujs in a loop to highlight memory leaks.
+        for (int i = 0; i < 5; ++i) {
+            for (int j = 0; j < 5; j++) {
+                runCujs();
+            }
+            sample();
+        }
+    }
+}
diff --git a/tests/SystemMemoryTest/host/src/com/android/tests/sysmem/host/Metrics.java b/tests/SystemMemoryTest/host/src/com/android/tests/sysmem/host/Metrics.java
new file mode 100644
index 000000000000..7de092a973ad
--- /dev/null
+++ b/tests/SystemMemoryTest/host/src/com/android/tests/sysmem/host/Metrics.java
@@ -0,0 +1,148 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+
+package com.android.tests.sysmem.host;
+
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.device.ITestDevice;
+import com.android.tradefed.result.FileInputStreamSource;
+import com.android.tradefed.result.LogDataType;
+import com.android.tradefed.testtype.DeviceJUnit4ClassRunner.TestLogData;
+import com.android.tradefed.testtype.DeviceJUnit4ClassRunner.TestMetrics;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.InputMismatchException;
+import java.util.Scanner;
+
+/**
+ * Utilities for sampling and reporting memory metrics.
+ */
+class Metrics {
+
+    private ITestDevice device;
+    private TestMetrics metrics;
+    private TestLogData logs;
+
+    /**
+     * Exception thrown in case of error sampling metrics.
+     */
+    public static class MetricsException extends Exception {
+        public MetricsException(String msg) {
+            super(msg);
+        }
+
+        MetricsException(String msg, Exception cause) {
+            super(msg, cause);
+        }
+    }
+
+    /**
+     * Constructs a metrics instance that will output high level metrics and
+     * more detailed breakdowns using the given <code>metrics</code> and
+     * <code>logs</code> objects.
+     *
+     * @param device the device to sample metrics from
+     * @param metrics where to log the high level metrics when taking a sample
+     * @param logs where to log detailed breakdowns when taking a sample
+     */
+    public Metrics(ITestDevice device, TestMetrics metrics, TestLogData logs) {
+        this.device = device;
+        this.metrics = metrics;
+        this.logs = logs;
+    }
+
+    /**
+     * Returns the pid for the process with the given name.
+     */
+    private int getPidForProcess(String name)
+            throws DeviceNotAvailableException, IOException, MetricsException {
+        String psout = device.executeShellCommand("ps -A -o PID,CMD");
+        Scanner sc = new Scanner(psout);
+        try {
+            // ps output is of the form:
+            //  PID CMD            
+            //    1 init
+            //    2 kthreadd
+            //    ...
+            // 9693 ps
+            sc.nextLine();
+            while (sc.hasNextLine()) {
+                int pid = sc.nextInt();
+                String cmd = sc.next();
+
+                if (name.equals(cmd)) {
+                    return pid;
+                }
+            }
+        } catch (InputMismatchException e) {
+            throw new MetricsException("unexpected ps output format: " + psout, e);
+        }
+
+        throw new MetricsException("failed to get pid for process " + name);
+    }
+
+    /**
+     * Samples the current memory use on the system. Outputs high level test
+     * metrics and detailed breakdowns to the TestMetrics and TestLogData
+     * objects provided when constructing this Metrics instance. The metrics
+     * and log names are prefixed with the given label.
+     *
+     * @param label prefix to use for metrics and logs output for this sample.
+     */
+    void sample(String label) throws DeviceNotAvailableException, IOException, MetricsException {
+        // adb root access is required to get showmap
+        device.enableAdbRoot();
+
+        int pid = getPidForProcess("system_server");
+
+        // Read showmap for system server and add it as a test log
+        String showmap = device.executeShellCommand("showmap " + pid);
+        String showmapLabel = label + ".system_server.showmap";
+        File file = File.createTempFile(showmapLabel, "txt");
+        PrintStream ps = new PrintStream(file);
+        ps.print(showmap);
+        try (FileInputStreamSource dataStream = new FileInputStreamSource(file)) {
+            logs.addTestLog(showmapLabel, LogDataType.TEXT, dataStream);
+        }
+
+        // Extract VSS, PSS and RSS from the showmap and output them as metrics.
+        // The last lines of the showmap output looks something like:
+        // virtual                     shared   shared  private  private
+        //    size      RSS      PSS    clean    dirty    clean    dirty     swap  swapPSS   # object
+        //-------- -------- -------- -------- -------- -------- -------- -------- -------- ---- ------------------------------
+        //  928480   113016    24860    87348     7916     3632    14120     1968     1968 1900 TOTAL
+        try {
+            int pos = showmap.lastIndexOf("----");
+            Scanner sc = new Scanner(showmap.substring(pos));
+            sc.next();
+            long vss = sc.nextLong();
+            long rss = sc.nextLong();
+            long pss = sc.nextLong();
+
+            metrics.addTestMetric(String.format("%s.system_server.vss", label), Long.toString(vss));
+            metrics.addTestMetric(String.format("%s.system_server.rss", label), Long.toString(rss));
+            metrics.addTestMetric(String.format("%s.system_server.pss", label), Long.toString(pss));
+        } catch (InputMismatchException e) {
+            throw new MetricsException("unexpected showmap format", e);
+        }
+
+        // TODO: Experiment with other additional metrics.
+
+        // TODO: Consider launching an instrumentation to collect metrics from
+        // within the device itself.
+    }
+}