Store mitigation call history before reboot

As part of the effort to better support failure mitigation across
reboots, track the mitigation calls across reboots by storing them
along with other salient parts of MonitoredPackage. These values
are relative to the current uptime of the system, so that they will
be accurate when the system uptime is reset at the next boot.

Also refactored code to allow for testing the reading and writing
of MonitoredPackage objects, and added tests for this.

Test: atest PackageWatchdogTest
Bug: 171951174
Change-Id: Ia96cf3892886d8d77193ffc278fa1eb584fecdd3

2 files changed, 157 insertions, 19 deletions

diff --git a/services/core/java/com/android/server/PackageWatchdog.java b/services/core/java/com/android/server/PackageWatchdog.java
index 9bf63cbbb25e..b9a717ca36a8 100644
--- a/services/core/java/com/android/server/PackageWatchdog.java
+++ b/services/core/java/com/android/server/PackageWatchdog.java
@@ -52,9 +52,7 @@ import com.android.internal.util.XmlUtils;
 
 import libcore.io.IoUtils;
 
-import org.xmlpull.v1.XmlPullParser;
 import org.xmlpull.v1.XmlPullParserException;
-import org.xmlpull.v1.XmlSerializer;
 
 import java.io.File;
 import java.io.FileNotFoundException;
@@ -149,6 +147,7 @@ public class PackageWatchdog {
     private static final String ATTR_DURATION = "duration";
     private static final String ATTR_EXPLICIT_HEALTH_CHECK_DURATION = "health-check-duration";
     private static final String ATTR_PASSED_HEALTH_CHECK = "passed-health-check";
+    private static final String ATTR_MITIGATION_CALLS = "mitigation-calls";
 
     @GuardedBy("PackageWatchdog.class")
     private static PackageWatchdog sPackageWatchdog;
@@ -1067,6 +1066,33 @@ public class PackageWatchdog {
         }
     }
 
+    /** Convert a {@code LongArrayQueue} to a String of comma-separated values. */
+    public static String longArrayQueueToString(LongArrayQueue queue) {
+        if (queue.size() > 0) {
+            StringBuilder sb = new StringBuilder();
+            sb.append(queue.get(0));
+            for (int i = 1; i < queue.size(); i++) {
+                sb.append(",");
+                sb.append(queue.get(i));
+            }
+            return sb.toString();
+        }
+        return "";
+    }
+
+    /** Parse a comma-separated String of longs into a LongArrayQueue. */
+    public static LongArrayQueue parseLongArrayQueue(String commaSeparatedValues) {
+        LongArrayQueue result = new LongArrayQueue();
+        if (!TextUtils.isEmpty(commaSeparatedValues)) {
+            String[] values = commaSeparatedValues.split(",");
+            for (String value : values) {
+                result.addLast(Long.parseLong(value));
+            }
+        }
+        return result;
+    }
+
+
     /** Dump status of every observer in mAllObservers. */
     public void dump(IndentingPrintWriter pw) {
         pw.println("Package Watchdog status");
@@ -1240,16 +1266,7 @@ public class PackageWatchdog {
                 while (XmlUtils.nextElementWithin(parser, innerDepth)) {
                     if (TAG_PACKAGE.equals(parser.getName())) {
                         try {
-                            String packageName = parser.getAttributeValue(
-                                    null, ATTR_NAME);
-                            long duration = parser.getAttributeLong(
-                                    null, ATTR_DURATION);
-                            long healthCheckDuration = parser.getAttributeLong(
-                                    null, ATTR_EXPLICIT_HEALTH_CHECK_DURATION);
-                            boolean hasPassedHealthCheck = parser.getAttributeBoolean(
-                                    null, ATTR_PASSED_HEALTH_CHECK, false);
-                            MonitoredPackage pkg = watchdog.newMonitoredPackage(packageName,
-                                    duration, healthCheckDuration, hasPassedHealthCheck);
+                            MonitoredPackage pkg = watchdog.parseMonitoredPackage(parser);
                             if (pkg != null) {
                                 packages.add(pkg);
                             }
@@ -1305,16 +1322,31 @@ public class PackageWatchdog {
 
     MonitoredPackage newMonitoredPackage(
             String name, long durationMs, boolean hasPassedHealthCheck) {
-        return newMonitoredPackage(name, durationMs, Long.MAX_VALUE, hasPassedHealthCheck);
+        return newMonitoredPackage(name, durationMs, Long.MAX_VALUE, hasPassedHealthCheck,
+                new LongArrayQueue());
     }
 
     MonitoredPackage newMonitoredPackage(String name, long durationMs, long healthCheckDurationMs,
-            boolean hasPassedHealthCheck) {
+            boolean hasPassedHealthCheck, LongArrayQueue mitigationCalls) {
         VersionedPackage pkg = getVersionedPackage(name);
         if (pkg == null) {
             return null;
         }
-        return new MonitoredPackage(pkg, durationMs, healthCheckDurationMs, hasPassedHealthCheck);
+        return new MonitoredPackage(pkg, durationMs, healthCheckDurationMs,
+                hasPassedHealthCheck, mitigationCalls);
+    }
+
+    MonitoredPackage parseMonitoredPackage(TypedXmlPullParser parser)
+            throws XmlPullParserException {
+        String packageName = parser.getAttributeValue(null, ATTR_NAME);
+        long duration = parser.getAttributeLong(null, ATTR_DURATION);
+        long healthCheckDuration = parser.getAttributeLong(null,
+                        ATTR_EXPLICIT_HEALTH_CHECK_DURATION);
+        boolean hasPassedHealthCheck = parser.getAttributeBoolean(null, ATTR_PASSED_HEALTH_CHECK);
+        LongArrayQueue mitigationCalls = parseLongArrayQueue(
+                parser.getAttributeValue(null, ATTR_MITIGATION_CALLS));
+        return newMonitoredPackage(packageName,
+                duration, healthCheckDuration, hasPassedHealthCheck, mitigationCalls);
     }
 
     /**
@@ -1332,7 +1364,7 @@ public class PackageWatchdog {
         // Times when an observer was called to mitigate this package's failure. Sorted in
         // ascending order.
         @GuardedBy("mLock")
-        private final LongArrayQueue mMitigationCalls = new LongArrayQueue();
+        private final LongArrayQueue mMitigationCalls;
         // One of STATE_[ACTIVE|INACTIVE|PASSED|FAILED]. Updated on construction and after
         // methods that could change the health check state: handleElapsedTimeLocked and
         // tryPassHealthCheckLocked
@@ -1353,12 +1385,14 @@ public class PackageWatchdog {
         @GuardedBy("mLock")
         private long mHealthCheckDurationMs = Long.MAX_VALUE;
 
-        private MonitoredPackage(VersionedPackage pkg, long durationMs,
-                long healthCheckDurationMs, boolean hasPassedHealthCheck) {
+        MonitoredPackage(VersionedPackage pkg, long durationMs,
+                long healthCheckDurationMs, boolean hasPassedHealthCheck,
+                LongArrayQueue mitigationCalls) {
             mPackage = pkg;
             mDurationMs = durationMs;
             mHealthCheckDurationMs = healthCheckDurationMs;
             mHasPassedHealthCheck = hasPassedHealthCheck;
+            mMitigationCalls = mitigationCalls;
             updateHealthCheckStateLocked();
         }
 
@@ -1370,6 +1404,8 @@ public class PackageWatchdog {
             out.attributeLong(null, ATTR_DURATION, mDurationMs);
             out.attributeLong(null, ATTR_EXPLICIT_HEALTH_CHECK_DURATION, mHealthCheckDurationMs);
             out.attributeBoolean(null, ATTR_PASSED_HEALTH_CHECK, mHasPassedHealthCheck);
+            LongArrayQueue normalizedCalls = normalizeMitigationCalls();
+            out.attribute(null, ATTR_MITIGATION_CALLS, longArrayQueueToString(normalizedCalls));
             out.endTag(null, TAG_PACKAGE);
         }
 
@@ -1423,6 +1459,23 @@ public class PackageWatchdog {
         }
 
         /**
+         * Before writing to disk, make the mitigation call timestamps relative to the current
+         * system uptime. This is because they need to be relative to the uptime which will reset
+         * at the next boot.
+         *
+         * @return a LongArrayQueue of the mitigation calls relative to the current system uptime.
+         */
+        @GuardedBy("mLock")
+        public LongArrayQueue normalizeMitigationCalls() {
+            LongArrayQueue normalized = new LongArrayQueue();
+            final long now = mSystemClock.uptimeMillis();
+            for (int i = 0; i < mMitigationCalls.size(); i++) {
+                normalized.addLast(mMitigationCalls.get(i) - now);
+            }
+            return normalized;
+        }
+
+        /**
          * Sets the initial health check duration.
          *
          * @return the new health check state
@@ -1582,6 +1635,16 @@ public class PackageWatchdog {
         private long toPositive(long value) {
             return value > 0 ? value : Long.MAX_VALUE;
         }
+
+        /** Compares the equality of this object with another {@link MonitoredPackage}. */
+        @VisibleForTesting
+        boolean isEqualTo(MonitoredPackage pkg) {
+            return (getName().equals(pkg.getName()))
+                    && mDurationMs == pkg.mDurationMs
+                    && mHasPassedHealthCheck == pkg.mHasPassedHealthCheck
+                    && mHealthCheckDurationMs == pkg.mHealthCheckDurationMs
+                    && (mMitigationCalls.toString()).equals(pkg.mMitigationCalls.toString());
+        }
     }
 
     /**
diff --git a/tests/PackageWatchdog/src/com/android/server/PackageWatchdogTest.java b/tests/PackageWatchdog/src/com/android/server/PackageWatchdogTest.java
index 9738e58543e1..104758de49f1 100644
--- a/tests/PackageWatchdog/src/com/android/server/PackageWatchdogTest.java
+++ b/tests/PackageWatchdog/src/com/android/server/PackageWatchdogTest.java
@@ -22,6 +22,7 @@ import static com.android.dx.mockito.inline.extended.ExtendedMockito.doAnswer;
 
 import static com.google.common.truth.Truth.assertThat;
 
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 import static org.mockito.ArgumentMatchers.anyInt;
 import static org.mockito.ArgumentMatchers.anyLong;
@@ -43,10 +44,15 @@ import android.os.SystemProperties;
 import android.os.test.TestLooper;
 import android.provider.DeviceConfig;
 import android.util.AtomicFile;
+import android.util.LongArrayQueue;
+import android.util.TypedXmlPullParser;
+import android.util.TypedXmlSerializer;
+import android.util.Xml;
 
 import androidx.test.InstrumentationRegistry;
 
 import com.android.dx.mockito.inline.extended.ExtendedMockito;
+import com.android.internal.util.XmlUtils;
 import com.android.server.PackageWatchdog.HealthCheckState;
 import com.android.server.PackageWatchdog.MonitoredPackage;
 import com.android.server.PackageWatchdog.PackageHealthObserver;
@@ -64,6 +70,7 @@ import org.mockito.quality.Strictness;
 import org.mockito.stubbing.Answer;
 
 import java.io.File;
+import java.io.FileOutputStream;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -739,7 +746,8 @@ public class PackageWatchdogTest {
                 false /* hasPassedHealthCheck */);
         MonitoredPackage m2 = wd.newMonitoredPackage(APP_B, LONG_DURATION, false);
         MonitoredPackage m3 = wd.newMonitoredPackage(APP_C, LONG_DURATION, false);
-        MonitoredPackage m4 = wd.newMonitoredPackage(APP_D, LONG_DURATION, SHORT_DURATION, true);
+        MonitoredPackage m4 = wd.newMonitoredPackage(APP_D, LONG_DURATION, SHORT_DURATION, true,
+                new LongArrayQueue());
 
         // Verify transition: inactive -> active -> passed
         // Verify initially inactive
@@ -1210,6 +1218,73 @@ public class PackageWatchdogTest {
         assertThat(observer.mMitigationCounts).isEqualTo(List.of(1, 2, 3, 3, 2, 3));
     }
 
+    @Test
+    public void testNormalizingMitigationCalls()  {
+        PackageWatchdog watchdog = createWatchdog();
+
+        LongArrayQueue mitigationCalls = new LongArrayQueue();
+        mitigationCalls.addLast(1000);
+        mitigationCalls.addLast(2000);
+        mitigationCalls.addLast(3000);
+
+        MonitoredPackage pkg = watchdog.newMonitoredPackage(
+                "test", 123, 456, true, mitigationCalls);
+
+        // Make current system uptime 10000ms.
+        moveTimeForwardAndDispatch(9999);
+
+        LongArrayQueue expectedCalls = pkg.normalizeMitigationCalls();
+
+        assertThat(expectedCalls.size()).isEqualTo(mitigationCalls.size());
+
+        for (int i = 0; i < mitigationCalls.size(); i++) {
+            assertThat(expectedCalls.get(i)).isEqualTo(mitigationCalls.get(i) - 10000);
+        }
+    }
+
+    /**
+     * Ensure that a {@link MonitoredPackage} may be correctly written and read in order to persist
+     * across reboots.
+     */
+    @Test
+    public void testWritingAndReadingMonitoredPackage() throws Exception {
+        PackageWatchdog watchdog = createWatchdog();
+
+        LongArrayQueue mitigationCalls = new LongArrayQueue();
+        mitigationCalls.addLast(1000);
+        mitigationCalls.addLast(2000);
+        mitigationCalls.addLast(3000);
+        MonitoredPackage writePkg = watchdog.newMonitoredPackage(
+                "test.package", 1000, 2000, true, mitigationCalls);
+
+        // Move time forward so that the current uptime is 4000ms. Therefore, the written mitigation
+        // calls will each be reduced by 4000.
+        moveTimeForwardAndDispatch(3999);
+        LongArrayQueue expectedCalls = new LongArrayQueue();
+        expectedCalls.addLast(-3000);
+        expectedCalls.addLast(-2000);
+        expectedCalls.addLast(-1000);
+        MonitoredPackage expectedPkg = watchdog.newMonitoredPackage(
+                "test.package", 1000, 2000, true, expectedCalls);
+
+        // Write the package
+        File tmpFile = File.createTempFile("package-watchdog-test", ".xml");
+        AtomicFile testFile = new AtomicFile(tmpFile);
+        FileOutputStream stream = testFile.startWrite();
+        TypedXmlSerializer outputSerializer = Xml.resolveSerializer(stream);
+        outputSerializer.startDocument(null, true);
+        writePkg.writeLocked(outputSerializer);
+        outputSerializer.endDocument();
+        testFile.finishWrite(stream);
+
+        // Read the package
+        TypedXmlPullParser parser = Xml.resolvePullParser(testFile.openRead());
+        XmlUtils.beginDocument(parser, "package");
+        MonitoredPackage readPkg = watchdog.parseMonitoredPackage(parser);
+
+        assertTrue(readPkg.isEqualTo(expectedPkg));
+    }
+
     private void adoptShellPermissions(String... permissions) {
         InstrumentationRegistry
                 .getInstrumentation()