Integrate Rescue Party with Package Watchdog

Integrate Rescue Party as an observer for Package
Watchdog, for managing package failures. Rescue Party
will be a persistent observer, meaning it may receive
failure calls for packages it has not explicitly asked
to observe.

Remove app failure calls and thresholding logic from
Rescue Party. Remove obsolete Rescue Party tests
and add persistent observer tests to
PackageWatchdogTest.

Test: atest PackageWatchdogTest
Test: atest RescuePartyTest
Test: atest StagedRollbackTest
Bug: 136135457
Change-Id: I55ec0de48acd5434255811feba758d38c9304478

6 files changed, 292 insertions, 134 deletions

diff --git a/services/core/java/com/android/server/PackageWatchdog.java b/services/core/java/com/android/server/PackageWatchdog.java
index deff440aa0a6..7b4fd37f01c9 100644
--- a/services/core/java/com/android/server/PackageWatchdog.java
+++ b/services/core/java/com/android/server/PackageWatchdog.java
@@ -199,13 +199,14 @@ public class PackageWatchdog {
         mSystemClock = clock;
         mNumberOfNativeCrashPollsRemaining = NUMBER_OF_NATIVE_CRASH_POLLS;
         loadFromFile();
+        sPackageWatchdog = this;
     }
 
     /** Creates or gets singleton instance of PackageWatchdog. */
     public static PackageWatchdog getInstance(Context context) {
         synchronized (PackageWatchdog.class) {
             if (sPackageWatchdog == null) {
-                sPackageWatchdog = new PackageWatchdog(context);
+                new PackageWatchdog(context);
             }
             return sPackageWatchdog;
         }
diff --git a/services/core/java/com/android/server/RescueParty.java b/services/core/java/com/android/server/RescueParty.java
index fb1a962c0ef3..3dafc64391fd 100644
--- a/services/core/java/com/android/server/RescueParty.java
+++ b/services/core/java/com/android/server/RescueParty.java
@@ -18,8 +18,12 @@ package com.android.server;
 
 import static com.android.server.pm.PackageManagerServiceUtils.logCriticalInfo;
 
+import android.annotation.Nullable;
 import android.content.ContentResolver;
 import android.content.Context;
+import android.content.pm.ApplicationInfo;
+import android.content.pm.PackageManager;
+import android.content.pm.VersionedPackage;
 import android.os.Build;
 import android.os.Environment;
 import android.os.FileUtils;
@@ -36,8 +40,12 @@ import android.util.Slog;
 import android.util.SparseArray;
 import android.util.StatsLog;
 
+import com.android.internal.annotations.GuardedBy;
 import com.android.internal.annotations.VisibleForTesting;
 import com.android.internal.util.ArrayUtils;
+import com.android.server.PackageWatchdog.FailureReasons;
+import com.android.server.PackageWatchdog.PackageHealthObserver;
+import com.android.server.PackageWatchdog.PackageHealthObserverImpact;
 import com.android.server.am.SettingsToPropertiesMapper;
 import com.android.server.utils.FlagNamespaceUtils;
 
@@ -79,19 +87,30 @@ public class RescueParty {
     @VisibleForTesting
     static final long BOOT_TRIGGER_WINDOW_MILLIS = 600 * DateUtils.SECOND_IN_MILLIS;
     @VisibleForTesting
-    static final long PERSISTENT_APP_CRASH_TRIGGER_WINDOW_MILLIS = 30 * DateUtils.SECOND_IN_MILLIS;
-    @VisibleForTesting
     static final String TAG = "RescueParty";
 
+    private static final String NAME = "rescue-party-observer";
+
+
     private static final String PROP_DISABLE_RESCUE = "persist.sys.disable_rescue";
     private static final String PROP_RESCUE_BOOT_START = "sys.rescue_boot_start";
     private static final String PROP_VIRTUAL_DEVICE = "ro.hardware.virtual_device";
 
+    private static final int PERSISTENT_MASK = ApplicationInfo.FLAG_PERSISTENT
+            | ApplicationInfo.FLAG_SYSTEM;
+
+
     /** Threshold for boot loops */
     private static final Threshold sBoot = new BootThreshold();
     /** Threshold for app crash loops */
     private static SparseArray<Threshold> sApps = new SparseArray<>();
 
+    /** Register the Rescue Party observer as a Package Watchdog health observer */
+    public static void registerHealthObserver(Context context) {
+        PackageWatchdog.getInstance(context).registerHealthObserver(
+                RescuePartyObserver.getInstance(context));
+    }
+
     private static boolean isDisabled() {
         // Check if we're explicitly enabled for testing
         if (SystemProperties.getBoolean(PROP_ENABLE_RESCUE, false)) {
@@ -135,24 +154,6 @@ public class RescueParty {
     }
 
     /**
-     * Take note of a persistent app or apex module crash. If we notice too many of these
-     * events happening in rapid succession, we'll send out a rescue party.
-     */
-    public static void noteAppCrash(Context context, int uid) {
-        if (isDisabled()) return;
-        Threshold t = sApps.get(uid);
-        if (t == null) {
-            t = new AppThreshold(uid);
-            sApps.put(uid, t);
-        }
-        if (t.incrementAndTest()) {
-            t.reset();
-            incrementRescueLevel(t.uid);
-            executeRescueLevel(context);
-        }
-    }
-
-    /**
      * Check if we're currently attempting to reboot for a factory reset.
      */
     public static boolean isAttemptingFactoryReset() {
@@ -171,11 +172,6 @@ public class RescueParty {
     @VisibleForTesting
     static void resetAllThresholds() {
         sBoot.reset();
-
-        for (int i = 0; i < sApps.size(); i++) {
-            Threshold appThreshold = sApps.get(sApps.keyAt(i));
-            appThreshold.reset();
-        }
     }
 
     @VisibleForTesting
@@ -243,6 +239,28 @@ public class RescueParty {
                 FlagNamespaceUtils.NAMESPACE_NO_PACKAGE);
     }
 
+    private static int mapRescueLevelToUserImpact(int rescueLevel) {
+        switch(rescueLevel) {
+            case LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS:
+            case LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES:
+                return PackageHealthObserverImpact.USER_IMPACT_LOW;
+            case LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS:
+            case LEVEL_FACTORY_RESET:
+                return PackageHealthObserverImpact.USER_IMPACT_HIGH;
+            default:
+                return PackageHealthObserverImpact.USER_IMPACT_NONE;
+        }
+    }
+
+    private static int getPackageUid(Context context, String packageName) {
+        try {
+            return context.getPackageManager().getPackageUid(packageName, 0);
+        } catch (PackageManager.NameNotFoundException e) {
+            // Since UIDs are always >= 0, this value means the UID could not be determined.
+            return -1;
+        }
+    }
+
     private static void resetAllSettings(Context context, int mode) throws Exception {
         // Try our best to reset all settings possible, and once finished
         // rethrow any exception that we encountered
@@ -271,6 +289,89 @@ public class RescueParty {
     }
 
     /**
+     * Handle mitigation action for package failures. This observer will be register to Package
+     * Watchdog and will receive calls about package failures. This observer is persistent so it
+     * may choose to mitigate failures for packages it has not explicitly asked to observe.
+     */
+    public static class RescuePartyObserver implements PackageHealthObserver {
+
+        private Context mContext;
+
+        @GuardedBy("RescuePartyObserver.class")
+        static RescuePartyObserver sRescuePartyObserver;
+
+        private RescuePartyObserver(Context context) {
+            mContext = context;
+        }
+
+        /** Creates or gets singleton instance of RescueParty. */
+        public static RescuePartyObserver getInstance(Context context) {
+            synchronized (RescuePartyObserver.class) {
+                if (sRescuePartyObserver == null) {
+                    sRescuePartyObserver = new RescuePartyObserver(context);
+                }
+                return sRescuePartyObserver;
+            }
+        }
+
+        @Override
+        public int onHealthCheckFailed(@Nullable VersionedPackage failedPackage,
+                @FailureReasons int failureReason) {
+            if (failureReason == PackageWatchdog.FAILURE_REASON_APP_CRASH
+                    || failureReason == PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING) {
+                int rescueLevel = MathUtils.constrain(
+                        SystemProperties.getInt(PROP_RESCUE_LEVEL, LEVEL_NONE) + 1,
+                        LEVEL_NONE, LEVEL_FACTORY_RESET);
+                return mapRescueLevelToUserImpact(rescueLevel);
+            } else {
+                return PackageHealthObserverImpact.USER_IMPACT_NONE;
+            }
+        }
+
+        @Override
+        public boolean execute(@Nullable VersionedPackage failedPackage,
+                @FailureReasons int failureReason) {
+            if (isDisabled()) {
+                return false;
+            }
+            if (failureReason == PackageWatchdog.FAILURE_REASON_APP_CRASH
+                    || failureReason == PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING) {
+                int triggerUid = getPackageUid(mContext, failedPackage.getPackageName());
+                incrementRescueLevel(triggerUid);
+                executeRescueLevel(mContext);
+                return true;
+            } else {
+                return false;
+            }
+        }
+
+        @Override
+        public boolean isPersistent() {
+            return true;
+        }
+
+        @Override
+        public boolean mayObservePackage(String packageName) {
+            PackageManager pm = mContext.getPackageManager();
+            try {
+                // A package is a Mainline module if this is non-null
+                if (pm.getModuleInfo(packageName, 0) != null) {
+                    return true;
+                }
+                ApplicationInfo info = pm.getApplicationInfo(packageName, 0);
+                return (info.flags & PERSISTENT_MASK) == PERSISTENT_MASK;
+            } catch (PackageManager.NameNotFoundException e) {
+                return false;
+            }
+        }
+
+        @Override
+        public String getName() {
+            return NAME;
+        }
+    }
+
+    /**
      * Threshold that can be triggered if a number of events occur within a
      * window of time.
      */
@@ -349,27 +450,6 @@ public class RescueParty {
         }
     }
 
-    /**
-     * Specialization of {@link Threshold} for monitoring app crashes. It stores
-     * counters in memory.
-     */
-    private static class AppThreshold extends Threshold {
-        private int count;
-        private long start;
-
-        public AppThreshold(int uid) {
-            // We're interested in TRIGGER_COUNT events in any
-            // PERSISTENT_APP_CRASH_TRIGGER_WINDOW_MILLIS second period; apps crash pretty quickly
-            // so we can keep a tight leash on them.
-            super(uid, TRIGGER_COUNT, PERSISTENT_APP_CRASH_TRIGGER_WINDOW_MILLIS);
-        }
-
-        @Override public int getCount() { return count; }
-        @Override public void setCount(int count) { this.count = count; }
-        @Override public long getStart() { return start; }
-        @Override public void setStart(long start) { this.start = start; }
-    }
-
     private static int[] getAllUserIds() {
         int[] userIds = { UserHandle.USER_SYSTEM };
         try {
diff --git a/services/core/java/com/android/server/am/AppErrors.java b/services/core/java/com/android/server/am/AppErrors.java
index 5e48dcf91676..8071f52037e6 100644
--- a/services/core/java/com/android/server/am/AppErrors.java
+++ b/services/core/java/com/android/server/am/AppErrors.java
@@ -33,8 +33,6 @@ import android.content.ActivityNotFoundException;
 import android.content.Context;
 import android.content.Intent;
 import android.content.pm.ApplicationInfo;
-import android.content.pm.ModuleInfo;
-import android.content.pm.PackageManager;
 import android.content.pm.VersionedPackage;
 import android.net.Uri;
 import android.os.Binder;
@@ -56,7 +54,6 @@ import com.android.internal.app.ProcessMap;
 import com.android.internal.logging.MetricsLogger;
 import com.android.internal.logging.nano.MetricsProto;
 import com.android.server.PackageWatchdog;
-import com.android.server.RescueParty;
 import com.android.server.wm.WindowProcessController;
 
 import java.io.FileDescriptor;
@@ -423,28 +420,6 @@ class AppErrors {
         }
 
         if (r != null) {
-            boolean isApexModule = false;
-            try {
-                for (String androidPackage : r.getPackageList()) {
-                    ModuleInfo moduleInfo = mContext.getPackageManager().getModuleInfo(
-                            androidPackage, /*flags=*/ 0);
-                    if (moduleInfo != null) {
-                        isApexModule = true;
-                        break;
-                    }
-                }
-            } catch (IllegalStateException | PackageManager.NameNotFoundException e) {
-                // Call to PackageManager#getModuleInfo() can result in NameNotFoundException or
-                // IllegalStateException. In case they are thrown, there isn't much we can do
-                // other than proceed with app crash handling.
-            }
-
-            if (r.isPersistent() || isApexModule) {
-                // If a persistent app or apex module is stuck in a crash loop, the device isn't
-                // very usable, so we want to consider sending out a rescue party.
-                RescueParty.noteAppCrash(mContext, r.uid);
-            }
-
             mPackageWatchdog.onPackageFailure(r.getPackageListWithVersionCode(),
                     PackageWatchdog.FAILURE_REASON_APP_CRASH);
         }
diff --git a/services/java/com/android/server/SystemServer.java b/services/java/com/android/server/SystemServer.java
index bfec51c92651..b6a8ca447213 100644
--- a/services/java/com/android/server/SystemServer.java
+++ b/services/java/com/android/server/SystemServer.java
@@ -751,6 +751,7 @@ public final class SystemServer {
         // Now that we have the bare essentials of the OS up and running, take
         // note that we just booted, which might send out a rescue party if
         // we're stuck in a runtime restart loop.
+        RescueParty.registerHealthObserver(mSystemContext);
         RescueParty.noteBoot(mSystemContext);
 
         // Manages LEDs and display backlight so we need it to bring up the display.
diff --git a/services/tests/mockingservicestests/src/com/android/server/RescuePartyTest.java b/services/tests/mockingservicestests/src/com/android/server/RescuePartyTest.java
index 0d6020c3d06d..30d89d31ba64 100644
--- a/services/tests/mockingservicestests/src/com/android/server/RescuePartyTest.java
+++ b/services/tests/mockingservicestests/src/com/android/server/RescuePartyTest.java
@@ -32,6 +32,7 @@ import static org.mockito.ArgumentMatchers.isNull;
 
 import android.content.ContentResolver;
 import android.content.Context;
+import android.content.pm.VersionedPackage;
 import android.os.RecoverySystem;
 import android.os.SystemProperties;
 import android.os.UserHandle;
@@ -39,6 +40,8 @@ import android.provider.DeviceConfig;
 import android.provider.Settings;
 
 import com.android.dx.mockito.inline.extended.ExtendedMockito;
+import com.android.server.PackageWatchdog.PackageHealthObserverImpact;
+import com.android.server.RescueParty.RescuePartyObserver;
 import com.android.server.am.SettingsToPropertiesMapper;
 import com.android.server.utils.FlagNamespaceUtils;
 
@@ -57,13 +60,15 @@ import java.util.HashMap;
  * Test RescueParty.
  */
 public class RescuePartyTest {
-    private static final int PERSISTENT_APP_UID = 12;
     private static final long CURRENT_NETWORK_TIME_MILLIS = 0L;
     private static final String FAKE_NATIVE_NAMESPACE1 = "native1";
     private static final String FAKE_NATIVE_NAMESPACE2 = "native2";
     private static final String[] FAKE_RESET_NATIVE_NAMESPACES =
             {FAKE_NATIVE_NAMESPACE1, FAKE_NATIVE_NAMESPACE2};
 
+    private static VersionedPackage sFailingPackage = new VersionedPackage("com.package.name", 1);
+    private static final String PROP_DISABLE_RESCUE = "persist.sys.disable_rescue";
+
     private MockitoSession mSession;
 
     @Mock(answer = Answers.RETURNS_DEEP_STUBS)
@@ -182,25 +187,25 @@ public class RescuePartyTest {
 
     @Test
     public void testPersistentAppCrashDetectionWithExecutionForAllRescueLevels() {
-        notePersistentAppCrash(RescueParty.TRIGGER_COUNT);
+        notePersistentAppCrash();
 
         verifySettingsResets(Settings.RESET_MODE_UNTRUSTED_DEFAULTS);
         assertEquals(RescueParty.LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS,
                 SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, RescueParty.LEVEL_NONE));
 
-        notePersistentAppCrash(RescueParty.TRIGGER_COUNT);
+        notePersistentAppCrash();
 
         verifySettingsResets(Settings.RESET_MODE_UNTRUSTED_CHANGES);
         assertEquals(RescueParty.LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES,
                 SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, RescueParty.LEVEL_NONE));
 
-        notePersistentAppCrash(RescueParty.TRIGGER_COUNT);
+        notePersistentAppCrash();
 
         verifySettingsResets(Settings.RESET_MODE_TRUSTED_DEFAULTS);
         assertEquals(RescueParty.LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS,
                 SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, RescueParty.LEVEL_NONE));
 
-        notePersistentAppCrash(RescueParty.TRIGGER_COUNT);
+        notePersistentAppCrash();
 
         verify(() -> RecoverySystem.rebootPromptAndWipeUserData(mMockContext, RescueParty.TAG));
         assertEquals(RescueParty.LEVEL_FACTORY_RESET,
@@ -221,20 +226,6 @@ public class RescuePartyTest {
     }
 
     @Test
-    public void testPersistentAppCrashDetectionWithWrongInterval() {
-        notePersistentAppCrash(RescueParty.TRIGGER_COUNT - 1);
-
-        // last persistent app crash is just outside of the boot loop detection window
-        doReturn(CURRENT_NETWORK_TIME_MILLIS
-                + RescueParty.PERSISTENT_APP_CRASH_TRIGGER_WINDOW_MILLIS + 1)
-                .when(() -> RescueParty.getElapsedRealtime());
-        notePersistentAppCrash(/*numTimes=*/1);
-
-        assertEquals(RescueParty.LEVEL_NONE,
-                SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, RescueParty.LEVEL_NONE));
-    }
-
-    @Test
     public void testBootLoopDetectionWithProperInterval() {
         noteBoot(RescueParty.TRIGGER_COUNT - 1);
 
@@ -249,21 +240,6 @@ public class RescuePartyTest {
     }
 
     @Test
-    public void testPersistentAppCrashDetectionWithProperInterval() {
-        notePersistentAppCrash(RescueParty.TRIGGER_COUNT - 1);
-
-        // last persistent app crash is just inside of the boot loop detection window
-        doReturn(CURRENT_NETWORK_TIME_MILLIS
-                + RescueParty.PERSISTENT_APP_CRASH_TRIGGER_WINDOW_MILLIS)
-                .when(() -> RescueParty.getElapsedRealtime());
-        notePersistentAppCrash(/*numTimes=*/1);
-
-        verifySettingsResets(Settings.RESET_MODE_UNTRUSTED_DEFAULTS);
-        assertEquals(RescueParty.LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS,
-                SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, RescueParty.LEVEL_NONE));
-    }
-
-    @Test
     public void testBootLoopDetectionWithWrongTriggerCount() {
         noteBoot(RescueParty.TRIGGER_COUNT - 1);
         assertEquals(RescueParty.LEVEL_NONE,
@@ -271,13 +247,6 @@ public class RescuePartyTest {
     }
 
     @Test
-    public void testPersistentAppCrashDetectionWithWrongTriggerCount() {
-        notePersistentAppCrash(RescueParty.TRIGGER_COUNT - 1);
-        assertEquals(RescueParty.LEVEL_NONE,
-                SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, RescueParty.LEVEL_NONE));
-    }
-
-    @Test
     public void testIsAttemptingFactoryReset() {
         noteBoot(RescueParty.TRIGGER_COUNT * 4);
 
@@ -319,6 +288,77 @@ public class RescuePartyTest {
                         FAKE_NATIVE_NAMESPACE2, /*makeDefault=*/true));
     }
 
+    @Test
+    public void testExplicitlyEnablingAndDisablingRescue() {
+        SystemProperties.set(RescueParty.PROP_ENABLE_RESCUE, Boolean.toString(false));
+        SystemProperties.set(PROP_DISABLE_RESCUE, Boolean.toString(true));
+        assertEquals(RescuePartyObserver.getInstance(mMockContext).execute(sFailingPackage,
+                PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING), false);
+
+        SystemProperties.set(RescueParty.PROP_ENABLE_RESCUE, Boolean.toString(true));
+        assertTrue(RescuePartyObserver.getInstance(mMockContext).execute(sFailingPackage,
+                PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING));
+    }
+
+    @Test
+    public void testHealthCheckLevels() {
+        RescuePartyObserver observer = RescuePartyObserver.getInstance(mMockContext);
+
+        // Ensure that no action is taken for cases where the failure reason is unknown
+        SystemProperties.set(RescueParty.PROP_RESCUE_LEVEL, Integer.toString(
+                RescueParty.LEVEL_FACTORY_RESET));
+        assertEquals(observer.onHealthCheckFailed(null, PackageWatchdog.FAILURE_REASON_UNKNOWN),
+                PackageHealthObserverImpact.USER_IMPACT_NONE);
+
+        /*
+        For the following cases, ensure that the returned user impact corresponds with the user
+        impact of the next available rescue level, not the current one.
+         */
+        SystemProperties.set(RescueParty.PROP_RESCUE_LEVEL, Integer.toString(
+                RescueParty.LEVEL_NONE));
+        assertEquals(observer.onHealthCheckFailed(null,
+                PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING),
+                PackageHealthObserverImpact.USER_IMPACT_LOW);
+
+        SystemProperties.set(RescueParty.PROP_RESCUE_LEVEL, Integer.toString(
+                RescueParty.LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS));
+        assertEquals(observer.onHealthCheckFailed(null,
+                PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING),
+                PackageHealthObserverImpact.USER_IMPACT_LOW);
+
+
+        SystemProperties.set(RescueParty.PROP_RESCUE_LEVEL, Integer.toString(
+                RescueParty.LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES));
+        assertEquals(observer.onHealthCheckFailed(null,
+                PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING),
+                PackageHealthObserverImpact.USER_IMPACT_HIGH);
+
+
+        SystemProperties.set(RescueParty.PROP_RESCUE_LEVEL, Integer.toString(
+                RescueParty.LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS));
+        assertEquals(observer.onHealthCheckFailed(null,
+                PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING),
+                PackageHealthObserverImpact.USER_IMPACT_HIGH);
+
+
+        SystemProperties.set(RescueParty.PROP_RESCUE_LEVEL, Integer.toString(
+                RescueParty.LEVEL_FACTORY_RESET));
+        assertEquals(observer.onHealthCheckFailed(null,
+                PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING),
+                PackageHealthObserverImpact.USER_IMPACT_HIGH);
+    }
+
+    @Test
+    public void testRescueLevelIncrementsWhenExecuted() {
+        RescuePartyObserver observer = RescuePartyObserver.getInstance(mMockContext);
+        SystemProperties.set(RescueParty.PROP_RESCUE_LEVEL, Integer.toString(
+                RescueParty.LEVEL_NONE));
+        observer.execute(sFailingPackage,
+                PackageWatchdog.FAILURE_REASON_APP_CRASH);
+        assertEquals(SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, -1),
+                RescueParty.LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS);
+    }
+
     private void verifySettingsResets(int resetMode) {
         verify(() -> Settings.Global.resetToDefaultsAsUser(mMockContentResolver, null,
                 resetMode, UserHandle.USER_SYSTEM));
@@ -332,9 +372,8 @@ public class RescuePartyTest {
         }
     }
 
-    private void notePersistentAppCrash(int numTimes) {
-        for (int i = 0; i < numTimes; i++) {
-            RescueParty.noteAppCrash(mMockContext, PERSISTENT_APP_UID);
-        }
+    private void notePersistentAppCrash() {
+        RescuePartyObserver.getInstance(mMockContext).execute(new VersionedPackage(
+                "com.package.name", 1), PackageWatchdog.FAILURE_REASON_UNKNOWN);
     }
 }
diff --git a/tests/PackageWatchdog/src/com/android/server/PackageWatchdogTest.java b/tests/PackageWatchdog/src/com/android/server/PackageWatchdogTest.java
index b4cafe41662e..656628eb39d5 100644
--- a/tests/PackageWatchdog/src/com/android/server/PackageWatchdogTest.java
+++ b/tests/PackageWatchdog/src/com/android/server/PackageWatchdogTest.java
@@ -896,39 +896,78 @@ public class PackageWatchdogTest {
         assertThat(observer.mHealthCheckFailedPackages).containsExactly(APP_A);
     }
 
-    /** Test that observers execute correctly for different failure reasons */
+    /** Test that observers execute correctly for failures reasons that go through thresholding. */
     @Test
-    public void testFailureReasons() {
+    public void testNonImmediateFailureReasons() {
         PackageWatchdog watchdog = createWatchdog();
         TestObserver observer1 = new TestObserver(OBSERVER_NAME_1);
         TestObserver observer2 = new TestObserver(OBSERVER_NAME_2);
-        TestObserver observer3 = new TestObserver(OBSERVER_NAME_3);
-        TestObserver observer4 = new TestObserver(OBSERVER_NAME_4);
 
         watchdog.startObservingHealth(observer1, Arrays.asList(APP_A), SHORT_DURATION);
         watchdog.startObservingHealth(observer2, Arrays.asList(APP_B), SHORT_DURATION);
-        watchdog.startObservingHealth(observer3, Arrays.asList(APP_C), SHORT_DURATION);
-        watchdog.startObservingHealth(observer4, Arrays.asList(APP_D), SHORT_DURATION);
 
         raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A,
-                VERSION_CODE)), PackageWatchdog.FAILURE_REASON_NATIVE_CRASH);
-        raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_B,
-                VERSION_CODE)), PackageWatchdog.FAILURE_REASON_EXPLICIT_HEALTH_CHECK);
-        raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_C,
                 VERSION_CODE)), PackageWatchdog.FAILURE_REASON_APP_CRASH);
-        raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_D,
+        raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_B,
                 VERSION_CODE)), PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING);
 
         assertThat(observer1.getLastFailureReason()).isEqualTo(
-                PackageWatchdog.FAILURE_REASON_NATIVE_CRASH);
-        assertThat(observer2.getLastFailureReason()).isEqualTo(
-                PackageWatchdog.FAILURE_REASON_EXPLICIT_HEALTH_CHECK);
-        assertThat(observer3.getLastFailureReason()).isEqualTo(
                 PackageWatchdog.FAILURE_REASON_APP_CRASH);
-        assertThat(observer4.getLastFailureReason()).isEqualTo(
+        assertThat(observer2.getLastFailureReason()).isEqualTo(
                 PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING);
     }
 
+    /** Test that observers execute correctly for failures reasons that skip thresholding. */
+    @Test
+    public void testImmediateFailures() {
+        PackageWatchdog watchdog = createWatchdog();
+        TestObserver observer1 = new TestObserver(OBSERVER_NAME_1);
+
+        watchdog.startObservingHealth(observer1, Arrays.asList(APP_A), SHORT_DURATION);
+
+        raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A,
+                VERSION_CODE)), PackageWatchdog.FAILURE_REASON_NATIVE_CRASH);
+        raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_B,
+                VERSION_CODE)), PackageWatchdog.FAILURE_REASON_EXPLICIT_HEALTH_CHECK);
+
+        assertThat(observer1.mMitigatedPackages).containsExactly(APP_A, APP_B);
+    }
+
+    /**
+     * Test that a persistent observer will mitigate failures if it wishes to observe a package.
+     */
+    @Test
+    public void testPersistentObserverWatchesPackage() {
+        PackageWatchdog watchdog = createWatchdog();
+        TestObserver persistentObserver = new TestObserver(OBSERVER_NAME_1);
+        persistentObserver.setPersistent(true);
+        persistentObserver.setMayObservePackages(true);
+
+        watchdog.startObservingHealth(persistentObserver, Arrays.asList(APP_B), SHORT_DURATION);
+
+        raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A,
+                VERSION_CODE)), PackageWatchdog.FAILURE_REASON_UNKNOWN);
+        assertThat(persistentObserver.mHealthCheckFailedPackages).containsExactly(APP_A);
+    }
+
+    /**
+     * Test that a persistent observer will not mitigate failures if it does not wish to observe
+     * a given package.
+     */
+    @Test
+    public void testPersistentObserverDoesNotWatchPackage() {
+        PackageWatchdog watchdog = createWatchdog();
+        TestObserver persistentObserver = new TestObserver(OBSERVER_NAME_1);
+        persistentObserver.setPersistent(true);
+        persistentObserver.setMayObservePackages(false);
+
+        watchdog.startObservingHealth(persistentObserver, Arrays.asList(APP_B), SHORT_DURATION);
+
+        raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A,
+                VERSION_CODE)), PackageWatchdog.FAILURE_REASON_UNKNOWN);
+        assertThat(persistentObserver.mHealthCheckFailedPackages).isEmpty();
+    }
+
     private void adoptShellPermissions(String... permissions) {
         InstrumentationRegistry
                 .getInstrumentation()
@@ -964,7 +1003,12 @@ public class PackageWatchdogTest {
     /** Trigger package failures above the threshold. */
     private void raiseFatalFailureAndDispatch(PackageWatchdog watchdog,
             List<VersionedPackage> packages, int failureReason) {
-        for (int i = 0; i < watchdog.getTriggerFailureCount(); i++) {
+        long triggerFailureCount = watchdog.getTriggerFailureCount();
+        if (failureReason == PackageWatchdog.FAILURE_REASON_EXPLICIT_HEALTH_CHECK
+                || failureReason == PackageWatchdog.FAILURE_REASON_NATIVE_CRASH) {
+            triggerFailureCount = 1;
+        }
+        for (int i = 0; i < triggerFailureCount; i++) {
             watchdog.onPackageFailure(packages, failureReason);
         }
         mTestLooper.dispatchAll();
@@ -1000,6 +1044,8 @@ public class PackageWatchdogTest {
         private final String mName;
         private int mImpact;
         private int mLastFailureReason;
+        private boolean mIsPersistent = false;
+        private boolean mMayObservePackages = false;
         final List<String> mHealthCheckFailedPackages = new ArrayList<>();
         final List<String> mMitigatedPackages = new ArrayList<>();
 
@@ -1028,9 +1074,25 @@ public class PackageWatchdogTest {
             return mName;
         }
 
+        public boolean isPersistent() {
+            return mIsPersistent;
+        }
+
+        public boolean mayObservePackage(String packageName) {
+            return mMayObservePackages;
+        }
+
         public int getLastFailureReason() {
             return mLastFailureReason;
         }
+
+        public void setPersistent(boolean persistent) {
+            mIsPersistent = persistent;
+        }
+
+        public void setMayObservePackages(boolean mayObservePackages) {
+            mMayObservePackages = mayObservePackages;
+        }
     }
 
     private static class TestController extends ExplicitHealthCheckController {