Don't count proxied jobs toward scheduling limit.

1. Some API calls (such as SyncManager calls) can result in multiple job
schedule calls. Given that underlying implementation detail, we don't
count jobs scheduled via proxy towards the source app's schedule limit.
Calls using scheduleAsPackage for itself will still count.
2. Limit the number of times we log that a specific app has exceeded the
limit to once per minute.
3. Update the error message and documentation to help developers
identify what to do when they encounter the issue.

Bug: 160796417
Test: atest JobSchedulerServiceTest
Test: atest JobSchedulingTest
Change-Id: Ia682aa889ec2a26d373d0309e8c0324570f7d816
Merged-In: Ia682aa889ec2a26d373d0309e8c0324570f7d816

4 files changed, 170 insertions, 20 deletions

diff --git a/apex/jobscheduler/framework/java/android/app/job/JobScheduler.java b/apex/jobscheduler/framework/java/android/app/job/JobScheduler.java
index 231263579088..42725c51fd87 100644
--- a/apex/jobscheduler/framework/java/android/app/job/JobScheduler.java
+++ b/apex/jobscheduler/framework/java/android/app/job/JobScheduler.java
@@ -59,9 +59,9 @@ import java.util.List;
  *
  * <p class="caution"><strong>Note:</strong> Beginning with API 30
  * ({@link android.os.Build.VERSION_CODES#R}), JobScheduler will throttle runaway applications.
- * Calling {@link #schedule(JobInfo)} and other such methods with very high frequency is indicative
- * of an app bug and so, to make sure the system doesn't get overwhelmed, JobScheduler will begin
- * to throttle apps that show buggy behavior, regardless of target SDK version.
+ * Calling {@link #schedule(JobInfo)} and other such methods with very high frequency can have a
+ * high cost and so, to make sure the system doesn't get overwhelmed, JobScheduler will begin
+ * to throttle apps, regardless of target SDK version.
  */
 @SystemService(Context.JOB_SCHEDULER_SERVICE)
 public abstract class JobScheduler {
@@ -74,9 +74,16 @@ public abstract class JobScheduler {
     public @interface Result {}
 
     /**
-     * Returned from {@link #schedule(JobInfo)} when an invalid parameter was supplied. This can occur
-     * if the run-time for your job is too short, or perhaps the system can't resolve the
-     * requisite {@link JobService} in your package.
+     * Returned from {@link #schedule(JobInfo)} if a job wasn't scheduled successfully. Scheduling
+     * can fail for a variety of reasons, including, but not limited to:
+     * <ul>
+     * <li>an invalid parameter was supplied (eg. the run-time for your job is too short, or the
+     * system can't resolve the requisite {@link JobService} in your package)</li>
+     * <li>the app has too many jobs scheduled</li>
+     * <li>the app has tried to schedule too many jobs in a short amount of time</li>
+     * </ul>
+     * Attempting to schedule the job again immediately after receiving this result will not
+     * guarantee a successful schedule.
      */
     public static final int RESULT_FAILURE = 0;
     /**
@@ -89,6 +96,11 @@ public abstract class JobScheduler {
      * ID with the new information in the {@link JobInfo}.  If a job with the given ID is currently
      * running, it will be stopped.
      *
+     * <p class="caution"><strong>Note:</strong> Scheduling a job can have a high cost, even if it's
+     * rescheduling the same job and the job didn't execute, especially on platform versions before
+     * version {@link android.os.Build.VERSION_CODES#Q}. As such, the system may throttle calls to
+     * this API if calls are made too frequently in a short amount of time.
+     *
      * @param job The job you wish scheduled. See
      * {@link android.app.job.JobInfo.Builder JobInfo.Builder} for more detail on the sorts of jobs
      * you can schedule.
diff --git a/apex/jobscheduler/service/java/com/android/server/job/JobSchedulerService.java b/apex/jobscheduler/service/java/com/android/server/job/JobSchedulerService.java
index e88865161dfa..871e40fc9dfe 100644
--- a/apex/jobscheduler/service/java/com/android/server/job/JobSchedulerService.java
+++ b/apex/jobscheduler/service/java/com/android/server/job/JobSchedulerService.java
@@ -255,6 +255,18 @@ public class JobSchedulerService extends com.android.server.SystemService
 
     private final CountQuotaTracker mQuotaTracker;
     private static final String QUOTA_TRACKER_SCHEDULE_PERSISTED_TAG = ".schedulePersisted()";
+    private static final String QUOTA_TRACKER_SCHEDULE_LOGGED =
+            ".schedulePersisted out-of-quota logged";
+    private static final Category QUOTA_TRACKER_CATEGORY_SCHEDULE_PERSISTED = new Category(
+            ".schedulePersisted()");
+    private static final Category QUOTA_TRACKER_CATEGORY_SCHEDULE_LOGGED = new Category(
+            ".schedulePersisted out-of-quota logged");
+    private static final Categorizer QUOTA_CATEGORIZER = (userId, packageName, tag) -> {
+        if (QUOTA_TRACKER_SCHEDULE_PERSISTED_TAG.equals(tag)) {
+            return QUOTA_TRACKER_CATEGORY_SCHEDULE_PERSISTED;
+        }
+        return QUOTA_TRACKER_CATEGORY_SCHEDULE_LOGGED;
+    };
 
     /**
      * Queue of pending jobs. The JobServiceContext class will receive jobs from this list
@@ -271,6 +283,7 @@ public class JobSchedulerService extends com.android.server.SystemService
     ActivityManagerInternal mActivityManagerInternal;
     IBatteryStats mBatteryStats;
     DeviceIdleInternal mLocalDeviceIdleController;
+    @VisibleForTesting
     AppStateTracker mAppStateTracker;
     final UsageStatsManagerInternal mUsageStats;
     private final AppStandbyInternal mAppStandbyInternal;
@@ -343,10 +356,7 @@ public class JobSchedulerService extends com.android.server.SystemService
                         final StateController sc = mControllers.get(controller);
                         sc.onConstantsUpdatedLocked();
                     }
-                    mQuotaTracker.setEnabled(mConstants.ENABLE_API_QUOTAS);
-                    mQuotaTracker.setCountLimit(Category.SINGLE_CATEGORY,
-                            mConstants.API_QUOTA_SCHEDULE_COUNT,
-                            mConstants.API_QUOTA_SCHEDULE_WINDOW_MS);
+                    updateQuotaTracker();
                 } catch (IllegalArgumentException e) {
                     // Failed to parse the settings string, log this and move on
                     // with defaults.
@@ -356,6 +366,14 @@ public class JobSchedulerService extends com.android.server.SystemService
         }
     }
 
+    @VisibleForTesting
+    void updateQuotaTracker() {
+        mQuotaTracker.setEnabled(mConstants.ENABLE_API_QUOTAS);
+        mQuotaTracker.setCountLimit(QUOTA_TRACKER_CATEGORY_SCHEDULE_PERSISTED,
+                mConstants.API_QUOTA_SCHEDULE_COUNT,
+                mConstants.API_QUOTA_SCHEDULE_WINDOW_MS);
+    }
+
     static class MaxJobCounts {
         private final KeyValueListParser.IntValue mTotal;
         private final KeyValueListParser.IntValue mMaxBg;
@@ -508,6 +526,8 @@ public class JobSchedulerService extends com.android.server.SystemService
         private static final String KEY_API_QUOTA_SCHEDULE_WINDOW_MS = "aq_schedule_window_ms";
         private static final String KEY_API_QUOTA_SCHEDULE_THROW_EXCEPTION =
                 "aq_schedule_throw_exception";
+        private static final String KEY_API_QUOTA_SCHEDULE_RETURN_FAILURE_RESULT =
+                "aq_schedule_return_failure";
 
         private static final int DEFAULT_MIN_READY_NON_ACTIVE_JOBS_COUNT = 5;
         private static final long DEFAULT_MAX_NON_ACTIVE_JOB_BATCH_DELAY_MS = 31 * MINUTE_IN_MILLIS;
@@ -521,6 +541,7 @@ public class JobSchedulerService extends com.android.server.SystemService
         private static final int DEFAULT_API_QUOTA_SCHEDULE_COUNT = 250;
         private static final long DEFAULT_API_QUOTA_SCHEDULE_WINDOW_MS = MINUTE_IN_MILLIS;
         private static final boolean DEFAULT_API_QUOTA_SCHEDULE_THROW_EXCEPTION = true;
+        private static final boolean DEFAULT_API_QUOTA_SCHEDULE_RETURN_FAILURE_RESULT = false;
 
         /**
          * Minimum # of non-ACTIVE jobs for which the JMS will be happy running some work early.
@@ -624,6 +645,11 @@ public class JobSchedulerService extends com.android.server.SystemService
          */
         public boolean API_QUOTA_SCHEDULE_THROW_EXCEPTION =
                 DEFAULT_API_QUOTA_SCHEDULE_THROW_EXCEPTION;
+        /**
+         * Whether or not to return a failure result when an app hits its schedule quota limit.
+         */
+        public boolean API_QUOTA_SCHEDULE_RETURN_FAILURE_RESULT =
+                DEFAULT_API_QUOTA_SCHEDULE_RETURN_FAILURE_RESULT;
 
         private final KeyValueListParser mParser = new KeyValueListParser(',');
 
@@ -679,6 +705,9 @@ public class JobSchedulerService extends com.android.server.SystemService
             API_QUOTA_SCHEDULE_THROW_EXCEPTION = mParser.getBoolean(
                     KEY_API_QUOTA_SCHEDULE_THROW_EXCEPTION,
                     DEFAULT_API_QUOTA_SCHEDULE_THROW_EXCEPTION);
+            API_QUOTA_SCHEDULE_RETURN_FAILURE_RESULT = mParser.getBoolean(
+                    KEY_API_QUOTA_SCHEDULE_RETURN_FAILURE_RESULT,
+                    DEFAULT_API_QUOTA_SCHEDULE_RETURN_FAILURE_RESULT);
         }
 
         void dump(IndentingPrintWriter pw) {
@@ -713,6 +742,8 @@ public class JobSchedulerService extends com.android.server.SystemService
             pw.printPair(KEY_API_QUOTA_SCHEDULE_WINDOW_MS, API_QUOTA_SCHEDULE_WINDOW_MS).println();
             pw.printPair(KEY_API_QUOTA_SCHEDULE_THROW_EXCEPTION,
                     API_QUOTA_SCHEDULE_THROW_EXCEPTION).println();
+            pw.printPair(KEY_API_QUOTA_SCHEDULE_RETURN_FAILURE_RESULT,
+                    API_QUOTA_SCHEDULE_RETURN_FAILURE_RESULT).println();
 
             pw.decreaseIndent();
         }
@@ -741,6 +772,8 @@ public class JobSchedulerService extends com.android.server.SystemService
             proto.write(ConstantsProto.API_QUOTA_SCHEDULE_WINDOW_MS, API_QUOTA_SCHEDULE_WINDOW_MS);
             proto.write(ConstantsProto.API_QUOTA_SCHEDULE_THROW_EXCEPTION,
                     API_QUOTA_SCHEDULE_THROW_EXCEPTION);
+            proto.write(ConstantsProto.API_QUOTA_SCHEDULE_RETURN_FAILURE_RESULT,
+                    API_QUOTA_SCHEDULE_RETURN_FAILURE_RESULT);
         }
     }
 
@@ -974,12 +1007,17 @@ public class JobSchedulerService extends com.android.server.SystemService
 
     public int scheduleAsPackage(JobInfo job, JobWorkItem work, int uId, String packageName,
             int userId, String tag) {
-        if (job.isPersisted()) {
-            // Only limit schedule calls for persisted jobs.
+        final String servicePkg = job.getService().getPackageName();
+        if (job.isPersisted() && (packageName == null || packageName.equals(servicePkg))) {
+            // Only limit schedule calls for persisted jobs scheduled by the app itself.
             final String pkg =
                     packageName == null ? job.getService().getPackageName() : packageName;
             if (!mQuotaTracker.isWithinQuota(userId, pkg, QUOTA_TRACKER_SCHEDULE_PERSISTED_TAG)) {
-                Slog.e(TAG, userId + "-" + pkg + " has called schedule() too many times");
+                if (mQuotaTracker.isWithinQuota(userId, pkg, QUOTA_TRACKER_SCHEDULE_LOGGED)) {
+                    // Don't log too frequently
+                    Slog.wtf(TAG, userId + "-" + pkg + " has called schedule() too many times");
+                    mQuotaTracker.noteEvent(userId, pkg, QUOTA_TRACKER_SCHEDULE_LOGGED);
+                }
                 mAppStandbyInternal.restrictApp(
                         pkg, userId, UsageStatsManager.REASON_SUB_FORCED_SYSTEM_FLAG_BUGGY);
                 if (mConstants.API_QUOTA_SCHEDULE_THROW_EXCEPTION) {
@@ -1005,13 +1043,17 @@ public class JobSchedulerService extends com.android.server.SystemService
                         // Only throw the exception for debuggable apps.
                         throw new LimitExceededException(
                                 "schedule()/enqueue() called more than "
-                                        + mQuotaTracker.getLimit(Category.SINGLE_CATEGORY)
+                                        + mQuotaTracker.getLimit(
+                                        QUOTA_TRACKER_CATEGORY_SCHEDULE_PERSISTED)
                                         + " times in the past "
-                                        + mQuotaTracker.getWindowSizeMs(Category.SINGLE_CATEGORY)
-                                        + "ms");
+                                        + mQuotaTracker.getWindowSizeMs(
+                                        QUOTA_TRACKER_CATEGORY_SCHEDULE_PERSISTED)
+                                        + "ms. See the documentation for more information.");
                     }
                 }
-                return JobScheduler.RESULT_FAILURE;
+                if (mConstants.API_QUOTA_SCHEDULE_RETURN_FAILURE_RESULT) {
+                    return JobScheduler.RESULT_FAILURE;
+                }
             }
             mQuotaTracker.noteEvent(userId, pkg, QUOTA_TRACKER_SCHEDULE_PERSISTED_TAG);
         }
@@ -1372,10 +1414,12 @@ public class JobSchedulerService extends com.android.server.SystemService
         // Set up the app standby bucketing tracker
         mStandbyTracker = new StandbyTracker();
         mUsageStats = LocalServices.getService(UsageStatsManagerInternal.class);
-        mQuotaTracker = new CountQuotaTracker(context, Categorizer.SINGLE_CATEGORIZER);
-        mQuotaTracker.setCountLimit(Category.SINGLE_CATEGORY,
+        mQuotaTracker = new CountQuotaTracker(context, QUOTA_CATEGORIZER);
+        mQuotaTracker.setCountLimit(QUOTA_TRACKER_CATEGORY_SCHEDULE_PERSISTED,
                 mConstants.API_QUOTA_SCHEDULE_COUNT,
                 mConstants.API_QUOTA_SCHEDULE_WINDOW_MS);
+        // Log at most once per minute.
+        mQuotaTracker.setCountLimit(QUOTA_TRACKER_CATEGORY_SCHEDULE_LOGGED, 1, 60_000);
 
         mAppStandbyInternal = LocalServices.getService(AppStandbyInternal.class);
         mAppStandbyInternal.addListener(mStandbyTracker);
diff --git a/core/proto/android/server/jobscheduler.proto b/core/proto/android/server/jobscheduler.proto
index ec99684bf636..f2f20e3ac12e 100644
--- a/core/proto/android/server/jobscheduler.proto
+++ b/core/proto/android/server/jobscheduler.proto
@@ -236,6 +236,8 @@ message ConstantsProto {
     optional int64 api_quota_schedule_window_ms = 33;
     // Whether or not to throw an exception when an app hits its schedule quota limit.
     optional bool api_quota_schedule_throw_exception = 34;
+    // Whether or not to return a failure result when an app hits its schedule quota limit.
+    optional bool api_quota_schedule_return_failure_result = 35;
 
     message QuotaController {
         option (.android.msg_privacy).dest = DEST_AUTOMATIC;
@@ -335,7 +337,7 @@ message ConstantsProto {
     // In this time after screen turns on, we increase job concurrency.
     optional int32 screen_off_job_concurrency_increase_delay_ms = 28;
 
-    // Next tag: 35
+    // Next tag: 36
 }
 
 // Next tag: 4
diff --git a/services/tests/mockingservicestests/src/com/android/server/job/JobSchedulerServiceTest.java b/services/tests/mockingservicestests/src/com/android/server/job/JobSchedulerServiceTest.java
index 730f303f036a..a462dc309c24 100644
--- a/services/tests/mockingservicestests/src/com/android/server/job/JobSchedulerServiceTest.java
+++ b/services/tests/mockingservicestests/src/com/android/server/job/JobSchedulerServiceTest.java
@@ -39,6 +39,7 @@ import android.app.ActivityManager;
 import android.app.ActivityManagerInternal;
 import android.app.IActivityManager;
 import android.app.job.JobInfo;
+import android.app.job.JobScheduler;
 import android.app.usage.UsageStatsManagerInternal;
 import android.content.ComponentName;
 import android.content.Context;
@@ -56,6 +57,7 @@ import android.os.SystemClock;
 import com.android.server.AppStateTracker;
 import com.android.server.DeviceIdleInternal;
 import com.android.server.LocalServices;
+import com.android.server.SystemServiceManager;
 import com.android.server.job.controllers.JobStatus;
 import com.android.server.usage.AppStandbyInternal;
 
@@ -82,6 +84,7 @@ public class JobSchedulerServiceTest {
     private class TestJobSchedulerService extends JobSchedulerService {
         TestJobSchedulerService(Context context) {
             super(context);
+            mAppStateTracker = mock(AppStateTracker.class);
         }
 
         @Override
@@ -136,6 +139,9 @@ public class JobSchedulerServiceTest {
         } catch (RemoteException e) {
             fail("registerUidObserver threw exception: " + e.getMessage());
         }
+        // Called by QuotaTracker
+        doReturn(mock(SystemServiceManager.class))
+                .when(() -> LocalServices.getService(SystemServiceManager.class));
 
         JobSchedulerService.sSystemClock = Clock.fixed(Clock.systemUTC().instant(), ZoneOffset.UTC);
         JobSchedulerService.sElapsedRealtimeClock =
@@ -750,4 +756,90 @@ public class JobSchedulerServiceTest {
         maybeQueueFunctor.postProcess();
         assertEquals(3, mService.mPendingJobs.size());
     }
+
+    /** Tests that jobs scheduled by the app itself are counted towards scheduling limits. */
+    @Test
+    public void testScheduleLimiting_RegularSchedule_Blocked() {
+        mService.mConstants.ENABLE_API_QUOTAS = true;
+        mService.mConstants.API_QUOTA_SCHEDULE_COUNT = 300;
+        mService.mConstants.API_QUOTA_SCHEDULE_WINDOW_MS = 300000;
+        mService.mConstants.API_QUOTA_SCHEDULE_THROW_EXCEPTION = false;
+        mService.mConstants.API_QUOTA_SCHEDULE_RETURN_FAILURE_RESULT = true;
+        mService.updateQuotaTracker();
+
+        final JobInfo job = createJobInfo().setPersisted(true).build();
+        for (int i = 0; i < 500; ++i) {
+            final int expected =
+                    i < 300 ? JobScheduler.RESULT_SUCCESS : JobScheduler.RESULT_FAILURE;
+            assertEquals("Got unexpected result for schedule #" + (i + 1),
+                    expected,
+                    mService.scheduleAsPackage(job, null, 10123, null, 0, ""));
+        }
+    }
+
+    /**
+     * Tests that jobs scheduled by the app itself succeed even if the app is above the scheduling
+     * limit.
+     */
+    @Test
+    public void testScheduleLimiting_RegularSchedule_Allowed() {
+        mService.mConstants.ENABLE_API_QUOTAS = true;
+        mService.mConstants.API_QUOTA_SCHEDULE_COUNT = 300;
+        mService.mConstants.API_QUOTA_SCHEDULE_WINDOW_MS = 300000;
+        mService.mConstants.API_QUOTA_SCHEDULE_THROW_EXCEPTION = false;
+        mService.mConstants.API_QUOTA_SCHEDULE_RETURN_FAILURE_RESULT = false;
+        mService.updateQuotaTracker();
+
+        final JobInfo job = createJobInfo().setPersisted(true).build();
+        for (int i = 0; i < 500; ++i) {
+            assertEquals("Got unexpected result for schedule #" + (i + 1),
+                    JobScheduler.RESULT_SUCCESS,
+                    mService.scheduleAsPackage(job, null, 10123, null, 0, ""));
+        }
+    }
+
+    /**
+     * Tests that jobs scheduled through a proxy (eg. system server) don't count towards scheduling
+     * limits.
+     */
+    @Test
+    public void testScheduleLimiting_Proxy() {
+        mService.mConstants.ENABLE_API_QUOTAS = true;
+        mService.mConstants.API_QUOTA_SCHEDULE_COUNT = 300;
+        mService.mConstants.API_QUOTA_SCHEDULE_WINDOW_MS = 300000;
+        mService.mConstants.API_QUOTA_SCHEDULE_THROW_EXCEPTION = false;
+        mService.mConstants.API_QUOTA_SCHEDULE_RETURN_FAILURE_RESULT = true;
+        mService.updateQuotaTracker();
+
+        final JobInfo job = createJobInfo().setPersisted(true).build();
+        for (int i = 0; i < 500; ++i) {
+            assertEquals("Got unexpected result for schedule #" + (i + 1),
+                    JobScheduler.RESULT_SUCCESS,
+                    mService.scheduleAsPackage(job, null, 10123, "proxied.package", 0, ""));
+        }
+    }
+
+    /**
+     * Tests that jobs scheduled by an app for itself as if through a proxy are counted towards
+     * scheduling limits.
+     */
+    @Test
+    public void testScheduleLimiting_SelfProxy() {
+        mService.mConstants.ENABLE_API_QUOTAS = true;
+        mService.mConstants.API_QUOTA_SCHEDULE_COUNT = 300;
+        mService.mConstants.API_QUOTA_SCHEDULE_WINDOW_MS = 300000;
+        mService.mConstants.API_QUOTA_SCHEDULE_THROW_EXCEPTION = false;
+        mService.mConstants.API_QUOTA_SCHEDULE_RETURN_FAILURE_RESULT = true;
+        mService.updateQuotaTracker();
+
+        final JobInfo job = createJobInfo().setPersisted(true).build();
+        for (int i = 0; i < 500; ++i) {
+            final int expected =
+                    i < 300 ? JobScheduler.RESULT_SUCCESS : JobScheduler.RESULT_FAILURE;
+            assertEquals("Got unexpected result for schedule #" + (i + 1),
+                    expected,
+                    mService.scheduleAsPackage(job, null, 10123, job.getService().getPackageName(),
+                            0, ""));
+        }
+    }
 }