diff options
author | Varun Shah <varunshah@google.com> | 2019-10-07 14:41:12 -0700 |
---|---|---|
committer | Varun Shah <varunshah@google.com> | 2019-10-17 18:11:37 -0700 |
commit | 23033b0046709140355cf7062bc52611087a1b6c (patch) | |
tree | 0c12e7d0d4ba4c84afd24e1b37af15a2bab2ca89 | |
parent | 358d5006234b1e5f0feb95f72ef42e592d28e661 (diff) |
Do not retain UsageStats for uninstalled packages.
When packages are removed, remove their associated token mappings in the
packages token data stored in UsageStats. When data is being persisted
to disk, the data for removed packages will not be written to disk.
Additionally, when any of the query APIs are called, data that is read
from disk for which a mapping does not exist will not be returned.
The data deletion uses a lazy techinque to avoid heavy costs of reading
and writing all of the usage stats data on every package removal.
Bug: 135484470
Test: atest IntervalStatsTests
Test: atest UsageStatsDatabaseTest
Change-Id: Ie32d65b47f86071c6a814a8b21e4be060519e598
7 files changed, 249 insertions, 19 deletions
diff --git a/services/tests/servicestests/src/com/android/server/usage/UsageStatsDatabaseTest.java b/services/tests/servicestests/src/com/android/server/usage/UsageStatsDatabaseTest.java index 3a0ad4d76ef3..df0c37a9856c 100644 --- a/services/tests/servicestests/src/com/android/server/usage/UsageStatsDatabaseTest.java +++ b/services/tests/servicestests/src/com/android/server/usage/UsageStatsDatabaseTest.java @@ -74,6 +74,7 @@ public class UsageStatsDatabaseTest { mContext = InstrumentationRegistry.getTargetContext(); mTestDir = new File(mContext.getFilesDir(), "UsageStatsDatabaseTest"); mUsageStatsDatabase = new UsageStatsDatabase(mTestDir); + mUsageStatsDatabase.readMappingsLocked(); mUsageStatsDatabase.init(1); populateIntervalStats(); clearUsageStatsFiles(); @@ -388,6 +389,7 @@ public class UsageStatsDatabaseTest { void runVersionChangeTest(int oldVersion, int newVersion, int interval) throws IOException { // Write IntervalStats to disk in old version format UsageStatsDatabase prevDB = new UsageStatsDatabase(mTestDir, oldVersion); + prevDB.readMappingsLocked(); prevDB.init(1); prevDB.putUsageStats(interval, mIntervalStats); if (oldVersion >= 5) { @@ -396,6 +398,7 @@ public class UsageStatsDatabaseTest { // Simulate an upgrade to a new version and read from the disk UsageStatsDatabase newDB = new UsageStatsDatabase(mTestDir, newVersion); + newDB.readMappingsLocked(); newDB.init(mEndTime); List<IntervalStats> stats = newDB.queryUsageStats(interval, 0, mEndTime, mIntervalStatsVerifier); @@ -415,6 +418,7 @@ public class UsageStatsDatabaseTest { */ void runBackupRestoreTest(int version) throws IOException { UsageStatsDatabase prevDB = new UsageStatsDatabase(mTestDir); + prevDB.readMappingsLocked(); prevDB.init(1); prevDB.putUsageStats(UsageStatsManager.INTERVAL_DAILY, mIntervalStats); // Create a backup with a specific version @@ -423,6 +427,7 @@ public class UsageStatsDatabaseTest { clearUsageStatsFiles(); UsageStatsDatabase newDB = new UsageStatsDatabase(mTestDir); + newDB.readMappingsLocked(); newDB.init(1); // Attempt to restore the usage stats from the backup newDB.applyRestoredPayload(KEY_USAGE_STATS, blob); @@ -539,12 +544,14 @@ public class UsageStatsDatabaseTest { private void compareObfuscatedData(int interval) throws IOException { // Write IntervalStats to disk UsageStatsDatabase prevDB = new UsageStatsDatabase(mTestDir, 5); + prevDB.readMappingsLocked(); prevDB.init(1); prevDB.putUsageStats(interval, mIntervalStats); prevDB.writeMappingsLocked(); // Read IntervalStats from disk into a new db UsageStatsDatabase newDB = new UsageStatsDatabase(mTestDir, 5); + newDB.readMappingsLocked(); newDB.init(mEndTime); List<IntervalStats> stats = newDB.queryUsageStats(interval, 0, mEndTime, mIntervalStatsVerifier); @@ -561,4 +568,40 @@ public class UsageStatsDatabaseTest { compareObfuscatedData(UsageStatsManager.INTERVAL_MONTHLY); compareObfuscatedData(UsageStatsManager.INTERVAL_YEARLY); } + + private void verifyPackageNotRetained(int interval) throws IOException { + UsageStatsDatabase db = new UsageStatsDatabase(mTestDir, 5); + db.readMappingsLocked(); + db.init(1); + db.putUsageStats(interval, mIntervalStats); + + final String removedPackage = "fake.package.name0"; + // invoke handler call directly from test to remove package + db.onPackageRemoved(removedPackage, System.currentTimeMillis()); + + List<IntervalStats> stats = db.queryUsageStats(interval, 0, mEndTime, + mIntervalStatsVerifier); + for (int i = 0; i < stats.size(); i++) { + final IntervalStats stat = stats.get(i); + if (stat.packageStats.containsKey(removedPackage)) { + fail("Found removed package " + removedPackage + " in package stats."); + return; + } + for (int j = 0; j < stat.events.size(); j++) { + final Event event = stat.events.get(j); + if (removedPackage.equals(event.mPackage)) { + fail("Found an event from removed package " + removedPackage); + return; + } + } + } + } + + @Test + public void testPackageRetention() throws IOException { + verifyPackageNotRetained(UsageStatsManager.INTERVAL_DAILY); + verifyPackageNotRetained(UsageStatsManager.INTERVAL_WEEKLY); + verifyPackageNotRetained(UsageStatsManager.INTERVAL_MONTHLY); + verifyPackageNotRetained(UsageStatsManager.INTERVAL_YEARLY); + } } diff --git a/services/usage/java/com/android/server/usage/IntervalStats.java b/services/usage/java/com/android/server/usage/IntervalStats.java index 7ea669d1f0a2..46b261b64192 100644 --- a/services/usage/java/com/android/server/usage/IntervalStats.java +++ b/services/usage/java/com/android/server/usage/IntervalStats.java @@ -454,8 +454,7 @@ public class IntervalStats { for (int statsIndex = 0; statsIndex < usageStatsSize; statsIndex++) { final int packageToken = packageStatsObfuscated.keyAt(statsIndex); final UsageStats usageStats = packageStatsObfuscated.valueAt(statsIndex); - usageStats.mPackageName = packagesTokenData.getString(packageToken, - PackagesTokenData.PACKAGE_NAME_INDEX); + usageStats.mPackageName = packagesTokenData.getPackageString(packageToken); if (usageStats.mPackageName == null) { Slog.e(TAG, "Unable to parse usage stats package " + packageToken); continue; @@ -501,8 +500,7 @@ public class IntervalStats { for (int i = this.events.size() - 1; i >= 0; i--) { final Event event = this.events.get(i); final int packageToken = event.mPackageToken; - event.mPackage = packagesTokenData.getString(packageToken, - PackagesTokenData.PACKAGE_NAME_INDEX); + event.mPackage = packagesTokenData.getPackageString(packageToken); if (event.mPackage == null) { Slog.e(TAG, "Unable to parse event package " + packageToken); this.events.remove(i); @@ -586,7 +584,12 @@ public class IntervalStats { continue; } - final int packageToken = packagesTokenData.getPackageTokenOrAdd(packageName); + final int packageToken = packagesTokenData.getPackageTokenOrAdd( + packageName, usageStats.mEndTimeStamp); + // don't obfuscate stats whose packages have been removed + if (packageToken == PackagesTokenData.UNASSIGNED_TOKEN) { + continue; + } usageStats.mPackageToken = packageToken; // Update chooser counts. final int chooserActionsSize = usageStats.mChooserCounts.size(); @@ -619,14 +622,19 @@ public class IntervalStats { * task root package and class names, and shortcut and notification channel ids. */ private void obfuscateEventsData(PackagesTokenData packagesTokenData) { - final int eventSize = events.size(); - for (int i = 0; i < eventSize; i++) { + for (int i = events.size() - 1; i >= 0; i--) { final Event event = events.get(i); if (event == null) { continue; } - final int packageToken = packagesTokenData.getPackageTokenOrAdd(event.mPackage); + final int packageToken = packagesTokenData.getPackageTokenOrAdd( + event.mPackage, event.mTimeStamp); + // don't obfuscate events from packages that have been removed + if (packageToken == PackagesTokenData.UNASSIGNED_TOKEN) { + events.remove(i); + continue; + } event.mPackageToken = packageToken; if (!TextUtils.isEmpty(event.mClass)) { event.mClassToken = packagesTokenData.getTokenOrAdd(packageToken, diff --git a/services/usage/java/com/android/server/usage/PackagesTokenData.java b/services/usage/java/com/android/server/usage/PackagesTokenData.java index 3beee678d7ff..4bf08a49af0f 100644 --- a/services/usage/java/com/android/server/usage/PackagesTokenData.java +++ b/services/usage/java/com/android/server/usage/PackagesTokenData.java @@ -29,14 +29,14 @@ import java.util.ArrayList; */ public final class PackagesTokenData { /** - * The default token for any string that hasn't been tokenized yet. + * The package name is always stored at index 0 in {@code tokensToPackagesMap}. */ - public static final int UNASSIGNED_TOKEN = -1; + private static final int PACKAGE_NAME_INDEX = 0; /** - * The package name is always stored at index 0 in {@code tokensToPackagesMap}. + * The default token for any string that hasn't been tokenized yet. */ - public static final int PACKAGE_NAME_INDEX = 0; + public static final int UNASSIGNED_TOKEN = -1; /** * The main token counter for each package. @@ -52,6 +52,10 @@ public final class PackagesTokenData { * map of the {@code tokenToPackagesMap} in this class, mainly for an O(1) access to the tokens. */ public final ArrayMap<String, ArrayMap<String, Integer>> packagesToTokensMap = new ArrayMap<>(); + /** + * Stores a map of packages that were removed and when they were removed. + */ + public final ArrayMap<String, Long> removedPackagesMap = new ArrayMap<>(); public PackagesTokenData() { } @@ -61,9 +65,26 @@ public final class PackagesTokenData { * created and the relevant mappings are updated. * * @param packageName the package name whose token is being fetched + * @param timeStamp the time stamp of the event or end time of the usage stats; used to verify + * the package hasn't been removed * @return the mapped token */ - public int getPackageTokenOrAdd(String packageName) { + public int getPackageTokenOrAdd(String packageName, long timeStamp) { + final Long timeRemoved = removedPackagesMap.get(packageName); + if (timeRemoved != null && timeRemoved > timeStamp) { + return UNASSIGNED_TOKEN; // package was removed + /* + Note: instead of querying Package Manager each time for a list of packages to verify + if this package is still installed, it's more efficient to check the internal list of + removed packages and verify with the incoming time stamp. Although rare, it is possible + that some asynchronous function is triggered after a package is removed and the + time stamp passed into this function is not accurate. We'll have to keep the respective + event/usage stat until the next time the device reboots and the mappings are cleaned. + Additionally, this is a data class with some helper methods - it doesn't make sense to + overload it with references to other services. + */ + } + ArrayMap<String, Integer> packageTokensMap = packagesToTokensMap.get(packageName); if (packageTokensMap == null) { packageTokensMap = new ArrayMap<>(); @@ -104,6 +125,20 @@ public final class PackagesTokenData { } /** + * Fetches the package name for the given token. + * + * @param packageToken the package token representing the package name + * @return the string representing the given token or {@code null} if not found + */ + public String getPackageString(int packageToken) { + final ArrayList<String> packageStrings = tokensToPackagesMap.get(packageToken); + if (packageStrings == null) { + return null; + } + return packageStrings.get(PACKAGE_NAME_INDEX); + } + + /** * Fetches the string represented by the given token. * * @param packageToken the package token for which this token belongs to @@ -121,4 +156,21 @@ public final class PackagesTokenData { return null; } } + + /** + * Removes the package from all known mappings. + * + * @param packageName the package to be removed + * @param timeRemoved the time stamp of when the package was removed + */ + public void removePackage(String packageName, long timeRemoved) { + removedPackagesMap.put(packageName, timeRemoved); + + if (!packagesToTokensMap.containsKey(packageName)) { + return; + } + final int packageToken = packagesToTokensMap.get(packageName).get(packageName); + packagesToTokensMap.remove(packageName); + tokensToPackagesMap.delete(packageToken); + } } diff --git a/services/usage/java/com/android/server/usage/UsageStatsDatabase.java b/services/usage/java/com/android/server/usage/UsageStatsDatabase.java index 5c785f76d4db..db7ed1f58c1a 100644 --- a/services/usage/java/com/android/server/usage/UsageStatsDatabase.java +++ b/services/usage/java/com/android/server/usage/UsageStatsDatabase.java @@ -17,6 +17,7 @@ package com.android.server.usage; import android.app.usage.TimeSparseArray; +import android.app.usage.UsageEvents; import android.app.usage.UsageStats; import android.app.usage.UsageStatsManager; import android.os.Build; @@ -125,7 +126,7 @@ public class UsageStatsDatabase { // The obfuscated packages to tokens mappings file private final File mPackageMappingsFile; // Holds all of the data related to the obfuscated packages and their token mappings. - private final PackagesTokenData mPackagesTokenData = new PackagesTokenData(); + final PackagesTokenData mPackagesTokenData = new PackagesTokenData(); /** * UsageStatsDatabase constructor that allows setting the version number. @@ -159,8 +160,6 @@ public class UsageStatsDatabase { */ public void init(long currentTimeMillis) { synchronized (mLock) { - readMappingsLocked(); - for (File f : mIntervalDirs) { f.mkdirs(); if (!f.exists()) { @@ -538,6 +537,12 @@ public class UsageStatsDatabase { } } + void onPackageRemoved(String packageName, long timeRemoved) { + synchronized (mLock) { + mPackagesTokenData.removePackage(packageName, timeRemoved); + } + } + public void onTimeChanged(long timeDiffMillis) { synchronized (mLock) { StringBuilder logBuilder = new StringBuilder(); @@ -612,6 +617,37 @@ public class UsageStatsDatabase { } /** + * Filter out those stats from the given stats that belong to removed packages. Filtering out + * all of the stats at once has an amortized cost for future calls. + */ + void filterStats(IntervalStats stats) { + if (mPackagesTokenData.removedPackagesMap.isEmpty()) { + return; + } + final ArrayMap<String, Long> removedPackagesMap = mPackagesTokenData.removedPackagesMap; + + // filter out package usage stats + final int removedPackagesSize = removedPackagesMap.size(); + for (int i = 0; i < removedPackagesSize; i++) { + final String removedPackage = removedPackagesMap.keyAt(i); + final UsageStats usageStats = stats.packageStats.get(removedPackage); + if (usageStats != null && usageStats.mEndTimeStamp < removedPackagesMap.valueAt(i)) { + stats.packageStats.remove(removedPackage); + } + } + + // filter out events + final int eventsSize = stats.events.size(); + for (int i = stats.events.size() - 1; i >= 0; i--) { + final UsageEvents.Event event = stats.events.get(i); + final Long timeRemoved = removedPackagesMap.get(event.mPackage); + if (timeRemoved != null && timeRemoved > event.mTimeStamp) { + stats.events.remove(i); + } + } + } + + /** * Figures out what to extract from the given IntervalStats object. */ public interface StatCombiner<T> { @@ -954,7 +990,7 @@ public class UsageStatsDatabase { * Reads the obfuscated data file from disk containing the tokens to packages mappings and * rebuilds the packages to tokens mappings based on that data. */ - private void readMappingsLocked() { + public void readMappingsLocked() { if (!mPackageMappingsFile.exists()) { return; // package mappings file is missing - recreate mappings on next write. } diff --git a/services/usage/java/com/android/server/usage/UsageStatsService.java b/services/usage/java/com/android/server/usage/UsageStatsService.java index b12d9008689a..f007bd379713 100644 --- a/services/usage/java/com/android/server/usage/UsageStatsService.java +++ b/services/usage/java/com/android/server/usage/UsageStatsService.java @@ -141,6 +141,7 @@ public class UsageStatsService extends SystemService implements static final int MSG_UID_STATE_CHANGED = 3; static final int MSG_REPORT_EVENT_TO_ALL_USERID = 4; static final int MSG_UNLOCKED_USER = 5; + static final int MSG_PACKAGE_REMOVED = 6; private final Object mLock = new Object(); Handler mHandler; @@ -148,7 +149,6 @@ public class UsageStatsService extends SystemService implements UserManager mUserManager; PackageManager mPackageManager; PackageManagerInternal mPackageManagerInternal; - PackageMonitor mPackageMonitor; IDeviceIdleController mDeviceIdleController; // Do not use directly. Call getDpmInternal() instead DevicePolicyManagerInternal mDpmInternal; @@ -164,6 +164,8 @@ public class UsageStatsService extends SystemService implements /** Manages app time limit observers */ AppTimeLimitController mAppTimeLimit; + private final PackageMonitor mPackageMonitor = new MyPackageMonitor(); + // A map maintaining a queue of events to be reported per user. private final SparseArray<LinkedList<Event>> mReportedEvents = new SparseArray<>(); final SparseArray<ArraySet<String>> mUsageReporters = new SparseArray(); @@ -246,6 +248,8 @@ public class UsageStatsService extends SystemService implements mAppStandby.addListener(mStandbyChangeListener); + mPackageMonitor.register(getContext(), null, UserHandle.ALL, true); + IntentFilter filter = new IntentFilter(Intent.ACTION_USER_REMOVED); filter.addAction(Intent.ACTION_USER_STARTED); getContext().registerReceiverAsUser(new UserActionsReceiver(), UserHandle.ALL, filter, @@ -846,6 +850,26 @@ public class UsageStatsService extends SystemService implements } /** + * Called by the Handler for message MSG_PACKAGE_REMOVED. + */ + private void onPackageRemoved(int userId, String packageName) { + synchronized (mLock) { + final long timeRemoved = System.currentTimeMillis(); + if (!mUserUnlockedStates.get(userId, false)) { + // If user is not unlocked and a package is removed for them, we will handle it + // when the user service is initialized and package manager is queried. + return; + } + final UserUsageStatsService userService = mUserState.get(userId); + if (userService == null) { + return; + } + + userService.onPackageRemoved(packageName, timeRemoved); + } + } + + /** * Called by the Binder stub. */ List<UsageStats> queryUsageStats(int userId, int bucketType, long beginTime, long endTime, @@ -1162,7 +1186,9 @@ public class UsageStatsService extends SystemService implements case MSG_REMOVE_USER: onUserRemoved(msg.arg1); break; - + case MSG_PACKAGE_REMOVED: + onPackageRemoved(msg.arg1, (String) msg.obj); + break; case MSG_UID_STATE_CHANGED: { final int uid = msg.arg1; final int procState = msg.arg2; @@ -2112,4 +2138,13 @@ public class UsageStatsService extends SystemService implements return mAppTimeLimit.getAppUsageLimit(packageName, user); } } + + private class MyPackageMonitor extends PackageMonitor { + @Override + public void onPackageRemoved(String packageName, int uid) { + mHandler.obtainMessage(MSG_PACKAGE_REMOVED, getChangingUserId(), 0, packageName) + .sendToTarget(); + super.onPackageRemoved(packageName, uid); + } + } } diff --git a/services/usage/java/com/android/server/usage/UserUsageStatsService.java b/services/usage/java/com/android/server/usage/UserUsageStatsService.java index ec6caded01a7..23df1c553de2 100644 --- a/services/usage/java/com/android/server/usage/UserUsageStatsService.java +++ b/services/usage/java/com/android/server/usage/UserUsageStatsService.java @@ -34,7 +34,10 @@ import android.app.usage.UsageEvents.Event; import android.app.usage.UsageStats; import android.app.usage.UsageStatsManager; import android.content.Context; +import android.content.pm.ApplicationInfo; +import android.content.pm.PackageManagerInternal; import android.content.res.Configuration; +import android.os.Process; import android.os.SystemClock; import android.text.format.DateUtils; import android.util.ArrayMap; @@ -44,6 +47,7 @@ import android.util.Slog; import android.util.SparseIntArray; import com.android.internal.util.IndentingPrintWriter; +import com.android.server.LocalServices; import com.android.server.usage.UsageStatsDatabase.StatCombiner; import java.io.File; @@ -51,6 +55,7 @@ import java.io.IOException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; /** @@ -108,6 +113,7 @@ class UserUsageStatsService { } void init(final long currentTimeMillis) { + readPackageMappingsLocked(); mDatabase.init(currentTimeMillis); int nullCount = 0; @@ -169,6 +175,54 @@ class UserUsageStatsService { persistActiveStats(); } + void onPackageRemoved(String packageName, long timeRemoved) { + mDatabase.onPackageRemoved(packageName, timeRemoved); + } + + private void readPackageMappingsLocked() { + mDatabase.readMappingsLocked(); + cleanUpPackageMappingsLocked(); + } + + /** + * Queries Package Manager for a list of installed packages and removes those packages from + * mPackagesTokenData which are not installed any more. + * This will only happen once per device boot, when the user is unlocked for the first time. + */ + private void cleanUpPackageMappingsLocked() { + final long timeNow = System.currentTimeMillis(); + /* + Note (b/142501248): PackageManagerInternal#getInstalledApplications is not lightweight. + Once its implementation is updated, or it's replaced with a better alternative, update + the call here to use it. For now, using the heavy #getInstalledApplications is okay since + this clean-up is only performed once every boot. + */ + final PackageManagerInternal packageManagerInternal = + LocalServices.getService(PackageManagerInternal.class); + if (packageManagerInternal == null) { + return; + } + final List<ApplicationInfo> installedPackages = + packageManagerInternal.getInstalledApplications(0, mUserId, Process.SYSTEM_UID); + // convert the package list to a set for easy look-ups + final HashSet<String> packagesSet = new HashSet<>(installedPackages.size()); + for (int i = installedPackages.size() - 1; i >= 0; i--) { + packagesSet.add(installedPackages.get(i).packageName); + } + final List<String> removedPackages = new ArrayList<>(); + // populate list of packages that are found in the mappings but not in the installed list + for (int i = mDatabase.mPackagesTokenData.packagesToTokensMap.size() - 1; i >= 0; i--) { + if (!packagesSet.contains(mDatabase.mPackagesTokenData.packagesToTokensMap.keyAt(i))) { + removedPackages.add(mDatabase.mPackagesTokenData.packagesToTokensMap.keyAt(i)); + } + } + + // remove packages in the mappings that are no longer installed + for (int i = removedPackages.size() - 1; i >= 0; i--) { + mDatabase.mPackagesTokenData.removePackage(removedPackages.get(i), timeNow); + } + } + private void onTimeChanged(long oldTime, long newTime) { persistActiveStats(); mDatabase.onTimeChanged(newTime - oldTime); @@ -400,6 +454,7 @@ class UserUsageStatsService { if (results == null) { results = new ArrayList<>(); } + mDatabase.filterStats(currentStats); combiner.combine(currentStats, true, results); } diff --git a/tests/UsageStatsPerfTests/src/com/android/frameworks/perftests/usage/tests/UsageStatsDatabasePerfTest.java b/tests/UsageStatsPerfTests/src/com/android/frameworks/perftests/usage/tests/UsageStatsDatabasePerfTest.java index 62aef876a2eb..7e8a13470c35 100644 --- a/tests/UsageStatsPerfTests/src/com/android/frameworks/perftests/usage/tests/UsageStatsDatabasePerfTest.java +++ b/tests/UsageStatsPerfTests/src/com/android/frameworks/perftests/usage/tests/UsageStatsDatabasePerfTest.java @@ -80,6 +80,7 @@ public class UsageStatsDatabasePerfTest { sContext = InstrumentationRegistry.getTargetContext(); mTestDir = new File(sContext.getFilesDir(), "UsageStatsDatabasePerfTest"); sUsageStatsDatabase = new UsageStatsDatabase(mTestDir); + sUsageStatsDatabase.readMappingsLocked(); sUsageStatsDatabase.init(1); } |