diff options
Diffstat (limited to 'cmds')
201 files changed, 12821 insertions, 2508 deletions
diff --git a/cmds/am/src/com/android/commands/am/Am.java b/cmds/am/src/com/android/commands/am/Am.java index c04e61b77274..41d546f6d603 100644 --- a/cmds/am/src/com/android/commands/am/Am.java +++ b/cmds/am/src/com/android/commands/am/Am.java @@ -174,6 +174,8 @@ public class Am extends BaseCommand { instrument.noWindowAnimation = true; } else if (opt.equals("--no-hidden-api-checks")) { instrument.disableHiddenApiChecks = true; + } else if (opt.equals("--no-isolated-storage")) { + instrument.disableIsolatedStorage = true; } else if (opt.equals("--user")) { instrument.userId = parseUserArg(nextArgRequired()); } else if (opt.equals("--abi")) { diff --git a/cmds/am/src/com/android/commands/am/Instrument.java b/cmds/am/src/com/android/commands/am/Instrument.java index 0dade0b2ba81..70baa8702ba9 100644 --- a/cmds/am/src/com/android/commands/am/Instrument.java +++ b/cmds/am/src/com/android/commands/am/Instrument.java @@ -16,6 +16,9 @@ package com.android.commands.am; +import static android.app.ActivityManager.INSTR_FLAG_DISABLE_HIDDEN_API_CHECKS; +import static android.app.ActivityManager.INSTR_FLAG_MOUNT_EXTERNAL_STORAGE_FULL; + import android.app.IActivityManager; import android.app.IInstrumentationWatcher; import android.app.Instrumentation; @@ -74,16 +77,13 @@ public class Instrument { String logPath = null; public boolean noWindowAnimation = false; public boolean disableHiddenApiChecks = false; + public boolean disableIsolatedStorage = false; public String abi = null; public int userId = UserHandle.USER_CURRENT; public Bundle args = new Bundle(); // Required public String componentNameArg; - // Disable hidden API checks for the newly started instrumentation. - // Must be kept in sync with ActivityManagerService. - private static final int INSTRUMENTATION_FLAG_DISABLE_HIDDEN_API_CHECKS = 1 << 0; - /** * Construct the instrument command runner. */ @@ -480,7 +480,13 @@ public class Instrument { } // Start the instrumentation - int flags = disableHiddenApiChecks ? INSTRUMENTATION_FLAG_DISABLE_HIDDEN_API_CHECKS : 0; + int flags = 0; + if (disableHiddenApiChecks) { + flags |= INSTR_FLAG_DISABLE_HIDDEN_API_CHECKS; + } + if (disableIsolatedStorage) { + flags |= INSTR_FLAG_MOUNT_EXTERNAL_STORAGE_FULL; + } if (!mAm.startInstrumentation(cn, profileFile, flags, args, watcher, connection, userId, abi)) { throw new AndroidException("INSTRUMENTATION_FAILED: " + cn.flattenToString()); diff --git a/cmds/bmgr/src/com/android/commands/bmgr/Bmgr.java b/cmds/bmgr/src/com/android/commands/bmgr/Bmgr.java index 84a04e5ad6e3..a826ec7c717e 100644 --- a/cmds/bmgr/src/com/android/commands/bmgr/Bmgr.java +++ b/cmds/bmgr/src/com/android/commands/bmgr/Bmgr.java @@ -16,10 +16,13 @@ package com.android.commands.bmgr; +import android.annotation.IntDef; import android.app.backup.BackupManager; +import android.app.backup.BackupManagerMonitor; import android.app.backup.BackupProgress; import android.app.backup.BackupTransport; import android.app.backup.IBackupManager; +import android.app.backup.IBackupManagerMonitor; import android.app.backup.IBackupObserver; import android.app.backup.IRestoreObserver; import android.app.backup.IRestoreSession; @@ -28,6 +31,7 @@ import android.app.backup.RestoreSet; import android.content.ComponentName; import android.content.pm.IPackageManager; import android.content.pm.PackageInfo; +import android.os.Bundle; import android.os.RemoteException; import android.os.ServiceManager; import android.os.SystemClock; @@ -36,10 +40,13 @@ import android.util.ArraySet; import com.android.internal.annotations.GuardedBy; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.CountDownLatch; public final class Bmgr { @@ -71,9 +78,7 @@ public final class Bmgr { return; } - mBmgr = IBackupManager.Stub.asInterface(ServiceManager.getService("backup")); - if (mBmgr == null) { - System.err.println(BMGR_NOT_RUNNING_ERR); + if (!isBmgrActive()) { return; } @@ -150,6 +155,27 @@ public final class Bmgr { showUsage(); } + private boolean isBmgrActive() { + mBmgr = IBackupManager.Stub.asInterface(ServiceManager.getService("backup")); + if (mBmgr == null) { + System.err.println(BMGR_NOT_RUNNING_ERR); + return false; + } + + try { + if (!mBmgr.isBackupServiceActive(UserHandle.USER_SYSTEM)) { + System.err.println(BMGR_NOT_RUNNING_ERR); + return false; + } + } catch (RemoteException e) { + System.err.println(e.toString()); + System.err.println(BMGR_NOT_RUNNING_ERR); + return false; + } + + return true; + } + private String enableToString(boolean enabled) { return enabled ? "enabled" : "disabled"; } @@ -228,7 +254,7 @@ public final class Bmgr { } // IBackupObserver generically usable for any backup/init operation - abstract class Observer extends IBackupObserver.Stub { + private static abstract class Observer extends IBackupObserver.Stub { private final Object trigger = new Object(); @GuardedBy("trigger") @@ -276,7 +302,7 @@ public final class Bmgr { } } - class BackupObserver extends Observer { + private static class BackupObserver extends Observer { @Override public void onUpdate(String currentPackage, BackupProgress backupProgress) { super.onUpdate(currentPackage, backupProgress); @@ -328,7 +354,7 @@ public final class Bmgr { } } - private void backupNowAllPackages(boolean nonIncrementalBackup) { + private void backupNowAllPackages(boolean nonIncrementalBackup, @Monitor int monitorState) { int userId = UserHandle.USER_SYSTEM; IPackageManager mPm = IPackageManager.Stub.asInterface(ServiceManager.getService("package")); @@ -353,20 +379,27 @@ public final class Bmgr { System.err.println(e.toString()); System.err.println(BMGR_NOT_RUNNING_ERR); } - backupNowPackages(Arrays.asList(filteredPackages), nonIncrementalBackup); + backupNowPackages(Arrays.asList(filteredPackages), nonIncrementalBackup, monitorState); } } - private void backupNowPackages(List<String> packages, boolean nonIncrementalBackup) { + private void backupNowPackages( + List<String> packages, boolean nonIncrementalBackup, @Monitor int monitorState) { int flags = 0; if (nonIncrementalBackup) { flags |= BackupManager.FLAG_NON_INCREMENTAL_BACKUP; } try { BackupObserver observer = new BackupObserver(); - // TODO: implement monitor here? - int err = mBmgr.requestBackup(packages.toArray(new String[packages.size()]), observer, - null, flags); + BackupMonitor monitor = + (monitorState != Monitor.OFF) + ? new BackupMonitor(monitorState == Monitor.VERBOSE) + : null; + int err = mBmgr.requestBackup( + packages.toArray(new String[packages.size()]), + observer, + monitor, + flags); if (err == 0) { // Off and running -- wait for the backup to complete observer.waitForCompletion(); @@ -383,6 +416,7 @@ public final class Bmgr { String pkg; boolean backupAll = false; boolean nonIncrementalBackup = false; + @Monitor int monitor = Monitor.OFF; ArrayList<String> allPkgs = new ArrayList<String>(); while ((pkg = nextArg()) != null) { if (pkg.equals("--all")) { @@ -391,6 +425,10 @@ public final class Bmgr { nonIncrementalBackup = true; } else if (pkg.equals("--incremental")) { nonIncrementalBackup = false; + } else if (pkg.equals("--monitor")) { + monitor = Monitor.NORMAL; + } else if (pkg.equals("--monitor-verbose")) { + monitor = Monitor.VERBOSE; } else { if (!allPkgs.contains(pkg)) { allPkgs.add(pkg); @@ -401,14 +439,14 @@ public final class Bmgr { if (allPkgs.size() == 0) { System.out.println("Running " + (nonIncrementalBackup ? "non-" : "") + "incremental backup for all packages."); - backupNowAllPackages(nonIncrementalBackup); + backupNowAllPackages(nonIncrementalBackup, monitor); } else { System.err.println("Provide only '--all' flag or list of packages."); } } else if (allPkgs.size() > 0) { System.out.println("Running " + (nonIncrementalBackup ? "non-" : "") + "incremental backup for " + allPkgs.size() +" requested packages."); - backupNowPackages(allPkgs, nonIncrementalBackup); + backupNowPackages(allPkgs, nonIncrementalBackup, monitor); } else { System.err.println("Provide '--all' flag or list of packages."); } @@ -704,34 +742,11 @@ public final class Bmgr { return; } } - - System.out.println("done"); } private void doRestorePackage(String pkg) { - try { - mRestore = mBmgr.beginRestoreSession(pkg, null); - if (mRestore == null) { - System.err.println(BMGR_NOT_RUNNING_ERR); - return; - } - - RestoreObserver observer = new RestoreObserver(); - // TODO implement monitor here - int err = mRestore.restorePackage(pkg, observer, null ); - if (err == 0) { - // Off and running -- wait for the restore to complete - observer.waitForCompletion(); - } else { - System.err.println("Unable to restore package " + pkg); - } - - // And finally shut down the session - mRestore.endRestoreSession(); - } catch (RemoteException e) { - System.err.println(e.toString()); - System.err.println(BMGR_NOT_RUNNING_ERR); - } + System.err.println("The syntax 'restore <package>' is no longer supported, please use "); + System.err.println("'restore <token> <package>'."); } private void doRestoreAll(long token, HashSet<String> filter) { @@ -784,6 +799,8 @@ public final class Bmgr { // once the restore has finished, close down the session and we're done mRestore.endRestoreSession(); + + System.out.println("done"); } catch (RemoteException e) { System.err.println(e.toString()); System.err.println(BMGR_NOT_RUNNING_ERR); @@ -823,12 +840,12 @@ public final class Bmgr { System.err.println(" bmgr transport WHICH|-c WHICH_COMPONENT"); System.err.println(" bmgr restore TOKEN"); System.err.println(" bmgr restore TOKEN PACKAGE..."); - System.err.println(" bmgr restore PACKAGE"); System.err.println(" bmgr run"); System.err.println(" bmgr wipe TRANSPORT PACKAGE"); System.err.println(" bmgr fullbackup PACKAGE..."); - System.err.println(" bmgr backupnow --all|PACKAGE..."); + System.err.println(" bmgr backupnow [--monitor|--monitor-verbose] --all|PACKAGE..."); System.err.println(" bmgr cancel backups"); + System.err.println(" bmgr init TRANSPORT..."); System.err.println(""); System.err.println("The 'backup' command schedules a backup pass for the named package."); System.err.println("Note that the backup pass will effectively be a no-op if the package"); @@ -867,10 +884,6 @@ public final class Bmgr { System.err.println("'restore' operation supplying only a token, but applies a filter to the"); System.err.println("set of applications to be restored."); System.err.println(""); - System.err.println("The 'restore' command when given just a package name intiates a restore of"); - System.err.println("just that one package according to the restore set selection algorithm"); - System.err.println("used by the RestoreSession.restorePackage() method."); - System.err.println(""); System.err.println("The 'run' command causes any scheduled backup operation to be initiated"); System.err.println("immediately, without the usual waiting period for batching together"); System.err.println("data changes."); @@ -885,9 +898,168 @@ public final class Bmgr { System.err.println(""); System.err.println("The 'backupnow' command runs an immediate backup for one or more packages."); System.err.println(" --all flag runs backup for all eligible packages."); + System.err.println(" --monitor flag prints monitor events."); + System.err.println(" --monitor-verbose flag prints monitor events with all keys."); System.err.println("For each package it will run key/value or full data backup "); System.err.println("depending on the package's manifest declarations."); System.err.println("The data is sent via the currently active transport."); + System.err.println(""); System.err.println("The 'cancel backups' command cancels all running backups."); + System.err.println(""); + System.err.println("The 'init' command initializes the given transports, wiping all data"); + System.err.println("from their backing data stores."); + } + + private static class BackupMonitor extends IBackupManagerMonitor.Stub { + private final boolean mVerbose; + + private BackupMonitor(boolean verbose) { + mVerbose = verbose; + } + + @Override + public void onEvent(Bundle event) throws RemoteException { + StringBuilder out = new StringBuilder(); + int id = event.getInt(BackupManagerMonitor.EXTRA_LOG_EVENT_ID); + int category = event.getInt(BackupManagerMonitor.EXTRA_LOG_EVENT_CATEGORY); + out.append("=> Event{").append(eventCategoryToString(category)); + out.append(" / ").append(eventIdToString(id)); + String packageName = event.getString(BackupManagerMonitor.EXTRA_LOG_EVENT_PACKAGE_NAME); + if (packageName != null) { + out.append(" : package = ").append(packageName); + if (event.containsKey(BackupManagerMonitor.EXTRA_LOG_EVENT_PACKAGE_LONG_VERSION)) { + long version = + event.getLong( + BackupManagerMonitor.EXTRA_LOG_EVENT_PACKAGE_LONG_VERSION); + out.append("(v").append(version).append(")"); + } + } + if (mVerbose) { + Set<String> remainingKeys = new ArraySet<>(event.keySet()); + remainingKeys.remove(BackupManagerMonitor.EXTRA_LOG_EVENT_ID); + remainingKeys.remove(BackupManagerMonitor.EXTRA_LOG_EVENT_CATEGORY); + remainingKeys.remove(BackupManagerMonitor.EXTRA_LOG_EVENT_PACKAGE_NAME); + remainingKeys.remove(BackupManagerMonitor.EXTRA_LOG_EVENT_PACKAGE_LONG_VERSION); + remainingKeys.remove(BackupManagerMonitor.EXTRA_LOG_EVENT_PACKAGE_VERSION); + if (!remainingKeys.isEmpty()) { + out.append(", other keys ="); + for (String key : remainingKeys) { + out.append(" ").append(key); + } + } + } + out.append("}"); + System.out.println(out.toString()); + } + } + + private static String eventCategoryToString(int eventCategory) { + switch (eventCategory) { + case BackupManagerMonitor.LOG_EVENT_CATEGORY_TRANSPORT: + return "TRANSPORT"; + case BackupManagerMonitor.LOG_EVENT_CATEGORY_AGENT: + return "AGENT"; + case BackupManagerMonitor.LOG_EVENT_CATEGORY_BACKUP_MANAGER_POLICY: + return "BACKUP_MANAGER_POLICY"; + default: + return "UNKNOWN_CATEGORY"; + } + } + + private static String eventIdToString(int eventId) { + switch (eventId) { + case BackupManagerMonitor.LOG_EVENT_ID_FULL_BACKUP_CANCEL: + return "FULL_BACKUP_CANCEL"; + case BackupManagerMonitor.LOG_EVENT_ID_ILLEGAL_KEY: + return "ILLEGAL_KEY"; + case BackupManagerMonitor.LOG_EVENT_ID_NO_DATA_TO_SEND: + return "NO_DATA_TO_SEND"; + case BackupManagerMonitor.LOG_EVENT_ID_PACKAGE_INELIGIBLE: + return "PACKAGE_INELIGIBLE"; + case BackupManagerMonitor.LOG_EVENT_ID_PACKAGE_KEY_VALUE_PARTICIPANT: + return "PACKAGE_KEY_VALUE_PARTICIPANT"; + case BackupManagerMonitor.LOG_EVENT_ID_PACKAGE_STOPPED: + return "PACKAGE_STOPPED"; + case BackupManagerMonitor.LOG_EVENT_ID_PACKAGE_NOT_FOUND: + return "PACKAGE_NOT_FOUND"; + case BackupManagerMonitor.LOG_EVENT_ID_BACKUP_DISABLED: + return "BACKUP_DISABLED"; + case BackupManagerMonitor.LOG_EVENT_ID_DEVICE_NOT_PROVISIONED: + return "DEVICE_NOT_PROVISIONED"; + case BackupManagerMonitor.LOG_EVENT_ID_PACKAGE_TRANSPORT_NOT_PRESENT: + return "PACKAGE_TRANSPORT_NOT_PRESENT"; + case BackupManagerMonitor.LOG_EVENT_ID_ERROR_PREFLIGHT: + return "ERROR_PREFLIGHT"; + case BackupManagerMonitor.LOG_EVENT_ID_QUOTA_HIT_PREFLIGHT: + return "QUOTA_HIT_PREFLIGHT"; + case BackupManagerMonitor.LOG_EVENT_ID_EXCEPTION_FULL_BACKUP: + return "EXCEPTION_FULL_BACKUP"; + case BackupManagerMonitor.LOG_EVENT_ID_KEY_VALUE_BACKUP_CANCEL: + return "KEY_VALUE_BACKUP_CANCEL"; + case BackupManagerMonitor.LOG_EVENT_ID_NO_RESTORE_METADATA_AVAILABLE: + return "NO_RESTORE_METADATA_AVAILABLE"; + case BackupManagerMonitor.LOG_EVENT_ID_NO_PM_METADATA_RECEIVED: + return "NO_PM_METADATA_RECEIVED"; + case BackupManagerMonitor.LOG_EVENT_ID_PM_AGENT_HAS_NO_METADATA: + return "PM_AGENT_HAS_NO_METADATA"; + case BackupManagerMonitor.LOG_EVENT_ID_LOST_TRANSPORT: + return "LOST_TRANSPORT"; + case BackupManagerMonitor.LOG_EVENT_ID_PACKAGE_NOT_PRESENT: + return "PACKAGE_NOT_PRESENT"; + case BackupManagerMonitor.LOG_EVENT_ID_RESTORE_VERSION_HIGHER: + return "RESTORE_VERSION_HIGHER"; + case BackupManagerMonitor.LOG_EVENT_ID_APP_HAS_NO_AGENT: + return "APP_HAS_NO_AGENT"; + case BackupManagerMonitor.LOG_EVENT_ID_SIGNATURE_MISMATCH: + return "SIGNATURE_MISMATCH"; + case BackupManagerMonitor.LOG_EVENT_ID_CANT_FIND_AGENT: + return "CANT_FIND_AGENT"; + case BackupManagerMonitor.LOG_EVENT_ID_KEY_VALUE_RESTORE_TIMEOUT: + return "KEY_VALUE_RESTORE_TIMEOUT"; + case BackupManagerMonitor.LOG_EVENT_ID_RESTORE_ANY_VERSION: + return "RESTORE_ANY_VERSION"; + case BackupManagerMonitor.LOG_EVENT_ID_VERSIONS_MATCH: + return "VERSIONS_MATCH"; + case BackupManagerMonitor.LOG_EVENT_ID_VERSION_OF_BACKUP_OLDER: + return "VERSION_OF_BACKUP_OLDER"; + case BackupManagerMonitor.LOG_EVENT_ID_FULL_RESTORE_SIGNATURE_MISMATCH: + return "FULL_RESTORE_SIGNATURE_MISMATCH"; + case BackupManagerMonitor.LOG_EVENT_ID_SYSTEM_APP_NO_AGENT: + return "SYSTEM_APP_NO_AGENT"; + case BackupManagerMonitor.LOG_EVENT_ID_FULL_RESTORE_ALLOW_BACKUP_FALSE: + return "FULL_RESTORE_ALLOW_BACKUP_FALSE"; + case BackupManagerMonitor.LOG_EVENT_ID_APK_NOT_INSTALLED: + return "APK_NOT_INSTALLED"; + case BackupManagerMonitor.LOG_EVENT_ID_CANNOT_RESTORE_WITHOUT_APK: + return "CANNOT_RESTORE_WITHOUT_APK"; + case BackupManagerMonitor.LOG_EVENT_ID_MISSING_SIGNATURE: + return "MISSING_SIGNATURE"; + case BackupManagerMonitor.LOG_EVENT_ID_EXPECTED_DIFFERENT_PACKAGE: + return "EXPECTED_DIFFERENT_PACKAGE"; + case BackupManagerMonitor.LOG_EVENT_ID_UNKNOWN_VERSION: + return "UNKNOWN_VERSION"; + case BackupManagerMonitor.LOG_EVENT_ID_FULL_RESTORE_TIMEOUT: + return "FULL_RESTORE_TIMEOUT"; + case BackupManagerMonitor.LOG_EVENT_ID_CORRUPT_MANIFEST: + return "CORRUPT_MANIFEST"; + case BackupManagerMonitor.LOG_EVENT_ID_WIDGET_METADATA_MISMATCH: + return "WIDGET_METADATA_MISMATCH"; + case BackupManagerMonitor.LOG_EVENT_ID_WIDGET_UNKNOWN_VERSION: + return "WIDGET_UNKNOWN_VERSION"; + case BackupManagerMonitor.LOG_EVENT_ID_NO_PACKAGES: + return "NO_PACKAGES"; + case BackupManagerMonitor.LOG_EVENT_ID_TRANSPORT_IS_NULL: + return "TRANSPORT_IS_NULL"; + default: + return "UNKNOWN_ID"; + } + } + + @IntDef({Monitor.OFF, Monitor.NORMAL, Monitor.VERBOSE}) + @Retention(RetentionPolicy.SOURCE) + private @interface Monitor { + int OFF = 0; + int NORMAL = 1; + int VERBOSE = 2; } } diff --git a/cmds/bootanimation/Android.mk b/cmds/bootanimation/Android.mk index e5d35b3b8a0e..6943dab0acbe 100644 --- a/cmds/bootanimation/Android.mk +++ b/cmds/bootanimation/Android.mk @@ -27,7 +27,12 @@ ifeq ($(PRODUCT_IOT),true) LOCAL_SHARED_LIBRARIES += \ libandroidthings \ + libandroidthings_protos \ libchrome \ + libprotobuf-cpp-lite \ + +LOCAL_STATIC_LIBRARIES += \ + libjsoncpp LOCAL_SRC_FILES += \ iot/iotbootanimation_main.cpp \ @@ -94,3 +99,5 @@ LOCAL_32_BIT_ONLY := true endif include ${BUILD_SHARED_LIBRARY} + +include $(call all-makefiles-under,$(LOCAL_PATH)) diff --git a/cmds/bootanimation/BootAnimation.cpp b/cmds/bootanimation/BootAnimation.cpp index ed6c25dc49c3..5dcb392b002d 100644 --- a/cmds/bootanimation/BootAnimation.cpp +++ b/cmds/bootanimation/BootAnimation.cpp @@ -114,7 +114,7 @@ BootAnimation::BootAnimation(sp<Callbacks> callbacks) void BootAnimation::onFirstRef() { status_t err = mSession->linkToComposerDeath(this); - ALOGE_IF(err, "linkToComposerDeath failed (%s) ", strerror(-err)); + SLOGE_IF(err, "linkToComposerDeath failed (%s) ", strerror(-err)); if (err == NO_ERROR) { run("BootAnimation", PRIORITY_DISPLAY); } @@ -128,7 +128,7 @@ sp<SurfaceComposerClient> BootAnimation::session() const { void BootAnimation::binderDied(const wp<IBinder>&) { // woah, surfaceflinger died! - ALOGD("SurfaceFlinger died, exiting..."); + SLOGD("SurfaceFlinger died, exiting..."); // calling requestExit() is not enough here because the Surface code // might be blocked on a condition variable that will never be updated. @@ -360,7 +360,7 @@ bool BootAnimation::threadLoop() bool BootAnimation::android() { - ALOGD("%sAnimationShownTiming start time: %" PRId64 "ms", mShuttingDown ? "Shutdown" : "Boot", + SLOGD("%sAnimationShownTiming start time: %" PRId64 "ms", mShuttingDown ? "Shutdown" : "Boot", elapsedRealtime()); initTexture(&mAndroid[0], mAssets, "images/android-logo-mask.png"); initTexture(&mAndroid[1], mAssets, "images/android-logo-shine.png"); @@ -508,14 +508,14 @@ static bool parseColor(const char str[7], float color[3]) { static bool readFile(ZipFileRO* zip, const char* name, String8& outString) { ZipEntryRO entry = zip->findEntryByName(name); - ALOGE_IF(!entry, "couldn't find %s", name); + SLOGE_IF(!entry, "couldn't find %s", name); if (!entry) { return false; } FileMap* entryMap = zip->createEntryFileMap(entry); zip->releaseEntry(entry); - ALOGE_IF(!entryMap, "entryMap is null"); + SLOGE_IF(!entryMap, "entryMap is null"); if (!entryMap) { return false; } @@ -616,7 +616,7 @@ void BootAnimation::drawClock(const Font& font, const int xPos, const int yPos) size_t length = strftime(timeBuff, TIME_LENGTH, timeFormat, timeInfo); if (length != TIME_LENGTH - 1) { - ALOGE("Couldn't format time; abandoning boot animation clock"); + SLOGE("Couldn't format time; abandoning boot animation clock"); mClockEnabled = false; return; } @@ -654,13 +654,13 @@ bool BootAnimation::parseAnimationDesc(Animation& animation) char pathType; if (sscanf(l, "%d %d %d", &width, &height, &fps) == 3) { - // ALOGD("> w=%d, h=%d, fps=%d", width, height, fps); + // SLOGD("> w=%d, h=%d, fps=%d", width, height, fps); animation.width = width; animation.height = height; animation.fps = fps; } else if (sscanf(l, " %c %d %d %s #%6s %16s %16s", &pathType, &count, &pause, path, color, clockPos1, clockPos2) >= 4) { - //ALOGD("> type=%c, count=%d, pause=%d, path=%s, color=%s, clockPos1=%s, clockPos2=%s", + //SLOGD("> type=%c, count=%d, pause=%d, path=%s, color=%s, clockPos1=%s, clockPos2=%s", // pathType, count, pause, path, color, clockPos1, clockPos2); Animation::Part part; part.playUntilComplete = pathType == 'c'; @@ -670,7 +670,7 @@ bool BootAnimation::parseAnimationDesc(Animation& animation) part.audioData = NULL; part.animation = NULL; if (!parseColor(color, part.backgroundColor)) { - ALOGE("> invalid color '#%s'", color); + SLOGE("> invalid color '#%s'", color); part.backgroundColor[0] = 0.0f; part.backgroundColor[1] = 0.0f; part.backgroundColor[2] = 0.0f; @@ -679,7 +679,7 @@ bool BootAnimation::parseAnimationDesc(Animation& animation) animation.parts.add(part); } else if (strcmp(l, "$SYSTEM") == 0) { - // ALOGD("> SYSTEM"); + // SLOGD("> SYSTEM"); Animation::Part part; part.playUntilComplete = false; part.count = 1; @@ -710,7 +710,7 @@ bool BootAnimation::preloadZip(Animation& animation) while ((entry = zip->nextEntry(cookie)) != NULL) { const int foundEntryName = zip->getEntryFileName(entry, name, ANIM_ENTRY_NAME_MAX); if (foundEntryName > ANIM_ENTRY_NAME_MAX || foundEntryName == -1) { - ALOGE("Error fetching entry file name"); + SLOGE("Error fetching entry file name"); continue; } @@ -754,7 +754,7 @@ bool BootAnimation::preloadZip(Animation& animation) } } } else { - ALOGE("bootanimation.zip is compressed; must be only stored"); + SLOGE("bootanimation.zip is compressed; must be only stored"); } } } @@ -782,7 +782,7 @@ bool BootAnimation::preloadZip(Animation& animation) frame.trimX = x; frame.trimY = y; } else { - ALOGE("Error parsing trim.txt, line: %s", lineStr); + SLOGE("Error parsing trim.txt, line: %s", lineStr); break; } } @@ -860,12 +860,12 @@ bool BootAnimation::movie() mTimeCheckThread = nullptr; } - releaseAnimation(animation); - if (clockFontInitialized) { glDeleteTextures(1, &animation->clockFont.texture.name); } + releaseAnimation(animation); + return false; } @@ -876,7 +876,7 @@ bool BootAnimation::playAnimation(const Animation& animation) const int animationX = (mWidth - animation.width) / 2; const int animationY = (mHeight - animation.height) / 2; - ALOGD("%sAnimationShownTiming start time: %" PRId64 "ms", mShuttingDown ? "Shutdown" : "Boot", + SLOGD("%sAnimationShownTiming start time: %" PRId64 "ms", mShuttingDown ? "Shutdown" : "Boot", elapsedRealtime()); for (size_t i=0 ; i<pcount ; i++) { const Animation::Part& part(animation.parts[i]); @@ -949,7 +949,7 @@ bool BootAnimation::playAnimation(const Animation& animation) nsecs_t now = systemTime(); nsecs_t delay = frameDuration - (now - lastFrame); - //ALOGD("%lld, %lld", ns2ms(now - lastFrame), ns2ms(delay)); + //SLOGD("%lld, %lld", ns2ms(now - lastFrame), ns2ms(delay)); lastFrame = now; if (delay > 0) { @@ -1048,13 +1048,13 @@ void BootAnimation::releaseAnimation(Animation* animation) const BootAnimation::Animation* BootAnimation::loadAnimation(const String8& fn) { if (mLoadedFiles.indexOf(fn) >= 0) { - ALOGE("File \"%s\" is already loaded. Cyclic ref is not allowed", + SLOGE("File \"%s\" is already loaded. Cyclic ref is not allowed", fn.string()); return NULL; } ZipFileRO *zip = ZipFileRO::open(fn); if (zip == NULL) { - ALOGE("Failed to open animation zip \"%s\": %s", + SLOGE("Failed to open animation zip \"%s\": %s", fn.string(), strerror(errno)); return NULL; } @@ -1143,7 +1143,7 @@ bool BootAnimation::TimeCheckThread::doThreadLoop() { if (pollResult == 0) { return true; } else if (pollResult < 0) { - ALOGE("Could not poll inotify events"); + SLOGE("Could not poll inotify events"); return false; } @@ -1152,7 +1152,7 @@ bool BootAnimation::TimeCheckThread::doThreadLoop() { if (length == 0) { return true; } else if (length < 0) { - ALOGE("Could not read inotify events"); + SLOGE("Could not read inotify events"); return false; } @@ -1183,7 +1183,7 @@ void BootAnimation::TimeCheckThread::addTimeDirWatch() { status_t BootAnimation::TimeCheckThread::readyToRun() { mInotifyFd = inotify_init(); if (mInotifyFd < 0) { - ALOGE("Could not initialize inotify fd"); + SLOGE("Could not initialize inotify fd"); return NO_INIT; } @@ -1191,7 +1191,7 @@ status_t BootAnimation::TimeCheckThread::readyToRun() { if (mSystemWd < 0) { close(mInotifyFd); mInotifyFd = -1; - ALOGE("Could not add watch for %s", SYSTEM_DATA_DIR_PATH); + SLOGE("Could not add watch for %s", SYSTEM_DATA_DIR_PATH); return NO_INIT; } diff --git a/cmds/bootanimation/BootAnimation.h b/cmds/bootanimation/BootAnimation.h index b4699d884681..498eebce5999 100644 --- a/cmds/bootanimation/BootAnimation.h +++ b/cmds/bootanimation/BootAnimation.h @@ -22,6 +22,7 @@ #include <androidfw/AssetManager.h> #include <utils/Thread.h> +#include <binder/IBinder.h> #include <EGL/egl.h> #include <GLES/gl.h> diff --git a/cmds/bootanimation/BootAnimationUtil.cpp b/cmds/bootanimation/BootAnimationUtil.cpp index 7718daf61d81..1e417e938359 100644 --- a/cmds/bootanimation/BootAnimationUtil.cpp +++ b/cmds/bootanimation/BootAnimationUtil.cpp @@ -16,14 +16,30 @@ #include "BootAnimationUtil.h" +#include <vector> #include <inttypes.h> #include <binder/IServiceManager.h> #include <cutils/properties.h> #include <utils/Log.h> #include <utils/SystemClock.h> +#include <android-base/properties.h> namespace android { +namespace { + +static constexpr char PLAY_SOUND_PROP_NAME[] = "persist.sys.bootanim.play_sound"; +static constexpr char BOOT_COMPLETED_PROP_NAME[] = "sys.boot_completed"; +static constexpr char POWER_CTL_PROP_NAME[] = "sys.powerctl"; +static constexpr char BOOTREASON_PROP_NAME[] = "ro.boot.bootreason"; +static const std::vector<std::string> PLAY_SOUND_BOOTREASON_BLACKLIST { + "kernel_panic", + "Panic", + "Watchdog", +}; + +} // namespace + bool bootAnimationDisabled() { char value[PROPERTY_VALUE_MAX]; @@ -58,4 +74,31 @@ void waitForSurfaceFlinger() { } } +bool playSoundsAllowed() { + // Only play sounds for system boots, not runtime restarts. + if (android::base::GetBoolProperty(BOOT_COMPLETED_PROP_NAME, false)) { + return false; + } + // no audio while shutting down + if (!android::base::GetProperty(POWER_CTL_PROP_NAME, "").empty()) { + return false; + } + // Read the system property to see if we should play the sound. + // If it's not present, default to allowed. + if (!property_get_bool(PLAY_SOUND_PROP_NAME, 1)) { + return false; + } + + // Don't play sounds if this is a reboot due to an error. + char bootreason[PROPERTY_VALUE_MAX]; + if (property_get(BOOTREASON_PROP_NAME, bootreason, nullptr) > 0) { + for (const auto& str : PLAY_SOUND_BOOTREASON_BLACKLIST) { + if (strcasecmp(str.c_str(), bootreason) == 0) { + return false; + } + } + } + return true; +} + } // namespace android diff --git a/cmds/bootanimation/BootAnimationUtil.h b/cmds/bootanimation/BootAnimationUtil.h index 60987cd1ccd1..1e1140a51763 100644 --- a/cmds/bootanimation/BootAnimationUtil.h +++ b/cmds/bootanimation/BootAnimationUtil.h @@ -14,6 +14,9 @@ * limitations under the License. */ +#ifndef ANDROID_BOOTANIMATION_UTIL_H +#define ANDROID_BOOTANIMATION_UTIL_H + namespace android { // Returns true if boot animation is disabled. @@ -22,4 +25,8 @@ bool bootAnimationDisabled(); // Waits until the surface flinger is up. void waitForSurfaceFlinger(); +// Returns whether sounds should be played during current boot. +bool playSoundsAllowed(); } // namespace android + +#endif // ANDROID_BOOTANIMATION_UTIL_H diff --git a/cmds/bootanimation/audioplay.cpp b/cmds/bootanimation/audioplay.cpp index c546072e733a..874aab08862e 100644 --- a/cmds/bootanimation/audioplay.cpp +++ b/cmds/bootanimation/audioplay.cpp @@ -17,22 +17,27 @@ // cribbed from samples/native-audio -#include "audioplay.h" - #define CHATTY ALOGD #define LOG_TAG "audioplay" +#include "audioplay.h" + #include <string.h> #include <utils/Log.h> +#include <utils/threads.h> // for native audio #include <SLES/OpenSLES.h> #include <SLES/OpenSLES_Android.h> +#include "BootAnimationUtil.h" + namespace audioplay { namespace { +using namespace android; + // engine interfaces static SLObjectItf engineObject = NULL; static SLEngineItf engineEngine; @@ -305,6 +310,74 @@ bool parseClipBuf(const uint8_t* clipBuf, int clipBufSize, const ChunkFormat** o return true; } +class InitAudioThread : public Thread { +public: + InitAudioThread(uint8_t* exampleAudioData, int exampleAudioLength) + : Thread(false), + mExampleAudioData(exampleAudioData), + mExampleAudioLength(exampleAudioLength) {} +private: + virtual bool threadLoop() { + audioplay::create(mExampleAudioData, mExampleAudioLength); + // Exit immediately + return false; + } + + uint8_t* mExampleAudioData; + int mExampleAudioLength; +}; + +// Typedef to aid readability. +typedef android::BootAnimation::Animation Animation; + +class AudioAnimationCallbacks : public android::BootAnimation::Callbacks { +public: + void init(const Vector<Animation::Part>& parts) override { + const Animation::Part* partWithAudio = nullptr; + for (const Animation::Part& part : parts) { + if (part.audioData != nullptr) { + partWithAudio = ∂ + break; + } + } + + if (partWithAudio == nullptr) { + return; + } + + ALOGD("found audio.wav, creating playback engine"); + // The audioData is used to initialize the audio system. Different data + // can be played later for other parts BUT the assumption is that they + // will all be the same format and only the format of this audioData + // will work correctly. + initAudioThread = new InitAudioThread(partWithAudio->audioData, + partWithAudio->audioLength); + initAudioThread->run("BootAnimation::InitAudioThread", PRIORITY_NORMAL); + }; + + void playPart(int partNumber, const Animation::Part& part, int playNumber) override { + // only play audio file the first time we animate the part + if (playNumber == 0 && part.audioData && playSoundsAllowed()) { + ALOGD("playing clip for part%d, size=%d", + partNumber, part.audioLength); + // Block until the audio engine is finished initializing. + if (initAudioThread != nullptr) { + initAudioThread->join(); + } + audioplay::playClip(part.audioData, part.audioLength); + } + }; + + void shutdown() override { + // we've finally played everything we're going to play + audioplay::setPlaying(false); + audioplay::destroy(); + }; + +private: + sp<InitAudioThread> initAudioThread = nullptr; +}; + } // namespace bool create(const uint8_t* exampleClipBuf, int exampleClipBufSize) { @@ -397,4 +470,8 @@ void destroy() { } } +sp<BootAnimation::Callbacks> createAnimationCallbacks() { + return new AudioAnimationCallbacks(); +} + } // namespace audioplay diff --git a/cmds/bootanimation/audioplay.h b/cmds/bootanimation/audioplay.h index 0e5705af0ad0..4704a702d50b 100644 --- a/cmds/bootanimation/audioplay.h +++ b/cmds/bootanimation/audioplay.h @@ -20,6 +20,8 @@ #include <string.h> +#include "BootAnimation.h" + namespace audioplay { // Initializes the engine with an example of the type of WAV clip to play. @@ -32,6 +34,9 @@ bool playClip(const uint8_t* buf, int size); void setPlaying(bool isPlaying); void destroy(); +// Generates callbacks to integrate the audioplay system with the BootAnimation. +android::sp<android::BootAnimation::Callbacks> createAnimationCallbacks(); + } #endif // AUDIOPLAY_H_ diff --git a/cmds/bootanimation/bootanimation_main.cpp b/cmds/bootanimation/bootanimation_main.cpp index 8501982d071c..a52a5e92a840 100644 --- a/cmds/bootanimation/bootanimation_main.cpp +++ b/cmds/bootanimation/bootanimation_main.cpp @@ -26,8 +26,6 @@ #include <sys/resource.h> #include <utils/Log.h> #include <utils/SystemClock.h> -#include <utils/threads.h> -#include <android-base/properties.h> #include "BootAnimation.h" #include "BootAnimationUtil.h" @@ -35,113 +33,6 @@ using namespace android; -// --------------------------------------------------------------------------- - -namespace { - -// Create a typedef for readability. -typedef android::BootAnimation::Animation Animation; - -static const char PLAY_SOUND_PROP_NAME[] = "persist.sys.bootanim.play_sound"; -static const char BOOT_COMPLETED_PROP_NAME[] = "sys.boot_completed"; -static const char POWER_CTL_PROP_NAME[] = "sys.powerctl"; -static const char BOOTREASON_PROP_NAME[] = "ro.boot.bootreason"; -static const std::vector<std::string> PLAY_SOUND_BOOTREASON_BLACKLIST { - "kernel_panic", - "Panic", - "Watchdog", -}; - -class InitAudioThread : public Thread { -public: - InitAudioThread(uint8_t* exampleAudioData, int exampleAudioLength) - : Thread(false), - mExampleAudioData(exampleAudioData), - mExampleAudioLength(exampleAudioLength) {} -private: - virtual bool threadLoop() { - audioplay::create(mExampleAudioData, mExampleAudioLength); - // Exit immediately - return false; - } - - uint8_t* mExampleAudioData; - int mExampleAudioLength; -}; - -bool playSoundsAllowed() { - // Only play sounds for system boots, not runtime restarts. - if (android::base::GetBoolProperty(BOOT_COMPLETED_PROP_NAME, false)) { - return false; - } - // no audio while shutting down - if (!android::base::GetProperty(POWER_CTL_PROP_NAME, "").empty()) { - return false; - } - // Read the system property to see if we should play the sound. - // If it's not present, default to allowed. - if (!property_get_bool(PLAY_SOUND_PROP_NAME, 1)) { - return false; - } - - // Don't play sounds if this is a reboot due to an error. - char bootreason[PROPERTY_VALUE_MAX]; - if (property_get(BOOTREASON_PROP_NAME, bootreason, nullptr) > 0) { - for (const auto& str : PLAY_SOUND_BOOTREASON_BLACKLIST) { - if (strcasecmp(str.c_str(), bootreason) == 0) { - return false; - } - } - } - return true; -} - -class AudioAnimationCallbacks : public android::BootAnimation::Callbacks { -public: - void init(const Vector<Animation::Part>& parts) override { - const Animation::Part* partWithAudio = nullptr; - for (const Animation::Part& part : parts) { - if (part.audioData != nullptr) { - partWithAudio = ∂ - } - } - - if (partWithAudio == nullptr) { - return; - } - - ALOGD("found audio.wav, creating playback engine"); - initAudioThread = new InitAudioThread(partWithAudio->audioData, - partWithAudio->audioLength); - initAudioThread->run("BootAnimation::InitAudioThread", PRIORITY_NORMAL); - }; - - void playPart(int partNumber, const Animation::Part& part, int playNumber) override { - // only play audio file the first time we animate the part - if (playNumber == 0 && part.audioData && playSoundsAllowed()) { - ALOGD("playing clip for part%d, size=%d", - partNumber, part.audioLength); - // Block until the audio engine is finished initializing. - if (initAudioThread != nullptr) { - initAudioThread->join(); - } - audioplay::playClip(part.audioData, part.audioLength); - } - }; - - void shutdown() override { - // we've finally played everything we're going to play - audioplay::setPlaying(false); - audioplay::destroy(); - }; - -private: - sp<InitAudioThread> initAudioThread = nullptr; -}; - -} // namespace - - int main() { setpriority(PRIO_PROCESS, 0, ANDROID_PRIORITY_DISPLAY); @@ -156,7 +47,7 @@ int main() waitForSurfaceFlinger(); // create the boot animation object - sp<BootAnimation> boot = new BootAnimation(new AudioAnimationCallbacks()); + sp<BootAnimation> boot = new BootAnimation(audioplay::createAnimationCallbacks()); ALOGV("Boot animation set up. Joining pool."); IPCThreadState::self()->joinThreadPool(); diff --git a/cmds/bootanimation/iot/Android.mk b/cmds/bootanimation/iot/Android.mk new file mode 100644 index 000000000000..3d288e4e111b --- /dev/null +++ b/cmds/bootanimation/iot/Android.mk @@ -0,0 +1,43 @@ +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +LOCAL_PATH:= $(call my-dir) + +ifeq ($(PRODUCT_IOT),true) + +# libbootanimation_iot_test +# =========================================================== +include $(CLEAR_VARS) +LOCAL_MODULE := libbootanimation_iot_test +LOCAL_CFLAGS := -Wall -Werror -Wunused -Wunreachable-code + +LOCAL_SHARED_LIBRARIES := \ + libandroidthings \ + libandroidthings_protos \ + libbase \ + libchrome \ + liblog \ + libprotobuf-cpp-lite \ + +LOCAL_STATIC_LIBRARIES += \ + libjsoncpp + +LOCAL_SRC_FILES := \ + BootParameters.cpp \ + BootParameters_test.cpp \ + +include $(BUILD_NATIVE_TEST) + +endif # PRODUCT_IOT diff --git a/cmds/bootanimation/iot/BootAction.cpp b/cmds/bootanimation/iot/BootAction.cpp index fa797444d569..8b55147110bc 100644 --- a/cmds/bootanimation/iot/BootAction.cpp +++ b/cmds/bootanimation/iot/BootAction.cpp @@ -32,7 +32,7 @@ BootAction::~BootAction() { } bool BootAction::init(const std::string& libraryPath, - const std::vector<ABootActionParameter>& parameters) { + const std::unique_ptr<BootParameters>& bootParameters) { APeripheralManagerClient* client = nullptr; ALOGD("Connecting to peripheralmanager"); // Wait for peripheral manager to come up. @@ -77,9 +77,32 @@ bool BootAction::init(const std::string& libraryPath, mLibStartPart = reinterpret_cast<libStartPart>(loaded); } - ALOGD("Entering boot_action_init"); - bool result = mLibInit(parameters.data(), parameters.size()); - ALOGD("Returned from boot_action_init"); + // SilentBoot is considered optional, if it isn't exported by the library + // and the boot is silent, no method is called. + loaded = nullptr; + if (!loadSymbol("boot_action_silent_boot", &loaded) || loaded == nullptr) { + ALOGW("No boot_action_silent_boot found, boot action will not be " + "executed during a silent boot."); + } else { + mLibSilentBoot = reinterpret_cast<libInit>(loaded); + } + + bool result = true; + const auto& parameters = bootParameters->getParameters(); + if (bootParameters->isSilentBoot()) { + if (mLibSilentBoot != nullptr) { + ALOGD("Entering boot_action_silent_boot"); + result = mLibSilentBoot(parameters.data(), parameters.size()); + ALOGD("Returned from boot_action_silent_boot"); + } else { + ALOGW("Skipping missing boot_action_silent_boot"); + } + } else { + ALOGD("Entering boot_action_init"); + result = mLibInit(parameters.data(), parameters.size()); + ALOGD("Returned from boot_action_init"); + } + return result; } @@ -99,7 +122,7 @@ void BootAction::shutdown() { bool BootAction::loadSymbol(const char* symbol, void** loaded) { *loaded = dlsym(mLibHandle, symbol); - if (loaded == nullptr) { + if (*loaded == nullptr) { ALOGE("Unable to load symbol : %s :: %s", symbol, dlerror()); return false; } diff --git a/cmds/bootanimation/iot/BootAction.h b/cmds/bootanimation/iot/BootAction.h index 5e2495fe6c51..7119c35db0f9 100644 --- a/cmds/bootanimation/iot/BootAction.h +++ b/cmds/bootanimation/iot/BootAction.h @@ -20,6 +20,8 @@ #include <string> #include <vector> +#include "BootParameters.h" + #include <boot_action/boot_action.h> // libandroidthings native API. #include <utils/RefBase.h> @@ -31,7 +33,7 @@ public: // libraryPath is a fully qualified path to the target .so library. bool init(const std::string& libraryPath, - const std::vector<ABootActionParameter>& parameters); + const std::unique_ptr<BootParameters>& bootParameters); // The animation is going to start playing partNumber for the playCount'th // time, update the action as needed. @@ -45,7 +47,7 @@ public: private: typedef bool (*libInit)(const ABootActionParameter* parameters, - size_t num_parameters); + size_t numParameters); typedef void (*libStartPart)(int partNumber, int playNumber); typedef void (*libShutdown)(); @@ -55,6 +57,9 @@ private: libInit mLibInit = nullptr; libStartPart mLibStartPart = nullptr; libShutdown mLibShutdown = nullptr; + + // Called only if the boot is silent. + libInit mLibSilentBoot = nullptr; }; } // namespace android diff --git a/cmds/bootanimation/iot/BootParameters.cpp b/cmds/bootanimation/iot/BootParameters.cpp index da6ad0d1f08f..30a9b2895c44 100644 --- a/cmds/bootanimation/iot/BootParameters.cpp +++ b/cmds/bootanimation/iot/BootParameters.cpp @@ -18,45 +18,52 @@ #define LOG_TAG "BootParameters" +#include <errno.h> #include <fcntl.h> -#include <string> - #include <android-base/file.h> -#include <base/json/json_parser.h> -#include <base/json/json_reader.h> -#include <base/json/json_value_converter.h> +#include <json/json.h> #include <utils/Log.h> -using android::base::RemoveFileIfExists; using android::base::ReadFileToString; -using base::JSONReader; -using base::JSONValueConverter; -using base::Value; +using android::base::RemoveFileIfExists; +using android::base::WriteStringToFile; +using Json::ArrayIndex; +using Json::Reader; +using Json::Value; namespace android { namespace { -// Brightness and volume are stored as integer strings in next_boot.json. -// They are divided by this constant to produce the actual float values in -// range [0.0, 1.0]. This constant must match its counterpart in -// DeviceManager. -constexpr const float kFloatScaleFactor = 1000.0f; +// Keys for deprecated parameters. Devices that OTA from N to O and that used +// the hidden BootParameters API will store these in the JSON blob. To support +// the transition from N to O, these keys are mapped to the new parameters. +constexpr const char *kKeyLegacyVolume = "volume"; +constexpr const char *kKeyLegacyAnimationsDisabled = "boot_animation_disabled"; +constexpr const char *kKeyLegacyParamNames = "param_names"; +constexpr const char *kKeyLegacyParamValues = "param_values"; + +constexpr const char *kNextBootFile = "/data/misc/bootanimation/next_boot.proto"; +constexpr const char *kLastBootFile = "/data/misc/bootanimation/last_boot.proto"; -constexpr const char* kNextBootFile = "/data/misc/bootanimation/next_boot.json"; -constexpr const char* kLastBootFile = "/data/misc/bootanimation/last_boot.json"; +constexpr const char *kLegacyNextBootFile = "/data/misc/bootanimation/next_boot.json"; +constexpr const char *kLegacyLastBootFile = "/data/misc/bootanimation/last_boot.json"; -void swapBootConfigs() { - // rename() will fail if next_boot.json doesn't exist, so delete - // last_boot.json manually first. +void removeLegacyFiles() { std::string err; - if (!RemoveFileIfExists(kLastBootFile, &err)) - ALOGE("Unable to delete last boot file: %s", err.c_str()); + if (!RemoveFileIfExists(kLegacyLastBootFile, &err)) { + ALOGW("Unable to delete %s: %s", kLegacyLastBootFile, err.c_str()); + } - if (rename(kNextBootFile, kLastBootFile) && errno != ENOENT) - ALOGE("Unable to swap boot files: %s", strerror(errno)); + err.clear(); + if (!RemoveFileIfExists(kLegacyNextBootFile, &err)) { + ALOGW("Unable to delete %s: %s", kLegacyNextBootFile, err.c_str()); + } +} +void createNextBootFile() { + errno = 0; int fd = open(kNextBootFile, O_CREAT, DEFFILEMODE); if (fd == -1) { ALOGE("Unable to create next boot file: %s", strerror(errno)); @@ -71,54 +78,120 @@ void swapBootConfigs() { } // namespace -BootParameters::SavedBootParameters::SavedBootParameters() - : brightness(-kFloatScaleFactor), volume(-kFloatScaleFactor) {} - -void BootParameters::SavedBootParameters::RegisterJSONConverter( - JSONValueConverter<SavedBootParameters>* converter) { - converter->RegisterIntField("brightness", &SavedBootParameters::brightness); - converter->RegisterIntField("volume", &SavedBootParameters::volume); - converter->RegisterRepeatedString("param_names", - &SavedBootParameters::param_names); - converter->RegisterRepeatedString("param_values", - &SavedBootParameters::param_values); +// Renames the 'next' boot file to the 'last' file and reads its contents. +bool BootParameters::swapAndLoadBootConfigContents(const char *lastBootFile, + const char *nextBootFile, + std::string *contents) { + if (!ReadFileToString(nextBootFile, contents)) { + RemoveFileIfExists(lastBootFile); + return false; + } + + errno = 0; + if (rename(nextBootFile, lastBootFile) && errno != ENOENT) + ALOGE("Unable to swap boot files: %s", strerror(errno)); + + return true; } BootParameters::BootParameters() { - swapBootConfigs(); loadParameters(); } +// Saves the boot parameters state to disk so the framework can read it. +void BootParameters::storeParameters() { + errno = 0; + if (!WriteStringToFile(mProto.SerializeAsString(), kLastBootFile)) { + ALOGE("Failed to write boot parameters to %s: %s", kLastBootFile, strerror(errno)); + } + + // WriteStringToFile sets the file permissions to 0666, but these are not + // honored by the system. + errno = 0; + if (chmod(kLastBootFile, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH)) { + ALOGE("Failed to set permissions for %s: %s", kLastBootFile, strerror(errno)); + } +} + +// Load the boot parameters from disk, try the old location and format if the +// file does not exist. Note: +// - Parse errors result in defaults being used (a normal boot). +// - Legacy boot parameters default to a silent boot. void BootParameters::loadParameters() { + // Precedence is given to the new file format (.proto). std::string contents; - if (!ReadFileToString(kLastBootFile, &contents)) { - if (errno != ENOENT) - ALOGE("Unable to read from %s: %s", kLastBootFile, strerror(errno)); + if (swapAndLoadBootConfigContents(kLastBootFile, kNextBootFile, &contents)) { + parseBootParameters(contents); + } else if (swapAndLoadBootConfigContents(kLegacyLastBootFile, kLegacyNextBootFile, &contents)) { + parseLegacyBootParameters(contents); + storeParameters(); + removeLegacyFiles(); + } + + createNextBootFile(); +} +void BootParameters::parseBootParameters(const std::string &contents) { + if (!mProto.ParseFromString(contents)) { + ALOGW("Failed to parse parameters from %s", kLastBootFile); return; } - std::unique_ptr<Value> json = JSONReader::Read(contents); - if (json.get() == nullptr) { + loadStateFromProto(); +} + +// Parses the JSON in the proto. +void BootParameters::parseLegacyBootParameters(const std::string &contents) { + Value json; + if (!Reader().parse(contents, json)) { + ALOGW("Failed to parse parameters from %s", kLegacyLastBootFile); return; } - JSONValueConverter<SavedBootParameters> converter; - if (converter.Convert(*(json.get()), &mRawParameters)) { - mBrightness = mRawParameters.brightness / kFloatScaleFactor; - mVolume = mRawParameters.volume / kFloatScaleFactor; - - if (mRawParameters.param_names.size() == mRawParameters.param_values.size()) { - for (size_t i = 0; i < mRawParameters.param_names.size(); i++) { - mParameters.push_back({ - .key = mRawParameters.param_names[i]->c_str(), - .value = mRawParameters.param_values[i]->c_str() - }); + int volume = 0; + bool bootAnimationDisabled = true; + + Value &jsonValue = json[kKeyLegacyVolume]; + if (jsonValue.isIntegral()) { + volume = jsonValue.asInt(); + } + + jsonValue = json[kKeyLegacyAnimationsDisabled]; + if (jsonValue.isIntegral()) { + bootAnimationDisabled = jsonValue.asInt() == 1; + } + + // Assume a silent boot unless all of the following are true - + // 1. The volume is neither 0 nor -1000 (the legacy default value). + // 2. The boot animations are explicitly enabled. + // Note: brightness was never used. + mProto.set_silent_boot((volume == 0) || (volume == -1000) || bootAnimationDisabled); + + Value &keys = json[kKeyLegacyParamNames]; + Value &values = json[kKeyLegacyParamValues]; + if (keys.isArray() && values.isArray() && (keys.size() == values.size())) { + for (ArrayIndex i = 0; i < keys.size(); ++i) { + auto &key = keys[i]; + auto &value = values[i]; + if (key.isString() && value.isString()) { + auto userParameter = mProto.add_user_parameter(); + userParameter->set_key(key.asString()); + userParameter->set_value(value.asString()); } - } else { - ALOGW("Parameter names and values size mismatch"); } } + + loadStateFromProto(); +} + +void BootParameters::loadStateFromProto() { + // A missing key returns a safe, default value. + // Ignore invalid or missing parameters. + mIsSilentBoot = mProto.silent_boot(); + + for (const auto ¶m : mProto.user_parameter()) { + mParameters.push_back({.key = param.key().c_str(), .value = param.value().c_str()}); + } } } // namespace android diff --git a/cmds/bootanimation/iot/BootParameters.h b/cmds/bootanimation/iot/BootParameters.h index c10bd44bc2ca..cbd1ca61cfc3 100644 --- a/cmds/bootanimation/iot/BootParameters.h +++ b/cmds/bootanimation/iot/BootParameters.h @@ -18,10 +18,11 @@ #define _BOOTANIMATION_BOOT_PARAMETERS_H_ #include <list> +#include <string> #include <vector> -#include <base/json/json_value_converter.h> #include <boot_action/boot_action.h> // libandroidthings native API. +#include <boot_parameters.pb.h> namespace android { @@ -32,39 +33,39 @@ public: // to clear the parameters for next boot. BootParameters(); - // Returns true if volume/brightness were explicitly set on reboot. - bool hasVolume() const { return mVolume >= 0; } - bool hasBrightness() const { return mBrightness >= 0; } - - // Returns volume/brightness in [0,1], or -1 if unset. - float getVolume() const { return mVolume; } - float getBrightness() const { return mBrightness; } + // Returns whether or not this is a silent boot. + bool isSilentBoot() const { return mIsSilentBoot; } // Returns the additional boot parameters that were set on reboot. const std::vector<ABootActionParameter>& getParameters() const { return mParameters; } -private: - // Raw boot saved_parameters loaded from .json. - struct SavedBootParameters { - int brightness; - int volume; - std::vector<std::unique_ptr<std::string>> param_names; - std::vector<std::unique_ptr<std::string>> param_values; + // Exposed for testing. Sets the parameters to the serialized proto. + void parseBootParameters(const std::string &contents); + + // For devices that OTA from N to O. + // Exposed for testing. Sets the parameters to the raw JSON. + void parseLegacyBootParameters(const std::string &contents); - SavedBootParameters(); - static void RegisterJSONConverter( - ::base::JSONValueConverter<SavedBootParameters>* converter); - }; + // Exposed for testing. Loads the contents from |nextBootFile| and replaces + // |lastBootFile| with |nextBootFile|. + static bool swapAndLoadBootConfigContents(const char *lastBootFile, const char *nextBootFile, + std::string *contents); + private: void loadParameters(); - float mVolume = -1.f; - float mBrightness = -1.f; + // Replaces the legacy JSON blob with the updated version, allowing the + // framework to read it. + void storeParameters(); + + void loadStateFromProto(); + + bool mIsSilentBoot = false; + std::vector<ABootActionParameter> mParameters; - // ABootActionParameter is just a raw pointer so we need to keep the - // original strings around to avoid losing them. - SavedBootParameters mRawParameters; + // Store the proto because mParameters makes a shallow copy. + android::things::proto::BootParameters mProto; }; } // namespace android diff --git a/cmds/bootanimation/iot/BootParameters_test.cpp b/cmds/bootanimation/iot/BootParameters_test.cpp new file mode 100644 index 000000000000..d55bce6eecc3 --- /dev/null +++ b/cmds/bootanimation/iot/BootParameters_test.cpp @@ -0,0 +1,263 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "BootParameters.h" + +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> + +#include <android-base/file.h> +#include <android-base/test_utils.h> +#include <boot_parameters.pb.h> +#include <gtest/gtest.h> + +namespace android { + +namespace { + +TEST(BootParametersTest, TestNoBootParametersIsNotSilent) { + android::things::proto::BootParameters proto; + + BootParameters bootParameters = BootParameters(); + bootParameters.parseBootParameters(proto.SerializeAsString()); + + ASSERT_FALSE(bootParameters.isSilentBoot()); + ASSERT_EQ(0u, bootParameters.getParameters().size()); +} + +TEST(BootParametersTest, TestParseIsSilent) { + android::things::proto::BootParameters proto; + proto.set_silent_boot(true); + + BootParameters bootParameters = BootParameters(); + bootParameters.parseBootParameters(proto.SerializeAsString()); + + ASSERT_TRUE(bootParameters.isSilentBoot()); +} + +TEST(BootParametersTest, TestParseIsNotSilent) { + android::things::proto::BootParameters proto; + proto.set_silent_boot(false); + + BootParameters bootParameters = BootParameters(); + bootParameters.parseBootParameters(proto.SerializeAsString()); + + ASSERT_FALSE(bootParameters.isSilentBoot()); +} + +TEST(BootParametersTest, TestParseBootParameters) { + android::things::proto::BootParameters proto; + proto.set_silent_boot(false); + + auto userParameter = proto.add_user_parameter(); + userParameter->set_key("key1"); + userParameter->set_value("value1"); + + userParameter = proto.add_user_parameter(); + userParameter->set_key("key2"); + userParameter->set_value("value2"); + + BootParameters bootParameters = BootParameters(); + bootParameters.parseBootParameters(proto.SerializeAsString()); + + auto ¶meters = bootParameters.getParameters(); + ASSERT_EQ(2u, parameters.size()); + ASSERT_STREQ(parameters[0].key, "key1"); + ASSERT_STREQ(parameters[0].value, "value1"); + ASSERT_STREQ(parameters[1].key, "key2"); + ASSERT_STREQ(parameters[1].value, "value2"); +} + +TEST(BootParametersTest, TestParseLegacyDisableBootAnimationIsSilent) { + BootParameters bootParameters = BootParameters(); + bootParameters.parseLegacyBootParameters(R"( + { + "brightness":200, + "volume":100, + "boot_animation_disabled":1, + "param_names":[], + "param_values":[] + } + )"); + + ASSERT_TRUE(bootParameters.isSilentBoot()); +} + +TEST(BootParametersTest, TestParseLegacyZeroVolumeIsSilent) { + BootParameters bootParameters = BootParameters(); + bootParameters.parseLegacyBootParameters(R"( + { + "brightness":200, + "volume":0, + "boot_animation_disabled":0, + "param_names":[], + "param_values":[] + } + )"); + + ASSERT_TRUE(bootParameters.isSilentBoot()); +} + +TEST(BootParametersTest, TestParseLegacyDefaultVolumeIsSilent) { + BootParameters bootParameters = BootParameters(); + bootParameters.parseLegacyBootParameters(R"( + { + "brightness":200, + "volume":-1000, + "boot_animation_disabled":0, + "param_names":[], + "param_values":[] + } + )"); + + ASSERT_TRUE(bootParameters.isSilentBoot()); +} + +TEST(BootParametersTest, TestParseLegacyNotSilent) { + BootParameters bootParameters = BootParameters(); + bootParameters.parseLegacyBootParameters(R"( + { + "brightness":200, + "volume":500, + "boot_animation_disabled":0, + "param_names":[], + "param_values":[] + } + )"); + + ASSERT_FALSE(bootParameters.isSilentBoot()); +} + +TEST(BootParametersTest, TestParseLegacyParameters) { + BootParameters bootParameters = BootParameters(); + bootParameters.parseLegacyBootParameters(R"( + { + "brightness":200, + "volume":100, + "boot_animation_disabled":1, + "param_names":["key1", "key2"], + "param_values":["value1", "value2"] + } + )"); + + auto parameters = bootParameters.getParameters(); + ASSERT_EQ(2u, parameters.size()); + ASSERT_STREQ(parameters[0].key, "key1"); + ASSERT_STREQ(parameters[0].value, "value1"); + ASSERT_STREQ(parameters[1].key, "key2"); + ASSERT_STREQ(parameters[1].value, "value2"); +} + +TEST(BootParametersTest, TestParseLegacyZeroParameters) { + BootParameters bootParameters = BootParameters(); + bootParameters.parseLegacyBootParameters(R"( + { + "brightness":200, + "volume":100, + "boot_animation_disabled":1, + "param_names":[], + "param_values":[] + } + )"); + + ASSERT_EQ(0u, bootParameters.getParameters().size()); +} + +TEST(BootParametersTest, TestMalformedLegacyParametersAreSkipped) { + BootParameters bootParameters = BootParameters(); + bootParameters.parseLegacyBootParameters(R"( + { + "brightness":500, + "volume":500, + "boot_animation_disabled":0, + "param_names":["key1", "key2"], + "param_values":[1, "value2"] + } + )"); + + auto parameters = bootParameters.getParameters(); + ASSERT_EQ(1u, parameters.size()); + ASSERT_STREQ(parameters[0].key, "key2"); + ASSERT_STREQ(parameters[0].value, "value2"); +} + +TEST(BootParametersTest, TestLegacyUnequalParameterSizesAreSkipped) { + BootParameters bootParameters = BootParameters(); + bootParameters.parseLegacyBootParameters(R"( + { + "brightness":500, + "volume":500, + "boot_animation_disabled":0, + "param_names":["key1", "key2"], + "param_values":["value1"] + } + )"); + + ASSERT_EQ(0u, bootParameters.getParameters().size()); +} + +TEST(BootParametersTest, TestMissingLegacyBootParametersIsSilent) { + BootParameters bootParameters = BootParameters(); + bootParameters.parseLegacyBootParameters(R"( + { + "brightness":500 + } + )"); + + EXPECT_TRUE(bootParameters.isSilentBoot()); + ASSERT_EQ(0u, bootParameters.getParameters().size()); +} + +TEST(BootParametersTest, TestLastFileIsRemovedOnError) { + TemporaryFile lastFile; + TemporaryDir tempDir; + std::string nonExistentFilePath(std::string(tempDir.path) + "/nonexistent"); + std::string contents; + + BootParameters::swapAndLoadBootConfigContents(lastFile.path, nonExistentFilePath.c_str(), + &contents); + + struct stat buf; + ASSERT_EQ(-1, lstat(lastFile.path, &buf)); + ASSERT_TRUE(contents.empty()); +} + +TEST(BootParametersTest, TestNextFileIsRemovedLastFileExistsOnSuccess) { + TemporaryFile lastFile; + TemporaryFile nextFile; + + base::WriteStringToFile("foo", nextFile.path); + + std::string contents; + // Expected side effects: + // - |next_file| is moved to |last_file| + // - |contents| is the contents of |next_file| before being moved. + BootParameters::swapAndLoadBootConfigContents(lastFile.path, nextFile.path, &contents); + + struct stat buf; + ASSERT_EQ(0, lstat(lastFile.path, &buf)); + ASSERT_EQ(-1, lstat(nextFile.path, &buf)); + ASSERT_EQ(contents, "foo"); + + contents.clear(); + ASSERT_TRUE(base::ReadFileToString(lastFile.path, &contents)); + ASSERT_EQ(contents, "foo"); +} + +} // namespace + +} // namespace android diff --git a/cmds/bootanimation/iot/iotbootanimation_main.cpp b/cmds/bootanimation/iot/iotbootanimation_main.cpp index 00cef430135e..2a3d3766ab38 100644 --- a/cmds/bootanimation/iot/iotbootanimation_main.cpp +++ b/cmds/bootanimation/iot/iotbootanimation_main.cpp @@ -59,7 +59,7 @@ public: } mBootAction = new BootAction(); - if (!mBootAction->init(library_path, mBootParameters->getParameters())) { + if (!mBootAction->init(library_path, mBootParameters)) { mBootAction = NULL; } }; @@ -116,8 +116,16 @@ int main() { sp<ProcessState> proc(ProcessState::self()); ProcessState::self()->startThreadPool(); - sp<BootAnimation> boot = new BootAnimation( - new BootActionAnimationCallbacks(std::move(bootParameters))); + bool isSilentBoot = bootParameters->isSilentBoot(); + sp<BootActionAnimationCallbacks> callbacks = + new BootActionAnimationCallbacks(std::move(bootParameters)); + + // On silent boot, animations aren't displayed. + if (isSilentBoot) { + callbacks->init({}); + } else { + sp<BootAnimation> boot = new BootAnimation(callbacks); + } IPCThreadState::self()->joinThreadPool(); return 0; diff --git a/cmds/content/src/com/android/commands/content/Content.java b/cmds/content/src/com/android/commands/content/Content.java index 6e0bd3a81d84..1597c8c9c2b2 100644 --- a/cmds/content/src/com/android/commands/content/Content.java +++ b/cmds/content/src/com/android/commands/content/Content.java @@ -33,11 +33,7 @@ import android.os.Process; import android.os.UserHandle; import android.text.TextUtils; -import libcore.io.Streams; - import java.io.FileDescriptor; -import java.io.FileInputStream; -import java.io.FileOutputStream; /** * This class is a command line utility for manipulating content. A client @@ -462,7 +458,7 @@ public class Content { IBinder token = new Binder(); try { ContentProviderHolder holder = activityManager.getContentProviderExternal( - providerName, mUserId, token); + providerName, mUserId, token, "*cmd*"); if (holder == null) { throw new IllegalStateException("Could not find provider: " + providerName); } @@ -470,7 +466,8 @@ public class Content { onExecute(provider); } finally { if (provider != null) { - activityManager.removeContentProviderExternal(providerName, token); + activityManager.removeContentProviderExternalAsUser( + providerName, token, mUserId); } } } catch (Exception e) { diff --git a/cmds/dpm/src/com/android/commands/dpm/Dpm.java b/cmds/dpm/src/com/android/commands/dpm/Dpm.java index 7c1a5557a1e9..376b13cd371e 100644 --- a/cmds/dpm/src/com/android/commands/dpm/Dpm.java +++ b/cmds/dpm/src/com/android/commands/dpm/Dpm.java @@ -46,6 +46,7 @@ public final class Dpm extends BaseCommand { private static final String COMMAND_SET_PROFILE_OWNER = "set-profile-owner"; private static final String COMMAND_REMOVE_ACTIVE_ADMIN = "remove-active-admin"; private static final String COMMAND_CLEAR_FREEZE_PERIOD_RECORD = "clear-freeze-period-record"; + private static final String COMMAND_FORCE_NETWORK_LOGS = "force-network-logs"; private static final String COMMAND_FORCE_SECURITY_LOGS = "force-security-logs"; private IDevicePolicyManager mDevicePolicyManager; @@ -84,6 +85,9 @@ public final class Dpm extends BaseCommand { "feature development to prevent triggering restriction on setting freeze " + "periods.\n" + "\n" + + "dpm " + COMMAND_FORCE_NETWORK_LOGS + ": makes all network logs available to " + + "the DPC and triggers DeviceAdminReceiver.onNetworkLogsAvailable() if needed.\n" + + "\n" + "dpm " + COMMAND_FORCE_SECURITY_LOGS + ": makes all security logs available to " + "the DPC and triggers DeviceAdminReceiver.onSecurityLogsAvailable() if needed."); } @@ -114,6 +118,9 @@ public final class Dpm extends BaseCommand { case COMMAND_CLEAR_FREEZE_PERIOD_RECORD: runClearFreezePeriodRecord(); break; + case COMMAND_FORCE_NETWORK_LOGS: + runForceNetworkLogs(); + break; case COMMAND_FORCE_SECURITY_LOGS: runForceSecurityLogs(); break; @@ -122,6 +129,18 @@ public final class Dpm extends BaseCommand { } } + private void runForceNetworkLogs() throws RemoteException, InterruptedException { + while (true) { + final long toWait = mDevicePolicyManager.forceNetworkLogs(); + if (toWait == 0) { + break; + } + System.out.println("We have to wait for " + toWait + " milliseconds..."); + Thread.sleep(toWait); + } + System.out.println("Success"); + } + private void runForceSecurityLogs() throws RemoteException, InterruptedException { while (true) { final long toWait = mDevicePolicyManager.forceSecurityLogs(); diff --git a/cmds/hid/jni/com_android_commands_hid_Device.cpp b/cmds/hid/jni/com_android_commands_hid_Device.cpp index 5cc4fc4c16b2..b3e287bae76a 100644 --- a/cmds/hid/jni/com_android_commands_hid_Device.cpp +++ b/cmds/hid/jni/com_android_commands_hid_Device.cpp @@ -42,7 +42,6 @@ namespace android { namespace uhid { static const char* UHID_PATH = "/dev/uhid"; -static const size_t UHID_MAX_NAME_LENGTH = 128; static struct { jmethodID onDeviceOpen; @@ -90,8 +89,13 @@ JNIEnv* DeviceCallback::getJNIEnv() { } Device* Device::open(int32_t id, const char* name, int32_t vid, int32_t pid, - std::unique_ptr<uint8_t[]> descriptor, size_t descriptorSize, - std::unique_ptr<DeviceCallback> callback) { + std::vector<uint8_t> descriptor, std::unique_ptr<DeviceCallback> callback) { + + size_t size = descriptor.size(); + if (size > HID_MAX_DESCRIPTOR_SIZE) { + LOGE("Received invalid hid report with descriptor size %zu, skipping", size); + return nullptr; + } int fd = ::open(UHID_PATH, O_RDWR | O_CLOEXEC); if (fd < 0) { @@ -102,10 +106,10 @@ Device* Device::open(int32_t id, const char* name, int32_t vid, int32_t pid, struct uhid_event ev; memset(&ev, 0, sizeof(ev)); ev.type = UHID_CREATE2; - strncpy((char*)ev.u.create2.name, name, UHID_MAX_NAME_LENGTH); - memcpy(&ev.u.create2.rd_data, descriptor.get(), - descriptorSize * sizeof(ev.u.create2.rd_data[0])); - ev.u.create2.rd_size = descriptorSize; + strlcpy(reinterpret_cast<char*>(ev.u.create2.name), name, sizeof(ev.u.create2.name)); + memcpy(&ev.u.create2.rd_data, descriptor.data(), + size * sizeof(ev.u.create2.rd_data[0])); + ev.u.create2.rd_size = size; ev.u.create2.bus = BUS_BLUETOOTH; ev.u.create2.vendor = vid; ev.u.create2.product = pid; @@ -156,12 +160,17 @@ Device::~Device() { mFd = -1; } -void Device::sendReport(uint8_t* report, size_t reportSize) { +void Device::sendReport(const std::vector<uint8_t>& report) const { + if (report.size() > UHID_DATA_MAX) { + LOGE("Received invalid report of size %zu, skipping", report.size()); + return; + } + struct uhid_event ev; memset(&ev, 0, sizeof(ev)); ev.type = UHID_INPUT2; - ev.u.input2.size = reportSize; - memcpy(&ev.u.input2.data, report, reportSize); + ev.u.input2.size = report.size(); + memcpy(&ev.u.input2.data, report.data(), report.size() * sizeof(ev.u.input2.data[0])); ssize_t ret = TEMP_FAILURE_RETRY(::write(mFd, &ev, sizeof(ev))); if (ret < 0 || ret != sizeof(ev)) { LOGE("Failed to send hid event: %s", strerror(errno)); @@ -191,12 +200,13 @@ int Device::handleEvents(int events) { } // namespace uhid -std::unique_ptr<uint8_t[]> getData(JNIEnv* env, jbyteArray javaArray, size_t& outSize) { +std::vector<uint8_t> getData(JNIEnv* env, jbyteArray javaArray) { ScopedByteArrayRO scopedArray(env, javaArray); - outSize = scopedArray.size(); - std::unique_ptr<uint8_t[]> data(new uint8_t[outSize]); - for (size_t i = 0; i < outSize; i++) { - data[i] = static_cast<uint8_t>(scopedArray[i]); + size_t size = scopedArray.size(); + std::vector<uint8_t> data; + data.reserve(size); + for (size_t i = 0; i < size; i++) { + data.push_back(static_cast<uint8_t>(scopedArray[i])); } return data; } @@ -208,23 +218,20 @@ static jlong openDevice(JNIEnv* env, jclass /* clazz */, jstring rawName, jint i return 0; } - size_t size; - std::unique_ptr<uint8_t[]> desc = getData(env, rawDescriptor, size); + std::vector<uint8_t> desc = getData(env, rawDescriptor); std::unique_ptr<uhid::DeviceCallback> cb(new uhid::DeviceCallback(env, callback)); uhid::Device* d = uhid::Device::open( - id, reinterpret_cast<const char*>(name.c_str()), vid, pid, - std::move(desc), size, std::move(cb)); + id, reinterpret_cast<const char*>(name.c_str()), vid, pid, desc, std::move(cb)); return reinterpret_cast<jlong>(d); } static void sendReport(JNIEnv* env, jclass /* clazz */, jlong ptr, jbyteArray rawReport) { - size_t size; - std::unique_ptr<uint8_t[]> report = getData(env, rawReport, size); + std::vector<uint8_t> report = getData(env, rawReport); uhid::Device* d = reinterpret_cast<uhid::Device*>(ptr); if (d) { - d->sendReport(report.get(), size); + d->sendReport(report); } else { LOGE("Could not send report, Device* is null!"); } diff --git a/cmds/hid/jni/com_android_commands_hid_Device.h b/cmds/hid/jni/com_android_commands_hid_Device.h index 149456d8c10d..61a1f760697f 100644 --- a/cmds/hid/jni/com_android_commands_hid_Device.h +++ b/cmds/hid/jni/com_android_commands_hid_Device.h @@ -15,6 +15,7 @@ */ #include <memory> +#include <vector> #include <jni.h> @@ -38,13 +39,12 @@ private: class Device { public: static Device* open(int32_t id, const char* name, int32_t vid, int32_t pid, - std::unique_ptr<uint8_t[]> descriptor, size_t descriptorSize, - std::unique_ptr<DeviceCallback> callback); + std::vector<uint8_t> descriptor, std::unique_ptr<DeviceCallback> callback); Device(int32_t id, int fd, std::unique_ptr<DeviceCallback> callback); ~Device(); - void sendReport(uint8_t* report, size_t reportSize); + void sendReport(const std::vector<uint8_t>& report) const; void close(); int handleEvents(int events); diff --git a/cmds/idmap2/.clang-format b/cmds/idmap2/.clang-format new file mode 100644 index 000000000000..c91502a257f3 --- /dev/null +++ b/cmds/idmap2/.clang-format @@ -0,0 +1,7 @@ +BasedOnStyle: Google +ColumnLimit: 100 +AllowShortBlocksOnASingleLine: false +AllowShortFunctionsOnASingleLine: false +CommentPragmas: NOLINT:.* +DerivePointerAlignment: false +PointerAlignment: Left diff --git a/cmds/idmap2/Android.bp b/cmds/idmap2/Android.bp new file mode 100644 index 000000000000..5a6c813fd202 --- /dev/null +++ b/cmds/idmap2/Android.bp @@ -0,0 +1,191 @@ +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +cc_library { + name: "libidmap2", + host_supported: true, + tidy: true, + tidy_flags: [ + "-system-headers", + "-warnings-as-errors=*", + ], + srcs: [ + "libidmap2/BinaryStreamVisitor.cpp", + "libidmap2/CommandLineOptions.cpp", + "libidmap2/FileUtils.cpp", + "libidmap2/Idmap.cpp", + "libidmap2/PrettyPrintVisitor.cpp", + "libidmap2/RawPrintVisitor.cpp", + "libidmap2/ResourceUtils.cpp", + "libidmap2/Xml.cpp", + "libidmap2/ZipFile.cpp", + ], + export_include_dirs: ["include"], + target: { + android: { + static: { + enabled: false, + }, + shared_libs: [ + "libandroidfw", + "libbase", + "libutils", + "libziparchive", + ], + }, + host: { + shared: { + enabled: false, + }, + static_libs: [ + "libandroidfw", + "libbase", + "libutils", + "libziparchive", + ], + }, + }, +} + +cc_test { + name: "idmap2_tests", + host_supported: true, + tidy: true, + tidy_flags: [ + "-system-headers", + "-warnings-as-errors=*", + ], + srcs: [ + "tests/BinaryStreamVisitorTests.cpp", + "tests/CommandLineOptionsTests.cpp", + "tests/FileUtilsTests.cpp", + "tests/Idmap2BinaryTests.cpp", + "tests/IdmapTests.cpp", + "tests/Main.cpp", + "tests/PrettyPrintVisitorTests.cpp", + "tests/RawPrintVisitorTests.cpp", + "tests/ResourceUtilsTests.cpp", + "tests/XmlTests.cpp", + "tests/ZipFileTests.cpp", + ], + required: [ + "idmap2", + ], + static_libs: ["libgmock"], + target: { + android: { + shared_libs: [ + "libandroidfw", + "libbase", + "libidmap2", + "liblog", + "libutils", + "libz", + "libziparchive", + ], + }, + host: { + static_libs: [ + "libandroidfw", + "libbase", + "libidmap2", + "liblog", + "libutils", + "libziparchive", + ], + shared_libs: [ + "libz", + ], + }, + }, + data: ["tests/data/**/*.apk"], +} + +cc_binary { + name: "idmap2", + host_supported: true, + tidy: true, + tidy_flags: [ + "-system-headers", + "-warnings-as-errors=*", + ], + srcs: [ + "idmap2/Create.cpp", + "idmap2/Dump.cpp", + "idmap2/Lookup.cpp", + "idmap2/Main.cpp", + "idmap2/Scan.cpp", + "idmap2/Verify.cpp", + ], + target: { + android: { + shared_libs: [ + "libandroidfw", + "libbase", + "libidmap2", + "libutils", + "libziparchive", + ], + }, + host: { + static_libs: [ + "libandroidfw", + "libbase", + "libidmap2", + "liblog", + "libutils", + "libziparchive", + ], + shared_libs: [ + "libz", + ], + }, + }, +} + +cc_binary { + name: "idmap2d", + host_supported: false, + tidy: true, + tidy_checks: [ + // remove google-default-arguments or clang-tidy will complain about + // the auto-generated file IIdmap2.cpp + "-google-default-arguments", + ], + tidy_flags: [ + "-system-headers", + "-warnings-as-errors=*", + ], + srcs: [ + ":idmap2_aidl", + "idmap2d/Idmap2Service.cpp", + "idmap2d/Main.cpp", + ], + shared_libs: [ + "libandroidfw", + "libbase", + "libbinder", + "libcutils", + "libidmap2", + "libutils", + "libziparchive", + ], +} + +filegroup { + name: "idmap2_aidl", + srcs: [ + "idmap2d/aidl/android/os/IIdmap2.aidl", + ], +} diff --git a/cmds/idmap2/AndroidTest.xml b/cmds/idmap2/AndroidTest.xml new file mode 100644 index 000000000000..5147f4e6cb4c --- /dev/null +++ b/cmds/idmap2/AndroidTest.xml @@ -0,0 +1,26 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2018 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration description="Config for idmap2_tests"> + <target_preparer class="com.android.tradefed.targetprep.PushFilePreparer"> + <option name="cleanup" value="true" /> + <option name="push" value="idmap2_tests->/data/local/tmp/idmap2_tests" /> + </target_preparer> + <option name="test-suite-tag" value="idmap2_tests" /> + <test class="com.android.tradefed.testtype.GTest" > + <option name="native-test-device-path" value="/data/local/tmp" /> + <option name="module-name" value="idmap2_tests" /> + </test> +</configuration> diff --git a/cmds/idmap2/CPPLINT.cfg b/cmds/idmap2/CPPLINT.cfg new file mode 100644 index 000000000000..9dc6b4a77380 --- /dev/null +++ b/cmds/idmap2/CPPLINT.cfg @@ -0,0 +1,18 @@ +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set noparent +linelength=100 +root=.. +filter=+build/include_alpha diff --git a/cmds/idmap2/OWNERS b/cmds/idmap2/OWNERS new file mode 100644 index 000000000000..23ec5ab0d1f3 --- /dev/null +++ b/cmds/idmap2/OWNERS @@ -0,0 +1,2 @@ +set noparent +toddke@google.com diff --git a/cmds/idmap2/TEST_MAPPING b/cmds/idmap2/TEST_MAPPING new file mode 100644 index 000000000000..26ccf038cba2 --- /dev/null +++ b/cmds/idmap2/TEST_MAPPING @@ -0,0 +1,7 @@ +{ + "presubmit" : [ + { + "name" : "idmap2_tests" + } + ] +} diff --git a/cmds/idmap2/idmap2/Commands.h b/cmds/idmap2/idmap2/Commands.h new file mode 100644 index 000000000000..dcc69b30743d --- /dev/null +++ b/cmds/idmap2/idmap2/Commands.h @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IDMAP2_IDMAP2_COMMANDS_H_ +#define IDMAP2_IDMAP2_COMMANDS_H_ + +#include <string> +#include <vector> + +bool Create(const std::vector<std::string>& args, std::ostream& out_error); +bool Dump(const std::vector<std::string>& args, std::ostream& out_error); +bool Lookup(const std::vector<std::string>& args, std::ostream& out_error); +bool Scan(const std::vector<std::string>& args, std::ostream& out_error); +bool Verify(const std::vector<std::string>& args, std::ostream& out_error); + +#endif // IDMAP2_IDMAP2_COMMANDS_H_ diff --git a/cmds/idmap2/idmap2/Create.cpp b/cmds/idmap2/idmap2/Create.cpp new file mode 100644 index 000000000000..291eaeb9c211 --- /dev/null +++ b/cmds/idmap2/idmap2/Create.cpp @@ -0,0 +1,86 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <sys/stat.h> // umask +#include <sys/types.h> // umask +#include <fstream> +#include <memory> +#include <ostream> +#include <sstream> +#include <string> +#include <vector> + +#include "idmap2/BinaryStreamVisitor.h" +#include "idmap2/CommandLineOptions.h" +#include "idmap2/FileUtils.h" +#include "idmap2/Idmap.h" + +using android::ApkAssets; +using android::idmap2::BinaryStreamVisitor; +using android::idmap2::CommandLineOptions; +using android::idmap2::Idmap; + +bool Create(const std::vector<std::string>& args, std::ostream& out_error) { + std::string target_apk_path, overlay_apk_path, idmap_path; + + const CommandLineOptions opts = + CommandLineOptions("idmap2 create") + .MandatoryOption("--target-apk-path", + "input: path to apk which will have its resources overlaid", + &target_apk_path) + .MandatoryOption("--overlay-apk-path", + "input: path to apk which contains the new resource values", + &overlay_apk_path) + .MandatoryOption("--idmap-path", "output: path to where to write idmap file", + &idmap_path); + if (!opts.Parse(args, out_error)) { + return false; + } + + const std::unique_ptr<const ApkAssets> target_apk = ApkAssets::Load(target_apk_path); + if (!target_apk) { + out_error << "error: failed to load apk " << target_apk_path << std::endl; + return false; + } + + const std::unique_ptr<const ApkAssets> overlay_apk = ApkAssets::Load(overlay_apk_path); + if (!overlay_apk) { + out_error << "error: failed to load apk " << overlay_apk_path << std::endl; + return false; + } + + const std::unique_ptr<const Idmap> idmap = + Idmap::FromApkAssets(target_apk_path, *target_apk, overlay_apk_path, *overlay_apk, out_error); + if (!idmap) { + return false; + } + + umask(0133); // u=rw,g=r,o=r + std::ofstream fout(idmap_path); + if (fout.fail()) { + out_error << "failed to open idmap path " << idmap_path << std::endl; + return false; + } + BinaryStreamVisitor visitor(fout); + idmap->accept(&visitor); + fout.close(); + if (fout.fail()) { + out_error << "failed to write to idmap path " << idmap_path << std::endl; + return false; + } + + return true; +} diff --git a/cmds/idmap2/idmap2/Dump.cpp b/cmds/idmap2/idmap2/Dump.cpp new file mode 100644 index 000000000000..c8cdcfa6d3dc --- /dev/null +++ b/cmds/idmap2/idmap2/Dump.cpp @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fstream> +#include <iostream> +#include <memory> +#include <string> +#include <vector> + +#include "idmap2/CommandLineOptions.h" +#include "idmap2/Idmap.h" +#include "idmap2/PrettyPrintVisitor.h" +#include "idmap2/RawPrintVisitor.h" + +using android::idmap2::CommandLineOptions; +using android::idmap2::Idmap; +using android::idmap2::PrettyPrintVisitor; +using android::idmap2::RawPrintVisitor; + +bool Dump(const std::vector<std::string>& args, std::ostream& out_error) { + std::string idmap_path; + bool verbose; + + const CommandLineOptions opts = + CommandLineOptions("idmap2 dump") + .MandatoryOption("--idmap-path", "input: path to idmap file to pretty-print", &idmap_path) + .OptionalFlag("--verbose", "annotate every byte of the idmap", &verbose); + if (!opts.Parse(args, out_error)) { + return false; + } + std::ifstream fin(idmap_path); + const std::unique_ptr<const Idmap> idmap = Idmap::FromBinaryStream(fin, out_error); + fin.close(); + if (!idmap) { + return false; + } + + if (verbose) { + RawPrintVisitor visitor(std::cout); + idmap->accept(&visitor); + } else { + PrettyPrintVisitor visitor(std::cout); + idmap->accept(&visitor); + } + + return true; +} diff --git a/cmds/idmap2/idmap2/Lookup.cpp b/cmds/idmap2/idmap2/Lookup.cpp new file mode 100644 index 000000000000..1191e6a27b07 --- /dev/null +++ b/cmds/idmap2/idmap2/Lookup.cpp @@ -0,0 +1,249 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <algorithm> +#include <fstream> +#include <iterator> +#include <memory> +#include <ostream> +#include <string> +#include <utility> +#include <vector> + +#include "android-base/macros.h" +#include "android-base/stringprintf.h" +#include "androidfw/ApkAssets.h" +#include "androidfw/AssetManager2.h" +#include "androidfw/ConfigDescription.h" +#include "androidfw/ResourceUtils.h" +#include "androidfw/StringPiece.h" +#include "androidfw/Util.h" +#include "utils/String16.h" +#include "utils/String8.h" + +#include "idmap2/CommandLineOptions.h" +#include "idmap2/Idmap.h" +#include "idmap2/Xml.h" +#include "idmap2/ZipFile.h" + +using android::ApkAssets; +using android::ApkAssetsCookie; +using android::AssetManager2; +using android::ConfigDescription; +using android::is_valid_resid; +using android::kInvalidCookie; +using android::Res_value; +using android::ResStringPool; +using android::ResTable_config; +using android::String16; +using android::String8; +using android::StringPiece16; +using android::base::StringPrintf; +using android::idmap2::CommandLineOptions; +using android::idmap2::IdmapHeader; +using android::idmap2::ResourceId; +using android::idmap2::Xml; +using android::idmap2::ZipFile; +using android::util::Utf16ToUtf8; + +namespace { +std::pair<bool, ResourceId> WARN_UNUSED ParseResReference(const AssetManager2& am, + const std::string& res, + const std::string& fallback_package) { + // first, try to parse as a hex number + char* endptr = nullptr; + ResourceId resid; + resid = strtol(res.c_str(), &endptr, 16); + if (*endptr == '\0') { + return std::make_pair(true, resid); + } + + // next, try to parse as a package:type/name string + resid = am.GetResourceId(res, "", fallback_package); + if (is_valid_resid(resid)) { + return std::make_pair(true, resid); + } + + // end of the road: res could not be parsed + return std::make_pair(false, 0); +} + +std::pair<bool, std::string> WARN_UNUSED GetValue(const AssetManager2& am, ResourceId resid) { + Res_value value; + ResTable_config config; + uint32_t flags; + ApkAssetsCookie cookie = am.GetResource(resid, false, 0, &value, &config, &flags); + if (cookie == kInvalidCookie) { + return std::make_pair(false, ""); + } + + std::string out; + + // TODO(martenkongstad): use optional parameter GetResource(..., std::string* + // stacktrace = NULL) instead + out.append(StringPrintf("cookie=%d ", cookie)); + + out.append("config='"); + out.append(config.toString().c_str()); + out.append("' value="); + + switch (value.dataType) { + case Res_value::TYPE_INT_DEC: + out.append(StringPrintf("%d", value.data)); + break; + case Res_value::TYPE_INT_HEX: + out.append(StringPrintf("0x%08x", value.data)); + break; + case Res_value::TYPE_INT_BOOLEAN: + out.append(value.data != 0 ? "true" : "false"); + break; + case Res_value::TYPE_STRING: { + const ResStringPool* pool = am.GetStringPoolForCookie(cookie); + size_t len; + if (pool->isUTF8()) { + const char* str = pool->string8At(value.data, &len); + out.append(str, len); + } else { + const char16_t* str16 = pool->stringAt(value.data, &len); + out += Utf16ToUtf8(StringPiece16(str16, len)); + } + } break; + default: + out.append(StringPrintf("dataType=0x%02x data=0x%08x", value.dataType, value.data)); + break; + } + return std::make_pair(true, out); +} + +std::pair<bool, std::string> GetTargetPackageNameFromManifest(const std::string& apk_path) { + const auto zip = ZipFile::Open(apk_path); + if (!zip) { + return std::make_pair(false, ""); + } + const auto entry = zip->Uncompress("AndroidManifest.xml"); + if (!entry) { + return std::make_pair(false, ""); + } + const auto xml = Xml::Create(entry->buf, entry->size); + if (!xml) { + return std::make_pair(false, ""); + } + const auto tag = xml->FindTag("overlay"); + if (!tag) { + return std::make_pair(false, ""); + } + const auto iter = tag->find("targetPackage"); + if (iter == tag->end()) { + return std::make_pair(false, ""); + } + return std::make_pair(true, iter->second); +} +} // namespace + +bool Lookup(const std::vector<std::string>& args, std::ostream& out_error) { + std::vector<std::string> idmap_paths; + std::string config_str, resid_str; + const CommandLineOptions opts = + CommandLineOptions("idmap2 lookup") + .MandatoryOption("--idmap-path", "input: path to idmap file to load", &idmap_paths) + .MandatoryOption("--config", "configuration to use", &config_str) + .MandatoryOption("--resid", + "Resource ID (in the target package; '0xpptteeee' or " + "'[package:]type/name') to look up", + &resid_str); + + if (!opts.Parse(args, out_error)) { + return false; + } + + ConfigDescription config; + if (!ConfigDescription::Parse(config_str, &config)) { + out_error << "error: failed to parse config" << std::endl; + return false; + } + + std::vector<std::unique_ptr<const ApkAssets>> apk_assets; + std::string target_path; + std::string target_package_name; + for (size_t i = 0; i < idmap_paths.size(); i++) { + const auto& idmap_path = idmap_paths[i]; + std::fstream fin(idmap_path); + auto idmap_header = IdmapHeader::FromBinaryStream(fin); + fin.close(); + if (!idmap_header) { + out_error << "error: failed to read idmap from " << idmap_path << std::endl; + return false; + } + + if (i == 0) { + target_path = idmap_header->GetTargetPath().to_string(); + auto target_apk = ApkAssets::Load(target_path); + if (!target_apk) { + out_error << "error: failed to read target apk from " << target_path << std::endl; + return false; + } + apk_assets.push_back(std::move(target_apk)); + + bool lookup_ok; + std::tie(lookup_ok, target_package_name) = + GetTargetPackageNameFromManifest(idmap_header->GetOverlayPath().to_string()); + if (!lookup_ok) { + out_error << "error: failed to parse android:targetPackage from overlay manifest" + << std::endl; + return false; + } + } else if (target_path != idmap_header->GetTargetPath()) { + out_error << "error: different target APKs (expected target APK " << target_path << " but " + << idmap_path << " has target APK " << idmap_header->GetTargetPath() << ")" + << std::endl; + return false; + } + + auto overlay_apk = ApkAssets::LoadOverlay(idmap_path); + if (!overlay_apk) { + out_error << "error: failed to read overlay apk from " << idmap_header->GetOverlayPath() + << std::endl; + return false; + } + apk_assets.push_back(std::move(overlay_apk)); + } + + // AssetManager2::SetApkAssets requires raw ApkAssets pointers, not unique_ptrs + std::vector<const ApkAssets*> raw_pointer_apk_assets; + std::transform(apk_assets.cbegin(), apk_assets.cend(), std::back_inserter(raw_pointer_apk_assets), + [](const auto& p) -> const ApkAssets* { return p.get(); }); + AssetManager2 am; + am.SetApkAssets(raw_pointer_apk_assets); + am.SetConfiguration(config); + + ResourceId resid; + bool lookup_ok; + std::tie(lookup_ok, resid) = ParseResReference(am, resid_str, target_package_name); + if (!lookup_ok) { + out_error << "error: failed to parse resource ID" << std::endl; + return false; + } + + std::string value; + std::tie(lookup_ok, value) = GetValue(am, resid); + if (!lookup_ok) { + out_error << StringPrintf("error: resource 0x%08x not found", resid) << std::endl; + return false; + } + std::cout << value << std::endl; + + return true; +} diff --git a/cmds/idmap2/idmap2/Main.cpp b/cmds/idmap2/idmap2/Main.cpp new file mode 100644 index 000000000000..5d9ea778915a --- /dev/null +++ b/cmds/idmap2/idmap2/Main.cpp @@ -0,0 +1,67 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <cstdlib> // EXIT_{FAILURE,SUCCESS} +#include <functional> +#include <iostream> +#include <map> +#include <memory> +#include <ostream> +#include <string> +#include <vector> + +#include "idmap2/CommandLineOptions.h" + +#include "Commands.h" + +using android::idmap2::CommandLineOptions; + +typedef std::map<std::string, std::function<int(const std::vector<std::string>&, std::ostream&)>> + NameToFunctionMap; + +static void PrintUsage(const NameToFunctionMap& commands, std::ostream& out) { + out << "usage: idmap2 ["; + for (auto iter = commands.cbegin(); iter != commands.cend(); iter++) { + if (iter != commands.cbegin()) { + out << "|"; + } + out << iter->first; + } + out << "]" << std::endl; +} + +int main(int argc, char** argv) { + const NameToFunctionMap commands = { + {"create", Create}, {"dump", Dump}, {"lookup", Lookup}, {"scan", Scan}, {"verify", Verify}, + }; + if (argc <= 1) { + PrintUsage(commands, std::cerr); + return EXIT_FAILURE; + } + const std::unique_ptr<std::vector<std::string>> args = + CommandLineOptions::ConvertArgvToVector(argc - 1, const_cast<const char**>(argv + 1)); + if (!args) { + std::cerr << "error: failed to parse command line options" << std::endl; + return EXIT_FAILURE; + } + const auto iter = commands.find(argv[1]); + if (iter == commands.end()) { + std::cerr << argv[1] << ": command not found" << std::endl; + PrintUsage(commands, std::cerr); + return EXIT_FAILURE; + } + return iter->second(*args, std::cerr) ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/cmds/idmap2/idmap2/Scan.cpp b/cmds/idmap2/idmap2/Scan.cpp new file mode 100644 index 000000000000..00c49e38e8b2 --- /dev/null +++ b/cmds/idmap2/idmap2/Scan.cpp @@ -0,0 +1,160 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <dirent.h> +#include <fstream> +#include <memory> +#include <ostream> +#include <set> +#include <sstream> +#include <string> +#include <vector> + +#include "idmap2/CommandLineOptions.h" +#include "idmap2/FileUtils.h" +#include "idmap2/Idmap.h" +#include "idmap2/Xml.h" +#include "idmap2/ZipFile.h" + +#include "Commands.h" + +using android::idmap2::CommandLineOptions; +using android::idmap2::Idmap; +using android::idmap2::MemoryChunk; +using android::idmap2::Xml; +using android::idmap2::ZipFile; +using android::idmap2::utils::FindFiles; + +namespace { +std::unique_ptr<std::vector<std::string>> FindApkFiles(const std::vector<std::string>& dirs, + bool recursive, std::ostream& out_error) { + const auto predicate = [](unsigned char type, const std::string& path) -> bool { + static constexpr size_t kExtLen = 4; // strlen(".apk") + return type == DT_REG && path.size() > kExtLen && + !path.compare(path.size() - kExtLen, kExtLen, ".apk"); + }; + // pass apk paths through a set to filter out duplicates + std::set<std::string> paths; + for (const auto& dir : dirs) { + const auto apk_paths = FindFiles(dir, recursive, predicate); + if (!apk_paths) { + out_error << "error: failed to open directory " << dir << std::endl; + return nullptr; + } + paths.insert(apk_paths->cbegin(), apk_paths->cend()); + } + return std::unique_ptr<std::vector<std::string>>( + new std::vector<std::string>(paths.cbegin(), paths.cend())); +} +} // namespace + +bool Scan(const std::vector<std::string>& args, std::ostream& out_error) { + std::vector<std::string> input_directories; + std::string target_package_name, target_apk_path, output_directory; + bool recursive = false; + + const CommandLineOptions opts = + CommandLineOptions("idmap2 scan") + .MandatoryOption("--input-directory", "directory containing overlay apks to scan", + &input_directories) + .OptionalFlag("--recursive", "also scan subfolders of overlay-directory", &recursive) + .MandatoryOption("--target-package-name", "package name of target package", + &target_package_name) + .MandatoryOption("--target-apk-path", "path to target apk", &target_apk_path) + .MandatoryOption("--output-directory", + "directory in which to write artifacts (idmap files and overlays.list)", + &output_directory); + if (!opts.Parse(args, out_error)) { + return false; + } + + const auto apk_paths = FindApkFiles(input_directories, recursive, out_error); + if (!apk_paths) { + return false; + } + + std::vector<std::string> interesting_apks; + for (const std::string& path : *apk_paths) { + std::unique_ptr<const ZipFile> zip = ZipFile::Open(path); + if (!zip) { + out_error << "error: failed to open " << path << " as a zip file" << std::endl; + return false; + } + + std::unique_ptr<const MemoryChunk> entry = zip->Uncompress("AndroidManifest.xml"); + if (!entry) { + out_error << "error: failed to uncompress AndroidManifest.xml from " << path << std::endl; + return false; + } + + std::unique_ptr<const Xml> xml = Xml::Create(entry->buf, entry->size); + if (!xml) { + out_error << "error: failed to parse AndroidManifest.xml from " << path << std::endl; + continue; + } + + const auto tag = xml->FindTag("overlay"); + if (!tag) { + continue; + } + + auto iter = tag->find("isStatic"); + if (iter == tag->end() || std::stoul(iter->second) == 0u) { + continue; + } + + iter = tag->find("targetPackage"); + if (iter == tag->end() || iter->second != target_package_name) { + continue; + } + + iter = tag->find("priority"); + if (iter == tag->end()) { + continue; + } + + const int priority = std::stoi(iter->second); + if (priority < 0) { + continue; + } + + interesting_apks.insert( + std::lower_bound(interesting_apks.begin(), interesting_apks.end(), path), path); + } + + std::stringstream stream; + for (auto iter = interesting_apks.cbegin(); iter != interesting_apks.cend(); ++iter) { + const std::string idmap_path = Idmap::CanonicalIdmapPathFor(output_directory, *iter); + std::stringstream dev_null; + if (!Verify(std::vector<std::string>({"--idmap-path", idmap_path}), dev_null) && + !Create(std::vector<std::string>({ + "--target-apk-path", + target_apk_path, + "--overlay-apk-path", + *iter, + "--idmap-path", + idmap_path, + }), + out_error)) { + return false; + } + stream << idmap_path << std::endl; + } + + std::cout << stream.str(); + + return true; +} diff --git a/cmds/idmap2/idmap2/Verify.cpp b/cmds/idmap2/idmap2/Verify.cpp new file mode 100644 index 000000000000..b5fa438b5b9f --- /dev/null +++ b/cmds/idmap2/idmap2/Verify.cpp @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fstream> +#include <memory> +#include <string> +#include <vector> + +#include "idmap2/CommandLineOptions.h" +#include "idmap2/Idmap.h" + +using android::idmap2::CommandLineOptions; +using android::idmap2::IdmapHeader; + +bool Verify(const std::vector<std::string>& args, std::ostream& out_error) { + std::string idmap_path; + const CommandLineOptions opts = + CommandLineOptions("idmap2 verify") + .MandatoryOption("--idmap-path", "input: path to idmap file to verify", &idmap_path); + if (!opts.Parse(args, out_error)) { + return false; + } + + std::ifstream fin(idmap_path); + const std::unique_ptr<const IdmapHeader> header = IdmapHeader::FromBinaryStream(fin); + fin.close(); + if (!header) { + out_error << "error: failed to parse idmap header" << std::endl; + return false; + } + + return header->IsUpToDate(out_error); +} diff --git a/cmds/idmap2/idmap2d/Idmap2Service.cpp b/cmds/idmap2/idmap2d/Idmap2Service.cpp new file mode 100644 index 000000000000..cf72cb94da2c --- /dev/null +++ b/cmds/idmap2/idmap2d/Idmap2Service.cpp @@ -0,0 +1,138 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <sys/stat.h> // umask +#include <sys/types.h> // umask +#include <unistd.h> + +#include <cerrno> +#include <cstring> +#include <fstream> +#include <memory> +#include <ostream> +#include <string> + +#include "android-base/macros.h" +#include "utils/String8.h" +#include "utils/Trace.h" + +#include "idmap2/BinaryStreamVisitor.h" +#include "idmap2/FileUtils.h" +#include "idmap2/Idmap.h" + +#include "idmap2d/Idmap2Service.h" + +using android::binder::Status; +using android::idmap2::BinaryStreamVisitor; +using android::idmap2::Idmap; +using android::idmap2::IdmapHeader; + +namespace { + +static constexpr const char* kIdmapCacheDir = "/data/resource-cache"; + +Status ok() { + return Status::ok(); +} + +Status error(const std::string& msg) { + LOG(ERROR) << msg; + return Status::fromExceptionCode(Status::EX_NONE, msg.c_str()); +} + +} // namespace + +namespace android { +namespace os { + +Status Idmap2Service::getIdmapPath(const std::string& overlay_apk_path, + int32_t user_id ATTRIBUTE_UNUSED, std::string* _aidl_return) { + assert(_aidl_return); + *_aidl_return = Idmap::CanonicalIdmapPathFor(kIdmapCacheDir, overlay_apk_path); + return ok(); +} + +Status Idmap2Service::removeIdmap(const std::string& overlay_apk_path, + int32_t user_id ATTRIBUTE_UNUSED, bool* _aidl_return) { + assert(_aidl_return); + const std::string idmap_path = Idmap::CanonicalIdmapPathFor(kIdmapCacheDir, overlay_apk_path); + if (unlink(idmap_path.c_str()) == 0) { + *_aidl_return = true; + return ok(); + } else { + *_aidl_return = false; + return error("failed to unlink " + idmap_path + ": " + strerror(errno)); + } +} + +Status Idmap2Service::createIdmap(const std::string& target_apk_path, + const std::string& overlay_apk_path, int32_t user_id, + std::unique_ptr<std::string>* _aidl_return) { + assert(_aidl_return); + std::stringstream trace; + trace << __FUNCTION__ << " " << target_apk_path << " " << overlay_apk_path << " " + << std::to_string(user_id); + ATRACE_NAME(trace.str().c_str()); + std::cout << trace.str() << std::endl; + + _aidl_return->reset(nullptr); + + const std::string idmap_path = Idmap::CanonicalIdmapPathFor(kIdmapCacheDir, overlay_apk_path); + std::ifstream fin(idmap_path); + const std::unique_ptr<const IdmapHeader> header = IdmapHeader::FromBinaryStream(fin); + fin.close(); + // do not reuse error stream from IsUpToDate below, or error messages will be + // polluted with irrelevant data + std::stringstream dev_null; + if (header && header->IsUpToDate(dev_null)) { + return ok(); + } + + const std::unique_ptr<const ApkAssets> target_apk = ApkAssets::Load(target_apk_path); + if (!target_apk) { + return error("failed to load apk " + target_apk_path); + } + + const std::unique_ptr<const ApkAssets> overlay_apk = ApkAssets::Load(overlay_apk_path); + if (!overlay_apk) { + return error("failed to load apk " + overlay_apk_path); + } + + std::stringstream err; + const std::unique_ptr<const Idmap> idmap = + Idmap::FromApkAssets(target_apk_path, *target_apk, overlay_apk_path, *overlay_apk, err); + if (!idmap) { + return error(err.str()); + } + + umask(0133); // u=rw,g=r,o=r + std::ofstream fout(idmap_path); + if (fout.fail()) { + return error("failed to open idmap path " + idmap_path); + } + BinaryStreamVisitor visitor(fout); + idmap->accept(&visitor); + fout.close(); + if (fout.fail()) { + return error("failed to write to idmap path " + idmap_path); + } + + _aidl_return->reset(new std::string(idmap_path)); + return ok(); +} + +} // namespace os +} // namespace android diff --git a/cmds/idmap2/idmap2d/Idmap2Service.h b/cmds/idmap2/idmap2d/Idmap2Service.h new file mode 100644 index 000000000000..2b32042d6aa3 --- /dev/null +++ b/cmds/idmap2/idmap2d/Idmap2Service.h @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IDMAP2_IDMAP2D_IDMAP2SERVICE_H_ +#define IDMAP2_IDMAP2D_IDMAP2SERVICE_H_ + +#include <android-base/unique_fd.h> +#include <binder/BinderService.h> + +#include <memory> +#include <string> + +#include "android/os/BnIdmap2.h" + +namespace android { +namespace os { +class Idmap2Service : public BinderService<Idmap2Service>, public BnIdmap2 { + public: + static char const* getServiceName() { + return "idmap"; + } + + binder::Status getIdmapPath(const std::string& overlay_apk_path, int32_t user_id, + std::string* _aidl_return); + + binder::Status removeIdmap(const std::string& overlay_apk_path, int32_t user_id, + bool* _aidl_return); + + binder::Status createIdmap(const std::string& target_apk_path, + const std::string& overlay_apk_path, int32_t user_id, + std::unique_ptr<std::string>* _aidl_return); +}; +} // namespace os +} // namespace android + +#endif // IDMAP2_IDMAP2D_IDMAP2SERVICE_H_ diff --git a/cmds/idmap2/idmap2d/Main.cpp b/cmds/idmap2/idmap2d/Main.cpp new file mode 100644 index 000000000000..d64a87b8ee15 --- /dev/null +++ b/cmds/idmap2/idmap2d/Main.cpp @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define ATRACE_TAG ATRACE_TAG_RESOURCES + +#include <binder/BinderService.h> +#include <binder/IPCThreadState.h> +#include <binder/ProcessState.h> + +#include <cstdlib> // EXIT_{FAILURE,SUCCESS} + +#include <iostream> +#include <sstream> + +#include "android-base/macros.h" + +#include "Idmap2Service.h" + +using android::BinderService; +using android::IPCThreadState; +using android::ProcessState; +using android::sp; +using android::status_t; +using android::os::Idmap2Service; + +int main(int argc ATTRIBUTE_UNUSED, char** argv ATTRIBUTE_UNUSED) { + IPCThreadState::self()->disableBackgroundScheduling(true); + status_t ret = BinderService<Idmap2Service>::publish(); + if (ret != android::OK) { + return EXIT_FAILURE; + } + sp<ProcessState> ps(ProcessState::self()); + ps->startThreadPool(); + ps->giveThreadPoolName(); + IPCThreadState::self()->joinThreadPool(); + return EXIT_SUCCESS; +} diff --git a/cmds/idmap2/idmap2d/aidl/android/os/IIdmap2.aidl b/cmds/idmap2/idmap2d/aidl/android/os/IIdmap2.aidl new file mode 100644 index 000000000000..5d196101a7a6 --- /dev/null +++ b/cmds/idmap2/idmap2d/aidl/android/os/IIdmap2.aidl @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.os; + +/** + * @hide + */ +interface IIdmap2 { + @utf8InCpp String getIdmapPath(@utf8InCpp String overlayApkPath, int userId); + boolean removeIdmap(@utf8InCpp String overlayApkPath, int userId); + @nullable @utf8InCpp String createIdmap(@utf8InCpp String targetApkPath, + @utf8InCpp String overlayApkPath, int userId); +} diff --git a/cmds/idmap2/include/idmap2/BinaryStreamVisitor.h b/cmds/idmap2/include/idmap2/BinaryStreamVisitor.h new file mode 100644 index 000000000000..2368aeadbc9f --- /dev/null +++ b/cmds/idmap2/include/idmap2/BinaryStreamVisitor.h @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IDMAP2_INCLUDE_IDMAP2_BINARYSTREAMVISITOR_H_ +#define IDMAP2_INCLUDE_IDMAP2_BINARYSTREAMVISITOR_H_ + +#include <cstdint> +#include <iostream> +#include <string> + +#include "idmap2/Idmap.h" + +namespace android { +namespace idmap2 { + +class BinaryStreamVisitor : public Visitor { + public: + explicit BinaryStreamVisitor(std::ostream& stream) : stream_(stream) { + } + virtual void visit(const Idmap& idmap); + virtual void visit(const IdmapHeader& header); + virtual void visit(const IdmapData& data); + virtual void visit(const IdmapData::Header& header); + virtual void visit(const IdmapData::TypeEntry& type_entry); + + private: + void Write16(uint16_t value); + void Write32(uint32_t value); + void WriteString(const StringPiece& value); + std::ostream& stream_; +}; + +} // namespace idmap2 +} // namespace android + +#endif // IDMAP2_INCLUDE_IDMAP2_BINARYSTREAMVISITOR_H_ diff --git a/cmds/idmap2/include/idmap2/CommandLineOptions.h b/cmds/idmap2/include/idmap2/CommandLineOptions.h new file mode 100644 index 000000000000..f3aa68b8d1a2 --- /dev/null +++ b/cmds/idmap2/include/idmap2/CommandLineOptions.h @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IDMAP2_INCLUDE_IDMAP2_COMMANDLINEOPTIONS_H_ +#define IDMAP2_INCLUDE_IDMAP2_COMMANDLINEOPTIONS_H_ + +#include <functional> +#include <memory> +#include <ostream> +#include <string> +#include <vector> + +namespace android { +namespace idmap2 { + +/* + * Utility class to convert a command line, including options (--path foo.txt), + * into data structures (options.path = "foo.txt"). + */ +class CommandLineOptions { + public: + static std::unique_ptr<std::vector<std::string>> ConvertArgvToVector(int argc, const char** argv); + + explicit CommandLineOptions(const std::string& name) : name_(name) { + } + + CommandLineOptions& OptionalFlag(const std::string& name, const std::string& description, + bool* value); + CommandLineOptions& MandatoryOption(const std::string& name, const std::string& description, + std::string* value); + CommandLineOptions& MandatoryOption(const std::string& name, const std::string& description, + std::vector<std::string>* value); + CommandLineOptions& OptionalOption(const std::string& name, const std::string& description, + std::string* value); + bool Parse(const std::vector<std::string>& argv, std::ostream& outError) const; + void Usage(std::ostream& out) const; + + private: + struct Option { + std::string name; + std::string description; + std::function<void(const std::string& value)> action; + enum { + COUNT_OPTIONAL, + COUNT_EXACTLY_ONCE, + COUNT_ONCE_OR_MORE, + } count; + bool argument; + }; + + mutable std::vector<Option> options_; + std::string name_; +}; + +} // namespace idmap2 +} // namespace android + +#endif // IDMAP2_INCLUDE_IDMAP2_COMMANDLINEOPTIONS_H_ diff --git a/cmds/idmap2/include/idmap2/FileUtils.h b/cmds/idmap2/include/idmap2/FileUtils.h new file mode 100644 index 000000000000..05c6d31d395d --- /dev/null +++ b/cmds/idmap2/include/idmap2/FileUtils.h @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IDMAP2_INCLUDE_IDMAP2_FILEUTILS_H_ +#define IDMAP2_INCLUDE_IDMAP2_FILEUTILS_H_ + +#include <functional> +#include <memory> +#include <string> +#include <vector> + +namespace android { +namespace idmap2 { +namespace utils { +typedef std::function<bool(unsigned char type /* DT_* from dirent.h */, const std::string& path)> + FindFilesPredicate; +std::unique_ptr<std::vector<std::string>> FindFiles(const std::string& root, bool recurse, + const FindFilesPredicate& predicate); + +std::unique_ptr<std::string> ReadFile(int fd); + +std::unique_ptr<std::string> ReadFile(const std::string& path); + +} // namespace utils +} // namespace idmap2 +} // namespace android + +#endif // IDMAP2_INCLUDE_IDMAP2_FILEUTILS_H_ diff --git a/cmds/idmap2/include/idmap2/Idmap.h b/cmds/idmap2/include/idmap2/Idmap.h new file mode 100644 index 000000000000..837b7c5264d5 --- /dev/null +++ b/cmds/idmap2/include/idmap2/Idmap.h @@ -0,0 +1,277 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * # idmap file format (current version) + * + * idmap := header data* + * header := magic version target_crc overlay_crc target_path overlay_path + * data := data_header data_block* + * data_header := target_package_id types_count + * data_block := target_type overlay_type entry_count entry_offset entry* + * overlay_path := string + * target_path := string + * entry := <uint32_t> + * entry_count := <uint16_t> + * entry_offset := <uint16_t> + * magic := <uint32_t> + * overlay_crc := <uint32_t> + * overlay_type := <uint16_t> + * string := <uint8_t>[256] + * target_crc := <uint32_t> + * target_package_id := <uint16_t> + * target_type := <uint16_t> + * types_count := <uint16_t> + * version := <uint32_t> + * + * + * # idmap file format changelog + * ## v1 + * - Identical to idmap v1. + */ + +#ifndef IDMAP2_INCLUDE_IDMAP2_IDMAP_H_ +#define IDMAP2_INCLUDE_IDMAP2_IDMAP_H_ + +#include <iostream> +#include <memory> +#include <string> +#include <vector> + +#include "android-base/macros.h" + +#include "androidfw/ApkAssets.h" +#include "androidfw/ResourceTypes.h" +#include "androidfw/StringPiece.h" + +namespace android { +namespace idmap2 { + +class Idmap; +class Visitor; + +// use typedefs to let the compiler warn us about implicit casts +typedef uint32_t ResourceId; // 0xpptteeee +typedef uint8_t PackageId; // pp in 0xpptteeee +typedef uint8_t TypeId; // tt in 0xpptteeee +typedef uint16_t EntryId; // eeee in 0xpptteeee + +static constexpr const ResourceId kPadding = 0xffffffffu; + +static constexpr const EntryId kNoEntry = 0xffffu; + +// magic number: all idmap files start with this +static constexpr const uint32_t kIdmapMagic = android::kIdmapMagic; + +// current version of the idmap binary format; must be incremented when the format is changed +static constexpr const uint32_t kIdmapCurrentVersion = android::kIdmapCurrentVersion; + +// strings in the idmap are encoded char arrays of length 'kIdmapStringLength' (including mandatory +// terminating null) +static constexpr const size_t kIdmapStringLength = 256; + +class IdmapHeader { + public: + static std::unique_ptr<const IdmapHeader> FromBinaryStream(std::istream& stream); + + inline uint32_t GetMagic() const { + return magic_; + } + + inline uint32_t GetVersion() const { + return version_; + } + + inline uint32_t GetTargetCrc() const { + return target_crc_; + } + + inline uint32_t GetOverlayCrc() const { + return overlay_crc_; + } + + inline StringPiece GetTargetPath() const { + return StringPiece(target_path_); + } + + inline StringPiece GetOverlayPath() const { + return StringPiece(overlay_path_); + } + + // Invariant: anytime the idmap data encoding is changed, the idmap version + // field *must* be incremented. Because of this, we know that if the idmap + // header is up-to-date the entire file is up-to-date. + bool IsUpToDate(std::ostream& out_error) const; + + void accept(Visitor* v) const; + + private: + IdmapHeader() { + } + + uint32_t magic_; + uint32_t version_; + uint32_t target_crc_; + uint32_t overlay_crc_; + char target_path_[kIdmapStringLength]; + char overlay_path_[kIdmapStringLength]; + + friend Idmap; + DISALLOW_COPY_AND_ASSIGN(IdmapHeader); +}; + +class IdmapData { + public: + class Header { + public: + static std::unique_ptr<const Header> FromBinaryStream(std::istream& stream); + + inline PackageId GetTargetPackageId() const { + return target_package_id_; + } + + inline uint16_t GetTypeCount() const { + return type_count_; + } + + void accept(Visitor* v) const; + + private: + Header() { + } + + PackageId target_package_id_; + uint16_t type_count_; + + friend Idmap; + DISALLOW_COPY_AND_ASSIGN(Header); + }; + + class TypeEntry { + public: + static std::unique_ptr<const TypeEntry> FromBinaryStream(std::istream& stream); + + inline TypeId GetTargetTypeId() const { + return target_type_id_; + } + + inline TypeId GetOverlayTypeId() const { + return overlay_type_id_; + } + + inline uint16_t GetEntryCount() const { + return entries_.size(); + } + + inline uint16_t GetEntryOffset() const { + return entry_offset_; + } + + inline EntryId GetEntry(size_t i) const { + return i < entries_.size() ? entries_[i] : 0xffffu; + } + + void accept(Visitor* v) const; + + private: + TypeEntry() { + } + + TypeId target_type_id_; + TypeId overlay_type_id_; + uint16_t entry_offset_; + std::vector<EntryId> entries_; + + friend Idmap; + DISALLOW_COPY_AND_ASSIGN(TypeEntry); + }; + + static std::unique_ptr<const IdmapData> FromBinaryStream(std::istream& stream); + + inline const std::unique_ptr<const Header>& GetHeader() const { + return header_; + } + + inline const std::vector<std::unique_ptr<const TypeEntry>>& GetTypeEntries() const { + return type_entries_; + } + + void accept(Visitor* v) const; + + private: + IdmapData() { + } + + std::unique_ptr<const Header> header_; + std::vector<std::unique_ptr<const TypeEntry>> type_entries_; + + friend Idmap; + DISALLOW_COPY_AND_ASSIGN(IdmapData); +}; + +class Idmap { + public: + static std::string CanonicalIdmapPathFor(const std::string& absolute_dir, + const std::string& absolute_apk_path); + + static std::unique_ptr<const Idmap> FromBinaryStream(std::istream& stream, + std::ostream& out_error); + + // In the current version of idmap, the first package in each resources.arsc + // file is used; change this in the next version of idmap to use a named + // package instead; also update FromApkAssets to take additional parameters: + // the target and overlay package names + static std::unique_ptr<const Idmap> FromApkAssets(const std::string& target_apk_path, + const ApkAssets& target_apk_assets, + const std::string& overlay_apk_path, + const ApkAssets& overlay_apk_assets, + std::ostream& out_error); + + inline const std::unique_ptr<const IdmapHeader>& GetHeader() const { + return header_; + } + + inline const std::vector<std::unique_ptr<const IdmapData>>& GetData() const { + return data_; + } + + void accept(Visitor* v) const; + + private: + Idmap() { + } + + std::unique_ptr<const IdmapHeader> header_; + std::vector<std::unique_ptr<const IdmapData>> data_; + + DISALLOW_COPY_AND_ASSIGN(Idmap); +}; + +class Visitor { + public: + virtual ~Visitor() { + } + virtual void visit(const Idmap& idmap) = 0; + virtual void visit(const IdmapHeader& header) = 0; + virtual void visit(const IdmapData& data) = 0; + virtual void visit(const IdmapData::Header& header) = 0; + virtual void visit(const IdmapData::TypeEntry& type_entry) = 0; +}; + +} // namespace idmap2 +} // namespace android + +#endif // IDMAP2_INCLUDE_IDMAP2_IDMAP_H_ diff --git a/cmds/idmap2/include/idmap2/PrettyPrintVisitor.h b/cmds/idmap2/include/idmap2/PrettyPrintVisitor.h new file mode 100644 index 000000000000..c388f4b94251 --- /dev/null +++ b/cmds/idmap2/include/idmap2/PrettyPrintVisitor.h @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IDMAP2_INCLUDE_IDMAP2_PRETTYPRINTVISITOR_H_ +#define IDMAP2_INCLUDE_IDMAP2_PRETTYPRINTVISITOR_H_ + +#include <iostream> +#include <memory> + +#include "androidfw/AssetManager2.h" + +#include "idmap2/Idmap.h" + +namespace android { + +class ApkAssets; + +namespace idmap2 { + +class PrettyPrintVisitor : public Visitor { + public: + explicit PrettyPrintVisitor(std::ostream& stream) : stream_(stream) { + } + virtual void visit(const Idmap& idmap); + virtual void visit(const IdmapHeader& header); + virtual void visit(const IdmapData& data); + virtual void visit(const IdmapData::Header& header); + virtual void visit(const IdmapData::TypeEntry& type_entry); + + private: + std::ostream& stream_; + std::unique_ptr<const ApkAssets> target_apk_; + AssetManager2 target_am_; + PackageId last_seen_package_id_; +}; + +} // namespace idmap2 +} // namespace android + +#endif // IDMAP2_INCLUDE_IDMAP2_PRETTYPRINTVISITOR_H_ diff --git a/cmds/idmap2/include/idmap2/RawPrintVisitor.h b/cmds/idmap2/include/idmap2/RawPrintVisitor.h new file mode 100644 index 000000000000..7e33b3b06fc3 --- /dev/null +++ b/cmds/idmap2/include/idmap2/RawPrintVisitor.h @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IDMAP2_INCLUDE_IDMAP2_RAWPRINTVISITOR_H_ +#define IDMAP2_INCLUDE_IDMAP2_RAWPRINTVISITOR_H_ + +#include <iostream> +#include <memory> +#include <string> + +#include "androidfw/AssetManager2.h" + +#include "idmap2/Idmap.h" + +namespace android { + +class ApkAssets; + +namespace idmap2 { + +class RawPrintVisitor : public Visitor { + public: + explicit RawPrintVisitor(std::ostream& stream) : stream_(stream), offset_(0) { + } + virtual void visit(const Idmap& idmap); + virtual void visit(const IdmapHeader& header); + virtual void visit(const IdmapData& data); + virtual void visit(const IdmapData::Header& header); + virtual void visit(const IdmapData::TypeEntry& type_entry); + + private: + void print(uint16_t value, const char* fmt, ...); + void print(uint32_t value, const char* fmt, ...); + void print(const std::string& value, const char* fmt, ...); + + std::ostream& stream_; + std::unique_ptr<const ApkAssets> target_apk_; + AssetManager2 target_am_; + size_t offset_; + PackageId last_seen_package_id_; +}; + +} // namespace idmap2 +} // namespace android + +#endif // IDMAP2_INCLUDE_IDMAP2_RAWPRINTVISITOR_H_ diff --git a/cmds/idmap2/include/idmap2/ResourceUtils.h b/cmds/idmap2/include/idmap2/ResourceUtils.h new file mode 100644 index 000000000000..88a835b6439c --- /dev/null +++ b/cmds/idmap2/include/idmap2/ResourceUtils.h @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IDMAP2_INCLUDE_IDMAP2_RESOURCEUTILS_H_ +#define IDMAP2_INCLUDE_IDMAP2_RESOURCEUTILS_H_ + +#include <string> +#include <utility> + +#include "android-base/macros.h" +#include "androidfw/AssetManager2.h" + +#include "idmap2/Idmap.h" + +namespace android { +namespace idmap2 { +namespace utils { + +std::pair<bool, std::string> WARN_UNUSED ResToTypeEntryName(const AssetManager2& am, + ResourceId resid); + +} // namespace utils +} // namespace idmap2 +} // namespace android + +#endif // IDMAP2_INCLUDE_IDMAP2_RESOURCEUTILS_H_ diff --git a/cmds/idmap2/include/idmap2/Xml.h b/cmds/idmap2/include/idmap2/Xml.h new file mode 100644 index 000000000000..9ab5ec454750 --- /dev/null +++ b/cmds/idmap2/include/idmap2/Xml.h @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IDMAP2_INCLUDE_IDMAP2_XML_H_ +#define IDMAP2_INCLUDE_IDMAP2_XML_H_ + +#include <map> +#include <memory> +#include <string> + +#include "android-base/macros.h" +#include "androidfw/ResourceTypes.h" +#include "utils/String16.h" + +namespace android { +namespace idmap2 { + +class Xml { + public: + static std::unique_ptr<const Xml> Create(const uint8_t* data, size_t size, bool copyData = false); + + std::unique_ptr<std::map<std::string, std::string>> FindTag(const std::string& name) const; + + ~Xml(); + + private: + Xml() { + } + + mutable ResXMLTree xml_; + + DISALLOW_COPY_AND_ASSIGN(Xml); +}; + +} // namespace idmap2 +} // namespace android + +#endif // IDMAP2_INCLUDE_IDMAP2_XML_H_ diff --git a/cmds/idmap2/include/idmap2/ZipFile.h b/cmds/idmap2/include/idmap2/ZipFile.h new file mode 100644 index 000000000000..328bd367adfc --- /dev/null +++ b/cmds/idmap2/include/idmap2/ZipFile.h @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IDMAP2_INCLUDE_IDMAP2_ZIPFILE_H_ +#define IDMAP2_INCLUDE_IDMAP2_ZIPFILE_H_ + +#include <memory> +#include <string> +#include <utility> + +#include "android-base/macros.h" +#include "ziparchive/zip_archive.h" + +namespace android { +namespace idmap2 { + +struct MemoryChunk { + size_t size; + uint8_t buf[0]; + + static std::unique_ptr<MemoryChunk> Allocate(size_t size); + + private: + MemoryChunk() { + } +}; + +class ZipFile { + public: + static std::unique_ptr<const ZipFile> Open(const std::string& path); + + std::unique_ptr<const MemoryChunk> Uncompress(const std::string& entryPath) const; + std::pair<bool, uint32_t> Crc(const std::string& entryPath) const; + + ~ZipFile(); + + private: + explicit ZipFile(const ::ZipArchiveHandle handle) : handle_(handle) { + } + + const ::ZipArchiveHandle handle_; + + DISALLOW_COPY_AND_ASSIGN(ZipFile); +}; + +} // namespace idmap2 +} // namespace android + +#endif // IDMAP2_INCLUDE_IDMAP2_ZIPFILE_H_ diff --git a/cmds/idmap2/libidmap2/BinaryStreamVisitor.cpp b/cmds/idmap2/libidmap2/BinaryStreamVisitor.cpp new file mode 100644 index 000000000000..29969a23250b --- /dev/null +++ b/cmds/idmap2/libidmap2/BinaryStreamVisitor.cpp @@ -0,0 +1,81 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <algorithm> +#include <cstring> +#include <string> + +#include "android-base/macros.h" + +#include "idmap2/BinaryStreamVisitor.h" + +namespace android { +namespace idmap2 { + +void BinaryStreamVisitor::Write16(uint16_t value) { + uint16_t x = htodl(value); + stream_.write(reinterpret_cast<char*>(&x), sizeof(uint16_t)); +} + +void BinaryStreamVisitor::Write32(uint32_t value) { + uint32_t x = htodl(value); + stream_.write(reinterpret_cast<char*>(&x), sizeof(uint32_t)); +} + +void BinaryStreamVisitor::WriteString(const StringPiece& value) { + char buf[kIdmapStringLength]; + memset(buf, 0, sizeof(buf)); + memcpy(buf, value.data(), std::min(value.size(), sizeof(buf))); + stream_.write(buf, sizeof(buf)); +} + +void BinaryStreamVisitor::visit(const Idmap& idmap ATTRIBUTE_UNUSED) { + // nothing to do +} + +void BinaryStreamVisitor::visit(const IdmapHeader& header) { + Write32(header.GetMagic()); + Write32(header.GetVersion()); + Write32(header.GetTargetCrc()); + Write32(header.GetOverlayCrc()); + WriteString(header.GetTargetPath()); + WriteString(header.GetOverlayPath()); +} + +void BinaryStreamVisitor::visit(const IdmapData& data ATTRIBUTE_UNUSED) { + // nothing to do +} + +void BinaryStreamVisitor::visit(const IdmapData::Header& header) { + Write16(header.GetTargetPackageId()); + Write16(header.GetTypeCount()); +} + +void BinaryStreamVisitor::visit(const IdmapData::TypeEntry& te) { + const uint16_t entryCount = te.GetEntryCount(); + + Write16(te.GetTargetTypeId()); + Write16(te.GetOverlayTypeId()); + Write16(entryCount); + Write16(te.GetEntryOffset()); + for (uint16_t i = 0; i < entryCount; i++) { + EntryId entry_id = te.GetEntry(i); + Write32(entry_id != kNoEntry ? static_cast<uint32_t>(entry_id) : kPadding); + } +} + +} // namespace idmap2 +} // namespace android diff --git a/cmds/idmap2/libidmap2/CommandLineOptions.cpp b/cmds/idmap2/libidmap2/CommandLineOptions.cpp new file mode 100644 index 000000000000..28c3797226e1 --- /dev/null +++ b/cmds/idmap2/libidmap2/CommandLineOptions.cpp @@ -0,0 +1,163 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <algorithm> +#include <iomanip> +#include <iostream> +#include <memory> +#include <set> +#include <string> +#include <vector> + +#include "android-base/macros.h" + +#include "idmap2/CommandLineOptions.h" + +namespace android { +namespace idmap2 { + +std::unique_ptr<std::vector<std::string>> CommandLineOptions::ConvertArgvToVector( + int argc, const char** argv) { + return std::unique_ptr<std::vector<std::string>>( + new std::vector<std::string>(argv + 1, argv + argc)); +} + +CommandLineOptions& CommandLineOptions::OptionalFlag(const std::string& name, + const std::string& description, bool* value) { + assert(value != nullptr); + auto func = [value](const std::string& arg ATTRIBUTE_UNUSED) -> void { *value = true; }; + options_.push_back(Option{name, description, func, Option::COUNT_OPTIONAL, false}); + return *this; +} + +CommandLineOptions& CommandLineOptions::MandatoryOption(const std::string& name, + const std::string& description, + std::string* value) { + assert(value != nullptr); + auto func = [value](const std::string& arg) -> void { *value = arg; }; + options_.push_back(Option{name, description, func, Option::COUNT_EXACTLY_ONCE, true}); + return *this; +} + +CommandLineOptions& CommandLineOptions::MandatoryOption(const std::string& name, + const std::string& description, + std::vector<std::string>* value) { + assert(value != nullptr); + auto func = [value](const std::string& arg) -> void { value->push_back(arg); }; + options_.push_back(Option{name, description, func, Option::COUNT_ONCE_OR_MORE, true}); + return *this; +} + +CommandLineOptions& CommandLineOptions::OptionalOption(const std::string& name, + const std::string& description, + std::string* value) { + assert(value != nullptr); + auto func = [value](const std::string& arg) -> void { *value = arg; }; + options_.push_back(Option{name, description, func, Option::COUNT_OPTIONAL, true}); + return *this; +} + +bool CommandLineOptions::Parse(const std::vector<std::string>& argv, std::ostream& outError) const { + const auto pivot = std::partition(options_.begin(), options_.end(), [](const Option& opt) { + return opt.count != Option::COUNT_OPTIONAL; + }); + std::set<std::string> mandatory_opts; + std::transform(options_.begin(), pivot, std::inserter(mandatory_opts, mandatory_opts.end()), + [](const Option& opt) -> std::string { return opt.name; }); + + const size_t argv_size = argv.size(); + for (size_t i = 0; i < argv_size; i++) { + const std::string arg = argv[i]; + if ("--help" == arg || "-h" == arg) { + Usage(outError); + return false; + } + bool match = false; + for (const Option& opt : options_) { + if (opt.name == arg) { + match = true; + + if (opt.argument) { + i++; + if (i >= argv_size) { + outError << "error: " << opt.name << ": missing argument" << std::endl; + Usage(outError); + return false; + } + } + opt.action(argv[i]); + mandatory_opts.erase(opt.name); + break; + } + } + if (!match) { + outError << "error: " << arg << ": unknown option" << std::endl; + Usage(outError); + return false; + } + } + + if (!mandatory_opts.empty()) { + for (auto iter = mandatory_opts.cbegin(); iter != mandatory_opts.cend(); ++iter) { + outError << "error: " << *iter << ": missing mandatory option" << std::endl; + } + Usage(outError); + return false; + } + return true; +} + +void CommandLineOptions::Usage(std::ostream& out) const { + size_t maxLength = 0; + out << "usage: " << name_; + for (const Option& opt : options_) { + const bool mandatory = opt.count != Option::COUNT_OPTIONAL; + out << " "; + if (!mandatory) { + out << "["; + } + if (opt.argument) { + out << opt.name << " arg"; + maxLength = std::max(maxLength, opt.name.size() + 4); + } else { + out << opt.name; + maxLength = std::max(maxLength, opt.name.size()); + } + if (!mandatory) { + out << "]"; + } + if (opt.count == Option::COUNT_ONCE_OR_MORE) { + out << " [" << opt.name << " arg [..]]"; + } + } + out << std::endl << std::endl; + for (const Option& opt : options_) { + out << std::left << std::setw(maxLength); + if (opt.argument) { + out << (opt.name + " arg"); + } else { + out << opt.name; + } + out << " " << opt.description; + if (opt.count == Option::COUNT_ONCE_OR_MORE) { + out << " (can be provided multiple times)"; + } + out << std::endl; + } +} + +} // namespace idmap2 +} // namespace android diff --git a/cmds/idmap2/libidmap2/FileUtils.cpp b/cmds/idmap2/libidmap2/FileUtils.cpp new file mode 100644 index 000000000000..4ac4c04d0bfc --- /dev/null +++ b/cmds/idmap2/libidmap2/FileUtils.cpp @@ -0,0 +1,82 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <dirent.h> +#include <errno.h> +#include <sys/types.h> +#include <unistd.h> + +#include <fstream> +#include <memory> +#include <string> +#include <utility> +#include <vector> + +#include "idmap2/FileUtils.h" + +namespace android { +namespace idmap2 { +namespace utils { + +std::unique_ptr<std::vector<std::string>> FindFiles(const std::string& root, bool recurse, + const FindFilesPredicate& predicate) { + DIR* dir = opendir(root.c_str()); + if (!dir) { + return nullptr; + } + std::unique_ptr<std::vector<std::string>> vector(new std::vector<std::string>()); + struct dirent* dirent; + while ((dirent = readdir(dir))) { + const std::string path = root + "/" + dirent->d_name; + if (predicate(dirent->d_type, path)) { + vector->push_back(path); + } + if (recurse && dirent->d_type == DT_DIR && strcmp(dirent->d_name, ".") != 0 && + strcmp(dirent->d_name, "..") != 0) { + auto sub_vector = FindFiles(path, recurse, predicate); + if (!sub_vector) { + closedir(dir); + return nullptr; + } + vector->insert(vector->end(), sub_vector->begin(), sub_vector->end()); + } + } + closedir(dir); + + return vector; +} + +std::unique_ptr<std::string> ReadFile(const std::string& path) { + std::unique_ptr<std::string> str(new std::string()); + std::ifstream fin(path); + str->append({std::istreambuf_iterator<char>(fin), std::istreambuf_iterator<char>()}); + fin.close(); + return str; +} + +std::unique_ptr<std::string> ReadFile(int fd) { + std::unique_ptr<std::string> str(new std::string()); + char buf[1024]; + ssize_t r; + while ((r = read(fd, buf, sizeof(buf))) > 0) { + str->append(buf, r); + } + return r == 0 ? std::move(str) : nullptr; +} + +} // namespace utils +} // namespace idmap2 +} // namespace android diff --git a/cmds/idmap2/libidmap2/Idmap.cpp b/cmds/idmap2/libidmap2/Idmap.cpp new file mode 100644 index 000000000000..5a47e301b66c --- /dev/null +++ b/cmds/idmap2/libidmap2/Idmap.cpp @@ -0,0 +1,443 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <algorithm> +#include <iostream> +#include <iterator> +#include <limits> +#include <map> +#include <memory> +#include <set> +#include <string> +#include <utility> +#include <vector> + +#include "android-base/macros.h" +#include "android-base/stringprintf.h" +#include "androidfw/AssetManager2.h" +#include "utils/String16.h" +#include "utils/String8.h" + +#include "idmap2/Idmap.h" +#include "idmap2/ResourceUtils.h" +#include "idmap2/ZipFile.h" + +namespace android { +namespace idmap2 { + +#define EXTRACT_TYPE(resid) ((0x00ff0000 & (resid)) >> 16) + +#define EXTRACT_ENTRY(resid) (0x0000ffff & (resid)) + +struct MatchingResources { + void Add(ResourceId target_resid, ResourceId overlay_resid) { + TypeId target_typeid = EXTRACT_TYPE(target_resid); + if (map.find(target_typeid) == map.end()) { + map.emplace(target_typeid, std::set<std::pair<ResourceId, ResourceId>>()); + } + map[target_typeid].insert(std::make_pair(target_resid, overlay_resid)); + } + + // target type id -> set { pair { overlay entry id, overlay entry id } } + std::map<TypeId, std::set<std::pair<ResourceId, ResourceId>>> map; +}; + +static bool WARN_UNUSED Read16(std::istream& stream, uint16_t* out) { + uint16_t value; + if (stream.read(reinterpret_cast<char*>(&value), sizeof(uint16_t))) { + *out = dtohl(value); + return true; + } + return false; +} + +static bool WARN_UNUSED Read32(std::istream& stream, uint32_t* out) { + uint32_t value; + if (stream.read(reinterpret_cast<char*>(&value), sizeof(uint32_t))) { + *out = dtohl(value); + return true; + } + return false; +} + +// a string is encoded as a kIdmapStringLength char array; the array is always null-terminated +static bool WARN_UNUSED ReadString(std::istream& stream, char out[kIdmapStringLength]) { + char buf[kIdmapStringLength]; + memset(buf, 0, sizeof(buf)); + if (!stream.read(buf, sizeof(buf))) { + return false; + } + if (buf[sizeof(buf) - 1] != '\0') { + return false; + } + memcpy(out, buf, sizeof(buf)); + return true; +} + +static ResourceId NameToResid(const AssetManager2& am, const std::string& name) { + return am.GetResourceId(name); +} + +// TODO(martenkongstad): scan for package name instead of assuming package at index 0 +// +// idmap version 0x01 naively assumes that the package to use is always the first ResTable_package +// in the resources.arsc blob. In most cases, there is only a single ResTable_package anyway, so +// this assumption tends to work out. That said, the correct thing to do is to scan +// resources.arsc for a package with a given name as read from the package manifest instead of +// relying on a hard-coded index. This however requires storing the package name in the idmap +// header, which in turn requires incrementing the idmap version. Because the initial version of +// idmap2 is compatible with idmap, this will have to wait for now. +static const LoadedPackage* GetPackageAtIndex0(const LoadedArsc& loaded_arsc) { + const std::vector<std::unique_ptr<const LoadedPackage>>& packages = loaded_arsc.GetPackages(); + if (packages.empty()) { + return nullptr; + } + int id = packages[0]->GetPackageId(); + return loaded_arsc.GetPackageById(id); +} + +std::unique_ptr<const IdmapHeader> IdmapHeader::FromBinaryStream(std::istream& stream) { + std::unique_ptr<IdmapHeader> idmap_header(new IdmapHeader()); + + if (!Read32(stream, &idmap_header->magic_) || !Read32(stream, &idmap_header->version_) || + !Read32(stream, &idmap_header->target_crc_) || !Read32(stream, &idmap_header->overlay_crc_) || + !ReadString(stream, idmap_header->target_path_) || + !ReadString(stream, idmap_header->overlay_path_)) { + return nullptr; + } + + return std::move(idmap_header); +} + +bool IdmapHeader::IsUpToDate(std::ostream& out_error) const { + if (magic_ != kIdmapMagic) { + out_error << base::StringPrintf("error: bad magic: actual 0x%08x, expected 0x%08x", magic_, + kIdmapMagic) + << std::endl; + return false; + } + + if (version_ != kIdmapCurrentVersion) { + out_error << base::StringPrintf("error: bad version: actual 0x%08x, expected 0x%08x", version_, + kIdmapCurrentVersion) + << std::endl; + return false; + } + + const std::unique_ptr<const ZipFile> target_zip = ZipFile::Open(target_path_); + if (!target_zip) { + out_error << "error: failed to open target " << target_path_ << std::endl; + return false; + } + + bool status; + uint32_t target_crc; + std::tie(status, target_crc) = target_zip->Crc("resources.arsc"); + if (!status) { + out_error << "error: failed to get target crc" << std::endl; + return false; + } + + if (target_crc_ != target_crc) { + out_error << base::StringPrintf( + "error: bad target crc: idmap version 0x%08x, file system version 0x%08x", + target_crc_, target_crc) + << std::endl; + return false; + } + + const std::unique_ptr<const ZipFile> overlay_zip = ZipFile::Open(overlay_path_); + if (!overlay_zip) { + out_error << "error: failed to open overlay " << overlay_path_ << std::endl; + return false; + } + + uint32_t overlay_crc; + std::tie(status, overlay_crc) = overlay_zip->Crc("resources.arsc"); + if (!status) { + out_error << "error: failed to get overlay crc" << std::endl; + return false; + } + + if (overlay_crc_ != overlay_crc) { + out_error << base::StringPrintf( + "error: bad overlay crc: idmap version 0x%08x, file system version 0x%08x", + overlay_crc_, overlay_crc) + << std::endl; + return false; + } + + return true; +} + +std::unique_ptr<const IdmapData::Header> IdmapData::Header::FromBinaryStream(std::istream& stream) { + std::unique_ptr<IdmapData::Header> idmap_data_header(new IdmapData::Header()); + + uint16_t target_package_id16; + if (!Read16(stream, &target_package_id16) || !Read16(stream, &idmap_data_header->type_count_)) { + return nullptr; + } + idmap_data_header->target_package_id_ = target_package_id16; + + return std::move(idmap_data_header); +} + +std::unique_ptr<const IdmapData::TypeEntry> IdmapData::TypeEntry::FromBinaryStream( + std::istream& stream) { + std::unique_ptr<IdmapData::TypeEntry> data(new IdmapData::TypeEntry()); + + uint16_t target_type16, overlay_type16, entry_count; + if (!Read16(stream, &target_type16) || !Read16(stream, &overlay_type16) || + !Read16(stream, &entry_count) || !Read16(stream, &data->entry_offset_)) { + return nullptr; + } + data->target_type_id_ = target_type16; + data->overlay_type_id_ = overlay_type16; + for (uint16_t i = 0; i < entry_count; i++) { + ResourceId resid; + if (!Read32(stream, &resid)) { + return nullptr; + } + data->entries_.push_back(resid); + } + + return std::move(data); +} + +std::unique_ptr<const IdmapData> IdmapData::FromBinaryStream(std::istream& stream) { + std::unique_ptr<IdmapData> data(new IdmapData()); + data->header_ = IdmapData::Header::FromBinaryStream(stream); + if (!data->header_) { + return nullptr; + } + for (size_t type_count = 0; type_count < data->header_->GetTypeCount(); type_count++) { + std::unique_ptr<const TypeEntry> type = IdmapData::TypeEntry::FromBinaryStream(stream); + if (!type) { + return nullptr; + } + data->type_entries_.push_back(std::move(type)); + } + return std::move(data); +} + +std::string Idmap::CanonicalIdmapPathFor(const std::string& absolute_dir, + const std::string& absolute_apk_path) { + assert(absolute_dir.size() > 0 && absolute_dir[0] == "/"); + assert(absolute_apk_path.size() > 0 && absolute_apk_path[0] == "/"); + std::string copy(++absolute_apk_path.cbegin(), absolute_apk_path.cend()); + replace(copy.begin(), copy.end(), '/', '@'); + return absolute_dir + "/" + copy + "@idmap"; +} + +std::unique_ptr<const Idmap> Idmap::FromBinaryStream(std::istream& stream, + std::ostream& out_error) { + std::unique_ptr<Idmap> idmap(new Idmap()); + + idmap->header_ = IdmapHeader::FromBinaryStream(stream); + if (!idmap->header_) { + out_error << "error: failed to parse idmap header" << std::endl; + return nullptr; + } + + // idmap version 0x01 does not specify the number of data blocks that follow + // the idmap header; assume exactly one data block + for (int i = 0; i < 1; i++) { + std::unique_ptr<const IdmapData> data = IdmapData::FromBinaryStream(stream); + if (!data) { + out_error << "error: failed to parse data block " << i << std::endl; + return nullptr; + } + idmap->data_.push_back(std::move(data)); + } + + return std::move(idmap); +} + +std::unique_ptr<const Idmap> Idmap::FromApkAssets(const std::string& target_apk_path, + const ApkAssets& target_apk_assets, + const std::string& overlay_apk_path, + const ApkAssets& overlay_apk_assets, + std::ostream& out_error) { + AssetManager2 target_asset_manager; + if (!target_asset_manager.SetApkAssets({&target_apk_assets}, true, false)) { + out_error << "error: failed to create target asset manager" << std::endl; + return nullptr; + } + + AssetManager2 overlay_asset_manager; + if (!overlay_asset_manager.SetApkAssets({&overlay_apk_assets}, true, false)) { + out_error << "error: failed to create overlay asset manager" << std::endl; + return nullptr; + } + + const LoadedArsc* target_arsc = target_apk_assets.GetLoadedArsc(); + if (!target_arsc) { + out_error << "error: failed to load target resources.arsc" << std::endl; + return nullptr; + } + + const LoadedArsc* overlay_arsc = overlay_apk_assets.GetLoadedArsc(); + if (!overlay_arsc) { + out_error << "error: failed to load overlay resources.arsc" << std::endl; + return nullptr; + } + + const LoadedPackage* target_pkg = GetPackageAtIndex0(*target_arsc); + if (!target_pkg) { + out_error << "error: failed to load target package from resources.arsc" << std::endl; + return nullptr; + } + + const LoadedPackage* overlay_pkg = GetPackageAtIndex0(*overlay_arsc); + if (!overlay_pkg) { + out_error << "error: failed to load overlay package from resources.arsc" << std::endl; + return nullptr; + } + + const std::unique_ptr<const ZipFile> target_zip = ZipFile::Open(target_apk_path); + if (!target_zip) { + out_error << "error: failed to open target as zip" << std::endl; + return nullptr; + } + + const std::unique_ptr<const ZipFile> overlay_zip = ZipFile::Open(overlay_apk_path); + if (!overlay_zip) { + out_error << "error: failed to open overlay as zip" << std::endl; + return nullptr; + } + + std::unique_ptr<IdmapHeader> header(new IdmapHeader()); + header->magic_ = kIdmapMagic; + header->version_ = kIdmapCurrentVersion; + bool crc_status; + std::tie(crc_status, header->target_crc_) = target_zip->Crc("resources.arsc"); + if (!crc_status) { + out_error << "error: failed to get zip crc for target" << std::endl; + return nullptr; + } + std::tie(crc_status, header->overlay_crc_) = overlay_zip->Crc("resources.arsc"); + if (!crc_status) { + out_error << "error: failed to get zip crc for overlay" << std::endl; + return nullptr; + } + + if (target_apk_path.size() > sizeof(header->target_path_)) { + out_error << "error: target apk path \"" << target_apk_path << "\" longer that maximum size " + << sizeof(header->target_path_) << std::endl; + return nullptr; + } + memset(header->target_path_, 0, sizeof(header->target_path_)); + memcpy(header->target_path_, target_apk_path.data(), target_apk_path.size()); + + if (overlay_apk_path.size() > sizeof(header->overlay_path_)) { + out_error << "error: overlay apk path \"" << overlay_apk_path << "\" longer that maximum size " + << sizeof(header->overlay_path_) << std::endl; + return nullptr; + } + memset(header->overlay_path_, 0, sizeof(header->overlay_path_)); + memcpy(header->overlay_path_, overlay_apk_path.data(), overlay_apk_path.size()); + + std::unique_ptr<Idmap> idmap(new Idmap()); + idmap->header_ = std::move(header); + + // find the resources that exist in both packages + MatchingResources matching_resources; + const auto end = overlay_pkg->end(); + for (auto iter = overlay_pkg->begin(); iter != end; ++iter) { + const ResourceId overlay_resid = *iter; + bool lookup_ok; + std::string name; + std::tie(lookup_ok, name) = utils::ResToTypeEntryName(overlay_asset_manager, overlay_resid); + if (!lookup_ok) { + continue; + } + // prepend "<package>:" to turn name into "<package>:<type>/<name>" + name = base::StringPrintf("%s:%s", target_pkg->GetPackageName().c_str(), name.c_str()); + const ResourceId target_resid = NameToResid(target_asset_manager, name); + if (target_resid == 0) { + continue; + } + matching_resources.Add(target_resid, overlay_resid); + } + + // encode idmap data + std::unique_ptr<IdmapData> data(new IdmapData()); + const auto types_end = matching_resources.map.cend(); + for (auto ti = matching_resources.map.cbegin(); ti != types_end; ++ti) { + auto ei = ti->second.cbegin(); + std::unique_ptr<IdmapData::TypeEntry> type(new IdmapData::TypeEntry()); + type->target_type_id_ = EXTRACT_TYPE(ei->first); + type->overlay_type_id_ = EXTRACT_TYPE(ei->second); + type->entry_offset_ = EXTRACT_ENTRY(ei->first); + EntryId last_target_entry = kNoEntry; + for (; ei != ti->second.cend(); ++ei) { + if (last_target_entry != kNoEntry) { + int count = EXTRACT_ENTRY(ei->first) - last_target_entry - 1; + type->entries_.insert(type->entries_.end(), count, kNoEntry); + } + type->entries_.push_back(EXTRACT_ENTRY(ei->second)); + last_target_entry = EXTRACT_ENTRY(ei->first); + } + data->type_entries_.push_back(std::move(type)); + } + + std::unique_ptr<IdmapData::Header> data_header(new IdmapData::Header()); + data_header->target_package_id_ = target_pkg->GetPackageId(); + data_header->type_count_ = data->type_entries_.size(); + data->header_ = std::move(data_header); + + idmap->data_.push_back(std::move(data)); + + return std::move(idmap); +} + +void IdmapHeader::accept(Visitor* v) const { + assert(v != nullptr); + v->visit(*this); +} + +void IdmapData::Header::accept(Visitor* v) const { + assert(v != nullptr); + v->visit(*this); +} + +void IdmapData::TypeEntry::accept(Visitor* v) const { + assert(v != nullptr); + v->visit(*this); +} + +void IdmapData::accept(Visitor* v) const { + assert(v != nullptr); + v->visit(*this); + header_->accept(v); + auto end = type_entries_.cend(); + for (auto iter = type_entries_.cbegin(); iter != end; ++iter) { + (*iter)->accept(v); + } +} + +void Idmap::accept(Visitor* v) const { + assert(v != nullptr); + v->visit(*this); + header_->accept(v); + auto end = data_.cend(); + for (auto iter = data_.cbegin(); iter != end; ++iter) { + (*iter)->accept(v); + } +} + +} // namespace idmap2 +} // namespace android diff --git a/cmds/idmap2/libidmap2/PrettyPrintVisitor.cpp b/cmds/idmap2/libidmap2/PrettyPrintVisitor.cpp new file mode 100644 index 000000000000..492e6f049d68 --- /dev/null +++ b/cmds/idmap2/libidmap2/PrettyPrintVisitor.cpp @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <string> +#include <utility> + +#include "android-base/macros.h" +#include "android-base/stringprintf.h" +#include "androidfw/ApkAssets.h" + +#include "idmap2/PrettyPrintVisitor.h" +#include "idmap2/ResourceUtils.h" + +namespace android { +namespace idmap2 { + +#define RESID(pkg, type, entry) (((pkg) << 24) | ((type) << 16) | (entry)) + +void PrettyPrintVisitor::visit(const Idmap& idmap ATTRIBUTE_UNUSED) { +} + +void PrettyPrintVisitor::visit(const IdmapHeader& header) { + stream_ << "target apk path : " << header.GetTargetPath() << std::endl + << "overlay apk path : " << header.GetOverlayPath() << std::endl; + + target_apk_ = ApkAssets::Load(header.GetTargetPath().to_string()); + if (target_apk_) { + target_am_.SetApkAssets({target_apk_.get()}); + } +} + +void PrettyPrintVisitor::visit(const IdmapData& data ATTRIBUTE_UNUSED) { +} + +void PrettyPrintVisitor::visit(const IdmapData::Header& header ATTRIBUTE_UNUSED) { + last_seen_package_id_ = header.GetTargetPackageId(); +} + +void PrettyPrintVisitor::visit(const IdmapData::TypeEntry& te) { + const bool target_package_loaded = !target_am_.GetApkAssets().empty(); + for (uint16_t i = 0; i < te.GetEntryCount(); i++) { + const EntryId entry = te.GetEntry(i); + if (entry == kNoEntry) { + continue; + } + + const ResourceId target_resid = + RESID(last_seen_package_id_, te.GetTargetTypeId(), te.GetEntryOffset() + i); + const ResourceId overlay_resid = RESID(last_seen_package_id_, te.GetOverlayTypeId(), entry); + + stream_ << base::StringPrintf("0x%08x -> 0x%08x", target_resid, overlay_resid); + if (target_package_loaded) { + bool lookup_ok; + std::string name; + std::tie(lookup_ok, name) = utils::ResToTypeEntryName(target_am_, target_resid); + if (lookup_ok) { + stream_ << " " << name; + } + } + stream_ << std::endl; + } +} + +} // namespace idmap2 +} // namespace android diff --git a/cmds/idmap2/libidmap2/RawPrintVisitor.cpp b/cmds/idmap2/libidmap2/RawPrintVisitor.cpp new file mode 100644 index 000000000000..57cfc8ef85b4 --- /dev/null +++ b/cmds/idmap2/libidmap2/RawPrintVisitor.cpp @@ -0,0 +1,131 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <cstdarg> +#include <string> +#include <utility> + +#include "android-base/macros.h" +#include "android-base/stringprintf.h" +#include "androidfw/ApkAssets.h" + +#include "idmap2/RawPrintVisitor.h" +#include "idmap2/ResourceUtils.h" + +using android::ApkAssets; + +namespace android { +namespace idmap2 { + +// verbatim copy fomr PrettyPrintVisitor.cpp, move to common utils +#define RESID(pkg, type, entry) (((pkg) << 24) | ((type) << 16) | (entry)) + +void RawPrintVisitor::visit(const Idmap& idmap ATTRIBUTE_UNUSED) { +} + +void RawPrintVisitor::visit(const IdmapHeader& header) { + print(header.GetMagic(), "magic"); + print(header.GetVersion(), "version"); + print(header.GetTargetCrc(), "target crc"); + print(header.GetOverlayCrc(), "overlay crc"); + print(header.GetTargetPath().to_string(), "target path"); + print(header.GetOverlayPath().to_string(), "overlay path"); + + target_apk_ = ApkAssets::Load(header.GetTargetPath().to_string()); + if (target_apk_) { + target_am_.SetApkAssets({target_apk_.get()}); + } +} + +void RawPrintVisitor::visit(const IdmapData& data ATTRIBUTE_UNUSED) { +} + +void RawPrintVisitor::visit(const IdmapData::Header& header) { + print(static_cast<uint16_t>(header.GetTargetPackageId()), "target package id"); + print(header.GetTypeCount(), "type count"); + last_seen_package_id_ = header.GetTargetPackageId(); +} + +void RawPrintVisitor::visit(const IdmapData::TypeEntry& te) { + const bool target_package_loaded = !target_am_.GetApkAssets().empty(); + + print(static_cast<uint16_t>(te.GetTargetTypeId()), "target type"); + print(static_cast<uint16_t>(te.GetOverlayTypeId()), "overlay type"); + print(static_cast<uint16_t>(te.GetEntryCount()), "entry count"); + print(static_cast<uint16_t>(te.GetEntryOffset()), "entry offset"); + + for (uint16_t i = 0; i < te.GetEntryCount(); i++) { + const EntryId entry = te.GetEntry(i); + if (entry == kNoEntry) { + print(kPadding, "no entry"); + } else { + const ResourceId target_resid = + RESID(last_seen_package_id_, te.GetTargetTypeId(), te.GetEntryOffset() + i); + const ResourceId overlay_resid = RESID(last_seen_package_id_, te.GetOverlayTypeId(), entry); + bool lookup_ok = false; + std::string name; + if (target_package_loaded) { + std::tie(lookup_ok, name) = utils::ResToTypeEntryName(target_am_, target_resid); + } + if (lookup_ok) { + print(static_cast<uint32_t>(entry), "0x%08x -> 0x%08x %s", target_resid, overlay_resid, + name.c_str()); + } else { + print(static_cast<uint32_t>(entry), "0x%08x -> 0x%08x", target_resid, overlay_resid); + } + } + } +} + +void RawPrintVisitor::print(uint16_t value, const char* fmt, ...) { + va_list ap; + va_start(ap, fmt); + std::string comment; + base::StringAppendV(&comment, fmt, ap); + va_end(ap); + + stream_ << base::StringPrintf("%08zx: %04x", offset_, value) << " " << comment << std::endl; + + offset_ += sizeof(uint16_t); +} + +void RawPrintVisitor::print(uint32_t value, const char* fmt, ...) { + va_list ap; + va_start(ap, fmt); + std::string comment; + base::StringAppendV(&comment, fmt, ap); + va_end(ap); + + stream_ << base::StringPrintf("%08zx: %08x", offset_, value) << " " << comment << std::endl; + + offset_ += sizeof(uint32_t); +} + +void RawPrintVisitor::print(const std::string& value, const char* fmt, ...) { + va_list ap; + va_start(ap, fmt); + std::string comment; + base::StringAppendV(&comment, fmt, ap); + va_end(ap); + + stream_ << base::StringPrintf("%08zx: ", offset_) << "........ " << comment << ": " << value + << std::endl; + + offset_ += kIdmapStringLength; +} + +} // namespace idmap2 +} // namespace android diff --git a/cmds/idmap2/libidmap2/ResourceUtils.cpp b/cmds/idmap2/libidmap2/ResourceUtils.cpp new file mode 100644 index 000000000000..e98f843931c8 --- /dev/null +++ b/cmds/idmap2/libidmap2/ResourceUtils.cpp @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <string> +#include <utility> + +#include "androidfw/StringPiece.h" +#include "androidfw/Util.h" + +#include "idmap2/ResourceUtils.h" + +using android::StringPiece16; +using android::util::Utf16ToUtf8; + +namespace android { +namespace idmap2 { +namespace utils { + +std::pair<bool, std::string> WARN_UNUSED ResToTypeEntryName(const AssetManager2& am, + ResourceId resid) { + AssetManager2::ResourceName name; + if (!am.GetResourceName(resid, &name)) { + return std::make_pair(false, ""); + } + std::string out; + if (name.type != nullptr) { + out.append(name.type, name.type_len); + } else { + out += Utf16ToUtf8(StringPiece16(name.type16, name.type_len)); + } + out.append("/"); + if (name.entry != nullptr) { + out.append(name.entry, name.entry_len); + } else { + out += Utf16ToUtf8(StringPiece16(name.entry16, name.entry_len)); + } + return std::make_pair(true, out); +} + +} // namespace utils +} // namespace idmap2 +} // namespace android diff --git a/cmds/idmap2/libidmap2/Xml.cpp b/cmds/idmap2/libidmap2/Xml.cpp new file mode 100644 index 000000000000..5543722ce4fe --- /dev/null +++ b/cmds/idmap2/libidmap2/Xml.cpp @@ -0,0 +1,82 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <map> +#include <memory> +#include <string> +#include <utility> + +#include "idmap2/Xml.h" + +namespace android { +namespace idmap2 { + +std::unique_ptr<const Xml> Xml::Create(const uint8_t* data, size_t size, bool copyData) { + std::unique_ptr<Xml> xml(new Xml()); + if (xml->xml_.setTo(data, size, copyData) != NO_ERROR) { + return nullptr; + } + return xml; +} + +std::unique_ptr<std::map<std::string, std::string>> Xml::FindTag(const std::string& name) const { + const String16 tag_to_find(name.c_str(), name.size()); + xml_.restart(); + ResXMLParser::event_code_t type; + do { + type = xml_.next(); + if (type == ResXMLParser::START_TAG) { + size_t len; + const String16 tag(xml_.getElementName(&len)); + if (tag == tag_to_find) { + std::unique_ptr<std::map<std::string, std::string>> map( + new std::map<std::string, std::string>()); + for (size_t i = 0; i < xml_.getAttributeCount(); i++) { + const String16 key16(xml_.getAttributeName(i, &len)); + std::string key = String8(key16).c_str(); + + std::string value; + switch (xml_.getAttributeDataType(i)) { + case Res_value::TYPE_STRING: { + const String16 value16(xml_.getAttributeStringValue(i, &len)); + value = String8(value16).c_str(); + } break; + case Res_value::TYPE_INT_DEC: + case Res_value::TYPE_INT_HEX: + case Res_value::TYPE_INT_BOOLEAN: { + Res_value resValue; + xml_.getAttributeValue(i, &resValue); + value = std::to_string(resValue.data); + } break; + default: + return nullptr; + } + + map->emplace(std::make_pair(key, value)); + } + return map; + } + } + } while (type != ResXMLParser::BAD_DOCUMENT && type != ResXMLParser::END_DOCUMENT); + return nullptr; +} + +Xml::~Xml() { + xml_.uninit(); +} + +} // namespace idmap2 +} // namespace android diff --git a/cmds/idmap2/libidmap2/ZipFile.cpp b/cmds/idmap2/libidmap2/ZipFile.cpp new file mode 100644 index 000000000000..3f2079a380d6 --- /dev/null +++ b/cmds/idmap2/libidmap2/ZipFile.cpp @@ -0,0 +1,67 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <memory> +#include <string> +#include <utility> + +#include "idmap2/ZipFile.h" + +namespace android { +namespace idmap2 { + +std::unique_ptr<MemoryChunk> MemoryChunk::Allocate(size_t size) { + void* ptr = ::operator new(sizeof(MemoryChunk) + size); + std::unique_ptr<MemoryChunk> chunk(reinterpret_cast<MemoryChunk*>(ptr)); + chunk->size = size; + return chunk; +} + +std::unique_ptr<const ZipFile> ZipFile::Open(const std::string& path) { + ::ZipArchiveHandle handle; + int32_t status = ::OpenArchive(path.c_str(), &handle); + if (status != 0) { + return nullptr; + } + return std::unique_ptr<ZipFile>(new ZipFile(handle)); +} + +ZipFile::~ZipFile() { + ::CloseArchive(handle_); +} + +std::unique_ptr<const MemoryChunk> ZipFile::Uncompress(const std::string& entryPath) const { + ::ZipEntry entry; + int32_t status = ::FindEntry(handle_, ::ZipString(entryPath.c_str()), &entry); + if (status != 0) { + return nullptr; + } + std::unique_ptr<MemoryChunk> chunk = MemoryChunk::Allocate(entry.uncompressed_length); + status = ::ExtractToMemory(handle_, &entry, chunk->buf, chunk->size); + if (status != 0) { + return nullptr; + } + return chunk; +} + +std::pair<bool, uint32_t> ZipFile::Crc(const std::string& entryPath) const { + ::ZipEntry entry; + int32_t status = ::FindEntry(handle_, ::ZipString(entryPath.c_str()), &entry); + return std::make_pair(status == 0, entry.crc32); +} + +} // namespace idmap2 +} // namespace android diff --git a/cmds/idmap2/static-checks.sh b/cmds/idmap2/static-checks.sh new file mode 100755 index 000000000000..560ccb692bb1 --- /dev/null +++ b/cmds/idmap2/static-checks.sh @@ -0,0 +1,121 @@ +#!/bin/bash +# +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +function _log() +{ + echo -e "$*" >&2 +} + +function _eval() +{ + local label="$1" + local cmd="$2" + local red="\e[31m" + local green="\e[32m" + local reset="\e[0m" + + _log "${green}[ RUN ]${reset} ${label}" + local output="$(eval "$cmd")" + if [[ -z "${output}" ]]; then + _log "${green}[ OK ]${reset} ${label}" + return 0 + else + echo "${output}" + _log "${red}[ FAILED ]${reset} ${label}" + errors=$((errors + 1)) + return 1 + fi +} + +function _clang_format() +{ + local path + local errors=0 + + for path in $cpp_files; do + local output="$(clang-format -style=file "$path" | diff $path -)" + if [[ "$output" ]]; then + echo "$path" + echo "$output" + errors=1 + fi + done + return $errors +} + +function _bpfmt() +{ + local output="$(bpfmt -s -d $bp_files)" + if [[ "$output" ]]; then + echo "$output" + return 1 + fi + return 0 +} + +function _cpplint() +{ + local cpplint="${ANDROID_BUILD_TOP}/tools/repohooks/tools/cpplint.py" + $cpplint --quiet $cpp_files +} + +function _parse_args() +{ + local opts + + opts="$(getopt -o cfh --long check,fix,help -- "$@")" + if [[ $? -ne 0 ]]; then + exit 1 + fi + eval set -- "$opts" + while true; do + case "$1" in + -c|--check) opt_mode="check"; shift ;; + -f|--fix) opt_mode="fix"; shift ;; + -h|--help) opt_mode="help"; shift ;; + *) break ;; + esac + done +} + +errors=0 +script="$(readlink -f "$BASH_SOURCE")" +prefix="$(dirname "$script")" +cpp_files="$(find "$prefix" -name '*.cpp' -or -name '*.h')" +bp_files="$(find "$prefix" -name 'Android.bp')" +opt_mode="check" + +_parse_args "$@" +if [[ $opt_mode == "check" ]]; then + _eval "clang-format" "_clang_format" + _eval "bpfmt" "_bpfmt" + _eval "cpplint" "_cpplint" + exit $errors +elif [[ $opt_mode == "fix" ]]; then + clang-format -style=file -i $cpp_files + bpfmt -s -w $bp_files + exit 0 +elif [[ $opt_mode == "help" ]]; then + echo "Run static analysis tools such as clang-format and cpplint on the idmap2" + echo "module. Optionally fix some of the issues found (--fix). Intended to be run" + echo "before merging any changes." + echo + echo "usage: $(basename $script) [--check|--fix|--help]" + exit 0 +else + exit 1 +fi diff --git a/cmds/idmap2/tests/BinaryStreamVisitorTests.cpp b/cmds/idmap2/tests/BinaryStreamVisitorTests.cpp new file mode 100644 index 000000000000..8b552dcc1265 --- /dev/null +++ b/cmds/idmap2/tests/BinaryStreamVisitorTests.cpp @@ -0,0 +1,129 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <memory> +#include <sstream> +#include <string> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "androidfw/ApkAssets.h" +#include "androidfw/Idmap.h" + +#include "idmap2/BinaryStreamVisitor.h" +#include "idmap2/Idmap.h" + +#include "TestHelpers.h" + +using ::testing::IsNull; +using ::testing::NotNull; + +namespace android { +namespace idmap2 { + +TEST(BinaryStreamVisitorTests, CreateBinaryStreamViaBinaryStreamVisitor) { + std::string raw(reinterpret_cast<const char*>(idmap_raw_data), idmap_raw_data_len); + std::istringstream raw_stream(raw); + + std::stringstream error; + std::unique_ptr<const Idmap> idmap1 = Idmap::FromBinaryStream(raw_stream, error); + ASSERT_THAT(idmap1, NotNull()); + + std::stringstream stream; + BinaryStreamVisitor visitor(stream); + idmap1->accept(&visitor); + + std::unique_ptr<const Idmap> idmap2 = Idmap::FromBinaryStream(stream, error); + ASSERT_THAT(idmap2, NotNull()); + + ASSERT_EQ(idmap1->GetHeader()->GetTargetCrc(), idmap2->GetHeader()->GetTargetCrc()); + ASSERT_EQ(idmap1->GetHeader()->GetTargetPath(), idmap2->GetHeader()->GetTargetPath()); + ASSERT_EQ(idmap1->GetData().size(), 1u); + ASSERT_EQ(idmap1->GetData().size(), idmap2->GetData().size()); + + const auto& data1 = idmap1->GetData()[0]; + const auto& data2 = idmap2->GetData()[0]; + + ASSERT_EQ(data1->GetHeader()->GetTargetPackageId(), data2->GetHeader()->GetTargetPackageId()); + ASSERT_EQ(data1->GetTypeEntries().size(), 2u); + ASSERT_EQ(data1->GetTypeEntries().size(), data2->GetTypeEntries().size()); + ASSERT_EQ(data1->GetTypeEntries()[0]->GetEntry(0), data2->GetTypeEntries()[0]->GetEntry(0)); + ASSERT_EQ(data1->GetTypeEntries()[0]->GetEntry(1), data2->GetTypeEntries()[0]->GetEntry(1)); + ASSERT_EQ(data1->GetTypeEntries()[0]->GetEntry(2), data2->GetTypeEntries()[0]->GetEntry(2)); + ASSERT_EQ(data1->GetTypeEntries()[1]->GetEntry(0), data2->GetTypeEntries()[1]->GetEntry(0)); + ASSERT_EQ(data1->GetTypeEntries()[1]->GetEntry(1), data2->GetTypeEntries()[1]->GetEntry(1)); + ASSERT_EQ(data1->GetTypeEntries()[1]->GetEntry(2), data2->GetTypeEntries()[1]->GetEntry(2)); +} + +TEST(BinaryStreamVisitorTests, CreateIdmapFromApkAssetsInteropWithLoadedIdmap) { + const std::string target_apk_path(GetTestDataPath() + "/target/target.apk"); + std::unique_ptr<const ApkAssets> target_apk = ApkAssets::Load(target_apk_path); + ASSERT_THAT(target_apk, NotNull()); + + const std::string overlay_apk_path(GetTestDataPath() + "/overlay/overlay.apk"); + std::unique_ptr<const ApkAssets> overlay_apk = ApkAssets::Load(overlay_apk_path); + ASSERT_THAT(overlay_apk, NotNull()); + + std::stringstream error; + std::unique_ptr<const Idmap> idmap = + Idmap::FromApkAssets(target_apk_path, *target_apk, overlay_apk_path, *overlay_apk, error); + ASSERT_THAT(idmap, NotNull()); + + std::stringstream stream; + BinaryStreamVisitor visitor(stream); + idmap->accept(&visitor); + const std::string str = stream.str(); + const StringPiece data(str); + std::unique_ptr<const LoadedIdmap> loaded_idmap = LoadedIdmap::Load(data); + ASSERT_THAT(loaded_idmap, NotNull()); + ASSERT_EQ(loaded_idmap->TargetPackageId(), 0x7f); + + const IdmapEntry_header* header = loaded_idmap->GetEntryMapForType(0x01); + ASSERT_THAT(header, NotNull()); + + EntryId entry; + bool success = LoadedIdmap::Lookup(header, 0x0000, &entry); + ASSERT_TRUE(success); + ASSERT_EQ(entry, 0x0000); + + header = loaded_idmap->GetEntryMapForType(0x02); + ASSERT_THAT(header, NotNull()); + + success = LoadedIdmap::Lookup(header, 0x0002, &entry); + ASSERT_FALSE(success); + + success = LoadedIdmap::Lookup(header, 0x0003, &entry); + ASSERT_TRUE(success); + ASSERT_EQ(entry, 0x0000); + + success = LoadedIdmap::Lookup(header, 0x0004, &entry); + ASSERT_FALSE(success); + + success = LoadedIdmap::Lookup(header, 0x0005, &entry); + ASSERT_TRUE(success); + ASSERT_EQ(entry, 0x0001); + + success = LoadedIdmap::Lookup(header, 0x0006, &entry); + ASSERT_TRUE(success); + ASSERT_EQ(entry, 0x0002); + + success = LoadedIdmap::Lookup(header, 0x0007, &entry); + ASSERT_FALSE(success); +} + +} // namespace idmap2 +} // namespace android diff --git a/cmds/idmap2/tests/CommandLineOptionsTests.cpp b/cmds/idmap2/tests/CommandLineOptionsTests.cpp new file mode 100644 index 000000000000..b04b25660ee4 --- /dev/null +++ b/cmds/idmap2/tests/CommandLineOptionsTests.cpp @@ -0,0 +1,192 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <sys/mman.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> + +#include <fstream> +#include <memory> +#include <sstream> +#include <string> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "android-base/file.h" +#include "androidfw/ApkAssets.h" +#include "androidfw/Idmap.h" +#include "androidfw/LoadedArsc.h" + +#include "idmap2/CommandLineOptions.h" +#include "idmap2/Idmap.h" + +#include "TestHelpers.h" + +using ::testing::NotNull; + +namespace android { +namespace idmap2 { + +TEST(CommandLineOptionsTests, Flag) { + bool foo = true, bar = false; + + CommandLineOptions opts = + CommandLineOptions("test").OptionalFlag("--foo", "", &foo).OptionalFlag("--bar", "", &bar); + + std::ostream fakeStdErr(nullptr); + bool success = opts.Parse({"--foo", "--bar"}, fakeStdErr); + ASSERT_TRUE(success); + ASSERT_TRUE(foo); + ASSERT_TRUE(bar); + + foo = bar = false; + success = opts.Parse({"--foo"}, fakeStdErr); + ASSERT_TRUE(success); + ASSERT_TRUE(foo); + ASSERT_FALSE(bar); +} + +TEST(CommandLineOptionsTests, MandatoryOption) { + std::string foo, bar; + CommandLineOptions opts = CommandLineOptions("test") + .MandatoryOption("--foo", "", &foo) + .MandatoryOption("--bar", "", &bar); + std::ostream fakeStdErr(nullptr); + bool success = opts.Parse({"--foo", "FOO", "--bar", "BAR"}, fakeStdErr); + ASSERT_TRUE(success); + ASSERT_EQ(foo, "FOO"); + ASSERT_EQ(bar, "BAR"); + + success = opts.Parse({"--foo"}, fakeStdErr); + ASSERT_FALSE(success); +} + +TEST(CommandLineOptionsTests, MandatoryOptionMultipleArgsButExpectedOnce) { + std::string foo; + CommandLineOptions opts = CommandLineOptions("test").MandatoryOption("--foo", "", &foo); + std::ostream fakeStdErr(nullptr); + bool success = opts.Parse({"--foo", "FIRST", "--foo", "SECOND"}, fakeStdErr); + ASSERT_TRUE(success); + ASSERT_EQ(foo, "SECOND"); +} + +TEST(CommandLineOptionsTests, MandatoryOptionMultipleArgsAndExpectedOnceOrMore) { + std::vector<std::string> args; + CommandLineOptions opts = CommandLineOptions("test").MandatoryOption("--foo", "", &args); + std::ostream fakeStdErr(nullptr); + bool success = opts.Parse({"--foo", "FOO", "--foo", "BAR"}, fakeStdErr); + ASSERT_TRUE(success); + ASSERT_EQ(args.size(), 2u); + ASSERT_EQ(args[0], "FOO"); + ASSERT_EQ(args[1], "BAR"); +} + +TEST(CommandLineOptionsTests, OptionalOption) { + std::string foo, bar; + CommandLineOptions opts = CommandLineOptions("test") + .OptionalOption("--foo", "", &foo) + .OptionalOption("--bar", "", &bar); + std::ostream fakeStdErr(nullptr); + bool success = opts.Parse({"--foo", "FOO", "--bar", "BAR"}, fakeStdErr); + ASSERT_TRUE(success); + ASSERT_EQ(foo, "FOO"); + ASSERT_EQ(bar, "BAR"); + + success = opts.Parse({"--foo", "BAZ"}, fakeStdErr); + ASSERT_TRUE(success); + ASSERT_EQ(foo, "BAZ"); + + success = opts.Parse({"--foo"}, fakeStdErr); + ASSERT_FALSE(success); + + success = opts.Parse({"--foo", "--bar", "BAR"}, fakeStdErr); + ASSERT_FALSE(success); + + success = opts.Parse({"--foo", "FOO", "--bar"}, fakeStdErr); + ASSERT_FALSE(success); +} + +TEST(CommandLineOptionsTests, CornerCases) { + std::string foo, bar; + bool baz = false; + CommandLineOptions opts = CommandLineOptions("test") + .MandatoryOption("--foo", "", &foo) + .OptionalFlag("--baz", "", &baz) + .OptionalOption("--bar", "", &bar); + std::ostream fakeStdErr(nullptr); + bool success = opts.Parse({"--unexpected"}, fakeStdErr); + ASSERT_FALSE(success); + + success = opts.Parse({"--bar", "BAR"}, fakeStdErr); + ASSERT_FALSE(success); + + success = opts.Parse({"--baz", "--foo", "FOO"}, fakeStdErr); + ASSERT_TRUE(success); + ASSERT_TRUE(baz); + ASSERT_EQ(foo, "FOO"); +} + +TEST(CommandLineOptionsTests, ConvertArgvToVector) { + const char* argv[] = { + "program-name", + "--foo", + "FOO", + nullptr, + }; + std::unique_ptr<std::vector<std::string>> v = CommandLineOptions::ConvertArgvToVector(3, argv); + ASSERT_EQ(v->size(), 2ul); + ASSERT_EQ((*v)[0], "--foo"); + ASSERT_EQ((*v)[1], "FOO"); +} + +TEST(CommandLineOptionsTests, ConvertArgvToVectorNoArgs) { + const char* argv[] = { + "program-name", + nullptr, + }; + std::unique_ptr<std::vector<std::string>> v = CommandLineOptions::ConvertArgvToVector(1, argv); + ASSERT_EQ(v->size(), 0ul); +} + +TEST(CommandLineOptionsTests, Usage) { + std::string arg1, arg2, arg3, arg4; + bool arg5 = false, arg6 = false; + std::vector<std::string> arg7; + CommandLineOptions opts = CommandLineOptions("test") + .MandatoryOption("--aa", "description-aa", &arg1) + .OptionalFlag("--bb", "description-bb", &arg5) + .OptionalOption("--cc", "description-cc", &arg2) + .OptionalOption("--dd", "description-dd", &arg3) + .MandatoryOption("--ee", "description-ee", &arg4) + .OptionalFlag("--ff", "description-ff", &arg6) + .MandatoryOption("--gg", "description-gg", &arg7); + std::stringstream stream; + opts.Usage(stream); + const std::string s = stream.str(); + ASSERT_NE(s.find("usage: test --aa arg [--bb] [--cc arg] [--dd arg] --ee arg [--ff] --gg arg " + "[--gg arg [..]]"), + std::string::npos); + ASSERT_NE(s.find("--aa arg description-aa"), std::string::npos); + ASSERT_NE(s.find("--ff description-ff"), std::string::npos); + ASSERT_NE(s.find("--gg arg description-gg (can be provided multiple times)"), + std::string::npos); +} + +} // namespace idmap2 +} // namespace android diff --git a/cmds/idmap2/tests/FileUtilsTests.cpp b/cmds/idmap2/tests/FileUtilsTests.cpp new file mode 100644 index 000000000000..0c6439ab8c0c --- /dev/null +++ b/cmds/idmap2/tests/FileUtilsTests.cpp @@ -0,0 +1,76 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <dirent.h> +#include <set> +#include <string> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "android-base/macros.h" + +#include "idmap2/FileUtils.h" + +#include "TestHelpers.h" + +using ::testing::NotNull; + +namespace android { +namespace idmap2 { +namespace utils { + +TEST(FileUtilsTests, FindFilesFindEverythingNonRecursive) { + const auto& root = GetTestDataPath(); + auto v = utils::FindFiles(root, false, + [](unsigned char type ATTRIBUTE_UNUSED, + const std::string& path ATTRIBUTE_UNUSED) -> bool { return true; }); + ASSERT_THAT(v, NotNull()); + ASSERT_EQ(v->size(), 4u); + ASSERT_EQ( + std::set<std::string>(v->begin(), v->end()), + std::set<std::string>({root + "/.", root + "/..", root + "/overlay", root + "/target"})); +} + +TEST(FileUtilsTests, FindFilesFindApkFilesRecursive) { + const auto& root = GetTestDataPath(); + auto v = utils::FindFiles(root, true, [](unsigned char type, const std::string& path) -> bool { + return type == DT_REG && path.size() > 4 && !path.compare(path.size() - 4, 4, ".apk"); + }); + ASSERT_THAT(v, NotNull()); + ASSERT_EQ(v->size(), 4u); + ASSERT_EQ(std::set<std::string>(v->begin(), v->end()), + std::set<std::string>({root + "/target/target.apk", root + "/overlay/overlay.apk", + root + "/overlay/overlay-static-1.apk", + root + "/overlay/overlay-static-2.apk"})); +} + +TEST(FileUtilsTests, ReadFile) { + int pipefd[2]; + ASSERT_EQ(pipe(pipefd), 0); + + ASSERT_EQ(write(pipefd[1], "foobar", 6), 6); + close(pipefd[1]); + + auto data = ReadFile(pipefd[0]); + ASSERT_THAT(data, NotNull()); + ASSERT_EQ(*data, "foobar"); + close(pipefd[0]); +} + +} // namespace utils +} // namespace idmap2 +} // namespace android diff --git a/cmds/idmap2/tests/Idmap2BinaryTests.cpp b/cmds/idmap2/tests/Idmap2BinaryTests.cpp new file mode 100644 index 000000000000..5c4e8576985b --- /dev/null +++ b/cmds/idmap2/tests/Idmap2BinaryTests.cpp @@ -0,0 +1,313 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * The tests in this file operate on a higher level than the tests in the other + * files. Here, all tests execute the idmap2 binary and only depend on + * libidmap2 to verify the output of idmap2. + */ +#include <fcntl.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <sys/wait.h> +#include <unistd.h> + +#include <cerrno> +#include <cstdlib> +#include <cstring> // strerror +#include <fstream> +#include <memory> +#include <sstream> +#include <string> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "androidfw/PosixUtils.h" +#include "idmap2/FileUtils.h" +#include "idmap2/Idmap.h" + +#include "TestHelpers.h" + +using ::android::util::ExecuteBinary; +using ::testing::NotNull; + +namespace android { +namespace idmap2 { + +class Idmap2BinaryTests : public Idmap2Tests {}; + +static void AssertIdmap(const Idmap& idmap, const std::string& target_apk_path, + const std::string& overlay_apk_path) { + // check that the idmap file looks reasonable (IdmapTests is responsible for + // more in-depth verification) + ASSERT_EQ(idmap.GetHeader()->GetMagic(), kIdmapMagic); + ASSERT_EQ(idmap.GetHeader()->GetVersion(), kIdmapCurrentVersion); + ASSERT_EQ(idmap.GetHeader()->GetTargetPath(), target_apk_path); + ASSERT_EQ(idmap.GetHeader()->GetOverlayPath(), overlay_apk_path); + ASSERT_EQ(idmap.GetData().size(), 1u); +} + +#define ASSERT_IDMAP(idmap_ref, target_apk_path, overlay_apk_path) \ + do { \ + ASSERT_NO_FATAL_FAILURE(AssertIdmap(idmap_ref, target_apk_path, overlay_apk_path)); \ + } while (0) + +TEST_F(Idmap2BinaryTests, Create) { + // clang-format off + auto result = ExecuteBinary({"idmap2", + "create", + "--target-apk-path", GetTargetApkPath(), + "--overlay-apk-path", GetOverlayApkPath(), + "--idmap-path", GetIdmapPath()}); + // clang-format on + ASSERT_THAT(result, NotNull()); + ASSERT_EQ(result->status, EXIT_SUCCESS) << result->stderr; + + struct stat st; + ASSERT_EQ(stat(GetIdmapPath().c_str(), &st), 0); + + std::stringstream error; + std::ifstream fin(GetIdmapPath()); + std::unique_ptr<const Idmap> idmap = Idmap::FromBinaryStream(fin, error); + fin.close(); + + ASSERT_THAT(idmap, NotNull()); + ASSERT_IDMAP(*idmap, GetTargetApkPath(), GetOverlayApkPath()); + + unlink(GetIdmapPath().c_str()); +} + +TEST_F(Idmap2BinaryTests, Dump) { + // clang-format off + auto result = ExecuteBinary({"idmap2", + "create", + "--target-apk-path", GetTargetApkPath(), + "--overlay-apk-path", GetOverlayApkPath(), + "--idmap-path", GetIdmapPath()}); + // clang-format on + ASSERT_THAT(result, NotNull()); + ASSERT_EQ(result->status, EXIT_SUCCESS) << result->stderr; + + // clang-format off + result = ExecuteBinary({"idmap2", + "dump", + "--idmap-path", GetIdmapPath()}); + // clang-format on + ASSERT_THAT(result, NotNull()); + ASSERT_EQ(result->status, EXIT_SUCCESS) << result->stderr; + ASSERT_NE(result->stdout.find("0x7f010000 -> 0x7f010000 integer/int1"), std::string::npos); + ASSERT_NE(result->stdout.find("0x7f020003 -> 0x7f020000 string/str1"), std::string::npos); + ASSERT_NE(result->stdout.find("0x7f020005 -> 0x7f020001 string/str3"), std::string::npos); + ASSERT_EQ(result->stdout.find("00000210: 007f target package id"), std::string::npos); + + // clang-format off + result = ExecuteBinary({"idmap2", + "dump", + "--verbose", + "--idmap-path", GetIdmapPath()}); + // clang-format on + ASSERT_THAT(result, NotNull()); + ASSERT_EQ(result->status, EXIT_SUCCESS) << result->stderr; + ASSERT_NE(result->stdout.find("00000000: 504d4449 magic"), std::string::npos); + ASSERT_NE(result->stdout.find("00000210: 007f target package id"), std::string::npos); + + // clang-format off + result = ExecuteBinary({"idmap2", + "dump", + "--verbose", + "--idmap-path", GetTestDataPath() + "/DOES-NOT-EXIST"}); + // clang-format on + ASSERT_THAT(result, NotNull()); + ASSERT_NE(result->status, EXIT_SUCCESS); + + unlink(GetIdmapPath().c_str()); +} + +TEST_F(Idmap2BinaryTests, Scan) { + const std::string overlay_static_1_apk_path = GetTestDataPath() + "/overlay/overlay-static-1.apk"; + const std::string overlay_static_2_apk_path = GetTestDataPath() + "/overlay/overlay-static-2.apk"; + const std::string idmap_static_1_path = + Idmap::CanonicalIdmapPathFor(GetTempDirPath(), overlay_static_1_apk_path); + const std::string idmap_static_2_path = + Idmap::CanonicalIdmapPathFor(GetTempDirPath(), overlay_static_2_apk_path); + + // single input directory, recursive + // clang-format off + auto result = ExecuteBinary({"idmap2", + "scan", + "--input-directory", GetTestDataPath(), + "--recursive", + "--target-package-name", "test.target", + "--target-apk-path", GetTargetApkPath(), + "--output-directory", GetTempDirPath()}); + // clang-format on + ASSERT_THAT(result, NotNull()); + ASSERT_EQ(result->status, EXIT_SUCCESS) << result->stderr; + std::stringstream expected; + expected << idmap_static_1_path << std::endl; + expected << idmap_static_2_path << std::endl; + ASSERT_EQ(result->stdout, expected.str()); + + std::stringstream error; + auto idmap_static_1_raw_string = utils::ReadFile(idmap_static_1_path); + auto idmap_static_1_raw_stream = std::istringstream(*idmap_static_1_raw_string); + auto idmap_static_1 = Idmap::FromBinaryStream(idmap_static_1_raw_stream, error); + ASSERT_THAT(idmap_static_1, NotNull()); + ASSERT_IDMAP(*idmap_static_1, GetTargetApkPath(), overlay_static_1_apk_path); + + auto idmap_static_2_raw_string = utils::ReadFile(idmap_static_2_path); + auto idmap_static_2_raw_stream = std::istringstream(*idmap_static_2_raw_string); + auto idmap_static_2 = Idmap::FromBinaryStream(idmap_static_2_raw_stream, error); + ASSERT_THAT(idmap_static_2, NotNull()); + ASSERT_IDMAP(*idmap_static_2, GetTargetApkPath(), overlay_static_2_apk_path); + + unlink(idmap_static_2_path.c_str()); + unlink(idmap_static_1_path.c_str()); + + // multiple input directories, non-recursive + // clang-format off + result = ExecuteBinary({"idmap2", + "scan", + "--input-directory", GetTestDataPath() + "/target", + "--input-directory", GetTestDataPath() + "/overlay", + "--target-package-name", "test.target", + "--target-apk-path", GetTargetApkPath(), + "--output-directory", GetTempDirPath()}); + // clang-format on + ASSERT_THAT(result, NotNull()); + ASSERT_EQ(result->status, EXIT_SUCCESS) << result->stderr; + ASSERT_EQ(result->stdout, expected.str()); + unlink(idmap_static_2_path.c_str()); + unlink(idmap_static_1_path.c_str()); + + // the same input directory given twice, but no duplicate entries + // clang-format off + result = ExecuteBinary({"idmap2", + "scan", + "--input-directory", GetTestDataPath(), + "--input-directory", GetTestDataPath(), + "--recursive", + "--target-package-name", "test.target", + "--target-apk-path", GetTargetApkPath(), + "--output-directory", GetTempDirPath()}); + // clang-format on + ASSERT_THAT(result, NotNull()); + ASSERT_EQ(result->status, EXIT_SUCCESS) << result->stderr; + ASSERT_EQ(result->stdout, expected.str()); + unlink(idmap_static_2_path.c_str()); + unlink(idmap_static_1_path.c_str()); + + // no APKs in input-directory: ok, but no output + // clang-format off + result = ExecuteBinary({"idmap2", + "scan", + "--input-directory", GetTempDirPath(), + "--target-package-name", "test.target", + "--target-apk-path", GetTargetApkPath(), + "--output-directory", GetTempDirPath()}); + // clang-format on + ASSERT_THAT(result, NotNull()); + ASSERT_EQ(result->status, EXIT_SUCCESS) << result->stderr; + ASSERT_EQ(result->stdout, ""); +} + +TEST_F(Idmap2BinaryTests, Lookup) { + // clang-format off + auto result = ExecuteBinary({"idmap2", + "create", + "--target-apk-path", GetTargetApkPath(), + "--overlay-apk-path", GetOverlayApkPath(), + "--idmap-path", GetIdmapPath()}); + // clang-format on + ASSERT_THAT(result, NotNull()); + ASSERT_EQ(result->status, EXIT_SUCCESS) << result->stderr; + + // clang-format off + result = ExecuteBinary({"idmap2", + "lookup", + "--idmap-path", GetIdmapPath(), + "--config", "", + "--resid", "0x7f020003"}); // string/str1 + // clang-format on + ASSERT_THAT(result, NotNull()); + ASSERT_EQ(result->status, EXIT_SUCCESS) << result->stderr; + ASSERT_NE(result->stdout.find("overlay-1"), std::string::npos); + ASSERT_EQ(result->stdout.find("overlay-1-sv"), std::string::npos); + + // clang-format off + result = ExecuteBinary({"idmap2", + "lookup", + "--idmap-path", GetIdmapPath(), + "--config", "", + "--resid", "test.target:string/str1"}); + // clang-format on + ASSERT_THAT(result, NotNull()); + ASSERT_EQ(result->status, EXIT_SUCCESS) << result->stderr; + ASSERT_NE(result->stdout.find("overlay-1"), std::string::npos); + ASSERT_EQ(result->stdout.find("overlay-1-sv"), std::string::npos); + + // clang-format off + result = ExecuteBinary({"idmap2", + "lookup", + "--idmap-path", GetIdmapPath(), + "--config", "sv", + "--resid", "test.target:string/str1"}); + // clang-format on + ASSERT_THAT(result, NotNull()); + ASSERT_EQ(result->status, EXIT_SUCCESS) << result->stderr; + ASSERT_NE(result->stdout.find("overlay-1-sv"), std::string::npos); + + unlink(GetIdmapPath().c_str()); +} + +TEST_F(Idmap2BinaryTests, InvalidCommandLineOptions) { + const std::string invalid_target_apk_path = GetTestDataPath() + "/DOES-NOT-EXIST"; + + // missing mandatory options + // clang-format off + auto result = ExecuteBinary({"idmap2", + "create"}); + // clang-format on + ASSERT_THAT(result, NotNull()); + ASSERT_NE(result->status, EXIT_SUCCESS); + + // missing argument to option + // clang-format off + result = ExecuteBinary({"idmap2", + "create", + "--target-apk-path", GetTargetApkPath(), + "--overlay-apk-path", GetOverlayApkPath(), + "--idmap-path"}); + // clang-format on + ASSERT_THAT(result, NotNull()); + ASSERT_NE(result->status, EXIT_SUCCESS); + + // invalid target apk path + // clang-format off + result = ExecuteBinary({"idmap2", + "create", + "--target-apk-path", invalid_target_apk_path, + "--overlay-apk-path", GetOverlayApkPath(), + "--idmap-path", GetIdmapPath()}); + // clang-format on + ASSERT_THAT(result, NotNull()); + ASSERT_NE(result->status, EXIT_SUCCESS); +} + +} // namespace idmap2 +} // namespace android diff --git a/cmds/idmap2/tests/IdmapTests.cpp b/cmds/idmap2/tests/IdmapTests.cpp new file mode 100644 index 000000000000..0379aa491682 --- /dev/null +++ b/cmds/idmap2/tests/IdmapTests.cpp @@ -0,0 +1,395 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <cstdio> // fclose + +#include <fstream> +#include <memory> +#include <sstream> +#include <string> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "android-base/macros.h" +#include "androidfw/ApkAssets.h" + +#include "idmap2/BinaryStreamVisitor.h" +#include "idmap2/CommandLineOptions.h" +#include "idmap2/Idmap.h" + +#include "TestHelpers.h" + +using ::testing::IsNull; +using ::testing::NotNull; + +namespace android { +namespace idmap2 { + +TEST(IdmapTests, TestCanonicalIdmapPathFor) { + ASSERT_EQ(Idmap::CanonicalIdmapPathFor("/foo", "/vendor/overlay/bar.apk"), + "/foo/vendor@overlay@bar.apk@idmap"); +} + +TEST(IdmapTests, CreateIdmapHeaderFromBinaryStream) { + std::string raw(reinterpret_cast<const char*>(idmap_raw_data), idmap_raw_data_len); + std::istringstream stream(raw); + std::unique_ptr<const IdmapHeader> header = IdmapHeader::FromBinaryStream(stream); + ASSERT_THAT(header, NotNull()); + ASSERT_EQ(header->GetMagic(), 0x504d4449u); + ASSERT_EQ(header->GetVersion(), 0x01u); + ASSERT_EQ(header->GetTargetCrc(), 0x1234u); + ASSERT_EQ(header->GetOverlayCrc(), 0x5678u); + ASSERT_EQ(header->GetTargetPath().to_string(), "target.apk"); + ASSERT_EQ(header->GetOverlayPath().to_string(), "overlay.apk"); +} + +TEST(IdmapTests, FailToCreateIdmapHeaderFromBinaryStreamIfPathTooLong) { + std::string raw(reinterpret_cast<const char*>(idmap_raw_data), idmap_raw_data_len); + // overwrite the target path string, including the terminating null, with '.' + for (size_t i = 0x10; i < 0x110; i++) { + raw[i] = '.'; + } + std::istringstream stream(raw); + std::unique_ptr<const IdmapHeader> header = IdmapHeader::FromBinaryStream(stream); + ASSERT_THAT(header, IsNull()); +} + +TEST(IdmapTests, CreateIdmapDataHeaderFromBinaryStream) { + const size_t offset = 0x210; + std::string raw(reinterpret_cast<const char*>(idmap_raw_data + offset), + idmap_raw_data_len - offset); + std::istringstream stream(raw); + + std::unique_ptr<const IdmapData::Header> header = IdmapData::Header::FromBinaryStream(stream); + ASSERT_THAT(header, NotNull()); + ASSERT_EQ(header->GetTargetPackageId(), 0x7fu); + ASSERT_EQ(header->GetTypeCount(), 2u); +} + +TEST(IdmapTests, CreateIdmapDataResourceTypeFromBinaryStream) { + const size_t offset = 0x214; + std::string raw(reinterpret_cast<const char*>(idmap_raw_data + offset), + idmap_raw_data_len - offset); + std::istringstream stream(raw); + + std::unique_ptr<const IdmapData::TypeEntry> data = IdmapData::TypeEntry::FromBinaryStream(stream); + ASSERT_THAT(data, NotNull()); + ASSERT_EQ(data->GetTargetTypeId(), 0x02u); + ASSERT_EQ(data->GetOverlayTypeId(), 0x02u); + ASSERT_EQ(data->GetEntryCount(), 1u); + ASSERT_EQ(data->GetEntryOffset(), 0u); + ASSERT_EQ(data->GetEntry(0), 0u); +} + +TEST(IdmapTests, CreateIdmapDataFromBinaryStream) { + const size_t offset = 0x210; + std::string raw(reinterpret_cast<const char*>(idmap_raw_data + offset), + idmap_raw_data_len - offset); + std::istringstream stream(raw); + + std::unique_ptr<const IdmapData> data = IdmapData::FromBinaryStream(stream); + ASSERT_THAT(data, NotNull()); + ASSERT_EQ(data->GetHeader()->GetTargetPackageId(), 0x7fu); + ASSERT_EQ(data->GetHeader()->GetTypeCount(), 2u); + const std::vector<std::unique_ptr<const IdmapData::TypeEntry>>& types = data->GetTypeEntries(); + ASSERT_EQ(types.size(), 2u); + + ASSERT_EQ(types[0]->GetTargetTypeId(), 0x02u); + ASSERT_EQ(types[0]->GetOverlayTypeId(), 0x02u); + ASSERT_EQ(types[0]->GetEntryCount(), 1u); + ASSERT_EQ(types[0]->GetEntryOffset(), 0u); + ASSERT_EQ(types[0]->GetEntry(0), 0x0000u); + + ASSERT_EQ(types[1]->GetTargetTypeId(), 0x03u); + ASSERT_EQ(types[1]->GetOverlayTypeId(), 0x03u); + ASSERT_EQ(types[1]->GetEntryCount(), 3u); + ASSERT_EQ(types[1]->GetEntryOffset(), 3u); + ASSERT_EQ(types[1]->GetEntry(0), 0x0000u); + ASSERT_EQ(types[1]->GetEntry(1), kNoEntry); + ASSERT_EQ(types[1]->GetEntry(2), 0x0001u); +} + +TEST(IdmapTests, CreateIdmapFromBinaryStream) { + std::string raw(reinterpret_cast<const char*>(idmap_raw_data), idmap_raw_data_len); + std::istringstream stream(raw); + + std::stringstream error; + std::unique_ptr<const Idmap> idmap = Idmap::FromBinaryStream(stream, error); + ASSERT_THAT(idmap, NotNull()); + + ASSERT_THAT(idmap->GetHeader(), NotNull()); + ASSERT_EQ(idmap->GetHeader()->GetMagic(), 0x504d4449u); + ASSERT_EQ(idmap->GetHeader()->GetVersion(), 0x01u); + ASSERT_EQ(idmap->GetHeader()->GetTargetCrc(), 0x1234u); + ASSERT_EQ(idmap->GetHeader()->GetOverlayCrc(), 0x5678u); + ASSERT_EQ(idmap->GetHeader()->GetTargetPath().to_string(), "target.apk"); + ASSERT_EQ(idmap->GetHeader()->GetOverlayPath().to_string(), "overlay.apk"); + + const std::vector<std::unique_ptr<const IdmapData>>& dataBlocks = idmap->GetData(); + ASSERT_EQ(dataBlocks.size(), 1u); + + const std::unique_ptr<const IdmapData>& data = dataBlocks[0]; + ASSERT_EQ(data->GetHeader()->GetTargetPackageId(), 0x7fu); + ASSERT_EQ(data->GetHeader()->GetTypeCount(), 2u); + const std::vector<std::unique_ptr<const IdmapData::TypeEntry>>& types = data->GetTypeEntries(); + ASSERT_EQ(types.size(), 2u); + + ASSERT_EQ(types[0]->GetTargetTypeId(), 0x02u); + ASSERT_EQ(types[0]->GetOverlayTypeId(), 0x02u); + ASSERT_EQ(types[0]->GetEntryCount(), 1u); + ASSERT_EQ(types[0]->GetEntryOffset(), 0u); + ASSERT_EQ(types[0]->GetEntry(0), 0x0000u); + + ASSERT_EQ(types[1]->GetTargetTypeId(), 0x03u); + ASSERT_EQ(types[1]->GetOverlayTypeId(), 0x03u); + ASSERT_EQ(types[1]->GetEntryCount(), 3u); + ASSERT_EQ(types[1]->GetEntryOffset(), 3u); + ASSERT_EQ(types[1]->GetEntry(0), 0x0000u); + ASSERT_EQ(types[1]->GetEntry(1), kNoEntry); + ASSERT_EQ(types[1]->GetEntry(2), 0x0001u); +} + +TEST(IdmapTests, GracefullyFailToCreateIdmapFromCorruptBinaryStream) { + std::string raw(reinterpret_cast<const char*>(idmap_raw_data), + 10); // data too small + std::istringstream stream(raw); + + std::stringstream error; + std::unique_ptr<const Idmap> idmap = Idmap::FromBinaryStream(stream, error); + ASSERT_THAT(idmap, IsNull()); +} + +TEST(IdmapTests, CreateIdmapFromApkAssets) { + const std::string target_apk_path(GetTestDataPath() + "/target/target.apk"); + std::unique_ptr<const ApkAssets> target_apk = ApkAssets::Load(target_apk_path); + ASSERT_THAT(target_apk, NotNull()); + + const std::string overlay_apk_path(GetTestDataPath() + "/overlay/overlay.apk"); + std::unique_ptr<const ApkAssets> overlay_apk = ApkAssets::Load(overlay_apk_path); + ASSERT_THAT(overlay_apk, NotNull()); + + std::stringstream error; + std::unique_ptr<const Idmap> idmap = + Idmap::FromApkAssets(target_apk_path, *target_apk, overlay_apk_path, *overlay_apk, error); + ASSERT_THAT(idmap, NotNull()); + + ASSERT_THAT(idmap->GetHeader(), NotNull()); + ASSERT_EQ(idmap->GetHeader()->GetMagic(), 0x504d4449u); + ASSERT_EQ(idmap->GetHeader()->GetVersion(), 0x01u); + ASSERT_EQ(idmap->GetHeader()->GetTargetCrc(), 0xf5ad1d1d); + ASSERT_EQ(idmap->GetHeader()->GetOverlayCrc(), 0xd470336b); + ASSERT_EQ(idmap->GetHeader()->GetTargetPath().to_string(), target_apk_path); + ASSERT_EQ(idmap->GetHeader()->GetOverlayPath(), overlay_apk_path); + ASSERT_EQ(idmap->GetHeader()->GetOverlayPath(), overlay_apk_path); + + const std::vector<std::unique_ptr<const IdmapData>>& dataBlocks = idmap->GetData(); + ASSERT_EQ(dataBlocks.size(), 1u); + + const std::unique_ptr<const IdmapData>& data = dataBlocks[0]; + + ASSERT_EQ(data->GetHeader()->GetTargetPackageId(), 0x7fu); + ASSERT_EQ(data->GetHeader()->GetTypeCount(), 2u); + + const std::vector<std::unique_ptr<const IdmapData::TypeEntry>>& types = data->GetTypeEntries(); + ASSERT_EQ(types.size(), 2u); + + ASSERT_EQ(types[0]->GetTargetTypeId(), 0x01u); + ASSERT_EQ(types[0]->GetOverlayTypeId(), 0x01u); + ASSERT_EQ(types[0]->GetEntryCount(), 1u); + ASSERT_EQ(types[0]->GetEntryOffset(), 0u); + ASSERT_EQ(types[0]->GetEntry(0), 0x0000u); + + ASSERT_EQ(types[1]->GetTargetTypeId(), 0x02u); + ASSERT_EQ(types[1]->GetOverlayTypeId(), 0x02u); + ASSERT_EQ(types[1]->GetEntryCount(), 4u); + ASSERT_EQ(types[1]->GetEntryOffset(), 3u); + ASSERT_EQ(types[1]->GetEntry(0), 0x0000u); + ASSERT_EQ(types[1]->GetEntry(1), kNoEntry); + ASSERT_EQ(types[1]->GetEntry(2), 0x0001u); + ASSERT_EQ(types[1]->GetEntry(3), 0x0002u); +} + +TEST(IdmapTests, FailToCreateIdmapFromApkAssetsIfPathTooLong) { + std::string target_apk_path(GetTestDataPath()); + for (int i = 0; i < 32; i++) { + target_apk_path += "/target/../"; + } + target_apk_path += "/target/target.apk"; + ASSERT_GT(target_apk_path.size(), kIdmapStringLength); + std::unique_ptr<const ApkAssets> target_apk = ApkAssets::Load(target_apk_path); + ASSERT_THAT(target_apk, NotNull()); + + const std::string overlay_apk_path(GetTestDataPath() + "/overlay/overlay.apk"); + std::unique_ptr<const ApkAssets> overlay_apk = ApkAssets::Load(overlay_apk_path); + ASSERT_THAT(overlay_apk, NotNull()); + + std::stringstream error; + std::unique_ptr<const Idmap> idmap = + Idmap::FromApkAssets(target_apk_path, *target_apk, overlay_apk_path, *overlay_apk, error); + ASSERT_THAT(idmap, IsNull()); +} + +TEST(IdmapTests, IdmapHeaderIsUpToDate) { + fclose(stderr); // silence expected warnings from libandroidfw + + const std::string target_apk_path(GetTestDataPath() + "/target/target.apk"); + std::unique_ptr<const ApkAssets> target_apk = ApkAssets::Load(target_apk_path); + ASSERT_THAT(target_apk, NotNull()); + + const std::string overlay_apk_path(GetTestDataPath() + "/overlay/overlay.apk"); + std::unique_ptr<const ApkAssets> overlay_apk = ApkAssets::Load(overlay_apk_path); + ASSERT_THAT(overlay_apk, NotNull()); + + std::stringstream error; + std::unique_ptr<const Idmap> idmap = + Idmap::FromApkAssets(target_apk_path, *target_apk, overlay_apk_path, *overlay_apk, error); + ASSERT_THAT(idmap, NotNull()); + + std::stringstream stream; + BinaryStreamVisitor visitor(stream); + idmap->accept(&visitor); + + std::unique_ptr<const IdmapHeader> header = IdmapHeader::FromBinaryStream(stream); + ASSERT_THAT(header, NotNull()); + ASSERT_TRUE(header->IsUpToDate(error)) << error.str(); + + // magic: bytes (0x0, 0x03) + std::string bad_magic_string(stream.str()); + bad_magic_string[0x0] = '.'; + bad_magic_string[0x1] = '.'; + bad_magic_string[0x2] = '.'; + bad_magic_string[0x3] = '.'; + std::stringstream bad_magic_stream(bad_magic_string); + std::unique_ptr<const IdmapHeader> bad_magic_header = + IdmapHeader::FromBinaryStream(bad_magic_stream); + ASSERT_THAT(bad_magic_header, NotNull()); + ASSERT_NE(header->GetMagic(), bad_magic_header->GetMagic()); + ASSERT_FALSE(bad_magic_header->IsUpToDate(error)); + + // version: bytes (0x4, 0x07) + std::string bad_version_string(stream.str()); + bad_version_string[0x4] = '.'; + bad_version_string[0x5] = '.'; + bad_version_string[0x6] = '.'; + bad_version_string[0x7] = '.'; + std::stringstream bad_version_stream(bad_version_string); + std::unique_ptr<const IdmapHeader> bad_version_header = + IdmapHeader::FromBinaryStream(bad_version_stream); + ASSERT_THAT(bad_version_header, NotNull()); + ASSERT_NE(header->GetVersion(), bad_version_header->GetVersion()); + ASSERT_FALSE(bad_version_header->IsUpToDate(error)); + + // target crc: bytes (0x8, 0xb) + std::string bad_target_crc_string(stream.str()); + bad_target_crc_string[0x8] = '.'; + bad_target_crc_string[0x9] = '.'; + bad_target_crc_string[0xa] = '.'; + bad_target_crc_string[0xb] = '.'; + std::stringstream bad_target_crc_stream(bad_target_crc_string); + std::unique_ptr<const IdmapHeader> bad_target_crc_header = + IdmapHeader::FromBinaryStream(bad_target_crc_stream); + ASSERT_THAT(bad_target_crc_header, NotNull()); + ASSERT_NE(header->GetTargetCrc(), bad_target_crc_header->GetTargetCrc()); + ASSERT_FALSE(bad_target_crc_header->IsUpToDate(error)); + + // overlay crc: bytes (0xc, 0xf) + std::string bad_overlay_crc_string(stream.str()); + bad_overlay_crc_string[0xc] = '.'; + bad_overlay_crc_string[0xd] = '.'; + bad_overlay_crc_string[0xe] = '.'; + bad_overlay_crc_string[0xf] = '.'; + std::stringstream bad_overlay_crc_stream(bad_overlay_crc_string); + std::unique_ptr<const IdmapHeader> bad_overlay_crc_header = + IdmapHeader::FromBinaryStream(bad_overlay_crc_stream); + ASSERT_THAT(bad_overlay_crc_header, NotNull()); + ASSERT_NE(header->GetOverlayCrc(), bad_overlay_crc_header->GetOverlayCrc()); + ASSERT_FALSE(bad_overlay_crc_header->IsUpToDate(error)); + + // target path: bytes (0x10, 0x10f) + std::string bad_target_path_string(stream.str()); + bad_target_path_string[0x10] = '\0'; + std::stringstream bad_target_path_stream(bad_target_path_string); + std::unique_ptr<const IdmapHeader> bad_target_path_header = + IdmapHeader::FromBinaryStream(bad_target_path_stream); + ASSERT_THAT(bad_target_path_header, NotNull()); + ASSERT_NE(header->GetTargetPath(), bad_target_path_header->GetTargetPath()); + ASSERT_FALSE(bad_target_path_header->IsUpToDate(error)); + + // overlay path: bytes (0x110, 0x20f) + std::string bad_overlay_path_string(stream.str()); + bad_overlay_path_string[0x110] = '\0'; + std::stringstream bad_overlay_path_stream(bad_overlay_path_string); + std::unique_ptr<const IdmapHeader> bad_overlay_path_header = + IdmapHeader::FromBinaryStream(bad_overlay_path_stream); + ASSERT_THAT(bad_overlay_path_header, NotNull()); + ASSERT_NE(header->GetOverlayPath(), bad_overlay_path_header->GetOverlayPath()); + ASSERT_FALSE(bad_overlay_path_header->IsUpToDate(error)); +} + +class TestVisitor : public Visitor { + public: + explicit TestVisitor(std::ostream& stream) : stream_(stream) { + } + + void visit(const Idmap& idmap ATTRIBUTE_UNUSED) { + stream_ << "TestVisitor::visit(Idmap)" << std::endl; + } + + void visit(const IdmapHeader& idmap ATTRIBUTE_UNUSED) { + stream_ << "TestVisitor::visit(IdmapHeader)" << std::endl; + } + + void visit(const IdmapData& idmap ATTRIBUTE_UNUSED) { + stream_ << "TestVisitor::visit(IdmapData)" << std::endl; + } + + void visit(const IdmapData::Header& idmap ATTRIBUTE_UNUSED) { + stream_ << "TestVisitor::visit(IdmapData::Header)" << std::endl; + } + + void visit(const IdmapData::TypeEntry& idmap ATTRIBUTE_UNUSED) { + stream_ << "TestVisitor::visit(IdmapData::TypeEntry)" << std::endl; + } + + private: + std::ostream& stream_; +}; + +TEST(IdmapTests, TestVisitor) { + std::string raw(reinterpret_cast<const char*>(idmap_raw_data), idmap_raw_data_len); + std::istringstream stream(raw); + + std::stringstream error; + std::unique_ptr<const Idmap> idmap = Idmap::FromBinaryStream(stream, error); + ASSERT_THAT(idmap, NotNull()); + + std::stringstream test_stream; + TestVisitor visitor(test_stream); + idmap->accept(&visitor); + + ASSERT_EQ(test_stream.str(), + "TestVisitor::visit(Idmap)\n" + "TestVisitor::visit(IdmapHeader)\n" + "TestVisitor::visit(IdmapData)\n" + "TestVisitor::visit(IdmapData::Header)\n" + "TestVisitor::visit(IdmapData::TypeEntry)\n" + "TestVisitor::visit(IdmapData::TypeEntry)\n"); +} + +} // namespace idmap2 +} // namespace android diff --git a/cmds/idmap2/tests/Main.cpp b/cmds/idmap2/tests/Main.cpp new file mode 100644 index 000000000000..f2469eaf57cc --- /dev/null +++ b/cmds/idmap2/tests/Main.cpp @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <string> + +#include "android-base/file.h" + +#include "gtest/gtest.h" + +#include "TestHelpers.h" + +namespace android { +namespace idmap2 { + +const std::string GetTestDataPath() { + return base::GetExecutableDirectory() + "/tests/data"; +} + +} // namespace idmap2 +} // namespace android + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/cmds/idmap2/tests/PrettyPrintVisitorTests.cpp b/cmds/idmap2/tests/PrettyPrintVisitorTests.cpp new file mode 100644 index 000000000000..da9779211f81 --- /dev/null +++ b/cmds/idmap2/tests/PrettyPrintVisitorTests.cpp @@ -0,0 +1,83 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <memory> +#include <sstream> +#include <string> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "androidfw/ApkAssets.h" +#include "androidfw/Idmap.h" + +#include "idmap2/Idmap.h" +#include "idmap2/PrettyPrintVisitor.h" + +#include "TestHelpers.h" + +using ::testing::IsNull; +using ::testing::NotNull; + +using android::ApkAssets; + +namespace android { +namespace idmap2 { + +TEST(PrettyPrintVisitorTests, CreatePrettyPrintVisitor) { + const std::string target_apk_path(GetTestDataPath() + "/target/target.apk"); + std::unique_ptr<const ApkAssets> target_apk = ApkAssets::Load(target_apk_path); + ASSERT_THAT(target_apk, NotNull()); + + const std::string overlay_apk_path(GetTestDataPath() + "/overlay/overlay.apk"); + std::unique_ptr<const ApkAssets> overlay_apk = ApkAssets::Load(overlay_apk_path); + ASSERT_THAT(overlay_apk, NotNull()); + + std::stringstream error; + std::unique_ptr<const Idmap> idmap = + Idmap::FromApkAssets(target_apk_path, *target_apk, overlay_apk_path, *overlay_apk, error); + ASSERT_THAT(idmap, NotNull()); + + std::stringstream stream; + PrettyPrintVisitor visitor(stream); + idmap->accept(&visitor); + + ASSERT_NE(stream.str().find("target apk path : "), std::string::npos); + ASSERT_NE(stream.str().find("overlay apk path : "), std::string::npos); + ASSERT_NE(stream.str().find("0x7f010000 -> 0x7f010000 integer/int1\n"), std::string::npos); +} + +TEST(PrettyPrintVisitorTests, CreatePrettyPrintVisitorWithoutAccessToApks) { + fclose(stderr); // silence expected warnings from libandroidfw + + std::string raw(reinterpret_cast<const char*>(idmap_raw_data), idmap_raw_data_len); + std::istringstream raw_stream(raw); + + std::stringstream error; + std::unique_ptr<const Idmap> idmap = Idmap::FromBinaryStream(raw_stream, error); + ASSERT_THAT(idmap, NotNull()); + + std::stringstream stream; + PrettyPrintVisitor visitor(stream); + idmap->accept(&visitor); + + ASSERT_NE(stream.str().find("target apk path : "), std::string::npos); + ASSERT_NE(stream.str().find("overlay apk path : "), std::string::npos); + ASSERT_NE(stream.str().find("0x7f020000 -> 0x7f020000\n"), std::string::npos); +} + +} // namespace idmap2 +} // namespace android diff --git a/cmds/idmap2/tests/RawPrintVisitorTests.cpp b/cmds/idmap2/tests/RawPrintVisitorTests.cpp new file mode 100644 index 000000000000..c28ce2e02ea9 --- /dev/null +++ b/cmds/idmap2/tests/RawPrintVisitorTests.cpp @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <cstdio> // fclose +#include <memory> +#include <sstream> +#include <string> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "idmap2/Idmap.h" +#include "idmap2/RawPrintVisitor.h" + +#include "TestHelpers.h" + +using ::testing::IsNull; +using ::testing::NotNull; + +namespace android { +namespace idmap2 { + +TEST(RawPrintVisitorTests, CreateRawPrintVisitor) { + const std::string target_apk_path(GetTestDataPath() + "/target/target.apk"); + std::unique_ptr<const ApkAssets> target_apk = ApkAssets::Load(target_apk_path); + ASSERT_THAT(target_apk, NotNull()); + + const std::string overlay_apk_path(GetTestDataPath() + "/overlay/overlay.apk"); + std::unique_ptr<const ApkAssets> overlay_apk = ApkAssets::Load(overlay_apk_path); + ASSERT_THAT(overlay_apk, NotNull()); + + std::stringstream error; + std::unique_ptr<const Idmap> idmap = + Idmap::FromApkAssets(target_apk_path, *target_apk, overlay_apk_path, *overlay_apk, error); + ASSERT_THAT(idmap, NotNull()); + + std::stringstream stream; + RawPrintVisitor visitor(stream); + idmap->accept(&visitor); + + ASSERT_NE(stream.str().find("00000000: 504d4449 magic\n"), std::string::npos); + ASSERT_NE(stream.str().find("00000004: 00000001 version\n"), std::string::npos); + ASSERT_NE(stream.str().find("00000008: f5ad1d1d target crc\n"), std::string::npos); + ASSERT_NE(stream.str().find("0000000c: d470336b overlay crc\n"), std::string::npos); + ASSERT_NE(stream.str().find("0000021c: 00000000 0x7f010000 -> 0x7f010000 integer/int1\n"), + std::string::npos); +} + +TEST(RawPrintVisitorTests, CreateRawPrintVisitorWithoutAccessToApks) { + fclose(stderr); // silence expected warnings from libandroidfw + + std::string raw(reinterpret_cast<const char*>(idmap_raw_data), idmap_raw_data_len); + std::istringstream raw_stream(raw); + + std::stringstream error; + std::unique_ptr<const Idmap> idmap = Idmap::FromBinaryStream(raw_stream, error); + ASSERT_THAT(idmap, NotNull()); + + std::stringstream stream; + RawPrintVisitor visitor(stream); + idmap->accept(&visitor); + + ASSERT_NE(stream.str().find("00000000: 504d4449 magic\n"), std::string::npos); + ASSERT_NE(stream.str().find("00000004: 00000001 version\n"), std::string::npos); + ASSERT_NE(stream.str().find("00000008: 00001234 target crc\n"), std::string::npos); + ASSERT_NE(stream.str().find("0000000c: 00005678 overlay crc\n"), std::string::npos); + ASSERT_NE(stream.str().find("0000021c: 00000000 0x7f020000 -> 0x7f020000\n"), std::string::npos); +} + +} // namespace idmap2 +} // namespace android diff --git a/cmds/idmap2/tests/ResourceUtilsTests.cpp b/cmds/idmap2/tests/ResourceUtilsTests.cpp new file mode 100644 index 000000000000..0547fa00de3d --- /dev/null +++ b/cmds/idmap2/tests/ResourceUtilsTests.cpp @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <memory> +#include <string> +#include <utility> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "androidfw/ApkAssets.h" +#include "idmap2/ResourceUtils.h" + +#include "TestHelpers.h" + +using ::testing::NotNull; + +namespace android { +namespace idmap2 { + +class ResourceUtilsTests : public Idmap2Tests { + protected: + void SetUp() override { + Idmap2Tests::SetUp(); + + apk_assets_ = ApkAssets::Load(GetTargetApkPath()); + ASSERT_THAT(apk_assets_, NotNull()); + + am_.SetApkAssets({apk_assets_.get()}); + } + + const AssetManager2& GetAssetManager() { + return am_; + } + + private: + AssetManager2 am_; + std::unique_ptr<const ApkAssets> apk_assets_; +}; + +TEST_F(ResourceUtilsTests, ResToTypeEntryName) { + bool lookup_ok; + std::string name; + std::tie(lookup_ok, name) = utils::ResToTypeEntryName(GetAssetManager(), 0x7f010000u); + ASSERT_TRUE(lookup_ok); + ASSERT_EQ(name, "integer/int1"); +} + +TEST_F(ResourceUtilsTests, ResToTypeEntryNameNoSuchResourceId) { + bool lookup_ok; + std::tie(lookup_ok, std::ignore) = utils::ResToTypeEntryName(GetAssetManager(), 0x7f123456u); + ASSERT_FALSE(lookup_ok); +} + +} // namespace idmap2 +} // namespace android diff --git a/cmds/idmap2/tests/TestHelpers.h b/cmds/idmap2/tests/TestHelpers.h new file mode 100644 index 000000000000..18dc541021c1 --- /dev/null +++ b/cmds/idmap2/tests/TestHelpers.h @@ -0,0 +1,168 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IDMAP2_TESTS_TESTHELPERS_H_ +#define IDMAP2_TESTS_TESTHELPERS_H_ + +#include <string> + +namespace android { +namespace idmap2 { + +const unsigned char idmap_raw_data[] = { + // IDMAP HEADER + // 0x0: magic + 0x49, 0x44, 0x4d, 0x50, + + // 0x4: version + 0x01, 0x00, 0x00, 0x00, + + // 0x8: target crc + 0x34, 0x12, 0x00, 0x00, + + // 0xc: overlay crc + 0x78, 0x56, 0x00, 0x00, + + // 0x10: target path "target.apk" + 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x2e, 0x61, 0x70, 0x6b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + + // 0x110: overlay path "overlay.apk" + 0x6f, 0x76, 0x65, 0x72, 0x6c, 0x61, 0x79, 0x2e, 0x61, 0x70, 0x6b, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + + // DATA HEADER + // 0x210: target package id + 0x7f, 0x00, + + // 0x212: types count + 0x02, 0x00, + + // DATA BLOCK + // 0x214: target type + 0x02, 0x00, + + // 0x216: overlay type + 0x02, 0x00, + + // 0x218: entry count + 0x01, 0x00, + + // 0x21a: entry offset + 0x00, 0x00, + + // 0x21c: entries + 0x00, 0x00, 0x00, 0x00, + + // DATA BLOCK + // 0x220: target type + 0x03, 0x00, + + // 0x222: overlay type + 0x03, 0x00, + + // 0x224: entry count + 0x03, 0x00, + + // 0x226: entry offset + 0x03, 0x00, + + // 0x228, 0x22c, 0x230: entries + 0x00, 0x00, 0x00, 0x00, + + 0xff, 0xff, 0xff, 0xff, + + 0x01, 0x00, 0x00, 0x00}; + +const unsigned int idmap_raw_data_len = 565; + +const std::string GetTestDataPath(); + +class Idmap2Tests : public testing::Test { + protected: + virtual void SetUp() { +#ifdef __ANDROID__ + tmp_dir_path_ = "/data/local/tmp/idmap2-tests-XXXXXX"; +#else + tmp_dir_path_ = "/tmp/idmap2-tests-XXXXXX"; +#endif + EXPECT_NE(mkdtemp(const_cast<char*>(tmp_dir_path_.c_str())), nullptr) + << "Failed to create temporary directory: " << strerror(errno); + target_apk_path_ = GetTestDataPath() + "/target/target.apk"; + overlay_apk_path_ = GetTestDataPath() + "/overlay/overlay.apk"; + idmap_path_ = tmp_dir_path_ + "/a.idmap"; + } + + virtual void TearDown() { + EXPECT_EQ(rmdir(tmp_dir_path_.c_str()), 0) + << "Failed to remove temporary directory " << tmp_dir_path_ << ": " << strerror(errno); + } + + const std::string& GetTempDirPath() { + return tmp_dir_path_; + } + + const std::string& GetTargetApkPath() { + return target_apk_path_; + } + + const std::string& GetOverlayApkPath() { + return overlay_apk_path_; + } + + const std::string& GetIdmapPath() { + return idmap_path_; + } + + private: + std::string tmp_dir_path_; + std::string target_apk_path_; + std::string overlay_apk_path_; + std::string idmap_path_; +}; + +} // namespace idmap2 +} // namespace android + +#endif // IDMAP2_TESTS_TESTHELPERS_H_ diff --git a/cmds/idmap2/tests/XmlTests.cpp b/cmds/idmap2/tests/XmlTests.cpp new file mode 100644 index 000000000000..97ff03e0f9e3 --- /dev/null +++ b/cmds/idmap2/tests/XmlTests.cpp @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <cstdio> // fclose + +#include "idmap2/Xml.h" +#include "idmap2/ZipFile.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "TestHelpers.h" + +using ::testing::IsNull; +using ::testing::NotNull; + +namespace android { +namespace idmap2 { + +TEST(XmlTests, Create) { + auto zip = ZipFile::Open(GetTestDataPath() + "/target/target.apk"); + ASSERT_THAT(zip, NotNull()); + + auto data = zip->Uncompress("AndroidManifest.xml"); + ASSERT_THAT(data, NotNull()); + + auto xml = Xml::Create(data->buf, data->size); + ASSERT_THAT(xml, NotNull()); + + fclose(stderr); // silence expected warnings from libandroidfw + const char* not_xml = "foo"; + auto fail = Xml::Create(reinterpret_cast<const uint8_t*>(not_xml), strlen(not_xml)); + ASSERT_THAT(fail, IsNull()); +} + +TEST(XmlTests, FindTag) { + auto zip = ZipFile::Open(GetTestDataPath() + "/target/target.apk"); + ASSERT_THAT(zip, NotNull()); + + auto data = zip->Uncompress("res/xml/test.xml"); + ASSERT_THAT(data, NotNull()); + + auto xml = Xml::Create(data->buf, data->size); + ASSERT_THAT(xml, NotNull()); + + auto attrs = xml->FindTag("c"); + ASSERT_THAT(attrs, NotNull()); + ASSERT_EQ(attrs->size(), 4u); + ASSERT_EQ(attrs->at("type_string"), "fortytwo"); + ASSERT_EQ(std::stoi(attrs->at("type_int_dec")), 42); + ASSERT_EQ(std::stoi(attrs->at("type_int_hex")), 42); + ASSERT_NE(std::stoul(attrs->at("type_int_boolean")), 0u); + + auto fail = xml->FindTag("does-not-exist"); + ASSERT_THAT(fail, IsNull()); +} + +} // namespace idmap2 +} // namespace android diff --git a/cmds/idmap2/tests/ZipFileTests.cpp b/cmds/idmap2/tests/ZipFileTests.cpp new file mode 100644 index 000000000000..a504d3126c05 --- /dev/null +++ b/cmds/idmap2/tests/ZipFileTests.cpp @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <cstdio> // fclose +#include <string> +#include <utility> + +#include "idmap2/ZipFile.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "TestHelpers.h" + +using ::testing::IsNull; +using ::testing::NotNull; + +namespace android { +namespace idmap2 { + +TEST(ZipFileTests, BasicOpen) { + auto zip = ZipFile::Open(GetTestDataPath() + "/target/target.apk"); + ASSERT_THAT(zip, NotNull()); + + fclose(stderr); // silence expected warnings from libziparchive + auto fail = ZipFile::Open(GetTestDataPath() + "/does-not-exist"); + ASSERT_THAT(fail, IsNull()); +} + +TEST(ZipFileTests, Crc) { + auto zip = ZipFile::Open(GetTestDataPath() + "/target/target.apk"); + ASSERT_THAT(zip, NotNull()); + + bool status; + uint32_t crc; + std::tie(status, crc) = zip->Crc("AndroidManifest.xml"); + ASSERT_TRUE(status); + ASSERT_EQ(crc, 0x762f3d24); + + std::tie(status, std::ignore) = zip->Crc("does-not-exist"); + ASSERT_FALSE(status); +} + +TEST(ZipFileTests, Uncompress) { + auto zip = ZipFile::Open(GetTestDataPath() + "/target/target.apk"); + ASSERT_THAT(zip, NotNull()); + + auto data = zip->Uncompress("assets/lorem-ipsum.txt"); + ASSERT_THAT(data, NotNull()); + const std::string lorem_ipsum("Lorem ipsum dolor sit amet.\n"); + ASSERT_THAT(data->size, lorem_ipsum.size()); + ASSERT_THAT(std::string(reinterpret_cast<const char*>(data->buf), data->size), lorem_ipsum); + + auto fail = zip->Uncompress("does-not-exist"); + ASSERT_THAT(fail, IsNull()); +} + +} // namespace idmap2 +} // namespace android diff --git a/cmds/idmap2/tests/data/overlay/AndroidManifest.xml b/cmds/idmap2/tests/data/overlay/AndroidManifest.xml new file mode 100644 index 000000000000..9f89d3121a82 --- /dev/null +++ b/cmds/idmap2/tests/data/overlay/AndroidManifest.xml @@ -0,0 +1,21 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2018 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<manifest + xmlns:android="http://schemas.android.com/apk/res/android" + package="test.overlay"> + <overlay + android:targetPackage="test.target" /> +</manifest> diff --git a/cmds/idmap2/tests/data/overlay/AndroidManifestStatic1.xml b/cmds/idmap2/tests/data/overlay/AndroidManifestStatic1.xml new file mode 100644 index 000000000000..39336cc7e76b --- /dev/null +++ b/cmds/idmap2/tests/data/overlay/AndroidManifestStatic1.xml @@ -0,0 +1,23 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2018 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<manifest + xmlns:android="http://schemas.android.com/apk/res/android" + package="test.overlay.static1"> + <overlay + android:targetPackage="test.target" + android:isStatic="true" + android:priority="1" /> +</manifest> diff --git a/cmds/idmap2/tests/data/overlay/AndroidManifestStatic2.xml b/cmds/idmap2/tests/data/overlay/AndroidManifestStatic2.xml new file mode 100644 index 000000000000..e1cc1758d8cc --- /dev/null +++ b/cmds/idmap2/tests/data/overlay/AndroidManifestStatic2.xml @@ -0,0 +1,23 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2018 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<manifest + xmlns:android="http://schemas.android.com/apk/res/android" + package="test.overlay.static2"> + <overlay + android:targetPackage="test.target" + android:isStatic="true" + android:priority="2" /> +</manifest> diff --git a/cmds/idmap2/tests/data/overlay/build b/cmds/idmap2/tests/data/overlay/build new file mode 100644 index 000000000000..cba108674005 --- /dev/null +++ b/cmds/idmap2/tests/data/overlay/build @@ -0,0 +1,40 @@ +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FRAMEWORK_RES_APK="$(gettop)/out/target/common/obj/APPS/framework-res_intermediates/package-export.apk" + +aapt2 compile --dir res -o compiled.flata + +aapt2 link \ + --no-resource-removal \ + -I "$FRAMEWORK_RES_APK" \ + --manifest AndroidManifest.xml \ + -o overlay.apk \ + compiled.flata + +aapt2 link \ + --no-resource-removal \ + -I "$FRAMEWORK_RES_APK" \ + --manifest AndroidManifestStatic1.xml \ + -o overlay-static-1.apk \ + compiled.flata + +aapt2 link \ + --no-resource-removal \ + -I "$FRAMEWORK_RES_APK" \ + --manifest AndroidManifestStatic2.xml \ + -o overlay-static-2.apk \ + compiled.flata + +rm compiled.flata diff --git a/cmds/idmap2/tests/data/overlay/overlay-static-1.apk b/cmds/idmap2/tests/data/overlay/overlay-static-1.apk Binary files differnew file mode 100644 index 000000000000..9a0f487522c8 --- /dev/null +++ b/cmds/idmap2/tests/data/overlay/overlay-static-1.apk diff --git a/cmds/idmap2/tests/data/overlay/overlay-static-2.apk b/cmds/idmap2/tests/data/overlay/overlay-static-2.apk Binary files differnew file mode 100644 index 000000000000..3fc31c7d11b0 --- /dev/null +++ b/cmds/idmap2/tests/data/overlay/overlay-static-2.apk diff --git a/cmds/idmap2/tests/data/overlay/overlay.apk b/cmds/idmap2/tests/data/overlay/overlay.apk Binary files differnew file mode 100644 index 000000000000..b4cd7cfc3248 --- /dev/null +++ b/cmds/idmap2/tests/data/overlay/overlay.apk diff --git a/cmds/idmap2/tests/data/overlay/res/values-sv/values.xml b/cmds/idmap2/tests/data/overlay/res/values-sv/values.xml new file mode 100644 index 000000000000..eed0b3dac1ab --- /dev/null +++ b/cmds/idmap2/tests/data/overlay/res/values-sv/values.xml @@ -0,0 +1,19 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2018 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<resources> + <string name="str1">overlay-1-sv</string> + <string name="str4">overlay-4-sv</string> +</resources> diff --git a/cmds/idmap2/tests/data/overlay/res/values/values.xml b/cmds/idmap2/tests/data/overlay/res/values/values.xml new file mode 100644 index 000000000000..815d1a88fa7b --- /dev/null +++ b/cmds/idmap2/tests/data/overlay/res/values/values.xml @@ -0,0 +1,21 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2018 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<resources> + <string name="str1">overlay-1</string> + <string name="str3">overlay-3</string> + <integer name="int1">-1</integer> + <integer name="not_in_target">-1</integer> +</resources> diff --git a/cmds/idmap2/tests/data/target/AndroidManifest.xml b/cmds/idmap2/tests/data/target/AndroidManifest.xml new file mode 100644 index 000000000000..3a861b4800fa --- /dev/null +++ b/cmds/idmap2/tests/data/target/AndroidManifest.xml @@ -0,0 +1,19 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2018 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<manifest + xmlns:android="http://schemas.android.com/apk/res/android" + package="test.target"> +</manifest> diff --git a/cmds/idmap2/tests/data/target/assets/lorem-ipsum.txt b/cmds/idmap2/tests/data/target/assets/lorem-ipsum.txt new file mode 100644 index 000000000000..d2cf010d36ff --- /dev/null +++ b/cmds/idmap2/tests/data/target/assets/lorem-ipsum.txt @@ -0,0 +1 @@ +Lorem ipsum dolor sit amet. diff --git a/cmds/idmap2/tests/data/target/build b/cmds/idmap2/tests/data/target/build new file mode 100644 index 000000000000..8569c4ff0a6b --- /dev/null +++ b/cmds/idmap2/tests/data/target/build @@ -0,0 +1,17 @@ +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +aapt2 compile --dir res -o compiled.flata +aapt2 link --manifest AndroidManifest.xml -A assets -o target.apk compiled.flata +rm compiled.flata diff --git a/cmds/idmap2/tests/data/target/res/values/values.xml b/cmds/idmap2/tests/data/target/res/values/values.xml new file mode 100644 index 000000000000..56bf0d60021a --- /dev/null +++ b/cmds/idmap2/tests/data/target/res/values/values.xml @@ -0,0 +1,28 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2018 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<resources> + <string name="a">a</string> + <string name="b">b</string> + <string name="c">c</string> + <string name="str1">target-1</string> + <string name="str2">target-2</string> + <string name="str3">target-3</string> + <string name="str4">target-4</string> + <string name="x">x</string> + <string name="y">y</string> + <string name="z">z</string> + <integer name="int1">1</integer> +</resources> diff --git a/cmds/idmap2/tests/data/target/res/xml/test.xml b/cmds/idmap2/tests/data/target/res/xml/test.xml new file mode 100644 index 000000000000..0fe21c6b6d0a --- /dev/null +++ b/cmds/idmap2/tests/data/target/res/xml/test.xml @@ -0,0 +1,25 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2018 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<a> + <b> + <c + type_string="fortytwo" + type_int_dec="42" + type_int_hex="0x2a" + type_int_boolean="true" + /> + </b> +</a> diff --git a/cmds/idmap2/tests/data/target/target.apk b/cmds/idmap2/tests/data/target/target.apk Binary files differnew file mode 100644 index 000000000000..18ecc276caae --- /dev/null +++ b/cmds/idmap2/tests/data/target/target.apk diff --git a/cmds/incident_helper/src/main.cpp b/cmds/incident_helper/src/main.cpp index 091410bce2dd..809a77163fb4 100644 --- a/cmds/incident_helper/src/main.cpp +++ b/cmds/incident_helper/src/main.cpp @@ -73,7 +73,8 @@ static TextParserBase* selectParser(int section) { case 2006: return new BatteryTypeParser(); default: - return NULL; + // Return no op parser when no specific ones are implemented. + return new NoopParser(); } } diff --git a/cmds/incidentd/src/FdBuffer.cpp b/cmds/incidentd/src/FdBuffer.cpp index 0885b13483c6..a8ef8311720d 100644 --- a/cmds/incidentd/src/FdBuffer.cpp +++ b/cmds/incidentd/src/FdBuffer.cpp @@ -71,7 +71,8 @@ status_t FdBuffer::read(int fd, int64_t timeout) { VLOG("return event has error %s", strerror(errno)); return errno != 0 ? -errno : UNKNOWN_ERROR; } else { - ssize_t amt = ::read(fd, mBuffer.writeBuffer(), mBuffer.currentToWrite()); + ssize_t amt = TEMP_FAILURE_RETRY( + ::read(fd, mBuffer.writeBuffer(), mBuffer.currentToWrite())); if (amt < 0) { if (errno == EAGAIN || errno == EWOULDBLOCK) { continue; @@ -182,9 +183,9 @@ status_t FdBuffer::readProcessedDataInStream(int fd, unique_fd toFd, unique_fd f if (cirSize != BUFFER_SIZE && pfds[0].fd != -1) { ssize_t amt; if (rpos >= wpos) { - amt = ::read(fd, cirBuf + rpos, BUFFER_SIZE - rpos); + amt = TEMP_FAILURE_RETRY(::read(fd, cirBuf + rpos, BUFFER_SIZE - rpos)); } else { - amt = ::read(fd, cirBuf + rpos, wpos - rpos); + amt = TEMP_FAILURE_RETRY(::read(fd, cirBuf + rpos, wpos - rpos)); } if (amt < 0) { if (!(errno == EAGAIN || errno == EWOULDBLOCK)) { @@ -204,9 +205,9 @@ status_t FdBuffer::readProcessedDataInStream(int fd, unique_fd toFd, unique_fd f if (cirSize > 0 && pfds[1].fd != -1) { ssize_t amt; if (rpos > wpos) { - amt = ::write(toFd.get(), cirBuf + wpos, rpos - wpos); + amt = TEMP_FAILURE_RETRY(::write(toFd.get(), cirBuf + wpos, rpos - wpos)); } else { - amt = ::write(toFd.get(), cirBuf + wpos, BUFFER_SIZE - wpos); + amt = TEMP_FAILURE_RETRY(::write(toFd.get(), cirBuf + wpos, BUFFER_SIZE - wpos)); } if (amt < 0) { if (!(errno == EAGAIN || errno == EWOULDBLOCK)) { @@ -235,7 +236,8 @@ status_t FdBuffer::readProcessedDataInStream(int fd, unique_fd toFd, unique_fd f } // read from parsing process - ssize_t amt = ::read(fromFd.get(), mBuffer.writeBuffer(), mBuffer.currentToWrite()); + ssize_t amt = TEMP_FAILURE_RETRY( + ::read(fromFd.get(), mBuffer.writeBuffer(), mBuffer.currentToWrite())); if (amt < 0) { if (!(errno == EAGAIN || errno == EWOULDBLOCK)) { VLOG("Fail to read fromFd %d: %s", fromFd.get(), strerror(errno)); diff --git a/cmds/incidentd/src/IncidentService.cpp b/cmds/incidentd/src/IncidentService.cpp index e305b5462b77..e92cf9444e15 100644 --- a/cmds/incidentd/src/IncidentService.cpp +++ b/cmds/incidentd/src/IncidentService.cpp @@ -314,6 +314,19 @@ status_t IncidentService::command(FILE* in, FILE* out, FILE* err, Vector<String8 mThrottler->dump(out); return NO_ERROR; } + if (!args[0].compare(String8("section"))) { + int id = atoi(args[1]); + int idx = 0; + while (SECTION_LIST[idx] != NULL) { + const Section* section = SECTION_LIST[idx]; + if (section->id == id) { + fprintf(out, "Section[%d] %s\n", id, section->name.string()); + break; + } + idx++; + } + return NO_ERROR; + } } return cmd_help(out); } @@ -321,8 +334,9 @@ status_t IncidentService::command(FILE* in, FILE* out, FILE* err, Vector<String8 status_t IncidentService::cmd_help(FILE* out) { fprintf(out, "usage: adb shell cmd incident privacy print <section_id>\n"); fprintf(out, "usage: adb shell cmd incident privacy parse <section_id> < proto.txt\n"); - fprintf(out, " Prints/parses for the section id.\n"); - fprintf(out, "\n"); + fprintf(out, " Prints/parses for the section id.\n\n"); + fprintf(out, "usage: adb shell cmd incident section <section_id>\n"); + fprintf(out, " Prints section id and its name.\n\n"); fprintf(out, "usage: adb shell cmd incident throttler\n"); fprintf(out, " Prints the current throttler state\n"); return NO_ERROR; diff --git a/cmds/incidentd/src/PrivacyBuffer.cpp b/cmds/incidentd/src/PrivacyBuffer.cpp index d753e5e6404e..7a8ebe394d51 100644 --- a/cmds/incidentd/src/PrivacyBuffer.cpp +++ b/cmds/incidentd/src/PrivacyBuffer.cpp @@ -86,8 +86,8 @@ status_t PrivacyBuffer::stripField(const Privacy* parentPolicy, const PrivacySpe // iterator will point to head of next field size_t currentAt = mData.rp()->pos(); writeFieldOrSkip(fieldTag, skip); - VLOG("[Depth %2d]Field %d %ss %d bytes", depth, fieldId, skip ? "skip" : "write", - (int)(get_varint_size(fieldTag) + mData.rp()->pos() - currentAt)); + VLOG("[Depth %2d]Field %d %ss %zu bytes", depth, fieldId, skip ? "skip" : "write", + get_varint_size(fieldTag) + mData.rp()->pos() - currentAt); return NO_ERROR; } // current field is message type and its sub-fields have extra privacy policies diff --git a/cmds/incidentd/src/Section.cpp b/cmds/incidentd/src/Section.cpp index 72a41036e595..cd48af95d08a 100644 --- a/cmds/incidentd/src/Section.cpp +++ b/cmds/incidentd/src/Section.cpp @@ -151,11 +151,10 @@ DONE: } // ================================================================================ -Section::Section(int i, int64_t timeoutMs, bool userdebugAndEngOnly, bool deviceSpecific) +Section::Section(int i, int64_t timeoutMs, bool userdebugAndEngOnly) : id(i), timeoutMs(timeoutMs), - userdebugAndEngOnly(userdebugAndEngOnly), - deviceSpecific(deviceSpecific) {} + userdebugAndEngOnly(userdebugAndEngOnly) {} Section::~Section() {} @@ -240,10 +239,10 @@ status_t MetadataSection::Execute(ReportRequestSet* requests) const { // ================================================================================ static inline bool isSysfs(const char* filename) { return strncmp(filename, "/sys/", 5) == 0; } -FileSection::FileSection(int id, const char* filename, const bool deviceSpecific, - const int64_t timeoutMs) - : Section(id, timeoutMs, false, deviceSpecific), mFilename(filename) { - name = filename; +FileSection::FileSection(int id, const char* filename, const int64_t timeoutMs) + : Section(id, timeoutMs, false), mFilename(filename) { + name = "file "; + name += filename; mIsSysfs = isSysfs(filename); } @@ -254,8 +253,10 @@ status_t FileSection::Execute(ReportRequestSet* requests) const { // add O_CLOEXEC to make sure it is closed when exec incident helper unique_fd fd(open(mFilename, O_RDONLY | O_CLOEXEC)); if (fd.get() == -1) { - ALOGW("FileSection '%s' failed to open file", this->name.string()); - return this->deviceSpecific ? NO_ERROR : -errno; + ALOGW("[%s] failed to open file", this->name.string()); + // There may be some devices/architectures that won't have the file. + // Just return here without an error. + return NO_ERROR; } FdBuffer buffer; @@ -263,13 +264,13 @@ status_t FileSection::Execute(ReportRequestSet* requests) const { Fpipe c2pPipe; // initiate pipes to pass data to/from incident_helper if (!p2cPipe.init() || !c2pPipe.init()) { - ALOGW("FileSection '%s' failed to setup pipes", this->name.string()); + ALOGW("[%s] failed to setup pipes", this->name.string()); return -errno; } pid_t pid = fork_execute_incident_helper(this->id, &p2cPipe, &c2pPipe); if (pid == -1) { - ALOGW("FileSection '%s' failed to fork", this->name.string()); + ALOGW("[%s] failed to fork", this->name.string()); return -errno; } @@ -279,7 +280,7 @@ status_t FileSection::Execute(ReportRequestSet* requests) const { this->timeoutMs, mIsSysfs); write_section_stats(requests->sectionStats(this->id), buffer); if (readStatus != NO_ERROR || buffer.timedOut()) { - ALOGW("FileSection '%s' failed to read data from incident helper: %s, timedout: %s", + ALOGW("[%s] failed to read data from incident helper: %s, timedout: %s", this->name.string(), strerror(-readStatus), buffer.timedOut() ? "true" : "false"); kill_child(pid); return readStatus; @@ -287,20 +288,11 @@ status_t FileSection::Execute(ReportRequestSet* requests) const { status_t ihStatus = wait_child(pid); if (ihStatus != NO_ERROR) { - ALOGW("FileSection '%s' abnormal child process: %s", this->name.string(), - strerror(-ihStatus)); + ALOGW("[%s] abnormal child process: %s", this->name.string(), strerror(-ihStatus)); return ihStatus; } - VLOG("FileSection '%s' wrote %zd bytes in %d ms", this->name.string(), buffer.size(), - (int)buffer.durationMs()); - status_t err = write_report_requests(this->id, buffer, requests); - if (err != NO_ERROR) { - ALOGW("FileSection '%s' failed writing: %s", this->name.string(), strerror(-err)); - return err; - } - - return NO_ERROR; + return write_report_requests(this->id, buffer, requests); } // ================================================================================ GZipSection::GZipSection(int id, const char* filename, ...) : Section(id) { @@ -329,9 +321,8 @@ status_t GZipSection::Execute(ReportRequestSet* requests) const { ALOGW("GZipSection failed to open file %s", mFilenames[index]); index++; // look at the next file. } - VLOG("GZipSection is using file %s, fd=%d", mFilenames[index], fd.get()); if (fd.get() == -1) { - ALOGW("GZipSection %s can't open all the files", this->name.string()); + ALOGW("[%s] can't open all the files", this->name.string()); return NO_ERROR; // e.g. LAST_KMSG will reach here in user build. } FdBuffer buffer; @@ -339,13 +330,13 @@ status_t GZipSection::Execute(ReportRequestSet* requests) const { Fpipe c2pPipe; // initiate pipes to pass data to/from gzip if (!p2cPipe.init() || !c2pPipe.init()) { - ALOGW("GZipSection '%s' failed to setup pipes", this->name.string()); + ALOGW("[%s] failed to setup pipes", this->name.string()); return -errno; } pid_t pid = fork_execute_cmd((char* const*)GZIP, &p2cPipe, &c2pPipe); if (pid == -1) { - ALOGW("GZipSection '%s' failed to fork", this->name.string()); + ALOGW("[%s] failed to fork", this->name.string()); return -errno; } // parent process @@ -364,24 +355,22 @@ status_t GZipSection::Execute(ReportRequestSet* requests) const { size_t editPos = internalBuffer->wp()->pos(); internalBuffer->wp()->move(8); // reserve 8 bytes for the varint of the data size. size_t dataBeginAt = internalBuffer->wp()->pos(); - VLOG("GZipSection '%s' editPos=%zd, dataBeginAt=%zd", this->name.string(), editPos, - dataBeginAt); + VLOG("[%s] editPos=%zu, dataBeginAt=%zu", this->name.string(), editPos, dataBeginAt); status_t readStatus = buffer.readProcessedDataInStream( fd.get(), std::move(p2cPipe.writeFd()), std::move(c2pPipe.readFd()), this->timeoutMs, isSysfs(mFilenames[index])); write_section_stats(requests->sectionStats(this->id), buffer); if (readStatus != NO_ERROR || buffer.timedOut()) { - ALOGW("GZipSection '%s' failed to read data from gzip: %s, timedout: %s", - this->name.string(), strerror(-readStatus), buffer.timedOut() ? "true" : "false"); + ALOGW("[%s] failed to read data from gzip: %s, timedout: %s", this->name.string(), + strerror(-readStatus), buffer.timedOut() ? "true" : "false"); kill_child(pid); return readStatus; } status_t gzipStatus = wait_child(pid); if (gzipStatus != NO_ERROR) { - ALOGW("GZipSection '%s' abnormal child process: %s", this->name.string(), - strerror(-gzipStatus)); + ALOGW("[%s] abnormal child process: %s", this->name.string(), strerror(-gzipStatus)); return gzipStatus; } // Revisit the actual size from gzip result and edit the internal buffer accordingly. @@ -389,15 +378,8 @@ status_t GZipSection::Execute(ReportRequestSet* requests) const { internalBuffer->wp()->rewind()->move(editPos); internalBuffer->writeRawVarint32(dataSize); internalBuffer->copy(dataBeginAt, dataSize); - VLOG("GZipSection '%s' wrote %zd bytes in %d ms, dataSize=%zd", this->name.string(), - buffer.size(), (int)buffer.durationMs(), dataSize); - status_t err = write_report_requests(this->id, buffer, requests); - if (err != NO_ERROR) { - ALOGW("GZipSection '%s' failed writing: %s", this->name.string(), strerror(-err)); - return err; - } - return NO_ERROR; + return write_report_requests(this->id, buffer, requests); } // ================================================================================ @@ -494,8 +476,7 @@ status_t WorkerThreadSection::Execute(ReportRequestSet* requests) const { err = buffer.read(data->pipe.readFd().get(), this->timeoutMs); if (err != NO_ERROR) { // TODO: Log this error into the incident report. - ALOGW("WorkerThreadSection '%s' reader failed with error '%s'", this->name.string(), - strerror(-err)); + ALOGW("[%s] reader failed with error '%s'", this->name.string(), strerror(-err)); } // Done with the read fd. The worker thread closes the write one so @@ -513,40 +494,26 @@ status_t WorkerThreadSection::Execute(ReportRequestSet* requests) const { if (data->workerError != NO_ERROR) { err = data->workerError; // TODO: Log this error into the incident report. - ALOGW("WorkerThreadSection '%s' worker failed with error '%s'", this->name.string(), - strerror(-err)); + ALOGW("[%s] worker failed with error '%s'", this->name.string(), strerror(-err)); } } } write_section_stats(requests->sectionStats(this->id), buffer); if (timedOut || buffer.timedOut()) { - ALOGW("WorkerThreadSection '%s' timed out", this->name.string()); + ALOGW("[%s] timed out", this->name.string()); return NO_ERROR; } - if (buffer.truncated()) { - // TODO: Log this into the incident report. - } - // TODO: There was an error with the command or buffering. Report that. For now // just exit with a log messasge. if (err != NO_ERROR) { - ALOGW("WorkerThreadSection '%s' failed with error '%s'", this->name.string(), - strerror(-err)); + ALOGW("[%s] failed with error '%s'", this->name.string(), strerror(-err)); return NO_ERROR; } // Write the data that was collected - VLOG("WorkerThreadSection '%s' wrote %zd bytes in %d ms", name.string(), buffer.size(), - (int)buffer.durationMs()); - err = write_report_requests(this->id, buffer, requests); - if (err != NO_ERROR) { - ALOGW("WorkerThreadSection '%s' failed writing: '%s'", this->name.string(), strerror(-err)); - return err; - } - - return NO_ERROR; + return write_report_requests(this->id, buffer, requests); } // ================================================================================ @@ -583,18 +550,18 @@ status_t CommandSection::Execute(ReportRequestSet* requests) const { Fpipe ihPipe; if (!cmdPipe.init() || !ihPipe.init()) { - ALOGW("CommandSection '%s' failed to setup pipes", this->name.string()); + ALOGW("[%s] failed to setup pipes", this->name.string()); return -errno; } pid_t cmdPid = fork_execute_cmd((char* const*)mCommand, NULL, &cmdPipe); if (cmdPid == -1) { - ALOGW("CommandSection '%s' failed to fork", this->name.string()); + ALOGW("[%s] failed to fork", this->name.string()); return -errno; } pid_t ihPid = fork_execute_incident_helper(this->id, &cmdPipe, &ihPipe); if (ihPid == -1) { - ALOGW("CommandSection '%s' failed to fork", this->name.string()); + ALOGW("[%s] failed to fork", this->name.string()); return -errno; } @@ -602,7 +569,7 @@ status_t CommandSection::Execute(ReportRequestSet* requests) const { status_t readStatus = buffer.read(ihPipe.readFd().get(), this->timeoutMs); write_section_stats(requests->sectionStats(this->id), buffer); if (readStatus != NO_ERROR || buffer.timedOut()) { - ALOGW("CommandSection '%s' failed to read data from incident helper: %s, timedout: %s", + ALOGW("[%s] failed to read data from incident helper: %s, timedout: %s", this->name.string(), strerror(-readStatus), buffer.timedOut() ? "true" : "false"); kill_child(cmdPid); kill_child(ihPid); @@ -614,20 +581,13 @@ status_t CommandSection::Execute(ReportRequestSet* requests) const { status_t cmdStatus = wait_child(cmdPid); status_t ihStatus = wait_child(ihPid); if (cmdStatus != NO_ERROR || ihStatus != NO_ERROR) { - ALOGW("CommandSection '%s' abnormal child processes, return status: command: %s, incident " + ALOGW("[%s] abnormal child processes, return status: command: %s, incident " "helper: %s", this->name.string(), strerror(-cmdStatus), strerror(-ihStatus)); return cmdStatus != NO_ERROR ? cmdStatus : ihStatus; } - VLOG("CommandSection '%s' wrote %zd bytes in %d ms", this->name.string(), buffer.size(), - (int)buffer.durationMs()); - status_t err = write_report_requests(this->id, buffer, requests); - if (err != NO_ERROR) { - ALOGW("CommandSection '%s' failed writing: %s", this->name.string(), strerror(-err)); - return err; - } - return NO_ERROR; + return write_report_requests(this->id, buffer, requests); } // ================================================================================ @@ -677,7 +637,7 @@ status_t DumpsysSection::BlockingCall(int pipeWriteFd) const { map<log_id_t, log_time> LogSection::gLastLogsRetrieved; LogSection::LogSection(int id, log_id_t logID) : WorkerThreadSection(id), mLogID(logID) { - name += "logcat "; + name = "logcat "; name += android_log_id_to_name(logID); switch (logID) { case LOG_ID_EVENTS: @@ -718,7 +678,7 @@ status_t LogSection::BlockingCall(int pipeWriteFd) const { android_logger_list_free); if (android_logger_open(loggers.get(), mLogID) == NULL) { - ALOGE("LogSection %s: Can't get logger.", this->name.string()); + ALOGE("[%s] Can't get logger.", this->name.string()); return -1; } @@ -734,7 +694,7 @@ status_t LogSection::BlockingCall(int pipeWriteFd) const { // err = -EAGAIN, graceful indication for ANDRODI_LOG_NONBLOCK that this is the end of data. if (err <= 0) { if (err != -EAGAIN) { - ALOGW("LogSection %s: fails to read a log_msg.\n", this->name.string()); + ALOGW("[%s] fails to read a log_msg.\n", this->name.string()); } // dump previous logs and don't consider this error a failure. break; @@ -805,7 +765,7 @@ status_t LogSection::BlockingCall(int pipeWriteFd) const { AndroidLogEntry entry; err = android_log_processLogBuffer(&msg.entry_v1, &entry); if (err != NO_ERROR) { - ALOGW("LogSection %s: fails to process to an entry.\n", this->name.string()); + ALOGW("[%s] fails to process to an entry.\n", this->name.string()); break; } lastTimestamp.tv_sec = entry.tv_sec; @@ -837,7 +797,7 @@ status_t LogSection::BlockingCall(int pipeWriteFd) const { TombstoneSection::TombstoneSection(int id, const char* type, const int64_t timeoutMs) : WorkerThreadSection(id, timeoutMs), mType(type) { - name += "tombstone "; + name = "tombstone "; name += type; } @@ -892,7 +852,7 @@ status_t TombstoneSection::BlockingCall(int pipeWriteFd) const { Fpipe dumpPipe; if (!dumpPipe.init()) { - ALOGW("TombstoneSection '%s' failed to setup dump pipe", this->name.string()); + ALOGW("[%s] failed to setup dump pipe", this->name.string()); err = -errno; break; } @@ -926,7 +886,7 @@ status_t TombstoneSection::BlockingCall(int pipeWriteFd) const { // Wait on the child to avoid it becoming a zombie process. status_t cStatus = wait_child(child); if (err != NO_ERROR) { - ALOGW("TombstoneSection '%s' failed to read stack dump: %d", this->name.string(), err); + ALOGW("[%s] failed to read stack dump: %d", this->name.string(), err); dumpPipe.readFd().reset(); break; } diff --git a/cmds/incidentd/src/Section.h b/cmds/incidentd/src/Section.h index 2f89370b0d26..86d956ff75d8 100644 --- a/cmds/incidentd/src/Section.h +++ b/cmds/incidentd/src/Section.h @@ -41,11 +41,9 @@ public: const int id; const int64_t timeoutMs; // each section must have a timeout const bool userdebugAndEngOnly; - const bool deviceSpecific; String8 name; - Section(int id, int64_t timeoutMs = REMOTE_CALL_TIMEOUT_MS, bool userdebugAndEngOnly = false, - bool deviceSpecific = false); + Section(int id, int64_t timeoutMs = REMOTE_CALL_TIMEOUT_MS, bool userdebugAndEngOnly = false); virtual ~Section(); virtual status_t Execute(ReportRequestSet* requests) const = 0; @@ -78,7 +76,7 @@ public: */ class FileSection : public Section { public: - FileSection(int id, const char* filename, bool deviceSpecific = false, + FileSection(int id, const char* filename, int64_t timeoutMs = 5000 /* 5 seconds */); virtual ~FileSection(); diff --git a/cmds/incidentd/src/Throttler.cpp b/cmds/incidentd/src/Throttler.cpp index 2b790ca14176..11136ecca091 100644 --- a/cmds/incidentd/src/Throttler.cpp +++ b/cmds/incidentd/src/Throttler.cpp @@ -18,6 +18,7 @@ #include "Throttler.h" +#include <inttypes.h> #include <utils/SystemClock.h> namespace android { @@ -42,15 +43,15 @@ bool Throttler::shouldThrottle() { } void Throttler::addReportSize(size_t reportByteSize) { - VLOG("The current request took %d bytes to dropbox", (int)reportByteSize); + VLOG("The current request took %zu bytes to dropbox", reportByteSize); mAccumulatedSize += reportByteSize; } void Throttler::dump(FILE* out) { - fprintf(out, "mSizeLimit=%d\n", (int)mSizeLimit); - fprintf(out, "mAccumulatedSize=%d\n", (int)mAccumulatedSize); - fprintf(out, "mRefractoryPeriodMs=%d\n", (int)mRefractoryPeriodMs); - fprintf(out, "mLastRefractoryMs=%d\n", (int)mLastRefractoryMs); + fprintf(out, "mSizeLimit=%zu\n", mSizeLimit); + fprintf(out, "mAccumulatedSize=%zu\n", mAccumulatedSize); + fprintf(out, "mRefractoryPeriodMs=%" PRIi64 "\n", mRefractoryPeriodMs); + fprintf(out, "mLastRefractoryMs=%" PRIi64 "\n", mLastRefractoryMs); } } // namespace incidentd diff --git a/cmds/incidentd/tests/Reporter_test.cpp b/cmds/incidentd/tests/Reporter_test.cpp index cf107c858cca..108690844280 100644 --- a/cmds/incidentd/tests/Reporter_test.cpp +++ b/cmds/incidentd/tests/Reporter_test.cpp @@ -176,7 +176,7 @@ TEST_F(ReporterTest, RunReportToGivenDirectory) { ASSERT_EQ(Reporter::REPORT_FINISHED, reporter->runReport(&size)); vector<string> results = InspectFiles(); - ASSERT_EQ((int)results.size(), 1); + ASSERT_EQ(results.size(), 1UL); EXPECT_EQ(results[0], "\n\x2" "\b\f\n\x6" diff --git a/cmds/incidentd/tests/Section_test.cpp b/cmds/incidentd/tests/Section_test.cpp index 3c338b3a36c8..9b684a060286 100644 --- a/cmds/incidentd/tests/Section_test.cpp +++ b/cmds/incidentd/tests/Section_test.cpp @@ -144,15 +144,15 @@ TEST_F(SectionTest, FileSection) { } TEST_F(SectionTest, FileSectionNotExist) { - FileSection fs1(NOOP_PARSER, "notexist", false, QUICK_TIMEOUT_MS); - ASSERT_EQ(NAME_NOT_FOUND, fs1.Execute(&requests)); + FileSection fs1(NOOP_PARSER, "notexist", QUICK_TIMEOUT_MS); + ASSERT_EQ(NO_ERROR, fs1.Execute(&requests)); - FileSection fs2(NOOP_PARSER, "notexist", true, QUICK_TIMEOUT_MS); + FileSection fs2(NOOP_PARSER, "notexist", QUICK_TIMEOUT_MS); ASSERT_EQ(NO_ERROR, fs2.Execute(&requests)); } TEST_F(SectionTest, FileSectionTimeout) { - FileSection fs(TIMEOUT_PARSER, tf.path, false, QUICK_TIMEOUT_MS); + FileSection fs(TIMEOUT_PARSER, tf.path, QUICK_TIMEOUT_MS); ASSERT_EQ(NO_ERROR, fs.Execute(&requests)); ASSERT_TRUE(requests.sectionStats(TIMEOUT_PARSER)->timed_out()); } diff --git a/cmds/input/src/com/android/commands/input/Input.java b/cmds/input/src/com/android/commands/input/Input.java index d3ec32076292..74edffb4738d 100644 --- a/cmds/input/src/com/android/commands/input/Input.java +++ b/cmds/input/src/com/android/commands/input/Input.java @@ -91,9 +91,6 @@ public class Input { if (args.length > start) { for (int i = start; i < args.length; i++) { int keyCode = KeyEvent.keyCodeFromString(args[i]); - if (keyCode == KeyEvent.KEYCODE_UNKNOWN) { - keyCode = KeyEvent.keyCodeFromString("KEYCODE_" + args[i]); - } sendKeyEvent(inputSource, keyCode, longpress); } return; diff --git a/cmds/screencap/screencap.cpp b/cmds/screencap/screencap.cpp index b11e84322dde..8fa298060d60 100644 --- a/cmds/screencap/screencap.cpp +++ b/cmds/screencap/screencap.cpp @@ -31,6 +31,7 @@ #include <gui/ISurfaceComposer.h> #include <ui/DisplayInfo.h> +#include <ui/GraphicTypes.h> #include <ui/PixelFormat.h> #include <system/graphics.h> @@ -74,12 +75,12 @@ static SkColorType flinger2skia(PixelFormat f) } } -static sk_sp<SkColorSpace> dataSpaceToColorSpace(android_dataspace d) +static sk_sp<SkColorSpace> dataSpaceToColorSpace(ui::Dataspace d) { switch (d) { - case HAL_DATASPACE_V0_SRGB: + case ui::Dataspace::V0_SRGB: return SkColorSpace::MakeSRGB(); - case HAL_DATASPACE_DISPLAY_P3: + case ui::Dataspace::DISPLAY_P3: return SkColorSpace::MakeRGB( SkColorSpace::kSRGB_RenderTargetGamma, SkColorSpace::kDCIP3_D65_Gamut); default: @@ -87,12 +88,26 @@ static sk_sp<SkColorSpace> dataSpaceToColorSpace(android_dataspace d) } } -static uint32_t dataSpaceToInt(android_dataspace d) +static ui::Dataspace pickBestDataspace(ui::ColorMode colorMode) +{ + switch (colorMode) { + case ui::ColorMode::SRGB: + return ui::Dataspace::V0_SRGB; + case ui::ColorMode::DISPLAY_P3: + case ui::ColorMode::BT2100_PQ: + case ui::ColorMode::BT2100_HLG: + return ui::Dataspace::DISPLAY_P3; + default: + return ui::Dataspace::V0_SRGB; + } +} + +static uint32_t dataSpaceToInt(ui::Dataspace d) { switch (d) { - case HAL_DATASPACE_V0_SRGB: + case ui::Dataspace::V0_SRGB: return COLORSPACE_SRGB; - case HAL_DATASPACE_DISPLAY_P3: + case ui::Dataspace::DISPLAY_P3: return COLORSPACE_DISPLAY_P3; default: return COLORSPACE_UNKNOWN; @@ -161,7 +176,6 @@ int main(int argc, char** argv) void* base = NULL; uint32_t w, s, h, f; - android_dataspace d; size_t size = 0; // Maps orientations from DisplayInfo to ISurfaceComposer @@ -197,9 +211,15 @@ int main(int argc, char** argv) uint32_t captureOrientation = ORIENTATION_MAP[displayOrientation]; sp<GraphicBuffer> outBuffer; - status_t result = ScreenshotClient::capture(display, Rect(), 0 /* reqWidth */, - 0 /* reqHeight */, INT32_MIN, INT32_MAX, /* all layers */ false, captureOrientation, - &outBuffer); + ui::Dataspace reqDataspace = + pickBestDataspace(SurfaceComposerClient::getActiveColorMode(display)); + + // Due to the fact that we hard code the way we write pixels into screenshot, + // we hard code RGBA_8888 here. + ui::PixelFormat reqPixelFormat = ui::PixelFormat::RGBA_8888; + status_t result = ScreenshotClient::capture(display, reqDataspace, reqPixelFormat, Rect(), + 0 /* reqWidth */, 0 /* reqHeight */, false, + captureOrientation, &outBuffer); if (result != NO_ERROR) { close(fd); return 1; @@ -207,7 +227,14 @@ int main(int argc, char** argv) result = outBuffer->lock(GraphicBuffer::USAGE_SW_READ_OFTEN, &base); - if (base == NULL) { + if (base == nullptr || result != NO_ERROR) { + String8 reason; + if (base == nullptr) { + reason = "Failed to write to buffer"; + } else { + reason.appendFormat("Error Code: %d", result); + } + fprintf(stderr, "Failed to take screenshot (%s)\n", reason.c_str()); close(fd); return 1; } @@ -216,12 +243,12 @@ int main(int argc, char** argv) h = outBuffer->getHeight(); s = outBuffer->getStride(); f = outBuffer->getPixelFormat(); - d = HAL_DATASPACE_UNKNOWN; size = s * h * bytesPerPixel(f); if (png) { const SkImageInfo info = - SkImageInfo::Make(w, h, flinger2skia(f), kPremul_SkAlphaType, dataSpaceToColorSpace(d)); + SkImageInfo::Make(w, h, flinger2skia(f), kPremul_SkAlphaType, + dataSpaceToColorSpace(reqDataspace)); SkPixmap pixmap(info, base, s * bytesPerPixel(f)); struct FDWStream final : public SkWStream { size_t fBytesWritten = 0; @@ -238,7 +265,7 @@ int main(int argc, char** argv) notifyMediaScanner(fn); } } else { - uint32_t c = dataSpaceToInt(d); + uint32_t c = dataSpaceToInt(reqDataspace); write(fd, &w, 4); write(fd, &h, 4); write(fd, &f, 4); @@ -255,4 +282,4 @@ int main(int argc, char** argv) } return 0; -}
\ No newline at end of file +} diff --git a/cmds/sm/src/com/android/commands/sm/Sm.java b/cmds/sm/src/com/android/commands/sm/Sm.java index 70e81dfe7404..783c8c45d603 100644 --- a/cmds/sm/src/com/android/commands/sm/Sm.java +++ b/cmds/sm/src/com/android/commands/sm/Sm.java @@ -101,6 +101,8 @@ public final class Sm { runFstrim(); } else if ("set-virtual-disk".equals(op)) { runSetVirtualDisk(); + } else if ("set-isolated-storage".equals(op)) { + runIsolatedStorage(); } else { throw new IllegalArgumentException(); } @@ -280,6 +282,20 @@ public final class Sm { StorageManager.DEBUG_VIRTUAL_DISK); } + public void runIsolatedStorage() throws RemoteException { + final boolean enableIsolatedStorage = Boolean.parseBoolean(nextArg()); + // Toggling isolated-storage state will result in a device reboot. So to avoid this command + // from erroring out (DeadSystemException), call setDebugFlags() in a separate thread. + new Thread(() -> { + try { + mSm.setDebugFlags(enableIsolatedStorage ? StorageManager.DEBUG_ISOLATED_STORAGE : 0, + StorageManager.DEBUG_ISOLATED_STORAGE); + } catch (RemoteException e) { + Log.e(TAG, "Encountered an error!", e); + } + }).start(); + } + public void runIdleMaint() throws RemoteException { final boolean im_run = "run".equals(nextArg()); if (im_run) { @@ -318,6 +334,8 @@ public final class Sm { System.err.println(""); System.err.println(" sm set-emulate-fbe [true|false]"); System.err.println(""); + System.err.println(" sm set-isolated-storage [true|false]"); + System.err.println(""); return 1; } } diff --git a/cmds/statsd/Android.bp b/cmds/statsd/Android.bp index 5c3d17e30585..a3cd8a3bb32b 100644 --- a/cmds/statsd/Android.bp +++ b/cmds/statsd/Android.bp @@ -68,17 +68,18 @@ cc_defaults { "src/config/ConfigListener.cpp", "src/config/ConfigManager.cpp", "src/external/Perfetto.cpp", + "src/external/Perfprofd.cpp", "src/external/StatsPuller.cpp", "src/external/StatsCompanionServicePuller.cpp", "src/external/SubsystemSleepStatePuller.cpp", "src/external/ResourceHealthManagerPuller.cpp", "src/external/ResourceThermalManagerPuller.cpp", - "src/external/StatsPullerManagerImpl.cpp", + "src/external/StatsPullerManager.cpp", "src/external/puller_util.cpp", "src/logd/LogEvent.cpp", "src/logd/LogListener.cpp", - "src/logd/LogReader.cpp", "src/matchers/CombinationLogMatchingTracker.cpp", + "src/matchers/EventMatcherWizard.cpp", "src/matchers/matcher_util.cpp", "src/matchers/SimpleLogMatchingTracker.cpp", "src/metrics/MetricProducer.cpp", @@ -92,7 +93,6 @@ cc_defaults { "src/metrics/MetricsManager.cpp", "src/metrics/metrics_manager_util.cpp", "src/packages/UidMap.cpp", - "src/perfetto/perfetto_config.proto", "src/storage/StorageManager.cpp", "src/StatsLogProcessor.cpp", "src/StatsService.cpp", @@ -102,6 +102,10 @@ cc_defaults { "src/HashableDimensionKey.cpp", "src/guardrail/StatsdStats.cpp", "src/socket/StatsSocketListener.cpp", + "src/shell/ShellSubscriber.cpp", + "src/shell/shell_config.proto", + + ":perfprofd_aidl", ], local_include_dirs: [ @@ -110,7 +114,6 @@ cc_defaults { static_libs: [ "libhealthhalutils", - "libplatformprotos", ], shared_libs: [ @@ -127,6 +130,7 @@ cc_defaults { "libhidlbase", "libhidltransport", "libhwbinder", + "android.frameworks.stats@1.0", "android.hardware.health@2.0", "android.hardware.power@1.0", "android.hardware.power@1.1", @@ -160,11 +164,14 @@ cc_binary { product_variables: { eng: { - // Enable sanitizer and allow very verbose printing on eng builds + // Enable sanitizer ONLY on eng builds + //sanitize: { + // address: true, + //}, + }, + debuggable: { + // Add a flag to enable stats log printing from statsd on debug builds. cflags: ["-DVERY_VERBOSE_PRINTING"], - sanitize: { - address: true, - }, }, }, @@ -174,6 +181,8 @@ cc_binary { shared_libs: ["libgtest_prod"], + vintf_fragments: ["android.frameworks.stats@1.0-service.xml"], + init_rc: ["statsd.rc"], } @@ -199,6 +208,7 @@ cc_test { "src/atom_field_options.proto", "src/atoms.proto", "src/stats_log.proto", + "src/shell/shell_data.proto", "tests/AlarmMonitor_test.cpp", "tests/anomaly/AlarmTracker_test.cpp", "tests/anomaly/AnomalyTracker_test.cpp", @@ -206,7 +216,6 @@ cc_test { "tests/external/puller_util_test.cpp", "tests/indexed_priority_queue_test.cpp", "tests/LogEntryMatcher_test.cpp", - "tests/LogReader_test.cpp", "tests/LogEvent_test.cpp", "tests/MetricsManager_test.cpp", "tests/StatsLogProcessor_test.cpp", @@ -227,6 +236,7 @@ cc_test { "tests/metrics/metrics_test_helper.cpp", "tests/statsd_test_util.cpp", "tests/e2e/WakelockDuration_e2e_test.cpp", + "tests/e2e/MetricActivation_e2e_test.cpp", "tests/e2e/MetricConditionLink_e2e_test.cpp", "tests/e2e/Alarm_e2e_test.cpp", "tests/e2e/Attribution_e2e_test.cpp", @@ -240,9 +250,13 @@ cc_test { "tests/e2e/Anomaly_duration_sum_e2e_test.cpp", "tests/e2e/ConfigTtl_e2e_test.cpp", "tests/e2e/PartialBucket_e2e_test.cpp", + "tests/shell/ShellSubscriber_test.cpp", ], - static_libs: ["libgmock"], + static_libs: [ + "libgmock", + "libplatformprotos", + ], proto: { type: "full", @@ -291,6 +305,10 @@ cc_benchmark { "-Wno-varargs" ], + static_libs: [ + "libplatformprotos", + ], + shared_libs: [ "libgtest_prod", "libstatslog", @@ -310,7 +328,6 @@ java_library { srcs: [ "src/stats_log.proto", "src/statsd_config.proto", - "src/perfetto/perfetto_config.proto", "src/atoms.proto", ], @@ -322,5 +339,3 @@ java_library { javacflags: ["-XepDisableAllChecks"], }, } - - diff --git a/cmds/statsd/android.frameworks.stats@1.0-service.xml b/cmds/statsd/android.frameworks.stats@1.0-service.xml new file mode 100644 index 000000000000..bb02f66a28b1 --- /dev/null +++ b/cmds/statsd/android.frameworks.stats@1.0-service.xml @@ -0,0 +1,11 @@ +<manifest version="1.0" type="framework"> + <hal> + <name>android.frameworks.stats</name> + <transport>hwbinder</transport> + <version>1.0</version> + <interface> + <name>IStats</name> + <instance>default</instance> + </interface> + </hal> +</manifest> diff --git a/cmds/statsd/benchmark/metric_util.cpp b/cmds/statsd/benchmark/metric_util.cpp index 50ed18d3e2b0..067b6eddf254 100644 --- a/cmds/statsd/benchmark/metric_util.cpp +++ b/cmds/statsd/benchmark/metric_util.cpp @@ -362,11 +362,12 @@ std::unique_ptr<LogEvent> CreateSyncEndEvent( sp<StatsLogProcessor> CreateStatsLogProcessor(const long timeBaseSec, const StatsdConfig& config, const ConfigKey& key) { sp<UidMap> uidMap = new UidMap(); + sp<StatsPullerManager> pullerManager = new StatsPullerManager(); sp<AlarmMonitor> anomalyAlarmMonitor; sp<AlarmMonitor> periodicAlarmMonitor; - sp<StatsLogProcessor> processor = new StatsLogProcessor( - uidMap, anomalyAlarmMonitor, periodicAlarmMonitor, timeBaseSec * NS_PER_SEC, - [](const ConfigKey&){return true;}); + sp<StatsLogProcessor> processor = + new StatsLogProcessor(uidMap, pullerManager, anomalyAlarmMonitor, periodicAlarmMonitor, + timeBaseSec * NS_PER_SEC, [](const ConfigKey&) { return true; }); processor->OnConfigUpdated(timeBaseSec * NS_PER_SEC, key, config); return processor; } diff --git a/cmds/statsd/src/FieldValue.cpp b/cmds/statsd/src/FieldValue.cpp index f150f074c52b..80ed80776829 100644 --- a/cmds/statsd/src/FieldValue.cpp +++ b/cmds/statsd/src/FieldValue.cpp @@ -18,6 +18,7 @@ #include "Log.h" #include "FieldValue.h" #include "HashableDimensionKey.h" +#include "math.h" namespace android { namespace os { @@ -141,9 +142,15 @@ Value::Value(const Value& from) { case FLOAT: float_value = from.float_value; break; + case DOUBLE: + double_value = from.double_value; + break; case STRING: str_value = from.str_value; break; + case STORAGE: + storage_value = from.storage_value; + break; default: break; } @@ -157,13 +164,36 @@ std::string Value::toString() const { return std::to_string(long_value) + "[L]"; case FLOAT: return std::to_string(float_value) + "[F]"; + case DOUBLE: + return std::to_string(double_value) + "[D]"; case STRING: return str_value + "[S]"; + case STORAGE: + return "bytes of size " + std::to_string(storage_value.size()) + "[ST]"; default: return "[UNKNOWN]"; } } +bool Value::isZero() const { + switch (type) { + case INT: + return int_value == 0; + case LONG: + return long_value == 0; + case FLOAT: + return fabs(float_value) <= std::numeric_limits<float>::epsilon(); + case DOUBLE: + return fabs(double_value) <= std::numeric_limits<double>::epsilon(); + case STRING: + return str_value.size() == 0; + case STORAGE: + return storage_value.size() == 0; + default: + return false; + } +} + bool Value::operator==(const Value& that) const { if (type != that.getType()) return false; @@ -174,8 +204,12 @@ bool Value::operator==(const Value& that) const { return long_value == that.long_value; case FLOAT: return float_value == that.float_value; + case DOUBLE: + return double_value == that.double_value; case STRING: return str_value == that.str_value; + case STORAGE: + return storage_value == that.storage_value; default: return false; } @@ -190,8 +224,12 @@ bool Value::operator!=(const Value& that) const { return long_value != that.long_value; case FLOAT: return float_value != that.float_value; + case DOUBLE: + return double_value != that.double_value; case STRING: return str_value != that.str_value; + case STORAGE: + return storage_value != that.storage_value; default: return false; } @@ -207,13 +245,169 @@ bool Value::operator<(const Value& that) const { return long_value < that.long_value; case FLOAT: return float_value < that.float_value; + case DOUBLE: + return double_value < that.double_value; case STRING: return str_value < that.str_value; + case STORAGE: + return storage_value < that.storage_value; default: return false; } } +bool Value::operator>(const Value& that) const { + if (type != that.getType()) return type > that.getType(); + + switch (type) { + case INT: + return int_value > that.int_value; + case LONG: + return long_value > that.long_value; + case FLOAT: + return float_value > that.float_value; + case DOUBLE: + return double_value > that.double_value; + case STRING: + return str_value > that.str_value; + case STORAGE: + return storage_value > that.storage_value; + default: + return false; + } +} + +bool Value::operator>=(const Value& that) const { + if (type != that.getType()) return type >= that.getType(); + + switch (type) { + case INT: + return int_value >= that.int_value; + case LONG: + return long_value >= that.long_value; + case FLOAT: + return float_value >= that.float_value; + case DOUBLE: + return double_value >= that.double_value; + case STRING: + return str_value >= that.str_value; + case STORAGE: + return storage_value >= that.storage_value; + default: + return false; + } +} + +Value Value::operator-(const Value& that) const { + Value v; + if (type != that.type) { + ALOGE("Can't operate on different value types, %d, %d", type, that.type); + return v; + } + if (type == STRING) { + ALOGE("Can't operate on string value type"); + return v; + } + + if (type == STORAGE) { + ALOGE("Can't operate on storage value type"); + return v; + } + + switch (type) { + case INT: + v.setInt(int_value - that.int_value); + break; + case LONG: + v.setLong(long_value - that.long_value); + break; + case FLOAT: + v.setFloat(float_value - that.float_value); + break; + case DOUBLE: + v.setDouble(double_value - that.double_value); + break; + default: + break; + } + return v; +} + +Value& Value::operator=(const Value& that) { + type = that.type; + switch (type) { + case INT: + int_value = that.int_value; + break; + case LONG: + long_value = that.long_value; + break; + case FLOAT: + float_value = that.float_value; + break; + case DOUBLE: + double_value = that.double_value; + break; + case STRING: + str_value = that.str_value; + break; + case STORAGE: + storage_value = that.storage_value; + break; + default: + break; + } + return *this; +} + +Value& Value::operator+=(const Value& that) { + if (type != that.type) { + ALOGE("Can't operate on different value types, %d, %d", type, that.type); + return *this; + } + if (type == STRING) { + ALOGE("Can't operate on string value type"); + return *this; + } + if (type == STORAGE) { + ALOGE("Can't operate on storage value type"); + return *this; + } + + switch (type) { + case INT: + int_value += that.int_value; + break; + case LONG: + long_value += that.long_value; + break; + case FLOAT: + float_value += that.float_value; + break; + case DOUBLE: + double_value += that.double_value; + break; + default: + break; + } + return *this; +} + +double Value::getDouble() const { + switch (type) { + case INT: + return int_value; + case LONG: + return long_value; + case FLOAT: + return float_value; + case DOUBLE: + return double_value; + default: + return 0; + } +} + bool equalDimensions(const std::vector<Matcher>& dimension_a, const std::vector<Matcher>& dimension_b) { bool eq = dimension_a.size() == dimension_b.size(); diff --git a/cmds/statsd/src/FieldValue.h b/cmds/statsd/src/FieldValue.h index 02c49b99c583..a5d00ac4e72b 100644 --- a/cmds/statsd/src/FieldValue.h +++ b/cmds/statsd/src/FieldValue.h @@ -32,7 +32,7 @@ const int32_t kLastBitMask = 0x80; const int32_t kClearLastBitDeco = 0x7f; const int32_t kClearAllPositionMatcherMask = 0xffff00ff; -enum Type { UNKNOWN, INT, LONG, FLOAT, STRING }; +enum Type { UNKNOWN, INT, LONG, FLOAT, DOUBLE, STRING, STORAGE }; int32_t getEncodedField(int32_t pos[], int32_t depth, bool includeDepth); @@ -212,7 +212,7 @@ public: * the result is equal to the Matcher Field. That's a bit wise AND operation + check if 2 ints are * equal. Nothing can beat the performance of this matching algorithm. * - * TODO: ADD EXAMPLE HERE. + * TODO(b/110561213): ADD EXAMPLE HERE. */ struct Matcher { Matcher(const Field& matcher, int32_t mask) : mMatcher(matcher), mMask(mask){}; @@ -283,11 +283,21 @@ struct Value { type = FLOAT; } + Value(double v) { + double_value = v; + type = DOUBLE; + } + Value(const std::string& v) { str_value = v; type = STRING; } + Value(const std::vector<uint8_t>& v) { + storage_value = v; + type = STORAGE; + } + void setInt(int32_t v) { int_value = v; type = INT; @@ -298,27 +308,48 @@ struct Value { type = LONG; } + void setFloat(float v) { + float_value = v; + type = FLOAT; + } + + void setDouble(double v) { + double_value = v; + type = DOUBLE; + } + union { int32_t int_value; int64_t long_value; float float_value; + double double_value; }; std::string str_value; + std::vector<uint8_t> storage_value; Type type; std::string toString() const; + bool isZero() const; + Type getType() const { return type; } + double getDouble() const; + Value(const Value& from); bool operator==(const Value& that) const; bool operator!=(const Value& that) const; bool operator<(const Value& that) const; + bool operator>(const Value& that) const; + bool operator>=(const Value& that) const; + Value operator-(const Value& that) const; + Value& operator+=(const Value& that); + Value& operator=(const Value& that); }; /** diff --git a/cmds/statsd/src/HashableDimensionKey.cpp b/cmds/statsd/src/HashableDimensionKey.cpp index 71030345b0aa..af8b3af6ea61 100644 --- a/cmds/statsd/src/HashableDimensionKey.cpp +++ b/cmds/statsd/src/HashableDimensionKey.cpp @@ -65,8 +65,6 @@ bool filterValues(const vector<Matcher>& matcherFields, const vector<FieldValue> for (const auto& value : values) { for (size_t i = 0; i < matcherFields.size(); ++i) { const auto& matcher = matcherFields[i]; - // TODO: potential optimization here to break early because all fields are naturally - // sorted. if (value.mField.matches(matcher)) { output->addValue(value); output->mutableValue(num_matches)->mField.setTag(value.mField.getTag()); @@ -196,4 +194,4 @@ bool MetricDimensionKey::operator<(const MetricDimensionKey& that) const { } // namespace statsd } // namespace os -} // namespace android
\ No newline at end of file +} // namespace android diff --git a/cmds/statsd/src/StatsLogProcessor.cpp b/cmds/statsd/src/StatsLogProcessor.cpp index e7adba4d194a..eb498f596141 100644 --- a/cmds/statsd/src/StatsLogProcessor.cpp +++ b/cmds/statsd/src/StatsLogProcessor.cpp @@ -75,18 +75,20 @@ const int FIELD_ID_STRINGS = 9; #define WRITE_DATA_COOL_DOWN_SEC 5 StatsLogProcessor::StatsLogProcessor(const sp<UidMap>& uidMap, + const sp<StatsPullerManager>& pullerManager, const sp<AlarmMonitor>& anomalyAlarmMonitor, const sp<AlarmMonitor>& periodicAlarmMonitor, const int64_t timeBaseNs, const std::function<bool(const ConfigKey&)>& sendBroadcast) : mUidMap(uidMap), + mPullerManager(pullerManager), mAnomalyAlarmMonitor(anomalyAlarmMonitor), mPeriodicAlarmMonitor(periodicAlarmMonitor), mSendBroadcast(sendBroadcast), mTimeBaseNs(timeBaseNs), mLargestTimestampSeen(0), mLastTimestampSeen(0) { - mStatsPullerManager.ForceClearPullerCache(); + mPullerManager->ForceClearPullerCache(); } StatsLogProcessor::~StatsLogProcessor() { @@ -155,17 +157,13 @@ void StatsLogProcessor::onIsolatedUidChangedEventLocked(const LogEvent& event) { if (is_create) { mUidMap->assignIsolatedUid(isolated_uid, parent_uid); } else { - mUidMap->removeIsolatedUid(isolated_uid, parent_uid); + mUidMap->removeIsolatedUid(isolated_uid); } } else { ALOGE("Failed to parse uid in the isolated uid change event."); } } -void StatsLogProcessor::OnLogEvent(LogEvent* event) { - OnLogEvent(event, false); -} - void StatsLogProcessor::resetConfigs() { std::lock_guard<std::mutex> lock(mMetricsMutex); resetConfigsLocked(getElapsedRealtimeNs()); @@ -179,7 +177,7 @@ void StatsLogProcessor::resetConfigsLocked(const int64_t timestampNs) { resetConfigsLocked(timestampNs, configKeys); } -void StatsLogProcessor::OnLogEvent(LogEvent* event, bool reconnected) { +void StatsLogProcessor::OnLogEvent(LogEvent* event) { std::lock_guard<std::mutex> lock(mMetricsMutex); #ifdef VERY_VERBOSE_PRINTING @@ -189,41 +187,6 @@ void StatsLogProcessor::OnLogEvent(LogEvent* event, bool reconnected) { #endif const int64_t currentTimestampNs = event->GetElapsedTimestampNs(); - if (reconnected && mLastTimestampSeen != 0) { - // LogReader tells us the connection has just been reset. Now we need - // to enter reconnection state to find the last CP. - mInReconnection = true; - } - - if (mInReconnection) { - // We see the checkpoint - if (currentTimestampNs == mLastTimestampSeen) { - mInReconnection = false; - // Found the CP. ignore this event, and we will start to read from next event. - return; - } - if (currentTimestampNs > mLargestTimestampSeen) { - // We see a new log but CP has not been found yet. Give up now. - mLogLossCount++; - mInReconnection = false; - StatsdStats::getInstance().noteLogLost(currentTimestampNs); - // Persist the data before we reset. Do we want this? - WriteDataToDiskLocked(CONFIG_RESET); - // We see fresher event before we see the checkpoint. We might have lost data. - // The best we can do is to reset. - resetConfigsLocked(currentTimestampNs); - } else { - // Still in search of the CP. Keep going. - return; - } - } - - mLogCount++; - mLastTimestampSeen = currentTimestampNs; - if (mLargestTimestampSeen < currentTimestampNs) { - mLargestTimestampSeen = currentTimestampNs; - } - resetIfConfigTtlExpiredLocked(currentTimestampNs); StatsdStats::getInstance().noteAtomLogged( @@ -241,7 +204,7 @@ void StatsLogProcessor::OnLogEvent(LogEvent* event, bool reconnected) { int64_t curTimeSec = getElapsedRealtimeSec(); if (curTimeSec - mLastPullerCacheClearTimeSec > StatsdStats::kPullerCacheClearIntervalSec) { - mStatsPullerManager.ClearPullerCacheIfNecessary(curTimeSec * NS_PER_SEC); + mPullerManager->ClearPullerCacheIfNecessary(curTimeSec * NS_PER_SEC); mLastPullerCacheClearTimeSec = curTimeSec; } @@ -269,8 +232,8 @@ void StatsLogProcessor::OnConfigUpdatedLocked( const int64_t timestampNs, const ConfigKey& key, const StatsdConfig& config) { VLOG("Updated configuration for key %s", key.ToString().c_str()); sp<MetricsManager> newMetricsManager = - new MetricsManager(key, config, mTimeBaseNs, timestampNs, mUidMap, - mAnomalyAlarmMonitor, mPeriodicAlarmMonitor); + new MetricsManager(key, config, mTimeBaseNs, timestampNs, mUidMap, mPullerManager, + mAnomalyAlarmMonitor, mPeriodicAlarmMonitor); if (newMetricsManager->isConfigValid()) { mUidMap->OnConfigUpdated(key); if (newMetricsManager->shouldAddUidMapListener()) { @@ -297,35 +260,40 @@ size_t StatsLogProcessor::GetMetricsSize(const ConfigKey& key) const { return it->second->byteSize(); } -void StatsLogProcessor::dumpStates(FILE* out, bool verbose) { +void StatsLogProcessor::dumpStates(int out, bool verbose) { std::lock_guard<std::mutex> lock(mMetricsMutex); - fprintf(out, "MetricsManager count: %lu\n", (unsigned long)mMetricsManagers.size()); + FILE* fout = fdopen(out, "w"); + if (fout == NULL) { + return; + } + fprintf(fout, "MetricsManager count: %lu\n", (unsigned long)mMetricsManagers.size()); for (auto metricsManager : mMetricsManagers) { - metricsManager.second->dumpStates(out, verbose); + metricsManager.second->dumpStates(fout, verbose); } + + fclose(fout); } /* - * onDumpReport dumps serialized ConfigMetricsReportList into outData. + * onDumpReport dumps serialized ConfigMetricsReportList into proto. */ void StatsLogProcessor::onDumpReport(const ConfigKey& key, const int64_t dumpTimeStampNs, const bool include_current_partial_bucket, + const bool erase_data, const DumpReportReason dumpReportReason, - vector<uint8_t>* outData) { + ProtoOutputStream* proto) { std::lock_guard<std::mutex> lock(mMetricsMutex); - ProtoOutputStream proto; - // Start of ConfigKey. - uint64_t configKeyToken = proto.start(FIELD_TYPE_MESSAGE | FIELD_ID_CONFIG_KEY); - proto.write(FIELD_TYPE_INT32 | FIELD_ID_UID, key.GetUid()); - proto.write(FIELD_TYPE_INT64 | FIELD_ID_ID, (long long)key.GetId()); - proto.end(configKeyToken); + uint64_t configKeyToken = proto->start(FIELD_TYPE_MESSAGE | FIELD_ID_CONFIG_KEY); + proto->write(FIELD_TYPE_INT32 | FIELD_ID_UID, key.GetUid()); + proto->write(FIELD_TYPE_INT64 | FIELD_ID_ID, (long long)key.GetId()); + proto->end(configKeyToken); // End of ConfigKey. // Then, check stats-data directory to see there's any file containing // ConfigMetricsReport from previous shutdowns to concatenate to reports. - StorageManager::appendConfigMetricsReport(key, &proto); + StorageManager::appendConfigMetricsReport(key, proto); auto it = mMetricsManagers.find(key); if (it != mMetricsManagers.end()) { @@ -335,14 +303,27 @@ void StatsLogProcessor::onDumpReport(const ConfigKey& key, const int64_t dumpTim // Start of ConfigMetricsReport (reports). uint64_t reportsToken = - proto.start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_REPORTS); + proto->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_REPORTS); onConfigMetricsReportLocked(key, dumpTimeStampNs, include_current_partial_bucket, - dumpReportReason, &proto); - proto.end(reportsToken); + erase_data, dumpReportReason, proto); + proto->end(reportsToken); // End of ConfigMetricsReport (reports). } else { ALOGW("Config source %s does not exist", key.ToString().c_str()); } +} + +/* + * onDumpReport dumps serialized ConfigMetricsReportList into outData. + */ +void StatsLogProcessor::onDumpReport(const ConfigKey& key, const int64_t dumpTimeStampNs, + const bool include_current_partial_bucket, + const bool erase_data, + const DumpReportReason dumpReportReason, + vector<uint8_t>* outData) { + ProtoOutputStream proto; + onDumpReport(key, dumpTimeStampNs, include_current_partial_bucket, erase_data, + dumpReportReason, &proto); if (outData != nullptr) { outData->clear(); @@ -366,6 +347,7 @@ void StatsLogProcessor::onDumpReport(const ConfigKey& key, const int64_t dumpTim void StatsLogProcessor::onConfigMetricsReportLocked(const ConfigKey& key, const int64_t dumpTimeStampNs, const bool include_current_partial_bucket, + const bool erase_data, const DumpReportReason dumpReportReason, ProtoOutputStream* proto) { // We already checked whether key exists in mMetricsManagers in @@ -382,7 +364,7 @@ void StatsLogProcessor::onConfigMetricsReportLocked(const ConfigKey& key, // First, fill in ConfigMetricsReport using current data on memory, which // starts from filling in StatsLogReport's. it->second->onDumpReport(dumpTimeStampNs, include_current_partial_bucket, - &str_set, proto); + erase_data, &str_set, proto); // Fill in UidMap if there is at least one metric to report. // This skips the uid map if it's an empty config. @@ -456,7 +438,7 @@ void StatsLogProcessor::OnConfigRemoved(const ConfigKey& key) { mLastBroadcastTimes.erase(key); if (mMetricsManagers.empty()) { - mStatsPullerManager.ForceClearPullerCache(); + mPullerManager->ForceClearPullerCache(); } } @@ -476,7 +458,7 @@ void StatsLogProcessor::flushIfNecessaryLocked( if (totalBytes > StatsdStats::kMaxMetricsBytesPerConfig) { // Too late. We need to start clearing data. metricsManager.dropData(timestampNs); - StatsdStats::getInstance().noteDataDropped(key); + StatsdStats::getInstance().noteDataDropped(key, totalBytes); VLOG("StatsD had to toss out metrics for %s", key.ToString().c_str()); } else if ((totalBytes > StatsdStats::kBytesPerConfigTriggerGetData) || (mOnDiskDataConfigs.find(key) != mOnDiskDataConfigs.end())) { @@ -513,7 +495,7 @@ void StatsLogProcessor::WriteDataToDiskLocked(const ConfigKey& key, } ProtoOutputStream proto; onConfigMetricsReportLocked(key, timestampNs, true /* include_current_partial_bucket*/, - dumpReportReason, &proto); + true /* erase_data */, dumpReportReason, &proto); string file_name = StringPrintf("%s/%ld_%d_%lld", STATS_DATA_DIR, (long)getWallClockSec(), key.GetUid(), (long long)key.GetId()); android::base::unique_fd fd(open(file_name.c_str(), @@ -551,7 +533,7 @@ void StatsLogProcessor::WriteDataToDisk(const DumpReportReason dumpReportReason) void StatsLogProcessor::informPullAlarmFired(const int64_t timestampNs) { std::lock_guard<std::mutex> lock(mMetricsMutex); - mStatsPullerManager.OnAlarmFired(timestampNs); + mPullerManager->OnAlarmFired(timestampNs); } int64_t StatsLogProcessor::getLastReportTimeNs(const ConfigKey& key) { diff --git a/cmds/statsd/src/StatsLogProcessor.h b/cmds/statsd/src/StatsLogProcessor.h index 86eb855825aa..a5ce9b65f899 100644 --- a/cmds/statsd/src/StatsLogProcessor.h +++ b/cmds/statsd/src/StatsLogProcessor.h @@ -18,7 +18,6 @@ #include <gtest/gtest_prod.h> #include "config/ConfigListener.h" -#include "logd/LogReader.h" #include "metrics/MetricsManager.h" #include "packages/UidMap.h" #include "external/StatsPullerManager.h" @@ -46,15 +45,13 @@ enum DumpReportReason { class StatsLogProcessor : public ConfigListener { public: - StatsLogProcessor(const sp<UidMap>& uidMap, const sp<AlarmMonitor>& anomalyAlarmMonitor, + StatsLogProcessor(const sp<UidMap>& uidMap, const sp<StatsPullerManager>& pullerManager, + const sp<AlarmMonitor>& anomalyAlarmMonitor, const sp<AlarmMonitor>& subscriberTriggerAlarmMonitor, const int64_t timeBaseNs, const std::function<bool(const ConfigKey&)>& sendBroadcast); virtual ~StatsLogProcessor(); - void OnLogEvent(LogEvent* event, bool reconnectionStarts); - - // for testing only. void OnLogEvent(LogEvent* event); void OnConfigUpdated(const int64_t timestampNs, const ConfigKey& key, @@ -64,8 +61,11 @@ public: size_t GetMetricsSize(const ConfigKey& key) const; void onDumpReport(const ConfigKey& key, const int64_t dumpTimeNs, - const bool include_current_partial_bucket, + const bool include_current_partial_bucket, const bool erase_data, const DumpReportReason dumpReportReason, vector<uint8_t>* outData); + void onDumpReport(const ConfigKey& key, const int64_t dumpTimeNs, + const bool include_current_partial_bucket, const bool erase_data, + const DumpReportReason dumpReportReason, ProtoOutputStream* proto); /* Tells MetricsManager that the alarms in alarmSet have fired. Modifies anomaly alarmSet. */ void onAnomalyAlarmFired( @@ -87,7 +87,7 @@ public: return mUidMap; } - void dumpStates(FILE* out, bool verbose); + void dumpStates(int outFd, bool verbose); void informPullAlarmFired(const int64_t timestampNs); @@ -127,7 +127,7 @@ private: sp<UidMap> mUidMap; // Reference to the UidMap to lookup app name and version for each uid. - StatsPullerManager mStatsPullerManager; + sp<StatsPullerManager> mPullerManager; // Reference to StatsPullerManager sp<AlarmMonitor> mAnomalyAlarmMonitor; @@ -144,6 +144,7 @@ private: void onConfigMetricsReportLocked(const ConfigKey& key, const int64_t dumpTimeStampNs, const bool include_current_partial_bucket, + const bool erase_data, const DumpReportReason dumpReportReason, util::ProtoOutputStream* proto); @@ -174,14 +175,6 @@ private: int64_t mLastTimestampSeen = 0; - bool mInReconnection = false; - - // Processed log count - uint64_t mLogCount = 0; - - // Log loss detected count - int mLogLossCount = 0; - long mLastPullerCacheClearTimeSec = 0; // Last time we wrote data to disk. @@ -208,7 +201,7 @@ private: FRIEND_TEST(GaugeMetricE2eTest, TestMultipleFieldsForPushedEvent); FRIEND_TEST(GaugeMetricE2eTest, TestRandomSamplePulledEvents); FRIEND_TEST(GaugeMetricE2eTest, TestRandomSamplePulledEvent_LateAlarm); - FRIEND_TEST(GaugeMetricE2eTest, TestAllConditionChangesSamplePulledEvents); + FRIEND_TEST(GaugeMetricE2eTest, TestConditionChangeToTrueSamplePulledEvents); FRIEND_TEST(ValueMetricE2eTest, TestPulledEvents); FRIEND_TEST(ValueMetricE2eTest, TestPulledEvents_LateAlarm); @@ -233,6 +226,7 @@ private: FRIEND_TEST(AlarmE2eTest, TestMultipleAlarms); FRIEND_TEST(ConfigTtlE2eTest, TestCountMetric); + FRIEND_TEST(MetricActivationE2eTest, TestCountMetric); }; } // namespace statsd diff --git a/cmds/statsd/src/StatsService.cpp b/cmds/statsd/src/StatsService.cpp index cb48a716231f..27685fc108a0 100644 --- a/cmds/statsd/src/StatsService.cpp +++ b/cmds/statsd/src/StatsService.cpp @@ -34,17 +34,20 @@ #include <dirent.h> #include <frameworks/base/cmds/statsd/src/statsd_config.pb.h> #include <private/android_filesystem_config.h> -#include <utils/Looper.h> -#include <utils/String16.h> #include <statslog.h> #include <stdio.h> #include <stdlib.h> #include <sys/system_properties.h> #include <unistd.h> +#include <utils/Looper.h> +#include <utils/String16.h> +#include <chrono> using namespace android; using android::base::StringPrintf; +using android::util::FIELD_COUNT_REPEATED; +using android::util::FIELD_TYPE_MESSAGE; namespace android { namespace os { @@ -57,6 +60,9 @@ constexpr const char* kOpUsage = "android:get_usage_stats"; #define STATS_SERVICE_DIR "/data/misc/stats-service" +// for StatsDataDumpProto +const int FIELD_ID_REPORTS_LIST = 1; + static binder::Status ok() { return binder::Status::ok(); } @@ -150,25 +156,26 @@ StatsService::StatsService(const sp<Looper>& handlerLooper) })) { mUidMap = new UidMap(); + mPullerManager = new StatsPullerManager(); StatsPuller::SetUidMap(mUidMap); mConfigManager = new ConfigManager(); - mProcessor = new StatsLogProcessor(mUidMap, mAnomalyAlarmMonitor, mPeriodicAlarmMonitor, - getElapsedRealtimeNs(), [this](const ConfigKey& key) { - sp<IStatsCompanionService> sc = getStatsCompanionService(); - auto receiver = mConfigManager->GetConfigReceiver(key); - if (sc == nullptr) { - VLOG("Could not find StatsCompanionService"); - return false; - } else if (receiver == nullptr) { - VLOG("Statscompanion could not find a broadcast receiver for %s", - key.ToString().c_str()); - return false; - } else { - sc->sendDataBroadcast(receiver, mProcessor->getLastReportTimeNs(key)); - return true; - } - } - ); + mProcessor = new StatsLogProcessor( + mUidMap, mPullerManager, mAnomalyAlarmMonitor, mPeriodicAlarmMonitor, + getElapsedRealtimeNs(), [this](const ConfigKey& key) { + sp<IStatsCompanionService> sc = getStatsCompanionService(); + auto receiver = mConfigManager->GetConfigReceiver(key); + if (sc == nullptr) { + VLOG("Could not find StatsCompanionService"); + return false; + } else if (receiver == nullptr) { + VLOG("Statscompanion could not find a broadcast receiver for %s", + key.ToString().c_str()); + return false; + } else { + sc->sendDataBroadcast(receiver, mProcessor->getLastReportTimeNs(key)); + return true; + } + }); mConfigManager->AddListener(mProcessor); @@ -213,30 +220,8 @@ status_t StatsService::onTransact(uint32_t code, const Parcel& data, Parcel* rep sp<IResultReceiver> resultReceiver = IResultReceiver::asInterface(data.readStrongBinder()); - FILE* fin = fdopen(in, "r"); - FILE* fout = fdopen(out, "w"); - FILE* ferr = fdopen(err, "w"); - - if (fin == NULL || fout == NULL || ferr == NULL) { - resultReceiver->send(NO_MEMORY); - } else { - err = command(fin, fout, ferr, args); - resultReceiver->send(err); - } - - if (fin != NULL) { - fflush(fin); - fclose(fin); - } - if (fout != NULL) { - fflush(fout); - fclose(fout); - } - if (fout != NULL) { - fflush(ferr); - fclose(ferr); - } - + err = command(in, out, err, args, resultReceiver); + resultReceiver->send(err); return NO_ERROR; } default: { return BnStatsManager::onTransact(code, data, reply, flags); } @@ -244,41 +229,53 @@ status_t StatsService::onTransact(uint32_t code, const Parcel& data, Parcel* rep } /** - * Write debugging data about statsd. + * Write data from statsd. + * Format for statsdStats: adb shell dumpsys stats --metadata [-v] [--proto] + * Format for data report: adb shell dumpsys stats [anything other than --metadata] [--proto] + * Anything ending in --proto will be in proto format. + * Anything without --metadata as the first argument will be report information. + * (bugreports call "adb shell dumpsys stats --dump-priority NORMAL -a --proto") + * TODO: Come up with a more robust method of enacting <serviceutils/PriorityDumper.h>. */ status_t StatsService::dump(int fd, const Vector<String16>& args) { if (!checkCallingPermission(String16(kPermissionDump))) { return PERMISSION_DENIED; } - FILE* out = fdopen(fd, "w"); - if (out == NULL) { - return NO_MEMORY; // the fd is already open + int lastArg = args.size() - 1; + bool asProto = false; + if (lastArg >= 0 && !args[lastArg].compare(String16("--proto"))) { // last argument + asProto = true; + lastArg--; } - - bool verbose = false; - bool proto = false; - if (args.size() > 0 && !args[0].compare(String16("-v"))) { - verbose = true; - } - if (args.size() > 0 && !args[args.size()-1].compare(String16("--proto"))) { - proto = true; + if (args.size() > 0 && !args[0].compare(String16("--metadata"))) { // first argument + // Request is to dump statsd stats. + bool verbose = false; + if (lastArg >= 0 && !args[lastArg].compare(String16("-v"))) { + verbose = true; + lastArg--; + } + dumpStatsdStats(fd, verbose, asProto); + } else { + // Request is to dump statsd report data. + if (asProto) { + dumpIncidentSection(fd); + } else { + dprintf(fd, "Non-proto format of stats data dump not available; see proto version.\n"); + } } - dump_impl(out, verbose, proto); - - fclose(out); return NO_ERROR; } /** * Write debugging data about statsd in text or proto format. */ -void StatsService::dump_impl(FILE* out, bool verbose, bool proto) { +void StatsService::dumpStatsdStats(int out, bool verbose, bool proto) { if (proto) { vector<uint8_t> data; StatsdStats::getInstance().dumpStats(&data, false); // does not reset statsdStats. for (size_t i = 0; i < data.size(); i ++) { - fprintf(out, "%c", data[i]); + dprintf(out, "%c", data[i]); } } else { StatsdStats::getInstance().dumpStats(out); @@ -287,9 +284,26 @@ void StatsService::dump_impl(FILE* out, bool verbose, bool proto) { } /** + * Write stats report data in StatsDataDumpProto incident section format. + */ +void StatsService::dumpIncidentSection(int out) { + ProtoOutputStream proto; + for (const ConfigKey& configKey : mConfigManager->GetAllConfigKeys()) { + uint64_t reportsListToken = + proto.start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_REPORTS_LIST); + mProcessor->onDumpReport(configKey, getElapsedRealtimeNs(), + true /* includeCurrentBucket */, false /* erase_data */, + ADB_DUMP, &proto); + proto.end(reportsListToken); + proto.flush(out); + } +} + +/** * Implementation of the adb shell cmd stats command. */ -status_t StatsService::command(FILE* in, FILE* out, FILE* err, Vector<String8>& args) { +status_t StatsService::command(int in, int out, int err, Vector<String8>& args, + sp<IResultReceiver> resultReceiver) { uid_t uid = IPCThreadState::self()->getCallingUid(); if (uid != AID_ROOT && uid != AID_SHELL) { return PERMISSION_DENIED; @@ -307,7 +321,7 @@ status_t StatsService::command(FILE* in, FILE* out, FILE* err, Vector<String8>& } if (!args[0].compare(String8("dump-report"))) { - return cmd_dump_report(out, err, args); + return cmd_dump_report(out, args); } if (!args[0].compare(String8("pull-source")) && args.size() > 1) { @@ -341,97 +355,106 @@ status_t StatsService::command(FILE* in, FILE* out, FILE* err, Vector<String8>& if (!args[0].compare(String8("print-logs"))) { return cmd_print_logs(out, args); } + if (!args[0].compare(String8("data-subscribe"))) { + if (mShellSubscriber == nullptr) { + mShellSubscriber = new ShellSubscriber(mUidMap, mPullerManager); + } + mShellSubscriber->startNewSubscription(in, out, resultReceiver); + return NO_ERROR; + } } print_cmd_help(out); return NO_ERROR; } -void StatsService::print_cmd_help(FILE* out) { - fprintf(out, +void StatsService::print_cmd_help(int out) { + dprintf(out, "usage: adb shell cmd stats print-stats-log [tag_required] " "[timestamp_nsec_optional]\n"); - fprintf(out, "\n"); - fprintf(out, "\n"); - fprintf(out, "usage: adb shell cmd stats meminfo\n"); - fprintf(out, "\n"); - fprintf(out, " Prints the malloc debug information. You need to run the following first: \n"); - fprintf(out, " # adb shell stop\n"); - fprintf(out, " # adb shell setprop libc.debug.malloc.program statsd \n"); - fprintf(out, " # adb shell setprop libc.debug.malloc.options backtrace \n"); - fprintf(out, " # adb shell start\n"); - fprintf(out, "\n"); - fprintf(out, "\n"); - fprintf(out, "usage: adb shell cmd stats print-uid-map [PKG]\n"); - fprintf(out, "\n"); - fprintf(out, " Prints the UID, app name, version mapping.\n"); - fprintf(out, " PKG Optional package name to print the uids of the package\n"); - fprintf(out, "\n"); - fprintf(out, "\n"); - fprintf(out, "usage: adb shell cmd stats pull-source [int] \n"); - fprintf(out, "\n"); - fprintf(out, " Prints the output of a pulled metrics source (int indicates source)\n"); - fprintf(out, "\n"); - fprintf(out, "\n"); - fprintf(out, "usage: adb shell cmd stats write-to-disk \n"); - fprintf(out, "\n"); - fprintf(out, " Flushes all data on memory to disk.\n"); - fprintf(out, "\n"); - fprintf(out, "\n"); - fprintf(out, "usage: adb shell cmd stats log-app-breadcrumb [UID] LABEL STATE\n"); - fprintf(out, " Writes an AppBreadcrumbReported event to the statslog buffer.\n"); - fprintf(out, " UID The uid to use. It is only possible to pass a UID\n"); - fprintf(out, " parameter on eng builds. If UID is omitted the calling\n"); - fprintf(out, " uid is used.\n"); - fprintf(out, " LABEL Integer in [0, 15], as per atoms.proto.\n"); - fprintf(out, " STATE Integer in [0, 3], as per atoms.proto.\n"); - fprintf(out, "\n"); - fprintf(out, "\n"); - fprintf(out, "usage: adb shell cmd stats config remove [UID] [NAME]\n"); - fprintf(out, "usage: adb shell cmd stats config update [UID] NAME\n"); - fprintf(out, "\n"); - fprintf(out, " Adds, updates or removes a configuration. The proto should be in\n"); - fprintf(out, " wire-encoded protobuf format and passed via stdin. If no UID and name is\n"); - fprintf(out, " provided, then all configs will be removed from memory and disk.\n"); - fprintf(out, "\n"); - fprintf(out, " UID The uid to use. It is only possible to pass the UID\n"); - fprintf(out, " parameter on eng builds. If UID is omitted the calling\n"); - fprintf(out, " uid is used.\n"); - fprintf(out, " NAME The per-uid name to use\n"); - fprintf(out, "\n"); - fprintf(out, "\n *Note: If both UID and NAME are omitted then all configs will\n"); - fprintf(out, "\n be removed from memory and disk!\n"); - fprintf(out, "\n"); - fprintf(out, "usage: adb shell cmd stats dump-report [UID] NAME [--proto]\n"); - fprintf(out, " Dump all metric data for a configuration.\n"); - fprintf(out, " UID The uid of the configuration. It is only possible to pass\n"); - fprintf(out, " the UID parameter on eng builds. If UID is omitted the\n"); - fprintf(out, " calling uid is used.\n"); - fprintf(out, " NAME The name of the configuration\n"); - fprintf(out, " --proto Print proto binary.\n"); - fprintf(out, "\n"); - fprintf(out, "\n"); - fprintf(out, "usage: adb shell cmd stats send-broadcast [UID] NAME\n"); - fprintf(out, " Send a broadcast that triggers the subscriber to fetch metrics.\n"); - fprintf(out, " UID The uid of the configuration. It is only possible to pass\n"); - fprintf(out, " the UID parameter on eng builds. If UID is omitted the\n"); - fprintf(out, " calling uid is used.\n"); - fprintf(out, " NAME The name of the configuration\n"); - fprintf(out, "\n"); - fprintf(out, "\n"); - fprintf(out, "usage: adb shell cmd stats print-stats\n"); - fprintf(out, " Prints some basic stats.\n"); - fprintf(out, " --proto Print proto binary instead of string format.\n"); - fprintf(out, "\n"); - fprintf(out, "\n"); - fprintf(out, "usage: adb shell cmd stats clear-puller-cache\n"); - fprintf(out, " Clear cached puller data.\n"); - fprintf(out, "\n"); - fprintf(out, "usage: adb shell cmd stats print-logs\n"); - fprintf(out, " Only works on eng build\n"); + dprintf(out, "\n"); + dprintf(out, "\n"); + dprintf(out, "usage: adb shell cmd stats meminfo\n"); + dprintf(out, "\n"); + dprintf(out, " Prints the malloc debug information. You need to run the following first: \n"); + dprintf(out, " # adb shell stop\n"); + dprintf(out, " # adb shell setprop libc.debug.malloc.program statsd \n"); + dprintf(out, " # adb shell setprop libc.debug.malloc.options backtrace \n"); + dprintf(out, " # adb shell start\n"); + dprintf(out, "\n"); + dprintf(out, "\n"); + dprintf(out, "usage: adb shell cmd stats print-uid-map [PKG]\n"); + dprintf(out, "\n"); + dprintf(out, " Prints the UID, app name, version mapping.\n"); + dprintf(out, " PKG Optional package name to print the uids of the package\n"); + dprintf(out, "\n"); + dprintf(out, "\n"); + dprintf(out, "usage: adb shell cmd stats pull-source [int] \n"); + dprintf(out, "\n"); + dprintf(out, " Prints the output of a pulled metrics source (int indicates source)\n"); + dprintf(out, "\n"); + dprintf(out, "\n"); + dprintf(out, "usage: adb shell cmd stats write-to-disk \n"); + dprintf(out, "\n"); + dprintf(out, " Flushes all data on memory to disk.\n"); + dprintf(out, "\n"); + dprintf(out, "\n"); + dprintf(out, "usage: adb shell cmd stats log-app-breadcrumb [UID] LABEL STATE\n"); + dprintf(out, " Writes an AppBreadcrumbReported event to the statslog buffer.\n"); + dprintf(out, " UID The uid to use. It is only possible to pass a UID\n"); + dprintf(out, " parameter on eng builds. If UID is omitted the calling\n"); + dprintf(out, " uid is used.\n"); + dprintf(out, " LABEL Integer in [0, 15], as per atoms.proto.\n"); + dprintf(out, " STATE Integer in [0, 3], as per atoms.proto.\n"); + dprintf(out, "\n"); + dprintf(out, "\n"); + dprintf(out, "usage: adb shell cmd stats config remove [UID] [NAME]\n"); + dprintf(out, "usage: adb shell cmd stats config update [UID] NAME\n"); + dprintf(out, "\n"); + dprintf(out, " Adds, updates or removes a configuration. The proto should be in\n"); + dprintf(out, " wire-encoded protobuf format and passed via stdin. If no UID and name is\n"); + dprintf(out, " provided, then all configs will be removed from memory and disk.\n"); + dprintf(out, "\n"); + dprintf(out, " UID The uid to use. It is only possible to pass the UID\n"); + dprintf(out, " parameter on eng builds. If UID is omitted the calling\n"); + dprintf(out, " uid is used.\n"); + dprintf(out, " NAME The per-uid name to use\n"); + dprintf(out, "\n"); + dprintf(out, "\n *Note: If both UID and NAME are omitted then all configs will\n"); + dprintf(out, "\n be removed from memory and disk!\n"); + dprintf(out, "\n"); + dprintf(out, + "usage: adb shell cmd stats dump-report [UID] NAME [--include_current_bucket] " + "[--proto]\n"); + dprintf(out, " Dump all metric data for a configuration.\n"); + dprintf(out, " UID The uid of the configuration. It is only possible to pass\n"); + dprintf(out, " the UID parameter on eng builds. If UID is omitted the\n"); + dprintf(out, " calling uid is used.\n"); + dprintf(out, " NAME The name of the configuration\n"); + dprintf(out, " --proto Print proto binary.\n"); + dprintf(out, "\n"); + dprintf(out, "\n"); + dprintf(out, "usage: adb shell cmd stats send-broadcast [UID] NAME\n"); + dprintf(out, " Send a broadcast that triggers the subscriber to fetch metrics.\n"); + dprintf(out, " UID The uid of the configuration. It is only possible to pass\n"); + dprintf(out, " the UID parameter on eng builds. If UID is omitted the\n"); + dprintf(out, " calling uid is used.\n"); + dprintf(out, " NAME The name of the configuration\n"); + dprintf(out, "\n"); + dprintf(out, "\n"); + dprintf(out, "usage: adb shell cmd stats print-stats\n"); + dprintf(out, " Prints some basic stats.\n"); + dprintf(out, " --proto Print proto binary instead of string format.\n"); + dprintf(out, "\n"); + dprintf(out, "\n"); + dprintf(out, "usage: adb shell cmd stats clear-puller-cache\n"); + dprintf(out, " Clear cached puller data.\n"); + dprintf(out, "\n"); + dprintf(out, "usage: adb shell cmd stats print-logs\n"); + dprintf(out, " Only works on eng build\n"); } -status_t StatsService::cmd_trigger_broadcast(FILE* out, Vector<String8>& args) { +status_t StatsService::cmd_trigger_broadcast(int out, Vector<String8>& args) { string name; bool good = false; int uid; @@ -439,7 +462,6 @@ status_t StatsService::cmd_trigger_broadcast(FILE* out, Vector<String8>& args) { if (argCount == 2) { // Automatically pick the UID uid = IPCThreadState::self()->getCallingUid(); - // TODO: What if this isn't a binder call? Should we fail? name.assign(args[1].c_str(), args[1].size()); good = true; } else if (argCount == 3) { @@ -456,9 +478,9 @@ status_t StatsService::cmd_trigger_broadcast(FILE* out, Vector<String8>& args) { } } } else { - fprintf(out, + dprintf(out, "The metrics can only be dumped for other UIDs on eng or userdebug " - "builds.\n"); + "builds.\n"); } } if (!good) { @@ -481,7 +503,7 @@ status_t StatsService::cmd_trigger_broadcast(FILE* out, Vector<String8>& args) { return NO_ERROR; } -status_t StatsService::cmd_config(FILE* in, FILE* out, FILE* err, Vector<String8>& args) { +status_t StatsService::cmd_config(int in, int out, int err, Vector<String8>& args) { const int argCount = args.size(); if (argCount >= 2) { if (args[1] == "update" || args[1] == "remove") { @@ -492,7 +514,6 @@ status_t StatsService::cmd_config(FILE* in, FILE* out, FILE* err, Vector<String8 if (argCount == 3) { // Automatically pick the UID uid = IPCThreadState::self()->getCallingUid(); - // TODO: What if this isn't a binder call? Should we fail? name.assign(args[2].c_str(), args[2].size()); good = true; } else if (argCount == 4) { @@ -509,7 +530,7 @@ status_t StatsService::cmd_config(FILE* in, FILE* out, FILE* err, Vector<String8 } } } else { - fprintf(err, + dprintf(err, "The config can only be set for other UIDs on eng or userdebug " "builds.\n"); } @@ -527,21 +548,21 @@ status_t StatsService::cmd_config(FILE* in, FILE* out, FILE* err, Vector<String8 char* endp; int64_t configID = strtoll(name.c_str(), &endp, 10); if (endp == name.c_str() || *endp != '\0') { - fprintf(err, "Error parsing config ID.\n"); + dprintf(err, "Error parsing config ID.\n"); return UNKNOWN_ERROR; } // Read stream into buffer. string buffer; - if (!android::base::ReadFdToString(fileno(in), &buffer)) { - fprintf(err, "Error reading stream for StatsConfig.\n"); + if (!android::base::ReadFdToString(in, &buffer)) { + dprintf(err, "Error reading stream for StatsConfig.\n"); return UNKNOWN_ERROR; } // Parse buffer. StatsdConfig config; if (!config.ParseFromString(buffer)) { - fprintf(err, "Error parsing proto stream for StatsConfig.\n"); + dprintf(err, "Error parsing proto stream for StatsConfig.\n"); return UNKNOWN_ERROR; } @@ -563,21 +584,25 @@ status_t StatsService::cmd_config(FILE* in, FILE* out, FILE* err, Vector<String8 return UNKNOWN_ERROR; } -status_t StatsService::cmd_dump_report(FILE* out, FILE* err, const Vector<String8>& args) { +status_t StatsService::cmd_dump_report(int out, const Vector<String8>& args) { if (mProcessor != nullptr) { int argCount = args.size(); bool good = false; bool proto = false; + bool includeCurrentBucket = false; int uid; string name; if (!std::strcmp("--proto", args[argCount-1].c_str())) { proto = true; argCount -= 1; } + if (!std::strcmp("--include_current_bucket", args[argCount-1].c_str())) { + includeCurrentBucket = true; + argCount -= 1; + } if (argCount == 2) { // Automatically pick the UID uid = IPCThreadState::self()->getCallingUid(); - // TODO: What if this isn't a binder call? Should we fail? name.assign(args[1].c_str(), args[1].size()); good = true; } else if (argCount == 3) { @@ -594,7 +619,7 @@ status_t StatsService::cmd_dump_report(FILE* out, FILE* err, const Vector<String } } } else { - fprintf(out, + dprintf(out, "The metrics can only be dumped for other UIDs on eng or userdebug " "builds.\n"); } @@ -602,15 +627,13 @@ status_t StatsService::cmd_dump_report(FILE* out, FILE* err, const Vector<String if (good) { vector<uint8_t> data; mProcessor->onDumpReport(ConfigKey(uid, StrToInt64(name)), getElapsedRealtimeNs(), - false /* include_current_bucket*/, ADB_DUMP, &data); - // TODO: print the returned StatsLogReport to file instead of printing to logcat. + includeCurrentBucket, true /* erase_data */, ADB_DUMP, &data); if (proto) { for (size_t i = 0; i < data.size(); i ++) { - fprintf(out, "%c", data[i]); + dprintf(out, "%c", data[i]); } } else { - fprintf(out, "Dump report for Config [%d,%s]\n", uid, name.c_str()); - fprintf(out, "See the StatsLogReport in logcat...\n"); + dprintf(out, "Non-proto stats data dump not currently supported.\n"); } return android::OK; } else { @@ -619,12 +642,12 @@ status_t StatsService::cmd_dump_report(FILE* out, FILE* err, const Vector<String return UNKNOWN_ERROR; } } else { - fprintf(out, "Log processor does not exist...\n"); + dprintf(out, "Log processor does not exist...\n"); return UNKNOWN_ERROR; } } -status_t StatsService::cmd_print_stats(FILE* out, const Vector<String8>& args) { +status_t StatsService::cmd_print_stats(int out, const Vector<String8>& args) { int argCount = args.size(); bool proto = false; if (!std::strcmp("--proto", args[argCount-1].c_str())) { @@ -636,13 +659,13 @@ status_t StatsService::cmd_print_stats(FILE* out, const Vector<String8>& args) { vector<uint8_t> data; statsdStats.dumpStats(&data, false); // does not reset statsdStats. for (size_t i = 0; i < data.size(); i ++) { - fprintf(out, "%c", data[i]); + dprintf(out, "%c", data[i]); } } else { vector<ConfigKey> configs = mConfigManager->GetAllConfigKeys(); for (const ConfigKey& key : configs) { - fprintf(out, "Config %s uses %zu bytes\n", key.ToString().c_str(), + dprintf(out, "Config %s uses %zu bytes\n", key.ToString().c_str(), mProcessor->GetMetricsSize(key)); } statsdStats.dumpStats(out); @@ -650,29 +673,29 @@ status_t StatsService::cmd_print_stats(FILE* out, const Vector<String8>& args) { return NO_ERROR; } -status_t StatsService::cmd_print_uid_map(FILE* out, const Vector<String8>& args) { +status_t StatsService::cmd_print_uid_map(int out, const Vector<String8>& args) { if (args.size() > 1) { string pkg; pkg.assign(args[1].c_str(), args[1].size()); auto uids = mUidMap->getAppUid(pkg); - fprintf(out, "%s -> [ ", pkg.c_str()); + dprintf(out, "%s -> [ ", pkg.c_str()); for (const auto& uid : uids) { - fprintf(out, "%d ", uid); + dprintf(out, "%d ", uid); } - fprintf(out, "]\n"); + dprintf(out, "]\n"); } else { mUidMap->printUidMap(out); } return NO_ERROR; } -status_t StatsService::cmd_write_data_to_disk(FILE* out) { - fprintf(out, "Writing data to disk\n"); +status_t StatsService::cmd_write_data_to_disk(int out) { + dprintf(out, "Writing data to disk\n"); mProcessor->WriteDataToDisk(ADB_DUMP); return NO_ERROR; } -status_t StatsService::cmd_log_app_breadcrumb(FILE* out, const Vector<String8>& args) { +status_t StatsService::cmd_log_app_breadcrumb(int out, const Vector<String8>& args) { bool good = false; int32_t uid; int32_t label; @@ -693,13 +716,13 @@ status_t StatsService::cmd_log_app_breadcrumb(FILE* out, const Vector<String8>& state = atoi(args[3].c_str()); good = true; } else { - fprintf(out, + dprintf(out, "Selecting a UID for writing AppBreadcrumb can only be done for other UIDs " - "on eng or userdebug builds.\n"); + "on eng or userdebug builds.\n"); } } if (good) { - fprintf(out, "Logging AppBreadcrumbReported(%d, %d, %d) to statslog.\n", uid, label, state); + dprintf(out, "Logging AppBreadcrumbReported(%d, %d, %d) to statslog.\n", uid, label, state); android::util::stats_write(android::util::APP_BREADCRUMB_REPORTED, uid, label, state); } else { print_cmd_help(out); @@ -708,46 +731,46 @@ status_t StatsService::cmd_log_app_breadcrumb(FILE* out, const Vector<String8>& return NO_ERROR; } -status_t StatsService::cmd_print_pulled_metrics(FILE* out, const Vector<String8>& args) { +status_t StatsService::cmd_print_pulled_metrics(int out, const Vector<String8>& args) { int s = atoi(args[1].c_str()); vector<shared_ptr<LogEvent> > stats; - if (mStatsPullerManager.Pull(s, getElapsedRealtimeNs(), &stats)) { + if (mPullerManager->Pull(s, getElapsedRealtimeNs(), &stats)) { for (const auto& it : stats) { - fprintf(out, "Pull from %d: %s\n", s, it->ToString().c_str()); + dprintf(out, "Pull from %d: %s\n", s, it->ToString().c_str()); } - fprintf(out, "Pull from %d: Received %zu elements\n", s, stats.size()); + dprintf(out, "Pull from %d: Received %zu elements\n", s, stats.size()); return NO_ERROR; } return UNKNOWN_ERROR; } -status_t StatsService::cmd_remove_all_configs(FILE* out) { - fprintf(out, "Removing all configs...\n"); +status_t StatsService::cmd_remove_all_configs(int out) { + dprintf(out, "Removing all configs...\n"); VLOG("StatsService::cmd_remove_all_configs was called"); mConfigManager->RemoveAllConfigs(); StorageManager::deleteAllFiles(STATS_SERVICE_DIR); return NO_ERROR; } -status_t StatsService::cmd_dump_memory_info(FILE* out) { - fprintf(out, "meminfo not available.\n"); +status_t StatsService::cmd_dump_memory_info(int out) { + dprintf(out, "meminfo not available.\n"); return NO_ERROR; } -status_t StatsService::cmd_clear_puller_cache(FILE* out) { +status_t StatsService::cmd_clear_puller_cache(int out) { IPCThreadState* ipc = IPCThreadState::self(); VLOG("StatsService::cmd_clear_puller_cache with Pid %i, Uid %i", ipc->getCallingPid(), ipc->getCallingUid()); if (checkCallingPermission(String16(kPermissionDump))) { - int cleared = mStatsPullerManager.ForceClearPullerCache(); - fprintf(out, "Puller removed %d cached data!\n", cleared); + int cleared = mPullerManager->ForceClearPullerCache(); + dprintf(out, "Puller removed %d cached data!\n", cleared); return NO_ERROR; } else { return PERMISSION_DENIED; } } -status_t StatsService::cmd_print_logs(FILE* out, const Vector<String8>& args) { +status_t StatsService::cmd_print_logs(int out, const Vector<String8>& args) { IPCThreadState* ipc = IPCThreadState::self(); VLOG("StatsService::cmd_print_logs with Pid %i, Uid %i", ipc->getCallingPid(), ipc->getCallingUid()); @@ -870,7 +893,7 @@ Status StatsService::statsCompanionReady() { } VLOG("StatsService::statsCompanionReady linking to statsCompanion."); IInterface::asBinder(statsCompanion)->linkToDeath(this); - mStatsPullerManager.SetStatsCompanionService(statsCompanion); + mPullerManager->SetStatsCompanionService(statsCompanion); mAnomalyAlarmMonitor->setStatsCompanionService(statsCompanion); mPeriodicAlarmMonitor->setStatsCompanionService(statsCompanion); SubscriberReporter::getInstance().setStatsCompanionService(statsCompanion); @@ -888,8 +911,11 @@ void StatsService::Terminate() { } } -void StatsService::OnLogEvent(LogEvent* event, bool reconnectionStarts) { - mProcessor->OnLogEvent(event, reconnectionStarts); +void StatsService::OnLogEvent(LogEvent* event) { + mProcessor->OnLogEvent(event); + if (mShellSubscriber != nullptr) { + mShellSubscriber->onLogEvent(*event); + } } Status StatsService::getData(int64_t key, const String16& packageName, vector<uint8_t>* output) { @@ -899,7 +925,7 @@ Status StatsService::getData(int64_t key, const String16& packageName, vector<ui VLOG("StatsService::getData with Pid %i, Uid %i", ipc->getCallingPid(), ipc->getCallingUid()); ConfigKey configKey(ipc->getCallingUid(), key); mProcessor->onDumpReport(configKey, getElapsedRealtimeNs(), false /* include_current_bucket*/, - GET_DATA_CALLED, output); + true /* erase_data */, GET_DATA_CALLED, output); return Status::ok(); } @@ -1010,18 +1036,72 @@ Status StatsService::sendAppBreadcrumbAtom(int32_t label, int32_t state) { return Status::ok(); } +hardware::Return<void> StatsService::reportSpeakerImpedance( + const SpeakerImpedance& speakerImpedance) { + LogEvent event(getWallClockSec() * NS_PER_SEC, getElapsedRealtimeNs(), speakerImpedance); + mProcessor->OnLogEvent(&event); + + return hardware::Void(); +} + +hardware::Return<void> StatsService::reportHardwareFailed(const HardwareFailed& hardwareFailed) { + LogEvent event(getWallClockSec() * NS_PER_SEC, getElapsedRealtimeNs(), hardwareFailed); + mProcessor->OnLogEvent(&event); + + return hardware::Void(); +} + +hardware::Return<void> StatsService::reportPhysicalDropDetected( + const PhysicalDropDetected& physicalDropDetected) { + LogEvent event(getWallClockSec() * NS_PER_SEC, getElapsedRealtimeNs(), physicalDropDetected); + mProcessor->OnLogEvent(&event); + + return hardware::Void(); +} + +hardware::Return<void> StatsService::reportChargeCycles(const ChargeCycles& chargeCycles) { + LogEvent event(getWallClockSec() * NS_PER_SEC, getElapsedRealtimeNs(), chargeCycles); + mProcessor->OnLogEvent(&event); + + return hardware::Void(); +} + +hardware::Return<void> StatsService::reportBatteryHealthSnapshot( + const BatteryHealthSnapshotArgs& batteryHealthSnapshotArgs) { + LogEvent event(getWallClockSec() * NS_PER_SEC, getElapsedRealtimeNs(), + batteryHealthSnapshotArgs); + mProcessor->OnLogEvent(&event); + + return hardware::Void(); +} + +hardware::Return<void> StatsService::reportSlowIo(const SlowIo& slowIo) { + LogEvent event(getWallClockSec() * NS_PER_SEC, getElapsedRealtimeNs(), slowIo); + mProcessor->OnLogEvent(&event); + + return hardware::Void(); +} + +hardware::Return<void> StatsService::reportBatteryCausedShutdown( + const BatteryCausedShutdown& batteryCausedShutdown) { + LogEvent event(getWallClockSec() * NS_PER_SEC, getElapsedRealtimeNs(), batteryCausedShutdown); + mProcessor->OnLogEvent(&event); + + return hardware::Void(); +} + void StatsService::binderDied(const wp <IBinder>& who) { ALOGW("statscompanion service died"); StatsdStats::getInstance().noteSystemServerRestart(getWallClockSec()); if (mProcessor != nullptr) { - ALOGW("Reset statsd upon system server restars."); + ALOGW("Reset statsd upon system server restarts."); mProcessor->WriteDataToDisk(STATSCOMPANION_DIED); mProcessor->resetConfigs(); } mAnomalyAlarmMonitor->setStatsCompanionService(nullptr); mPeriodicAlarmMonitor->setStatsCompanionService(nullptr); SubscriberReporter::getInstance().setStatsCompanionService(nullptr); - mStatsPullerManager.SetStatsCompanionService(nullptr); + mPullerManager->SetStatsCompanionService(nullptr); } } // namespace statsd diff --git a/cmds/statsd/src/StatsService.h b/cmds/statsd/src/StatsService.h index d8aab88ce9f9..4a5f05fef034 100644 --- a/cmds/statsd/src/StatsService.h +++ b/cmds/statsd/src/StatsService.h @@ -22,9 +22,13 @@ #include "anomaly/AlarmMonitor.h" #include "config/ConfigManager.h" #include "external/StatsPullerManager.h" +#include "logd/LogListener.h" #include "packages/UidMap.h" +#include "shell/ShellSubscriber.h" #include "statscompanion_util.h" +#include <android/frameworks/stats/1.0/IStats.h> +#include <android/frameworks/stats/1.0/types.h> #include <android/os/BnStatsManager.h> #include <android/os/IStatsCompanionService.h> #include <binder/IResultReceiver.h> @@ -36,6 +40,7 @@ using namespace android; using namespace android::base; using namespace android::binder; +using namespace android::frameworks::stats::V1_0; using namespace android::os; using namespace std; @@ -43,18 +48,23 @@ namespace android { namespace os { namespace statsd { -class StatsService : public BnStatsManager, public LogListener, public IBinder::DeathRecipient { +using android::hardware::Return; + +class StatsService : public BnStatsManager, + public LogListener, + public IStats, + public IBinder::DeathRecipient { public: StatsService(const sp<Looper>& handlerLooper); virtual ~StatsService(); /** The anomaly alarm registered with AlarmManager won't be updated by less than this. */ - // TODO: Consider making this configurable. And choose a good number. const uint32_t MIN_DIFF_TO_UPDATE_REGISTERED_ALARM_SECS = 5; virtual status_t onTransact(uint32_t code, const Parcel& data, Parcel* reply, uint32_t flags); virtual status_t dump(int fd, const Vector<String16>& args); - virtual status_t command(FILE* in, FILE* out, FILE* err, Vector<String8>& args); + virtual status_t command(int inFd, int outFd, int err, Vector<String8>& args, + sp<IResultReceiver> resultReceiver); virtual Status systemRunning(); virtual Status statsCompanionReady(); @@ -81,7 +91,7 @@ public: /** * Called by LogReader when there's a log event to process. */ - virtual void OnLogEvent(LogEvent* event, bool reconnectionStarts); + virtual void OnLogEvent(LogEvent* event); /** * Binder call for clients to request data for this configuration key. @@ -149,6 +159,44 @@ public: */ virtual Status sendAppBreadcrumbAtom(int32_t label, int32_t state) override; + /** + * Binder call to get SpeakerImpedance atom. + */ + virtual Return<void> reportSpeakerImpedance(const SpeakerImpedance& speakerImpedance) override; + + /** + * Binder call to get HardwareFailed atom. + */ + virtual Return<void> reportHardwareFailed(const HardwareFailed& hardwareFailed) override; + + /** + * Binder call to get PhysicalDropDetected atom. + */ + virtual Return<void> reportPhysicalDropDetected( + const PhysicalDropDetected& physicalDropDetected) override; + + /** + * Binder call to get ChargeCyclesReported atom. + */ + virtual Return<void> reportChargeCycles(const ChargeCycles& chargeCycles) override; + + /** + * Binder call to get BatteryHealthSnapshot atom. + */ + virtual Return<void> reportBatteryHealthSnapshot( + const BatteryHealthSnapshotArgs& batteryHealthSnapshotArgs) override; + + /** + * Binder call to get SlowIo atom. + */ + virtual Return<void> reportSlowIo(const SlowIo& slowIo) override; + + /** + * Binder call to get BatteryCausedShutdown atom. + */ + virtual Return<void> reportBatteryCausedShutdown( + const BatteryCausedShutdown& batteryCausedShutdown) override; + /** IBinder::DeathRecipient */ virtual void binderDied(const wp<IBinder>& who) override; @@ -165,75 +213,80 @@ private: uint32_t serial); /** - * Text or proto output of dumpsys. + * Proto output of statsd report data dumpsys, wrapped in a StatsDataDumpProto. + */ + void dumpIncidentSection(int outFd); + + /** + * Text or proto output of statsdStats dumpsys. */ - void dump_impl(FILE* out, bool verbose, bool proto); + void dumpStatsdStats(int outFd, bool verbose, bool proto); /** * Print usage information for the commands */ - void print_cmd_help(FILE* out); + void print_cmd_help(int out); /** * Trigger a broadcast. */ - status_t cmd_trigger_broadcast(FILE* out, Vector<String8>& args); + status_t cmd_trigger_broadcast(int outFd, Vector<String8>& args); /** * Handle the config sub-command. */ - status_t cmd_config(FILE* in, FILE* out, FILE* err, Vector<String8>& args); + status_t cmd_config(int inFd, int outFd, int err, Vector<String8>& args); /** * Prints some basic stats to std out. */ - status_t cmd_print_stats(FILE* out, const Vector<String8>& args); + status_t cmd_print_stats(int outFd, const Vector<String8>& args); /** * Print the event log. */ - status_t cmd_dump_report(FILE* out, FILE* err, const Vector<String8>& args); + status_t cmd_dump_report(int outFd, const Vector<String8>& args); /** * Print the mapping of uids to package names. */ - status_t cmd_print_uid_map(FILE* out, const Vector<String8>& args); + status_t cmd_print_uid_map(int outFd, const Vector<String8>& args); /** * Flush the data to disk. */ - status_t cmd_write_data_to_disk(FILE* out); + status_t cmd_write_data_to_disk(int outFd); /** * Write an AppBreadcrumbReported event to the StatsLog buffer, as if calling * StatsLog.write(APP_BREADCRUMB_REPORTED). */ - status_t cmd_log_app_breadcrumb(FILE* out, const Vector<String8>& args); + status_t cmd_log_app_breadcrumb(int outFd, const Vector<String8>& args); /** * Print contents of a pulled metrics source. */ - status_t cmd_print_pulled_metrics(FILE* out, const Vector<String8>& args); + status_t cmd_print_pulled_metrics(int outFd, const Vector<String8>& args); /** * Removes all configs stored on disk and on memory. */ - status_t cmd_remove_all_configs(FILE* out); + status_t cmd_remove_all_configs(int outFd); /* * Dump memory usage by statsd. */ - status_t cmd_dump_memory_info(FILE* out); + status_t cmd_dump_memory_info(int outFd); /* * Clear all puller cached data */ - status_t cmd_clear_puller_cache(FILE* out); + status_t cmd_clear_puller_cache(int outFd); /** * Print all stats logs received to logcat. */ - status_t cmd_print_logs(FILE* out, const Vector<String8>& args); + status_t cmd_print_logs(int outFd, const Vector<String8>& args); /** * Adds a configuration after checking permissions and obtaining UID from binder call. @@ -251,9 +304,9 @@ private: sp<UidMap> mUidMap; /** - * Fetches external metrics. + * Fetches external metrics */ - StatsPullerManager mStatsPullerManager; + sp<StatsPullerManager> mPullerManager; /** * Tracks the configurations that have been passed to statsd. @@ -280,6 +333,8 @@ private: */ bool mEngBuild; + sp<ShellSubscriber> mShellSubscriber; + FRIEND_TEST(StatsServiceTest, TestAddConfig_simple); FRIEND_TEST(StatsServiceTest, TestAddConfig_empty); FRIEND_TEST(StatsServiceTest, TestAddConfig_invalid); diff --git a/cmds/statsd/src/anomaly/AlarmMonitor.cpp b/cmds/statsd/src/anomaly/AlarmMonitor.cpp index 78f0c2b09537..bc36dadacddb 100644 --- a/cmds/statsd/src/anomaly/AlarmMonitor.cpp +++ b/cmds/statsd/src/anomaly/AlarmMonitor.cpp @@ -60,7 +60,7 @@ void AlarmMonitor::add(sp<const InternalAlarm> alarm) { ALOGW("Asked to add a 0-time alarm."); return; } - // TODO: Ensure that refractory period is respected. + // TODO(b/110563466): Ensure that refractory period is respected. VLOG("Adding alarm with time %u", alarm->timestampSec); mPq.push(alarm); if (mRegisteredAlarmTimeSec < 1 || diff --git a/cmds/statsd/src/anomaly/AnomalyTracker.cpp b/cmds/statsd/src/anomaly/AnomalyTracker.cpp index f32efee56d64..ee111cddcfd7 100644 --- a/cmds/statsd/src/anomaly/AnomalyTracker.cpp +++ b/cmds/statsd/src/anomaly/AnomalyTracker.cpp @@ -208,7 +208,8 @@ bool AnomalyTracker::detectAnomaly(const int64_t& currentBucketNum, } void AnomalyTracker::declareAnomaly(const int64_t& timestampNs, const MetricDimensionKey& key) { - // TODO: Why receive timestamp? RefractoryPeriod should always be based on real time right now. + // TODO(b/110563466): Why receive timestamp? RefractoryPeriod should always be based on + // real time right now. if (isInRefractoryPeriod(timestampNs, key)) { VLOG("Skipping anomaly declaration since within refractory period"); return; @@ -216,7 +217,8 @@ void AnomalyTracker::declareAnomaly(const int64_t& timestampNs, const MetricDime if (mAlert.has_refractory_period_secs()) { mRefractoryPeriodEndsSec[key] = ((timestampNs + NS_PER_SEC - 1) / NS_PER_SEC) // round up + mAlert.refractory_period_secs(); - // TODO: If we had access to the bucket_size_millis, consider calling resetStorage() + // TODO(b/110563466): If we had access to the bucket_size_millis, consider + // calling resetStorage() // if (mAlert.refractory_period_secs() > mNumOfPastBuckets * bucketSizeNs) {resetStorage();} } @@ -230,7 +232,7 @@ void AnomalyTracker::declareAnomaly(const int64_t& timestampNs, const MetricDime StatsdStats::getInstance().noteAnomalyDeclared(mConfigKey, mAlert.id()); - // TODO: This should also take in the const MetricDimensionKey& key? + // TODO(b/110564268): This should also take in the const MetricDimensionKey& key? android::util::stats_write(android::util::ANOMALY_DETECTED, mConfigKey.GetUid(), mConfigKey.GetId(), mAlert.id()); } diff --git a/cmds/statsd/src/anomaly/subscriber_util.cpp b/cmds/statsd/src/anomaly/subscriber_util.cpp index ee9e9c01a60a..9d37cdb2d4d7 100644 --- a/cmds/statsd/src/anomaly/subscriber_util.cpp +++ b/cmds/statsd/src/anomaly/subscriber_util.cpp @@ -22,6 +22,7 @@ #include <binder/IServiceManager.h> #include "external/Perfetto.h" +#include "external/Perfprofd.h" #include "frameworks/base/libs/incident/proto/android/os/header.pb.h" #include "subscriber/IncidentdReporter.h" #include "subscriber/SubscriberReporter.h" @@ -64,6 +65,12 @@ void triggerSubscribers(const int64_t rule_id, SubscriberReporter::getInstance().alertBroadcastSubscriber(configKey, subscription, dimensionKey); break; + case Subscription::SubscriberInformationCase::kPerfprofdDetails: + if (!CollectPerfprofdTraceAndUploadToDropbox(subscription.perfprofd_details(), + rule_id, configKey)) { + ALOGW("Failed to generate perfprofd traces."); + } + break; default: break; } diff --git a/cmds/statsd/src/atom_field_options.proto b/cmds/statsd/src/atom_field_options.proto index a2a03b14c073..e33bd8c97a07 100644 --- a/cmds/statsd/src/atom_field_options.proto +++ b/cmds/statsd/src/atom_field_options.proto @@ -36,8 +36,8 @@ enum StateField { // exclusive state field, and any number of primary key fields. // For example, // message UidProcessStateChanged { -// optional int32 uid = 1 [(stateFieldOption).option = PRIMARY]; -// optional android.app.ProcessStateEnum state = 2 [(stateFieldOption).option = EXCLUSIVE]; +// optional int32 uid = 1 [(state_field_option).option = PRIMARY]; +// optional android.app.ProcessStateEnum state = 2 [(state_field_option).option = EXCLUSIVE]; // } // Each of this UidProcessStateChanged atom represents a state change for a specific uid. // A new state automatically overrides the previous state. @@ -46,16 +46,16 @@ enum StateField { // the primary key. // For example: // message ThreadStateChanged { -// optional int32 pid = 1 [(stateFieldOption).option = PRIMARY]; -// optional int32 tid = 2 [(stateFieldOption).option = PRIMARY]; -// optional int32 state = 3 [(stateFieldOption).option = EXCLUSIVE]; +// optional int32 pid = 1 [(state_field_option).option = PRIMARY]; +// optional int32 tid = 2 [(state_field_option).option = PRIMARY]; +// optional int32 state = 3 [(state_field_option).option = EXCLUSIVE]; // } // // Sometimes, there is no primary key field, when the state is GLOBAL. // For example, // // message ScreenStateChanged { -// optional android.view.DisplayStateEnum state = 1 [(stateFieldOption).option = EXCLUSIVE]; +// optional android.view.DisplayStateEnum state = 1 [(state_field_option).option = EXCLUSIVE]; // } // // Only fields of primary types can be annotated. AttributionNode cannot be primary keys (and they @@ -64,10 +64,22 @@ message StateAtomFieldOption { optional StateField option = 1 [default = STATE_FIELD_UNSET]; } +// Used to generate StatsLog.write APIs. +enum LogMode { + MODE_UNSET = 0; + // Log fields as their actual types e.g., all primary data types. + // Or fields that are hardcoded in stats_log_api_gen tool e.g., AttributionNode + MODE_AUTOMATIC = 1; + // Log fields in their proto binary format. These fields will not be parsed in statsd + MODE_BYTES = 2; +} + extend google.protobuf.FieldOptions { // Flags to decorate an atom that presents a state change. - optional StateAtomFieldOption stateFieldOption = 50000; + optional StateAtomFieldOption state_field_option = 50000; // Flags to decorate the uid fields in an atom. optional bool is_uid = 50001 [default = false]; + + optional LogMode log_mode = 50002 [default = MODE_AUTOMATIC]; }
\ No newline at end of file diff --git a/cmds/statsd/src/atoms.proto b/cmds/statsd/src/atoms.proto index ac402c3d2caf..5620184d5038 100644 --- a/cmds/statsd/src/atoms.proto +++ b/cmds/statsd/src/atoms.proto @@ -22,10 +22,14 @@ option java_outer_classname = "AtomsProto"; import "frameworks/base/cmds/statsd/src/atom_field_options.proto"; import "frameworks/base/core/proto/android/app/enums.proto"; +import "frameworks/base/core/proto/android/app/settings_enums.proto"; import "frameworks/base/core/proto/android/app/job/enums.proto"; import "frameworks/base/core/proto/android/bluetooth/enums.proto"; import "frameworks/base/core/proto/android/os/enums.proto"; import "frameworks/base/core/proto/android/server/enums.proto"; +import "frameworks/base/core/proto/android/service/procstats_enum.proto"; +import "frameworks/base/core/proto/android/stats/enums.proto"; +import "frameworks/base/core/proto/android/stats/launcher/launcher.proto"; import "frameworks/base/core/proto/android/telecomm/enums.proto"; import "frameworks/base/core/proto/android/telephony/enums.proto"; import "frameworks/base/core/proto/android/view/enums.proto"; @@ -48,7 +52,7 @@ message Atom { oneof pushed { // For StatsLog reasons, 1 is illegal and will not work. Must start at 2. BleScanStateChanged ble_scan_state_changed = 2; - // 3 is available for use + ProcessStateChanged process_state_changed = 3; BleScanResultReceived ble_scan_result_received = 4; SensorStateChanged sensor_state_changed = 5; GpsScanStateChanged gps_scan_state_changed = 6; @@ -59,7 +63,12 @@ message Atom { LongPartialWakelockStateChanged long_partial_wakelock_state_changed = 11; MobileRadioPowerStateChanged mobile_radio_power_state_changed = 12; WifiRadioPowerStateChanged wifi_radio_power_state_changed = 13; - // 14 - 19 are available + ActivityManagerSleepStateChanged activity_manager_sleep_state_changed = 14; + MemoryFactorStateChanged memory_factor_state_changed = 15; + ExcessiveCpuUsageReported excessive_cpu_usage_reported = 16; + CachedKillReported cached_kill_reported = 17; + ProcessMemoryStatReported process_memory_stat_reported = 18; + LauncherUIChanged launcher_event = 19; BatterySaverModeStateChanged battery_saver_mode_state_changed = 20; DeviceIdleModeStateChanged device_idle_mode_state_changed = 21; DeviceIdlingModeStateChanged device_idling_mode_state_changed = 22; @@ -73,7 +82,8 @@ message Atom { BatteryLevelChanged battery_level_changed = 30; ChargingStateChanged charging_state_changed = 31; PluggedStateChanged plugged_state_changed = 32; - // 33 - 34 are available + InteractiveStateChanged interactive_state_changed = 33; + // 34 is available WakeupAlarmOccurred wakeup_alarm_occurred = 35; KernelWakeupReported kernel_wakeup_reported = 36; WifiLockStateChanged wifi_lock_state_changed = 37; @@ -84,7 +94,7 @@ message Atom { ActivityForegroundStateChanged activity_foreground_state_changed = 42; IsolatedUidChanged isolated_uid_changed = 43; PacketWakeupOccurred packet_wakeup_occurred = 44; - // 45 is available + WallClockTimeShifted wall_clock_time_shifted = 45; AnomalyDetected anomaly_detected = 46; AppBreadcrumbReported app_breadcrumb_reported = 47; AppStartOccurred app_start_occurred = 48; @@ -120,14 +130,27 @@ message Atom { AppCrashOccurred app_crash_occurred = 78; ANROccurred anr_occurred = 79; WTFOccurred wtf_occurred = 80; - PhoneServiceStateChanged phone_service_state_changed = 94; - PhoneStateChanged phone_state_changed = 95; LowMemReported low_mem_reported = 81; + GenericAtom generic_atom = 82; + KeyValuePairsAtom key_value_pairs_atom = 83; + VibratorStateChanged vibrator_state_changed = 84; + DeferredJobStatsReported deferred_job_stats_reported = 85; ThermalThrottlingStateChanged thermal_throttling = 86; + FingerprintAcquired fingerprint_acquired = 87; + FingerprintAuthenticated fingerprint_authenticated = 88; + FingerprintErrorOccurred fingerprint_error_occurred = 89; + Notification notification = 90; + BatteryHealthSnapshot battery_health_snapshot = 91; + SlowIo slow_io = 92; + BatteryCausedShutdown battery_caused_shutdown = 93; + PhoneServiceStateChanged phone_service_state_changed = 94; + PhoneStateChanged phone_state_changed = 95; + UserRestrictionChanged user_restriction_changed = 96; + SettingsUIChanged settings_ui_changed = 97; } // Pulled events will start at field 10000. - // Next: 10022 + // Next: 10038 oneof pulled { WifiBytesTransfer wifi_bytes_transfer = 10000; WifiBytesTransferByFgBg wifi_bytes_transfer_by_fg_bg = 10001; @@ -147,10 +170,26 @@ message Atom { SystemUptime system_uptime = 10015; CpuActiveTime cpu_active_time = 10016; CpuClusterTime cpu_cluster_time = 10017; - DiskSpace disk_space = 10018; + DiskSpace disk_space = 10018 [deprecated=true]; RemainingBatteryCapacity remaining_battery_capacity = 10019; FullBatteryCapacity full_battery_capacity = 10020; Temperature temperature = 10021; + BinderCalls binder_calls = 10022; + BinderCallsExceptions binder_calls_exceptions = 10023; + LooperStats looper_stats = 10024; + DiskStats disk_stats = 10025; + DirectoryUsage directory_usage = 10026; + AppSize app_size = 10027; + CategorySize category_size = 10028; + ProcStats proc_stats = 10029; + BatteryVoltage battery_voltage = 10030; + NumFingerprints num_fingerprints = 10031; + DiskIo disk_io = 10032; + PowerProfile power_profile = 10033; + ProcStats proc_stats_pkg_proc = 10034; + ProcessCpuTime process_cpu_time = 10035; + NativeProcessMemoryState native_process_memory_state = 10036; + CpuTimePerThreadFreq cpu_time_per_thread_freq = 10037; } // DO NOT USE field numbers above 100,000 in AOSP. @@ -173,6 +212,21 @@ message AttributionNode { optional string tag = 2; } +message KeyValuePair { + optional int32 key = 1; + oneof value { + int32 value_int = 2; + int64 value_long = 3; + string value_str = 4; + float value_float = 5; + } +} + +message KeyValuePairsAtom { + optional int32 uid = 1; + repeated KeyValuePair pairs = 2; +} + /* * ***************************************************************************** * Below are all of the individual atoms that are logged by Android via statsd. @@ -227,20 +281,127 @@ message ThermalThrottlingStateChanged { */ message ScreenStateChanged { // New screen state, from frameworks/base/core/proto/android/view/enums.proto. - optional android.view.DisplayStateEnum state = 1 [(stateFieldOption).option = EXCLUSIVE]; + optional android.view.DisplayStateEnum state = 1 [(state_field_option).option = EXCLUSIVE]; } /** - * Logs that the state of a process state, as per the activity manager, has changed. + * Logs that the process state of the uid, as determined by ActivityManager + * (i.e. the highest process state of that uid's processes) has changed. * * Logged from: * frameworks/base/services/core/java/com/android/server/am/BatteryStatsService.java */ message UidProcessStateChanged { - optional int32 uid = 1 [(stateFieldOption).option = PRIMARY, (is_uid) = true]; + optional int32 uid = 1 [(state_field_option).option = PRIMARY, (is_uid) = true]; + + // The state, from frameworks/base/core/proto/android/app/enums.proto. + optional android.app.ProcessStateEnum state = 2 [(state_field_option).option = EXCLUSIVE]; +} +/** + * Logs process state change of a process, as per the activity manager. + * + * Logged from: + * frameworks/base/services/core/java/com/android/server/am/ProcessRecord.java + */ +message ProcessStateChanged { + optional int32 uid = 1; + optional string process_name = 2; + optional string package_name = 3; + // TODO: remove this when validation is done + optional int64 version = 5; // The state, from frameworks/base/core/proto/android/app/enums.proto. - optional android.app.ProcessStateEnum state = 2 [(stateFieldOption).option = EXCLUSIVE]; + optional android.app.ProcessStateEnum state = 4; +} + +/** + * Logs when ActivityManagerService sleep state is changed. + * + * Logged from: + * frameworks/base/services/core/java/com/android/server/am/ActivityTaskManagerService.java + */ +message ActivityManagerSleepStateChanged { + // TODO: import frameworks proto + enum State { + UNKNOWN = 0; + ASLEEP = 1; + AWAKE = 2; + } + optional State state = 1 [(state_field_option).option = EXCLUSIVE]; +} + +/** + * Logs when system memory state changes. + * + * Logged from: + * frameworks/base/services/core/java/com/android/server/am/ActivityManagerService.java + */ +message MemoryFactorStateChanged { + // TODO: import frameworks proto + enum State { + MEMORY_UNKNOWN = 0; + NORMAL = 1; // normal. + MODERATE = 2; // moderate memory pressure. + LOW = 3; // low memory. + CRITICAL = 4; // critical memory. + + } + optional State factor = 1 [(state_field_option).option = EXCLUSIVE]; +} + +/** + * Logs when app is using too much cpu, according to ActivityManagerService. + * + * Logged from: + * frameworks/base/services/core/java/com/android/server/am/ActivityManagerService.java + */ +message ExcessiveCpuUsageReported { + optional int32 uid = 1; + optional string process_name = 2; + optional string package_name = 3; + // package version. TODO: remove this when validation is done + optional int64 version = 4; +} + +/** + * Logs when a cached process is killed, along with its pss. + * + * Logged from: + * frameworks/base/services/core/java/com/android/server/am/ActivityManagerService.java + */ +message CachedKillReported { + optional int32 uid = 1; + optional string process_name = 2; + optional string package_name = 3; + // TODO: remove this when validation is done + optional int64 version = 5; + optional int64 pss = 4; +} + +/** + * Logs when memory stats of a process is reported. + * + * Logged from: + * frameworks/base/services/core/java/com/android/server/am/ProcessRecord.java + */ +message ProcessMemoryStatReported { + optional int32 uid = 1; + optional string process_name = 2; + optional string package_name = 3; + //TODO: remove this when validation is done + optional int64 version = 9; + optional int64 pss = 4; + optional int64 uss = 5; + optional int64 rss = 6; + enum Type { + ADD_PSS_INTERNAL_SINGLE = 0; + ADD_PSS_INTERNAL_ALL_MEM = 1; + ADD_PSS_INTERNAL_ALL_POLL = 2; + ADD_PSS_EXTERNAL = 3; + ADD_PSS_EXTERNAL_SLOW = 4; + } + optional Type type = 7; + optional int64 duration = 8; } /** @@ -360,6 +521,22 @@ message SyncStateChanged { optional State state = 3; } +/* + * Deferred job stats. + * + * Logged from: + * frameworks/base/services/core/java/com/android/server/job/JobSchedulerService.java +*/ +message DeferredJobStatsReported { + repeated AttributionNode attribution_node = 1; + + // Number of jobs deferred. + optional int32 num_jobs_deferred = 2; + + // Time since the last job runs. + optional int64 time_since_last_job_millis = 3; +} + /** * Logs when a job scheduler job state changes. * @@ -468,7 +645,7 @@ message WakelockStateChanged { // The type (level) of the wakelock; e.g. a partial wakelock or a full wakelock. // From frameworks/base/core/proto/android/os/enums.proto. - optional android.os.WakeLockLevelEnum level = 2; + optional android.os.WakeLockLevelEnum type = 2; // The wakelock tag (Called tag in the Java API, sometimes name elsewhere). optional string tag = 3; @@ -505,6 +682,20 @@ message LongPartialWakelockStateChanged { } /** + * Logs when the device is interactive, according to the PowerManager Notifier. + * + * Logged from: + * frameworks/base/services/core/java/com/android/server/power/Notifier.java + */ +message InteractiveStateChanged { + enum State { + OFF = 0; + ON = 1; + } + optional State state = 1; +} + +/** * Logs Battery Saver state change. * * Logged from: @@ -594,6 +785,9 @@ message WakeupAlarmOccurred { // Name of the wakeup alarm. optional string tag = 2; + + // Name of source package (for historical reasons, since BatteryStats tracked it). + optional string package_name = 3; } /** @@ -601,7 +795,7 @@ message WakeupAlarmOccurred { * Changing from LOW to MEDIUM or HIGH can be considered the app waking the mobile radio. * * Logged from: - * frameworks/base/core/java/com/android/internal/os/BatteryStatsImpl.java + * frameworks/base/services/core/java/com/android/server/NetworkManagementService.java */ message MobileRadioPowerStateChanged { repeated AttributionNode attribution_node = 1; @@ -615,7 +809,7 @@ message MobileRadioPowerStateChanged { * Changing from LOW to MEDIUM or HIGH can be considered the app waking the wifi radio. * * Logged from: - * frameworks/base/core/java/com/android/internal/os/BatteryStatsImpl.java + * frameworks/base/services/core/java/com/android/server/NetworkManagementService.java */ message WifiRadioPowerStateChanged { repeated AttributionNode attribution_node = 1; @@ -628,7 +822,7 @@ message WifiRadioPowerStateChanged { * Logs kernel wakeup reasons and aborts. * * Logged from: - * frameworks/base/services/core/java/com/android/server/am/BatteryStatsService.java + * frameworks/base/core/java/com/android/internal/os/BatteryStatsImpl.java */ message KernelWakeupReported { // Name of the kernel wakeup reason (or abort). @@ -844,20 +1038,20 @@ message ResourceConfigurationChanged { // Bit mask of color capabilities of the screen. // Contains information about the color gamut and hdr mode of the screen. // See: https://d.android.com/reference/android/content/res/Configuration.html#colorMode - optional int32 colorMode = 1; + optional int32 color_mode = 1; // The target screen density being rendered to. // See: https://d.android.com/reference/android/content/res/Configuration.html#densityDpi - optional int32 densityDpi = 2; + optional int32 density_dpi = 2; // Current user preference for the scaling factor for fonts, // relative to the base density scaling. // See: https://d.android.com/reference/android/content/res/Configuration.html#fontScale - optional float fontScale = 3; + optional float font_scale = 3; // Flag indicating whether the hard keyboard is hidden. // See: https://d.android.com/reference/android/content/res/Configuration.html#hardKeyboardHidden - optional int32 hardKeyboardHidden = 4; + optional int32 hard_keyboard_hidden = 4; // The type of keyboard attached to the device. // See: https://d.android.com/reference/android/content/res/Configuration.html#keyboard @@ -865,7 +1059,7 @@ message ResourceConfigurationChanged { // Flag indicating whether any keyboard is available. Takes soft keyboards into account. // See: https://d.android.com/reference/android/content/res/Configuration.html#keyboardHidden - optional int32 keyboardHideen = 6; + optional int32 keyboard_hidden = 6; // IMSI MCC (Mobile Country Code), corresponding to mcc resource qualifier. // 0 if undefined. @@ -884,7 +1078,7 @@ message ResourceConfigurationChanged { // Flag indicating whether the navigation is available. // See: https://d.android.com/reference/android/content/res/Configuration.html#navigationHidden - optional int32 navigationHidden = 10; + optional int32 navigation_hidden = 10; // Overall orientation of the screen. // See: https://d.android.com/reference/android/content/res/Configuration.html#orientation @@ -892,24 +1086,24 @@ message ResourceConfigurationChanged { // The current height of the available screen space, in dp units. // See: https://d.android.com/reference/android/content/res/Configuration.html#screenHeightDp - optional int32 screenHeightDp = 12; + optional int32 screen_height_dp = 12; // Bit mask of overall layout of the screen. // Contains information about screen size, whether the screen is wider/taller // than normal, whether the screen layout is right-tl-left or left-to-right, // and whether the screen has a rounded shape. // See: https://d.android.com/reference/android/content/res/Configuration.html#screenLayout - optional int32 screenLayout = 13; + optional int32 screen_layout = 13; // Current width of the available screen space, in dp units. // See: https://d.android.com/reference/android/content/res/Configuration.html#screenWidthDp - optional int32 screenWidthDp = 14; + optional int32 screen_width_dp = 14; // The smallest screen size an application will see in normal operation. // This is the smallest value of both screenWidthDp and screenHeightDp // in portrait and landscape. // See: https://d.android.com/reference/android/content/res/Configuration.html#smallestScreenWidthDp - optional int32 smallestScreenWidthDp = 15; + optional int32 smallest_screen_width_dp = 15; // The type of touch screen attached to the device. // See: https://d.android.com/reference/android/content/res/Configuration.html#touchscreen @@ -920,7 +1114,7 @@ message ResourceConfigurationChanged { // Eg: NORMAL, DESK, CAR, TELEVISION, WATCH, VR_HEADSET // Also contains information about whether the device is in night mode. // See: https://d.android.com/reference/android/content/res/Configuration.html#uiMode - optional int32 uiMode = 17; + optional int32 ui_mode = 17; } @@ -1007,7 +1201,7 @@ message BluetoothEnabledStateChanged { // Eg. Airplane mode, crash, application request. optional android.bluetooth.EnableDisableReasonEnum reason = 3; // If the reason is an application request, this will be the package name. - optional string pkgName = 4; + optional string pkg_name = 4; } /** @@ -1134,6 +1328,68 @@ message ChargeCyclesReported { } /** + * Log battery health snapshot. + * + * Resistance, Voltage, Open Circuit Voltage, Temperature, and Charge Level + * are snapshotted periodically over 24hrs. + */ +message BatteryHealthSnapshot { + enum BatterySnapshotType { + UNKNOWN = 0; + MIN_TEMP = 1; // Snapshot at min batt temp over 24hrs. + MAX_TEMP = 2; // Snapshot at max batt temp over 24hrs. + MIN_RESISTANCE = 3; // Snapshot at min batt resistance over 24hrs. + MAX_RESISTANCE = 4; // Snapshot at max batt resistance over 24hrs. + MIN_VOLTAGE = 5; // Snapshot at min batt voltage over 24hrs. + MAX_VOLTAGE = 6; // Snapshot at max batt voltage over 24hrs. + MIN_CURRENT = 7; // Snapshot at min batt current over 24hrs. + MAX_CURRENT = 8; // Snapshot at max batt current over 24hrs. + MIN_BATT_LEVEL = 9; // Snapshot at min battery level (SoC) over 24hrs. + MAX_BATT_LEVEL = 10; // Snapshot at max battery level (SoC) over 24hrs. + AVG_RESISTANCE = 11; // Snapshot at average battery resistance over 24hrs. + } + optional BatterySnapshotType type = 1; + // Temperature, in 1/10ths of degree C. + optional int32 temperature_deci_celsius = 2; + // Voltage Battery Voltage, in microVolts. + optional int32 voltage_micro_volt = 3; + // Current Battery current, in microAmps. + optional int32 current_micro_amps = 4; + // OpenCircuitVoltage Battery Open Circuit Voltage, in microVolts. + optional int32 open_circuit_micro_volt = 5; + // Resistance Battery Resistance, in microOhms. + optional int32 resistance_micro_ohm = 6; + // Level Battery Level, as % of full. + optional int32 level_percent = 7; +} + +/** + * Log slow I/O operations on the primary storage. + */ +message SlowIo { + // Classifications of IO Operations. + enum IoOperation { + UNKNOWN = 0; + READ = 1; + WRITE = 2; + UNMAP = 3; + SYNC = 4; + } + optional IoOperation operation = 1; + + // The number of slow IO operations of this type over 24 hours. + optional int32 count = 2; +} + +/** + * Log battery caused shutdown with the last recorded voltage. + */ +message BatteryCausedShutdown { + // The last recorded battery voltage prior to shutdown. + optional int32 last_recorded_micro_volt = 1; +} + +/** * Logs the duration of a davey (jank of >=700ms) when it occurs * * Logged from: @@ -1185,6 +1441,48 @@ message PhoneStateChanged { optional State state = 1; } +message LauncherUIChanged { + optional android.stats.launcher.LauncherAction action = 1; + optional android.stats.launcher.LauncherState src_state = 2; + optional android.stats.launcher.LauncherState dst_state = 3; + optional android.stats.launcher.LauncherExtension extension = 4 [(log_mode) = MODE_BYTES]; + optional bool is_swipe_up_enabled = 5; +} + +/** + * Logs when Settings UI has changed. + * + * Logged from: + * packages/apps/Settings + */ +message SettingsUIChanged { + /** + * Where this SettingsUIChange event comes from. For example, if + * it's a PAGE_VISIBLE event, where the page is opened from. + */ + optional android.app.settings.PageId attribution = 1; + + /** + * What the UI action is. + */ + optional android.app.settings.Action action = 2; + + /** + * Where the action is happening + */ + optional android.app.settings.PageId pageId = 3; + + /** + * What preference changed in this event. + */ + optional string changedPreferenceKey = 4; + + /** + * The new value of the changed preference. + */ + optional int64 changedPreferenceIntValue = 5; +} + /** * Logs that a setting was updated. * Logged from: @@ -1331,6 +1629,25 @@ message ANROccurred { optional ForegroundState foreground_state = 6; } +/** + * Logs when the vibrator state changes. + * Logged from: + * frameworks/base/services/core/java/com/android/server/VibratorService.java + */ +message VibratorStateChanged { + repeated AttributionNode attribution_node = 1; + + enum State { + OFF = 0; + ON = 1; + } + optional State state = 2; + + // Duration (in milliseconds) requested to keep the vibrator on. + // Only applicable for State == ON. + optional int64 duration_millis = 3; +} + /* * Allows other apps to push events into statsd. * Logged from: @@ -1355,6 +1672,18 @@ message AppBreadcrumbReported { } /** + * Logs the wall-clock time when a significant wall-clock time shift occurs. + * For example, this could be due to the user manually changing the time. + * + * Logged from: + * frameworks/base/services/core/java/com/android/server/AlarmManagerService.java + */ +message WallClockTimeShifted { + // New wall-clock time in milliseconds, according to System.currentTimeMillis(). + optional int64 wall_clock_timestamp_millis = 1; +} + +/** * Logs when statsd detects an anomaly. * * Logged from: @@ -1533,6 +1862,7 @@ message ForegroundServiceStateChanged { message IsolatedUidChanged { // The host UID. Generally, we should attribute metrics from the isolated uid to the host uid. // NOTE: DO NOT annotate uid field in this atom. This atom is specially handled in statsd. + // This field is ignored when event == REMOVED. optional int32 parent_uid = 1; optional int32 isolated_uid = 2; @@ -1590,10 +1920,10 @@ message AppStartMemoryStateCaptured { optional string activity_name = 3; // # of page-faults - optional int64 pgfault = 4; + optional int64 page_fault = 4; // # of major page-faults - optional int64 pgmajfault = 5; + optional int64 page_major_fault = 5; // RSS optional int64 rss_in_bytes = 6; @@ -1633,13 +1963,13 @@ message LmkKillOccurred { optional string process_name = 2; // oom adj score. - optional int32 oom_score = 3; + optional int32 oom_adj_score = 3; // # of page-faults - optional int64 pgfault = 4; + optional int64 page_fault = 4; // # of major page-faults - optional int64 pgmajfault = 5; + optional int64 page_major_fault = 5; // RSS optional int64 rss_in_bytes = 6; @@ -1659,9 +1989,146 @@ message LmkKillOccurred { */ message AppDied { // timestamp(elapsedRealtime) of record creation - optional uint64 timestamp_millis = 1 [(stateFieldOption).option = EXCLUSIVE]; + optional uint64 timestamp_millis = 1 [(state_field_option).option = EXCLUSIVE]; +} + +/** + * An atom for generic metrics logging. Available from Android Q. + */ +message GenericAtom { + // The uid of the application that sent this custom atom. + optional int32 uid = 1 [(is_uid) = true]; + + // An event_id indicates the type of event. + optional android.stats.EventType event_id = 2; +} + +/** + * Logs when a fingerprint acquire event occurs. + * + * Logged from: + * frameworks/base/services/core/java/com/android/server/biometrics/fingerprint/FingerprintService.java + */ +message FingerprintAcquired { + // The associated user. Eg: 0 for owners, 10+ for others. + // Defined in android/os/UserHandle.java + optional int32 user = 1; + // If this acquire is for a crypto fingerprint. + // e.g. Secure purchases, unlock password storage. + optional bool is_crypto = 2; } +/** + * Logs when a fingerprint authentication event occurs. + * + * Logged from: + * frameworks/base/services/core/java/com/android/server/biometrics/fingerprint/FingerprintService.java + */ +message FingerprintAuthenticated { + // The associated user. Eg: 0 for owners, 10+ for others. + // Defined in android/os/UserHandle.java + optional int32 user = 1; + // If this authentication is for a crypto fingerprint. + // e.g. Secure purchases, unlock password storage. + optional bool is_crypto = 2; + // Whether or not this authentication was successful. + optional bool is_authenticated = 3; +} + +/** + * Logs when a fingerprint error occurs. + * + * Logged from: + * frameworks/base/services/core/java/com/android/server/biometrics/fingerprint/FingerprintService.java + */ +message FingerprintErrorOccurred { + // The associated user. Eg: 0 for owners, 10+ for others. + // Defined in android/os/UserHandle.java + optional int32 user = 1; + // If this error is for a crypto fingerprint. + // e.g. Secure purchases, unlock password storage. + optional bool is_crypto = 2; + + enum Error { + UNKNOWN = 0; + LOCKOUT = 1; + PERMANENT_LOCKOUT = 2; + } + // The type of error. + optional Error error = 3; +} + +message Notification { + + // Type of notification event. + enum Type { + TYPE_UNKNOWN = 0; + // Notification became visible to the user. + TYPE_OPEN = 1; + // Notification became hidden. + TYPE_CLOSE = 2; + // Notification switched to detail mode. + TYPE_DETAIL = 3; + // Notification was clicked. + TYPE_ACTION = 4; + // Notification was dismissed. + TYPE_DISMISS = 5; + // Notification switched to summary mode. The enum value of 14 is to + // match that of metrics_constants. + TYPE_COLLAPSE = 14; + } + optional Type type = 1; + + // Package name associated with the notification. + optional string package_name = 2; + + // Tag associated with notification. + optional string tag = 3; + + // Application-supplied ID associated with the notification. + optional int32 id = 4; + + // Index of notification in the notification panel. + optional int32 shade_index = 5; + + // The number of notifications in the notification panel. + optional int32 shade_count = 6; + + // Importance for the notification. + optional int32 importance = 7; + + // ID for the notification channel. + optional string channel_id = 8; + + // Importance for the notification channel. + optional int32 channel_importance = 9; + + // Application-supplied ID associated with the notifications group. + optional string group_id = 10; + + // Whether notification was a group summary. + optional bool group_summary = 11; + + // Reason for dismissal of a notification. This field is only meaningful for + // TYPE_DISMISS events. + optional int32 dismiss_reason = 12; + + // The first post time of notification, stable across updates. + optional int64 creation_millis = 13; + + // The most recent interruption time, or the creation time if no updates. + // Differs from update_millis because updates are filtered based on whether + // they actually interrupted the user. + optional int64 interruption_millis = 14; + + // The most recent update time, or the creation time if no updates. + optional int64 update_millis = 15; + + // The most recent visibility event. + optional int64 visible_millis = 16; +} + + ////////////////////////////////////////////////////////////////////// // Pulled atoms below this line // ////////////////////////////////////////////////////////////////////// @@ -1773,7 +2240,7 @@ message KernelWakelock { optional int32 version = 3; - optional int64 time = 4; + optional int64 time_micros = 4; } /** @@ -1837,11 +2304,11 @@ message WifiActivityInfo { // stack reported state // TODO: replace this with proto enum optional int32 stack_state = 2; - // tx time in ms + // tx time in millis optional uint64 controller_tx_time_millis = 3; - // rx time in ms + // rx time in millis optional uint64 controller_rx_time_millis = 4; - // idle time in ms + // idle time in millis optional uint64 controller_idle_time_millis = 5; // product of current(mA), voltage(V) and time(ms) optional uint64 controller_energy_used = 6; @@ -1853,9 +2320,9 @@ message WifiActivityInfo { message ModemActivityInfo { // timestamp(wall clock) of record creation optional uint64 timestamp_millis = 1; - // sleep time in ms. + // sleep time in millis. optional uint64 sleep_time_millis = 2; - // idle time in ms + // idle time in millis optional uint64 controller_idle_time_millis = 3; /** * Tx power index @@ -1890,11 +2357,11 @@ message BluetoothActivityInfo { optional uint64 timestamp_millis = 1; // bluetooth stack state optional int32 bluetooth_stack_state = 2; - // tx time in ms + // tx time in millis optional uint64 controller_tx_time_millis = 3; - // rx time in ms + // rx time in millis optional uint64 controller_rx_time_millis = 4; - // idle time in ms + // idle time in millis optional uint64 controller_idle_time_millis = 5; // product of current(mA), voltage(V) and time(ms) optional uint64 energy_used = 6; @@ -1911,13 +2378,13 @@ message ProcessMemoryState { optional string process_name = 2; // oom adj score. - optional int32 oom_score = 3; + optional int32 oom_adj_score = 3; // # of page-faults - optional int64 pgfault = 4; + optional int64 page_fault = 4; // # of major page-faults - optional int64 pgmajfault = 5; + optional int64 page_major_fault = 5; // RSS optional int64 rss_in_bytes = 6; @@ -1927,6 +2394,43 @@ message ProcessMemoryState { // SWAP optional int64 swap_in_bytes = 8; + + // RSS high watermark. + // Peak RSS usage of the process. Value is read from the VmHWM field in /proc/PID/status or + // from memory.max_usage_in_bytes under /dev/memcg if the device uses per-app memory cgroups. + optional int64 rss_high_watermark_in_bytes = 9; + + // Elapsed real time when the process started. + // Value is read from /proc/PID/stat, field 22. 0 if read from per-app memory cgroups. + optional int64 start_time_nanos = 10; +} + +/* + * Logs the memory stats for a native process (from procfs). + */ +message NativeProcessMemoryState { + // The uid if available. -1 means not available. + optional int32 uid = 1 [(is_uid) = true]; + + // The process name. + optional string process_name = 2; + + // # of page-faults + optional int64 page_fault = 3; + + // # of major page-faults + optional int64 page_major_fault = 4; + + // RSS + optional int64 rss_in_bytes = 5; + + // RSS high watermark. + // Peak RSS usage of the process. Value is read from the VmHWM field in /proc/PID/status. + optional int64 rss_high_watermark_in_bytes = 6; + + // Elapsed real time when the process started. + // Value is read from /proc/PID/stat, field 22. + optional int64 start_time_nanos = 7; } /* @@ -1994,7 +2498,7 @@ message DiskSpace { * frameworks/base/cmds/statsd/src/external/ResourceHealthManagerPuller.cpp */ message RemainingBatteryCapacity { - optional int32 charge_uAh = 1; + optional int32 charge_micro_ampere_hour = 1; } /** @@ -2003,7 +2507,17 @@ message RemainingBatteryCapacity { * frameworks/base/cmds/statsd/src/external/ResourceHealthManagerPuller.cpp */ message FullBatteryCapacity { - optional int32 capacity_uAh = 1; + optional int32 capacity_micro_ampere_hour = 1; +} + +/** + * Pulls battery voltage. + * Pulled from: + * frameworks/base/cmds/statsd/src/external/ResourceHealthManagerPuller.cpp + */ +message BatteryVoltage { + // The voltage of the battery, in millivolts. + optional int32 voltage_millivolt = 1; } /** @@ -2021,5 +2535,627 @@ message Temperature { optional string sensor_name = 2; // Temperature in tenths of a degree C. - optional int32 temperature_dC = 3; + optional int32 temperature_deci_celsius = 3; +} + +/** + * Pulls the statistics of calls to Binder. + * + * Binder stats will be reset every time the data is pulled. It means it can only be pulled by one + * config on the device. + * + * Next tag: 15 + */ +message BinderCalls { + // UID of the process responsible for the binder transaction. It will be set if the process + // executing the binder transaction attribute the transaction to another uid using + // Binder.setThreadWorkSource(). + // + // If not set, the value will be -1. + optional int32 uid = 1 [(is_uid) = true]; + // UID of the process executing the binder transaction. + optional int32 direct_caller_uid = 14; + // Fully qualified class name of the API call. + // + // This is a system server class name. + // + // TODO(gaillard): figure out if binder call stats includes data from isolated uids, if a uid + // gets recycled and we have isolated uids, we might attribute the data incorrectly. + // TODO(gaillard): there is a high dimensions cardinality, figure out if we should drop the less + // commonly used APIs. + optional string service_class_name = 2; + // Method name of the API call. It can also be a transaction code if we cannot + // resolve it to a name. See Binder#getTransactionName. + // + // This is a system server method name. + optional string service_method_name = 3; + // Total number of API calls. + optional int64 call_count = 4; + // True if the screen was interactive PowerManager#isInteractive at the end of the call. + optional bool screen_interactive = 13; + // Total number of API calls we have data recorded for. If we collected data for all the calls, + // call_count will be equal to recorded_call_count. + // + // If recorded_call_count is different than call_count, it means data collection has been + // sampled. All the fields below will be sampled in this case. + optional int64 recorded_call_count = 12; + // Number of exceptions thrown by the API. + optional int64 recorded_exception_count = 5; + // Total latency of all API calls. + // Average can be computed using total_latency_micros / recorded_call_count. + optional int64 recorded_total_latency_micros = 6; + // Maximum latency of one API call. + optional int64 recorded_max_latency_micros = 7; + // Total CPU usage of all API calls. + // Average can be computed using total_cpu_micros / recorded_call_count. + // Total can be computed using total_cpu_micros / recorded_call_count * call_count. + optional int64 recorded_total_cpu_micros = 8; + // Maximum CPU usage of one API call. + optional int64 recorded_max_cpu_micros = 9; + // Maximum parcel reply size of one API call. + optional int64 recorded_max_reply_size_bytes = 10; + // Maximum parcel request size of one API call. + optional int64 recorded_max_request_size_bytes = 11; +} + +/** + * Pulls the statistics of exceptions during calls to Binder. + * + * Binder stats are cumulative from boot unless somebody reset the data using + * > adb shell dumpsys binder_calls_stats --reset + */ +message BinderCallsExceptions { + // Exception class name, e.g. java.lang.IllegalArgumentException. + // + // This is an exception class name thrown by the system server. + optional string exception_class_name = 1; + // Total number of exceptions. + optional int64 exception_count = 2; +} + +/** + * Pulls the statistics of message dispatching on HandlerThreads. + * + * Looper stats will be reset every time the data is pulled. It means it can only be pulled by one + * config on the device. + * + * Next tag: 11 + */ +message LooperStats { + // The uid that made a call to the System Server and caused the message to be enqueued. + optional int32 uid = 1 [(is_uid) = true]; + + // Fully qualified class name of the handler target class. + // + // This field does not contain PII. This is a system server class name. + optional string handler_class_name = 2; + + // The name of the thread that runs the Looper. + // + // This field does not contain PII. This is a system server thread name. + optional string looper_thread_name = 3; + + // The name of the dispatched message. + // + // This field does not contain PII. This is a system server constant or class + // name. + optional string message_name = 4; + + // Total number of successfully dispatched messages. + optional int64 message_count = 5; + + // Total number of messages that failed dispatching. + optional int64 exception_count = 6; + + // Total number of processed messages we have data recorded for. If we + // collected data for all the messages, message_count will be equal to + // recorded_message_count. + // + // If recorded_message_count is different than message_count, it means data + // collection has been sampled. The fields below will be sampled in this case. + optional int64 recorded_message_count = 7; + + // Total latency of all processed messages. + // Average can be computed using recorded_total_latency_micros / + // recorded_message_count. + optional int64 recorded_total_latency_micros = 8; + + // Total CPU usage of all processed message. + // Average can be computed using recorded_total_cpu_micros / + // recorded_message_count. Total can be computed using + // recorded_total_cpu_micros / recorded_message_count * message_count. + optional int64 recorded_total_cpu_micros = 9; + + // True if the screen was interactive PowerManager#isInteractive at the end of the call. + optional bool screen_interactive = 10; + + // Max recorded CPU usage of all processed messages. + optional int64 recorded_max_cpu_micros = 11; + + // Max recorded latency of all processed messages. + optional int64 recorded_max_latency_micros = 12; + + // Total number of messages we tracked the dispatching delay for. If we + // collected data for all the messages, message_count will be equal to + // recorded_delay_message_count. + // + // If recorded_delay_message_count is different than message_count, it means data + // collection has been sampled or/and not all messages specified the target dispatch time. + // The fields below will be sampled in this case. + optional int64 recorded_delay_message_count = 13; + + // Total dispatching delay of all processed messages. + // Calculated as a difference between the target dispatching time (Message.when) + // and the actual dispatching time. + // Average can be computed using recorded_total_delay_millis / recorded_delay_message_count. + optional int64 recorded_total_delay_millis = 14; + + // Max dispatching delay of all processed messages. + // Calculated as a difference between the target dispatching time (Message.when) + // and the actual dispatching time. + optional int64 recorded_max_delay_millis = 15; +} + +/** + * Pulls disk information, such as write speed and latency. + */ +message DiskStats { + // Time taken to open, write 512B to, and close a file. + // -1 if error performing the check. + optional int64 data_write_latency_millis = 1; + + optional bool file_based_encryption = 2; + + // Recent disk write speed in kB/s. + // -1 if error querying storageed. + // 0 if data is unavailable. + optional int32 recent_disk_write_speed = 3; +} + + +/** + * Free and total bytes of the Data, Cache, and System partition. + */ +message DirectoryUsage { + enum Directory { + UNKNOWN = 0; + DATA = 1; + CACHE = 2; + SYSTEM = 3; + } + optional Directory directory = 1; + optional int64 free_bytes = 2; + optional int64 total_bytes = 3; +} + + +/** + * Size of an application: apk size, data size, and cache size. + * Reads from a cached file produced daily by DiskStatsLoggingService.java. + * Information is only reported for apps with the primary user (user 0). + * Sizes are aggregated by package name. + */ +message AppSize { + // Including uids will involve modifying diskstats logic. + optional string package_name = 1; + // App size in bytes. -1 if unavailable. + optional int64 app_size_bytes = 2; + // App data size in bytes. -1 if unavailable. + optional int64 app_data_size_bytes = 3; + // App cache size in bytes. -1 if unavailable. + optional int64 app_cache_size_bytes = 4; + // Time that the cache file was produced. + // Uses System.currentTimeMillis(), which is wall clock time. + optional int64 cache_time_millis = 5; +} + + +/** + * Size of a particular category. Eg: photos, videos. + * Reads from a cached file produced daily by DiskStatsLoggingService.java. + */ +message CategorySize { + enum Category { + UNKNOWN = 0; + APP_SIZE = 1; + APP_DATA_SIZE = 2; + APP_CACHE_SIZE = 3; + PHOTOS = 4; + VIDEOS = 5; + AUDIO = 6; + DOWNLOADS = 7; + SYSTEM = 8; + OTHER = 9; + } + optional Category category = 1; + // Category size in bytes. + optional int64 size_bytes = 2; + // Time that the cache file was produced. + // Uses System.currentTimeMillis(), which is wall clock time. + optional int64 cache_time_millis = 3; +} + +/** + * Pulls per uid I/O stats. The stats are cumulative since boot. + * + * Read/write bytes are I/O events from a storage device + * Read/write chars are data requested by read/write syscalls, and can be + * satisfied by caching. + * + * Pulled from StatsCompanionService, which reads proc/uid_io/stats. + */ +message DiskIo { + optional int32 uid = 1 [(is_uid) = true]; + optional int64 fg_chars_read = 2; + optional int64 fg_chars_write = 3; + optional int64 fg_bytes_read = 4; + optional int64 fg_bytes_write = 5; + optional int64 bg_chars_read = 6; + optional int64 bg_chars_write = 7; + optional int64 bg_bytes_read = 8; + optional int64 bg_bytes_write = 9; + optional int64 fg_fsync = 10; + optional int64 bg_fsync= 11; +} + + +/** + * Pulls the number of fingerprints for each user. + * + * Pulled from StatsCompanionService, which queries FingerprintManager. + */ +message NumFingerprints { + // The associated user. Eg: 0 for owners, 10+ for others. + // Defined in android/os/UserHandle.java + optional int32 user = 1; + // Number of fingerprints registered to that user. + optional int32 num_fingerprints = 2; +} + +message AggStats { + optional int64 min = 1; + + optional int64 average = 2; + + optional int64 max = 3; +} + +message ProcessStatsStateProto { + optional android.service.procstats.ScreenState screen_state = 1; + + optional android.service.procstats.MemoryState memory_state = 2; + + // this enum list is from frameworks/base/core/java/com/android/internal/app/procstats/ProcessStats.java + // and not frameworks/base/core/java/android/app/ActivityManager.java + optional android.service.procstats.ProcessState process_state = 3; + + // Millisecond uptime duration spent in this state + optional int64 duration_millis = 4; + + // Millisecond elapsed realtime duration spent in this state + optional int64 realtime_duration_millis = 9; + + // # of samples taken + optional int32 sample_size = 5; + + // PSS is memory reserved for this process + optional AggStats pss = 6; + + // USS is memory shared between processes, divided evenly for accounting + optional AggStats uss = 7; + + // RSS is memory resident for this process + optional AggStats rss = 8; +} + +// Next Tag: 7 +message ProcessStatsProto { + // Name of process. + optional string process = 1; + + // Uid of the process. + optional int32 uid = 2; + + // Information about how often kills occurred + message Kill { + // Count of excessive CPU kills + optional int32 cpu = 1; + + // Count of kills when cached + optional int32 cached = 2; + + // PSS stats during cached kill + optional AggStats cached_pss = 3; + } + optional Kill kill = 3; + + // Time and memory spent in various states. + repeated ProcessStatsStateProto states = 5; + + // Total time process has been running... screen_state, memory_state, and process_state + // will not be set. + optional ProcessStatsStateProto total_running_state = 6; +} + +message PackageServiceOperationStatsProto { + // Operate enum: Started, Foreground, Bound, Executing + optional android.service.procstats.ServiceOperationState operation = 1; + + // Number of times the service was in this operation. + optional int32 count = 2; + + // Information about a state the service can be in. + message StateStats { + // Screen state enum. + optional android.service.procstats.ScreenState screen_state = 1; + // Memory state enum. + optional android.service.procstats.MemoryState memory_state = 2; + + // duration in milliseconds. + optional int64 duration_millis = 3; + // Millisecond elapsed realtime duration spent in this state + optional int64 realtime_duration_millis = 4; + } + repeated StateStats state_stats = 3; +} + +message PackageServiceStatsProto { + // Name of service component. + optional string service_name = 1; + + // The operation stats. + // The package_name, package_uid, package_version, service_name will not be set to save space. + repeated PackageServiceOperationStatsProto operation_stats = 2; +} + +message PackageAssociationSourceProcessStatsProto { + // Uid of the process. + optional int32 process_uid = 1; + // Process name. + optional string process_name = 2; + + // Total count of the times this association appeared. + optional int32 total_count = 3; + + // Millisecond uptime total duration this association was around. + optional int64 total_duration_millis = 4; + + // Total count of the times this association became actively impacting its target process. + optional int32 active_count = 5; + + // Information on one source in this association. + message StateStats { + // Process state enum. + optional android.service.procstats.ProcessState process_state = 1; + // Millisecond uptime duration spent in this state + optional int64 duration_millis = 2; + // Millisecond elapsed realtime duration spent in this state + optional int64 realtime_duration_mmillis = 3; + } + repeated StateStats active_state_stats = 6; +} + +message PackageAssociationProcessStatsProto { + // Name of the target component. + optional string component_name = 1; + // Information on one source in this association. + repeated PackageAssociationSourceProcessStatsProto sources = 2; +} + + +message ProcessStatsPackageProto { + // Name of package. + optional string package = 1; + + // Uid of the package. + optional int32 uid = 2; + + // Version of the package. + optional int64 version = 3; + + // Stats for each process running with the package loaded in to it. + repeated ProcessStatsProto process_stats = 4; + + // Stats for each of the package's services. + repeated PackageServiceStatsProto service_stats = 5; + + // Stats for each association with the package. + repeated PackageAssociationProcessStatsProto association_stats = 6; +} + +message ProcessStatsSectionProto { + // Elapsed realtime at start of report. + optional int64 start_realtime_millis = 1; + + // Elapsed realtime at end of report. + optional int64 end_realtime_millis = 2; + + // CPU uptime at start of report. + optional int64 start_uptime_millis = 3; + + // CPU uptime at end of report. + optional int64 end_uptime_millis = 4; + + // System runtime library. e.g. "libdvm.so", "libart.so". + optional string runtime = 5; + + // whether kernel reports swapped pss. + optional bool has_swapped_pss = 6; + + // Data completeness. e.g. "complete", "partial", shutdown", or "sysprops". + enum Status { + STATUS_UNKNOWN = 0; + STATUS_COMPLETE = 1; + STATUS_PARTIAL = 2; + STATUS_SHUTDOWN = 3; + STATUS_SYSPROPS = 4; + } + repeated Status status = 7; + + // Stats for each process. + repeated ProcessStatsProto process_stats = 8; + + // Stats for each package. + repeated ProcessStatsPackageProto package_stats = 9; +} + +/** + * Pulled from ProcessStatsService.java + */ +message ProcStats { + optional ProcessStatsSectionProto proc_stats_section = 1; +} + +message PowerProfileProto { + optional double cpu_suspend = 1; + + optional double cpu_idle = 2; + + optional double cpu_active = 3; + + message CpuCluster { + optional int32 id = 1; + optional double cluster_power = 2; + optional int32 cores = 3; + repeated int64 speed = 4; + repeated double core_power = 5; + } + + repeated CpuCluster cpu_cluster = 40; + + optional double wifi_scan = 4; + + optional double wifi_on = 5; + + optional double wifi_active = 6; + + optional double wifi_controller_idle = 7; + + optional double wifi_controller_rx = 8; + + optional double wifi_controller_tx = 9; + + repeated double wifi_controller_tx_levels = 10; + + optional double wifi_controller_operating_voltage = 11; + + optional double bluetooth_controller_idle = 12; + + optional double bluetooth_controller_rx = 13; + + optional double bluetooth_controller_tx = 14; + + optional double bluetooth_controller_operating_voltage = 15; + + optional double modem_controller_sleep = 16; + + optional double modem_controller_idle = 17; + + optional double modem_controller_rx = 18; + + repeated double modem_controller_tx = 19; + + optional double modem_controller_operating_voltage = 20; + + optional double gps_on = 21; + + repeated double gps_signal_quality_based = 22; + + optional double gps_operating_voltage = 23; + + optional double bluetooth_on = 24; + + optional double bluetooth_active = 25; + + optional double bluetooth_at_cmd = 26; + + optional double ambient_display = 27; + + optional double screen_on = 28; + + optional double radio_on = 29; + + optional double radio_scanning = 30; + + optional double radio_active = 31; + + optional double screen_full = 32; + + optional double audio = 33; + + optional double video = 34; + + optional double flashlight = 35; + + optional double memory = 36; + + optional double camera = 37; + + optional double wifi_batched_scan = 38; + + optional double battery_capacity = 39; +} + +/** + * power_profile.xml and other constants for power model calculations. + * Pulled from PowerProfile.java + */ +message PowerProfile { + optional PowerProfileProto power_profile = 1; +} + +/** + * Logs when a user restriction was added or removed. + * + * Logged from: + * frameworks/base/services/devicepolicy/java/com/android/server/devicepolicy/DevicePolicyManagerService.java + */ +message UserRestrictionChanged { + // The raw string of the user restriction as defined in UserManager. + // Allowed values are defined in UserRestrictionsUtils#USER_RESTRICTIONS. + optional string restriction = 1; + // Whether the restriction is enabled or disabled. + optional bool enabled = 2; +} + +/** + * Pulls process user time and system time. Puller takes a snapshot of all pids + * in the system and returns cpu stats for those that are working at the time. + * Dead pids will be dropped. Kernel processes are excluded. + * Min cool-down is 5 sec. + */ +message ProcessCpuTime { + optional int32 uid = 1 [(is_uid) = true]; + + optional string process_name = 2; + // Process cpu time in user space, cumulative from boot/process start + optional int64 user_time_millis = 3; + // Process cpu time in system space, cumulative from boot/process start + optional int64 system_time_millis = 4; +} + +/** + * Pulls the CPU usage for each thread. + * + * Read from /proc/$PID/task/$TID/time_in_state files. + * + * TODO(mishaw): This is an experimental atom. Issues with big/little CPU frequencies, and + * time_in_state files not being present on some phones, have not been addressed. These should be + * considered before a public release. + */ +message CpuTimePerThreadFreq { + // UID that owns the process. + optional int32 uid = 1 [(is_uid) = true]; + // ID of the process. + optional uint32 process_id = 2; + // ID of the thread. + optional uint32 thread_id = 3; + // Name of the process taken from `/proc/$PID/cmdline`. + optional string process_name = 4; + // Name of the thread taken from `/proc/$PID/task/$TID/comm` + optional string thread_name = 5; + // What frequency the CPU was running at, in KHz + optional uint32 frequency_khz = 6; + // Time spent in frequency in milliseconds, since thread start. + optional uint32 time_millis = 7; } diff --git a/cmds/statsd/src/condition/condition_util.cpp b/cmds/statsd/src/condition/condition_util.cpp index 691356b5edc6..35e03e45c785 100644 --- a/cmds/statsd/src/condition/condition_util.cpp +++ b/cmds/statsd/src/condition/condition_util.cpp @@ -76,9 +76,9 @@ ConditionState evaluateCombinationCondition(const std::vector<int>& children, break; } case LogicalOperation::NOT: - newCondition = (conditionCache[children[0]] == ConditionState::kFalse) - ? ConditionState::kTrue - : ConditionState::kFalse; + newCondition = children.empty() ? ConditionState::kUnknown : + ((conditionCache[children[0]] == ConditionState::kFalse) ? + ConditionState::kTrue : ConditionState::kFalse); break; case LogicalOperation::NAND: newCondition = hasFalse ? ConditionState::kTrue : ConditionState::kFalse; diff --git a/cmds/statsd/src/config/ConfigManager.h b/cmds/statsd/src/config/ConfigManager.h index 611c34250a38..122e669057b0 100644 --- a/cmds/statsd/src/config/ConfigManager.h +++ b/cmds/statsd/src/config/ConfigManager.h @@ -31,9 +31,6 @@ namespace android { namespace os { namespace statsd { -// Util function to build a hard coded config with test metrics. -StatsdConfig build_fake_config(); - /** * Keeps track of which configurations have been set from various sources. */ diff --git a/cmds/statsd/src/external/Perfetto.cpp b/cmds/statsd/src/external/Perfetto.cpp index c1f9a643e153..42cc543f18a8 100644 --- a/cmds/statsd/src/external/Perfetto.cpp +++ b/cmds/statsd/src/external/Perfetto.cpp @@ -113,7 +113,7 @@ bool CollectPerfettoTraceAndUploadToDropbox(const PerfettoDetails& config, return false; } - std::string cfgProto = config.trace_config().SerializeAsString(); + const std::string& cfgProto = config.trace_config(); size_t bytesWritten = fwrite(cfgProto.data(), 1, cfgProto.size(), writePipeStream); fclose(writePipeStream); if (bytesWritten != cfgProto.size() || cfgProto.size() == 0) { diff --git a/cmds/statsd/src/external/Perfprofd.cpp b/cmds/statsd/src/external/Perfprofd.cpp new file mode 100644 index 000000000000..1678f104a07a --- /dev/null +++ b/cmds/statsd/src/external/Perfprofd.cpp @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Perfprofd.h" + +#define DEBUG false // STOPSHIP if true +#include "config/ConfigKey.h" +#include "Log.h" + +#include <errno.h> +#include <fcntl.h> +#include <sys/types.h> +#include <sys/wait.h> +#include <unistd.h> + +#include <string> + +#include <binder/IServiceManager.h> + +#include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" // Alert + +#include "android/os/IPerfProfd.h" + +namespace android { +namespace os { +namespace statsd { + +bool CollectPerfprofdTraceAndUploadToDropbox(const PerfprofdDetails& config, + int64_t alert_id, + const ConfigKey& configKey) { + VLOG("Starting trace collection through perfprofd"); + + if (!config.has_perfprofd_config()) { + ALOGE("The perfprofd trace config is empty, aborting"); + return false; + } + + sp<IPerfProfd> service = interface_cast<IPerfProfd>( + defaultServiceManager()->getService(android::String16("perfprofd"))); + if (service == NULL) { + ALOGE("Could not find perfprofd service"); + return false; + } + + auto* data = reinterpret_cast<const uint8_t*>(config.perfprofd_config().data()); + std::vector<uint8_t> proto_serialized(data, data + config.perfprofd_config().size()); + + // TODO: alert-id etc? + + binder::Status status = service->startProfilingProtobuf(proto_serialized); + if (status.isOk()) { + return true; + } + + ALOGE("Error starting perfprofd profiling: %s", status.toString8().c_str()); + return false; +} + +} // namespace statsd +} // namespace os +} // namespace android diff --git a/cmds/statsd/src/external/Perfprofd.h b/cmds/statsd/src/external/Perfprofd.h new file mode 100644 index 000000000000..b93fdf8e1cb2 --- /dev/null +++ b/cmds/statsd/src/external/Perfprofd.h @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <inttypes.h> + +namespace android { +namespace os { +namespace statsd { + +class ConfigKey; +class PerfprofdDetails; // Declared in statsd_config.pb.h + +// Starts the collection of a Perfprofd trace with the given |config|. +// The trace is uploaded to Dropbox by the perfprofd service once done. +// This method returns immediately after passing the config and does NOT wait +// for the full duration of the trace. +bool CollectPerfprofdTraceAndUploadToDropbox(const PerfprofdDetails& config, + int64_t alert_id, + const ConfigKey& configKey); + +} // namespace statsd +} // namespace os +} // namespace android diff --git a/cmds/statsd/src/external/ResourceHealthManagerPuller.cpp b/cmds/statsd/src/external/ResourceHealthManagerPuller.cpp index 3741202763b3..ae2cf74962c3 100644 --- a/cmds/statsd/src/external/ResourceHealthManagerPuller.cpp +++ b/cmds/statsd/src/external/ResourceHealthManagerPuller.cpp @@ -54,7 +54,7 @@ bool getHealthHal() { ResourceHealthManagerPuller::ResourceHealthManagerPuller(int tagId) : StatsPuller(tagId) { } -// TODO: add other health atoms (eg. Temperature). +// TODO(b/110565992): add other health atoms (eg. Temperature). bool ResourceHealthManagerPuller::PullInternal(vector<shared_ptr<LogEvent>>* data) { if (!getHealthHal()) { ALOGE("Health Hal not loaded"); @@ -67,6 +67,7 @@ bool ResourceHealthManagerPuller::PullInternal(vector<shared_ptr<LogEvent>>* dat data->clear(); bool result_success = true; + // Get the data from the Health HAL (hardware/interfaces/health/1.0/types.hal). Return<void> ret = gHealthHal->getHealthInfo([&](Result r, HealthInfo v) { if (r != Result::SUCCESS) { result_success = false; @@ -84,6 +85,12 @@ bool ResourceHealthManagerPuller::PullInternal(vector<shared_ptr<LogEvent>>* dat ptr->write(v.legacy.batteryFullCharge); ptr->init(); data->push_back(ptr); + } else if (mTagId == android::util::BATTERY_VOLTAGE) { + auto ptr = make_shared<LogEvent>(android::util::BATTERY_VOLTAGE, + wallClockTimestampNs, elapsedTimestampNs); + ptr->write(v.legacy.batteryVoltage); + ptr->init(); + data->push_back(ptr); } else { ALOGE("Unsupported tag in ResourceHealthManagerPuller: %d", mTagId); } diff --git a/cmds/statsd/src/external/StatsCompanionServicePuller.cpp b/cmds/statsd/src/external/StatsCompanionServicePuller.cpp index d953f50bb5d8..3eb05a90e3b4 100644 --- a/cmds/statsd/src/external/StatsCompanionServicePuller.cpp +++ b/cmds/statsd/src/external/StatsCompanionServicePuller.cpp @@ -36,8 +36,6 @@ namespace android { namespace os { namespace statsd { -const int kLogMsgHeaderSize = 28; - // The reading and parsing are implemented in Java. It is not difficult to port over. But for now // let StatsCompanionService handle that and send the data back. StatsCompanionServicePuller::StatsCompanionServicePuller(int tagId) : StatsPuller(tagId) { @@ -56,20 +54,12 @@ bool StatsCompanionServicePuller::PullInternal(vector<shared_ptr<LogEvent> >* da vector<StatsLogEventWrapper> returned_value; Status status = statsCompanionServiceCopy->pullData(mTagId, &returned_value); if (!status.isOk()) { - ALOGW("error pulling for %d", mTagId); + ALOGW("StatsCompanionServicePuller::pull failed for %d", mTagId); return false; } data->clear(); - int32_t timestampSec = getWallClockSec(); for (const StatsLogEventWrapper& it : returned_value) { - log_msg tmp; - tmp.entry_v1.len = it.bytes.size(); - // Manually set the header size to 28 bytes to match the pushed log events. - tmp.entry.hdr_size = kLogMsgHeaderSize; - tmp.entry_v1.sec = timestampSec; - // And set the received bytes starting after the 28 bytes reserved for header. - std::copy(it.bytes.begin(), it.bytes.end(), tmp.buf + kLogMsgHeaderSize); - data->push_back(make_shared<LogEvent>(tmp)); + data->push_back(make_shared<LogEvent>(it)); } VLOG("StatsCompanionServicePuller::pull succeeded for %d", mTagId); return true; diff --git a/cmds/statsd/src/external/StatsPuller.cpp b/cmds/statsd/src/external/StatsPuller.cpp index b29e979b5236..e3f251a5263d 100644 --- a/cmds/statsd/src/external/StatsPuller.cpp +++ b/cmds/statsd/src/external/StatsPuller.cpp @@ -18,10 +18,10 @@ #include "Log.h" #include "StatsPuller.h" +#include "StatsPullerManager.h" #include "guardrail/StatsdStats.h" #include "puller_util.h" #include "stats_log_util.h" -#include "StatsPullerManagerImpl.h" namespace android { namespace os { @@ -35,7 +35,7 @@ void StatsPuller::SetUidMap(const sp<UidMap>& uidMap) { mUidMap = uidMap; } // ValueMetric has a minimum bucket size of 10min so that we don't pull too frequently StatsPuller::StatsPuller(const int tagId) : mTagId(tagId) { - mCoolDownNs = StatsPullerManagerImpl::kAllPullAtomInfo.find(tagId)->second.coolDownNs; + mCoolDownNs = StatsPullerManager::kAllPullAtomInfo.find(tagId)->second.coolDownNs; VLOG("Puller for tag %d created. Cooldown set to %lld", mTagId, (long long)mCoolDownNs); } @@ -46,6 +46,7 @@ bool StatsPuller::Pull(const int64_t elapsedTimeNs, std::vector<std::shared_ptr< if (elapsedTimeNs - mLastPullTimeNs < mCoolDownNs) { (*data) = mCachedData; StatsdStats::getInstance().notePullFromCache(mTagId); + StatsdStats::getInstance().notePullDelay(mTagId, getElapsedRealtimeNs() - elapsedTimeNs); return true; } if (mMinPullIntervalNs > elapsedTimeNs - mLastPullTimeNs) { @@ -55,7 +56,9 @@ bool StatsPuller::Pull(const int64_t elapsedTimeNs, std::vector<std::shared_ptr< } mCachedData.clear(); mLastPullTimeNs = elapsedTimeNs; + int64_t pullStartTimeNs = getElapsedRealtimeNs(); bool ret = PullInternal(&mCachedData); + StatsdStats::getInstance().notePullTime(mTagId, getElapsedRealtimeNs() - pullStartTimeNs); for (const shared_ptr<LogEvent>& data : mCachedData) { data->setElapsedTimestampNs(elapsedTimeNs); data->setLogdWallClockTimestampNs(wallClockTimeNs); @@ -64,6 +67,7 @@ bool StatsPuller::Pull(const int64_t elapsedTimeNs, std::vector<std::shared_ptr< mergeIsolatedUidsToHostUid(mCachedData, mUidMap, mTagId); (*data) = mCachedData; } + StatsdStats::getInstance().notePullDelay(mTagId, getElapsedRealtimeNs() - elapsedTimeNs); return ret; } diff --git a/cmds/statsd/src/external/StatsPuller.h b/cmds/statsd/src/external/StatsPuller.h index caac677ee215..22cb2f5c2175 100644 --- a/cmds/statsd/src/external/StatsPuller.h +++ b/cmds/statsd/src/external/StatsPuller.h @@ -37,6 +37,8 @@ public: virtual ~StatsPuller() {} + // Pulls the data. The returned data will have elapsedTimeNs set as timeNs + // and will have wallClockTimeNs set as current wall clock time. bool Pull(const int64_t timeNs, std::vector<std::shared_ptr<LogEvent>>* data); // Clear cache immediately diff --git a/cmds/statsd/src/external/StatsPullerManagerImpl.cpp b/cmds/statsd/src/external/StatsPullerManager.cpp index c020f9c12b87..9633980420ef 100644 --- a/cmds/statsd/src/external/StatsPullerManagerImpl.cpp +++ b/cmds/statsd/src/external/StatsPullerManager.cpp @@ -29,7 +29,7 @@ #include "ResourceHealthManagerPuller.h" #include "ResourceThermalManagerPuller.h" #include "StatsCompanionServicePuller.h" -#include "StatsPullerManagerImpl.h" +#include "StatsPullerManager.h" #include "SubsystemSleepStatePuller.h" #include "statslog.h" @@ -49,7 +49,7 @@ namespace statsd { // Values smaller than this may require to update the alarm. const int64_t NO_ALARM_UPDATE = INT64_MAX; -const std::map<int, PullAtomInfo> StatsPullerManagerImpl::kAllPullAtomInfo = { +const std::map<int, PullAtomInfo> StatsPullerManager::kAllPullAtomInfo = { // wifi_bytes_transfer {android::util::WIFI_BYTES_TRANSFER, {{2, 3, 4, 5}, @@ -149,9 +149,6 @@ const std::map<int, PullAtomInfo> StatsPullerManagerImpl::kAllPullAtomInfo = { // system_uptime {android::util::SYSTEM_UPTIME, {{}, {}, 1 * NS_PER_SEC, new StatsCompanionServicePuller(android::util::SYSTEM_UPTIME)}}, - // disk_space - {android::util::DISK_SPACE, - {{}, {}, 1 * NS_PER_SEC, new StatsCompanionServicePuller(android::util::DISK_SPACE)}}, // remaining_battery_capacity {android::util::REMAINING_BATTERY_CAPACITY, {{}, @@ -164,19 +161,90 @@ const std::map<int, PullAtomInfo> StatsPullerManagerImpl::kAllPullAtomInfo = { {}, 1 * NS_PER_SEC, new ResourceHealthManagerPuller(android::util::FULL_BATTERY_CAPACITY)}}, + // battery_voltage + {android::util::BATTERY_VOLTAGE, + {{}, {}, 1 * NS_PER_SEC, new ResourceHealthManagerPuller(android::util::BATTERY_VOLTAGE)}}, // process_memory_state {android::util::PROCESS_MEMORY_STATE, - {{4, 5, 6, 7, 8}, - {2, 3}, + {{4, 5, 6, 7, 8, 9}, + {2, 3, 10}, 1 * NS_PER_SEC, new StatsCompanionServicePuller(android::util::PROCESS_MEMORY_STATE)}}, + // native_process_memory_state + {android::util::NATIVE_PROCESS_MEMORY_STATE, + {{3, 4, 5, 6}, + {2, 7}, + 1 * NS_PER_SEC, + new StatsCompanionServicePuller(android::util::NATIVE_PROCESS_MEMORY_STATE)}}, // temperature - {android::util::TEMPERATURE, {{}, {}, 1, new ResourceThermalManagerPuller()}}}; + {android::util::TEMPERATURE, {{}, {}, 1 * NS_PER_SEC, new ResourceThermalManagerPuller()}}, + // binder_calls + {android::util::BINDER_CALLS, + {{4, 5, 6, 8, 12}, + {2, 3, 7, 9, 10, 11, 13}, + 1 * NS_PER_SEC, + new StatsCompanionServicePuller(android::util::BINDER_CALLS)}}, + // binder_calls_exceptions + {android::util::BINDER_CALLS_EXCEPTIONS, + {{}, + {}, + 1 * NS_PER_SEC, + new StatsCompanionServicePuller(android::util::BINDER_CALLS_EXCEPTIONS)}}, + // looper_stats + {android::util::LOOPER_STATS, + {{5, 6, 7, 8, 9}, + {2, 3, 4, 10}, + 1 * NS_PER_SEC, + new StatsCompanionServicePuller(android::util::LOOPER_STATS)}}, + // Disk Stats + {android::util::DISK_STATS, + {{}, {}, 1 * NS_PER_SEC, new StatsCompanionServicePuller(android::util::DISK_STATS)}}, + // Directory usage + {android::util::DIRECTORY_USAGE, + {{}, {}, 1 * NS_PER_SEC, new StatsCompanionServicePuller(android::util::DIRECTORY_USAGE)}}, + // Size of app's code, data, and cache + {android::util::APP_SIZE, + {{}, {}, 1 * NS_PER_SEC, new StatsCompanionServicePuller(android::util::APP_SIZE)}}, + // Size of specific categories of files. Eg. Music. + {android::util::CATEGORY_SIZE, + {{}, {}, 1 * NS_PER_SEC, new StatsCompanionServicePuller(android::util::CATEGORY_SIZE)}}, + // Number of fingerprints registered to each user. + {android::util::NUM_FINGERPRINTS, + {{}, + {}, + 1 * NS_PER_SEC, + new StatsCompanionServicePuller(android::util::NUM_FINGERPRINTS)}}, + // ProcStats. + {android::util::PROC_STATS, + {{}, {}, 1 * NS_PER_SEC, new StatsCompanionServicePuller(android::util::PROC_STATS)}}, + // ProcStatsPkgProc. + {android::util::PROC_STATS_PKG_PROC, + {{}, {}, 1 * NS_PER_SEC, new StatsCompanionServicePuller(android::util::PROC_STATS_PKG_PROC)}}, + // Disk I/O stats per uid. + {android::util::DISK_IO, + {{2,3,4,5,6,7,8,9,10,11}, + {}, + 3 * NS_PER_SEC, + new StatsCompanionServicePuller(android::util::DISK_IO)}}, + // PowerProfile constants for power model calculations. + {android::util::POWER_PROFILE, + {{}, {}, 1 * NS_PER_SEC, new StatsCompanionServicePuller(android::util::POWER_PROFILE)}}, + // Process cpu stats. Min cool-down is 5 sec, inline with what AcitivityManagerService uses. + {android::util::PROCESS_CPU_TIME, + {{} /* additive fields */, {} /* non additive fields */, + 5 * NS_PER_SEC /* min cool-down in seconds*/, + new StatsCompanionServicePuller(android::util::PROCESS_CPU_TIME)}}, + {android::util::CPU_TIME_PER_THREAD_FREQ, + {{7}, + {2, 3, 4, 5, 6}, + 1 * NS_PER_SEC, + new StatsCompanionServicePuller(android::util::CPU_TIME_PER_THREAD_FREQ)}}, +}; -StatsPullerManagerImpl::StatsPullerManagerImpl() : mNextPullTimeNs(NO_ALARM_UPDATE) { +StatsPullerManager::StatsPullerManager() : mNextPullTimeNs(NO_ALARM_UPDATE) { } -bool StatsPullerManagerImpl::Pull(const int tagId, const int64_t timeNs, +bool StatsPullerManager::Pull(const int tagId, const int64_t timeNs, vector<shared_ptr<LogEvent>>* data) { VLOG("Initiating pulling %d", tagId); @@ -190,16 +258,11 @@ bool StatsPullerManagerImpl::Pull(const int tagId, const int64_t timeNs, } } -StatsPullerManagerImpl& StatsPullerManagerImpl::GetInstance() { - static StatsPullerManagerImpl instance; - return instance; -} - -bool StatsPullerManagerImpl::PullerForMatcherExists(int tagId) const { +bool StatsPullerManager::PullerForMatcherExists(int tagId) const { return kAllPullAtomInfo.find(tagId) != kAllPullAtomInfo.end(); } -void StatsPullerManagerImpl::updateAlarmLocked() { +void StatsPullerManager::updateAlarmLocked() { if (mNextPullTimeNs == NO_ALARM_UPDATE) { VLOG("No need to set alarms. Skipping"); return; @@ -214,7 +277,7 @@ void StatsPullerManagerImpl::updateAlarmLocked() { return; } -void StatsPullerManagerImpl::SetStatsCompanionService( +void StatsPullerManager::SetStatsCompanionService( sp<IStatsCompanionService> statsCompanionService) { AutoMutex _l(mLock); sp<IStatsCompanionService> tmpForLock = mStatsCompanionService; @@ -227,7 +290,7 @@ void StatsPullerManagerImpl::SetStatsCompanionService( } } -void StatsPullerManagerImpl::RegisterReceiver(int tagId, wp<PullDataReceiver> receiver, +void StatsPullerManager::RegisterReceiver(int tagId, wp<PullDataReceiver> receiver, int64_t nextPullTimeNs, int64_t intervalNs) { AutoMutex _l(mLock); auto& receivers = mReceivers[tagId]; @@ -262,7 +325,7 @@ void StatsPullerManagerImpl::RegisterReceiver(int tagId, wp<PullDataReceiver> re VLOG("Puller for tagId %d registered of %d", tagId, (int)receivers.size()); } -void StatsPullerManagerImpl::UnRegisterReceiver(int tagId, wp<PullDataReceiver> receiver) { +void StatsPullerManager::UnRegisterReceiver(int tagId, wp<PullDataReceiver> receiver) { AutoMutex _l(mLock); if (mReceivers.find(tagId) == mReceivers.end()) { VLOG("Unknown pull code or no receivers: %d", tagId); @@ -278,7 +341,7 @@ void StatsPullerManagerImpl::UnRegisterReceiver(int tagId, wp<PullDataReceiver> } } -void StatsPullerManagerImpl::OnAlarmFired(const int64_t currentTimeNs) { +void StatsPullerManager::OnAlarmFired(const int64_t currentTimeNs) { AutoMutex _l(mLock); int64_t minNextPullTimeNs = NO_ALARM_UPDATE; @@ -331,7 +394,7 @@ void StatsPullerManagerImpl::OnAlarmFired(const int64_t currentTimeNs) { updateAlarmLocked(); } -int StatsPullerManagerImpl::ForceClearPullerCache() { +int StatsPullerManager::ForceClearPullerCache() { int totalCleared = 0; for (const auto& pulledAtom : kAllPullAtomInfo) { totalCleared += pulledAtom.second.puller->ForceClearCache(); @@ -339,7 +402,7 @@ int StatsPullerManagerImpl::ForceClearPullerCache() { return totalCleared; } -int StatsPullerManagerImpl::ClearPullerCacheIfNecessary(int64_t timestampNs) { +int StatsPullerManager::ClearPullerCacheIfNecessary(int64_t timestampNs) { int totalCleared = 0; for (const auto& pulledAtom : kAllPullAtomInfo) { totalCleared += pulledAtom.second.puller->ClearCacheIfNecessary(timestampNs); diff --git a/cmds/statsd/src/external/StatsPullerManager.h b/cmds/statsd/src/external/StatsPullerManager.h index 50ffe17549c6..bbf5d9dc69db 100644 --- a/cmds/statsd/src/external/StatsPullerManager.h +++ b/cmds/statsd/src/external/StatsPullerManager.h @@ -16,54 +16,95 @@ #pragma once -#include "StatsPullerManagerImpl.h" +#include <android/os/IStatsCompanionService.h> +#include <binder/IServiceManager.h> +#include <utils/RefBase.h> +#include <utils/threads.h> +#include <list> +#include <string> +#include <unordered_map> +#include <vector> +#include "PullDataReceiver.h" +#include "StatsPuller.h" +#include "logd/LogEvent.h" namespace android { namespace os { namespace statsd { -class StatsPullerManager { - public: - virtual ~StatsPullerManager() {} +typedef struct { + // The field numbers of the fields that need to be summed when merging + // isolated uid with host uid. + std::vector<int> additiveFields; + // The field numbers of the fields that can't be merged when merging + // data belong to isolated uid and host uid. + std::vector<int> nonAdditiveFields; + // How long should the puller wait before doing an actual pull again. Default + // 1 sec. Set this to 0 if this is handled elsewhere. + int64_t coolDownNs = 1 * NS_PER_SEC; + // The actual puller + sp<StatsPuller> puller; +} PullAtomInfo; +class StatsPullerManager : public virtual RefBase { +public: + StatsPullerManager(); + + virtual ~StatsPullerManager() { + } + + // Registers a receiver for tagId. It will be pulled on the nextPullTimeNs + // and then every intervalNs thereafter. virtual void RegisterReceiver(int tagId, wp<PullDataReceiver> receiver, int64_t nextPullTimeNs, - int64_t intervalNs) { - mPullerManager.RegisterReceiver(tagId, receiver, nextPullTimeNs, intervalNs); - }; + int64_t intervalNs); - virtual void UnRegisterReceiver(int tagId, wp <PullDataReceiver> receiver) { - mPullerManager.UnRegisterReceiver(tagId, receiver); - }; + // Stop listening on a tagId. + virtual void UnRegisterReceiver(int tagId, wp<PullDataReceiver> receiver); // Verify if we know how to pull for this matcher - bool PullerForMatcherExists(int tagId) { - return mPullerManager.PullerForMatcherExists(tagId); - } + bool PullerForMatcherExists(int tagId) const; - void OnAlarmFired(const int64_t currentTimeNs) { - mPullerManager.OnAlarmFired(currentTimeNs); - } + void OnAlarmFired(const int64_t timeNs); - virtual bool Pull(const int tagId, const int64_t timesNs, - vector<std::shared_ptr<LogEvent>>* data) { - return mPullerManager.Pull(tagId, timesNs, data); - } + // Use respective puller to pull the data. The returned data will have + // elapsedTimeNs set as timeNs and will have wallClockTimeNs set as current + // wall clock time. + virtual bool Pull(const int tagId, const int64_t timeNs, + vector<std::shared_ptr<LogEvent>>* data); - int ForceClearPullerCache() { - return mPullerManager.ForceClearPullerCache(); - } + // Clear pull data cache immediately. + int ForceClearPullerCache(); - void SetStatsCompanionService(sp<IStatsCompanionService> statsCompanionService) { - mPullerManager.SetStatsCompanionService(statsCompanionService); - } + // Clear pull data cache if it is beyond respective cool down time. + int ClearPullerCacheIfNecessary(int64_t timestampNs); - int ClearPullerCacheIfNecessary(int64_t timestampNs) { - return mPullerManager.ClearPullerCacheIfNecessary(timestampNs); - } + void SetStatsCompanionService(sp<IStatsCompanionService> statsCompanionService); + + const static std::map<int, PullAtomInfo> kAllPullAtomInfo; + +private: + sp<IStatsCompanionService> mStatsCompanionService = nullptr; + + typedef struct { + int64_t nextPullTimeNs; + int64_t intervalNs; + wp<PullDataReceiver> receiver; + } ReceiverInfo; + + // mapping from simple matcher tagId to receivers + std::map<int, std::list<ReceiverInfo>> mReceivers; + + // locks for data receiver and StatsCompanionService changes + Mutex mLock; + + void updateAlarmLocked(); + + int64_t mNextPullTimeNs; - private: - StatsPullerManagerImpl - & mPullerManager = StatsPullerManagerImpl::GetInstance(); + FRIEND_TEST(GaugeMetricE2eTest, TestRandomSamplePulledEvents); + FRIEND_TEST(GaugeMetricE2eTest, TestRandomSamplePulledEvent_LateAlarm); + FRIEND_TEST(ValueMetricE2eTest, TestPulledEvents); + FRIEND_TEST(ValueMetricE2eTest, TestPulledEvents_LateAlarm); }; } // namespace statsd diff --git a/cmds/statsd/src/external/StatsPullerManagerImpl.h b/cmds/statsd/src/external/StatsPullerManagerImpl.h deleted file mode 100644 index 56d04b41c5d5..000000000000 --- a/cmds/statsd/src/external/StatsPullerManagerImpl.h +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright (C) 2017 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include <android/os/IStatsCompanionService.h> -#include <binder/IServiceManager.h> -#include <utils/RefBase.h> -#include <utils/threads.h> -#include <string> -#include <unordered_map> -#include <vector> -#include <list> -#include "PullDataReceiver.h" -#include "StatsPuller.h" -#include "logd/LogEvent.h" - -namespace android { -namespace os { -namespace statsd { - -typedef struct { - // The field numbers of the fields that need to be summed when merging - // isolated uid with host uid. - std::vector<int> additiveFields; - // The field numbers of the fields that can't be merged when merging - // data belong to isolated uid and host uid. - std::vector<int> nonAdditiveFields; - // How long should the puller wait before doing an actual pull again. Default - // 1 sec. Set this to 0 if this is handled elsewhere. - int64_t coolDownNs = 1 * NS_PER_SEC; - // The actual puller - sp<StatsPuller> puller; -} PullAtomInfo; - -class StatsPullerManagerImpl : public virtual RefBase { -public: - static StatsPullerManagerImpl& GetInstance(); - - void RegisterReceiver(int tagId, wp<PullDataReceiver> receiver, int64_t nextPullTimeNs, - int64_t intervalNs); - - void UnRegisterReceiver(int tagId, wp<PullDataReceiver> receiver); - - // Verify if we know how to pull for this matcher - bool PullerForMatcherExists(int tagId) const; - - void OnAlarmFired(const int64_t timeNs); - - bool Pull(const int tagId, const int64_t timeNs, vector<std::shared_ptr<LogEvent>>* data); - - int ForceClearPullerCache(); - - int ClearPullerCacheIfNecessary(int64_t timestampNs); - - void SetStatsCompanionService(sp<IStatsCompanionService> statsCompanionService); - - const static std::map<int, PullAtomInfo> kAllPullAtomInfo; - - private: - StatsPullerManagerImpl(); - - sp<IStatsCompanionService> mStatsCompanionService = nullptr; - - typedef struct { - int64_t nextPullTimeNs; - int64_t intervalNs; - wp<PullDataReceiver> receiver; - } ReceiverInfo; - - // mapping from simple matcher tagId to receivers - std::map<int, std::list<ReceiverInfo>> mReceivers; - - // locks for data receiver and StatsCompanionService changes - Mutex mLock; - - void updateAlarmLocked(); - - int64_t mNextPullTimeNs; - - FRIEND_TEST(GaugeMetricE2eTest, TestRandomSamplePulledEvents); - FRIEND_TEST(GaugeMetricE2eTest, TestRandomSamplePulledEvent_LateAlarm); - FRIEND_TEST(ValueMetricE2eTest, TestPulledEvents); - FRIEND_TEST(ValueMetricE2eTest, TestPulledEvents_LateAlarm); -}; - -} // namespace statsd -} // namespace os -} // namespace android diff --git a/cmds/statsd/src/external/puller_util.cpp b/cmds/statsd/src/external/puller_util.cpp index 57fe10e51bfc..ea7fa972cb9c 100644 --- a/cmds/statsd/src/external/puller_util.cpp +++ b/cmds/statsd/src/external/puller_util.cpp @@ -17,7 +17,7 @@ #define DEBUG false // STOPSHIP if true #include "Log.h" -#include "StatsPullerManagerImpl.h" +#include "StatsPullerManager.h" #include "puller_util.h" #include "statslog.h" @@ -107,8 +107,8 @@ bool tryMerge(vector<shared_ptr<LogEvent>>& data, int child_pos, const vector<in */ void mergeIsolatedUidsToHostUid(vector<shared_ptr<LogEvent>>& data, const sp<UidMap>& uidMap, int tagId) { - if (StatsPullerManagerImpl::kAllPullAtomInfo.find(tagId) == - StatsPullerManagerImpl::kAllPullAtomInfo.end()) { + if (StatsPullerManager::kAllPullAtomInfo.find(tagId) == + StatsPullerManager::kAllPullAtomInfo.end()) { VLOG("Unknown pull atom id %d", tagId); return; } @@ -121,9 +121,9 @@ void mergeIsolatedUidsToHostUid(vector<shared_ptr<LogEvent>>& data, const sp<Uid uidField = it->second; // uidField is the field number in proto, } const vector<int>& additiveFields = - StatsPullerManagerImpl::kAllPullAtomInfo.find(tagId)->second.additiveFields; + StatsPullerManager::kAllPullAtomInfo.find(tagId)->second.additiveFields; const vector<int>& nonAdditiveFields = - StatsPullerManagerImpl::kAllPullAtomInfo.find(tagId)->second.nonAdditiveFields; + StatsPullerManager::kAllPullAtomInfo.find(tagId)->second.nonAdditiveFields; // map of host uid to their position in the original vector map<int, vector<int>> hostPosition; diff --git a/cmds/statsd/src/guardrail/StatsdStats.cpp b/cmds/statsd/src/guardrail/StatsdStats.cpp index 764366fc420a..661768914c69 100644 --- a/cmds/statsd/src/guardrail/StatsdStats.cpp +++ b/cmds/statsd/src/guardrail/StatsdStats.cpp @@ -47,11 +47,9 @@ const int FIELD_ID_CONFIG_STATS = 3; const int FIELD_ID_ATOM_STATS = 7; const int FIELD_ID_UIDMAP_STATS = 8; const int FIELD_ID_ANOMALY_ALARM_STATS = 9; -// const int FIELD_ID_PULLED_ATOM_STATS = 10; // The proto is written in stats_log_util.cpp -const int FIELD_ID_LOGGER_ERROR_STATS = 11; const int FIELD_ID_PERIODIC_ALARM_STATS = 12; -const int FIELD_ID_LOG_LOSS_STATS = 14; const int FIELD_ID_SYSTEM_SERVER_RESTART = 15; +const int FIELD_ID_LOGGER_ERROR_STATS = 16; const int FIELD_ID_ATOM_STATS_TAG = 1; const int FIELD_ID_ATOM_STATS_COUNT = 2; @@ -59,8 +57,9 @@ const int FIELD_ID_ATOM_STATS_COUNT = 2; const int FIELD_ID_ANOMALY_ALARMS_REGISTERED = 1; const int FIELD_ID_PERIODIC_ALARMS_REGISTERED = 1; -const int FIELD_ID_LOGGER_STATS_TIME = 1; -const int FIELD_ID_LOGGER_STATS_ERROR_CODE = 2; +const int FIELD_ID_LOG_LOSS_STATS_TIME = 1; +const int FIELD_ID_LOG_LOSS_STATS_COUNT = 2; +const int FIELD_ID_LOG_LOSS_STATS_ERROR = 3; const int FIELD_ID_CONFIG_STATS_UID = 1; const int FIELD_ID_CONFIG_STATS_ID = 2; @@ -73,7 +72,8 @@ const int FIELD_ID_CONFIG_STATS_MATCHER_COUNT = 7; const int FIELD_ID_CONFIG_STATS_ALERT_COUNT = 8; const int FIELD_ID_CONFIG_STATS_VALID = 9; const int FIELD_ID_CONFIG_STATS_BROADCAST = 10; -const int FIELD_ID_CONFIG_STATS_DATA_DROP = 11; +const int FIELD_ID_CONFIG_STATS_DATA_DROP_TIME = 11; +const int FIELD_ID_CONFIG_STATS_DATA_DROP_BYTES = 21; const int FIELD_ID_CONFIG_STATS_DUMP_REPORT_TIME = 12; const int FIELD_ID_CONFIG_STATS_DUMP_REPORT_BYTES = 20; const int FIELD_ID_CONFIG_STATS_MATCHER_STATS = 13; @@ -100,10 +100,10 @@ const int FIELD_ID_UID_MAP_DROPPED_CHANGES = 3; const int FIELD_ID_UID_MAP_DELETED_APPS = 4; const std::map<int, std::pair<size_t, size_t>> StatsdStats::kAtomDimensionKeySizeLimitMap = { + {android::util::BINDER_CALLS, {6000, 10000}}, {android::util::CPU_TIME_PER_UID_FREQ, {6000, 10000}}, }; -// TODO: add stats for pulled atoms. StatsdStats::StatsdStats() { mPushedAtomStats.resize(android::util::kMaxPushedAtomId + 1); mStartTimeSec = getWallClockSec(); @@ -180,12 +180,12 @@ void StatsdStats::noteConfigReset(const ConfigKey& key) { noteConfigResetInternalLocked(key); } -void StatsdStats::noteLogLost(int64_t timestampNs) { +void StatsdStats::noteLogLost(int32_t wallClockTimeSec, int32_t count, int32_t lastError) { lock_guard<std::mutex> lock(mLock); - if (mLogLossTimestampNs.size() == kMaxLoggerErrors) { - mLogLossTimestampNs.pop_front(); + if (mLogLossStats.size() == kMaxLoggerErrors) { + mLogLossStats.pop_front(); } - mLogLossTimestampNs.push_back(timestampNs); + mLogLossStats.emplace_back(wallClockTimeSec, count, lastError); } void StatsdStats::noteBroadcastSent(const ConfigKey& key) { @@ -205,11 +205,11 @@ void StatsdStats::noteBroadcastSent(const ConfigKey& key, int32_t timeSec) { it->second->broadcast_sent_time_sec.push_back(timeSec); } -void StatsdStats::noteDataDropped(const ConfigKey& key) { - noteDataDropped(key, getWallClockSec()); +void StatsdStats::noteDataDropped(const ConfigKey& key, const size_t totalBytes) { + noteDataDropped(key, totalBytes, getWallClockSec()); } -void StatsdStats::noteDataDropped(const ConfigKey& key, int32_t timeSec) { +void StatsdStats::noteDataDropped(const ConfigKey& key, const size_t totalBytes, int32_t timeSec) { lock_guard<std::mutex> lock(mLock); auto it = mConfigStats.find(key); if (it == mConfigStats.end()) { @@ -218,8 +218,10 @@ void StatsdStats::noteDataDropped(const ConfigKey& key, int32_t timeSec) { } if (it->second->data_drop_time_sec.size() == kMaxTimestampCount) { it->second->data_drop_time_sec.pop_front(); + it->second->data_drop_bytes.pop_front(); } it->second->data_drop_time_sec.push_back(timeSec); + it->second->data_drop_bytes.push_back(totalBytes); } void StatsdStats::noteMetricsReportSent(const ConfigKey& key, const size_t num_bytes) { @@ -332,7 +334,8 @@ void StatsdStats::noteRegisteredPeriodicAlarmChanged() { void StatsdStats::updateMinPullIntervalSec(int pullAtomId, long intervalSec) { lock_guard<std::mutex> lock(mLock); - mPulledAtomStats[pullAtomId].minPullIntervalSec = intervalSec; + mPulledAtomStats[pullAtomId].minPullIntervalSec = + std::min(mPulledAtomStats[pullAtomId].minPullIntervalSec, intervalSec); } void StatsdStats::notePull(int pullAtomId) { @@ -345,6 +348,30 @@ void StatsdStats::notePullFromCache(int pullAtomId) { mPulledAtomStats[pullAtomId].totalPullFromCache++; } +void StatsdStats::notePullTime(int pullAtomId, int64_t pullTimeNs) { + lock_guard<std::mutex> lock(mLock); + auto& pullStats = mPulledAtomStats[pullAtomId]; + pullStats.maxPullTimeNs = std::max(pullStats.maxPullTimeNs, pullTimeNs); + pullStats.avgPullTimeNs = (pullStats.avgPullTimeNs * pullStats.numPullTime + pullTimeNs) / + (pullStats.numPullTime + 1); + pullStats.numPullTime += 1; +} + +void StatsdStats::notePullDelay(int pullAtomId, int64_t pullDelayNs) { + lock_guard<std::mutex> lock(mLock); + auto& pullStats = mPulledAtomStats[pullAtomId]; + pullStats.maxPullDelayNs = std::max(pullStats.maxPullDelayNs, pullDelayNs); + pullStats.avgPullDelayNs = + (pullStats.avgPullDelayNs * pullStats.numPullDelay + pullDelayNs) / + (pullStats.numPullDelay + 1); + pullStats.numPullDelay += 1; +} + +void StatsdStats::notePullDataError(int pullAtomId) { + lock_guard<std::mutex> lock(mLock); + mPulledAtomStats[pullAtomId].dataError++; +} + void StatsdStats::noteAtomLogged(int atomId, int32_t timeSec) { lock_guard<std::mutex> lock(mLock); @@ -365,15 +392,6 @@ void StatsdStats::noteSystemServerRestart(int32_t timeSec) { mSystemServerRestartSec.push_back(timeSec); } -void StatsdStats::noteLoggerError(int error) { - lock_guard<std::mutex> lock(mLock); - // grows strictly one at a time. so it won't > kMaxLoggerErrors - if (mLoggerErrors.size() == kMaxLoggerErrors) { - mLoggerErrors.pop_front(); - } - mLoggerErrors.push_back(std::make_pair(getWallClockSec(), error)); -} - void StatsdStats::reset() { lock_guard<std::mutex> lock(mLock); resetInternalLocked(); @@ -386,12 +404,12 @@ void StatsdStats::resetInternalLocked() { std::fill(mPushedAtomStats.begin(), mPushedAtomStats.end(), 0); mAnomalyAlarmRegisteredStats = 0; mPeriodicAlarmRegisteredStats = 0; - mLoggerErrors.clear(); mSystemServerRestartSec.clear(); - mLogLossTimestampNs.clear(); + mLogLossStats.clear(); for (auto& config : mConfigStats) { config.second->broadcast_sent_time_sec.clear(); config.second->data_drop_time_sec.clear(); + config.second->data_drop_bytes.clear(); config.second->dump_report_stats.clear(); config.second->annotations.clear(); config.second->matcher_stats.clear(); @@ -400,6 +418,17 @@ void StatsdStats::resetInternalLocked() { config.second->metric_dimension_in_condition_stats.clear(); config.second->alert_stats.clear(); } + for (auto& pullStats : mPulledAtomStats) { + pullStats.second.totalPull = 0; + pullStats.second.totalPullFromCache = 0; + pullStats.second.avgPullTimeNs = 0; + pullStats.second.maxPullTimeNs = 0; + pullStats.second.numPullTime = 0; + pullStats.second.avgPullDelayNs = 0; + pullStats.second.maxPullDelayNs = 0; + pullStats.second.numPullDelay = 0; + pullStats.second.dataError = 0; + } } string buildTimeString(int64_t timeSec) { @@ -410,36 +439,39 @@ string buildTimeString(int64_t timeSec) { return string(timeBuffer); } -void StatsdStats::dumpStats(FILE* out) const { +void StatsdStats::dumpStats(int out) const { lock_guard<std::mutex> lock(mLock); time_t t = mStartTimeSec; struct tm* tm = localtime(&t); char timeBuffer[80]; strftime(timeBuffer, sizeof(timeBuffer), "%Y-%m-%d %I:%M%p\n", tm); - fprintf(out, "Stats collection start second: %s\n", timeBuffer); - fprintf(out, "%lu Config in icebox: \n", (unsigned long)mIceBox.size()); + dprintf(out, "Stats collection start second: %s\n", timeBuffer); + dprintf(out, "%lu Config in icebox: \n", (unsigned long)mIceBox.size()); for (const auto& configStats : mIceBox) { - fprintf(out, + dprintf(out, "Config {%d_%lld}: creation=%d, deletion=%d, reset=%d, #metric=%d, #condition=%d, " "#matcher=%d, #alert=%d, valid=%d\n", configStats->uid, (long long)configStats->id, configStats->creation_time_sec, configStats->deletion_time_sec, configStats->reset_time_sec, - configStats->metric_count, - configStats->condition_count, configStats->matcher_count, configStats->alert_count, - configStats->is_valid); + configStats->metric_count, configStats->condition_count, configStats->matcher_count, + configStats->alert_count, configStats->is_valid); for (const auto& broadcastTime : configStats->broadcast_sent_time_sec) { - fprintf(out, "\tbroadcast time: %d\n", broadcastTime); + dprintf(out, "\tbroadcast time: %d\n", broadcastTime); } - for (const auto& dataDropTime : configStats->data_drop_time_sec) { - fprintf(out, "\tdata drop time: %d\n", dataDropTime); + auto dropTimePtr = configStats->data_drop_time_sec.begin(); + auto dropBytesPtr = configStats->data_drop_bytes.begin(); + for (int i = 0; i < (int)configStats->data_drop_time_sec.size(); + i++, dropTimePtr++, dropBytesPtr++) { + dprintf(out, "\tdata drop time: %d with size %lld", *dropTimePtr, + (long long)*dropBytesPtr); } } - fprintf(out, "%lu Active Configs\n", (unsigned long)mConfigStats.size()); + dprintf(out, "%lu Active Configs\n", (unsigned long)mConfigStats.size()); for (auto& pair : mConfigStats) { auto& configStats = pair.second; - fprintf(out, + dprintf(out, "Config {%d-%lld}: creation=%d, deletion=%d, #metric=%d, #condition=%d, " "#matcher=%d, #alert=%d, valid=%d\n", configStats->uid, (long long)configStats->id, configStats->creation_time_sec, @@ -447,89 +479,92 @@ void StatsdStats::dumpStats(FILE* out) const { configStats->condition_count, configStats->matcher_count, configStats->alert_count, configStats->is_valid); for (const auto& annotation : configStats->annotations) { - fprintf(out, "\tannotation: %lld, %d\n", (long long)annotation.first, + dprintf(out, "\tannotation: %lld, %d\n", (long long)annotation.first, annotation.second); } for (const auto& broadcastTime : configStats->broadcast_sent_time_sec) { - fprintf(out, "\tbroadcast time: %s(%lld)\n", - buildTimeString(broadcastTime).c_str(), (long long)broadcastTime); + dprintf(out, "\tbroadcast time: %s(%lld)\n", buildTimeString(broadcastTime).c_str(), + (long long)broadcastTime); } - for (const auto& dataDropTime : configStats->data_drop_time_sec) { - fprintf(out, "\tdata drop time: %s(%lld)\n", - buildTimeString(dataDropTime).c_str(), (long long)dataDropTime); + auto dropTimePtr = configStats->data_drop_time_sec.begin(); + auto dropBytesPtr = configStats->data_drop_bytes.begin(); + for (int i = 0; i < (int)configStats->data_drop_time_sec.size(); + i++, dropTimePtr++, dropBytesPtr++) { + dprintf(out, "\tdata drop time: %s(%lld) with %lld bytes\n", + buildTimeString(*dropTimePtr).c_str(), (long long)*dropTimePtr, + (long long)*dropBytesPtr); } for (const auto& dump : configStats->dump_report_stats) { - fprintf(out, "\tdump report time: %s(%lld) bytes: %lld\n", + dprintf(out, "\tdump report time: %s(%lld) bytes: %lld\n", buildTimeString(dump.first).c_str(), (long long)dump.first, (long long)dump.second); } for (const auto& stats : pair.second->matcher_stats) { - fprintf(out, "matcher %lld matched %d times\n", (long long)stats.first, stats.second); + dprintf(out, "matcher %lld matched %d times\n", (long long)stats.first, stats.second); } for (const auto& stats : pair.second->condition_stats) { - fprintf(out, "condition %lld max output tuple size %d\n", (long long)stats.first, + dprintf(out, "condition %lld max output tuple size %d\n", (long long)stats.first, stats.second); } for (const auto& stats : pair.second->condition_stats) { - fprintf(out, "metrics %lld max output tuple size %d\n", (long long)stats.first, + dprintf(out, "metrics %lld max output tuple size %d\n", (long long)stats.first, stats.second); } for (const auto& stats : pair.second->alert_stats) { - fprintf(out, "alert %lld declared %d times\n", (long long)stats.first, stats.second); + dprintf(out, "alert %lld declared %d times\n", (long long)stats.first, stats.second); } } - fprintf(out, "********Disk Usage stats***********\n"); + dprintf(out, "********Disk Usage stats***********\n"); StorageManager::printStats(out); - fprintf(out, "********Pushed Atom stats***********\n"); + dprintf(out, "********Pushed Atom stats***********\n"); const size_t atomCounts = mPushedAtomStats.size(); for (size_t i = 2; i < atomCounts; i++) { if (mPushedAtomStats[i] > 0) { - fprintf(out, "Atom %lu->%d\n", (unsigned long)i, mPushedAtomStats[i]); + dprintf(out, "Atom %lu->%d\n", (unsigned long)i, mPushedAtomStats[i]); } } - fprintf(out, "********Pulled Atom stats***********\n"); + dprintf(out, "********Pulled Atom stats***********\n"); for (const auto& pair : mPulledAtomStats) { - fprintf(out, "Atom %d->%ld, %ld, %ld\n", (int)pair.first, (long)pair.second.totalPull, - (long)pair.second.totalPullFromCache, (long)pair.second.minPullIntervalSec); + dprintf(out, + "Atom %d->(total pull)%ld, (pull from cache)%ld, (min pull interval)%ld, (average " + "pull time nanos)%lld, (max pull time nanos)%lld, (average pull delay nanos)%lld, " + "(max pull delay nanos)%lld, (data error)%ld\n", + (int)pair.first, (long)pair.second.totalPull, (long)pair.second.totalPullFromCache, + (long)pair.second.minPullIntervalSec, (long long)pair.second.avgPullTimeNs, + (long long)pair.second.maxPullTimeNs, (long long)pair.second.avgPullDelayNs, + (long long)pair.second.maxPullDelayNs, pair.second.dataError); } if (mAnomalyAlarmRegisteredStats > 0) { - fprintf(out, "********AnomalyAlarmStats stats***********\n"); - fprintf(out, "Anomaly alarm registrations: %d\n", mAnomalyAlarmRegisteredStats); + dprintf(out, "********AnomalyAlarmStats stats***********\n"); + dprintf(out, "Anomaly alarm registrations: %d\n", mAnomalyAlarmRegisteredStats); } if (mPeriodicAlarmRegisteredStats > 0) { - fprintf(out, "********SubscriberAlarmStats stats***********\n"); - fprintf(out, "Subscriber alarm registrations: %d\n", mPeriodicAlarmRegisteredStats); + dprintf(out, "********SubscriberAlarmStats stats***********\n"); + dprintf(out, "Subscriber alarm registrations: %d\n", mPeriodicAlarmRegisteredStats); } - fprintf(out, "UID map stats: bytes=%d, changes=%d, deleted=%d, changes lost=%d\n", + dprintf(out, "UID map stats: bytes=%d, changes=%d, deleted=%d, changes lost=%d\n", mUidMapStats.bytes_used, mUidMapStats.changes, mUidMapStats.deleted_apps, mUidMapStats.dropped_changes); - for (const auto& error : mLoggerErrors) { - time_t error_time = error.first; - struct tm* error_tm = localtime(&error_time); - char buffer[80]; - strftime(buffer, sizeof(buffer), "%Y-%m-%d %I:%M%p\n", error_tm); - fprintf(out, "Logger error %d at %s\n", error.second, buffer); - } - for (const auto& restart : mSystemServerRestartSec) { - fprintf(out, "System server restarts at %s(%lld)\n", - buildTimeString(restart).c_str(), (long long)restart); + dprintf(out, "System server restarts at %s(%lld)\n", buildTimeString(restart).c_str(), + (long long)restart); } - for (const auto& loss : mLogLossTimestampNs) { - fprintf(out, "Log loss detected at %lld (elapsedRealtimeNs)\n", (long long)loss); + for (const auto& loss : mLogLossStats) { + dprintf(out, "Log loss: %lld (wall clock sec) - %d (count) %d (last error)\n", + (long long)loss.mWallClockSec, loss.mCount, loss.mLastError); } } @@ -558,9 +593,15 @@ void addConfigStatsToProto(const ConfigStats& configStats, ProtoOutputStream* pr broadcast); } - for (const auto& drop : configStats.data_drop_time_sec) { - proto->write(FIELD_TYPE_INT32 | FIELD_ID_CONFIG_STATS_DATA_DROP | FIELD_COUNT_REPEATED, - drop); + for (const auto& drop_time : configStats.data_drop_time_sec) { + proto->write(FIELD_TYPE_INT32 | FIELD_ID_CONFIG_STATS_DATA_DROP_TIME | FIELD_COUNT_REPEATED, + drop_time); + } + + for (const auto& drop_bytes : configStats.data_drop_bytes) { + proto->write( + FIELD_TYPE_INT64 | FIELD_ID_CONFIG_STATS_DATA_DROP_BYTES | FIELD_COUNT_REPEATED, + (long long)drop_bytes); } for (const auto& dump : configStats.dump_report_stats) { @@ -677,19 +718,15 @@ void StatsdStats::dumpStats(std::vector<uint8_t>* output, bool reset) { proto.write(FIELD_TYPE_INT32 | FIELD_ID_UID_MAP_DELETED_APPS, mUidMapStats.deleted_apps); proto.end(uidMapToken); - for (const auto& error : mLoggerErrors) { + for (const auto& error : mLogLossStats) { uint64_t token = proto.start(FIELD_TYPE_MESSAGE | FIELD_ID_LOGGER_ERROR_STATS | FIELD_COUNT_REPEATED); - proto.write(FIELD_TYPE_INT32 | FIELD_ID_LOGGER_STATS_TIME, error.first); - proto.write(FIELD_TYPE_INT32 | FIELD_ID_LOGGER_STATS_ERROR_CODE, error.second); + proto.write(FIELD_TYPE_INT32 | FIELD_ID_LOG_LOSS_STATS_TIME, error.mWallClockSec); + proto.write(FIELD_TYPE_INT32 | FIELD_ID_LOG_LOSS_STATS_COUNT, error.mCount); + proto.write(FIELD_TYPE_INT32 | FIELD_ID_LOG_LOSS_STATS_ERROR, error.mLastError); proto.end(token); } - for (const auto& loss : mLogLossTimestampNs) { - proto.write(FIELD_TYPE_INT64 | FIELD_ID_LOG_LOSS_STATS | FIELD_COUNT_REPEATED, - (long long)loss); - } - for (const auto& restart : mSystemServerRestartSec) { proto.write(FIELD_TYPE_INT32 | FIELD_ID_SYSTEM_SERVER_RESTART | FIELD_COUNT_REPEATED, restart); diff --git a/cmds/statsd/src/guardrail/StatsdStats.h b/cmds/statsd/src/guardrail/StatsdStats.h index 74541d37b840..343709a2a1eb 100644 --- a/cmds/statsd/src/guardrail/StatsdStats.h +++ b/cmds/statsd/src/guardrail/StatsdStats.h @@ -43,6 +43,8 @@ struct ConfigStats { std::list<int32_t> broadcast_sent_time_sec; std::list<int32_t> data_drop_time_sec; + // Number of bytes dropped at corresponding time. + std::list<int64_t> data_drop_bytes; std::list<std::pair<int32_t, int64_t>> dump_report_stats; // Stores how many times a matcher have been matched. The map size is capped by kMaxConfigCount. @@ -86,7 +88,6 @@ public: static StatsdStats& getInstance(); ~StatsdStats(){}; - // TODO: set different limit if the device is low ram. const static int kDimensionKeySizeSoftLimit = 500; const static int kDimensionKeySizeHardLimit = 800; @@ -157,7 +158,7 @@ public: * Report a config has been removed. */ void noteConfigRemoved(const ConfigKey& key); - /** + /** * Report a config has been reset when ttl expires. */ void noteConfigReset(const ConfigKey& key); @@ -170,7 +171,7 @@ public: /** * Report a config's metrics data has been dropped. */ - void noteDataDropped(const ConfigKey& key); + void noteDataDropped(const ConfigKey& key, const size_t totalBytes); /** * Report metrics data report has been sent. @@ -203,7 +204,6 @@ public: */ void noteMetricDimensionSize(const ConfigKey& key, const int64_t& id, int size); - /** * Report the max size of output tuple of dimension in condition across dimensions in what. * @@ -263,29 +263,47 @@ public: void setUidMapChanges(int changes); void setCurrentUidMapMemory(int bytes); - // Update minimum interval between pulls for an pulled atom + /* + * Updates minimum interval between pulls for an pulled atom. + */ void updateMinPullIntervalSec(int pullAtomId, long intervalSec); - // Notify pull request for an atom + /* + * Notes an atom is pulled. + */ void notePull(int pullAtomId); - // Notify pull request for an atom served from cached data + /* + * Notes an atom is served from puller cache. + */ void notePullFromCache(int pullAtomId); - /** - * Records statsd met an error while reading from logd. + /* + * Notify data error for pulled atom. + */ + void notePullDataError(int pullAtomId); + + /* + * Records time for actual pulling, not including those served from cache and not including + * statsd processing delays. */ - void noteLoggerError(int error); + void notePullTime(int pullAtomId, int64_t pullTimeNs); /* - * Records when system server restarts. - */ + * Records pull delay for a pulled atom, including those served from cache and including statsd + * processing delays. + */ + void notePullDelay(int pullAtomId, int64_t pullDelayNs); + + /* + * Records when system server restarts. + */ void noteSystemServerRestart(int32_t timeSec); /** * Records statsd skipped an event. */ - void noteLogLost(int64_t timestamp); + void noteLogLost(int32_t wallClockTimeSec, int32_t count, int lastError); /** * Reset the historical stats. Including all stats in icebox, and the tracked stats about @@ -302,14 +320,21 @@ public: void dumpStats(std::vector<uint8_t>* buffer, bool reset); /** - * Output statsd stats in human readable format to [out] file. + * Output statsd stats in human readable format to [out] file descriptor. */ - void dumpStats(FILE* out) const; + void dumpStats(int outFd) const; typedef struct { - long totalPull; - long totalPullFromCache; - long minPullIntervalSec; + long totalPull = 0; + long totalPullFromCache = 0; + long minPullIntervalSec = LONG_MAX; + int64_t avgPullTimeNs = 0; + int64_t maxPullTimeNs = 0; + long numPullTime = 0; + int64_t avgPullDelayNs = 0; + int64_t maxPullDelayNs = 0; + long numPullDelay = 0; + long dataError = 0; } PulledAtomStats; private: @@ -339,11 +364,18 @@ private: // Maps PullAtomId to its stats. The size is capped by the puller atom counts. std::map<int, PulledAtomStats> mPulledAtomStats; - // Logd errors. Size capped by kMaxLoggerErrors. - std::list<const std::pair<int, int>> mLoggerErrors; + struct LogLossStats { + LogLossStats(int32_t sec, int32_t count, int32_t error) + : mWallClockSec(sec), mCount(count), mLastError(error) { + } + int32_t mWallClockSec; + int32_t mCount; + // error code defined in linux/errno.h + int32_t mLastError; + }; - // Timestamps when we detect log loss after logd reconnect. - std::list<int64_t> mLogLossTimestampNs; + // Timestamps when we detect log loss, and the number of logs lost. + std::list<LogLossStats> mLogLossStats; std::list<int32_t> mSystemServerRestartSec; @@ -360,7 +392,7 @@ private: void resetInternalLocked(); - void noteDataDropped(const ConfigKey& key, int32_t timeSec); + void noteDataDropped(const ConfigKey& key, const size_t totalBytes, int32_t timeSec); void noteMetricsReportSent(const ConfigKey& key, const size_t num_bytes, int32_t timeSec); @@ -376,6 +408,7 @@ private: FRIEND_TEST(StatsdStatsTest, TestTimestampThreshold); FRIEND_TEST(StatsdStatsTest, TestAnomalyMonitor); FRIEND_TEST(StatsdStatsTest, TestSystemServerCrash); + FRIEND_TEST(StatsdStatsTest, TestPullAtomStats); }; } // namespace statsd diff --git a/cmds/statsd/src/logd/LogEvent.cpp b/cmds/statsd/src/logd/LogEvent.cpp index 4e4f146d27ac..febb9229bc95 100644 --- a/cmds/statsd/src/logd/LogEvent.cpp +++ b/cmds/statsd/src/logd/LogEvent.cpp @@ -18,6 +18,7 @@ #include "logd/LogEvent.h" #include "stats_log_util.h" +#include "statslog.h" namespace android { namespace os { @@ -40,6 +41,44 @@ LogEvent::LogEvent(log_msg& msg) { } } +LogEvent::LogEvent(const StatsLogEventWrapper& statsLogEventWrapper) { + mTagId = statsLogEventWrapper.getTagId(); + mLogdTimestampNs = statsLogEventWrapper.getWallClockTimeNs(); + mElapsedTimestampNs = statsLogEventWrapper.getElapsedRealTimeNs(); + mLogUid = 0; + for (int i = 0; i < (int)statsLogEventWrapper.getElements().size(); i++) { + Field field(statsLogEventWrapper.getTagId(), getSimpleField(i + 1)); + switch (statsLogEventWrapper.getElements()[i].type) { + case android::os::StatsLogValue::STATS_LOG_VALUE_TYPE::INT: + mValues.push_back( + FieldValue(field, Value(statsLogEventWrapper.getElements()[i].int_value))); + break; + case android::os::StatsLogValue::STATS_LOG_VALUE_TYPE::LONG: + mValues.push_back( + FieldValue(field, Value(statsLogEventWrapper.getElements()[i].long_value))); + break; + case android::os::StatsLogValue::STATS_LOG_VALUE_TYPE::FLOAT: + mValues.push_back(FieldValue( + field, Value(statsLogEventWrapper.getElements()[i].float_value))); + break; + case android::os::StatsLogValue::STATS_LOG_VALUE_TYPE::DOUBLE: + mValues.push_back(FieldValue( + field, Value(statsLogEventWrapper.getElements()[i].double_value))); + break; + case android::os::StatsLogValue::STATS_LOG_VALUE_TYPE::STRING: + mValues.push_back( + FieldValue(field, Value(statsLogEventWrapper.getElements()[i].str_value))); + break; + case android::os::StatsLogValue::STATS_LOG_VALUE_TYPE::STORAGE: + mValues.push_back(FieldValue( + field, Value(statsLogEventWrapper.getElements()[i].storage_value))); + break; + default: + break; + } + } +} + LogEvent::LogEvent(int32_t tagId, int64_t wallClockTimestampNs, int64_t elapsedTimestampNs) { mLogdTimestampNs = wallClockTimestampNs; mTagId = tagId; @@ -51,10 +90,189 @@ LogEvent::LogEvent(int32_t tagId, int64_t wallClockTimestampNs, int64_t elapsedT } } -LogEvent::LogEvent(int32_t tagId, int64_t timestampNs) { +LogEvent::LogEvent(int32_t tagId, int64_t wallClockTimestampNs, int64_t elapsedTimestampNs, + int32_t uid, + const std::map<int32_t, int32_t>& int_map, + const std::map<int32_t, int64_t>& long_map, + const std::map<int32_t, std::string>& string_map, + const std::map<int32_t, float>& float_map) { + mLogdTimestampNs = wallClockTimestampNs; + mElapsedTimestampNs = elapsedTimestampNs; + mTagId = android::util::KEY_VALUE_PAIRS_ATOM; + mLogUid = uid; + + int pos[] = {1, 1, 1}; + + mValues.push_back(FieldValue(Field(mTagId, pos, 0 /* depth */), Value(uid))); + pos[0]++; + for (const auto&itr : int_map) { + pos[2] = 1; + mValues.push_back(FieldValue(Field(mTagId, pos, 2 /* depth */), Value(itr.first))); + pos[2] = 2; + mValues.push_back(FieldValue(Field(mTagId, pos, 2 /* depth */), Value(itr.second))); + mValues.back().mField.decorateLastPos(2); + pos[1]++; + } + + for (const auto&itr : long_map) { + pos[2] = 1; + mValues.push_back(FieldValue(Field(mTagId, pos, 2 /* depth */), Value(itr.first))); + pos[2] = 3; + mValues.push_back(FieldValue(Field(mTagId, pos, 2 /* depth */), Value(itr.second))); + mValues.back().mField.decorateLastPos(2); + pos[1]++; + } + + for (const auto&itr : string_map) { + pos[2] = 1; + mValues.push_back(FieldValue(Field(mTagId, pos, 2 /* depth */), Value(itr.first))); + pos[2] = 4; + mValues.push_back(FieldValue(Field(mTagId, pos, 2 /* depth */), Value(itr.second))); + mValues.back().mField.decorateLastPos(2); + pos[1]++; + } + + for (const auto&itr : float_map) { + pos[2] = 1; + mValues.push_back(FieldValue(Field(mTagId, pos, 2 /* depth */), Value(itr.first))); + pos[2] = 5; + mValues.push_back(FieldValue(Field(mTagId, pos, 2 /* depth */), Value(itr.second))); + mValues.back().mField.decorateLastPos(2); + pos[1]++; + } + if (!mValues.empty()) { + mValues.back().mField.decorateLastPos(1); + mValues.at(mValues.size() - 2).mField.decorateLastPos(1); + } +} + +LogEvent::LogEvent(int64_t wallClockTimestampNs, int64_t elapsedTimestampNs, + const SpeakerImpedance& speakerImpedance) { + mLogdTimestampNs = wallClockTimestampNs; + mElapsedTimestampNs = elapsedTimestampNs; + mTagId = android::util::SPEAKER_IMPEDANCE_REPORTED; + + mValues.push_back( + FieldValue(Field(mTagId, getSimpleField(1)), Value(speakerImpedance.speakerLocation))); + mValues.push_back( + FieldValue(Field(mTagId, getSimpleField(2)), Value(speakerImpedance.milliOhms))); + if (!mValues.empty()) { + mValues.back().mField.decorateLastPos(1); + } +} + +LogEvent::LogEvent(int64_t wallClockTimestampNs, int64_t elapsedTimestampNs, + const HardwareFailed& hardwareFailed) { + mLogdTimestampNs = wallClockTimestampNs; + mElapsedTimestampNs = elapsedTimestampNs; + mTagId = android::util::HARDWARE_FAILED; + + mValues.push_back(FieldValue(Field(mTagId, getSimpleField(1)), + Value(int32_t(hardwareFailed.hardwareType)))); + mValues.push_back( + FieldValue(Field(mTagId, getSimpleField(2)), Value(hardwareFailed.hardwareLocation))); + mValues.push_back( + FieldValue(Field(mTagId, getSimpleField(3)), Value(int32_t(hardwareFailed.errorCode)))); + if (!mValues.empty()) { + mValues.back().mField.decorateLastPos(1); + } +} + +LogEvent::LogEvent(int64_t wallClockTimestampNs, int64_t elapsedTimestampNs, + const PhysicalDropDetected& physicalDropDetected) { + mLogdTimestampNs = wallClockTimestampNs; + mElapsedTimestampNs = elapsedTimestampNs; + mTagId = android::util::PHYSICAL_DROP_DETECTED; + + mValues.push_back(FieldValue(Field(mTagId, getSimpleField(1)), + Value(int32_t(physicalDropDetected.confidencePctg)))); + mValues.push_back( + FieldValue(Field(mTagId, getSimpleField(2)), Value(physicalDropDetected.accelPeak))); + mValues.push_back(FieldValue(Field(mTagId, getSimpleField(3)), + Value(physicalDropDetected.freefallDuration))); + if (!mValues.empty()) { + mValues.back().mField.decorateLastPos(1); + } +} + +LogEvent::LogEvent(int64_t wallClockTimestampNs, int64_t elapsedTimestampNs, + const ChargeCycles& chargeCycles) { + mLogdTimestampNs = wallClockTimestampNs; + mElapsedTimestampNs = elapsedTimestampNs; + mTagId = android::util::CHARGE_CYCLES_REPORTED; + + for (size_t i = 0; i < chargeCycles.cycleBucket.size(); i++) { + mValues.push_back(FieldValue(Field(mTagId, getSimpleField(i + 1)), + Value(chargeCycles.cycleBucket[i]))); + } + + if (!mValues.empty()) { + mValues.back().mField.decorateLastPos(1); + } +} + +LogEvent::LogEvent(int64_t wallClockTimestampNs, int64_t elapsedTimestampNs, + const BatteryHealthSnapshotArgs& batteryHealthSnapshotArgs) { + mLogdTimestampNs = wallClockTimestampNs; + mElapsedTimestampNs = elapsedTimestampNs; + mTagId = android::util::BATTERY_HEALTH_SNAPSHOT; + + mValues.push_back(FieldValue(Field(mTagId, getSimpleField(1)), + Value(int32_t(batteryHealthSnapshotArgs.type)))); + mValues.push_back(FieldValue(Field(mTagId, getSimpleField(2)), + Value(batteryHealthSnapshotArgs.temperatureDeciC))); + mValues.push_back(FieldValue(Field(mTagId, getSimpleField(3)), + Value(batteryHealthSnapshotArgs.voltageMicroV))); + mValues.push_back(FieldValue(Field(mTagId, getSimpleField(4)), + Value(batteryHealthSnapshotArgs.currentMicroA))); + mValues.push_back(FieldValue(Field(mTagId, getSimpleField(5)), + Value(batteryHealthSnapshotArgs.openCircuitVoltageMicroV))); + mValues.push_back(FieldValue(Field(mTagId, getSimpleField(6)), + Value(batteryHealthSnapshotArgs.resistanceMicroOhm))); + mValues.push_back(FieldValue(Field(mTagId, getSimpleField(7)), + Value(batteryHealthSnapshotArgs.levelPercent))); + + if (!mValues.empty()) { + mValues.back().mField.decorateLastPos(1); + } +} + +LogEvent::LogEvent(int64_t wallClockTimestampNs, int64_t elapsedTimestampNs, const SlowIo& slowIo) { + mLogdTimestampNs = wallClockTimestampNs; + mElapsedTimestampNs = elapsedTimestampNs; + mTagId = android::util::SLOW_IO; + + int pos[] = {1}; + mValues.push_back( + FieldValue(Field(mTagId, getSimpleField(1)), Value(int32_t(slowIo.operation)))); + pos[0]++; + mValues.push_back(FieldValue(Field(mTagId, getSimpleField(2)), Value(slowIo.count))); + + if (!mValues.empty()) { + mValues.back().mField.decorateLastPos(1); + } +} + +LogEvent::LogEvent(int64_t wallClockTimestampNs, int64_t elapsedTimestampNs, + const BatteryCausedShutdown& batteryCausedShutdown) { + mLogdTimestampNs = wallClockTimestampNs; + mElapsedTimestampNs = elapsedTimestampNs; + mTagId = android::util::BATTERY_CAUSED_SHUTDOWN; + + mValues.push_back(FieldValue(Field(mTagId, getSimpleField(1)), + Value(batteryCausedShutdown.voltageMicroV))); + + if (!mValues.empty()) { + mValues.back().mField.decorateLastPos(1); + } +} + +LogEvent::LogEvent(int32_t tagId, int64_t timestampNs) : LogEvent(tagId, timestampNs, 0) {} + +LogEvent::LogEvent(int32_t tagId, int64_t timestampNs, int32_t uid) { mLogdTimestampNs = timestampNs; mTagId = tagId; - mLogUid = 0; + mLogUid = uid; mContext = create_android_logger(1937006964); // the event tag shared by all stats logs if (mContext) { android_log_write_int64(mContext, timestampNs); @@ -128,6 +346,68 @@ bool LogEvent::write(float value) { return false; } +bool LogEvent::writeKeyValuePairs(int32_t uid, + const std::map<int32_t, int32_t>& int_map, + const std::map<int32_t, int64_t>& long_map, + const std::map<int32_t, std::string>& string_map, + const std::map<int32_t, float>& float_map) { + if (mContext) { + if (android_log_write_list_begin(mContext) < 0) { + return false; + } + write(uid); + for (const auto& itr : int_map) { + if (android_log_write_list_begin(mContext) < 0) { + return false; + } + write(itr.first); + write(itr.second); + if (android_log_write_list_end(mContext) < 0) { + return false; + } + } + + for (const auto& itr : long_map) { + if (android_log_write_list_begin(mContext) < 0) { + return false; + } + write(itr.first); + write(itr.second); + if (android_log_write_list_end(mContext) < 0) { + return false; + } + } + + for (const auto& itr : string_map) { + if (android_log_write_list_begin(mContext) < 0) { + return false; + } + write(itr.first); + write(itr.second.c_str()); + if (android_log_write_list_end(mContext) < 0) { + return false; + } + } + + for (const auto& itr : float_map) { + if (android_log_write_list_begin(mContext) < 0) { + return false; + } + write(itr.first); + write(itr.second); + if (android_log_write_list_end(mContext) < 0) { + return false; + } + } + + if (android_log_write_list_end(mContext) < 0) { + return false; + } + return true; + } + return false; +} + bool LogEvent::write(const std::vector<AttributionNodeInternal>& nodes) { if (mContext) { if (android_log_write_list_begin(mContext) < 0) { @@ -178,6 +458,7 @@ void LogEvent::init(android_log_context context) { int i = 0; int depth = -1; int pos[] = {1, 1, 1}; + bool isKeyValuePairAtom = false; do { elem = android_log_read_next(context); switch ((int)elem.type) { @@ -185,6 +466,7 @@ void LogEvent::init(android_log_context context) { // elem at [0] is EVENT_TYPE_LIST, [1] is the timestamp, [2] is tag id. if (i == 2) { mTagId = elem.data.int32; + isKeyValuePairAtom = (mTagId == android::util::KEY_VALUE_PAIRS_ATOM); } else { if (depth < 0 || depth > 2) { return; @@ -202,6 +484,11 @@ void LogEvent::init(android_log_context context) { return; } + // Handles the oneof field in KeyValuePair atom. + if (isKeyValuePairAtom && depth == 2) { + pos[depth] = 4; + } + mValues.push_back(FieldValue(Field(mTagId, pos, depth), Value(elem.data.float32))); pos[depth]++; @@ -213,6 +500,10 @@ void LogEvent::init(android_log_context context) { return; } + // Handles the oneof field in KeyValuePair atom. + if (isKeyValuePairAtom && depth == 2) { + pos[depth] = 3; + } mValues.push_back(FieldValue(Field(mTagId, pos, depth), Value(string(elem.data.string, elem.len)))); @@ -227,6 +518,10 @@ void LogEvent::init(android_log_context context) { ALOGE("Depth > 2. Not supported!"); return; } + // Handles the oneof field in KeyValuePair atom. + if (isKeyValuePairAtom && depth == 2) { + pos[depth] = 2; + } mValues.push_back( FieldValue(Field(mTagId, pos, depth), Value((int64_t)elem.data.int64))); @@ -270,10 +565,14 @@ void LogEvent::init(android_log_context context) { } i++; } while ((elem.type != EVENT_TYPE_UNKNOWN) && !elem.complete); + if (isKeyValuePairAtom && mValues.size() > 0) { + mValues[0] = FieldValue(Field(android::util::KEY_VALUE_PAIRS_ATOM, getSimpleField(1)), + Value((int32_t)mLogUid)); + } } int64_t LogEvent::GetLong(size_t key, status_t* err) const { - // TODO: encapsulate the magical operations all in Field struct as a static function. + // TODO(b/110561208): encapsulate the magical operations in Field struct as static functions int field = getSimpleField(key); for (const auto& value : mValues) { if (value.mField.getField() == field) { diff --git a/cmds/statsd/src/logd/LogEvent.h b/cmds/statsd/src/logd/LogEvent.h index 24d624d9d9be..3d5b2abc7e3a 100644 --- a/cmds/statsd/src/logd/LogEvent.h +++ b/cmds/statsd/src/logd/LogEvent.h @@ -18,6 +18,8 @@ #include "FieldValue.h" +#include <android/frameworks/stats/1.0/types.h> +#include <android/os/StatsLogEventWrapper.h> #include <android/util/ProtoOutputStream.h> #include <log/log_event_list.h> #include <log/log_read.h> @@ -27,6 +29,8 @@ #include <string> #include <vector> +using namespace android::frameworks::stats::V1_0; + namespace android { namespace os { namespace statsd { @@ -61,6 +65,8 @@ public: */ explicit LogEvent(log_msg& msg); + explicit LogEvent(const StatsLogEventWrapper& statsLogEventWrapper); + /** * Constructs a LogEvent with synthetic data for testing. Must call init() before reading. */ @@ -69,6 +75,40 @@ public: // For testing. The timestamp is used as both elapsed real time and logd timestamp. explicit LogEvent(int32_t tagId, int64_t timestampNs); + // For testing. The timestamp is used as both elapsed real time and logd timestamp. + explicit LogEvent(int32_t tagId, int64_t timestampNs, int32_t uid); + + /** + * Constructs a KeyValuePairsAtom LogEvent from value maps. + */ + explicit LogEvent(int32_t tagId, int64_t wallClockTimestampNs, int64_t elapsedTimestampNs, + int32_t uid, + const std::map<int32_t, int32_t>& int_map, + const std::map<int32_t, int64_t>& long_map, + const std::map<int32_t, std::string>& string_map, + const std::map<int32_t, float>& float_map); + + explicit LogEvent(int64_t wallClockTimestampNs, int64_t elapsedTimestampNs, + const SpeakerImpedance& speakerImpedance); + + explicit LogEvent(int64_t wallClockTimestampNs, int64_t elapsedTimestampNs, + const HardwareFailed& hardwareFailed); + + explicit LogEvent(int64_t wallClockTimestampNs, int64_t elapsedTimestampNs, + const PhysicalDropDetected& physicalDropDetected); + + explicit LogEvent(int64_t wallClockTimestampNs, int64_t elapsedTimestampNs, + const ChargeCycles& chargeCycles); + + explicit LogEvent(int64_t wallClockTimestampNs, int64_t elapsedTimestampNs, + const BatteryHealthSnapshotArgs& batteryHealthSnapshotArgs); + + explicit LogEvent(int64_t wallClockTimestampNs, int64_t elapsedTimestampNs, + const SlowIo& slowIo); + + explicit LogEvent(int64_t wallClockTimestampNs, int64_t elapsedTimestampNs, + const BatteryCausedShutdown& batteryCausedShutdown); + ~LogEvent(); /** @@ -110,6 +150,11 @@ public: bool write(float value); bool write(const std::vector<AttributionNodeInternal>& nodes); bool write(const AttributionNodeInternal& node); + bool writeKeyValuePairs(int32_t uid, + const std::map<int32_t, int32_t>& int_map, + const std::map<int32_t, int64_t>& long_map, + const std::map<int32_t, std::string>& string_map, + const std::map<int32_t, float>& float_map); /** * Return a string representation of this event. diff --git a/cmds/statsd/src/logd/LogListener.cpp b/cmds/statsd/src/logd/LogListener.cpp index 6ac7978bbac9..ddb26f9fe565 100644 --- a/cmds/statsd/src/logd/LogListener.cpp +++ b/cmds/statsd/src/logd/LogListener.cpp @@ -14,17 +14,7 @@ * limitations under the License. */ -#include "logd/LogReader.h" - -#include <log/log_read.h> - -#include <utils/Errors.h> - -#include <time.h> -#include <unistd.h> - -using namespace android; -using namespace std; +#include "logd/LogListener.h" namespace android { namespace os { diff --git a/cmds/statsd/src/logd/LogListener.h b/cmds/statsd/src/logd/LogListener.h index f924040e3a7f..d8b06e9fab92 100644 --- a/cmds/statsd/src/logd/LogListener.h +++ b/cmds/statsd/src/logd/LogListener.h @@ -19,7 +19,6 @@ #include "logd/LogEvent.h" #include <utils/RefBase.h> -#include <vector> namespace android { namespace os { @@ -33,7 +32,7 @@ public: LogListener(); virtual ~LogListener(); - virtual void OnLogEvent(LogEvent* msg, bool reconnectionStarts) = 0; + virtual void OnLogEvent(LogEvent* msg) = 0; }; } // namespace statsd diff --git a/cmds/statsd/src/logd/LogReader.cpp b/cmds/statsd/src/logd/LogReader.cpp deleted file mode 100644 index 26ae6a3e0e2e..000000000000 --- a/cmds/statsd/src/logd/LogReader.cpp +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright (C) 2017 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "logd/LogReader.h" - -#include "guardrail/StatsdStats.h" - -#include <time.h> -#include <unistd.h> -#include <utils/Errors.h> - -using namespace android; -using namespace std; - -namespace android { -namespace os { -namespace statsd { - -#define SNOOZE_INITIAL_MS 100 -#define SNOOZE_MAX_MS (10 * 60 * 1000) // Ten minutes - -LogReader::LogReader(const sp<LogListener>& listener) : mListener(listener) { -} - -LogReader::~LogReader() { -} - -void LogReader::Run() { - int nextSnoozeMs = SNOOZE_INITIAL_MS; - - // In an ideal world, this outer loop will only ever run one iteration, but it - // exists to handle crashes in logd. The inner loop inside connect_and_read() - // reads from logd forever, but if that read fails, we fall out to the outer - // loop, do the backoff (resetting the backoff timeout if we successfully read - // something), and then try again. - while (true) { - // Connect and read - int lineCount = connect_and_read(); - - // Figure out how long to sleep. - if (lineCount > 0) { - // If we managed to read at least one line, reset the backoff - nextSnoozeMs = SNOOZE_INITIAL_MS; - } else { - // Otherwise, expontial backoff - nextSnoozeMs *= 1.5f; - if (nextSnoozeMs > 10 * 60 * 1000) { - // Don't wait for toooo long. - nextSnoozeMs = SNOOZE_MAX_MS; - } - } - - // Sleep - timespec ts; - timespec rem; - ts.tv_sec = nextSnoozeMs / 1000; - ts.tv_nsec = (nextSnoozeMs % 1000) * 1000000L; - while (nanosleep(&ts, &rem) == -1) { - if (errno == EINTR) { - ts = rem; - } - // other errors are basically impossible - } - } -} - -int LogReader::connect_and_read() { - int lineCount = 0; - status_t err; - logger_list* loggers; - logger* eventLogger; - - // Prepare the logging context - loggers = android_logger_list_alloc(ANDROID_LOG_RDONLY, - /* don't stop after N lines */ 0, - /* no pid restriction */ 0); - - // Open the buffer(s) - eventLogger = android_logger_open(loggers, LOG_ID_STATS); - - // Read forever - if (eventLogger) { - log_msg msg; - while (true) { - // Read a message - err = android_logger_list_read(loggers, &msg); - // err = 0 - no content, unexpected connection drop or EOF. - // err = +ive number - size of retrieved data from logger - // err = -ive number, OS supplied error _except_ for -EAGAIN - if (err <= 0) { - StatsdStats::getInstance().noteLoggerError(err); - fprintf(stderr, "logcat read failure: %s\n", strerror(err)); - break; - } - - // Record that we read one (used above to know how to snooze). - lineCount++; - - // Wrap it in a LogEvent object - LogEvent event(msg); - - // Call the listener - mListener->OnLogEvent(&event, - lineCount == 1 /* indicate whether it's a new connection */); - } - } - - // Free the logger list and close the individual loggers - android_logger_list_free(loggers); - - return lineCount; -} - -} // namespace statsd -} // namespace os -} // namespace android diff --git a/cmds/statsd/src/logd/LogReader.h b/cmds/statsd/src/logd/LogReader.h deleted file mode 100644 index c51074c19d9a..000000000000 --- a/cmds/statsd/src/logd/LogReader.h +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright (C) 2017 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef LOGREADER_H -#define LOGREADER_H - -#include "logd/LogListener.h" - -#include <utils/RefBase.h> - -#include <vector> - -namespace android { -namespace os { -namespace statsd { - -/** - * Class to read logs from logd. - */ -class LogReader : public virtual android::RefBase { -public: - /** - * Construct the LogReader with the event listener. (Which is StatsService) - */ - LogReader(const sp<LogListener>& listener); - - /** - * Destructor. - */ - virtual ~LogReader(); - - /** - * Run the main LogReader loop - */ - void Run(); - -private: - /** - * Who is going to get the events when they're read. - */ - sp<LogListener> mListener; - - /** - * Connect to a single instance of logd, and read until there's a read error. - * Logd can crash, exit, be killed etc. - * - * Returns the number of lines that were read. - */ - int connect_and_read(); -}; - -} // namespace statsd -} // namespace os -} // namespace android - -#endif // LOGREADER_H diff --git a/cmds/statsd/src/main.cpp b/cmds/statsd/src/main.cpp index 58bbd96af7cf..eddc86eca798 100644 --- a/cmds/statsd/src/main.cpp +++ b/cmds/statsd/src/main.cpp @@ -18,7 +18,6 @@ #include "Log.h" #include "StatsService.h" -#include "logd/LogReader.h" #include "socket/StatsSocketListener.h" #include <binder/IInterface.h> @@ -26,9 +25,12 @@ #include <binder/IServiceManager.h> #include <binder/ProcessState.h> #include <binder/Status.h> +#include <hidl/HidlTransportSupport.h> #include <utils/Looper.h> #include <utils/StrongPointer.h> +#include <memory> + #include <stdio.h> #include <sys/stat.h> #include <sys/types.h> @@ -37,9 +39,6 @@ using namespace android; using namespace android::os::statsd; -const bool kUseLogd = false; -const bool kUseStatsdSocket = true; - /** * Thread function data. */ @@ -47,55 +46,6 @@ struct log_reader_thread_data { sp<StatsService> service; }; -/** - * Thread func for where the log reader runs. - */ -static void* log_reader_thread_func(void* cookie) { - log_reader_thread_data* data = static_cast<log_reader_thread_data*>(cookie); - sp<LogReader> reader = new LogReader(data->service); - - // Run the read loop. Never returns. - reader->Run(); - - ALOGW("statsd LogReader.Run() is not supposed to return."); - - delete data; - return NULL; -} - -/** - * Creates and starts the thread to own the LogReader. - */ -static status_t start_log_reader_thread(const sp<StatsService>& service) { - status_t err; - pthread_attr_t attr; - pthread_t thread; - - // Thread data. - log_reader_thread_data* data = new log_reader_thread_data(); - data->service = service; - - // Create the thread - err = pthread_attr_init(&attr); - if (err != NO_ERROR) { - return err; - } - // TODO: Do we need to tweak thread priority? - err = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - if (err != NO_ERROR) { - pthread_attr_destroy(&attr); - return err; - } - err = pthread_create(&thread, &attr, log_reader_thread_func, static_cast<void*>(data)); - if (err != NO_ERROR) { - pthread_attr_destroy(&attr); - return err; - } - pthread_attr_destroy(&attr); - - return NO_ERROR; -} - sp<StatsService> gStatsService = nullptr; @@ -128,13 +78,23 @@ int main(int /*argc*/, char** /*argv*/) { ps->giveThreadPoolName(); IPCThreadState::self()->disableBackgroundScheduling(true); + ::android::hardware::configureRpcThreadpool(1 /*threads*/, false /*willJoin*/); + // Create the service gStatsService = new StatsService(looper); - if (defaultServiceManager()->addService(String16("stats"), gStatsService) != 0) { - ALOGE("Failed to add service"); + if (defaultServiceManager()->addService(String16("stats"), gStatsService, false, + IServiceManager::DUMP_FLAG_PRIORITY_NORMAL | IServiceManager::DUMP_FLAG_PROTO) + != 0) { + ALOGE("Failed to add service as AIDL service"); return -1; } + auto ret = gStatsService->registerAsService(); + if (ret != ::android::OK) { + ALOGE("Failed to add service as HIDL service"); + return 1; // or handle error + } + registerSigHandler(); gStatsService->sayHiToStatsCompanion(); @@ -143,22 +103,11 @@ int main(int /*argc*/, char** /*argv*/) { sp<StatsSocketListener> socketListener = new StatsSocketListener(gStatsService); - if (kUseLogd) { - ALOGI("using logd"); - // Start the log reader thread - status_t err = start_log_reader_thread(gStatsService); - if (err != NO_ERROR) { - return 1; - } - } - - if (kUseStatsdSocket) { ALOGI("using statsd socket"); // Backlog and /proc/sys/net/unix/max_dgram_qlen set to large value if (socketListener->startListener(600)) { exit(1); } - } // Loop forever -- the reports run on this thread in a handler, and the // binder calls remain responsive in their pool of one thread. diff --git a/cmds/statsd/src/matchers/EventMatcherWizard.cpp b/cmds/statsd/src/matchers/EventMatcherWizard.cpp new file mode 100644 index 000000000000..8418e9833509 --- /dev/null +++ b/cmds/statsd/src/matchers/EventMatcherWizard.cpp @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "EventMatcherWizard.h" +#include <unordered_set> + +namespace android { +namespace os { +namespace statsd { + +using std::map; +using std::string; +using std::vector; + +MatchingState EventMatcherWizard::matchLogEvent(const LogEvent& event, int matcher_index) { + if (matcher_index < 0 || matcher_index >= (int)mAllEventMatchers.size()) { + return MatchingState::kNotComputed; + } + vector<MatchingState> matcherCache(mAllEventMatchers.size(), MatchingState::kNotComputed); + mAllEventMatchers[matcher_index]->onLogEvent(event, mAllEventMatchers, matcherCache); + return matcherCache[matcher_index]; +} + +} // namespace statsd +} // namespace os +} // namespace android
\ No newline at end of file diff --git a/cmds/statsd/src/matchers/EventMatcherWizard.h b/cmds/statsd/src/matchers/EventMatcherWizard.h new file mode 100644 index 000000000000..57ec2b35ba32 --- /dev/null +++ b/cmds/statsd/src/matchers/EventMatcherWizard.h @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "LogMatchingTracker.h" + +namespace android { +namespace os { +namespace statsd { + +class EventMatcherWizard : public virtual android::RefBase { +public: + EventMatcherWizard(){}; // for testing + EventMatcherWizard(const std::vector<sp<LogMatchingTracker>>& eventTrackers) + : mAllEventMatchers(eventTrackers){}; + + virtual ~EventMatcherWizard(){}; + + MatchingState matchLogEvent(const LogEvent& event, int matcher_index); + +private: + std::vector<sp<LogMatchingTracker>> mAllEventMatchers; +}; + +} // namespace statsd +} // namespace os +} // namespace android diff --git a/cmds/statsd/src/matchers/LogMatchingTracker.h b/cmds/statsd/src/matchers/LogMatchingTracker.h index 4f30a047e256..88ab4e6f683a 100644 --- a/cmds/statsd/src/matchers/LogMatchingTracker.h +++ b/cmds/statsd/src/matchers/LogMatchingTracker.h @@ -86,8 +86,6 @@ protected: // The collection of the event tag ids that this LogMatchingTracker cares. So we can quickly // return kNotMatched when we receive an event with an id not in the list. This is especially // useful when we have a complex CombinationLogMatcherTracker. - // TODO: Consider use an array instead of stl set. In reality, the number of the tag ids a - // LogMatchingTracker cares is only a few. std::set<int> mAtomIds; }; diff --git a/cmds/statsd/src/metrics/CountMetricProducer.cpp b/cmds/statsd/src/metrics/CountMetricProducer.cpp index 43f53e057000..5ca88142f152 100644 --- a/cmds/statsd/src/metrics/CountMetricProducer.cpp +++ b/cmds/statsd/src/metrics/CountMetricProducer.cpp @@ -66,9 +66,8 @@ const int FIELD_ID_END_BUCKET_ELAPSED_MILLIS = 6; CountMetricProducer::CountMetricProducer(const ConfigKey& key, const CountMetric& metric, const int conditionIndex, const sp<ConditionWizard>& wizard, - const int64_t startTimeNs) - : MetricProducer(metric.id(), key, startTimeNs, conditionIndex, wizard) { - // TODO: evaluate initial conditions. and set mConditionMet. + const int64_t timeBaseNs, const int64_t startTimeNs) + : MetricProducer(metric.id(), key, timeBaseNs, conditionIndex, wizard) { if (metric.has_bucket()) { mBucketSizeNs = TimeUnitToBucketSizeInMillisGuardrailed(key.GetUid(), metric.bucket()) * 1000000; @@ -101,6 +100,10 @@ CountMetricProducer::CountMetricProducer(const ConfigKey& key, const CountMetric mConditionSliced = (metric.links().size() > 0) || (mDimensionsInCondition.size() > 0); + flushIfNeededLocked(startTimeNs); + // Adjust start for partial bucket + mCurrentBucketStartTimeNs = startTimeNs; + VLOG("metric %lld created. bucket size %lld start_time: %lld", (long long)metric.id(), (long long)mBucketSizeNs, (long long)mTimeBaseNs); } @@ -140,6 +143,7 @@ void CountMetricProducer::clearPastBucketsLocked(const int64_t dumpTimeNs) { void CountMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs, const bool include_current_partial_bucket, + const bool erase_data, std::set<string> *str_set, ProtoOutputStream* protoOutput) { if (include_current_partial_bucket) { @@ -227,7 +231,9 @@ void CountMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs, protoOutput->end(protoToken); - mPastBuckets.clear(); + if (erase_data) { + mPastBuckets.clear(); + } } void CountMetricProducer::dropDataLocked(const int64_t dropTimeNs) { diff --git a/cmds/statsd/src/metrics/CountMetricProducer.h b/cmds/statsd/src/metrics/CountMetricProducer.h index 139c0838fef0..1ac4426468d8 100644 --- a/cmds/statsd/src/metrics/CountMetricProducer.h +++ b/cmds/statsd/src/metrics/CountMetricProducer.h @@ -40,10 +40,9 @@ struct CountBucket { class CountMetricProducer : public MetricProducer { public: - // TODO: Pass in the start time from MetricsManager, it should be consistent for all metrics. CountMetricProducer(const ConfigKey& key, const CountMetric& countMetric, const int conditionIndex, const sp<ConditionWizard>& wizard, - const int64_t startTimeNs); + const int64_t timeBaseNs, const int64_t startTimeNs); virtual ~CountMetricProducer(); @@ -57,6 +56,7 @@ private: void onDumpReportLocked(const int64_t dumpTimeNs, const bool include_current_partial_bucket, + const bool erase_data, std::set<string> *str_set, android::util::ProtoOutputStream* protoOutput) override; @@ -80,7 +80,6 @@ private: void flushCurrentBucketLocked(const int64_t& eventTimeNs) override; - // TODO: Add a lock to mPastBuckets. std::unordered_map<MetricDimensionKey, std::vector<CountBucket>> mPastBuckets; // The current bucket (may be a partial bucket). @@ -100,6 +99,7 @@ private: FRIEND_TEST(CountMetricProducerTest, TestAnomalyDetectionUnSliced); FRIEND_TEST(CountMetricProducerTest, TestEventWithAppUpgrade); FRIEND_TEST(CountMetricProducerTest, TestEventWithAppUpgradeInNextBucket); + FRIEND_TEST(CountMetricProducerTest, TestFirstBucket); }; } // namespace statsd diff --git a/cmds/statsd/src/metrics/DurationMetricProducer.cpp b/cmds/statsd/src/metrics/DurationMetricProducer.cpp index 62237bc04642..35deffe5db97 100644 --- a/cmds/statsd/src/metrics/DurationMetricProducer.cpp +++ b/cmds/statsd/src/metrics/DurationMetricProducer.cpp @@ -68,17 +68,14 @@ DurationMetricProducer::DurationMetricProducer(const ConfigKey& key, const Durat const bool nesting, const sp<ConditionWizard>& wizard, const FieldMatcher& internalDimensions, - const int64_t startTimeNs) - : MetricProducer(metric.id(), key, startTimeNs, conditionIndex, wizard), + const int64_t timeBaseNs, const int64_t startTimeNs) + : MetricProducer(metric.id(), key, timeBaseNs, conditionIndex, wizard), mAggregationType(metric.aggregation_type()), mStartIndex(startIndex), mStopIndex(stopIndex), mStopAllIndex(stopAllIndex), mNested(nesting), mContainANYPositionInInternalDimensions(false) { - // TODO: The following boiler plate code appears in all MetricProducers, but we can't abstract - // them in the base class, because the proto generated CountMetric, and DurationMetric are - // not related. Maybe we should add a template in the future?? if (metric.has_bucket()) { mBucketSizeNs = TimeUnitToBucketSizeInMillisGuardrailed(key.GetUid(), metric.bucket()) * 1000000; @@ -131,6 +128,9 @@ DurationMetricProducer::DurationMetricProducer(const ConfigKey& key, const Durat mMetric2ConditionLinks.begin()->conditionFields); } } + flushIfNeededLocked(startTimeNs); + // Adjust start for partial bucket + mCurrentBucketStartTimeNs = startTimeNs; VLOG("metric %lld created. bucket size %lld start_time: %lld", (long long)metric.id(), (long long)mBucketSizeNs, (long long)mTimeBaseNs); } @@ -434,8 +434,6 @@ void DurationMetricProducer::onConditionChangedLocked(const bool conditionMet, VLOG("Metric %lld onConditionChanged", (long long)mMetricId); mCondition = conditionMet; flushIfNeededLocked(eventTime); - // TODO: need to populate the condition change time from the event which triggers the condition - // change, instead of using current time. for (auto& whatIt : mCurrentSlicedDurationTrackerMap) { for (auto& pair : whatIt.second) { pair.second->onConditionChanged(conditionMet, eventTime); @@ -455,6 +453,7 @@ void DurationMetricProducer::clearPastBucketsLocked(const int64_t dumpTimeNs) { void DurationMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs, const bool include_current_partial_bucket, + const bool erase_data, std::set<string> *str_set, ProtoOutputStream* protoOutput) { if (include_current_partial_bucket) { @@ -543,7 +542,9 @@ void DurationMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs, } protoOutput->end(protoToken); - mPastBuckets.clear(); + if (erase_data) { + mPastBuckets.clear(); + } } void DurationMetricProducer::flushIfNeededLocked(const int64_t& eventTimeNs) { @@ -783,7 +784,7 @@ void DurationMetricProducer::onMatchedLogEventLocked(const size_t matcherIndex, if (whatIt != mCurrentSlicedDurationTrackerMap.end()) { for (const auto& condIt : whatIt->second) { const bool cond = dimensionKeysInCondition.find(condIt.first) != - dimensionKeysInCondition.end(); + dimensionKeysInCondition.end() && condition; handleStartEvent(MetricDimensionKey(dimensionInWhat, condIt.first), conditionKey, cond, event); dimensionKeysInCondition.erase(condIt.first); diff --git a/cmds/statsd/src/metrics/DurationMetricProducer.h b/cmds/statsd/src/metrics/DurationMetricProducer.h index 88e455a3f1a1..1b830a3f69f5 100644 --- a/cmds/statsd/src/metrics/DurationMetricProducer.h +++ b/cmds/statsd/src/metrics/DurationMetricProducer.h @@ -42,7 +42,7 @@ public: const int conditionIndex, const size_t startIndex, const size_t stopIndex, const size_t stopAllIndex, const bool nesting, const sp<ConditionWizard>& wizard, - const FieldMatcher& internalDimensions, const int64_t startTimeNs); + const FieldMatcher& internalDimensions, const int64_t timeBaseNs, const int64_t startTimeNs); virtual ~DurationMetricProducer(); @@ -63,6 +63,7 @@ private: void onDumpReportLocked(const int64_t dumpTimeNs, const bool include_current_partial_bucket, + const bool erase_data, std::set<string> *str_set, android::util::ProtoOutputStream* protoOutput) override; @@ -115,7 +116,6 @@ private: ConditionState mUnSlicedPartCondition; // Save the past buckets and we can clear when the StatsLogReport is dumped. - // TODO: Add a lock to mPastBuckets. std::unordered_map<MetricDimensionKey, std::vector<DurationBucket>> mPastBuckets; // The duration trackers in the current bucket. @@ -142,6 +142,7 @@ private: FRIEND_TEST(DurationMetricTrackerTest, TestMaxDurationWithUpgrade); FRIEND_TEST(DurationMetricTrackerTest, TestMaxDurationWithUpgradeInNextBucket); FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicates); + FRIEND_TEST(DurationMetricTrackerTest, TestFirstBucket); }; } // namespace statsd diff --git a/cmds/statsd/src/metrics/EventMetricProducer.cpp b/cmds/statsd/src/metrics/EventMetricProducer.cpp index eec90fc8d053..a18e406b74ca 100644 --- a/cmds/statsd/src/metrics/EventMetricProducer.cpp +++ b/cmds/statsd/src/metrics/EventMetricProducer.cpp @@ -49,7 +49,6 @@ const int FIELD_ID_DATA = 1; // for EventMetricData const int FIELD_ID_ELAPSED_TIMESTAMP_NANOS = 1; const int FIELD_ID_ATOMS = 2; -const int FIELD_ID_WALL_CLOCK_TIMESTAMP_NANOS = 3; EventMetricProducer::EventMetricProducer(const ConfigKey& key, const EventMetric& metric, const int conditionIndex, @@ -106,6 +105,7 @@ void EventMetricProducer::clearPastBucketsLocked(const int64_t dumpTimeNs) { void EventMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs, const bool include_current_partial_bucket, + const bool erase_data, std::set<string> *str_set, ProtoOutputStream* protoOutput) { if (mProto->size() <= 0) { @@ -121,7 +121,9 @@ void EventMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs, protoOutput->write(FIELD_TYPE_MESSAGE | FIELD_ID_EVENT_METRICS, reinterpret_cast<char*>(buffer.get()->data()), buffer.get()->size()); - mProto->clear(); + if (erase_data) { + mProto->clear(); + } } void EventMetricProducer::onConditionChangedLocked(const bool conditionMet, @@ -146,13 +148,9 @@ void EventMetricProducer::onMatchedLogEventInternalLocked( if (truncateTimestamp) { mProto->write(FIELD_TYPE_INT64 | FIELD_ID_ELAPSED_TIMESTAMP_NANOS, (long long)truncateTimestampNsToFiveMinutes(event.GetElapsedTimestampNs())); - mProto->write(FIELD_TYPE_INT64 | FIELD_ID_WALL_CLOCK_TIMESTAMP_NANOS, - (long long)truncateTimestampNsToFiveMinutes(getWallClockNs())); } else { mProto->write(FIELD_TYPE_INT64 | FIELD_ID_ELAPSED_TIMESTAMP_NANOS, (long long)event.GetElapsedTimestampNs()); - mProto->write(FIELD_TYPE_INT64 | FIELD_ID_WALL_CLOCK_TIMESTAMP_NANOS, - (long long)getWallClockNs()); } uint64_t eventToken = mProto->start(FIELD_TYPE_MESSAGE | FIELD_ID_ATOMS); diff --git a/cmds/statsd/src/metrics/EventMetricProducer.h b/cmds/statsd/src/metrics/EventMetricProducer.h index 62d1105e0514..96adfdd48743 100644 --- a/cmds/statsd/src/metrics/EventMetricProducer.h +++ b/cmds/statsd/src/metrics/EventMetricProducer.h @@ -33,7 +33,6 @@ namespace statsd { class EventMetricProducer : public MetricProducer { public: - // TODO: Pass in the start time from MetricsManager, it should be consistent for all metrics. EventMetricProducer(const ConfigKey& key, const EventMetric& eventMetric, const int conditionIndex, const sp<ConditionWizard>& wizard, const int64_t startTimeNs); @@ -48,6 +47,7 @@ private: void onDumpReportLocked(const int64_t dumpTimeNs, const bool include_current_partial_bucket, + const bool erase_data, std::set<string> *str_set, android::util::ProtoOutputStream* protoOutput) override; void clearPastBucketsLocked(const int64_t dumpTimeNs) override; diff --git a/cmds/statsd/src/metrics/GaugeMetricProducer.cpp b/cmds/statsd/src/metrics/GaugeMetricProducer.cpp index aabd3616e2fe..05103a962c33 100644 --- a/cmds/statsd/src/metrics/GaugeMetricProducer.cpp +++ b/cmds/statsd/src/metrics/GaugeMetricProducer.cpp @@ -63,19 +63,27 @@ const int FIELD_ID_DIMENSION_LEAF_IN_CONDITION = 5; // for GaugeBucketInfo const int FIELD_ID_ATOM = 3; const int FIELD_ID_ELAPSED_ATOM_TIMESTAMP = 4; -const int FIELD_ID_WALL_CLOCK_ATOM_TIMESTAMP = 5; const int FIELD_ID_BUCKET_NUM = 6; const int FIELD_ID_START_BUCKET_ELAPSED_MILLIS = 7; const int FIELD_ID_END_BUCKET_ELAPSED_MILLIS = 8; GaugeMetricProducer::GaugeMetricProducer(const ConfigKey& key, const GaugeMetric& metric, const int conditionIndex, - const sp<ConditionWizard>& wizard, const int pullTagId, + const sp<ConditionWizard>& wizard, + const int whatMatcherIndex, + const sp<EventMatcherWizard>& matcherWizard, + const int pullTagId, + const int triggerAtomId, const int atomId, const int64_t timeBaseNs, const int64_t startTimeNs, - shared_ptr<StatsPullerManager> statsPullerManager) + const sp<StatsPullerManager>& pullerManager) : MetricProducer(metric.id(), key, timeBaseNs, conditionIndex, wizard), - mStatsPullerManager(statsPullerManager), + mWhatMatcherIndex(whatMatcherIndex), + mEventMatcherWizard(matcherWizard), + mPullerManager(pullerManager), mPullTagId(pullTagId), + mTriggerAtomId(triggerAtomId), + mAtomId(atomId), + mIsPulled(pullTagId != -1), mMinBucketSizeNs(metric.min_bucket_size_nanos()), mDimensionSoftLimit(StatsdStats::kAtomDimensionKeySizeLimitMap.find(pullTagId) != StatsdStats::kAtomDimensionKeySizeLimitMap.end() @@ -101,7 +109,6 @@ GaugeMetricProducer::GaugeMetricProducer(const ConfigKey& key, const GaugeMetric translateFieldMatcher(metric.gauge_fields_filter().fields(), &mFieldMatchers); } - // TODO: use UidMap if uid->pkg_name is required if (metric.has_dimensions_in_what()) { translateFieldMatcher(metric.dimensions_in_what(), &mDimensionsInWhat); mContainANYPositionInDimensionsInWhat = HasPositionANY(metric.dimensions_in_what()); @@ -126,9 +133,15 @@ GaugeMetricProducer::GaugeMetricProducer(const ConfigKey& key, const GaugeMetric flushIfNeededLocked(startTimeNs); // Kicks off the puller immediately. - if (mPullTagId != -1 && mSamplingType == GaugeMetric::RANDOM_ONE_SAMPLE) { - mStatsPullerManager->RegisterReceiver( - mPullTagId, this, getCurrentBucketEndTimeNs(), mBucketSizeNs); + if (mIsPulled && mSamplingType == GaugeMetric::RANDOM_ONE_SAMPLE) { + mPullerManager->RegisterReceiver(mPullTagId, this, getCurrentBucketEndTimeNs(), + mBucketSizeNs); + } + + // Adjust start for partial bucket + mCurrentBucketStartTimeNs = startTimeNs; + if (mIsPulled) { + pullAndMatchEventsLocked(startTimeNs); } VLOG("Gauge metric %lld created. bucket size %lld start_time: %lld sliced %d", @@ -136,19 +149,10 @@ GaugeMetricProducer::GaugeMetricProducer(const ConfigKey& key, const GaugeMetric mConditionSliced); } -// for testing -GaugeMetricProducer::GaugeMetricProducer(const ConfigKey& key, const GaugeMetric& metric, - const int conditionIndex, - const sp<ConditionWizard>& wizard, const int pullTagId, - const int64_t timeBaseNs, const int64_t startTimeNs) - : GaugeMetricProducer(key, metric, conditionIndex, wizard, pullTagId, timeBaseNs, startTimeNs, - make_shared<StatsPullerManager>()) { -} - GaugeMetricProducer::~GaugeMetricProducer() { VLOG("~GaugeMetricProducer() called"); - if (mPullTagId != -1 && mSamplingType == GaugeMetric::RANDOM_ONE_SAMPLE) { - mStatsPullerManager->UnRegisterReceiver(mPullTagId, this); + if (mIsPulled && mSamplingType == GaugeMetric::RANDOM_ONE_SAMPLE) { + mPullerManager->UnRegisterReceiver(mPullTagId, this); } } @@ -178,6 +182,7 @@ void GaugeMetricProducer::clearPastBucketsLocked(const int64_t dumpTimeNs) { void GaugeMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs, const bool include_current_partial_bucket, + const bool erase_data, std::set<string> *str_set, ProtoOutputStream* protoOutput) { VLOG("Gauge metric %lld report now...", (long long)mMetricId); @@ -222,7 +227,6 @@ void GaugeMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs, (long long)(NanoToMillis(pair.second))); protoOutput->end(wrapperToken); } - mSkippedBuckets.clear(); for (const auto& pair : mPastBuckets) { const MetricDimensionKey& dimensionKey = pair.first; @@ -275,27 +279,20 @@ void GaugeMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs, uint64_t atomsToken = protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_ATOM); - writeFieldValueTreeToStream(mTagId, *(atom.mFields), protoOutput); + writeFieldValueTreeToStream(mAtomId, *(atom.mFields), protoOutput); protoOutput->end(atomsToken); } const bool truncateTimestamp = android::util::AtomsInfo::kNotTruncatingTimestampAtomWhiteList.find( - mTagId) == + mAtomId) == android::util::AtomsInfo::kNotTruncatingTimestampAtomWhiteList.end(); for (const auto& atom : bucket.mGaugeAtoms) { const int64_t elapsedTimestampNs = truncateTimestamp ? truncateTimestampNsToFiveMinutes(atom.mElapsedTimestamps) : atom.mElapsedTimestamps; - const int64_t wallClockNs = truncateTimestamp ? - truncateTimestampNsToFiveMinutes(atom.mWallClockTimestampNs) : - atom.mWallClockTimestampNs; protoOutput->write( FIELD_TYPE_INT64 | FIELD_COUNT_REPEATED | FIELD_ID_ELAPSED_ATOM_TIMESTAMP, (long long)elapsedTimestampNs); - protoOutput->write( - FIELD_TYPE_INT64 | FIELD_COUNT_REPEATED | - FIELD_ID_WALL_CLOCK_ATOM_TIMESTAMP, - (long long)wallClockNs); } } protoOutput->end(bucketInfoToken); @@ -307,11 +304,14 @@ void GaugeMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs, } protoOutput->end(protoToken); - mPastBuckets.clear(); - // TODO: Clear mDimensionKeyMap once the report is dumped. + + if (erase_data) { + mPastBuckets.clear(); + mSkippedBuckets.clear(); + } } -void GaugeMetricProducer::pullLocked(const int64_t timestampNs) { +void GaugeMetricProducer::pullAndMatchEventsLocked(const int64_t timestampNs) { bool triggerPuller = false; switch(mSamplingType) { // When the metric wants to do random sampling and there is already one gauge atom for the @@ -334,15 +334,16 @@ void GaugeMetricProducer::pullLocked(const int64_t timestampNs) { if (!triggerPuller) { return; } - vector<std::shared_ptr<LogEvent>> allData; - if (!mStatsPullerManager->Pull(mPullTagId, timestampNs, &allData)) { - ALOGE("Gauge Stats puller failed for tag: %d", mPullTagId); + if (!mPullerManager->Pull(mPullTagId, timestampNs, &allData)) { + ALOGE("Gauge Stats puller failed for tag: %d at %lld", mPullTagId, (long long)timestampNs); return; } - for (const auto& data : allData) { - onMatchedLogEventLocked(0, *data); + if (mEventMatcherWizard->matchLogEvent( + *data, mWhatMatcherIndex) == MatchingState::kMatched) { + onMatchedLogEventLocked(mWhatMatcherIndex, *data); + } } } @@ -351,9 +352,8 @@ void GaugeMetricProducer::onConditionChangedLocked(const bool conditionMet, VLOG("GaugeMetric %lld onConditionChanged", (long long)mMetricId); flushIfNeededLocked(eventTimeNs); mCondition = conditionMet; - - if (mPullTagId != -1) { - pullLocked(eventTimeNs); + if (mIsPulled) { + pullAndMatchEventsLocked(eventTimeNs); } // else: Push mode. No need to proactively pull the gauge data. } @@ -365,19 +365,32 @@ void GaugeMetricProducer::onSlicedConditionMayChangeLocked(bool overallCondition // If the condition is sliced, mCondition is true if any of the dimensions is true. And we will // pull for every dimension. mCondition = overallCondition; - if (mPullTagId != -1) { - pullLocked(eventTimeNs); + if (mIsPulled) { + pullAndMatchEventsLocked(eventTimeNs); } // else: Push mode. No need to proactively pull the gauge data. } std::shared_ptr<vector<FieldValue>> GaugeMetricProducer::getGaugeFields(const LogEvent& event) { + std::shared_ptr<vector<FieldValue>> gaugeFields; if (mFieldMatchers.size() > 0) { - std::shared_ptr<vector<FieldValue>> gaugeFields = std::make_shared<vector<FieldValue>>(); + gaugeFields = std::make_shared<vector<FieldValue>>(); filterGaugeValues(mFieldMatchers, event.getValues(), gaugeFields.get()); - return gaugeFields; } else { - return std::make_shared<vector<FieldValue>>(event.getValues()); + gaugeFields = std::make_shared<vector<FieldValue>>(event.getValues()); + } + // Trim all dimension fields from output. Dimensions will appear in output report and will + // benefit from dictionary encoding. For large pulled atoms, this can give the benefit of + // optional repeated field. + for (const auto& field : mDimensionsInWhat) { + for (auto it = gaugeFields->begin(); it != gaugeFields->end();) { + if (it->mField.matches(field)) { + it = gaugeFields->erase(it); + } else { + it++; + } + } } + return gaugeFields; } void GaugeMetricProducer::onDataPulled(const std::vector<std::shared_ptr<LogEvent>>& allData) { @@ -386,7 +399,10 @@ void GaugeMetricProducer::onDataPulled(const std::vector<std::shared_ptr<LogEven return; } for (const auto& data : allData) { - onMatchedLogEventLocked(0, *data); + if (mEventMatcherWizard->matchLogEvent( + *data, mWhatMatcherIndex) == MatchingState::kMatched) { + onMatchedLogEventLocked(mWhatMatcherIndex, *data); + } } } @@ -417,7 +433,6 @@ void GaugeMetricProducer::onMatchedLogEventInternalLocked( return; } int64_t eventTimeNs = event.GetElapsedTimestampNs(); - mTagId = event.GetTagId(); if (eventTimeNs < mCurrentBucketStartTimeNs) { VLOG("Gauge Skip event due to late arrival: %lld vs %lld", (long long)eventTimeNs, (long long)mCurrentBucketStartTimeNs); @@ -425,6 +440,11 @@ void GaugeMetricProducer::onMatchedLogEventInternalLocked( } flushIfNeededLocked(eventTimeNs); + if (mTriggerAtomId == event.GetTagId()) { + pullAndMatchEventsLocked(eventTimeNs); + return; + } + // When gauge metric wants to randomly sample the output atom, we just simply use the first // gauge in the given bucket. if (mCurrentSlicedBucket->find(eventKey) != mCurrentSlicedBucket->end() && @@ -437,7 +457,7 @@ void GaugeMetricProducer::onMatchedLogEventInternalLocked( if ((*mCurrentSlicedBucket)[eventKey].size() >= mGaugeAtomsPerDimensionLimit) { return; } - GaugeAtom gaugeAtom(getGaugeFields(event), eventTimeNs, getWallClockNs()); + GaugeAtom gaugeAtom(getGaugeFields(event), eventTimeNs); (*mCurrentSlicedBucket)[eventKey].push_back(gaugeAtom); // Anomaly detection on gauge metric only works when there is one numeric // field specified. diff --git a/cmds/statsd/src/metrics/GaugeMetricProducer.h b/cmds/statsd/src/metrics/GaugeMetricProducer.h index c74f7927dfac..5866139047ae 100644 --- a/cmds/statsd/src/metrics/GaugeMetricProducer.h +++ b/cmds/statsd/src/metrics/GaugeMetricProducer.h @@ -24,6 +24,7 @@ #include "../external/PullDataReceiver.h" #include "../external/StatsPullerManager.h" #include "../matchers/matcher_util.h" +#include "../matchers/EventMatcherWizard.h" #include "MetricProducer.h" #include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" #include "../stats_util.h" @@ -33,12 +34,11 @@ namespace os { namespace statsd { struct GaugeAtom { - GaugeAtom(std::shared_ptr<vector<FieldValue>> fields, int64_t elapsedTimeNs, int wallClockNs) - : mFields(fields), mElapsedTimestamps(elapsedTimeNs), mWallClockTimestampNs(wallClockNs) { + GaugeAtom(std::shared_ptr<vector<FieldValue>> fields, int64_t elapsedTimeNs) + : mFields(fields), mElapsedTimestamps(elapsedTimeNs) { } std::shared_ptr<vector<FieldValue>> mFields; int64_t mElapsedTimestamps; - int64_t mWallClockTimestampNs; }; struct GaugeBucket { @@ -57,8 +57,12 @@ typedef std::unordered_map<MetricDimensionKey, std::vector<GaugeAtom>> class GaugeMetricProducer : public virtual MetricProducer, public virtual PullDataReceiver { public: GaugeMetricProducer(const ConfigKey& key, const GaugeMetric& gaugeMetric, - const int conditionIndex, const sp<ConditionWizard>& wizard, - const int pullTagId, const int64_t timeBaseNs, const int64_t startTimeNs); + const int conditionIndex, const sp<ConditionWizard>& conditionWizard, + const int whatMatcherIndex, + const sp<EventMatcherWizard>& matcherWizard, + const int pullTagId, const int triggerAtomId, const int atomId, + const int64_t timeBaseNs, const int64_t startTimeNs, + const sp<StatsPullerManager>& pullerManager); virtual ~GaugeMetricProducer(); @@ -76,8 +80,8 @@ public: } flushCurrentBucketLocked(eventTimeNs); mCurrentBucketStartTimeNs = eventTimeNs; - if (mPullTagId != -1) { - pullLocked(eventTimeNs); + if (mIsPulled) { + pullAndMatchEventsLocked(eventTimeNs); } }; @@ -90,17 +94,11 @@ protected: private: void onDumpReportLocked(const int64_t dumpTimeNs, const bool include_current_partial_bucket, + const bool erase_data, std::set<string> *str_set, android::util::ProtoOutputStream* protoOutput) override; void clearPastBucketsLocked(const int64_t dumpTimeNs) override; - // for testing - GaugeMetricProducer(const ConfigKey& key, const GaugeMetric& gaugeMetric, - const int conditionIndex, const sp<ConditionWizard>& wizard, - const int pullTagId, - const int64_t timeBaseNs, const int64_t startTimeNs, - std::shared_ptr<StatsPullerManager> statsPullerManager); - // Internal interface to handle condition change. void onConditionChangedLocked(const bool conditionMet, const int64_t eventTime) override; @@ -119,16 +117,26 @@ private: void flushCurrentBucketLocked(const int64_t& eventTimeNs) override; - void pullLocked(const int64_t timestampNs); + void pullAndMatchEventsLocked(const int64_t timestampNs); - int mTagId; + const int mWhatMatcherIndex; - std::shared_ptr<StatsPullerManager> mStatsPullerManager; + sp<EventMatcherWizard> mEventMatcherWizard; + + sp<StatsPullerManager> mPullerManager; // tagId for pulled data. -1 if this is not pulled const int mPullTagId; + // tagId for atoms that trigger the pulling, if any + const int mTriggerAtomId; + + // tagId for output atom + const int mAtomId; + + // if this is pulled metric + const bool mIsPulled; + // Save the past buckets and we can clear when the StatsLogReport is dumped. - // TODO: Add a lock to mPastBuckets. std::unordered_map<MetricDimensionKey, std::vector<GaugeBucket>> mPastBuckets; // The current partial bucket. @@ -166,12 +174,15 @@ private: const size_t mGaugeAtomsPerDimensionLimit; - FRIEND_TEST(GaugeMetricProducerTest, TestWithCondition); - FRIEND_TEST(GaugeMetricProducerTest, TestWithSlicedCondition); - FRIEND_TEST(GaugeMetricProducerTest, TestNoCondition); + FRIEND_TEST(GaugeMetricProducerTest, TestPulledEventsWithCondition); + FRIEND_TEST(GaugeMetricProducerTest, TestPulledEventsWithSlicedCondition); + FRIEND_TEST(GaugeMetricProducerTest, TestPulledEventsNoCondition); FRIEND_TEST(GaugeMetricProducerTest, TestPushedEventsWithUpgrade); FRIEND_TEST(GaugeMetricProducerTest, TestPulledWithUpgrade); - FRIEND_TEST(GaugeMetricProducerTest, TestAnomalyDetection); + FRIEND_TEST(GaugeMetricProducerTest, TestPulledEventsAnomalyDetection); + FRIEND_TEST(GaugeMetricProducerTest, TestFirstBucket); + FRIEND_TEST(GaugeMetricProducerTest, TestPullOnTrigger); + FRIEND_TEST(GaugeMetricProducerTest, TestRemoveDimensionInOutput); }; } // namespace statsd diff --git a/cmds/statsd/src/metrics/MetricProducer.cpp b/cmds/statsd/src/metrics/MetricProducer.cpp index df081817232e..f87849edae7a 100644 --- a/cmds/statsd/src/metrics/MetricProducer.cpp +++ b/cmds/statsd/src/metrics/MetricProducer.cpp @@ -64,8 +64,54 @@ void MetricProducer::onMatchedLogEventLocked(const size_t matcherIndex, const Lo onMatchedLogEventInternalLocked( matcherIndex, metricKey, conditionKey, condition, event); } +} + +bool MetricProducer::evaluateActiveStateLocked(int64_t elapsedTimestampNs) { + bool isActive = mEventActivationMap.empty(); + for (auto& it : mEventActivationMap) { + if (it.second.state == ActivationState::kActive && + elapsedTimestampNs > it.second.ttl_ns + it.second.activation_ns) { + it.second.state = ActivationState::kNotActive; + } + if (it.second.state == ActivationState::kActive) { + isActive = true; + } + } + return isActive; +} + +void MetricProducer::flushIfExpire(int64_t elapsedTimestampNs) { + std::lock_guard<std::mutex> lock(mMutex); + if (!mIsActive) { + return; + } + mIsActive = evaluateActiveStateLocked(elapsedTimestampNs); + if (!mIsActive) { + flushLocked(elapsedTimestampNs); + } +} + +void MetricProducer::addActivation(int activationTrackerIndex, int64_t ttl_seconds) { + std::lock_guard<std::mutex> lock(mMutex); + // When a metric producer does not depend on any activation, its mIsActive is true. + // Therefor, if this is the 1st activation, mIsActive will turn to false. Otherwise it does not + // change. + if (mEventActivationMap.empty()) { + mIsActive = false; + } + mEventActivationMap[activationTrackerIndex].ttl_ns = ttl_seconds * NS_PER_SEC; +} + +void MetricProducer::activateLocked(int activationTrackerIndex, int64_t elapsedTimestampNs) { + auto it = mEventActivationMap.find(activationTrackerIndex); + if (it == mEventActivationMap.end()) { + return; + } + it->second.activation_ns = elapsedTimestampNs; + it->second.state = ActivationState::kActive; + mIsActive = true; +} - } } // namespace statsd } // namespace os diff --git a/cmds/statsd/src/metrics/MetricProducer.h b/cmds/statsd/src/metrics/MetricProducer.h index 6fe4bfb47a1f..127cbbde1a3a 100644 --- a/cmds/statsd/src/metrics/MetricProducer.h +++ b/cmds/statsd/src/metrics/MetricProducer.h @@ -34,6 +34,17 @@ namespace android { namespace os { namespace statsd { +// If the metric has no activation requirement, it will be active once the metric producer is +// created. +// If the metric needs to be activated by atoms, the metric producer will start +// with kNotActive state, turn to kActive when the activation event arrives, become kNotActive +// when it reaches the duration limit (timebomb). If the activation event arrives again before +// or after it expires, the event producer will be re-activated and ttl will be reset. +enum ActivationState { + kNotActive = 0, + kActive = 1, +}; + // A MetricProducer is responsible for compute one single metrics, creating stats log report, and // writing the report to dropbox. MetricProducers should respond to package changes as required in // PackageInfoListener, but if none of the metrics are slicing by package name, then the update can @@ -54,7 +65,8 @@ public: mContainANYPositionInDimensionsInWhat(false), mSliceByPositionALL(false), mSameConditionDimensionsInTracker(false), - mHasLinksToAllConditionDimensionsInTracker(false) { + mHasLinksToAllConditionDimensionsInTracker(false), + mIsActive(true) { } virtual ~MetricProducer(){}; @@ -93,17 +105,23 @@ public: // Consume the parsed stats log entry that already matched the "what" of the metric. void onMatchedLogEvent(const size_t matcherIndex, const LogEvent& event) { std::lock_guard<std::mutex> lock(mMutex); - onMatchedLogEventLocked(matcherIndex, event); + if (mIsActive) { + onMatchedLogEventLocked(matcherIndex, event); + } } void onConditionChanged(const bool condition, const int64_t eventTime) { std::lock_guard<std::mutex> lock(mMutex); - onConditionChangedLocked(condition, eventTime); + if (mIsActive) { + onConditionChangedLocked(condition, eventTime); + } } void onSlicedConditionMayChange(bool overallCondition, const int64_t eventTime) { std::lock_guard<std::mutex> lock(mMutex); - onSlicedConditionMayChangeLocked(overallCondition, eventTime); + if (mIsActive) { + onSlicedConditionMayChangeLocked(overallCondition, eventTime); + } } bool isConditionSliced() const { @@ -115,10 +133,12 @@ public: // This method clears all the past buckets. void onDumpReport(const int64_t dumpTimeNs, const bool include_current_partial_bucket, + const bool erase_data, std::set<string> *str_set, android::util::ProtoOutputStream* protoOutput) { std::lock_guard<std::mutex> lock(mMutex); - return onDumpReportLocked(dumpTimeNs, include_current_partial_bucket, str_set, protoOutput); + return onDumpReportLocked(dumpTimeNs, include_current_partial_bucket, erase_data, + str_set, protoOutput); } void clearPastBuckets(const int64_t dumpTimeNs) { @@ -177,18 +197,32 @@ public: return mCurrentBucketNum; } + void activate(int activationTrackerIndex, int64_t elapsedTimestampNs) { + std::lock_guard<std::mutex> lock(mMutex); + activateLocked(activationTrackerIndex, elapsedTimestampNs); + } + + void addActivation(int activationTrackerIndex, int64_t ttl_seconds); + + void flushIfExpire(int64_t elapsedTimestampNs); + protected: virtual void onConditionChangedLocked(const bool condition, const int64_t eventTime) = 0; virtual void onSlicedConditionMayChangeLocked(bool overallCondition, const int64_t eventTime) = 0; virtual void onDumpReportLocked(const int64_t dumpTimeNs, const bool include_current_partial_bucket, + const bool erase_data, std::set<string> *str_set, android::util::ProtoOutputStream* protoOutput) = 0; virtual void clearPastBucketsLocked(const int64_t dumpTimeNs) = 0; virtual size_t byteSizeLocked() const = 0; virtual void dumpStatesLocked(FILE* out, bool verbose) const = 0; + bool evaluateActiveStateLocked(int64_t elapsedTimestampNs); + + void activateLocked(int activationTrackerIndex, int64_t elapsedTimestampNs); + /** * Flushes the current bucket if the eventTime is after the current bucket's end time. This will also flush the current partial bucket in memory. @@ -198,9 +232,9 @@ protected: /** * Flushes all the data including the current partial bucket. */ - virtual void flushLocked(const int64_t& eventTime) { - flushIfNeededLocked(eventTime); - flushCurrentBucketLocked(eventTime); + virtual void flushLocked(const int64_t& eventTimeNs) { + flushIfNeededLocked(eventTimeNs); + flushCurrentBucketLocked(eventTimeNs); }; /** @@ -295,6 +329,21 @@ protected: virtual void onMatchedLogEventLocked(const size_t matcherIndex, const LogEvent& event); mutable std::mutex mMutex; + + struct Activation { + Activation() : ttl_ns(0), activation_ns(0), state(ActivationState::kNotActive) {} + + int64_t ttl_ns; + int64_t activation_ns; + ActivationState state; + }; + // When the metric producer has multiple activations, these activations are ORed to determine + // whether the metric producer is ready to generate metrics. + std::unordered_map<int, Activation> mEventActivationMap; + + bool mIsActive; + + FRIEND_TEST(MetricActivationE2eTest, TestCountMetric); }; } // namespace statsd diff --git a/cmds/statsd/src/metrics/MetricsManager.cpp b/cmds/statsd/src/metrics/MetricsManager.cpp index 4fac0e1a141b..4244d5bed23b 100644 --- a/cmds/statsd/src/metrics/MetricsManager.cpp +++ b/cmds/statsd/src/metrics/MetricsManager.cpp @@ -56,10 +56,12 @@ const int FIELD_ID_ANNOTATIONS_INT32 = 2; MetricsManager::MetricsManager(const ConfigKey& key, const StatsdConfig& config, const int64_t timeBaseNs, const int64_t currentTimeNs, - const sp<UidMap> &uidMap, + const sp<UidMap>& uidMap, + const sp<StatsPullerManager>& pullerManager, const sp<AlarmMonitor>& anomalyAlarmMonitor, const sp<AlarmMonitor>& periodicAlarmMonitor) - : mConfigKey(key), mUidMap(uidMap), + : mConfigKey(key), + mUidMap(uidMap), mTtlNs(config.has_ttl_in_seconds() ? config.ttl_in_seconds() * NS_PER_SEC : -1), mTtlEndNs(-1), mLastReportTimeNs(currentTimeNs), @@ -67,12 +69,12 @@ MetricsManager::MetricsManager(const ConfigKey& key, const StatsdConfig& config, // Init the ttl end timestamp. refreshTtl(timeBaseNs); - mConfigValid = - initStatsdConfig(key, config, *uidMap, anomalyAlarmMonitor, periodicAlarmMonitor, - timeBaseNs, currentTimeNs, mTagIds, mAllAtomMatchers, - mAllConditionTrackers, mAllMetricProducers, mAllAnomalyTrackers, - mAllPeriodicAlarmTrackers, mConditionToMetricMap, mTrackerToMetricMap, - mTrackerToConditionMap, mNoReportMetricIds); + mConfigValid = initStatsdConfig( + key, config, *uidMap, pullerManager, anomalyAlarmMonitor, periodicAlarmMonitor, + timeBaseNs, currentTimeNs, mTagIds, mAllAtomMatchers, mAllConditionTrackers, + mAllMetricProducers, mAllAnomalyTrackers, mAllPeriodicAlarmTrackers, + mConditionToMetricMap, mTrackerToMetricMap, mTrackerToConditionMap, + mActivationAtomTrackerToMetricMap, mMetricIndexesWithActivation, mNoReportMetricIds); mHashStringsInReport = config.hash_strings_in_metric_report(); @@ -195,6 +197,7 @@ void MetricsManager::dropData(const int64_t dropTimeNs) { void MetricsManager::onDumpReport(const int64_t dumpTimeStampNs, const bool include_current_partial_bucket, + const bool erase_data, std::set<string> *str_set, ProtoOutputStream* protoOutput) { VLOG("=========================Metric Reports Start=========================="); @@ -204,11 +207,11 @@ void MetricsManager::onDumpReport(const int64_t dumpTimeStampNs, uint64_t token = protoOutput->start( FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_METRICS); if (mHashStringsInReport) { - producer->onDumpReport(dumpTimeStampNs, include_current_partial_bucket, str_set, - protoOutput); + producer->onDumpReport(dumpTimeStampNs, include_current_partial_bucket, erase_data, + str_set, protoOutput); } else { - producer->onDumpReport(dumpTimeStampNs, include_current_partial_bucket, nullptr, - protoOutput); + producer->onDumpReport(dumpTimeStampNs, include_current_partial_bucket, erase_data, + nullptr, protoOutput); } protoOutput->end(token); } else { @@ -237,7 +240,6 @@ void MetricsManager::onLogEvent(const LogEvent& event) { if (event.GetTagId() == android::util::APP_BREADCRUMB_REPORTED) { // Check that app breadcrumb reported fields are valid. - // TODO: Find a way to make these checks easier to maintain. status_t err = NO_ERROR; // Uid is 3rd from last field and must match the caller's uid, @@ -298,7 +300,12 @@ void MetricsManager::onLogEvent(const LogEvent& event) { } int tagId = event.GetTagId(); - int64_t eventTime = event.GetElapsedTimestampNs(); + int64_t eventTimeNs = event.GetElapsedTimestampNs(); + + for (int metric : mMetricIndexesWithActivation) { + mAllMetricProducers[metric]->flushIfExpire(eventTimeNs); + } + if (mTagIds.find(tagId) == mTagIds.end()) { // not interesting... return; @@ -310,6 +317,14 @@ void MetricsManager::onLogEvent(const LogEvent& event) { matcher->onLogEvent(event, mAllAtomMatchers, matcherCache); } + for (const auto& it : mActivationAtomTrackerToMetricMap) { + if (matcherCache[it.first] == MatchingState::kMatched) { + for (int metricIndex : it.second) { + mAllMetricProducers[metricIndex]->activate(it.first, eventTimeNs); + } + } + } + // A bitmap to see which ConditionTracker needs to be re-evaluated. vector<bool> conditionToBeEvaluated(mAllConditionTrackers.size(), false); @@ -347,13 +362,13 @@ void MetricsManager::onLogEvent(const LogEvent& event) { // Push the new condition to it directly. if (!mAllMetricProducers[metricIndex]->isConditionSliced()) { mAllMetricProducers[metricIndex]->onConditionChanged(conditionCache[i], - eventTime); + eventTimeNs); // metric cares about sliced conditions, and it may have changed. Send // notification, and the metric can query the sliced conditions that are // interesting to it. } else { mAllMetricProducers[metricIndex]->onSlicedConditionMayChange(conditionCache[i], - eventTime); + eventTimeNs); } } } diff --git a/cmds/statsd/src/metrics/MetricsManager.h b/cmds/statsd/src/metrics/MetricsManager.h index 6f4db48def86..a4672b68f2bc 100644 --- a/cmds/statsd/src/metrics/MetricsManager.h +++ b/cmds/statsd/src/metrics/MetricsManager.h @@ -16,6 +16,7 @@ #pragma once +#include "external/StatsPullerManager.h" #include "anomaly/AlarmMonitor.h" #include "anomaly/AlarmTracker.h" #include "anomaly/AnomalyTracker.h" @@ -36,9 +37,10 @@ namespace statsd { // A MetricsManager is responsible for managing metrics from one single config source. class MetricsManager : public PackageInfoListener { public: - MetricsManager(const ConfigKey& configKey, const StatsdConfig& config, - const int64_t timeBaseNs, const int64_t currentTimeNs, - const sp<UidMap>& uidMap, const sp<AlarmMonitor>& anomalyAlarmMonitor, + MetricsManager(const ConfigKey& configKey, const StatsdConfig& config, const int64_t timeBaseNs, + const int64_t currentTimeNs, const sp<UidMap>& uidMap, + const sp<StatsPullerManager>& pullerManager, + const sp<AlarmMonitor>& anomalyAlarmMonitor, const sp<AlarmMonitor>& periodicAlarmMonitor); virtual ~MetricsManager(); @@ -105,6 +107,7 @@ public: virtual void onDumpReport(const int64_t dumpTimeNs, const bool include_current_partial_bucket, + const bool erase_data, std::set<string> *str_set, android::util::ProtoOutputStream* protoOutput); @@ -193,6 +196,11 @@ private: // maps from ConditionTracker to MetricProducer std::unordered_map<int, std::vector<int>> mConditionToMetricMap; + // maps from life span triggering event to MetricProducers. + std::unordered_map<int, std::vector<int>> mActivationAtomTrackerToMetricMap; + + std::vector<int> mMetricIndexesWithActivation; + void initLogSourceWhiteList(); // The metrics that don't need to be uploaded or even reported. @@ -205,7 +213,7 @@ private: FRIEND_TEST(GaugeMetricE2eTest, TestMultipleFieldsForPushedEvent); FRIEND_TEST(GaugeMetricE2eTest, TestRandomSamplePulledEvents); FRIEND_TEST(GaugeMetricE2eTest, TestRandomSamplePulledEvent_LateAlarm); - FRIEND_TEST(GaugeMetricE2eTest, TestAllConditionChangesSamplePulledEvents); + FRIEND_TEST(GaugeMetricE2eTest, TestConditionChangeToTrueSamplePulledEvents); FRIEND_TEST(ValueMetricE2eTest, TestPulledEvents); FRIEND_TEST(ValueMetricE2eTest, TestPulledEvents_LateAlarm); FRIEND_TEST(DimensionInConditionE2eTest, TestCreateCountMetric_NoLink_OR_CombinationCondition); @@ -228,6 +236,7 @@ private: FRIEND_TEST(AlarmE2eTest, TestMultipleAlarms); FRIEND_TEST(ConfigTtlE2eTest, TestCountMetric); + FRIEND_TEST(MetricActivationE2eTest, TestCountMetric); }; } // namespace statsd diff --git a/cmds/statsd/src/metrics/ValueMetricProducer.cpp b/cmds/statsd/src/metrics/ValueMetricProducer.cpp index 41e55cb27f5e..a34df8aabea2 100644 --- a/cmds/statsd/src/metrics/ValueMetricProducer.cpp +++ b/cmds/statsd/src/metrics/ValueMetricProducer.cpp @@ -27,7 +27,7 @@ using android::util::FIELD_COUNT_REPEATED; using android::util::FIELD_TYPE_BOOL; -using android::util::FIELD_TYPE_FLOAT; +using android::util::FIELD_TYPE_DOUBLE; using android::util::FIELD_TYPE_INT32; using android::util::FIELD_TYPE_INT64; using android::util::FIELD_TYPE_MESSAGE; @@ -64,21 +64,31 @@ const int FIELD_ID_BUCKET_INFO = 3; const int FIELD_ID_DIMENSION_LEAF_IN_WHAT = 4; const int FIELD_ID_DIMENSION_LEAF_IN_CONDITION = 5; // for ValueBucketInfo -const int FIELD_ID_VALUE = 3; +const int FIELD_ID_VALUE_INDEX = 1; +const int FIELD_ID_VALUE_LONG = 2; +const int FIELD_ID_VALUE_DOUBLE = 3; +const int FIELD_ID_VALUES = 9; const int FIELD_ID_BUCKET_NUM = 4; const int FIELD_ID_START_BUCKET_ELAPSED_MILLIS = 5; const int FIELD_ID_END_BUCKET_ELAPSED_MILLIS = 6; // ValueMetric has a minimum bucket size of 10min so that we don't pull too frequently -ValueMetricProducer::ValueMetricProducer(const ConfigKey& key, const ValueMetric& metric, +ValueMetricProducer::ValueMetricProducer(const ConfigKey& key, + const ValueMetric& metric, const int conditionIndex, - const sp<ConditionWizard>& wizard, const int pullTagId, - const int64_t timeBaseNs, const int64_t startTimestampNs, - shared_ptr<StatsPullerManager> statsPullerManager) - : MetricProducer(metric.id(), key, timeBaseNs, conditionIndex, wizard), - mValueField(metric.value_field()), - mStatsPullerManager(statsPullerManager), + const sp<ConditionWizard>& conditionWizard, + const int whatMatcherIndex, + const sp<EventMatcherWizard>& matcherWizard, + const int pullTagId, + const int64_t timeBaseNs, + const int64_t startTimeNs, + const sp<StatsPullerManager>& pullerManager) + : MetricProducer(metric.id(), key, timeBaseNs, conditionIndex, conditionWizard), + mWhatMatcherIndex(whatMatcherIndex), + mEventMatcherWizard(matcherWizard), + mPullerManager(pullerManager), mPullTagId(pullTagId), + mIsPulled(pullTagId != -1), mMinBucketSizeNs(metric.min_bucket_size_nanos()), mDimensionSoftLimit(StatsdStats::kAtomDimensionKeySizeLimitMap.find(pullTagId) != StatsdStats::kAtomDimensionKeySizeLimitMap.end() @@ -88,8 +98,11 @@ ValueMetricProducer::ValueMetricProducer(const ConfigKey& key, const ValueMetric StatsdStats::kAtomDimensionKeySizeLimitMap.end() ? StatsdStats::kAtomDimensionKeySizeLimitMap.at(pullTagId).second : StatsdStats::kDimensionKeySizeHardLimit), - mUseAbsoluteValueOnReset(metric.use_absolute_value_on_reset()) { - // TODO: valuemetric for pushed events may need unlimited bucket length + mUseAbsoluteValueOnReset(metric.use_absolute_value_on_reset()), + mAggregationType(metric.aggregation_type()), + mUseDiff(metric.has_use_diff() ? metric.use_diff() : (mIsPulled ? true : false)), + mValueDirection(metric.value_direction()), + mSkipZeroDiffOutput(metric.skip_zero_diff_output()) { int64_t bucketSizeMills = 0; if (metric.has_bucket()) { bucketSizeMills = TimeUnitToBucketSizeInMillisGuardrailed(key.GetUid(), metric.bucket()); @@ -98,6 +111,9 @@ ValueMetricProducer::ValueMetricProducer(const ConfigKey& key, const ValueMetric } mBucketSizeNs = bucketSizeMills * 1000000; + + translateFieldMatcher(metric.value_field(), &mFieldMatchers); + if (metric.has_dimensions_in_what()) { translateFieldMatcher(metric.dimensions_in_what(), &mDimensionsInWhat); mContainANYPositionInDimensionsInWhat = HasPositionANY(metric.dimensions_in_what()); @@ -117,37 +133,33 @@ ValueMetricProducer::ValueMetricProducer(const ConfigKey& key, const ValueMetric } } - if (mValueField.child_size() > 0) { - mField = mValueField.child(0).field(); - } mConditionSliced = (metric.links().size() > 0) || (mDimensionsInCondition.size() > 0); mSliceByPositionALL = HasPositionALL(metric.dimensions_in_what()) || - HasPositionALL(metric.dimensions_in_condition()); + HasPositionALL(metric.dimensions_in_condition()); - // Kicks off the puller immediately. - flushIfNeededLocked(startTimestampNs); - if (mPullTagId != -1) { - mStatsPullerManager->RegisterReceiver( - mPullTagId, this, mCurrentBucketStartTimeNs + mBucketSizeNs, mBucketSizeNs); - } + flushIfNeededLocked(startTimeNs); - VLOG("value metric %lld created. bucket size %lld start_time: %lld", - (long long)metric.id(), (long long)mBucketSizeNs, (long long)mTimeBaseNs); -} + if (mIsPulled) { + mPullerManager->RegisterReceiver(mPullTagId, this, getCurrentBucketEndTimeNs(), + mBucketSizeNs); + } -// for testing -ValueMetricProducer::ValueMetricProducer(const ConfigKey& key, const ValueMetric& metric, - const int conditionIndex, - const sp<ConditionWizard>& wizard, const int pullTagId, - const int64_t timeBaseNs, const int64_t startTimeNs) - : ValueMetricProducer(key, metric, conditionIndex, wizard, pullTagId, timeBaseNs, startTimeNs, - make_shared<StatsPullerManager>()) { + // Only do this for partial buckets like first bucket. All other buckets should use + // flushIfNeeded to adjust start and end to bucket boundaries. + // Adjust start for partial bucket + mCurrentBucketStartTimeNs = startTimeNs; + // Kicks off the puller immediately if condition is true and diff based. + if (mIsPulled && mCondition && mUseDiff) { + pullAndMatchEventsLocked(startTimeNs); + } + VLOG("value metric %lld created. bucket size %lld start_time: %lld", (long long)metric.id(), + (long long)mBucketSizeNs, (long long)mTimeBaseNs); } ValueMetricProducer::~ValueMetricProducer() { VLOG("~ValueMetricProducer() called"); - if (mPullTagId != -1) { - mStatsPullerManager->UnRegisterReceiver(mPullTagId, this); + if (mIsPulled) { + mPullerManager->UnRegisterReceiver(mPullTagId, this); } } @@ -169,6 +181,7 @@ void ValueMetricProducer::clearPastBucketsLocked(const int64_t dumpTimeNs) { void ValueMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs, const bool include_current_partial_bucket, + const bool erase_data, std::set<string> *str_set, ProtoOutputStream* protoOutput) { VLOG("metric %lld dump report now...", (long long)mMetricId); @@ -186,14 +199,14 @@ void ValueMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs, // Fills the dimension path if not slicing by ALL. if (!mSliceByPositionALL) { if (!mDimensionsInWhat.empty()) { - uint64_t dimenPathToken = protoOutput->start( - FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_PATH_IN_WHAT); + uint64_t dimenPathToken = + protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_PATH_IN_WHAT); writeDimensionPathToProto(mDimensionsInWhat, protoOutput); protoOutput->end(dimenPathToken); } if (!mDimensionsInCondition.empty()) { - uint64_t dimenPathToken = protoOutput->start( - FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_PATH_IN_CONDITION); + uint64_t dimenPathToken = + protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_PATH_IN_CONDITION); writeDimensionPathToProto(mDimensionsInCondition, protoOutput); protoOutput->end(dimenPathToken); } @@ -210,7 +223,6 @@ void ValueMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs, (long long)(NanoToMillis(pair.second))); protoOutput->end(wrapperToken); } - mSkippedBuckets.clear(); for (const auto& pair : mPastBuckets) { const MetricDimensionKey& dimensionKey = pair.first; @@ -220,15 +232,15 @@ void ValueMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs, // First fill dimension. if (mSliceByPositionALL) { - uint64_t dimensionToken = protoOutput->start( - FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_WHAT); + uint64_t dimensionToken = + protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_WHAT); writeDimensionToProto(dimensionKey.getDimensionKeyInWhat(), str_set, protoOutput); protoOutput->end(dimensionToken); if (dimensionKey.hasDimensionKeyInCondition()) { - uint64_t dimensionInConditionToken = protoOutput->start( - FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_CONDITION); - writeDimensionToProto(dimensionKey.getDimensionKeyInCondition(), - str_set, protoOutput); + uint64_t dimensionInConditionToken = + protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_CONDITION); + writeDimensionToProto(dimensionKey.getDimensionKeyInCondition(), str_set, + protoOutput); protoOutput->end(dimensionInConditionToken); } } else { @@ -236,8 +248,8 @@ void ValueMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs, FIELD_ID_DIMENSION_LEAF_IN_WHAT, str_set, protoOutput); if (dimensionKey.hasDimensionKeyInCondition()) { writeDimensionLeafNodesToProto(dimensionKey.getDimensionKeyInCondition(), - FIELD_ID_DIMENSION_LEAF_IN_CONDITION, - str_set, protoOutput); + FIELD_ID_DIMENSION_LEAF_IN_CONDITION, str_set, + protoOutput); } } @@ -255,24 +267,43 @@ void ValueMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs, protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_BUCKET_NUM, (long long)(getBucketNumFromEndTimeNs(bucket.mBucketEndNs))); } - - protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_VALUE, (long long)bucket.mValue); + for (int i = 0; i < (int)bucket.valueIndex.size(); i ++) { + int index = bucket.valueIndex[i]; + const Value& value = bucket.values[i]; + uint64_t valueToken = protoOutput->start( + FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_VALUES); + protoOutput->write(FIELD_TYPE_INT32 | FIELD_ID_VALUE_INDEX, + index); + if (value.getType() == LONG) { + protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_VALUE_LONG, + (long long)value.long_value); + VLOG("\t bucket [%lld - %lld] value %d: %lld", (long long)bucket.mBucketStartNs, + (long long)bucket.mBucketEndNs, index, (long long)value.long_value); + } else if (value.getType() == DOUBLE) { + protoOutput->write(FIELD_TYPE_DOUBLE | FIELD_ID_VALUE_DOUBLE, + value.double_value); + VLOG("\t bucket [%lld - %lld] value %d: %.2f", (long long)bucket.mBucketStartNs, + (long long)bucket.mBucketEndNs, index, value.double_value); + } else { + VLOG("Wrong value type for ValueMetric output: %d", value.getType()); + } + protoOutput->end(valueToken); + } protoOutput->end(bucketInfoToken); - VLOG("\t bucket [%lld - %lld] count: %lld", (long long)bucket.mBucketStartNs, - (long long)bucket.mBucketEndNs, (long long)bucket.mValue); } protoOutput->end(wrapperToken); } protoOutput->end(protoToken); VLOG("metric %lld dump report now...", (long long)mMetricId); - mPastBuckets.clear(); + if (erase_data) { + mPastBuckets.clear(); + mSkippedBuckets.clear(); + } } void ValueMetricProducer::onConditionChangedLocked(const bool condition, const int64_t eventTimeNs) { - mCondition = condition; - if (eventTimeNs < mCurrentBucketStartTimeNs) { VLOG("Skip event due to late arrival: %lld vs %lld", (long long)eventTimeNs, (long long)mCurrentBucketStartTimeNs); @@ -281,44 +312,72 @@ void ValueMetricProducer::onConditionChangedLocked(const bool condition, flushIfNeededLocked(eventTimeNs); - if (mPullTagId != -1) { - vector<shared_ptr<LogEvent>> allData; - if (mStatsPullerManager->Pull(mPullTagId, eventTimeNs, &allData)) { - if (allData.size() == 0) { - return; + // Pull on condition changes. + if (mIsPulled && (mCondition != condition)) { + pullAndMatchEventsLocked(eventTimeNs); + } + + // when condition change from true to false, clear diff base + if (mUseDiff && mCondition && !condition) { + for (auto& slice : mCurrentSlicedBucket) { + for (auto& interval : slice.second) { + interval.hasBase = false; } - for (const auto& data : allData) { - onMatchedLogEventLocked(0, *data); + } + } + + mCondition = condition; +} + +void ValueMetricProducer::pullAndMatchEventsLocked(const int64_t timestampNs) { + vector<std::shared_ptr<LogEvent>> allData; + if (mPullerManager->Pull(mPullTagId, timestampNs, &allData)) { + if (allData.size() == 0) { + return; + } + for (const auto& data : allData) { + if (mEventMatcherWizard->matchLogEvent( + *data, mWhatMatcherIndex) == MatchingState::kMatched) { + onMatchedLogEventLocked(mWhatMatcherIndex, *data); } } - return; } } +int64_t ValueMetricProducer::calcPreviousBucketEndTime(const int64_t currentTimeNs) { + return mTimeBaseNs + ((currentTimeNs - mTimeBaseNs) / mBucketSizeNs) * mBucketSizeNs; +} + void ValueMetricProducer::onDataPulled(const std::vector<std::shared_ptr<LogEvent>>& allData) { std::lock_guard<std::mutex> lock(mMutex); - - if (mCondition == true || mConditionTrackerIndex < 0) { + if (mCondition) { if (allData.size() == 0) { + VLOG("Data pulled is empty"); return; } // For scheduled pulled data, the effective event time is snap to the nearest - // bucket boundary to make bucket finalize. + // bucket end. In the case of waking up from a deep sleep state, we will + // attribute to the previous bucket end. If the sleep was long but not very long, we + // will be in the immediate next bucket. Previous bucket may get a larger number as + // we pull at a later time than real bucket end. + // If the sleep was very long, we skip more than one bucket before sleep. In this case, + // if the diff base will be cleared and this new data will serve as new diff base. int64_t realEventTime = allData.at(0)->GetElapsedTimestampNs(); - int64_t eventTime = mTimeBaseNs + - ((realEventTime - mTimeBaseNs) / mBucketSizeNs) * mBucketSizeNs; - - mCondition = false; - for (const auto& data : allData) { - data->setElapsedTimestampNs(eventTime - 1); - onMatchedLogEventLocked(0, *data); + int64_t bucketEndTime = calcPreviousBucketEndTime(realEventTime) - 1; + if (bucketEndTime < mCurrentBucketStartTimeNs) { + VLOG("Skip bucket end pull due to late arrival: %lld vs %lld", (long long)bucketEndTime, + (long long)mCurrentBucketStartTimeNs); + return; } - - mCondition = true; for (const auto& data : allData) { - data->setElapsedTimestampNs(eventTime); - onMatchedLogEventLocked(0, *data); + if (mEventMatcherWizard->matchLogEvent(*data, mWhatMatcherIndex) == + MatchingState::kMatched) { + data->setElapsedTimestampNs(bucketEndTime); + onMatchedLogEventLocked(mWhatMatcherIndex, *data); + } } + } else { + VLOG("No need to commit data on condition false."); } } @@ -331,10 +390,12 @@ void ValueMetricProducer::dumpStatesLocked(FILE* out, bool verbose) const { (unsigned long)mCurrentSlicedBucket.size()); if (verbose) { for (const auto& it : mCurrentSlicedBucket) { - fprintf(out, "\t(what)%s\t(condition)%s (value)%lld\n", - it.first.getDimensionKeyInWhat().toString().c_str(), - it.first.getDimensionKeyInCondition().toString().c_str(), - (unsigned long long)it.second.sum); + for (const auto& interval : it.second) { + fprintf(out, "\t(what)%s\t(condition)%s (value)%s\n", + it.first.getDimensionKeyInWhat().toString().c_str(), + it.first.getDimensionKeyInCondition().toString().c_str(), + interval.value.toString().c_str()); + } } } } @@ -350,8 +411,8 @@ bool ValueMetricProducer::hitGuardRailLocked(const MetricDimensionKey& newKey) { StatsdStats::getInstance().noteMetricDimensionSize(mConfigKey, mMetricId, newTupleCount); // 2. Don't add more tuples, we are above the allowed threshold. Drop the data. if (newTupleCount > mDimensionHardLimit) { - ALOGE("ValueMetric %lld dropping data for dimension key %s", - (long long)mMetricId, newKey.toString().c_str()); + ALOGE("ValueMetric %lld dropping data for dimension key %s", (long long)mMetricId, + newKey.toString().c_str()); return true; } } @@ -359,10 +420,35 @@ bool ValueMetricProducer::hitGuardRailLocked(const MetricDimensionKey& newKey) { return false; } -void ValueMetricProducer::onMatchedLogEventInternalLocked( - const size_t matcherIndex, const MetricDimensionKey& eventKey, - const ConditionKey& conditionKey, bool condition, - const LogEvent& event) { +bool getDoubleOrLong(const LogEvent& event, const Matcher& matcher, Value& ret) { + for (const FieldValue& value : event.getValues()) { + if (value.mField.matches(matcher)) { + switch (value.mValue.type) { + case INT: + ret.setLong(value.mValue.int_value); + break; + case LONG: + ret.setLong(value.mValue.long_value); + break; + case FLOAT: + ret.setDouble(value.mValue.float_value); + break; + case DOUBLE: + ret.setDouble(value.mValue.double_value); + break; + default: + break; + } + return true; + } + } + return false; +} + +void ValueMetricProducer::onMatchedLogEventInternalLocked(const size_t matcherIndex, + const MetricDimensionKey& eventKey, + const ConditionKey& conditionKey, + bool condition, const LogEvent& event) { int64_t eventTimeNs = event.GetElapsedTimestampNs(); if (eventTimeNs < mCurrentBucketStartTimeNs) { VLOG("Skip event due to late arrival: %lld vs %lld", (long long)eventTimeNs, @@ -372,56 +458,101 @@ void ValueMetricProducer::onMatchedLogEventInternalLocked( flushIfNeededLocked(eventTimeNs); - if (hitGuardRailLocked(eventKey)) { + // For pulled data, we already check condition when we decide to pull or + // in onDataPulled. So take all of them. + // For pushed data, just check condition. + if (!(mIsPulled || condition)) { + VLOG("ValueMetric skip event because condition is false"); return; } - Interval& interval = mCurrentSlicedBucket[eventKey]; - int error = 0; - const int64_t value = event.GetLong(mField, &error); - if (error < 0) { + if (hitGuardRailLocked(eventKey)) { return; } + vector<Interval>& multiIntervals = mCurrentSlicedBucket[eventKey]; + if (multiIntervals.size() < mFieldMatchers.size()) { + VLOG("Resizing number of intervals to %d", (int)mFieldMatchers.size()); + multiIntervals.resize(mFieldMatchers.size()); + } - if (mPullTagId != -1) { // for pulled events - if (mCondition == true) { - if (!interval.startUpdated) { - interval.start = value; - interval.startUpdated = true; - } else { - // skip it if there is already value recorded for the start - VLOG("Already recorded value for this dimension %s", eventKey.toString().c_str()); + for (int i = 0; i < (int)mFieldMatchers.size(); i++) { + const Matcher& matcher = mFieldMatchers[i]; + Interval& interval = multiIntervals[i]; + interval.valueIndex = i; + Value value; + if (!getDoubleOrLong(event, matcher, value)) { + VLOG("Failed to get value %d from event %s", i, event.ToString().c_str()); + return; + } + + if (mUseDiff) { + // no base. just update base and return. + if (!interval.hasBase) { + interval.base = value; + interval.hasBase = true; + return; } - } else { - // Generally we expect value to be monotonically increasing. - // If not, take absolute value or drop it, based on config. - if (interval.startUpdated) { - if (value >= interval.start) { - interval.sum += (value - interval.start); - interval.hasValue = true; - } else { - if (mUseAbsoluteValueOnReset) { - interval.sum += value; - interval.hasValue = true; + Value diff; + switch (mValueDirection) { + case ValueMetric::INCREASING: + if (value >= interval.base) { + diff = value - interval.base; + } else if (mUseAbsoluteValueOnReset) { + diff = value; } else { - VLOG("Dropping data for atom %d, prev: %lld, now: %lld", mPullTagId, - (long long)interval.start, (long long)value); + VLOG("Unexpected decreasing value"); + StatsdStats::getInstance().notePullDataError(mPullTagId); + interval.base = value; + return; } - } - interval.startUpdated = false; - } else { - VLOG("No start for matching end %lld", (long long)value); - interval.tainted += 1; + break; + case ValueMetric::DECREASING: + if (interval.base >= value) { + diff = interval.base - value; + } else if (mUseAbsoluteValueOnReset) { + diff = value; + } else { + VLOG("Unexpected increasing value"); + StatsdStats::getInstance().notePullDataError(mPullTagId); + interval.base = value; + return; + } + break; + case ValueMetric::ANY: + diff = value - interval.base; + break; + default: + break; } + interval.base = value; + value = diff; } - } else { // for pushed events, only accumulate when condition is true - if (mCondition == true || mConditionTrackerIndex < 0) { - interval.sum += value; + + if (interval.hasValue) { + switch (mAggregationType) { + case ValueMetric::SUM: + // for AVG, we add up and take average when flushing the bucket + case ValueMetric::AVG: + interval.value += value; + break; + case ValueMetric::MIN: + interval.value = std::min(value, interval.value); + break; + case ValueMetric::MAX: + interval.value = std::max(value, interval.value); + break; + default: + break; + } + } else { + interval.value = value; interval.hasValue = true; } + interval.sampleSize += 1; } - long wholeBucketVal = interval.sum; + // TODO: propgate proper values down stream when anomaly support doubles + long wholeBucketVal = multiIntervals[0].value.long_value; auto prev = mCurrentFullBucket.find(eventKey); if (prev != mCurrentFullBucket.end()) { wholeBucketVal += prev->second; @@ -448,6 +579,12 @@ void ValueMetricProducer::flushIfNeededLocked(const int64_t& eventTimeNs) { if (numBucketsForward > 1) { VLOG("Skipping forward %lld buckets", (long long)numBucketsForward); + // take base again in future good bucket. + for (auto& slice : mCurrentSlicedBucket) { + for (auto& interval : slice.second) { + interval.hasBase = false; + } + } } VLOG("metric %lld: new bucket start time: %lld", (long long)mMetricId, (long long)mCurrentBucketStartTimeNs); @@ -458,37 +595,46 @@ void ValueMetricProducer::flushCurrentBucketLocked(const int64_t& eventTimeNs) { (int)mCurrentSlicedBucket.size()); int64_t fullBucketEndTimeNs = getCurrentBucketEndTimeNs(); - ValueBucket info; - info.mBucketStartNs = mCurrentBucketStartTimeNs; - if (eventTimeNs < fullBucketEndTimeNs) { - info.mBucketEndNs = eventTimeNs; - } else { - info.mBucketEndNs = fullBucketEndTimeNs; - } + int64_t bucketEndTime = eventTimeNs < fullBucketEndTimeNs ? eventTimeNs : fullBucketEndTimeNs; - if (info.mBucketEndNs - mCurrentBucketStartTimeNs >= mMinBucketSizeNs) { + if (bucketEndTime - mCurrentBucketStartTimeNs >= mMinBucketSizeNs) { // The current bucket is large enough to keep. - int tainted = 0; for (const auto& slice : mCurrentSlicedBucket) { - tainted += slice.second.tainted; - tainted += slice.second.startUpdated; - if (slice.second.hasValue) { - info.mValue = slice.second.sum; - // it will auto create new vector of ValuebucketInfo if the key is not found. + ValueBucket bucket; + bucket.mBucketStartNs = mCurrentBucketStartTimeNs; + bucket.mBucketEndNs = bucketEndTime; + for (const auto& interval : slice.second) { + if (interval.hasValue) { + // skip the output if the diff is zero + if (mSkipZeroDiffOutput && mUseDiff && interval.value.isZero()) { + continue; + } + bucket.valueIndex.push_back(interval.valueIndex); + if (mAggregationType != ValueMetric::AVG) { + bucket.values.push_back(interval.value); + } else { + double sum = interval.value.type == LONG ? (double)interval.value.long_value + : interval.value.double_value; + bucket.values.push_back(Value((double)sum / interval.sampleSize)); + } + } + } + // it will auto create new vector of ValuebucketInfo if the key is not found. + if (bucket.valueIndex.size() > 0) { auto& bucketList = mPastBuckets[slice.first]; - bucketList.push_back(info); + bucketList.push_back(bucket); } } - VLOG("%d tainted pairs in the bucket", tainted); } else { - mSkippedBuckets.emplace_back(info.mBucketStartNs, info.mBucketEndNs); + mSkippedBuckets.emplace_back(mCurrentBucketStartTimeNs, bucketEndTime); } if (eventTimeNs > fullBucketEndTimeNs) { // If full bucket, send to anomaly tracker. // Accumulate partial buckets with current value and then send to anomaly tracker. if (mCurrentFullBucket.size() > 0) { for (const auto& slice : mCurrentSlicedBucket) { - mCurrentFullBucket[slice.first] += slice.second.sum; + // TODO: fix this when anomaly can accept double values + mCurrentFullBucket[slice.first] += slice.second[0].value.long_value; } for (const auto& slice : mCurrentFullBucket) { for (auto& tracker : mAnomalyTrackers) { @@ -503,7 +649,9 @@ void ValueMetricProducer::flushCurrentBucketLocked(const int64_t& eventTimeNs) { for (const auto& slice : mCurrentSlicedBucket) { for (auto& tracker : mAnomalyTrackers) { if (tracker != nullptr) { - tracker->addPastBucket(slice.first, slice.second.sum, mCurrentBucketNum); + // TODO: fix this when anomaly can accept double values + tracker->addPastBucket(slice.first, slice.second[0].value.long_value, + mCurrentBucketNum); } } } @@ -511,12 +659,18 @@ void ValueMetricProducer::flushCurrentBucketLocked(const int64_t& eventTimeNs) { } else { // Accumulate partial bucket. for (const auto& slice : mCurrentSlicedBucket) { - mCurrentFullBucket[slice.first] += slice.second.sum; + // TODO: fix this when anomaly can accept double values + mCurrentFullBucket[slice.first] += slice.second[0].value.long_value; } } // Reset counters - mCurrentSlicedBucket.clear(); + for (auto& slice : mCurrentSlicedBucket) { + for (auto& interval : slice.second) { + interval.hasValue = false; + interval.sampleSize = 0; + } + } } size_t ValueMetricProducer::byteSizeLocked() const { diff --git a/cmds/statsd/src/metrics/ValueMetricProducer.h b/cmds/statsd/src/metrics/ValueMetricProducer.h index cb6b051cd484..c3912eebbc11 100644 --- a/cmds/statsd/src/metrics/ValueMetricProducer.h +++ b/cmds/statsd/src/metrics/ValueMetricProducer.h @@ -23,6 +23,8 @@ #include "../condition/ConditionTracker.h" #include "../external/PullDataReceiver.h" #include "../external/StatsPullerManager.h" +#include "../matchers/EventMatcherWizard.h" +#include "../stats_log_util.h" #include "MetricProducer.h" #include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" @@ -33,52 +35,33 @@ namespace statsd { struct ValueBucket { int64_t mBucketStartNs; int64_t mBucketEndNs; - int64_t mValue; + std::vector<int> valueIndex; + std::vector<Value> values; }; class ValueMetricProducer : public virtual MetricProducer, public virtual PullDataReceiver { public: ValueMetricProducer(const ConfigKey& key, const ValueMetric& valueMetric, - const int conditionIndex, const sp<ConditionWizard>& wizard, - const int pullTagId, const int64_t timeBaseNs, const int64_t startTimeNs); + const int conditionIndex, const sp<ConditionWizard>& conditionWizard, + const int whatMatcherIndex, + const sp<EventMatcherWizard>& matcherWizard, + const int pullTagId, const int64_t timeBaseNs, const int64_t startTimeNs, + const sp<StatsPullerManager>& pullerManager); virtual ~ValueMetricProducer(); + // Process data pulled on bucket boundary. void onDataPulled(const std::vector<std::shared_ptr<LogEvent>>& data) override; // ValueMetric needs special logic if it's a pulled atom. void notifyAppUpgrade(const int64_t& eventTimeNs, const string& apk, const int uid, const int64_t version) override { std::lock_guard<std::mutex> lock(mMutex); - - if (mPullTagId != -1 && (mCondition == true || mConditionTrackerIndex < 0) ) { - vector<shared_ptr<LogEvent>> allData; - mStatsPullerManager->Pull(mPullTagId, eventTimeNs, &allData); - if (allData.size() == 0) { - // This shouldn't happen since this valuemetric is not useful now. - } - - // Pretend the pulled data occurs right before the app upgrade event. - mCondition = false; - for (const auto& data : allData) { - data->setElapsedTimestampNs(eventTimeNs - 1); - onMatchedLogEventLocked(0, *data); - } - - flushCurrentBucketLocked(eventTimeNs); - mCurrentBucketStartTimeNs = eventTimeNs; - - mCondition = true; - for (const auto& data : allData) { - data->setElapsedTimestampNs(eventTimeNs); - onMatchedLogEventLocked(0, *data); - } - } else { - // For pushed value metric or pulled metric where condition is not true, - // we simply flush and reset the current bucket start. - flushCurrentBucketLocked(eventTimeNs); - mCurrentBucketStartTimeNs = eventTimeNs; + if (mIsPulled && mCondition) { + pullAndMatchEventsLocked(eventTimeNs - 1); } + flushCurrentBucketLocked(eventTimeNs); + mCurrentBucketStartTimeNs = eventTimeNs; }; protected: @@ -90,6 +73,7 @@ protected: private: void onDumpReportLocked(const int64_t dumpTimeNs, const bool include_current_partial_bucket, + const bool erase_data, std::set<string> *str_set, android::util::ProtoOutputStream* protoOutput) override; void clearPastBucketsLocked(const int64_t dumpTimeNs) override; @@ -112,44 +96,46 @@ private: void dropDataLocked(const int64_t dropTimeNs) override; - const FieldMatcher mValueField; + // Calculate previous bucket end time based on current time. + int64_t calcPreviousBucketEndTime(const int64_t currentTimeNs); - std::shared_ptr<StatsPullerManager> mStatsPullerManager; + const int mWhatMatcherIndex; - // for testing - ValueMetricProducer(const ConfigKey& key, const ValueMetric& valueMetric, - const int conditionIndex, const sp<ConditionWizard>& wizard, - const int pullTagId, const int64_t timeBaseNs, const int64_t startTimeNs, - std::shared_ptr<StatsPullerManager> statsPullerManager); + sp<EventMatcherWizard> mEventMatcherWizard; + + sp<StatsPullerManager> mPullerManager; + + // Value fields for matching. + std::vector<Matcher> mFieldMatchers; // tagId for pulled data. -1 if this is not pulled const int mPullTagId; - int mField; + // if this is pulled metric + const bool mIsPulled; - // internal state of a bucket. + // internal state of an ongoing aggregation bucket. typedef struct { - // Pulled data always come in pair of <start, end>. This holds the value - // for start. The diff (end - start) is added to sum. - int64_t start; - // Whether the start data point is updated - bool startUpdated; - // If end data point comes before the start, record this pair as tainted - // and the value is not added to the running sum. - int tainted; - // Running sum of known pairs in this bucket - int64_t sum; + // Index in multi value aggregation. + int valueIndex; + // Holds current base value of the dimension. Take diff and update if necessary. + Value base; + // Whether there is a base to diff to. + bool hasBase; + // Current value, depending on the aggregation type. + Value value; + // Number of samples collected. + int sampleSize; // If this dimension has any non-tainted value. If not, don't report the // dimension. bool hasValue; } Interval; - std::unordered_map<MetricDimensionKey, Interval> mCurrentSlicedBucket; + std::unordered_map<MetricDimensionKey, std::vector<Interval>> mCurrentSlicedBucket; std::unordered_map<MetricDimensionKey, int64_t> mCurrentFullBucket; // Save the past buckets and we can clear when the StatsLogReport is dumped. - // TODO: Add a lock to mPastBuckets. std::unordered_map<MetricDimensionKey, std::vector<ValueBucket>> mPastBuckets; // Pairs of (elapsed start, elapsed end) denoting buckets that were skipped. @@ -160,6 +146,8 @@ private: // Util function to check whether the specified dimension hits the guardrail. bool hitGuardRailLocked(const MetricDimensionKey& newKey); + void pullAndMatchEventsLocked(const int64_t timestampNs); + static const size_t kBucketSize = sizeof(ValueBucket{}); const size_t mDimensionSoftLimit; @@ -168,7 +156,16 @@ private: const bool mUseAbsoluteValueOnReset; - FRIEND_TEST(ValueMetricProducerTest, TestNonDimensionalEvents); + const ValueMetric::AggregationType mAggregationType; + + const bool mUseDiff; + + const ValueMetric::ValueDirection mValueDirection; + + const bool mSkipZeroDiffOutput; + + FRIEND_TEST(ValueMetricProducerTest, TestPulledEventsNoCondition); + FRIEND_TEST(ValueMetricProducerTest, TestPulledEventsWithFiltering); FRIEND_TEST(ValueMetricProducerTest, TestPulledEventsTakeAbsoluteValueOnReset); FRIEND_TEST(ValueMetricProducerTest, TestPulledEventsTakeZeroOnReset); FRIEND_TEST(ValueMetricProducerTest, TestEventsWithNonSlicedCondition); @@ -181,7 +178,13 @@ private: FRIEND_TEST(ValueMetricProducerTest, TestBucketBoundaryNoCondition); FRIEND_TEST(ValueMetricProducerTest, TestBucketBoundaryWithCondition); FRIEND_TEST(ValueMetricProducerTest, TestBucketBoundaryWithCondition2); - FRIEND_TEST(ValueMetricProducerTest, TestBucketBoundaryWithCondition3); + FRIEND_TEST(ValueMetricProducerTest, TestPushedAggregateMin); + FRIEND_TEST(ValueMetricProducerTest, TestPushedAggregateMax); + FRIEND_TEST(ValueMetricProducerTest, TestPushedAggregateAvg); + FRIEND_TEST(ValueMetricProducerTest, TestPushedAggregateSum); + FRIEND_TEST(ValueMetricProducerTest, TestFirstBucket); + FRIEND_TEST(ValueMetricProducerTest, TestCalcPreviousBucketEndTime); + FRIEND_TEST(ValueMetricProducerTest, TestSkipZeroDiffOutput); }; } // namespace statsd diff --git a/cmds/statsd/src/metrics/duration_helper/DurationTracker.h b/cmds/statsd/src/metrics/duration_helper/DurationTracker.h index 149b3189dfba..ccb1d4359e89 100644 --- a/cmds/statsd/src/metrics/duration_helper/DurationTracker.h +++ b/cmds/statsd/src/metrics/duration_helper/DurationTracker.h @@ -44,7 +44,6 @@ struct DurationInfo { int64_t lastStartTime; // existing duration in current bucket. int64_t lastDuration; - // TODO: Optimize the way we track sliced condition in duration metrics. // cache the HashableDimensionKeys we need to query the condition for this duration event. ConditionKey conditionKeys; diff --git a/cmds/statsd/src/metrics/duration_helper/OringDurationTracker.cpp b/cmds/statsd/src/metrics/duration_helper/OringDurationTracker.cpp index b833dfc79a22..956383a99eea 100644 --- a/cmds/statsd/src/metrics/duration_helper/OringDurationTracker.cpp +++ b/cmds/statsd/src/metrics/duration_helper/OringDurationTracker.cpp @@ -326,7 +326,6 @@ void OringDurationTracker::onConditionChanged(bool condition, const int64_t time int64_t OringDurationTracker::predictAnomalyTimestampNs( const DurationAnomalyTracker& anomalyTracker, const int64_t eventTimestampNs) const { - // TODO: Unit-test this and see if it can be done more efficiently (e.g. use int32). // The anomaly threshold. const int64_t thresholdNs = anomalyTracker.getAnomalyThreshold(); diff --git a/cmds/statsd/src/metrics/metrics_manager_util.cpp b/cmds/statsd/src/metrics/metrics_manager_util.cpp index 811a00e47ae5..47b037646325 100644 --- a/cmds/statsd/src/metrics/metrics_manager_util.cpp +++ b/cmds/statsd/src/metrics/metrics_manager_util.cpp @@ -25,6 +25,7 @@ #include "../external/StatsPullerManager.h" #include "../matchers/CombinationLogMatchingTracker.h" #include "../matchers/SimpleLogMatchingTracker.h" +#include "../matchers/EventMatcherWizard.h" #include "../metrics/CountMetricProducer.h" #include "../metrics/DurationMetricProducer.h" #include "../metrics/EventMetricProducer.h" @@ -82,6 +83,28 @@ bool handleMetricWithLogTrackers(const int64_t what, const int metricIndex, return true; } +bool handlePullMetricTriggerWithLogTrackers( + const int64_t trigger, const int metricIndex, + const vector<sp<LogMatchingTracker>>& allAtomMatchers, + const unordered_map<int64_t, int>& logTrackerMap, + unordered_map<int, std::vector<int>>& trackerToMetricMap, int& logTrackerIndex) { + auto logTrackerIt = logTrackerMap.find(trigger); + if (logTrackerIt == logTrackerMap.end()) { + ALOGW("cannot find the AtomMatcher \"%lld\" in config", (long long)trigger); + return false; + } + if (allAtomMatchers[logTrackerIt->second]->getAtomIds().size() > 1) { + ALOGE("AtomMatcher \"%lld\" has more than one tag ids." + "Trigger can only be one atom type.", + (long long)trigger); + return false; + } + logTrackerIndex = logTrackerIt->second; + auto& metric_list = trackerToMetricMap[logTrackerIndex]; + metric_list.push_back(metricIndex); + return true; +} + bool handleMetricWithConditions( const int64_t condition, const int metricIndex, const unordered_map<int64_t, int>& conditionTrackerMap, @@ -103,7 +126,6 @@ bool handleMetricWithConditions( } allConditionTrackers[condition_it->second]->setSliced(true); allConditionTrackers[it->second]->setSliced(true); - // TODO: We need to verify the link is valid. } conditionIndex = condition_it->second; @@ -169,7 +191,6 @@ bool initLogTrackers(const StatsdConfig& config, const UidMap& uidMap, bool isStateTracker(const SimplePredicate& simplePredicate, vector<Matcher>* primaryKeys) { // 1. must not have "stop". must have "dimension" if (!simplePredicate.has_stop() && simplePredicate.has_dimensions()) { - // TODO: need to check the start atom matcher too. auto it = android::util::AtomsInfo::kStateAtomsFieldOptions.find( simplePredicate.dimensions().field()); // 2. must be based on a state atom. @@ -262,9 +283,10 @@ bool initConditions(const ConfigKey& key, const StatsdConfig& config, return true; } -bool initMetrics(const ConfigKey& key, const StatsdConfig& config, - const int64_t timeBaseTimeNs, const int64_t currentTimeNs, - UidMap& uidMap, const unordered_map<int64_t, int>& logTrackerMap, +bool initMetrics(const ConfigKey& key, const StatsdConfig& config, const int64_t timeBaseTimeNs, + const int64_t currentTimeNs, UidMap& uidMap, + const sp<StatsPullerManager>& pullerManager, + const unordered_map<int64_t, int>& logTrackerMap, const unordered_map<int64_t, int>& conditionTrackerMap, const vector<sp<LogMatchingTracker>>& allAtomMatchers, vector<sp<ConditionTracker>>& allConditionTrackers, @@ -273,6 +295,7 @@ bool initMetrics(const ConfigKey& key, const StatsdConfig& config, unordered_map<int, std::vector<int>>& trackerToMetricMap, unordered_map<int64_t, int>& metricMap, std::set<int64_t>& noReportMetricIds) { sp<ConditionWizard> wizard = new ConditionWizard(allConditionTrackers); + sp<EventMatcherWizard> matcherWizard = new EventMatcherWizard(allAtomMatchers); const int allMetricsCount = config.count_metric_size() + config.duration_metric_size() + config.event_metric_size() + config.value_metric_size(); allMetricProducers.reserve(allMetricsCount); @@ -313,7 +336,7 @@ bool initMetrics(const ConfigKey& key, const StatsdConfig& config, } sp<MetricProducer> countProducer = - new CountMetricProducer(key, metric, conditionIndex, wizard, timeBaseTimeNs); + new CountMetricProducer(key, metric, conditionIndex, wizard, timeBaseTimeNs, currentTimeNs); allMetricProducers.push_back(countProducer); } @@ -383,7 +406,7 @@ bool initMetrics(const ConfigKey& key, const StatsdConfig& config, sp<MetricProducer> durationMetric = new DurationMetricProducer( key, metric, conditionIndex, trackerIndices[0], trackerIndices[1], - trackerIndices[2], nesting, wizard, internalDimensions, timeBaseTimeNs); + trackerIndices[2], nesting, wizard, internalDimensions, timeBaseTimeNs, currentTimeNs); allMetricProducers.push_back(durationMetric); } @@ -465,9 +488,9 @@ bool initMetrics(const ConfigKey& key, const StatsdConfig& config, } } - sp<MetricProducer> valueProducer = new ValueMetricProducer(key, metric, conditionIndex, - wizard, pullTagId, - timeBaseTimeNs, currentTimeNs); + sp<MetricProducer> valueProducer = new ValueMetricProducer( + key, metric, conditionIndex, wizard, trackerIndex, matcherWizard, pullTagId, + timeBaseTimeNs, currentTimeNs, pullerManager); allMetricProducers.push_back(valueProducer); } @@ -503,13 +526,29 @@ bool initMetrics(const ConfigKey& key, const StatsdConfig& config, } sp<LogMatchingTracker> atomMatcher = allAtomMatchers.at(trackerIndex); - // If it is pulled atom, it should be simple matcher with one tagId. + // For GaugeMetric atom, it should be simple matcher with one tagId. if (atomMatcher->getAtomIds().size() != 1) { return false; } int atomTagId = *(atomMatcher->getAtomIds().begin()); int pullTagId = statsPullerManager.PullerForMatcherExists(atomTagId) ? atomTagId : -1; + int triggerTrackerIndex; + int triggerAtomId = -1; + if (pullTagId != -1 && metric.has_trigger_event()) { + // event_trigger should be used with ALL_CONDITION_CHANGES + if (metric.sampling_type() != GaugeMetric::ALL_CONDITION_CHANGES) { + return false; + } + if (!handlePullMetricTriggerWithLogTrackers(metric.trigger_event(), metricIndex, + allAtomMatchers, logTrackerMap, + trackerToMetricMap, triggerTrackerIndex)) { + return false; + } + sp<LogMatchingTracker> triggerAtomMatcher = allAtomMatchers.at(triggerTrackerIndex); + triggerAtomId = *(triggerAtomMatcher->getAtomIds().begin()); + } + int conditionIndex = -1; if (metric.has_condition()) { bool good = handleMetricWithConditions( @@ -526,7 +565,9 @@ bool initMetrics(const ConfigKey& key, const StatsdConfig& config, } sp<MetricProducer> gaugeProducer = new GaugeMetricProducer( - key, metric, conditionIndex, wizard, pullTagId, timeBaseTimeNs, currentTimeNs); + key, metric, conditionIndex, wizard, + trackerIndex, matcherWizard, pullTagId, triggerAtomId, atomTagId, + timeBaseTimeNs, currentTimeNs, pullerManager); allMetricProducers.push_back(gaugeProducer); } for (int i = 0; i < config.no_report_metric_size(); ++i) { @@ -644,11 +685,49 @@ bool initAlarms(const StatsdConfig& config, const ConfigKey& key, return true; } +bool initMetricActivations(const ConfigKey& key, const StatsdConfig& config, + const int64_t currentTimeNs, + const unordered_map<int64_t, int> &logEventTrackerMap, + const unordered_map<int64_t, int> &metricProducerMap, + vector<sp<MetricProducer>>& allMetricProducers, + unordered_map<int, std::vector<int>>& activationAtomTrackerToMetricMap, + vector<int>& metricsWithActivation) { + for (int i = 0; i < config.metric_activation_size(); ++i) { + const MetricActivation& metric_activation = config.metric_activation(i); + auto itr = metricProducerMap.find(metric_activation.metric_id()); + if (itr == metricProducerMap.end()) { + ALOGE("Metric id not found in metric activation: %lld", + (long long)metric_activation.metric_id()); + return false; + } + const int metricTrackerIndex = itr->second; + if (metricTrackerIndex < 0 || metricTrackerIndex >= (int)allMetricProducers.size()) { + ALOGE("Invalid metric tracker index."); + return false; + } + metricsWithActivation.push_back(metricTrackerIndex); + for (int j = 0; j < metric_activation.event_activation_size(); ++j) { + const EventActivation& activation = metric_activation.event_activation(j); + auto logTrackerIt = logEventTrackerMap.find(activation.atom_matcher_id()); + if (logTrackerIt == logEventTrackerMap.end()) { + ALOGE("Atom matcher not found for event activation."); + return false; + } + const int atomMatcherIndex = logTrackerIt->second; + activationAtomTrackerToMetricMap[atomMatcherIndex].push_back( + metricTrackerIndex); + allMetricProducers[metricTrackerIndex]->addActivation( + atomMatcherIndex, activation.ttl_seconds()); + } + } + return true; +} + bool initStatsdConfig(const ConfigKey& key, const StatsdConfig& config, UidMap& uidMap, + const sp<StatsPullerManager>& pullerManager, const sp<AlarmMonitor>& anomalyAlarmMonitor, - const sp<AlarmMonitor>& periodicAlarmMonitor, - const int64_t timeBaseNs, const int64_t currentTimeNs, - set<int>& allTagIds, + const sp<AlarmMonitor>& periodicAlarmMonitor, const int64_t timeBaseNs, + const int64_t currentTimeNs, set<int>& allTagIds, vector<sp<LogMatchingTracker>>& allAtomMatchers, vector<sp<ConditionTracker>>& allConditionTrackers, vector<sp<MetricProducer>>& allMetricProducers, @@ -657,6 +736,8 @@ bool initStatsdConfig(const ConfigKey& key, const StatsdConfig& config, UidMap& unordered_map<int, std::vector<int>>& conditionToMetricMap, unordered_map<int, std::vector<int>>& trackerToMetricMap, unordered_map<int, std::vector<int>>& trackerToConditionMap, + unordered_map<int, std::vector<int>>& activationAtomTrackerToMetricMap, + vector<int>& metricsWithActivation, std::set<int64_t>& noReportMetricIds) { unordered_map<int64_t, int> logTrackerMap; unordered_map<int64_t, int> conditionTrackerMap; @@ -674,9 +755,8 @@ bool initStatsdConfig(const ConfigKey& key, const StatsdConfig& config, UidMap& return false; } - if (!initMetrics(key, config, timeBaseNs, currentTimeNs, uidMap, - logTrackerMap, conditionTrackerMap, - allAtomMatchers, allConditionTrackers, allMetricProducers, + if (!initMetrics(key, config, timeBaseNs, currentTimeNs, uidMap, pullerManager, logTrackerMap, + conditionTrackerMap, allAtomMatchers, allConditionTrackers, allMetricProducers, conditionToMetricMap, trackerToMetricMap, metricProducerMap, noReportMetricIds)) { ALOGE("initMetricProducers failed"); @@ -692,6 +772,11 @@ bool initStatsdConfig(const ConfigKey& key, const StatsdConfig& config, UidMap& ALOGE("initAlarms failed"); return false; } + if (!initMetricActivations(key, config, currentTimeNs, logTrackerMap, metricProducerMap, + allMetricProducers, activationAtomTrackerToMetricMap, metricsWithActivation)) { + ALOGE("initMetricActivations failed"); + return false; + } return true; } diff --git a/cmds/statsd/src/metrics/metrics_manager_util.h b/cmds/statsd/src/metrics/metrics_manager_util.h index d749bf43c9be..9ffcedae4962 100644 --- a/cmds/statsd/src/metrics/metrics_manager_util.h +++ b/cmds/statsd/src/metrics/metrics_manager_util.h @@ -23,7 +23,7 @@ #include "../anomaly/AlarmTracker.h" #include "../condition/ConditionTracker.h" -#include "../external/StatsPullerManagerImpl.h" +#include "../external/StatsPullerManager.h" #include "../matchers/LogMatchingTracker.h" #include "../metrics/MetricProducer.h" @@ -81,9 +81,8 @@ bool initConditions(const ConfigKey& key, const StatsdConfig& config, // the list of MetricProducer index // [trackerToMetricMap]: contains the mapping from log tracker to MetricProducer index. bool initMetrics( - const ConfigKey& key, const StatsdConfig& config, - const int64_t timeBaseTimeNs, const int64_t currentTimeNs, - UidMap& uidMap, + const ConfigKey& key, const StatsdConfig& config, const int64_t timeBaseTimeNs, + const int64_t currentTimeNs, UidMap& uidMap, const sp<StatsPullerManager>& pullerManager, const std::unordered_map<int64_t, int>& logTrackerMap, const std::unordered_map<int64_t, int>& conditionTrackerMap, const std::unordered_map<int, std::vector<MetricConditionLink>>& eventConditionLinks, @@ -97,10 +96,10 @@ bool initMetrics( // Initialize MetricsManager from StatsdConfig. // Parameters are the members of MetricsManager. See MetricsManager for declaration. bool initStatsdConfig(const ConfigKey& key, const StatsdConfig& config, UidMap& uidMap, + const sp<StatsPullerManager>& pullerManager, const sp<AlarmMonitor>& anomalyAlarmMonitor, - const sp<AlarmMonitor>& periodicAlarmMonitor, - const int64_t timeBaseNs, const int64_t currentTimeNs, - std::set<int>& allTagIds, + const sp<AlarmMonitor>& periodicAlarmMonitor, const int64_t timeBaseNs, + const int64_t currentTimeNs, std::set<int>& allTagIds, std::vector<sp<LogMatchingTracker>>& allAtomMatchers, std::vector<sp<ConditionTracker>>& allConditionTrackers, std::vector<sp<MetricProducer>>& allMetricProducers, @@ -109,6 +108,8 @@ bool initStatsdConfig(const ConfigKey& key, const StatsdConfig& config, UidMap& std::unordered_map<int, std::vector<int>>& conditionToMetricMap, std::unordered_map<int, std::vector<int>>& trackerToMetricMap, std::unordered_map<int, std::vector<int>>& trackerToConditionMap, + unordered_map<int, std::vector<int>>& lifeSpanEventTrackerToMetricMap, + vector<int>& metricsWithLifeSpan, std::set<int64_t>& noReportMetricIds); bool isStateTracker(const SimplePredicate& simplePredicate, std::vector<Matcher>* primaryKeys); diff --git a/cmds/statsd/src/packages/UidMap.cpp b/cmds/statsd/src/packages/UidMap.cpp index 73ac9686889e..37a00673959c 100644 --- a/cmds/statsd/src/packages/UidMap.cpp +++ b/cmds/statsd/src/packages/UidMap.cpp @@ -272,7 +272,7 @@ void UidMap::assignIsolatedUid(int isolatedUid, int parentUid) { mIsolatedUidMap[isolatedUid] = parentUid; } -void UidMap::removeIsolatedUid(int isolatedUid, int parentUid) { +void UidMap::removeIsolatedUid(int isolatedUid) { lock_guard<mutex> lock(mIsolatedMutex); auto it = mIsolatedUidMap.find(isolatedUid); @@ -386,12 +386,12 @@ void UidMap::appendUidMap(const int64_t& timestamp, const ConfigKey& key, StatsdStats::getInstance().setUidMapChanges(mChanges.size()); } -void UidMap::printUidMap(FILE* out) const { +void UidMap::printUidMap(int out) const { lock_guard<mutex> lock(mMutex); for (const auto& kv : mMap) { if (!kv.second.deleted) { - fprintf(out, "%s, v%" PRId64 " (%i)\n", kv.first.second.c_str(), kv.second.versionCode, + dprintf(out, "%s, v%" PRId64 " (%i)\n", kv.first.second.c_str(), kv.second.versionCode, kv.first.first); } } diff --git a/cmds/statsd/src/packages/UidMap.h b/cmds/statsd/src/packages/UidMap.h index 5e42cd18de32..4598369f1222 100644 --- a/cmds/statsd/src/packages/UidMap.h +++ b/cmds/statsd/src/packages/UidMap.h @@ -103,7 +103,7 @@ public: // Helper for debugging contents of this uid map. Can be triggered with: // adb shell cmd stats print-uid-map - void printUidMap(FILE* out) const; + void printUidMap(int outFd) const; // Commands for indicating to the map that a producer should be notified if an app is updated. // This allows the metric producer to distinguish when the same uid or app represents a @@ -119,7 +119,7 @@ public: void OnConfigRemoved(const ConfigKey& key); void assignIsolatedUid(int isolatedUid, int parentUid); - void removeIsolatedUid(int isolatedUid, int parentUid); + void removeIsolatedUid(int isolatedUid); // Returns the host uid if it exists. Otherwise, returns the same uid that was passed-in. virtual int getHostUidOrSelf(int uid) const; @@ -146,7 +146,6 @@ private: void getListenerListCopyLocked(std::vector<wp<PackageInfoListener>>* output); - // TODO: Use shared_mutex for improved read-locking if a library can be found in Android. mutable mutex mMutex; mutable mutex mIsolatedMutex; diff --git a/cmds/statsd/src/perfetto/perfetto_config.proto b/cmds/statsd/src/perfetto/perfetto_config.proto deleted file mode 100644 index 56d12f8d81d4..000000000000 --- a/cmds/statsd/src/perfetto/perfetto_config.proto +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (C) 2017 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -syntax = "proto2"; - -package perfetto.protos; - -message DataSourceConfig { - message FtraceConfig { - repeated string event_names = 1; - } - - optional string name = 1; - - optional uint32 target_buffer = 2; - - optional FtraceConfig ftrace_config = 100; -} - -message TraceConfig { - message BufferConfig { - optional uint32 size_kb = 1; - - enum OptimizeFor { - DEFAULT = 0; - ONE_SHOT_READ = 1; - - } - optional OptimizeFor optimize_for = 3; - - enum FillPolicy { - UNSPECIFIED = 0; - RING_BUFFER = 1; - } - optional FillPolicy fill_policy = 4; - } - repeated BufferConfig buffers = 1; - - message DataSource { - optional protos.DataSourceConfig config = 1; - - repeated string producer_name_filter = 2; - } - repeated DataSource data_sources = 2; - - optional uint32 duration_ms = 3; -} diff --git a/cmds/statsd/src/shell/ShellSubscriber.cpp b/cmds/statsd/src/shell/ShellSubscriber.cpp new file mode 100644 index 000000000000..dffff7a96269 --- /dev/null +++ b/cmds/statsd/src/shell/ShellSubscriber.cpp @@ -0,0 +1,228 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#define DEBUG true // STOPSHIP if true +#include "Log.h" + +#include "ShellSubscriber.h" + +#include <android-base/file.h> +#include "matchers/matcher_util.h" +#include "stats_log_util.h" + +using android::util::ProtoOutputStream; + +namespace android { +namespace os { +namespace statsd { + +const static int FIELD_ID_ATOM = 1; + +void ShellSubscriber::startNewSubscription(int in, int out, sp<IResultReceiver> resultReceiver) { + VLOG("start new shell subscription"); + { + std::lock_guard<std::mutex> lock(mMutex); + if (mResultReceiver != nullptr) { + VLOG("Only one shell subscriber is allowed."); + return; + } + mInput = in; + mOutput = out; + mResultReceiver = resultReceiver; + IInterface::asBinder(mResultReceiver)->linkToDeath(this); + } + + // Note that the following is blocking, and it's intended as we cannot return until the shell + // cmd exits, otherwise all resources & FDs will be automatically closed. + + // Read config forever until EOF is reached. Clients may send multiple configs -- each new + // config replace the previous one. + readConfig(in); + + // Now we have read an EOF we now wait for the semaphore until the client exits. + VLOG("Now wait for client to exit"); + std::unique_lock<std::mutex> lk(mMutex); + mShellDied.wait(lk, [this, resultReceiver] { return mResultReceiver != resultReceiver; }); +} + +void ShellSubscriber::updateConfig(const ShellSubscription& config) { + std::lock_guard<std::mutex> lock(mMutex); + mPushedMatchers.clear(); + mPulledInfo.clear(); + + for (const auto& pushed : config.pushed()) { + mPushedMatchers.push_back(pushed); + VLOG("adding matcher for atom %d", pushed.atom_id()); + } + + int64_t token = getElapsedRealtimeNs(); + mPullToken = token; + + int64_t minInterval = -1; + for (const auto& pulled : config.pulled()) { + // All intervals need to be multiples of the min interval. + if (minInterval < 0 || pulled.freq_millis() < minInterval) { + minInterval = pulled.freq_millis(); + } + + mPulledInfo.emplace_back(pulled.matcher(), pulled.freq_millis()); + VLOG("adding matcher for pulled atom %d", pulled.matcher().atom_id()); + } + + if (mPulledInfo.size() > 0 && minInterval > 0) { + // This thread is guaranteed to terminate after it detects the token is different or + // cleaned up. + std::thread puller([token, minInterval, this] { startPull(token, minInterval); }); + puller.detach(); + } +} + +void ShellSubscriber::writeToOutputLocked(const vector<std::shared_ptr<LogEvent>>& data, + const SimpleAtomMatcher& matcher) { + if (mOutput == 0) return; + int count = 0; + mProto.clear(); + for (const auto& event : data) { + VLOG("%s", event->ToString().c_str()); + if (matchesSimple(*mUidMap, matcher, *event)) { + VLOG("matched"); + count++; + uint64_t atomToken = mProto.start(util::FIELD_TYPE_MESSAGE | + util::FIELD_COUNT_REPEATED | FIELD_ID_ATOM); + event->ToProto(mProto); + mProto.end(atomToken); + } + } + + if (count > 0) { + // First write the payload size. + size_t bufferSize = mProto.size(); + write(mOutput, &bufferSize, sizeof(bufferSize)); + VLOG("%d atoms, proto size: %zu", count, bufferSize); + // Then write the payload. + mProto.flush(mOutput); + } + mProto.clear(); +} + +void ShellSubscriber::startPull(int64_t token, int64_t intervalMillis) { + while (1) { + int64_t nowMillis = getElapsedRealtimeMillis(); + { + std::lock_guard<std::mutex> lock(mMutex); + if (mPulledInfo.size() == 0 || mPullToken != token) { + VLOG("Pulling thread %lld done!", (long long)token); + return; + } + for (auto& pullInfo : mPulledInfo) { + if (pullInfo.mPrevPullElapsedRealtimeMs + pullInfo.mInterval < nowMillis) { + VLOG("pull atom %d now", pullInfo.mPullerMatcher.atom_id()); + + vector<std::shared_ptr<LogEvent>> data; + mPullerMgr->Pull(pullInfo.mPullerMatcher.atom_id(), nowMillis * 1000000L, + &data); + VLOG("pulled %zu atoms", data.size()); + if (data.size() > 0) { + writeToOutputLocked(data, pullInfo.mPullerMatcher); + } + pullInfo.mPrevPullElapsedRealtimeMs = nowMillis; + } + } + } + VLOG("Pulling thread %lld sleep....", (long long)token); + std::this_thread::sleep_for(std::chrono::milliseconds(intervalMillis)); + } +} + +void ShellSubscriber::readConfig(int in) { + if (in <= 0) { + return; + } + + while (1) { + size_t bufferSize = 0; + int result = 0; + if ((result = read(in, &bufferSize, sizeof(bufferSize))) == 0) { + VLOG("Done reading"); + break; + } else if (result < 0 || result != sizeof(bufferSize)) { + ALOGE("Error reading config size"); + break; + } + + vector<uint8_t> buffer(bufferSize); + if ((result = read(in, buffer.data(), bufferSize)) > 0 && ((size_t)result) == bufferSize) { + ShellSubscription config; + if (config.ParseFromArray(buffer.data(), bufferSize)) { + updateConfig(config); + } else { + ALOGE("error parsing the config"); + break; + } + } else { + VLOG("Error reading the config, returned: %d, expecting %zu", result, bufferSize); + break; + } + } +} + +void ShellSubscriber::cleanUpLocked() { + // The file descriptors will be closed by binder. + mInput = 0; + mOutput = 0; + mResultReceiver = nullptr; + mPushedMatchers.clear(); + mPulledInfo.clear(); + mPullToken = 0; + VLOG("done clean up"); +} + +void ShellSubscriber::onLogEvent(const LogEvent& event) { + std::lock_guard<std::mutex> lock(mMutex); + + if (mOutput <= 0) { + return; + } + for (const auto& matcher : mPushedMatchers) { + if (matchesSimple(*mUidMap, matcher, event)) { + VLOG("%s", event.ToString().c_str()); + uint64_t atomToken = mProto.start(util::FIELD_TYPE_MESSAGE | + util::FIELD_COUNT_REPEATED | FIELD_ID_ATOM); + event.ToProto(mProto); + mProto.end(atomToken); + // First write the payload size. + size_t bufferSize = mProto.size(); + write(mOutput, &bufferSize, sizeof(bufferSize)); + + // Then write the payload. + mProto.flush(mOutput); + mProto.clear(); + break; + } + } +} + +void ShellSubscriber::binderDied(const wp<IBinder>& who) { + { + VLOG("Shell exits"); + std::lock_guard<std::mutex> lock(mMutex); + cleanUpLocked(); + } + mShellDied.notify_all(); +} + +} // namespace statsd +} // namespace os +} // namespace android diff --git a/cmds/statsd/src/shell/ShellSubscriber.h b/cmds/statsd/src/shell/ShellSubscriber.h new file mode 100644 index 000000000000..5401f31ce68c --- /dev/null +++ b/cmds/statsd/src/shell/ShellSubscriber.h @@ -0,0 +1,119 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "logd/LogEvent.h" + +#include <android/util/ProtoOutputStream.h> +#include <binder/IResultReceiver.h> +#include <condition_variable> +#include <mutex> +#include <string> +#include <thread> +#include "external/StatsPullerManager.h" +#include "frameworks/base/cmds/statsd/src/shell/shell_config.pb.h" +#include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" +#include "packages/UidMap.h" + +namespace android { +namespace os { +namespace statsd { + +/** + * Handles atoms subscription via shell cmd. + * + * A shell subscription lasts *until shell exits*. Unlike config based clients, a shell client + * communicates with statsd via file descriptors. They can subscribe pushed and pulled atoms. + * The atoms are sent back to the client in real time, as opposed to + * keeping the data in memory. Shell clients do not subscribe aggregated metrics, as they are + * responsible for doing the aggregation after receiving the atom events. + * + * Shell client pass ShellSubscription in the proto binary format. Client can update the + * subscription by sending a new subscription. The new subscription would replace the old one. + * Input data stream format is: + * + * |size_t|subscription proto|size_t|subscription proto|.... + * + * statsd sends the events back in Atom proto binary format. Each Atom message is preceded + * with sizeof(size_t) bytes indicating the size of the proto message payload. + * + * The stream would be in the following format: + * |size_t|shellData proto|size_t|shellData proto|.... + * + * Only one shell subscriber allowed at a time, because each shell subscriber blocks one thread + * until it exits. + */ +class ShellSubscriber : public virtual IBinder::DeathRecipient { +public: + ShellSubscriber(sp<UidMap> uidMap, sp<StatsPullerManager> pullerMgr) + : mUidMap(uidMap), mPullerMgr(pullerMgr){}; + + /** + * Start a new subscription. + */ + void startNewSubscription(int inFd, int outFd, sp<IResultReceiver> resultReceiver); + + void binderDied(const wp<IBinder>& who); + + void onLogEvent(const LogEvent& event); + +private: + struct PullInfo { + PullInfo(const SimpleAtomMatcher& matcher, int64_t interval) + : mPullerMatcher(matcher), mInterval(interval), mPrevPullElapsedRealtimeMs(0) { + } + SimpleAtomMatcher mPullerMatcher; + int64_t mInterval; + int64_t mPrevPullElapsedRealtimeMs; + }; + void readConfig(int in); + + void updateConfig(const ShellSubscription& config); + + void startPull(int64_t token, int64_t intervalMillis); + + void cleanUpLocked(); + + void writeToOutputLocked(const vector<std::shared_ptr<LogEvent>>& data, + const SimpleAtomMatcher& matcher); + + sp<UidMap> mUidMap; + + sp<StatsPullerManager> mPullerMgr; + + android::util::ProtoOutputStream mProto; + + mutable std::mutex mMutex; + + std::condition_variable mShellDied; // semaphore for waiting until shell exits. + + int mInput; // The input file descriptor + + int mOutput; // The output file descriptor + + sp<IResultReceiver> mResultReceiver; + + std::vector<SimpleAtomMatcher> mPushedMatchers; + + std::vector<PullInfo> mPulledInfo; + + int64_t mPullToken = 0; // A unique token to identify a puller thread. +}; + +} // namespace statsd +} // namespace os +} // namespace android diff --git a/cmds/statsd/src/shell/shell_config.proto b/cmds/statsd/src/shell/shell_config.proto new file mode 100644 index 000000000000..73cb49a61821 --- /dev/null +++ b/cmds/statsd/src/shell/shell_config.proto @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto2"; + +package android.os.statsd; + +option java_package = "com.android.os"; +option java_outer_classname = "ShellConfig"; + +import "frameworks/base/cmds/statsd/src/statsd_config.proto"; + +message PulledAtomSubscription { + optional SimpleAtomMatcher matcher = 1; + + /* gap between two pulls in milliseconds */ + optional int32 freq_millis = 2; +} + +message ShellSubscription { + repeated SimpleAtomMatcher pushed = 1; + repeated PulledAtomSubscription pulled = 2; +}
\ No newline at end of file diff --git a/cmds/statsd/src/shell/shell_data.proto b/cmds/statsd/src/shell/shell_data.proto new file mode 100644 index 000000000000..236bdbdd31f6 --- /dev/null +++ b/cmds/statsd/src/shell/shell_data.proto @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto2"; + +package android.os.statsd; + +option java_package = "com.android.os.statsd"; +option java_outer_classname = "ShellDataProto"; + +import "frameworks/base/cmds/statsd/src/atoms.proto"; + +// The output of shell subscription, including both pulled and pushed subscriptions. +message ShellData { + repeated Atom atom = 1; +} diff --git a/cmds/statsd/src/socket/StatsSocketListener.cpp b/cmds/statsd/src/socket/StatsSocketListener.cpp index 0392d6756292..6bb8cda07281 100755 --- a/cmds/statsd/src/socket/StatsSocketListener.cpp +++ b/cmds/statsd/src/socket/StatsSocketListener.cpp @@ -99,6 +99,24 @@ bool StatsSocketListener::onDataAvailable(SocketClient* cli) { char* ptr = ((char*)buffer) + sizeof(android_log_header_t); n -= sizeof(android_log_header_t); + // When a log failed to write to statsd socket (e.g., due ot EBUSY), a special message would + // be sent to statsd when the socket communication becomes available again. + // The format is android_log_event_int_t with a single integer in the payload indicating the + // number of logs that failed. (*FORMAT MUST BE IN SYNC WITH system/core/libstats*) + // Note that all normal stats logs are in the format of event_list, so there won't be confusion. + // + // TODO(b/80538532): In addition to log it in StatsdStats, we should properly reset the config. + if (n == sizeof(android_log_event_int_t)) { + android_log_event_int_t* int_event = reinterpret_cast<android_log_event_int_t*>(ptr); + if (int_event->payload.type == EVENT_TYPE_INT) { + ALOGE("Found dropped events: %d error %d", int_event->payload.data, + int_event->header.tag); + StatsdStats::getInstance().noteLogLost((int32_t)getWallClockSec(), + int_event->payload.data, int_event->header.tag); + return true; + } + } + log_msg msg; msg.entry.len = n; @@ -111,7 +129,7 @@ bool StatsSocketListener::onDataAvailable(SocketClient* cli) { LogEvent event(msg); // Call the listener - mListener->OnLogEvent(&event, false /*reconnected, N/A in statsd socket*/); + mListener->OnLogEvent(&event); return true; } diff --git a/cmds/statsd/src/stats_log.proto b/cmds/statsd/src/stats_log.proto index 1c70d88f6cf3..5d0f3d1db8c9 100644 --- a/cmds/statsd/src/stats_log.proto +++ b/cmds/statsd/src/stats_log.proto @@ -46,7 +46,7 @@ message EventMetricData { optional Atom atom = 2; - optional int64 wall_clock_timestamp_nanos = 3; + optional int64 wall_clock_timestamp_nanos = 3 [deprecated = true]; } message CountBucketInfo { @@ -106,7 +106,23 @@ message ValueBucketInfo { optional int64 end_bucket_elapsed_nanos = 2; - optional int64 value = 3; + optional int64 value = 3 [deprecated = true]; + + oneof single_value { + int64 value_long = 7 [deprecated = true]; + + double value_double = 8 [deprecated = true]; + } + + message Value { + optional int32 index = 1; + oneof value { + int64 value_long = 2; + double value_double = 3; + } + } + + repeated Value values = 9; optional int64 bucket_num = 4; @@ -136,7 +152,7 @@ message GaugeBucketInfo { repeated int64 elapsed_timestamp_nanos = 4; - repeated int64 wall_clock_timestamp_nanos = 5; + repeated int64 wall_clock_timestamp_nanos = 5 [deprecated = true]; optional int64 bucket_num = 6; @@ -322,6 +338,7 @@ message StatsdStatsReport { optional bool is_valid = 9; repeated int32 broadcast_sent_time_sec = 10; repeated int32 data_drop_time_sec = 11; + repeated int64 data_drop_bytes = 21; repeated int32 dump_report_time_sec = 12; repeated int32 dump_report_data_size = 20; repeated MatcherStats matcher_stats = 13; @@ -363,6 +380,11 @@ message StatsdStatsReport { optional int64 total_pull = 2; optional int64 total_pull_from_cache = 3; optional int64 min_pull_interval_sec = 4; + optional int64 average_pull_time_nanos = 5; + optional int64 max_pull_time_nanos = 6; + optional int64 average_pull_delay_nanos = 7; + optional int64 max_pull_delay_nanos = 8; + optional int64 data_error = 9; } repeated PulledAtomStats pulled_atom_stats = 10; @@ -386,4 +408,11 @@ message StatsdStatsReport { repeated int64 log_loss_stats = 14; repeated int32 system_restart_sec = 15; + + message LogLossStats { + optional int32 detected_time_sec = 1; + optional int32 count = 2; + optional int32 last_error = 3; + } + repeated LogLossStats detected_log_loss = 16; } diff --git a/cmds/statsd/src/stats_log_util.cpp b/cmds/statsd/src/stats_log_util.cpp index a0ab3e46e719..504c5864f2ec 100644 --- a/cmds/statsd/src/stats_log_util.cpp +++ b/cmds/statsd/src/stats_log_util.cpp @@ -25,15 +25,16 @@ #include <utils/Log.h> #include <utils/SystemClock.h> +using android::util::AtomsInfo; using android::util::FIELD_COUNT_REPEATED; using android::util::FIELD_TYPE_BOOL; +using android::util::FIELD_TYPE_FIXED64; using android::util::FIELD_TYPE_FLOAT; using android::util::FIELD_TYPE_INT32; using android::util::FIELD_TYPE_INT64; -using android::util::FIELD_TYPE_UINT64; -using android::util::FIELD_TYPE_FIXED64; using android::util::FIELD_TYPE_MESSAGE; using android::util::FIELD_TYPE_STRING; +using android::util::FIELD_TYPE_UINT64; using android::util::ProtoOutputStream; namespace android { @@ -58,6 +59,11 @@ const int FIELD_ID_PULL_ATOM_ID = 1; const int FIELD_ID_TOTAL_PULL = 2; const int FIELD_ID_TOTAL_PULL_FROM_CACHE = 3; const int FIELD_ID_MIN_PULL_INTERVAL_SEC = 4; +const int FIELD_ID_AVERAGE_PULL_TIME_NANOS = 5; +const int FIELD_ID_MAX_PULL_TIME_NANOS = 6; +const int FIELD_ID_AVERAGE_PULL_DELAY_NANOS = 7; +const int FIELD_ID_MAX_PULL_DELAY_NANOS = 8; +const int FIELD_ID_DATA_ERROR = 9; namespace { @@ -294,8 +300,9 @@ void writeDimensionPathToProto(const std::vector<Matcher>& fieldMatchers, // } // // -void writeFieldValueTreeToStreamHelper(const std::vector<FieldValue>& dims, size_t* index, - int depth, int prefix, ProtoOutputStream* protoOutput) { +void writeFieldValueTreeToStreamHelper(int tagId, const std::vector<FieldValue>& dims, + size_t* index, int depth, int prefix, + ProtoOutputStream* protoOutput) { size_t count = dims.size(); while (*index < count) { const auto& dim = dims[*index]; @@ -319,8 +326,35 @@ void writeFieldValueTreeToStreamHelper(const std::vector<FieldValue>& dims, size case FLOAT: protoOutput->write(FIELD_TYPE_FLOAT | fieldNum, dim.mValue.float_value); break; - case STRING: - protoOutput->write(FIELD_TYPE_STRING | fieldNum, dim.mValue.str_value); + case STRING: { + bool isBytesField = false; + // Bytes field is logged via string format in log_msg format. So here we check + // if this string field is a byte field. + std::map<int, std::vector<int>>::const_iterator itr; + if (depth == 0 && (itr = AtomsInfo::kBytesFieldAtoms.find(tagId)) != + AtomsInfo::kBytesFieldAtoms.end()) { + const std::vector<int>& bytesFields = itr->second; + for (int bytesField : bytesFields) { + if (bytesField == fieldNum) { + // This is a bytes field + isBytesField = true; + break; + } + } + } + if (isBytesField) { + protoOutput->write(FIELD_TYPE_MESSAGE | fieldNum, + (const char*)dim.mValue.str_value.c_str(), + dim.mValue.str_value.length()); + } else { + protoOutput->write(FIELD_TYPE_STRING | fieldNum, dim.mValue.str_value); + } + break; + } + case STORAGE: + protoOutput->write(FIELD_TYPE_MESSAGE | fieldNum, + (const char*)dim.mValue.storage_value.data(), + dim.mValue.storage_value.size()); break; default: break; @@ -337,7 +371,7 @@ void writeFieldValueTreeToStreamHelper(const std::vector<FieldValue>& dims, size } // Directly jump to the leaf value because the repeated position field is implied // by the position of the sub msg in the parent field. - writeFieldValueTreeToStreamHelper(dims, index, valueDepth, + writeFieldValueTreeToStreamHelper(tagId, dims, index, valueDepth, dim.mField.getPrefix(valueDepth), protoOutput); if (msg_token != 0) { protoOutput->end(msg_token); @@ -354,7 +388,7 @@ void writeFieldValueTreeToStream(int tagId, const std::vector<FieldValue>& value uint64_t atomToken = protoOutput->start(FIELD_TYPE_MESSAGE | tagId); size_t index = 0; - writeFieldValueTreeToStreamHelper(values, &index, 0, 0, protoOutput); + writeFieldValueTreeToStreamHelper(tagId, values, &index, 0, 0, protoOutput); protoOutput->end(atomToken); } @@ -405,6 +439,15 @@ void writePullerStatsToStream(const std::pair<int, StatsdStats::PulledAtomStats> (long long)pair.second.totalPullFromCache); protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_MIN_PULL_INTERVAL_SEC, (long long)pair.second.minPullIntervalSec); + protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_AVERAGE_PULL_TIME_NANOS, + (long long)pair.second.avgPullTimeNs); + protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_MAX_PULL_TIME_NANOS, + (long long)pair.second.maxPullTimeNs); + protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_AVERAGE_PULL_DELAY_NANOS, + (long long)pair.second.avgPullDelayNs); + protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_MAX_PULL_DELAY_NANOS, + (long long)pair.second.maxPullDelayNs); + protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_DATA_ERROR, (long long)pair.second.dataError); protoOutput->end(token); } diff --git a/cmds/statsd/src/stats_log_util.h b/cmds/statsd/src/stats_log_util.h index b8f6850ddc29..61f31eb3fa17 100644 --- a/cmds/statsd/src/stats_log_util.h +++ b/cmds/statsd/src/stats_log_util.h @@ -21,6 +21,7 @@ #include "HashableDimensionKey.h" #include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" #include "guardrail/StatsdStats.h" +#include "statslog.h" namespace android { namespace os { @@ -87,6 +88,10 @@ bool parseProtoOutputStream(util::ProtoOutputStream& protoOutput, T* message) { // Returns the truncated timestamp. int64_t truncateTimestampNsToFiveMinutes(int64_t timestampNs); +inline bool isPushedAtom(int atomId) { + return atomId <= util::kMaxPushedAtomId && atomId > 1; +} + } // namespace statsd } // namespace os } // namespace android diff --git a/cmds/statsd/src/stats_util.h b/cmds/statsd/src/stats_util.h index 5fcb16111f97..cfc411fdd25f 100644 --- a/cmds/statsd/src/stats_util.h +++ b/cmds/statsd/src/stats_util.h @@ -17,7 +17,6 @@ #pragma once #include "HashableDimensionKey.h" -#include "logd/LogReader.h" #include <unordered_map> diff --git a/cmds/statsd/src/statsd_config.proto b/cmds/statsd/src/statsd_config.proto index a6b9b7841707..aa789c799056 100644 --- a/cmds/statsd/src/statsd_config.proto +++ b/cmds/statsd/src/statsd_config.proto @@ -21,8 +21,6 @@ package android.os.statsd; option java_package = "com.android.internal.os"; option java_outer_classname = "StatsdConfigProto"; -import "frameworks/base/cmds/statsd/src/perfetto/perfetto_config.proto"; - enum Position { POSITION_UNKNOWN = 0; @@ -263,12 +261,27 @@ message ValueMetric { enum AggregationType { SUM = 1; + MIN = 2; + MAX = 3; + AVG = 4; } optional AggregationType aggregation_type = 8 [default = SUM]; optional int64 min_bucket_size_nanos = 10; optional bool use_absolute_value_on_reset = 11 [default = false]; + + optional bool use_diff = 12; + + enum ValueDirection { + UNKNOWN = 0; + INCREASING = 1; + DECREASING = 2; + ANY = 3; + } + optional ValueDirection value_direction = 13 [default = INCREASING]; + + optional bool skip_zero_diff_output = 14 [default = true]; } message Alert { @@ -302,7 +315,21 @@ message IncidentdDetails { } message PerfettoDetails { - optional perfetto.protos.TraceConfig trace_config = 1; + // The |trace_config| field is a proto-encoded message of type + // perfetto.protos.TraceConfig defined in + // //external/perfetto/protos/perfetto/config/. On device, + // statsd doesn't need to deserialize the message as it's just + // passed binary-encoded to the perfetto cmdline client. + optional bytes trace_config = 1; +} + +message PerfprofdDetails { + // The |perfprofd_config| field is a proto-encoded message of type + // android.perfprofd.ProfilingConfig defined in + // //system/extras/perfprofd/. On device, statsd doesn't need to + // deserialize the message as it's just passed binary-encoded to + // the perfprofd service. + optional bytes perfprofd_config = 1; } message BroadcastSubscriberDetails { @@ -326,11 +353,23 @@ message Subscription { IncidentdDetails incidentd_details = 4; PerfettoDetails perfetto_details = 5; BroadcastSubscriberDetails broadcast_subscriber_details = 6; + PerfprofdDetails perfprofd_details = 8; } optional float probability_of_informing = 7 [default = 1.1]; } +message EventActivation { + optional int64 atom_matcher_id = 1; + optional int64 ttl_seconds = 2; +} + +message MetricActivation { + optional int64 metric_id = 1; + + repeated EventActivation event_activation = 2; +} + message StatsdConfig { optional int64 id = 1; @@ -368,6 +407,8 @@ message StatsdConfig { optional bool hash_strings_in_metric_report = 16 [default = true]; + repeated MetricActivation metric_activation = 17; + // Field number 1000 is reserved for later use. reserved 1000; } diff --git a/cmds/statsd/src/storage/StorageManager.cpp b/cmds/statsd/src/storage/StorageManager.cpp index 1f8181266b65..2f19a02ecafe 100644 --- a/cmds/statsd/src/storage/StorageManager.cpp +++ b/cmds/statsd/src/storage/StorageManager.cpp @@ -57,7 +57,7 @@ static void parseFileName(char* name, int64_t* result) { } // When index ends before hitting 3, file name is corrupted. We // intentionally put -1 at index 0 to indicate the error to caller. - // TODO: consider removing files with unexpected name format. + // TODO(b/110563137): consider removing files with unexpected name format. if (index < 3) { result[0] = -1; } @@ -392,13 +392,13 @@ void StorageManager::trimToFit(const char* path) { } } -void StorageManager::printStats(FILE* out) { - printDirStats(out, STATS_SERVICE_DIR); - printDirStats(out, STATS_DATA_DIR); +void StorageManager::printStats(int outFd) { + printDirStats(outFd, STATS_SERVICE_DIR); + printDirStats(outFd, STATS_DATA_DIR); } -void StorageManager::printDirStats(FILE* out, const char* path) { - fprintf(out, "Printing stats of %s\n", path); +void StorageManager::printDirStats(int outFd, const char* path) { + dprintf(outFd, "Printing stats of %s\n", path); unique_ptr<DIR, decltype(&closedir)> dir(opendir(path), closedir); if (dir == NULL) { VLOG("Path %s does not exist", path); @@ -418,25 +418,22 @@ void StorageManager::printDirStats(FILE* out, const char* path) { int64_t timestamp = result[0]; int64_t uid = result[1]; int64_t configID = result[2]; - fprintf(out, "\t #%d, Last updated: %lld, UID: %d, Config ID: %lld", - fileCount + 1, - (long long)timestamp, - (int)uid, - (long long)configID); + dprintf(outFd, "\t #%d, Last updated: %lld, UID: %d, Config ID: %lld", fileCount + 1, + (long long)timestamp, (int)uid, (long long)configID); string file_name = getFilePath(path, timestamp, uid, configID); ifstream file(file_name.c_str(), ifstream::in | ifstream::binary); if (file.is_open()) { file.seekg(0, ios::end); int fileSize = file.tellg(); file.close(); - fprintf(out, ", File Size: %d bytes", fileSize); + dprintf(outFd, ", File Size: %d bytes", fileSize); totalFileSize += fileSize; } - fprintf(out, "\n"); + dprintf(outFd, "\n"); fileCount++; } - fprintf(out, "\tTotal number of files: %d, Total size of files: %d bytes.\n", - fileCount, totalFileSize); + dprintf(outFd, "\tTotal number of files: %d, Total size of files: %d bytes.\n", fileCount, + totalFileSize); } } // namespace statsd diff --git a/cmds/statsd/src/storage/StorageManager.h b/cmds/statsd/src/storage/StorageManager.h index 4840f3c4fa34..8fbc89e4f657 100644 --- a/cmds/statsd/src/storage/StorageManager.h +++ b/cmds/statsd/src/storage/StorageManager.h @@ -100,13 +100,13 @@ public: /** * Prints disk usage statistics related to statsd. */ - static void printStats(FILE* out); + static void printStats(int out); private: /** * Prints disk usage statistics about a directory related to statsd. */ - static void printDirStats(FILE* out, const char* path); + static void printDirStats(int out, const char* path); }; } // namespace statsd diff --git a/cmds/statsd/tests/FieldValue_test.cpp b/cmds/statsd/tests/FieldValue_test.cpp index c253bc19d641..a9305accb1be 100644 --- a/cmds/statsd/tests/FieldValue_test.cpp +++ b/cmds/statsd/tests/FieldValue_test.cpp @@ -312,7 +312,8 @@ TEST(AtomMatcherTest, TestSubscriberDimensionWrite) { dim.addValue(FieldValue(field4, value4)); SubscriberReporter::getStatsDimensionsValue(dim); - // TODO: can't test anything here because SubscriberReport class doesn't have any read api. + // TODO(b/110562792): can't test anything here because StatsDimensionsValue class doesn't + // have any read api. } TEST(AtomMatcherTest, TestWriteDimensionToProto) { @@ -483,4 +484,4 @@ TEST(AtomMatcherTest, TestWriteAtomToProto) { } // namespace android #else GTEST_LOG_(INFO) << "This test does nothing.\n"; -#endif
\ No newline at end of file +#endif diff --git a/cmds/statsd/tests/LogEvent_test.cpp b/cmds/statsd/tests/LogEvent_test.cpp index 2fcde29fbbdb..6384757b7fc8 100644 --- a/cmds/statsd/tests/LogEvent_test.cpp +++ b/cmds/statsd/tests/LogEvent_test.cpp @@ -12,9 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "src/logd/LogEvent.h" #include <gtest/gtest.h> #include <log/log_event_list.h> -#include "src/logd/LogEvent.h" +#include "frameworks/base/cmds/statsd/src/atoms.pb.h" +#include "frameworks/base/core/proto/android/stats/launcher/launcher.pb.h" #ifdef __ANDROID__ @@ -22,6 +24,9 @@ namespace android { namespace os { namespace statsd { +using std::string; +using util::ProtoOutputStream; + TEST(LogEventTest, TestLogParsing) { LogEvent event1(1, 2000); @@ -89,6 +94,122 @@ TEST(LogEventTest, TestLogParsing) { EXPECT_EQ((float)1.1, item7.mValue.float_value); } +TEST(LogEventTest, TestKeyValuePairsAtomParsing) { + LogEvent event1(83, 2000, 1000); + std::map<int32_t, int32_t> int_map; + std::map<int32_t, int64_t> long_map; + std::map<int32_t, std::string> string_map; + std::map<int32_t, float> float_map; + + int_map[11] = 123; + int_map[22] = 345; + + long_map[33] = 678L; + long_map[44] = 890L; + + string_map[1] = "test2"; + string_map[2] = "test1"; + + float_map[111] = 2.2f; + float_map[222] = 1.1f; + + EXPECT_TRUE(event1.writeKeyValuePairs(0, // Logging side logs 0 uid. + int_map, + long_map, + string_map, + float_map)); + event1.init(); + + EXPECT_EQ(83, event1.GetTagId()); + const auto& items = event1.getValues(); + EXPECT_EQ((size_t)17, items.size()); + + const FieldValue& item0 = event1.getValues()[0]; + EXPECT_EQ(0x10000, item0.mField.getField()); + EXPECT_EQ(Type::INT, item0.mValue.getType()); + EXPECT_EQ(1000, item0.mValue.int_value); + + const FieldValue& item1 = event1.getValues()[1]; + EXPECT_EQ(0x2010201, item1.mField.getField()); + EXPECT_EQ(Type::INT, item1.mValue.getType()); + EXPECT_EQ(11, item1.mValue.int_value); + + const FieldValue& item2 = event1.getValues()[2]; + EXPECT_EQ(0x2010282, item2.mField.getField()); + EXPECT_EQ(Type::INT, item2.mValue.getType()); + EXPECT_EQ(123, item2.mValue.int_value); + + const FieldValue& item3 = event1.getValues()[3]; + EXPECT_EQ(0x2010301, item3.mField.getField()); + EXPECT_EQ(Type::INT, item3.mValue.getType()); + EXPECT_EQ(22, item3.mValue.int_value); + + const FieldValue& item4 = event1.getValues()[4]; + EXPECT_EQ(0x2010382, item4.mField.getField()); + EXPECT_EQ(Type::INT, item4.mValue.getType()); + EXPECT_EQ(345, item4.mValue.int_value); + + const FieldValue& item5 = event1.getValues()[5]; + EXPECT_EQ(0x2010401, item5.mField.getField()); + EXPECT_EQ(Type::INT, item5.mValue.getType()); + EXPECT_EQ(33, item5.mValue.int_value); + + const FieldValue& item6 = event1.getValues()[6]; + EXPECT_EQ(0x2010482, item6.mField.getField()); + EXPECT_EQ(Type::LONG, item6.mValue.getType()); + EXPECT_EQ(678L, item6.mValue.int_value); + + const FieldValue& item7 = event1.getValues()[7]; + EXPECT_EQ(0x2010501, item7.mField.getField()); + EXPECT_EQ(Type::INT, item7.mValue.getType()); + EXPECT_EQ(44, item7.mValue.int_value); + + const FieldValue& item8 = event1.getValues()[8]; + EXPECT_EQ(0x2010582, item8.mField.getField()); + EXPECT_EQ(Type::LONG, item8.mValue.getType()); + EXPECT_EQ(890L, item8.mValue.int_value); + + const FieldValue& item9 = event1.getValues()[9]; + EXPECT_EQ(0x2010601, item9.mField.getField()); + EXPECT_EQ(Type::INT, item9.mValue.getType()); + EXPECT_EQ(1, item9.mValue.int_value); + + const FieldValue& item10 = event1.getValues()[10]; + EXPECT_EQ(0x2010683, item10.mField.getField()); + EXPECT_EQ(Type::STRING, item10.mValue.getType()); + EXPECT_EQ("test2", item10.mValue.str_value); + + const FieldValue& item11 = event1.getValues()[11]; + EXPECT_EQ(0x2010701, item11.mField.getField()); + EXPECT_EQ(Type::INT, item11.mValue.getType()); + EXPECT_EQ(2, item11.mValue.int_value); + + const FieldValue& item12 = event1.getValues()[12]; + EXPECT_EQ(0x2010783, item12.mField.getField()); + EXPECT_EQ(Type::STRING, item12.mValue.getType()); + EXPECT_EQ("test1", item12.mValue.str_value); + + const FieldValue& item13 = event1.getValues()[13]; + EXPECT_EQ(0x2010801, item13.mField.getField()); + EXPECT_EQ(Type::INT, item13.mValue.getType()); + EXPECT_EQ(111, item13.mValue.int_value); + + const FieldValue& item14 = event1.getValues()[14]; + EXPECT_EQ(0x2010884, item14.mField.getField()); + EXPECT_EQ(Type::FLOAT, item14.mValue.getType()); + EXPECT_EQ(2.2f, item14.mValue.float_value); + + const FieldValue& item15 = event1.getValues()[15]; + EXPECT_EQ(0x2018901, item15.mField.getField()); + EXPECT_EQ(Type::INT, item15.mValue.getType()); + EXPECT_EQ(222, item15.mValue.int_value); + + const FieldValue& item16 = event1.getValues()[16]; + EXPECT_EQ(0x2018984, item16.mField.getField()); + EXPECT_EQ(Type::FLOAT, item16.mValue.getType()); + EXPECT_EQ(1.1f, item16.mValue.float_value); +} + TEST(LogEventTest, TestLogParsing2) { LogEvent event1(1, 2000); @@ -158,10 +279,176 @@ TEST(LogEventTest, TestLogParsing2) { EXPECT_EQ((float)1.1, item7.mValue.float_value); } +TEST(LogEventTest, TestKeyValuePairsEvent) { + std::map<int32_t, int32_t> int_map; + std::map<int32_t, int64_t> long_map; + std::map<int32_t, std::string> string_map; + std::map<int32_t, float> float_map; + + int_map[11] = 123; + int_map[22] = 345; + + long_map[33] = 678L; + long_map[44] = 890L; + + string_map[1] = "test2"; + string_map[2] = "test1"; + + float_map[111] = 2.2f; + float_map[222] = 1.1f; + + LogEvent event1(83, 2000, 2001, 10001, int_map, long_map, string_map, float_map); + event1.init(); + + EXPECT_EQ(83, event1.GetTagId()); + EXPECT_EQ((int64_t)2000, event1.GetLogdTimestampNs()); + EXPECT_EQ((int64_t)2001, event1.GetElapsedTimestampNs()); + EXPECT_EQ((int64_t)10001, event1.GetUid()); + + const auto& items = event1.getValues(); + EXPECT_EQ((size_t)17, items.size()); + + const FieldValue& item0 = event1.getValues()[0]; + EXPECT_EQ(0x00010000, item0.mField.getField()); + EXPECT_EQ(Type::INT, item0.mValue.getType()); + EXPECT_EQ(10001, item0.mValue.int_value); + + const FieldValue& item1 = event1.getValues()[1]; + EXPECT_EQ(0x2020101, item1.mField.getField()); + EXPECT_EQ(Type::INT, item1.mValue.getType()); + EXPECT_EQ(11, item1.mValue.int_value); + + const FieldValue& item2 = event1.getValues()[2]; + EXPECT_EQ(0x2020182, item2.mField.getField()); + EXPECT_EQ(Type::INT, item2.mValue.getType()); + EXPECT_EQ(123, item2.mValue.int_value); + + const FieldValue& item3 = event1.getValues()[3]; + EXPECT_EQ(0x2020201, item3.mField.getField()); + EXPECT_EQ(Type::INT, item3.mValue.getType()); + EXPECT_EQ(22, item3.mValue.int_value); + + const FieldValue& item4 = event1.getValues()[4]; + EXPECT_EQ(0x2020282, item4.mField.getField()); + EXPECT_EQ(Type::INT, item4.mValue.getType()); + EXPECT_EQ(345, item4.mValue.int_value); + + const FieldValue& item5 = event1.getValues()[5]; + EXPECT_EQ(0x2020301, item5.mField.getField()); + EXPECT_EQ(Type::INT, item5.mValue.getType()); + EXPECT_EQ(33, item5.mValue.int_value); + + const FieldValue& item6 = event1.getValues()[6]; + EXPECT_EQ(0x2020383, item6.mField.getField()); + EXPECT_EQ(Type::LONG, item6.mValue.getType()); + EXPECT_EQ(678L, item6.mValue.long_value); + + const FieldValue& item7 = event1.getValues()[7]; + EXPECT_EQ(0x2020401, item7.mField.getField()); + EXPECT_EQ(Type::INT, item7.mValue.getType()); + EXPECT_EQ(44, item7.mValue.int_value); + + const FieldValue& item8 = event1.getValues()[8]; + EXPECT_EQ(0x2020483, item8.mField.getField()); + EXPECT_EQ(Type::LONG, item8.mValue.getType()); + EXPECT_EQ(890L, item8.mValue.long_value); + + const FieldValue& item9 = event1.getValues()[9]; + EXPECT_EQ(0x2020501, item9.mField.getField()); + EXPECT_EQ(Type::INT, item9.mValue.getType()); + EXPECT_EQ(1, item9.mValue.int_value); + + const FieldValue& item10 = event1.getValues()[10]; + EXPECT_EQ(0x2020584, item10.mField.getField()); + EXPECT_EQ(Type::STRING, item10.mValue.getType()); + EXPECT_EQ("test2", item10.mValue.str_value); + + const FieldValue& item11 = event1.getValues()[11]; + EXPECT_EQ(0x2020601, item11.mField.getField()); + EXPECT_EQ(Type::INT, item11.mValue.getType()); + EXPECT_EQ(2, item11.mValue.int_value); + + const FieldValue& item12 = event1.getValues()[12]; + EXPECT_EQ(0x2020684, item12.mField.getField()); + EXPECT_EQ(Type::STRING, item12.mValue.getType()); + EXPECT_EQ("test1", item12.mValue.str_value); + + const FieldValue& item13 = event1.getValues()[13]; + EXPECT_EQ(0x2020701, item13.mField.getField()); + EXPECT_EQ(Type::INT, item13.mValue.getType()); + EXPECT_EQ(111, item13.mValue.int_value); + + const FieldValue& item14 = event1.getValues()[14]; + EXPECT_EQ(0x2020785, item14.mField.getField()); + EXPECT_EQ(Type::FLOAT, item14.mValue.getType()); + EXPECT_EQ(2.2f, item14.mValue.float_value); + + const FieldValue& item15 = event1.getValues()[15]; + EXPECT_EQ(0x2028801, item15.mField.getField()); + EXPECT_EQ(Type::INT, item15.mValue.getType()); + EXPECT_EQ(222, item15.mValue.int_value); + + const FieldValue& item16 = event1.getValues()[16]; + EXPECT_EQ(0x2028885, item16.mField.getField()); + EXPECT_EQ(Type::FLOAT, item16.mValue.getType()); + EXPECT_EQ(1.1f, item16.mValue.float_value); +} + + +TEST(LogEventTest, TestBinaryFieldAtom) { + Atom launcherAtom; + auto launcher_event = launcherAtom.mutable_launcher_event(); + launcher_event->set_action(stats::launcher::LauncherAction::LONGPRESS); + launcher_event->set_src_state(stats::launcher::LauncherState::OVERVIEW); + launcher_event->set_dst_state(stats::launcher::LauncherState::ALLAPPS); + + auto extension = launcher_event->mutable_extension(); + + auto src_target = extension->add_src_target(); + src_target->set_type(stats::launcher::LauncherTarget_Type_ITEM_TYPE); + src_target->set_item(stats::launcher::LauncherTarget_Item_FOLDER_ICON); + + auto dst_target = extension->add_dst_target(); + dst_target->set_type(stats::launcher::LauncherTarget_Type_ITEM_TYPE); + dst_target->set_item(stats::launcher::LauncherTarget_Item_WIDGET); + + string extension_str; + extension->SerializeToString(&extension_str); + + LogEvent event1(Atom::kLauncherEventFieldNumber, 1000); + + event1.write((int32_t)stats::launcher::LauncherAction::LONGPRESS); + event1.write((int32_t)stats::launcher::LauncherState::OVERVIEW); + event1.write((int64_t)stats::launcher::LauncherState::ALLAPPS); + event1.write(extension_str); + event1.init(); + + ProtoOutputStream proto; + event1.ToProto(proto); + + std::vector<uint8_t> outData; + outData.resize(proto.size()); + size_t pos = 0; + auto iter = proto.data(); + while (iter.readBuffer() != NULL) { + size_t toRead = iter.currentToRead(); + std::memcpy(&(outData[pos]), iter.readBuffer(), toRead); + pos += toRead; + iter.rp()->move(toRead); + } + + std::string result_str(outData.begin(), outData.end()); + std::string orig_str; + launcherAtom.SerializeToString(&orig_str); + + EXPECT_EQ(orig_str, result_str); +} + + } // namespace statsd } // namespace os } // namespace android #else GTEST_LOG_(INFO) << "This test does nothing.\n"; -#endif
\ No newline at end of file +#endif diff --git a/cmds/statsd/tests/MetricsManager_test.cpp b/cmds/statsd/tests/MetricsManager_test.cpp index 07378dbcce1a..f8184d8aa14c 100644 --- a/cmds/statsd/tests/MetricsManager_test.cpp +++ b/cmds/statsd/tests/MetricsManager_test.cpp @@ -39,8 +39,6 @@ using android::os::statsd::Predicate; #ifdef __ANDROID__ -// TODO: ADD MORE TEST CASES. - const ConfigKey kConfigKey(0, 12345); const long timeBaseSec = 1000; @@ -271,6 +269,7 @@ StatsdConfig buildCirclePredicates() { TEST(MetricsManagerTest, TestGoodConfig) { UidMap uidMap; + sp<StatsPullerManager> pullerManager = new StatsPullerManager(); sp<AlarmMonitor> anomalyAlarmMonitor; sp<AlarmMonitor> periodicAlarmMonitor; StatsdConfig config = buildGoodConfig(); @@ -283,14 +282,16 @@ TEST(MetricsManagerTest, TestGoodConfig) { unordered_map<int, std::vector<int>> conditionToMetricMap; unordered_map<int, std::vector<int>> trackerToMetricMap; unordered_map<int, std::vector<int>> trackerToConditionMap; + unordered_map<int, std::vector<int>> lifeSpanEventTrackerToMetricMap; + vector<int> metricsWithLifeSpan; std::set<int64_t> noReportMetricIds; - EXPECT_TRUE(initStatsdConfig(kConfigKey, config, uidMap, - anomalyAlarmMonitor, periodicAlarmMonitor, - timeBaseSec, timeBaseSec, allTagIds, allAtomMatchers, - allConditionTrackers, allMetricProducers, allAnomalyTrackers, - allAlarmTrackers, - conditionToMetricMap, trackerToMetricMap, trackerToConditionMap, + EXPECT_TRUE(initStatsdConfig(kConfigKey, config, uidMap, pullerManager, anomalyAlarmMonitor, + periodicAlarmMonitor, timeBaseSec, timeBaseSec, allTagIds, + allAtomMatchers, allConditionTrackers, allMetricProducers, + allAnomalyTrackers, allAlarmTrackers, conditionToMetricMap, + trackerToMetricMap, trackerToConditionMap, + lifeSpanEventTrackerToMetricMap, metricsWithLifeSpan, noReportMetricIds)); EXPECT_EQ(1u, allMetricProducers.size()); EXPECT_EQ(1u, allAnomalyTrackers.size()); @@ -299,6 +300,7 @@ TEST(MetricsManagerTest, TestGoodConfig) { TEST(MetricsManagerTest, TestDimensionMetricsWithMultiTags) { UidMap uidMap; + sp<StatsPullerManager> pullerManager = new StatsPullerManager(); sp<AlarmMonitor> anomalyAlarmMonitor; sp<AlarmMonitor> periodicAlarmMonitor; StatsdConfig config = buildDimensionMetricsWithMultiTags(); @@ -311,19 +313,22 @@ TEST(MetricsManagerTest, TestDimensionMetricsWithMultiTags) { unordered_map<int, std::vector<int>> conditionToMetricMap; unordered_map<int, std::vector<int>> trackerToMetricMap; unordered_map<int, std::vector<int>> trackerToConditionMap; + unordered_map<int, std::vector<int>> lifeSpanEventTrackerToMetricMap; + vector<int> metricsWithLifeSpan; std::set<int64_t> noReportMetricIds; - EXPECT_FALSE(initStatsdConfig(kConfigKey, config, uidMap, - anomalyAlarmMonitor, periodicAlarmMonitor, - timeBaseSec, timeBaseSec, allTagIds, allAtomMatchers, - allConditionTrackers, allMetricProducers, allAnomalyTrackers, - allAlarmTrackers, - conditionToMetricMap, trackerToMetricMap, trackerToConditionMap, + EXPECT_FALSE(initStatsdConfig(kConfigKey, config, uidMap, pullerManager, anomalyAlarmMonitor, + periodicAlarmMonitor, timeBaseSec, timeBaseSec, allTagIds, + allAtomMatchers, allConditionTrackers, allMetricProducers, + allAnomalyTrackers, allAlarmTrackers, conditionToMetricMap, + trackerToMetricMap, trackerToConditionMap, + lifeSpanEventTrackerToMetricMap, metricsWithLifeSpan, noReportMetricIds)); } TEST(MetricsManagerTest, TestCircleLogMatcherDependency) { UidMap uidMap; + sp<StatsPullerManager> pullerManager = new StatsPullerManager(); sp<AlarmMonitor> anomalyAlarmMonitor; sp<AlarmMonitor> periodicAlarmMonitor; StatsdConfig config = buildCircleMatchers(); @@ -336,19 +341,22 @@ TEST(MetricsManagerTest, TestCircleLogMatcherDependency) { unordered_map<int, std::vector<int>> conditionToMetricMap; unordered_map<int, std::vector<int>> trackerToMetricMap; unordered_map<int, std::vector<int>> trackerToConditionMap; + unordered_map<int, std::vector<int>> lifeSpanEventTrackerToMetricMap; + vector<int> metricsWithLifeSpan; std::set<int64_t> noReportMetricIds; - EXPECT_FALSE(initStatsdConfig(kConfigKey, config, uidMap, - anomalyAlarmMonitor, periodicAlarmMonitor, - timeBaseSec, timeBaseSec, allTagIds, allAtomMatchers, - allConditionTrackers, allMetricProducers, allAnomalyTrackers, - allAlarmTrackers, - conditionToMetricMap, trackerToMetricMap, trackerToConditionMap, + EXPECT_FALSE(initStatsdConfig(kConfigKey, config, uidMap, pullerManager, anomalyAlarmMonitor, + periodicAlarmMonitor, timeBaseSec, timeBaseSec, allTagIds, + allAtomMatchers, allConditionTrackers, allMetricProducers, + allAnomalyTrackers, allAlarmTrackers, conditionToMetricMap, + trackerToMetricMap, trackerToConditionMap, + lifeSpanEventTrackerToMetricMap, metricsWithLifeSpan, noReportMetricIds)); } TEST(MetricsManagerTest, TestMissingMatchers) { UidMap uidMap; + sp<StatsPullerManager> pullerManager = new StatsPullerManager(); sp<AlarmMonitor> anomalyAlarmMonitor; sp<AlarmMonitor> periodicAlarmMonitor; StatsdConfig config = buildMissingMatchers(); @@ -361,18 +369,21 @@ TEST(MetricsManagerTest, TestMissingMatchers) { unordered_map<int, std::vector<int>> conditionToMetricMap; unordered_map<int, std::vector<int>> trackerToMetricMap; unordered_map<int, std::vector<int>> trackerToConditionMap; + unordered_map<int, std::vector<int>> lifeSpanEventTrackerToMetricMap; + vector<int> metricsWithLifeSpan; std::set<int64_t> noReportMetricIds; - EXPECT_FALSE(initStatsdConfig(kConfigKey, config, uidMap, - anomalyAlarmMonitor, periodicAlarmMonitor, - timeBaseSec, timeBaseSec, allTagIds, allAtomMatchers, - allConditionTrackers, allMetricProducers, allAnomalyTrackers, - allAlarmTrackers, - conditionToMetricMap, trackerToMetricMap, trackerToConditionMap, + EXPECT_FALSE(initStatsdConfig(kConfigKey, config, uidMap, pullerManager, anomalyAlarmMonitor, + periodicAlarmMonitor, timeBaseSec, timeBaseSec, allTagIds, + allAtomMatchers, allConditionTrackers, allMetricProducers, + allAnomalyTrackers, allAlarmTrackers, conditionToMetricMap, + trackerToMetricMap, trackerToConditionMap, + lifeSpanEventTrackerToMetricMap, metricsWithLifeSpan, noReportMetricIds)); } TEST(MetricsManagerTest, TestMissingPredicate) { UidMap uidMap; + sp<StatsPullerManager> pullerManager = new StatsPullerManager(); sp<AlarmMonitor> anomalyAlarmMonitor; sp<AlarmMonitor> periodicAlarmMonitor; StatsdConfig config = buildMissingPredicate(); @@ -385,18 +396,21 @@ TEST(MetricsManagerTest, TestMissingPredicate) { unordered_map<int, std::vector<int>> conditionToMetricMap; unordered_map<int, std::vector<int>> trackerToMetricMap; unordered_map<int, std::vector<int>> trackerToConditionMap; + unordered_map<int, std::vector<int>> lifeSpanEventTrackerToMetricMap; + vector<int> metricsWithLifeSpan; std::set<int64_t> noReportMetricIds; - EXPECT_FALSE(initStatsdConfig(kConfigKey, config, uidMap, - anomalyAlarmMonitor, periodicAlarmMonitor, - timeBaseSec, timeBaseSec, allTagIds, allAtomMatchers, - allConditionTrackers, allMetricProducers, allAnomalyTrackers, - allAlarmTrackers, - conditionToMetricMap, trackerToMetricMap, trackerToConditionMap, + EXPECT_FALSE(initStatsdConfig(kConfigKey, config, uidMap, pullerManager, anomalyAlarmMonitor, + periodicAlarmMonitor, timeBaseSec, timeBaseSec, allTagIds, + allAtomMatchers, allConditionTrackers, allMetricProducers, + allAnomalyTrackers, allAlarmTrackers, conditionToMetricMap, + trackerToMetricMap, trackerToConditionMap, + lifeSpanEventTrackerToMetricMap, metricsWithLifeSpan, noReportMetricIds)); } TEST(MetricsManagerTest, TestCirclePredicateDependency) { UidMap uidMap; + sp<StatsPullerManager> pullerManager = new StatsPullerManager(); sp<AlarmMonitor> anomalyAlarmMonitor; sp<AlarmMonitor> periodicAlarmMonitor; StatsdConfig config = buildCirclePredicates(); @@ -409,19 +423,22 @@ TEST(MetricsManagerTest, TestCirclePredicateDependency) { unordered_map<int, std::vector<int>> conditionToMetricMap; unordered_map<int, std::vector<int>> trackerToMetricMap; unordered_map<int, std::vector<int>> trackerToConditionMap; + unordered_map<int, std::vector<int>> lifeSpanEventTrackerToMetricMap; + vector<int> metricsWithLifeSpan; std::set<int64_t> noReportMetricIds; - EXPECT_FALSE(initStatsdConfig(kConfigKey, config, uidMap, - anomalyAlarmMonitor, periodicAlarmMonitor, - timeBaseSec, timeBaseSec, allTagIds, allAtomMatchers, - allConditionTrackers, allMetricProducers, allAnomalyTrackers, - allAlarmTrackers, - conditionToMetricMap, trackerToMetricMap, trackerToConditionMap, + EXPECT_FALSE(initStatsdConfig(kConfigKey, config, uidMap, pullerManager, anomalyAlarmMonitor, + periodicAlarmMonitor, timeBaseSec, timeBaseSec, allTagIds, + allAtomMatchers, allConditionTrackers, allMetricProducers, + allAnomalyTrackers, allAlarmTrackers, conditionToMetricMap, + trackerToMetricMap, trackerToConditionMap, + lifeSpanEventTrackerToMetricMap, metricsWithLifeSpan, noReportMetricIds)); } TEST(MetricsManagerTest, testAlertWithUnknownMetric) { UidMap uidMap; + sp<StatsPullerManager> pullerManager = new StatsPullerManager(); sp<AlarmMonitor> anomalyAlarmMonitor; sp<AlarmMonitor> periodicAlarmMonitor; StatsdConfig config = buildAlertWithUnknownMetric(); @@ -434,14 +451,16 @@ TEST(MetricsManagerTest, testAlertWithUnknownMetric) { unordered_map<int, std::vector<int>> conditionToMetricMap; unordered_map<int, std::vector<int>> trackerToMetricMap; unordered_map<int, std::vector<int>> trackerToConditionMap; + unordered_map<int, std::vector<int>> lifeSpanEventTrackerToMetricMap; + vector<int> metricsWithLifeSpan; std::set<int64_t> noReportMetricIds; - EXPECT_FALSE(initStatsdConfig(kConfigKey, config, uidMap, - anomalyAlarmMonitor, periodicAlarmMonitor, - timeBaseSec, timeBaseSec, allTagIds, allAtomMatchers, - allConditionTrackers, allMetricProducers, allAnomalyTrackers, - allAlarmTrackers, - conditionToMetricMap, trackerToMetricMap, trackerToConditionMap, + EXPECT_FALSE(initStatsdConfig(kConfigKey, config, uidMap, pullerManager, anomalyAlarmMonitor, + periodicAlarmMonitor, timeBaseSec, timeBaseSec, allTagIds, + allAtomMatchers, allConditionTrackers, allMetricProducers, + allAnomalyTrackers, allAlarmTrackers, conditionToMetricMap, + trackerToMetricMap, trackerToConditionMap, + lifeSpanEventTrackerToMetricMap, metricsWithLifeSpan, noReportMetricIds)); } diff --git a/cmds/statsd/tests/StatsLogProcessor_test.cpp b/cmds/statsd/tests/StatsLogProcessor_test.cpp index 76f3d8181dee..8864252bcf4b 100644 --- a/cmds/statsd/tests/StatsLogProcessor_test.cpp +++ b/cmds/statsd/tests/StatsLogProcessor_test.cpp @@ -44,13 +44,13 @@ using android::util::ProtoOutputStream; */ class MockMetricsManager : public MetricsManager { public: - MockMetricsManager() : MetricsManager( - ConfigKey(1, 12345), StatsdConfig(), 1000, 1000, - new UidMap(), - new AlarmMonitor(10, [](const sp<IStatsCompanionService>&, int64_t){}, - [](const sp<IStatsCompanionService>&){}), - new AlarmMonitor(10, [](const sp<IStatsCompanionService>&, int64_t){}, - [](const sp<IStatsCompanionService>&){})) { + MockMetricsManager() + : MetricsManager(ConfigKey(1, 12345), StatsdConfig(), 1000, 1000, new UidMap(), + new StatsPullerManager(), + new AlarmMonitor(10, [](const sp<IStatsCompanionService>&, int64_t) {}, + [](const sp<IStatsCompanionService>&) {}), + new AlarmMonitor(10, [](const sp<IStatsCompanionService>&, int64_t) {}, + [](const sp<IStatsCompanionService>&) {})) { } MOCK_METHOD0(byteSize, size_t()); @@ -60,11 +60,12 @@ public: TEST(StatsLogProcessorTest, TestRateLimitByteSize) { sp<UidMap> m = new UidMap(); + sp<StatsPullerManager> pullerManager = new StatsPullerManager(); sp<AlarmMonitor> anomalyAlarmMonitor; sp<AlarmMonitor> periodicAlarmMonitor; // Construct the processor with a dummy sendBroadcast function that does nothing. - StatsLogProcessor p(m, anomalyAlarmMonitor, periodicAlarmMonitor, 0, - [](const ConfigKey& key) {return true;}); + StatsLogProcessor p(m, pullerManager, anomalyAlarmMonitor, periodicAlarmMonitor, 0, + [](const ConfigKey& key) { return true; }); MockMetricsManager mockMetricsManager; @@ -79,11 +80,15 @@ TEST(StatsLogProcessorTest, TestRateLimitByteSize) { TEST(StatsLogProcessorTest, TestRateLimitBroadcast) { sp<UidMap> m = new UidMap(); + sp<StatsPullerManager> pullerManager = new StatsPullerManager(); sp<AlarmMonitor> anomalyAlarmMonitor; sp<AlarmMonitor> subscriberAlarmMonitor; int broadcastCount = 0; - StatsLogProcessor p(m, anomalyAlarmMonitor, subscriberAlarmMonitor, 0, - [&broadcastCount](const ConfigKey& key) { broadcastCount++; return true;}); + StatsLogProcessor p(m, pullerManager, anomalyAlarmMonitor, subscriberAlarmMonitor, 0, + [&broadcastCount](const ConfigKey& key) { + broadcastCount++; + return true; + }); MockMetricsManager mockMetricsManager; @@ -105,11 +110,15 @@ TEST(StatsLogProcessorTest, TestRateLimitBroadcast) { TEST(StatsLogProcessorTest, TestDropWhenByteSizeTooLarge) { sp<UidMap> m = new UidMap(); + sp<StatsPullerManager> pullerManager = new StatsPullerManager(); sp<AlarmMonitor> anomalyAlarmMonitor; sp<AlarmMonitor> subscriberAlarmMonitor; int broadcastCount = 0; - StatsLogProcessor p(m, anomalyAlarmMonitor, subscriberAlarmMonitor, 0, - [&broadcastCount](const ConfigKey& key) { broadcastCount++; return true;}); + StatsLogProcessor p(m, pullerManager, anomalyAlarmMonitor, subscriberAlarmMonitor, 0, + [&broadcastCount](const ConfigKey& key) { + broadcastCount++; + return true; + }); MockMetricsManager mockMetricsManager; @@ -143,19 +152,23 @@ StatsdConfig MakeConfig(bool includeMetric) { TEST(StatsLogProcessorTest, TestUidMapHasSnapshot) { // Setup simple config key corresponding to empty config. sp<UidMap> m = new UidMap(); + sp<StatsPullerManager> pullerManager = new StatsPullerManager(); m->updateMap(1, {1, 2}, {1, 2}, {String16("p1"), String16("p2")}); sp<AlarmMonitor> anomalyAlarmMonitor; sp<AlarmMonitor> subscriberAlarmMonitor; int broadcastCount = 0; - StatsLogProcessor p(m, anomalyAlarmMonitor, subscriberAlarmMonitor, 0, - [&broadcastCount](const ConfigKey& key) { broadcastCount++; return true;}); + StatsLogProcessor p(m, pullerManager, anomalyAlarmMonitor, subscriberAlarmMonitor, 0, + [&broadcastCount](const ConfigKey& key) { + broadcastCount++; + return true; + }); ConfigKey key(3, 4); StatsdConfig config = MakeConfig(true); p.OnConfigUpdated(0, key, config); // Expect to get no metrics, but snapshot specified above in uidmap. vector<uint8_t> bytes; - p.onDumpReport(key, 1, false, ADB_DUMP, &bytes); + p.onDumpReport(key, 1, false, true, ADB_DUMP, &bytes); ConfigMetricsReportList output; output.ParseFromArray(bytes.data(), bytes.size()); @@ -168,19 +181,23 @@ TEST(StatsLogProcessorTest, TestUidMapHasSnapshot) { TEST(StatsLogProcessorTest, TestEmptyConfigHasNoUidMap) { // Setup simple config key corresponding to empty config. sp<UidMap> m = new UidMap(); + sp<StatsPullerManager> pullerManager = new StatsPullerManager(); m->updateMap(1, {1, 2}, {1, 2}, {String16("p1"), String16("p2")}); sp<AlarmMonitor> anomalyAlarmMonitor; sp<AlarmMonitor> subscriberAlarmMonitor; int broadcastCount = 0; - StatsLogProcessor p(m, anomalyAlarmMonitor, subscriberAlarmMonitor, 0, - [&broadcastCount](const ConfigKey& key) { broadcastCount++; return true;}); + StatsLogProcessor p(m, pullerManager, anomalyAlarmMonitor, subscriberAlarmMonitor, 0, + [&broadcastCount](const ConfigKey& key) { + broadcastCount++; + return true; + }); ConfigKey key(3, 4); StatsdConfig config = MakeConfig(false); p.OnConfigUpdated(0, key, config); // Expect to get no metrics, but snapshot specified above in uidmap. vector<uint8_t> bytes; - p.onDumpReport(key, 1, false, ADB_DUMP, &bytes); + p.onDumpReport(key, 1, false, true, ADB_DUMP, &bytes); ConfigMetricsReportList output; output.ParseFromArray(bytes.data(), bytes.size()); @@ -191,11 +208,15 @@ TEST(StatsLogProcessorTest, TestEmptyConfigHasNoUidMap) { TEST(StatsLogProcessorTest, TestReportIncludesSubConfig) { // Setup simple config key corresponding to empty config. sp<UidMap> m = new UidMap(); + sp<StatsPullerManager> pullerManager = new StatsPullerManager(); sp<AlarmMonitor> anomalyAlarmMonitor; sp<AlarmMonitor> subscriberAlarmMonitor; int broadcastCount = 0; - StatsLogProcessor p(m, anomalyAlarmMonitor, subscriberAlarmMonitor, 0, - [&broadcastCount](const ConfigKey& key) { broadcastCount++; return true;}); + StatsLogProcessor p(m, pullerManager, anomalyAlarmMonitor, subscriberAlarmMonitor, 0, + [&broadcastCount](const ConfigKey& key) { + broadcastCount++; + return true; + }); ConfigKey key(3, 4); StatsdConfig config; auto annotation = config.add_annotation(); @@ -206,7 +227,7 @@ TEST(StatsLogProcessorTest, TestReportIncludesSubConfig) { // Expect to get no metrics, but snapshot specified above in uidmap. vector<uint8_t> bytes; - p.onDumpReport(key, 1, false, ADB_DUMP, &bytes); + p.onDumpReport(key, 1, false, true, ADB_DUMP, &bytes); ConfigMetricsReportList output; output.ParseFromArray(bytes.data(), bytes.size()); @@ -217,128 +238,6 @@ TEST(StatsLogProcessorTest, TestReportIncludesSubConfig) { EXPECT_EQ(2, report.annotation(0).field_int32()); } -TEST(StatsLogProcessorTest, TestOutOfOrderLogs) { - // Setup simple config key corresponding to empty config. - sp<UidMap> m = new UidMap(); - sp<AlarmMonitor> anomalyAlarmMonitor; - sp<AlarmMonitor> subscriberAlarmMonitor; - int broadcastCount = 0; - StatsLogProcessor p(m, anomalyAlarmMonitor, subscriberAlarmMonitor, 0, - [&broadcastCount](const ConfigKey& key) { broadcastCount++; return true;}); - - LogEvent event1(0, 1 /*logd timestamp*/, 1001 /*elapsedRealtime*/); - event1.init(); - - LogEvent event2(0, 2, 1002); - event2.init(); - - LogEvent event3(0, 3, 1005); - event3.init(); - - LogEvent event4(0, 4, 1004); - event4.init(); - - // <----- Reconnection happens - - LogEvent event5(0, 5, 999); - event5.init(); - - LogEvent event6(0, 6, 2000); - event6.init(); - - // <----- Reconnection happens - - LogEvent event7(0, 7, 3000); - event7.init(); - - // first event ever - p.OnLogEvent(&event1, true); - EXPECT_EQ(1UL, p.mLogCount); - EXPECT_EQ(1001LL, p.mLargestTimestampSeen); - EXPECT_EQ(1001LL, p.mLastTimestampSeen); - - p.OnLogEvent(&event2, false); - EXPECT_EQ(2UL, p.mLogCount); - EXPECT_EQ(1002LL, p.mLargestTimestampSeen); - EXPECT_EQ(1002LL, p.mLastTimestampSeen); - - p.OnLogEvent(&event3, false); - EXPECT_EQ(3UL, p.mLogCount); - EXPECT_EQ(1005LL, p.mLargestTimestampSeen); - EXPECT_EQ(1005LL, p.mLastTimestampSeen); - - p.OnLogEvent(&event4, false); - EXPECT_EQ(4UL, p.mLogCount); - EXPECT_EQ(1005LL, p.mLargestTimestampSeen); - EXPECT_EQ(1004LL, p.mLastTimestampSeen); - EXPECT_FALSE(p.mInReconnection); - - // Reconnect happens, event1 out of buffer. Read event2 - p.OnLogEvent(&event2, true); - EXPECT_EQ(4UL, p.mLogCount); - EXPECT_EQ(1005LL, p.mLargestTimestampSeen); - EXPECT_EQ(1004LL, p.mLastTimestampSeen); - EXPECT_TRUE(p.mInReconnection); - - p.OnLogEvent(&event3, false); - EXPECT_EQ(4UL, p.mLogCount); - EXPECT_EQ(1005LL, p.mLargestTimestampSeen); - EXPECT_EQ(1004LL, p.mLastTimestampSeen); - EXPECT_TRUE(p.mInReconnection); - - p.OnLogEvent(&event4, false); - EXPECT_EQ(4UL, p.mLogCount); - EXPECT_EQ(1005LL, p.mLargestTimestampSeen); - EXPECT_EQ(1004LL, p.mLastTimestampSeen); - EXPECT_FALSE(p.mInReconnection); - - // Fresh event comes. - p.OnLogEvent(&event5, false); - EXPECT_EQ(5UL, p.mLogCount); - EXPECT_EQ(1005LL, p.mLargestTimestampSeen); - EXPECT_EQ(999LL, p.mLastTimestampSeen); - - p.OnLogEvent(&event6, false); - EXPECT_EQ(6UL, p.mLogCount); - EXPECT_EQ(2000LL, p.mLargestTimestampSeen); - EXPECT_EQ(2000LL, p.mLastTimestampSeen); - - // Reconnect happens, read from event4 - p.OnLogEvent(&event4, true); - EXPECT_EQ(6UL, p.mLogCount); - EXPECT_EQ(2000LL, p.mLargestTimestampSeen); - EXPECT_EQ(2000LL, p.mLastTimestampSeen); - EXPECT_TRUE(p.mInReconnection); - - p.OnLogEvent(&event5, false); - EXPECT_EQ(6UL, p.mLogCount); - EXPECT_EQ(2000LL, p.mLargestTimestampSeen); - EXPECT_EQ(2000LL, p.mLastTimestampSeen); - EXPECT_TRUE(p.mInReconnection); - - // Before we get out of reconnection state, it reconnects again. - p.OnLogEvent(&event5, true); - EXPECT_EQ(6UL, p.mLogCount); - EXPECT_EQ(2000LL, p.mLargestTimestampSeen); - EXPECT_EQ(2000LL, p.mLastTimestampSeen); - EXPECT_TRUE(p.mInReconnection); - - p.OnLogEvent(&event6, false); - EXPECT_EQ(6UL, p.mLogCount); - EXPECT_EQ(2000LL, p.mLargestTimestampSeen); - EXPECT_EQ(2000LL, p.mLastTimestampSeen); - EXPECT_FALSE(p.mInReconnection); - EXPECT_EQ(0, p.mLogLossCount); - - // it reconnects again. All old events are gone. We lose CP. - p.OnLogEvent(&event7, true); - EXPECT_EQ(7UL, p.mLogCount); - EXPECT_EQ(3000LL, p.mLargestTimestampSeen); - EXPECT_EQ(3000LL, p.mLastTimestampSeen); - EXPECT_EQ(1, p.mLogLossCount); - EXPECT_FALSE(p.mInReconnection); -} - #else GTEST_LOG_(INFO) << "This test does nothing.\n"; #endif diff --git a/cmds/statsd/tests/UidMap_test.cpp b/cmds/statsd/tests/UidMap_test.cpp index e23131d7b45d..99082cc647f6 100644 --- a/cmds/statsd/tests/UidMap_test.cpp +++ b/cmds/statsd/tests/UidMap_test.cpp @@ -40,11 +40,12 @@ const string kApp2 = "app2.sharing.1"; TEST(UidMapTest, TestIsolatedUID) { sp<UidMap> m = new UidMap(); + sp<StatsPullerManager> pullerManager = new StatsPullerManager(); sp<AlarmMonitor> anomalyAlarmMonitor; sp<AlarmMonitor> subscriberAlarmMonitor; // Construct the processor with a dummy sendBroadcast function that does nothing. - StatsLogProcessor p(m, anomalyAlarmMonitor, subscriberAlarmMonitor, 0, - [](const ConfigKey& key) {return true;}); + StatsLogProcessor p(m, pullerManager, anomalyAlarmMonitor, subscriberAlarmMonitor, 0, + [](const ConfigKey& key) { return true; }); LogEvent addEvent(android::util::ISOLATED_UID_CHANGED, 1); addEvent.write(100); // parent UID addEvent.write(101); // isolated UID diff --git a/cmds/statsd/tests/anomaly/AnomalyTracker_test.cpp b/cmds/statsd/tests/anomaly/AnomalyTracker_test.cpp index 218d52a5c046..79bed52f0202 100644 --- a/cmds/statsd/tests/anomaly/AnomalyTracker_test.cpp +++ b/cmds/statsd/tests/anomaly/AnomalyTracker_test.cpp @@ -305,10 +305,10 @@ TEST(AnomalyTrackerTest, TestSparseBuckets) { EXPECT_EQ(anomalyTracker.getSumOverPastBuckets(keyB), 2LL); EXPECT_EQ(anomalyTracker.getSumOverPastBuckets(keyC), 1LL); EXPECT_TRUE(detectAnomaliesPass(anomalyTracker, 16, bucket16, {keyB}, {keyA, keyC, keyD})); - // TODO: after detectAnomaly fix: EXPECT_EQ(anomalyTracker.mSumOverPastBuckets.size(), 0UL); + EXPECT_EQ(anomalyTracker.mSumOverPastBuckets.size(), 0UL); EXPECT_EQ(anomalyTracker.mMostRecentBucketNum, 15L); detectAndDeclareAnomalies(anomalyTracker, 16, bucket16, eventTimestamp2); - // TODO: after detectAnomaly fix: EXPECT_EQ(anomalyTracker.mSumOverPastBuckets.size(), 0UL); + EXPECT_EQ(anomalyTracker.mSumOverPastBuckets.size(), 0UL); EXPECT_EQ(anomalyTracker.mMostRecentBucketNum, 15L); checkRefractoryTimes(anomalyTracker, eventTimestamp2, refractoryPeriodSec, {{keyA, -1}, {keyB, eventTimestamp2}, {keyC, -1}, {keyD, -1}, {keyE, -1}}); @@ -366,7 +366,7 @@ TEST(AnomalyTrackerTest, TestSparseBuckets) { EXPECT_EQ(anomalyTracker.getSumOverPastBuckets(keyC), 1LL); EXPECT_TRUE(detectAnomaliesPass(anomalyTracker, 25, bucket25, {}, {keyA, keyB, keyC, keyD})); EXPECT_EQ(anomalyTracker.mMostRecentBucketNum, 24L); - // TODO: after detectAnomaly fix: EXPECT_EQ(anomalyTracker.mSumOverPastBuckets.size(), 0UL); + EXPECT_EQ(anomalyTracker.mSumOverPastBuckets.size(), 0UL); detectAndDeclareAnomalies(anomalyTracker, 25, bucket25, eventTimestamp5); checkRefractoryTimes(anomalyTracker, eventTimestamp5, refractoryPeriodSec, {{keyA, -1}, {keyB, eventTimestamp4}, {keyC, -1}, {keyD, -1}, {keyE, -1}}); @@ -374,14 +374,14 @@ TEST(AnomalyTrackerTest, TestSparseBuckets) { // Add past bucket #25 anomalyTracker.addPastBucket(bucket25, 25); EXPECT_EQ(anomalyTracker.mMostRecentBucketNum, 25L); - // TODO: after detectAnomaly fix: EXPECT_EQ(anomalyTracker.mSumOverPastBuckets.size(), 1UL); + EXPECT_EQ(anomalyTracker.mSumOverPastBuckets.size(), 1UL); EXPECT_EQ(anomalyTracker.getSumOverPastBuckets(keyD), 1LL); EXPECT_TRUE(detectAnomaliesPass(anomalyTracker, 28, bucket28, {}, {keyA, keyB, keyC, keyD, keyE})); EXPECT_EQ(anomalyTracker.mMostRecentBucketNum, 27L); - // TODO: after detectAnomaly fix: EXPECT_EQ(anomalyTracker.mSumOverPastBuckets.size(), 0UL); + EXPECT_EQ(anomalyTracker.mSumOverPastBuckets.size(), 0UL); detectAndDeclareAnomalies(anomalyTracker, 28, bucket28, eventTimestamp6); - // TODO: after detectAnomaly fix: EXPECT_EQ(anomalyTracker.mSumOverPastBuckets.size(), 0UL); + EXPECT_EQ(anomalyTracker.mSumOverPastBuckets.size(), 0UL); checkRefractoryTimes(anomalyTracker, eventTimestamp6, refractoryPeriodSec, {{keyA, -1}, {keyB, -1}, {keyC, -1}, {keyD, -1}, {keyE, -1}}); @@ -390,9 +390,9 @@ TEST(AnomalyTrackerTest, TestSparseBuckets) { EXPECT_TRUE(detectAnomaliesPass(anomalyTracker, 28, bucket28, {keyE}, {keyA, keyB, keyC, keyD})); EXPECT_EQ(anomalyTracker.mMostRecentBucketNum, 27L); - // TODO: after detectAnomaly fix: EXPECT_EQ(anomalyTracker.mSumOverPastBuckets.size(), 0UL); + EXPECT_EQ(anomalyTracker.mSumOverPastBuckets.size(), 0UL); detectAndDeclareAnomalies(anomalyTracker, 28, bucket28, eventTimestamp6 + 7); - // TODO: after detectAnomaly fix: EXPECT_EQ(anomalyTracker.mSumOverPastBuckets.size(), 0UL); + EXPECT_EQ(anomalyTracker.mSumOverPastBuckets.size(), 0UL); checkRefractoryTimes(anomalyTracker, eventTimestamp6, refractoryPeriodSec, {{keyA, -1}, {keyB, -1}, {keyC, -1}, {keyD, -1}, {keyE, eventTimestamp6 + 7}}); } diff --git a/cmds/statsd/tests/condition/CombinationConditionTracker_test.cpp b/cmds/statsd/tests/condition/CombinationConditionTracker_test.cpp index 23d69267d1c0..6529d65a5825 100644 --- a/cmds/statsd/tests/condition/CombinationConditionTracker_test.cpp +++ b/cmds/statsd/tests/condition/CombinationConditionTracker_test.cpp @@ -40,6 +40,7 @@ TEST(ConditionTrackerTest, TestUnknownCondition) { EXPECT_EQ(evaluateCombinationCondition(children, operation, conditionResults), ConditionState::kUnknown); } + TEST(ConditionTrackerTest, TestAndCondition) { // Set up the matcher LogicalOperation operation = LogicalOperation::AND; @@ -103,6 +104,11 @@ TEST(ConditionTrackerTest, TestNotCondition) { conditionResults.clear(); conditionResults.push_back(ConditionState::kFalse); EXPECT_TRUE(evaluateCombinationCondition(children, operation, conditionResults)); + + children.clear(); + conditionResults.clear(); + EXPECT_EQ(evaluateCombinationCondition(children, operation, conditionResults), + ConditionState::kUnknown); } TEST(ConditionTrackerTest, TestNandCondition) { diff --git a/cmds/statsd/tests/e2e/Attribution_e2e_test.cpp b/cmds/statsd/tests/e2e/Attribution_e2e_test.cpp index a8fcc8163656..5c47af797eea 100644 --- a/cmds/statsd/tests/e2e/Attribution_e2e_test.cpp +++ b/cmds/statsd/tests/e2e/Attribution_e2e_test.cpp @@ -144,8 +144,8 @@ TEST(AttributionE2eTest, TestAttributionMatchAndSliceByFirstUid) { } ConfigMetricsReportList reports; vector<uint8_t> buffer; - processor->onDumpReport(cfgKey, bucketStartTimeNs + 4 * bucketSizeNs + 1, false, ADB_DUMP, - &buffer); + processor->onDumpReport(cfgKey, bucketStartTimeNs + 4 * bucketSizeNs + 1, false, true, + ADB_DUMP, &buffer); EXPECT_TRUE(buffer.size() > 0); EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); backfillDimensionPath(&reports); @@ -290,8 +290,8 @@ TEST(AttributionE2eTest, TestAttributionMatchAndSliceByChain) { } ConfigMetricsReportList reports; vector<uint8_t> buffer; - processor->onDumpReport(cfgKey, bucketStartTimeNs + 4 * bucketSizeNs + 1, false, ADB_DUMP, - &buffer); + processor->onDumpReport(cfgKey, bucketStartTimeNs + 4 * bucketSizeNs + 1, false, true, + ADB_DUMP, &buffer); EXPECT_TRUE(buffer.size() > 0); EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); backfillDimensionPath(&reports); diff --git a/cmds/statsd/tests/e2e/DimensionInCondition_e2e_combination_AND_cond_test.cpp b/cmds/statsd/tests/e2e/DimensionInCondition_e2e_combination_AND_cond_test.cpp index f03821432cc1..a8914da4b7fe 100644 --- a/cmds/statsd/tests/e2e/DimensionInCondition_e2e_combination_AND_cond_test.cpp +++ b/cmds/statsd/tests/e2e/DimensionInCondition_e2e_combination_AND_cond_test.cpp @@ -81,6 +81,34 @@ StatsdConfig CreateDurationMetricConfig_NoLink_AND_CombinationCondition( } // namespace +/* + The following test has the following input. + +{ 10000000002 10000000002 (8)9999[I], [S], job0[S], 1[I], } +{ 10000000010 10000000010 (7)111[I], App1[S], 222[I], GMSCoreModule1[S], 222[I], GMSCoreModule2[S], ReadEmail[S], 1[I], } +{ 10000000011 10000000011 (29)1[I], } +{ 10000000040 10000000040 (29)2[I], } +{ 10000000050 10000000050 (7)111[I], App1[S], 222[I], GMSCoreModule1[S], 222[I], GMSCoreModule2[S], ReadEmail[S], 0[I], } +{ 10000000101 10000000101 (8)9999[I], [S], job0[S], 0[I], } +{ 10000000102 10000000102 (29)1[I], } +{ 10000000200 10000000200 (7)111[I], App1[S], 222[I], GMSCoreModule1[S], 222[I], GMSCoreModule2[S], ReadEmail[S], 1[I], } +{ 10000000201 10000000201 (8)9999[I], [S], job2[S], 1[I], } +{ 10000000400 10000000400 (7)111[I], App1[S], 222[I], GMSCoreModule1[S], 222[I], GMSCoreModule2[S], ReadDoc[S], 1[I], } +{ 10000000401 10000000401 (7)333[I], App2[S], 222[I], GMSCoreModule1[S], 555[I], GMSCoreModule2[S], ReadEmail[S], 1[I], } +{ 10000000450 10000000450 (29)2[I], } +{ 10000000500 10000000500 (8)9999[I], [S], job2[S], 0[I], } +{ 10000000600 10000000600 (8)8888[I], [S], job2[S], 1[I], } +{ 10000000650 10000000650 (29)1[I], } +{ 309999999999 309999999999 (7)111[I], App1[S], 222[I], GMSCoreModule1[S], 222[I], GMSCoreModule2[S], ReadDoc[S], 0[I], } +{ 310000000100 310000000100 (29)2[I], } +{ 310000000300 310000000300 (7)111[I], App1[S], 222[I], GMSCoreModule1[S], 222[I], GMSCoreModule2[S], ReadEmail[S], 0[I], } +{ 310000000600 310000000600 (8)8888[I], [S], job1[S], 1[I], } +{ 310000000640 310000000640 (29)1[I], } +{ 310000000650 310000000650 (29)2[I], } +{ 310000000700 310000000700 (7)333[I], App2[S], 222[I], GMSCoreModule1[S], 555[I], GMSCoreModule2[S], ReadEmail[S], 0[I], } +{ 310000000850 310000000850 (8)8888[I], [S], job2[S], 0[I], } +{ 310000000900 310000000900 (8)8888[I], [S], job1[S], 0[I], } +*/ TEST(DimensionInConditionE2eTest, TestDurationMetric_NoLink_AND_CombinationCondition) { for (const bool hashStringInReport : { true, false }) { for (bool isDimensionInConditionSubSetOfConditionTrackerDimension : { true, false }) { @@ -184,7 +212,7 @@ TEST(DimensionInConditionE2eTest, TestDurationMetric_NoLink_AND_CombinationCondi ConfigMetricsReportList reports; vector<uint8_t> buffer; processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, - ADB_DUMP, &buffer); + true, ADB_DUMP, &buffer); EXPECT_TRUE(buffer.size() > 0); EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); backfillDimensionPath(&reports); @@ -250,7 +278,7 @@ TEST(DimensionInConditionE2eTest, TestDurationMetric_NoLink_AND_CombinationCondi EXPECT_EQ(data.bucket_info(0).start_bucket_elapsed_nanos(), bucketStartTimeNs); EXPECT_EQ(data.bucket_info(0).end_bucket_elapsed_nanos(), bucketStartTimeNs + bucketSizeNs); - EXPECT_EQ(data.bucket_info(0).duration_nanos(), 450 - 201 + bucketSizeNs - 600); + EXPECT_EQ(data.bucket_info(0).duration_nanos(), 450 - 201 + bucketSizeNs - 650); EXPECT_EQ(data.bucket_info(1).duration_nanos(), 100); EXPECT_EQ(data.bucket_info(1).start_bucket_elapsed_nanos(), bucketStartTimeNs + bucketSizeNs); @@ -269,7 +297,7 @@ TEST(DimensionInConditionE2eTest, TestDurationMetric_NoLink_AND_CombinationCondi data.dimensions_in_condition(), android::util::SYNC_STATE_CHANGED, 333, "App2"); EXPECT_EQ(data.bucket_info_size(), 2); - EXPECT_EQ(data.bucket_info(0).duration_nanos(), 450 - 401 + bucketSizeNs - 600); + EXPECT_EQ(data.bucket_info(0).duration_nanos(), 450 - 401 + bucketSizeNs - 650); EXPECT_EQ(data.bucket_info(0).start_bucket_elapsed_nanos(), bucketStartTimeNs); EXPECT_EQ(data.bucket_info(0).end_bucket_elapsed_nanos(), bucketStartTimeNs + bucketSizeNs); @@ -331,7 +359,7 @@ TEST(DimensionInConditionE2eTest, TestDurationMetric_NoLink_AND_CombinationCondi EXPECT_EQ(data.bucket_info(0).end_bucket_elapsed_nanos(), bucketStartTimeNs + bucketSizeNs); EXPECT_EQ(data.bucket_info(0).duration_nanos(), 450 - 201); - EXPECT_EQ(data.bucket_info(1).duration_nanos(), bucketSizeNs - 600 + 100); + EXPECT_EQ(data.bucket_info(1).duration_nanos(), bucketSizeNs - 650 + 100); EXPECT_EQ(data.bucket_info(1).start_bucket_elapsed_nanos(), bucketStartTimeNs + bucketSizeNs); EXPECT_EQ(data.bucket_info(1).end_bucket_elapsed_nanos(), @@ -353,7 +381,7 @@ TEST(DimensionInConditionE2eTest, TestDurationMetric_NoLink_AND_CombinationCondi EXPECT_EQ(data.bucket_info(0).start_bucket_elapsed_nanos(), bucketStartTimeNs); EXPECT_EQ(data.bucket_info(0).end_bucket_elapsed_nanos(), bucketStartTimeNs + bucketSizeNs); - EXPECT_EQ(data.bucket_info(1).duration_nanos(), bucketSizeNs - 600 + 110); + EXPECT_EQ(data.bucket_info(1).duration_nanos(), bucketSizeNs - 650 + 110); EXPECT_EQ(data.bucket_info(1).start_bucket_elapsed_nanos(), bucketStartTimeNs + bucketSizeNs); EXPECT_EQ(data.bucket_info(1).end_bucket_elapsed_nanos(), @@ -520,7 +548,7 @@ TEST(DimensionInConditionE2eTest, TestDurationMetric_Link_AND_CombinationConditi ConfigMetricsReportList reports; vector<uint8_t> buffer; processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, - ADB_DUMP, &buffer); + true, ADB_DUMP, &buffer); EXPECT_TRUE(buffer.size() > 0); EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); backfillDimensionPath(&reports); @@ -770,7 +798,7 @@ TEST(DimensionInConditionE2eTest, TestDurationMetric_PartialLink_AND_Combination ConfigMetricsReportList reports; vector<uint8_t> buffer; processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, - ADB_DUMP, &buffer); + true, ADB_DUMP, &buffer); EXPECT_TRUE(buffer.size() > 0); EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); backfillDimensionPath(&reports); diff --git a/cmds/statsd/tests/e2e/DimensionInCondition_e2e_combination_OR_cond_test.cpp b/cmds/statsd/tests/e2e/DimensionInCondition_e2e_combination_OR_cond_test.cpp index c5a8a2eba4ae..621b6ed6ddbf 100644 --- a/cmds/statsd/tests/e2e/DimensionInCondition_e2e_combination_OR_cond_test.cpp +++ b/cmds/statsd/tests/e2e/DimensionInCondition_e2e_combination_OR_cond_test.cpp @@ -130,8 +130,8 @@ TEST(DimensionInConditionE2eTest, TestCreateCountMetric_NoLink_OR_CombinationCon ConfigMetricsReportList reports; vector<uint8_t> buffer; - processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, ADB_DUMP, - &buffer); + processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, true, + ADB_DUMP, &buffer); EXPECT_TRUE(buffer.size() > 0); EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); backfillDimensionPath(&reports); @@ -346,8 +346,8 @@ TEST(DimensionInConditionE2eTest, TestCreateCountMetric_Link_OR_CombinationCondi ConfigMetricsReportList reports; vector<uint8_t> buffer; - processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, ADB_DUMP, - &buffer); + processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, true, + ADB_DUMP, &buffer); EXPECT_TRUE(buffer.size() > 0); EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); backfillDimensionPath(&reports); @@ -530,8 +530,8 @@ TEST(DimensionInConditionE2eTest, TestDurationMetric_NoLink_OR_CombinationCondit ConfigMetricsReportList reports; vector<uint8_t> buffer; - processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, ADB_DUMP, - &buffer); + processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, true, + ADB_DUMP, &buffer); EXPECT_TRUE(buffer.size() > 0); EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); backfillDimensionPath(&reports); @@ -732,8 +732,8 @@ TEST(DimensionInConditionE2eTest, TestDurationMetric_Link_OR_CombinationConditio ConfigMetricsReportList reports; vector<uint8_t> buffer; - processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, ADB_DUMP, - &buffer); + processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, true, + ADB_DUMP, &buffer); EXPECT_TRUE(buffer.size() > 0); EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); backfillDimensionPath(&reports); diff --git a/cmds/statsd/tests/e2e/DimensionInCondition_e2e_simple_cond_test.cpp b/cmds/statsd/tests/e2e/DimensionInCondition_e2e_simple_cond_test.cpp index 5bcc9ee8513d..9f8acaf18422 100644 --- a/cmds/statsd/tests/e2e/DimensionInCondition_e2e_simple_cond_test.cpp +++ b/cmds/statsd/tests/e2e/DimensionInCondition_e2e_simple_cond_test.cpp @@ -143,7 +143,7 @@ TEST(DimensionInConditionE2eTest, TestDurationMetric_NoLink_SimpleCondition) { ConfigMetricsReportList reports; vector<uint8_t> buffer; processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, - ADB_DUMP, &buffer); + true, ADB_DUMP, &buffer); EXPECT_TRUE(buffer.size() > 0); EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); backfillDimensionPath(&reports); @@ -438,7 +438,7 @@ TEST(DimensionInConditionE2eTest, TestDurationMetric_Link_SimpleCondition) { ConfigMetricsReportList reports; vector<uint8_t> buffer; processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, - ADB_DUMP, &buffer); + true, ADB_DUMP, &buffer); EXPECT_TRUE(buffer.size() > 0); EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); backfillDimensionPath(&reports); @@ -658,8 +658,8 @@ TEST(DimensionInConditionE2eTest, TestDurationMetric_PartialLink_SimpleCondition ConfigMetricsReportList reports; vector<uint8_t> buffer; - processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, ADB_DUMP, - &buffer); + processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, true, + ADB_DUMP, &buffer); EXPECT_TRUE(buffer.size() > 0); EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); backfillDimensionPath(&reports); diff --git a/cmds/statsd/tests/e2e/GaugeMetric_e2e_pull_test.cpp b/cmds/statsd/tests/e2e/GaugeMetric_e2e_pull_test.cpp index eca5690de478..2d090e02a42a 100644 --- a/cmds/statsd/tests/e2e/GaugeMetric_e2e_pull_test.cpp +++ b/cmds/statsd/tests/e2e/GaugeMetric_e2e_pull_test.cpp @@ -48,6 +48,7 @@ StatsdConfig CreateStatsdConfig(const GaugeMetric::SamplingType sampling_type) { *gaugeMetric->mutable_dimensions_in_what() = CreateDimensions(android::util::TEMPERATURE, {2/* sensor name field */ }); gaugeMetric->set_bucket(FIVE_MINUTES); + config.set_hash_strings_in_metric_report(false); return config; } @@ -66,7 +67,7 @@ TEST(GaugeMetricE2eTest, TestRandomSamplePulledEvents) { baseTimeNs, configAddedTimeNs, config, cfgKey); EXPECT_EQ(processor->mMetricsManagers.size(), 1u); EXPECT_TRUE(processor->mMetricsManagers.begin()->second->isConfigValid()); - processor->mStatsPullerManager.ForceClearPullerCache(); + processor->mPullerManager->ForceClearPullerCache(); int startBucketNum = processor->mMetricsManagers.begin()->second-> mAllMetricProducers[0]->getCurrentBucketNum(); @@ -74,12 +75,11 @@ TEST(GaugeMetricE2eTest, TestRandomSamplePulledEvents) { // When creating the config, the gauge metric producer should register the alarm at the // end of the current bucket. - EXPECT_EQ((size_t)1, StatsPullerManagerImpl::GetInstance().mReceivers.size()); + EXPECT_EQ((size_t)1, processor->mPullerManager->mReceivers.size()); EXPECT_EQ(bucketSizeNs, - StatsPullerManagerImpl::GetInstance().mReceivers.begin()-> - second.front().intervalNs); - int64_t& nextPullTimeNs = StatsPullerManagerImpl::GetInstance().mReceivers.begin()-> - second.front().nextPullTimeNs; + processor->mPullerManager->mReceivers.begin()->second.front().intervalNs); + int64_t& nextPullTimeNs = + processor->mPullerManager->mReceivers.begin()->second.front().nextPullTimeNs; EXPECT_EQ(baseTimeNs + startBucketNum * bucketSizeNs + bucketSizeNs, nextPullTimeNs); auto screenOffEvent = CreateScreenStateChangedEvent(android::view::DISPLAY_STATE_OFF, @@ -123,8 +123,8 @@ TEST(GaugeMetricE2eTest, TestRandomSamplePulledEvents) { ConfigMetricsReportList reports; vector<uint8_t> buffer; - processor->onDumpReport(cfgKey, configAddedTimeNs + 7 * bucketSizeNs + 10, false, ADB_DUMP, - &buffer); + processor->onDumpReport(cfgKey, configAddedTimeNs + 7 * bucketSizeNs + 10, false, true, + ADB_DUMP, &buffer); EXPECT_TRUE(buffer.size() > 0); EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); backfillDimensionPath(&reports); @@ -148,11 +148,11 @@ TEST(GaugeMetricE2eTest, TestRandomSamplePulledEvents) { EXPECT_EQ(1, data.bucket_info(0).atom_size()); EXPECT_EQ(1, data.bucket_info(0).elapsed_timestamp_nanos_size()); EXPECT_EQ(configAddedTimeNs + 55, data.bucket_info(0).elapsed_timestamp_nanos(0)); - EXPECT_EQ(1, data.bucket_info(0).wall_clock_timestamp_nanos_size()); + EXPECT_EQ(0, data.bucket_info(0).wall_clock_timestamp_nanos_size()); EXPECT_EQ(baseTimeNs + 2 * bucketSizeNs, data.bucket_info(0).start_bucket_elapsed_nanos()); EXPECT_EQ(baseTimeNs + 3 * bucketSizeNs, data.bucket_info(0).end_bucket_elapsed_nanos()); - EXPECT_FALSE(data.bucket_info(0).atom(0).temperature().sensor_name().empty()); - EXPECT_GT(data.bucket_info(0).atom(0).temperature().temperature_dc(), 0); + EXPECT_TRUE(data.bucket_info(0).atom(0).temperature().sensor_name().empty()); + EXPECT_GT(data.bucket_info(0).atom(0).temperature().temperature_deci_celsius(), 0); EXPECT_EQ(1, data.bucket_info(1).atom_size()); EXPECT_EQ(baseTimeNs + 3 * bucketSizeNs + 1, @@ -160,8 +160,8 @@ TEST(GaugeMetricE2eTest, TestRandomSamplePulledEvents) { EXPECT_EQ(configAddedTimeNs + 55, data.bucket_info(0).elapsed_timestamp_nanos(0)); EXPECT_EQ(baseTimeNs + 3 * bucketSizeNs, data.bucket_info(1).start_bucket_elapsed_nanos()); EXPECT_EQ(baseTimeNs + 4 * bucketSizeNs, data.bucket_info(1).end_bucket_elapsed_nanos()); - EXPECT_FALSE(data.bucket_info(1).atom(0).temperature().sensor_name().empty()); - EXPECT_GT(data.bucket_info(1).atom(0).temperature().temperature_dc(), 0); + EXPECT_TRUE(data.bucket_info(1).atom(0).temperature().sensor_name().empty()); + EXPECT_GT(data.bucket_info(1).atom(0).temperature().temperature_deci_celsius(), 0); EXPECT_EQ(1, data.bucket_info(2).atom_size()); EXPECT_EQ(1, data.bucket_info(2).elapsed_timestamp_nanos_size()); @@ -169,8 +169,8 @@ TEST(GaugeMetricE2eTest, TestRandomSamplePulledEvents) { data.bucket_info(2).elapsed_timestamp_nanos(0)); EXPECT_EQ(baseTimeNs + 4 * bucketSizeNs, data.bucket_info(2).start_bucket_elapsed_nanos()); EXPECT_EQ(baseTimeNs + 5 * bucketSizeNs, data.bucket_info(2).end_bucket_elapsed_nanos()); - EXPECT_FALSE(data.bucket_info(2).atom(0).temperature().sensor_name().empty()); - EXPECT_GT(data.bucket_info(2).atom(0).temperature().temperature_dc(), 0); + EXPECT_TRUE(data.bucket_info(2).atom(0).temperature().sensor_name().empty()); + EXPECT_GT(data.bucket_info(2).atom(0).temperature().temperature_deci_celsius(), 0); EXPECT_EQ(1, data.bucket_info(3).atom_size()); EXPECT_EQ(1, data.bucket_info(3).elapsed_timestamp_nanos_size()); @@ -178,8 +178,8 @@ TEST(GaugeMetricE2eTest, TestRandomSamplePulledEvents) { data.bucket_info(3).elapsed_timestamp_nanos(0)); EXPECT_EQ(baseTimeNs + 5 * bucketSizeNs, data.bucket_info(3).start_bucket_elapsed_nanos()); EXPECT_EQ(baseTimeNs + 6 * bucketSizeNs, data.bucket_info(3).end_bucket_elapsed_nanos()); - EXPECT_FALSE(data.bucket_info(3).atom(0).temperature().sensor_name().empty()); - EXPECT_GT(data.bucket_info(3).atom(0).temperature().temperature_dc(), 0); + EXPECT_TRUE(data.bucket_info(3).atom(0).temperature().sensor_name().empty()); + EXPECT_GT(data.bucket_info(3).atom(0).temperature().temperature_deci_celsius(), 0); EXPECT_EQ(1, data.bucket_info(4).atom_size()); EXPECT_EQ(1, data.bucket_info(4).elapsed_timestamp_nanos_size()); @@ -187,8 +187,8 @@ TEST(GaugeMetricE2eTest, TestRandomSamplePulledEvents) { data.bucket_info(4).elapsed_timestamp_nanos(0)); EXPECT_EQ(baseTimeNs + 7 * bucketSizeNs, data.bucket_info(4).start_bucket_elapsed_nanos()); EXPECT_EQ(baseTimeNs + 8 * bucketSizeNs, data.bucket_info(4).end_bucket_elapsed_nanos()); - EXPECT_FALSE(data.bucket_info(4).atom(0).temperature().sensor_name().empty()); - EXPECT_GT(data.bucket_info(4).atom(0).temperature().temperature_dc(), 0); + EXPECT_TRUE(data.bucket_info(4).atom(0).temperature().sensor_name().empty()); + EXPECT_GT(data.bucket_info(4).atom(0).temperature().temperature_deci_celsius(), 0); EXPECT_EQ(1, data.bucket_info(5).atom_size()); EXPECT_EQ(1, data.bucket_info(5).elapsed_timestamp_nanos_size()); @@ -196,12 +196,12 @@ TEST(GaugeMetricE2eTest, TestRandomSamplePulledEvents) { data.bucket_info(5).elapsed_timestamp_nanos(0)); EXPECT_EQ(baseTimeNs + 8 * bucketSizeNs, data.bucket_info(5).start_bucket_elapsed_nanos()); EXPECT_EQ(baseTimeNs + 9 * bucketSizeNs, data.bucket_info(5).end_bucket_elapsed_nanos()); - EXPECT_FALSE(data.bucket_info(5).atom(0).temperature().sensor_name().empty()); - EXPECT_GT(data.bucket_info(5).atom(0).temperature().temperature_dc(), 0); + EXPECT_TRUE(data.bucket_info(5).atom(0).temperature().sensor_name().empty()); + EXPECT_GT(data.bucket_info(5).atom(0).temperature().temperature_deci_celsius(), 0); } -TEST(GaugeMetricE2eTest, TestAllConditionChangesSamplePulledEvents) { - auto config = CreateStatsdConfig(GaugeMetric::ALL_CONDITION_CHANGES); +TEST(GaugeMetricE2eTest, TestConditionChangeToTrueSamplePulledEvents) { + auto config = CreateStatsdConfig(GaugeMetric::CONDITION_CHANGE_TO_TRUE); int64_t baseTimeNs = 10 * NS_PER_SEC; int64_t configAddedTimeNs = 10 * 60 * NS_PER_SEC + baseTimeNs; int64_t bucketSizeNs = @@ -212,7 +212,7 @@ TEST(GaugeMetricE2eTest, TestAllConditionChangesSamplePulledEvents) { baseTimeNs, configAddedTimeNs, config, cfgKey); EXPECT_EQ(processor->mMetricsManagers.size(), 1u); EXPECT_TRUE(processor->mMetricsManagers.begin()->second->isConfigValid()); - processor->mStatsPullerManager.ForceClearPullerCache(); + processor->mPullerManager->ForceClearPullerCache(); int startBucketNum = processor->mMetricsManagers.begin()->second-> mAllMetricProducers[0]->getCurrentBucketNum(); @@ -246,8 +246,8 @@ TEST(GaugeMetricE2eTest, TestAllConditionChangesSamplePulledEvents) { ConfigMetricsReportList reports; vector<uint8_t> buffer; - processor->onDumpReport(cfgKey, configAddedTimeNs + 8 * bucketSizeNs + 10, false, ADB_DUMP, - &buffer); + processor->onDumpReport(cfgKey, configAddedTimeNs + 8 * bucketSizeNs + 10, false, true, + ADB_DUMP, &buffer); EXPECT_TRUE(buffer.size() > 0); EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); backfillDimensionPath(&reports); @@ -271,11 +271,11 @@ TEST(GaugeMetricE2eTest, TestAllConditionChangesSamplePulledEvents) { EXPECT_EQ(1, data.bucket_info(0).atom_size()); EXPECT_EQ(1, data.bucket_info(0).elapsed_timestamp_nanos_size()); EXPECT_EQ(configAddedTimeNs + 55, data.bucket_info(0).elapsed_timestamp_nanos(0)); - EXPECT_EQ(1, data.bucket_info(0).wall_clock_timestamp_nanos_size()); + EXPECT_EQ(0, data.bucket_info(0).wall_clock_timestamp_nanos_size()); EXPECT_EQ(baseTimeNs + 2 * bucketSizeNs, data.bucket_info(0).start_bucket_elapsed_nanos()); EXPECT_EQ(baseTimeNs + 3 * bucketSizeNs, data.bucket_info(0).end_bucket_elapsed_nanos()); - EXPECT_FALSE(data.bucket_info(0).atom(0).temperature().sensor_name().empty()); - EXPECT_GT(data.bucket_info(0).atom(0).temperature().temperature_dc(), 0); + EXPECT_TRUE(data.bucket_info(0).atom(0).temperature().sensor_name().empty()); + EXPECT_GT(data.bucket_info(0).atom(0).temperature().temperature_deci_celsius(), 0); EXPECT_EQ(1, data.bucket_info(1).atom_size()); EXPECT_EQ(baseTimeNs + 3 * bucketSizeNs + 100, @@ -283,8 +283,8 @@ TEST(GaugeMetricE2eTest, TestAllConditionChangesSamplePulledEvents) { EXPECT_EQ(configAddedTimeNs + 55, data.bucket_info(0).elapsed_timestamp_nanos(0)); EXPECT_EQ(baseTimeNs + 3 * bucketSizeNs, data.bucket_info(1).start_bucket_elapsed_nanos()); EXPECT_EQ(baseTimeNs + 4 * bucketSizeNs, data.bucket_info(1).end_bucket_elapsed_nanos()); - EXPECT_FALSE(data.bucket_info(1).atom(0).temperature().sensor_name().empty()); - EXPECT_GT(data.bucket_info(1).atom(0).temperature().temperature_dc(), 0); + EXPECT_TRUE(data.bucket_info(1).atom(0).temperature().sensor_name().empty()); + EXPECT_GT(data.bucket_info(1).atom(0).temperature().temperature_deci_celsius(), 0); EXPECT_EQ(2, data.bucket_info(2).atom_size()); EXPECT_EQ(2, data.bucket_info(2).elapsed_timestamp_nanos_size()); @@ -294,10 +294,10 @@ TEST(GaugeMetricE2eTest, TestAllConditionChangesSamplePulledEvents) { data.bucket_info(2).elapsed_timestamp_nanos(1)); EXPECT_EQ(baseTimeNs + 7 * bucketSizeNs, data.bucket_info(2).start_bucket_elapsed_nanos()); EXPECT_EQ(baseTimeNs + 8 * bucketSizeNs, data.bucket_info(2).end_bucket_elapsed_nanos()); - EXPECT_FALSE(data.bucket_info(2).atom(0).temperature().sensor_name().empty()); - EXPECT_GT(data.bucket_info(2).atom(0).temperature().temperature_dc(), 0); - EXPECT_FALSE(data.bucket_info(2).atom(1).temperature().sensor_name().empty()); - EXPECT_GT(data.bucket_info(2).atom(1).temperature().temperature_dc(), 0); + EXPECT_TRUE(data.bucket_info(2).atom(0).temperature().sensor_name().empty()); + EXPECT_GT(data.bucket_info(2).atom(0).temperature().temperature_deci_celsius(), 0); + EXPECT_TRUE(data.bucket_info(2).atom(1).temperature().sensor_name().empty()); + EXPECT_GT(data.bucket_info(2).atom(1).temperature().temperature_deci_celsius(), 0); } @@ -313,7 +313,7 @@ TEST(GaugeMetricE2eTest, TestRandomSamplePulledEvent_LateAlarm) { baseTimeNs, configAddedTimeNs, config, cfgKey); EXPECT_EQ(processor->mMetricsManagers.size(), 1u); EXPECT_TRUE(processor->mMetricsManagers.begin()->second->isConfigValid()); - processor->mStatsPullerManager.ForceClearPullerCache(); + processor->mPullerManager->ForceClearPullerCache(); int startBucketNum = processor->mMetricsManagers.begin()->second-> mAllMetricProducers[0]->getCurrentBucketNum(); @@ -321,12 +321,11 @@ TEST(GaugeMetricE2eTest, TestRandomSamplePulledEvent_LateAlarm) { // When creating the config, the gauge metric producer should register the alarm at the // end of the current bucket. - EXPECT_EQ((size_t)1, StatsPullerManagerImpl::GetInstance().mReceivers.size()); + EXPECT_EQ((size_t)1, processor->mPullerManager->mReceivers.size()); EXPECT_EQ(bucketSizeNs, - StatsPullerManagerImpl::GetInstance().mReceivers.begin()-> - second.front().intervalNs); - int64_t& nextPullTimeNs = StatsPullerManagerImpl::GetInstance().mReceivers.begin()-> - second.front().nextPullTimeNs; + processor->mPullerManager->mReceivers.begin()->second.front().intervalNs); + int64_t& nextPullTimeNs = + processor->mPullerManager->mReceivers.begin()->second.front().nextPullTimeNs; EXPECT_EQ(baseTimeNs + startBucketNum * bucketSizeNs + bucketSizeNs, nextPullTimeNs); auto screenOffEvent = CreateScreenStateChangedEvent(android::view::DISPLAY_STATE_OFF, @@ -351,8 +350,8 @@ TEST(GaugeMetricE2eTest, TestRandomSamplePulledEvent_LateAlarm) { ConfigMetricsReportList reports; vector<uint8_t> buffer; - processor->onDumpReport(cfgKey, configAddedTimeNs + 7 * bucketSizeNs + 10, false, ADB_DUMP, - &buffer); + processor->onDumpReport(cfgKey, configAddedTimeNs + 7 * bucketSizeNs + 10, false, true, + ADB_DUMP, &buffer); EXPECT_TRUE(buffer.size() > 0); EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); backfillDimensionPath(&reports); @@ -376,11 +375,10 @@ TEST(GaugeMetricE2eTest, TestRandomSamplePulledEvent_LateAlarm) { EXPECT_EQ(1, data.bucket_info(0).atom_size()); EXPECT_EQ(1, data.bucket_info(0).elapsed_timestamp_nanos_size()); EXPECT_EQ(configAddedTimeNs + 55, data.bucket_info(0).elapsed_timestamp_nanos(0)); - EXPECT_EQ(1, data.bucket_info(0).wall_clock_timestamp_nanos_size()); EXPECT_EQ(baseTimeNs + 2 * bucketSizeNs, data.bucket_info(0).start_bucket_elapsed_nanos()); EXPECT_EQ(baseTimeNs + 3 * bucketSizeNs, data.bucket_info(0).end_bucket_elapsed_nanos()); - EXPECT_FALSE(data.bucket_info(0).atom(0).temperature().sensor_name().empty()); - EXPECT_GT(data.bucket_info(0).atom(0).temperature().temperature_dc(), 0); + EXPECT_TRUE(data.bucket_info(0).atom(0).temperature().sensor_name().empty()); + EXPECT_GT(data.bucket_info(0).atom(0).temperature().temperature_deci_celsius(), 0); EXPECT_EQ(1, data.bucket_info(1).atom_size()); EXPECT_EQ(configAddedTimeNs + 3 * bucketSizeNs + 11, @@ -388,8 +386,8 @@ TEST(GaugeMetricE2eTest, TestRandomSamplePulledEvent_LateAlarm) { EXPECT_EQ(configAddedTimeNs + 55, data.bucket_info(0).elapsed_timestamp_nanos(0)); EXPECT_EQ(baseTimeNs + 5 * bucketSizeNs, data.bucket_info(1).start_bucket_elapsed_nanos()); EXPECT_EQ(baseTimeNs + 6 * bucketSizeNs, data.bucket_info(1).end_bucket_elapsed_nanos()); - EXPECT_FALSE(data.bucket_info(1).atom(0).temperature().sensor_name().empty()); - EXPECT_GT(data.bucket_info(1).atom(0).temperature().temperature_dc(), 0); + EXPECT_TRUE(data.bucket_info(1).atom(0).temperature().sensor_name().empty()); + EXPECT_GT(data.bucket_info(1).atom(0).temperature().temperature_deci_celsius(), 0); EXPECT_EQ(1, data.bucket_info(2).atom_size()); EXPECT_EQ(1, data.bucket_info(2).elapsed_timestamp_nanos_size()); @@ -397,8 +395,8 @@ TEST(GaugeMetricE2eTest, TestRandomSamplePulledEvent_LateAlarm) { data.bucket_info(2).elapsed_timestamp_nanos(0)); EXPECT_EQ(baseTimeNs + 6 * bucketSizeNs, data.bucket_info(2).start_bucket_elapsed_nanos()); EXPECT_EQ(baseTimeNs + 7 * bucketSizeNs, data.bucket_info(2).end_bucket_elapsed_nanos()); - EXPECT_FALSE(data.bucket_info(2).atom(0).temperature().sensor_name().empty()); - EXPECT_GT(data.bucket_info(2).atom(0).temperature().temperature_dc(), 0); + EXPECT_TRUE(data.bucket_info(2).atom(0).temperature().sensor_name().empty()); + EXPECT_GT(data.bucket_info(2).atom(0).temperature().temperature_deci_celsius(), 0); } diff --git a/cmds/statsd/tests/e2e/GaugeMetric_e2e_push_test.cpp b/cmds/statsd/tests/e2e/GaugeMetric_e2e_push_test.cpp index 3de8d0da2c3b..71afedfa6c8f 100644 --- a/cmds/statsd/tests/e2e/GaugeMetric_e2e_push_test.cpp +++ b/cmds/statsd/tests/e2e/GaugeMetric_e2e_push_test.cpp @@ -149,8 +149,8 @@ TEST(GaugeMetricE2eTest, TestMultipleFieldsForPushedEvent) { } ConfigMetricsReportList reports; vector<uint8_t> buffer; - processor->onDumpReport(cfgKey, bucketStartTimeNs + 3 * bucketSizeNs, false, ADB_DUMP, - &buffer); + processor->onDumpReport(cfgKey, bucketStartTimeNs + 3 * bucketSizeNs, false, true, + ADB_DUMP, &buffer); EXPECT_TRUE(buffer.size() > 0); EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); backfillDimensionPath(&reports); @@ -173,7 +173,7 @@ TEST(GaugeMetricE2eTest, TestMultipleFieldsForPushedEvent) { if (sampling_type == GaugeMetric::ALL_CONDITION_CHANGES) { EXPECT_EQ(2, data.bucket_info(0).atom_size()); EXPECT_EQ(2, data.bucket_info(0).elapsed_timestamp_nanos_size()); - EXPECT_EQ(2, data.bucket_info(0).wall_clock_timestamp_nanos_size()); + EXPECT_EQ(0, data.bucket_info(0).wall_clock_timestamp_nanos_size()); EXPECT_EQ(bucketStartTimeNs, data.bucket_info(0).start_bucket_elapsed_nanos()); EXPECT_EQ(bucketStartTimeNs + bucketSizeNs, data.bucket_info(0).end_bucket_elapsed_nanos()); @@ -192,7 +192,6 @@ TEST(GaugeMetricE2eTest, TestMultipleFieldsForPushedEvent) { EXPECT_EQ(1, data.bucket_info(1).atom_size()); EXPECT_EQ(1, data.bucket_info(1).elapsed_timestamp_nanos_size()); - EXPECT_EQ(1, data.bucket_info(1).wall_clock_timestamp_nanos_size()); EXPECT_EQ(bucketStartTimeNs + bucketSizeNs, data.bucket_info(1).start_bucket_elapsed_nanos()); EXPECT_EQ(bucketStartTimeNs + 2 * bucketSizeNs, @@ -206,7 +205,6 @@ TEST(GaugeMetricE2eTest, TestMultipleFieldsForPushedEvent) { EXPECT_EQ(2, data.bucket_info(2).atom_size()); EXPECT_EQ(2, data.bucket_info(2).elapsed_timestamp_nanos_size()); - EXPECT_EQ(2, data.bucket_info(2).wall_clock_timestamp_nanos_size()); EXPECT_EQ(bucketStartTimeNs + 2 * bucketSizeNs, data.bucket_info(2).start_bucket_elapsed_nanos()); EXPECT_EQ(bucketStartTimeNs + 3 * bucketSizeNs, @@ -226,7 +224,6 @@ TEST(GaugeMetricE2eTest, TestMultipleFieldsForPushedEvent) { } else { EXPECT_EQ(1, data.bucket_info(0).atom_size()); EXPECT_EQ(1, data.bucket_info(0).elapsed_timestamp_nanos_size()); - EXPECT_EQ(1, data.bucket_info(0).wall_clock_timestamp_nanos_size()); EXPECT_EQ(bucketStartTimeNs, data.bucket_info(0).start_bucket_elapsed_nanos()); EXPECT_EQ(bucketStartTimeNs + bucketSizeNs, data.bucket_info(0).end_bucket_elapsed_nanos()); @@ -239,7 +236,6 @@ TEST(GaugeMetricE2eTest, TestMultipleFieldsForPushedEvent) { EXPECT_EQ(1, data.bucket_info(1).atom_size()); EXPECT_EQ(1, data.bucket_info(1).elapsed_timestamp_nanos_size()); - EXPECT_EQ(1, data.bucket_info(1).wall_clock_timestamp_nanos_size()); EXPECT_EQ(bucketStartTimeNs + bucketSizeNs, data.bucket_info(1).start_bucket_elapsed_nanos()); EXPECT_EQ(bucketStartTimeNs + 2 * bucketSizeNs, @@ -253,7 +249,6 @@ TEST(GaugeMetricE2eTest, TestMultipleFieldsForPushedEvent) { EXPECT_EQ(1, data.bucket_info(2).atom_size()); EXPECT_EQ(1, data.bucket_info(2).elapsed_timestamp_nanos_size()); - EXPECT_EQ(1, data.bucket_info(2).wall_clock_timestamp_nanos_size()); EXPECT_EQ(bucketStartTimeNs + 2 * bucketSizeNs, data.bucket_info(2).start_bucket_elapsed_nanos()); EXPECT_EQ(bucketStartTimeNs + 3 * bucketSizeNs, @@ -276,7 +271,6 @@ TEST(GaugeMetricE2eTest, TestMultipleFieldsForPushedEvent) { EXPECT_EQ(1, data.bucket_info_size()); EXPECT_EQ(1, data.bucket_info(0).atom_size()); EXPECT_EQ(1, data.bucket_info(0).elapsed_timestamp_nanos_size()); - EXPECT_EQ(1, data.bucket_info(0).wall_clock_timestamp_nanos_size()); EXPECT_EQ(bucketStartTimeNs + 2 * bucketSizeNs, data.bucket_info(0).start_bucket_elapsed_nanos()); EXPECT_EQ(bucketStartTimeNs + 3 * bucketSizeNs, diff --git a/cmds/statsd/tests/e2e/MetricActivation_e2e_test.cpp b/cmds/statsd/tests/e2e/MetricActivation_e2e_test.cpp new file mode 100644 index 000000000000..29e86f3f9456 --- /dev/null +++ b/cmds/statsd/tests/e2e/MetricActivation_e2e_test.cpp @@ -0,0 +1,242 @@ +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <gtest/gtest.h> + +#include "src/StatsLogProcessor.h" +#include "src/stats_log_util.h" +#include "tests/statsd_test_util.h" + +#include <vector> + +namespace android { +namespace os { +namespace statsd { + +#ifdef __ANDROID__ + +namespace { + +StatsdConfig CreateStatsdConfig() { + StatsdConfig config; + config.add_allowed_log_source("AID_ROOT"); // LogEvent defaults to UID of root. + auto crashMatcher = CreateProcessCrashAtomMatcher(); + auto screenOnMatcher = CreateScreenTurnedOnAtomMatcher(); + auto saverModeMatcher = CreateBatterySaverModeStartAtomMatcher(); + + *config.add_atom_matcher() = saverModeMatcher; + *config.add_atom_matcher() = crashMatcher; + *config.add_atom_matcher() = screenOnMatcher; + + int64_t metricId = 123456; + auto countMetric = config.add_count_metric(); + countMetric->set_id(metricId); + countMetric->set_what(crashMatcher.id()); + countMetric->set_bucket(FIVE_MINUTES); + countMetric->mutable_dimensions_in_what()->set_field( + android::util::PROCESS_LIFE_CYCLE_STATE_CHANGED); + countMetric->mutable_dimensions_in_what()->add_child()->set_field(1); // uid field + + auto metric_activation1 = config.add_metric_activation(); + metric_activation1->set_metric_id(metricId); + auto event_activation1 = metric_activation1->add_event_activation(); + event_activation1->set_atom_matcher_id(saverModeMatcher.id()); + event_activation1->set_ttl_seconds(60 * 6); // 6 minutes + auto event_activation2 = metric_activation1->add_event_activation(); + event_activation2->set_atom_matcher_id(screenOnMatcher.id()); + event_activation2->set_ttl_seconds(60 * 2); // 2 minutes + + return config; +} + +} // namespace + +TEST(MetricActivationE2eTest, TestCountMetric) { + auto config = CreateStatsdConfig(); + + int64_t bucketStartTimeNs = 10000000000; + int64_t bucketSizeNs = + TimeUnitToBucketSizeInMillis(config.count_metric(0).bucket()) * 1000000; + + ConfigKey cfgKey; + auto processor = CreateStatsLogProcessor(bucketStartTimeNs, bucketStartTimeNs, config, cfgKey); + EXPECT_EQ(processor->mMetricsManagers.size(), 1u); + EXPECT_TRUE(processor->mMetricsManagers.begin()->second->isConfigValid()); + sp<MetricProducer> metricProducer = + processor->mMetricsManagers.begin()->second->mAllMetricProducers[0]; + auto& eventActivationMap = metricProducer->mEventActivationMap; + + EXPECT_FALSE(metricProducer->mIsActive); + // Two activations: one is triggered by battery saver mode (tracker index 0), the other is + // triggered by screen on event (tracker index 2). + EXPECT_EQ(eventActivationMap.size(), 2u); + EXPECT_TRUE(eventActivationMap.find(0) != eventActivationMap.end()); + EXPECT_TRUE(eventActivationMap.find(2) != eventActivationMap.end()); + EXPECT_EQ(eventActivationMap[0].state, ActivationState::kNotActive); + EXPECT_EQ(eventActivationMap[0].activation_ns, 0); + EXPECT_EQ(eventActivationMap[0].ttl_ns, 60 * 6 * NS_PER_SEC); + EXPECT_EQ(eventActivationMap[2].state, ActivationState::kNotActive); + EXPECT_EQ(eventActivationMap[2].activation_ns, 0); + EXPECT_EQ(eventActivationMap[2].ttl_ns, 60 * 2 * NS_PER_SEC); + + std::unique_ptr<LogEvent> event; + + event = CreateAppCrashEvent(111, bucketStartTimeNs + 5); + processor->OnLogEvent(event.get()); + EXPECT_FALSE(metricProducer->mIsActive); + + // Activated by battery save mode. + event = CreateBatterySaverOnEvent(bucketStartTimeNs + 10); + processor->OnLogEvent(event.get()); + EXPECT_TRUE(metricProducer->mIsActive); + EXPECT_EQ(eventActivationMap[0].state, ActivationState::kActive); + EXPECT_EQ(eventActivationMap[0].activation_ns, bucketStartTimeNs + 10); + EXPECT_EQ(eventActivationMap[0].ttl_ns, 60 * 6 * NS_PER_SEC); + EXPECT_EQ(eventActivationMap[2].state, ActivationState::kNotActive); + EXPECT_EQ(eventActivationMap[2].activation_ns, 0); + EXPECT_EQ(eventActivationMap[2].ttl_ns, 60 * 2 * NS_PER_SEC); + + // First processed event. + event = CreateAppCrashEvent(222, bucketStartTimeNs + 15); + processor->OnLogEvent(event.get()); + + // Activated by screen on event. + event = CreateScreenStateChangedEvent(android::view::DISPLAY_STATE_ON, + bucketStartTimeNs + 20); + processor->OnLogEvent(event.get()); + EXPECT_TRUE(metricProducer->mIsActive); + EXPECT_EQ(eventActivationMap[0].state, ActivationState::kActive); + EXPECT_EQ(eventActivationMap[0].activation_ns, bucketStartTimeNs + 10); + EXPECT_EQ(eventActivationMap[0].ttl_ns, 60 * 6 * NS_PER_SEC); + EXPECT_EQ(eventActivationMap[2].state, ActivationState::kActive); + EXPECT_EQ(eventActivationMap[2].activation_ns, bucketStartTimeNs + 20); + EXPECT_EQ(eventActivationMap[2].ttl_ns, 60 * 2 * NS_PER_SEC); + + // 2nd processed event. + // The activation by screen_on event expires, but the one by battery save mode is still active. + event = CreateAppCrashEvent(333, bucketStartTimeNs + NS_PER_SEC * 60 * 2 + 25); + processor->OnLogEvent(event.get()); + EXPECT_TRUE(metricProducer->mIsActive); + EXPECT_EQ(eventActivationMap[0].state, ActivationState::kActive); + EXPECT_EQ(eventActivationMap[0].activation_ns, bucketStartTimeNs + 10); + EXPECT_EQ(eventActivationMap[0].ttl_ns, 60 * 6 * NS_PER_SEC); + EXPECT_EQ(eventActivationMap[2].state, ActivationState::kNotActive); + EXPECT_EQ(eventActivationMap[2].activation_ns, bucketStartTimeNs + 20); + EXPECT_EQ(eventActivationMap[2].ttl_ns, 60 * 2 * NS_PER_SEC); + + // 3rd processed event. + event = CreateAppCrashEvent(444, bucketStartTimeNs + NS_PER_SEC * 60 * 5 + 25); + processor->OnLogEvent(event.get()); + + // All activations expired. + event = CreateAppCrashEvent(555, bucketStartTimeNs + NS_PER_SEC * 60 * 8); + processor->OnLogEvent(event.get()); + EXPECT_FALSE(metricProducer->mIsActive); + EXPECT_EQ(eventActivationMap[0].state, ActivationState::kNotActive); + EXPECT_EQ(eventActivationMap[0].activation_ns, bucketStartTimeNs + 10); + EXPECT_EQ(eventActivationMap[0].ttl_ns, 60 * 6 * NS_PER_SEC); + EXPECT_EQ(eventActivationMap[2].state, ActivationState::kNotActive); + EXPECT_EQ(eventActivationMap[2].activation_ns, bucketStartTimeNs + 20); + EXPECT_EQ(eventActivationMap[2].ttl_ns, 60 * 2 * NS_PER_SEC); + + // Re-activate. + event = CreateScreenStateChangedEvent(android::view::DISPLAY_STATE_ON, + bucketStartTimeNs + NS_PER_SEC * 60 * 10 + 10); + processor->OnLogEvent(event.get()); + EXPECT_TRUE(metricProducer->mIsActive); + EXPECT_EQ(eventActivationMap[0].state, ActivationState::kNotActive); + EXPECT_EQ(eventActivationMap[0].activation_ns, bucketStartTimeNs + 10); + EXPECT_EQ(eventActivationMap[0].ttl_ns, 60 * 6 * NS_PER_SEC); + EXPECT_EQ(eventActivationMap[2].state, ActivationState::kActive); + EXPECT_EQ(eventActivationMap[2].activation_ns, bucketStartTimeNs + NS_PER_SEC * 60 * 10 + 10); + EXPECT_EQ(eventActivationMap[2].ttl_ns, 60 * 2 * NS_PER_SEC); + + event = CreateAppCrashEvent(666, bucketStartTimeNs + NS_PER_SEC * 60 * 11 + 1); + processor->OnLogEvent(event.get()); + + ConfigMetricsReportList reports; + vector<uint8_t> buffer; + processor->onDumpReport(cfgKey, bucketStartTimeNs + NS_PER_SEC * 60 * 15 + 1, false, true, + ADB_DUMP, &buffer); + EXPECT_TRUE(buffer.size() > 0); + EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); + backfillDimensionPath(&reports); + backfillStartEndTimestamp(&reports); + EXPECT_EQ(1, reports.reports_size()); + EXPECT_EQ(1, reports.reports(0).metrics_size()); + EXPECT_EQ(4, reports.reports(0).metrics(0).count_metrics().data_size()); + + StatsLogReport::CountMetricDataWrapper countMetrics; + sortMetricDataByDimensionsValue( + reports.reports(0).metrics(0).count_metrics(), &countMetrics); + EXPECT_EQ(4, countMetrics.data_size()); + + auto data = countMetrics.data(0); + EXPECT_EQ(android::util::PROCESS_LIFE_CYCLE_STATE_CHANGED, data.dimensions_in_what().field()); + EXPECT_EQ(1, data.dimensions_in_what().value_tuple().dimensions_value_size()); + EXPECT_EQ(1 /* uid field */, + data.dimensions_in_what().value_tuple().dimensions_value(0).field()); + EXPECT_EQ(222, data.dimensions_in_what().value_tuple().dimensions_value(0).value_int()); + EXPECT_EQ(1, data.bucket_info_size()); + EXPECT_EQ(1, data.bucket_info(0).count()); + EXPECT_EQ(bucketStartTimeNs, data.bucket_info(0).start_bucket_elapsed_nanos()); + EXPECT_EQ(bucketStartTimeNs + bucketSizeNs, data.bucket_info(0).end_bucket_elapsed_nanos()); + + data = countMetrics.data(1); + EXPECT_EQ(android::util::PROCESS_LIFE_CYCLE_STATE_CHANGED, data.dimensions_in_what().field()); + EXPECT_EQ(1, data.dimensions_in_what().value_tuple().dimensions_value_size()); + EXPECT_EQ(1 /* uid field */, + data.dimensions_in_what().value_tuple().dimensions_value(0).field()); + EXPECT_EQ(333, data.dimensions_in_what().value_tuple().dimensions_value(0).value_int()); + EXPECT_EQ(1, data.bucket_info_size()); + EXPECT_EQ(1, data.bucket_info(0).count()); + EXPECT_EQ(bucketStartTimeNs, data.bucket_info(0).start_bucket_elapsed_nanos()); + EXPECT_EQ(bucketStartTimeNs + bucketSizeNs, data.bucket_info(0).end_bucket_elapsed_nanos()); + + data = countMetrics.data(2); + EXPECT_EQ(android::util::PROCESS_LIFE_CYCLE_STATE_CHANGED, data.dimensions_in_what().field()); + EXPECT_EQ(1, data.dimensions_in_what().value_tuple().dimensions_value_size()); + EXPECT_EQ(1 /* uid field */, + data.dimensions_in_what().value_tuple().dimensions_value(0).field()); + EXPECT_EQ(444, data.dimensions_in_what().value_tuple().dimensions_value(0).value_int()); + EXPECT_EQ(1, data.bucket_info_size()); + EXPECT_EQ(1, data.bucket_info(0).count()); + // Partial bucket as metric is deactivated. + EXPECT_EQ(bucketStartTimeNs + bucketSizeNs, data.bucket_info(0).start_bucket_elapsed_nanos()); + EXPECT_EQ(bucketStartTimeNs + NS_PER_SEC * 60 * 8, + data.bucket_info(0).end_bucket_elapsed_nanos()); + + data = countMetrics.data(3); + EXPECT_EQ(android::util::PROCESS_LIFE_CYCLE_STATE_CHANGED, data.dimensions_in_what().field()); + EXPECT_EQ(1, data.dimensions_in_what().value_tuple().dimensions_value_size()); + EXPECT_EQ(1 /* uid field */, + data.dimensions_in_what().value_tuple().dimensions_value(0).field()); + EXPECT_EQ(666, data.dimensions_in_what().value_tuple().dimensions_value(0).value_int()); + EXPECT_EQ(1, data.bucket_info_size()); + EXPECT_EQ(1, data.bucket_info(0).count()); + EXPECT_EQ(bucketStartTimeNs + 2 * bucketSizeNs, + data.bucket_info(0).start_bucket_elapsed_nanos()); + EXPECT_EQ(bucketStartTimeNs + 3 * bucketSizeNs, + data.bucket_info(0).end_bucket_elapsed_nanos()); + +} + + +#else +GTEST_LOG_(INFO) << "This test does nothing.\n"; +#endif + +} // namespace statsd +} // namespace os +} // namespace android diff --git a/cmds/statsd/tests/e2e/MetricConditionLink_e2e_test.cpp b/cmds/statsd/tests/e2e/MetricConditionLink_e2e_test.cpp index 11aaab00d88c..9349c857c5b3 100644 --- a/cmds/statsd/tests/e2e/MetricConditionLink_e2e_test.cpp +++ b/cmds/statsd/tests/e2e/MetricConditionLink_e2e_test.cpp @@ -99,7 +99,6 @@ StatsdConfig CreateStatsdConfig() { // If we want to test multiple dump data, we must do it in separate tests, because in the e2e tests, // we should use the real API which will clear the data after dump data is called. -// TODO: better refactor the code so that the tests are not so verbose. TEST(MetricConditionLinkE2eTest, TestMultiplePredicatesAndLinks1) { auto config = CreateStatsdConfig(); uint64_t bucketStartTimeNs = 10000000000; @@ -200,8 +199,8 @@ TEST(MetricConditionLinkE2eTest, TestMultiplePredicatesAndLinks1) { } ConfigMetricsReportList reports; vector<uint8_t> buffer; - processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs - 1, false, ADB_DUMP, - &buffer); + processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs - 1, false, true, + ADB_DUMP, &buffer); EXPECT_TRUE(buffer.size() > 0); EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); backfillDimensionPath(&reports); @@ -319,8 +318,8 @@ TEST(MetricConditionLinkE2eTest, TestMultiplePredicatesAndLinks2) { ConfigMetricsReportList reports; vector<uint8_t> buffer; - processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, ADB_DUMP, - &buffer); + processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, true, + ADB_DUMP, &buffer); EXPECT_TRUE(buffer.size() > 0); EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); backfillDimensionPath(&reports); diff --git a/cmds/statsd/tests/e2e/PartialBucket_e2e_test.cpp b/cmds/statsd/tests/e2e/PartialBucket_e2e_test.cpp index 67acd6154176..3cb553fd9a16 100644 --- a/cmds/statsd/tests/e2e/PartialBucket_e2e_test.cpp +++ b/cmds/statsd/tests/e2e/PartialBucket_e2e_test.cpp @@ -46,7 +46,7 @@ ConfigMetricsReport GetReports(sp<StatsLogProcessor> processor, int64_t timestam IPCThreadState* ipc = IPCThreadState::self(); ConfigKey configKey(ipc->getCallingUid(), kConfigKey); processor->onDumpReport(configKey, timestamp, include_current /* include_current_bucket*/, - ADB_DUMP, &output); + true /* erase_data */, ADB_DUMP, &output); ConfigMetricsReportList reports; reports.ParseFromArray(output.data(), output.size()); EXPECT_EQ(1, reports.reports_size()); diff --git a/cmds/statsd/tests/e2e/ValueMetric_pull_e2e_test.cpp b/cmds/statsd/tests/e2e/ValueMetric_pull_e2e_test.cpp index dd28d3611b4f..abf1ab176ea7 100644 --- a/cmds/statsd/tests/e2e/ValueMetric_pull_e2e_test.cpp +++ b/cmds/statsd/tests/e2e/ValueMetric_pull_e2e_test.cpp @@ -49,6 +49,7 @@ StatsdConfig CreateStatsdConfig() { CreateDimensions(android::util::TEMPERATURE, {2/* sensor name field */ }); valueMetric->set_bucket(FIVE_MINUTES); valueMetric->set_use_absolute_value_on_reset(true); + valueMetric->set_skip_zero_diff_output(false); return config; } @@ -66,7 +67,7 @@ TEST(ValueMetricE2eTest, TestPulledEvents) { baseTimeNs, configAddedTimeNs, config, cfgKey); EXPECT_EQ(processor->mMetricsManagers.size(), 1u); EXPECT_TRUE(processor->mMetricsManagers.begin()->second->isConfigValid()); - processor->mStatsPullerManager.ForceClearPullerCache(); + processor->mPullerManager->ForceClearPullerCache(); int startBucketNum = processor->mMetricsManagers.begin()->second-> mAllMetricProducers[0]->getCurrentBucketNum(); @@ -74,12 +75,11 @@ TEST(ValueMetricE2eTest, TestPulledEvents) { // When creating the config, the gauge metric producer should register the alarm at the // end of the current bucket. - EXPECT_EQ((size_t)1, StatsPullerManagerImpl::GetInstance().mReceivers.size()); + EXPECT_EQ((size_t)1, processor->mPullerManager->mReceivers.size()); EXPECT_EQ(bucketSizeNs, - StatsPullerManagerImpl::GetInstance().mReceivers.begin()-> - second.front().intervalNs); - int64_t& expectedPullTimeNs = StatsPullerManagerImpl::GetInstance().mReceivers.begin()-> - second.front().nextPullTimeNs; + processor->mPullerManager->mReceivers.begin()->second.front().intervalNs); + int64_t& expectedPullTimeNs = + processor->mPullerManager->mReceivers.begin()->second.front().nextPullTimeNs; EXPECT_EQ(baseTimeNs + startBucketNum * bucketSizeNs + bucketSizeNs, expectedPullTimeNs); auto screenOffEvent = CreateScreenStateChangedEvent(android::view::DISPLAY_STATE_OFF, @@ -118,8 +118,8 @@ TEST(ValueMetricE2eTest, TestPulledEvents) { ConfigMetricsReportList reports; vector<uint8_t> buffer; - processor->onDumpReport(cfgKey, configAddedTimeNs + 7 * bucketSizeNs + 10, false, ADB_DUMP, - &buffer); + processor->onDumpReport(cfgKey, configAddedTimeNs + 7 * bucketSizeNs + 10, false, true, + ADB_DUMP, &buffer); EXPECT_TRUE(buffer.size() > 0); EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); backfillDimensionPath(&reports); @@ -142,23 +142,23 @@ TEST(ValueMetricE2eTest, TestPulledEvents) { EXPECT_EQ(baseTimeNs + 2 * bucketSizeNs, data.bucket_info(0).start_bucket_elapsed_nanos()); EXPECT_EQ(baseTimeNs + 3 * bucketSizeNs, data.bucket_info(0).end_bucket_elapsed_nanos()); - EXPECT_TRUE(data.bucket_info(0).has_value()); + EXPECT_EQ(1, data.bucket_info(0).values_size()); EXPECT_EQ(baseTimeNs + 3 * bucketSizeNs, data.bucket_info(1).start_bucket_elapsed_nanos()); EXPECT_EQ(baseTimeNs + 4 * bucketSizeNs, data.bucket_info(1).end_bucket_elapsed_nanos()); - EXPECT_TRUE(data.bucket_info(1).has_value()); + EXPECT_EQ(1, data.bucket_info(1).values_size()); EXPECT_EQ(baseTimeNs + 4 * bucketSizeNs, data.bucket_info(2).start_bucket_elapsed_nanos()); EXPECT_EQ(baseTimeNs + 5 * bucketSizeNs, data.bucket_info(2).end_bucket_elapsed_nanos()); - EXPECT_TRUE(data.bucket_info(2).has_value()); + EXPECT_EQ(1, data.bucket_info(2).values_size()); EXPECT_EQ(baseTimeNs + 6 * bucketSizeNs, data.bucket_info(3).start_bucket_elapsed_nanos()); EXPECT_EQ(baseTimeNs + 7 * bucketSizeNs, data.bucket_info(3).end_bucket_elapsed_nanos()); - EXPECT_TRUE(data.bucket_info(3).has_value()); + EXPECT_EQ(1, data.bucket_info(3).values_size()); EXPECT_EQ(baseTimeNs + 7 * bucketSizeNs, data.bucket_info(4).start_bucket_elapsed_nanos()); EXPECT_EQ(baseTimeNs + 8 * bucketSizeNs, data.bucket_info(4).end_bucket_elapsed_nanos()); - EXPECT_TRUE(data.bucket_info(4).has_value()); + EXPECT_EQ(1, data.bucket_info(4).values_size()); } TEST(ValueMetricE2eTest, TestPulledEvents_LateAlarm) { @@ -173,7 +173,7 @@ TEST(ValueMetricE2eTest, TestPulledEvents_LateAlarm) { baseTimeNs, configAddedTimeNs, config, cfgKey); EXPECT_EQ(processor->mMetricsManagers.size(), 1u); EXPECT_TRUE(processor->mMetricsManagers.begin()->second->isConfigValid()); - processor->mStatsPullerManager.ForceClearPullerCache(); + processor->mPullerManager->ForceClearPullerCache(); int startBucketNum = processor->mMetricsManagers.begin()->second-> mAllMetricProducers[0]->getCurrentBucketNum(); @@ -181,12 +181,11 @@ TEST(ValueMetricE2eTest, TestPulledEvents_LateAlarm) { // When creating the config, the gauge metric producer should register the alarm at the // end of the current bucket. - EXPECT_EQ((size_t)1, StatsPullerManagerImpl::GetInstance().mReceivers.size()); + EXPECT_EQ((size_t)1, processor->mPullerManager->mReceivers.size()); EXPECT_EQ(bucketSizeNs, - StatsPullerManagerImpl::GetInstance().mReceivers.begin()-> - second.front().intervalNs); - int64_t& expectedPullTimeNs = StatsPullerManagerImpl::GetInstance().mReceivers.begin()-> - second.front().nextPullTimeNs; + processor->mPullerManager->mReceivers.begin()->second.front().intervalNs); + int64_t& expectedPullTimeNs = + processor->mPullerManager->mReceivers.begin()->second.front().nextPullTimeNs; EXPECT_EQ(baseTimeNs + startBucketNum * bucketSizeNs + bucketSizeNs, expectedPullTimeNs); // Screen off/on/off events. @@ -226,8 +225,8 @@ TEST(ValueMetricE2eTest, TestPulledEvents_LateAlarm) { ConfigMetricsReportList reports; vector<uint8_t> buffer; - processor->onDumpReport(cfgKey, configAddedTimeNs + 9 * bucketSizeNs + 10, false, ADB_DUMP, - &buffer); + processor->onDumpReport(cfgKey, configAddedTimeNs + 9 * bucketSizeNs + 10, false, true, + ADB_DUMP, &buffer); EXPECT_TRUE(buffer.size() > 0); EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); backfillDimensionPath(&reports); @@ -250,15 +249,15 @@ TEST(ValueMetricE2eTest, TestPulledEvents_LateAlarm) { EXPECT_EQ(baseTimeNs + 2 * bucketSizeNs, data.bucket_info(0).start_bucket_elapsed_nanos()); EXPECT_EQ(baseTimeNs + 3 * bucketSizeNs, data.bucket_info(0).end_bucket_elapsed_nanos()); - EXPECT_TRUE(data.bucket_info(0).has_value()); + EXPECT_EQ(1, data.bucket_info(0).values_size()); EXPECT_EQ(baseTimeNs + 8 * bucketSizeNs, data.bucket_info(1).start_bucket_elapsed_nanos()); EXPECT_EQ(baseTimeNs + 9 * bucketSizeNs, data.bucket_info(1).end_bucket_elapsed_nanos()); - EXPECT_TRUE(data.bucket_info(1).has_value()); + EXPECT_EQ(1, data.bucket_info(1).values_size()); EXPECT_EQ(baseTimeNs + 9 * bucketSizeNs, data.bucket_info(2).start_bucket_elapsed_nanos()); EXPECT_EQ(baseTimeNs + 10 * bucketSizeNs, data.bucket_info(2).end_bucket_elapsed_nanos()); - EXPECT_TRUE(data.bucket_info(2).has_value()); + EXPECT_EQ(1, data.bucket_info(2).values_size()); } #else diff --git a/cmds/statsd/tests/e2e/WakelockDuration_e2e_test.cpp b/cmds/statsd/tests/e2e/WakelockDuration_e2e_test.cpp index b9d0c62cf596..6d1317cb5dee 100644 --- a/cmds/statsd/tests/e2e/WakelockDuration_e2e_test.cpp +++ b/cmds/statsd/tests/e2e/WakelockDuration_e2e_test.cpp @@ -127,8 +127,8 @@ TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForSumDuration1) FeedEvents(config, processor); vector<uint8_t> buffer; ConfigMetricsReportList reports; - processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs - 1, false, ADB_DUMP, - &buffer); + processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs - 1, false, true, + ADB_DUMP, &buffer); EXPECT_TRUE(buffer.size() > 0); EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); backfillDimensionPath(&reports); @@ -164,8 +164,8 @@ TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForSumDuration2) FeedEvents(config, processor); vector<uint8_t> buffer; ConfigMetricsReportList reports; - processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, ADB_DUMP, - &buffer); + processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, true, + ADB_DUMP, &buffer); EXPECT_TRUE(buffer.size() > 0); EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); backfillDimensionPath(&reports); @@ -215,8 +215,8 @@ TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForSumDuration3) processor->OnLogEvent(event.get()); } - processor->onDumpReport(cfgKey, bucketStartTimeNs + 6 * bucketSizeNs + 1, false, ADB_DUMP, - &buffer); + processor->onDumpReport(cfgKey, bucketStartTimeNs + 6 * bucketSizeNs + 1, false, true, + ADB_DUMP, &buffer); EXPECT_TRUE(buffer.size() > 0); EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); backfillDimensionPath(&reports); @@ -248,8 +248,8 @@ TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForMaxDuration1) FeedEvents(config, processor); ConfigMetricsReportList reports; vector<uint8_t> buffer; - processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs - 1, false, ADB_DUMP, - &buffer); + processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs - 1, false, true, + ADB_DUMP, &buffer); EXPECT_TRUE(buffer.size() > 0); EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); @@ -277,8 +277,8 @@ TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForMaxDuration2) FeedEvents(config, processor); ConfigMetricsReportList reports; vector<uint8_t> buffer; - processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, ADB_DUMP, - &buffer); + processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, false, true, + ADB_DUMP, &buffer); EXPECT_TRUE(buffer.size() > 0); EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); backfillDimensionPath(&reports); @@ -323,8 +323,8 @@ TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForMaxDuration3) processor->OnLogEvent(event.get()); } - processor->onDumpReport(cfgKey, bucketStartTimeNs + 6 * bucketSizeNs + 1, false, ADB_DUMP, - &buffer); + processor->onDumpReport(cfgKey, bucketStartTimeNs + 6 * bucketSizeNs + 1, false, true, + ADB_DUMP, &buffer); EXPECT_TRUE(buffer.size() > 0); EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); backfillDimensionPath(&reports); diff --git a/cmds/statsd/tests/guardrail/StatsdStats_test.cpp b/cmds/statsd/tests/guardrail/StatsdStats_test.cpp index 967ef3c5af63..6069516e9666 100644 --- a/cmds/statsd/tests/guardrail/StatsdStats_test.cpp +++ b/cmds/statsd/tests/guardrail/StatsdStats_test.cpp @@ -126,7 +126,7 @@ TEST(StatsdStatsTest, TestSubStats) { stats.noteBroadcastSent(key); // data drop -> 1 - stats.noteDataDropped(key); + stats.noteDataDropped(key, 123); // dump report -> 3 stats.noteMetricsReportSent(key, 0); @@ -142,6 +142,8 @@ TEST(StatsdStatsTest, TestSubStats) { const auto& configReport = report.config_stats(0); EXPECT_EQ(2, configReport.broadcast_sent_time_sec_size()); EXPECT_EQ(1, configReport.data_drop_time_sec_size()); + EXPECT_EQ(1, configReport.data_drop_bytes_size()); + EXPECT_EQ(123, configReport.data_drop_bytes(0)); EXPECT_EQ(3, configReport.dump_report_time_sec_size()); EXPECT_EQ(3, configReport.dump_report_data_size_size()); EXPECT_EQ(1, configReport.annotation_size()); @@ -242,6 +244,39 @@ TEST(StatsdStatsTest, TestAtomLog) { EXPECT_TRUE(sensorAtomGood); } +TEST(StatsdStatsTest, TestPullAtomStats) { + StatsdStats stats; + + stats.updateMinPullIntervalSec(android::util::DISK_SPACE, 3333L); + stats.updateMinPullIntervalSec(android::util::DISK_SPACE, 2222L); + stats.updateMinPullIntervalSec(android::util::DISK_SPACE, 4444L); + + stats.notePull(android::util::DISK_SPACE); + stats.notePullTime(android::util::DISK_SPACE, 1111L); + stats.notePullDelay(android::util::DISK_SPACE, 1111L); + stats.notePull(android::util::DISK_SPACE); + stats.notePullTime(android::util::DISK_SPACE, 3333L); + stats.notePullDelay(android::util::DISK_SPACE, 3335L); + stats.notePull(android::util::DISK_SPACE); + stats.notePullFromCache(android::util::DISK_SPACE); + + vector<uint8_t> output; + stats.dumpStats(&output, false); + StatsdStatsReport report; + bool good = report.ParseFromArray(&output[0], output.size()); + EXPECT_TRUE(good); + + EXPECT_EQ(1, report.pulled_atom_stats_size()); + + EXPECT_EQ(android::util::DISK_SPACE, report.pulled_atom_stats(0).atom_id()); + EXPECT_EQ(3, report.pulled_atom_stats(0).total_pull()); + EXPECT_EQ(1, report.pulled_atom_stats(0).total_pull_from_cache()); + EXPECT_EQ(2222L, report.pulled_atom_stats(0).min_pull_interval_sec()); + EXPECT_EQ(2222L, report.pulled_atom_stats(0).average_pull_time_nanos()); + EXPECT_EQ(3333L, report.pulled_atom_stats(0).max_pull_time_nanos()); + EXPECT_EQ(2223L, report.pulled_atom_stats(0).average_pull_delay_nanos()); + EXPECT_EQ(3335L, report.pulled_atom_stats(0).max_pull_delay_nanos()); +} TEST(StatsdStatsTest, TestAnomalyMonitor) { StatsdStats stats; @@ -275,7 +310,7 @@ TEST(StatsdStatsTest, TestTimestampThreshold) { int32_t newTimestamp = 10000; // now it should trigger removing oldest timestamp - stats.noteDataDropped(key, 10000); + stats.noteDataDropped(key, 123, 10000); stats.noteBroadcastSent(key, 10000); stats.noteMetricsReportSent(key, 0, 10000); @@ -295,6 +330,7 @@ TEST(StatsdStatsTest, TestTimestampThreshold) { // the last timestamp is the newest timestamp. EXPECT_EQ(newTimestamp, configStats->broadcast_sent_time_sec.back()); EXPECT_EQ(newTimestamp, configStats->data_drop_time_sec.back()); + EXPECT_EQ(123, configStats->data_drop_bytes.back()); EXPECT_EQ(newTimestamp, configStats->dump_report_stats.back().first); } diff --git a/cmds/statsd/tests/metrics/CountMetricProducer_test.cpp b/cmds/statsd/tests/metrics/CountMetricProducer_test.cpp index 9a8919e98f6d..67c704eb87fd 100644 --- a/cmds/statsd/tests/metrics/CountMetricProducer_test.cpp +++ b/cmds/statsd/tests/metrics/CountMetricProducer_test.cpp @@ -37,6 +37,19 @@ namespace statsd { const ConfigKey kConfigKey(0, 12345); +TEST(CountMetricProducerTest, TestFirstBucket) { + CountMetric metric; + metric.set_id(1); + metric.set_bucket(ONE_MINUTE); + sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); + + CountMetricProducer countProducer(kConfigKey, metric, -1 /*-1 meaning no condition*/, wizard, + 5, 600 * NS_PER_SEC + NS_PER_SEC/2); + EXPECT_EQ(600500000000, countProducer.mCurrentBucketStartTimeNs); + EXPECT_EQ(10, countProducer.mCurrentBucketNum); + EXPECT_EQ(660000000005, countProducer.getCurrentBucketEndTimeNs()); +} + TEST(CountMetricProducerTest, TestNonDimensionalEvents) { int64_t bucketStartTimeNs = 10000000000; int64_t bucketSizeNs = TimeUnitToBucketSizeInMillis(ONE_MINUTE) * 1000000LL; @@ -56,8 +69,7 @@ TEST(CountMetricProducerTest, TestNonDimensionalEvents) { sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); CountMetricProducer countProducer(kConfigKey, metric, -1 /*-1 meaning no condition*/, wizard, - bucketStartTimeNs); - countProducer.setBucketSize(60 * NS_PER_SEC); + bucketStartTimeNs, bucketStartTimeNs); // 2 events in bucket 1. countProducer.onMatchedLogEvent(1 /*log matcher index*/, event1); @@ -119,8 +131,7 @@ TEST(CountMetricProducerTest, TestEventsWithNonSlicedCondition) { sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); - CountMetricProducer countProducer(kConfigKey, metric, 1, wizard, bucketStartTimeNs); - countProducer.setBucketSize(60 * NS_PER_SEC); + CountMetricProducer countProducer(kConfigKey, metric, 1, wizard, bucketStartTimeNs, bucketStartTimeNs); countProducer.onConditionChanged(true, bucketStartTimeNs); countProducer.onMatchedLogEvent(1 /*matcher index*/, event1); @@ -181,8 +192,7 @@ TEST(CountMetricProducerTest, TestEventsWithSlicedCondition) { EXPECT_CALL(*wizard, query(_, key2, _, _, _, _)).WillOnce(Return(ConditionState::kTrue)); CountMetricProducer countProducer(kConfigKey, metric, 1 /*condition tracker index*/, wizard, - bucketStartTimeNs); - countProducer.setBucketSize(60 * NS_PER_SEC); + bucketStartTimeNs, bucketStartTimeNs); countProducer.onMatchedLogEvent(1 /*log matcher index*/, event1); countProducer.flushIfNeededLocked(bucketStartTimeNs + 1); @@ -221,8 +231,7 @@ TEST(CountMetricProducerTest, TestEventWithAppUpgrade) { event1.init(); sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); CountMetricProducer countProducer(kConfigKey, metric, -1 /* no condition */, wizard, - bucketStartTimeNs); - countProducer.setBucketSize(60 * NS_PER_SEC); + bucketStartTimeNs, bucketStartTimeNs); sp<AnomalyTracker> anomalyTracker = countProducer.addAnomalyTracker(alert, alarmMonitor); EXPECT_TRUE(anomalyTracker != nullptr); @@ -280,8 +289,7 @@ TEST(CountMetricProducerTest, TestEventWithAppUpgradeInNextBucket) { event1.init(); sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); CountMetricProducer countProducer(kConfigKey, metric, -1 /* no condition */, wizard, - bucketStartTimeNs); - countProducer.setBucketSize(60 * NS_PER_SEC); + bucketStartTimeNs, bucketStartTimeNs); // Bucket is flushed yet. countProducer.onMatchedLogEvent(1 /*log matcher index*/, event1); @@ -337,8 +345,7 @@ TEST(CountMetricProducerTest, TestAnomalyDetectionUnSliced) { sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); CountMetricProducer countProducer(kConfigKey, metric, -1 /*-1 meaning no condition*/, wizard, - bucketStartTimeNs); - countProducer.setBucketSize(60 * NS_PER_SEC); + bucketStartTimeNs, bucketStartTimeNs); sp<AnomalyTracker> anomalyTracker = countProducer.addAnomalyTracker(alert, alarmMonitor); diff --git a/cmds/statsd/tests/metrics/DurationMetricProducer_test.cpp b/cmds/statsd/tests/metrics/DurationMetricProducer_test.cpp index 7ef8c5bd6a1b..b54096441d3f 100644 --- a/cmds/statsd/tests/metrics/DurationMetricProducer_test.cpp +++ b/cmds/statsd/tests/metrics/DurationMetricProducer_test.cpp @@ -39,6 +39,23 @@ namespace statsd { const ConfigKey kConfigKey(0, 12345); +TEST(DurationMetricTrackerTest, TestFirstBucket) { + sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); + DurationMetric metric; + metric.set_id(1); + metric.set_bucket(ONE_MINUTE); + metric.set_aggregation_type(DurationMetric_AggregationType_SUM); + + FieldMatcher dimensions; + DurationMetricProducer durationProducer( + kConfigKey, metric, -1 /*no condition*/, 1 /* start index */, 2 /* stop index */, + 3 /* stop_all index */, false /*nesting*/, wizard, dimensions, 5, 600 * NS_PER_SEC + NS_PER_SEC/2); + + EXPECT_EQ(600500000000, durationProducer.mCurrentBucketStartTimeNs); + EXPECT_EQ(10, durationProducer.mCurrentBucketNum); + EXPECT_EQ(660000000005, durationProducer.getCurrentBucketEndTimeNs()); +} + TEST(DurationMetricTrackerTest, TestNoCondition) { sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); int64_t bucketStartTimeNs = 10000000000; @@ -58,8 +75,7 @@ TEST(DurationMetricTrackerTest, TestNoCondition) { FieldMatcher dimensions; DurationMetricProducer durationProducer( kConfigKey, metric, -1 /*no condition*/, 1 /* start index */, 2 /* stop index */, - 3 /* stop_all index */, false /*nesting*/, wizard, dimensions, bucketStartTimeNs); - durationProducer.setBucketSize(60 * NS_PER_SEC); + 3 /* stop_all index */, false /*nesting*/, wizard, dimensions, bucketStartTimeNs, bucketStartTimeNs); durationProducer.onMatchedLogEvent(1 /* start index*/, event1); durationProducer.onMatchedLogEvent(2 /* stop index*/, event2); @@ -100,8 +116,7 @@ TEST(DurationMetricTrackerTest, TestNonSlicedCondition) { FieldMatcher dimensions; DurationMetricProducer durationProducer( kConfigKey, metric, 0 /* condition index */, 1 /* start index */, 2 /* stop index */, - 3 /* stop_all index */, false /*nesting*/, wizard, dimensions, bucketStartTimeNs); - durationProducer.setBucketSize(60 * NS_PER_SEC); + 3 /* stop_all index */, false /*nesting*/, wizard, dimensions, bucketStartTimeNs, bucketStartTimeNs); EXPECT_FALSE(durationProducer.mCondition); EXPECT_FALSE(durationProducer.isConditionSliced()); @@ -151,8 +166,7 @@ TEST(DurationMetricTrackerTest, TestSumDurationWithUpgrade) { FieldMatcher dimensions; DurationMetricProducer durationProducer( kConfigKey, metric, -1 /* no condition */, 1 /* start index */, 2 /* stop index */, - 3 /* stop_all index */, false /*nesting*/, wizard, dimensions, bucketStartTimeNs); - durationProducer.setBucketSize(60 * NS_PER_SEC); + 3 /* stop_all index */, false /*nesting*/, wizard, dimensions, bucketStartTimeNs, bucketStartTimeNs); LogEvent start_event(tagId, startTimeNs); start_event.init(); @@ -206,8 +220,7 @@ TEST(DurationMetricTrackerTest, TestSumDurationWithUpgradeInFollowingBucket) { FieldMatcher dimensions; DurationMetricProducer durationProducer( kConfigKey, metric, -1 /* no condition */, 1 /* start index */, 2 /* stop index */, - 3 /* stop_all index */, false /*nesting*/, wizard, dimensions, bucketStartTimeNs); - durationProducer.setBucketSize(60 * NS_PER_SEC); + 3 /* stop_all index */, false /*nesting*/, wizard, dimensions, bucketStartTimeNs, bucketStartTimeNs); LogEvent start_event(tagId, startTimeNs); start_event.init(); @@ -261,8 +274,7 @@ TEST(DurationMetricTrackerTest, TestSumDurationAnomalyWithUpgrade) { FieldMatcher dimensions; DurationMetricProducer durationProducer( kConfigKey, metric, -1 /* no condition */, 1 /* start index */, 2 /* stop index */, - 3 /* stop_all index */, false /*nesting*/, wizard, dimensions, bucketStartTimeNs); - durationProducer.setBucketSize(60 * NS_PER_SEC); + 3 /* stop_all index */, false /*nesting*/, wizard, dimensions, bucketStartTimeNs, bucketStartTimeNs); sp<AnomalyTracker> anomalyTracker = durationProducer.addAnomalyTracker(alert, alarmMonitor); EXPECT_TRUE(anomalyTracker != nullptr); @@ -300,8 +312,7 @@ TEST(DurationMetricTrackerTest, TestMaxDurationWithUpgrade) { FieldMatcher dimensions; DurationMetricProducer durationProducer( kConfigKey, metric, -1 /* no condition */, 1 /* start index */, 2 /* stop index */, - 3 /* stop_all index */, false /*nesting*/, wizard, dimensions, bucketStartTimeNs); - durationProducer.setBucketSize(60 * NS_PER_SEC); + 3 /* stop_all index */, false /*nesting*/, wizard, dimensions, bucketStartTimeNs, bucketStartTimeNs); LogEvent start_event(tagId, startTimeNs); start_event.init(); @@ -348,8 +359,7 @@ TEST(DurationMetricTrackerTest, TestMaxDurationWithUpgradeInNextBucket) { FieldMatcher dimensions; DurationMetricProducer durationProducer( kConfigKey, metric, -1 /* no condition */, 1 /* start index */, 2 /* stop index */, - 3 /* stop_all index */, false /*nesting*/, wizard, dimensions, bucketStartTimeNs); - durationProducer.setBucketSize(60 * NS_PER_SEC); + 3 /* stop_all index */, false /*nesting*/, wizard, dimensions, bucketStartTimeNs, bucketStartTimeNs); LogEvent start_event(tagId, startTimeNs); start_event.init(); diff --git a/cmds/statsd/tests/metrics/EventMetricProducer_test.cpp b/cmds/statsd/tests/metrics/EventMetricProducer_test.cpp index 3a1546641d45..d2fd95c818cf 100644 --- a/cmds/statsd/tests/metrics/EventMetricProducer_test.cpp +++ b/cmds/statsd/tests/metrics/EventMetricProducer_test.cpp @@ -54,8 +54,8 @@ TEST(EventMetricProducerTest, TestNoCondition) { eventProducer.onMatchedLogEvent(1 /*matcher index*/, event1); eventProducer.onMatchedLogEvent(1 /*matcher index*/, event2); - // TODO: get the report and check the content after the ProtoOutputStream change is done. - // eventProducer.onDumpReport(); + // TODO(b/110561136): get the report and check the content after the ProtoOutputStream change + // is done eventProducer.onDumpReport(); } TEST(EventMetricProducerTest, TestEventsWithNonSlicedCondition) { diff --git a/cmds/statsd/tests/metrics/GaugeMetricProducer_test.cpp b/cmds/statsd/tests/metrics/GaugeMetricProducer_test.cpp index 698ce727e688..60bd4a7e07d9 100644 --- a/cmds/statsd/tests/metrics/GaugeMetricProducer_test.cpp +++ b/cmds/statsd/tests/metrics/GaugeMetricProducer_test.cpp @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "src/matchers/SimpleLogMatchingTracker.h" #include "src/metrics/GaugeMetricProducer.h" #include "src/stats_log_util.h" #include "logd/LogEvent.h" @@ -40,6 +41,8 @@ namespace statsd { const ConfigKey kConfigKey(0, 12345); const int tagId = 1; const int64_t metricId = 123; +const int64_t atomMatcherId = 678; +const int logEventMatcherIndex = 0; const int64_t bucketStartTimeNs = 10 * NS_PER_SEC; const int64_t bucketSizeNs = TimeUnitToBucketSizeInMillis(ONE_MINUTE) * 1000000LL; const int64_t bucket2StartTimeNs = bucketStartTimeNs + bucketSizeNs; @@ -47,7 +50,10 @@ const int64_t bucket3StartTimeNs = bucketStartTimeNs + 2 * bucketSizeNs; const int64_t bucket4StartTimeNs = bucketStartTimeNs + 3 * bucketSizeNs; const int64_t eventUpgradeTimeNs = bucketStartTimeNs + 15 * NS_PER_SEC; -TEST(GaugeMetricProducerTest, TestNoCondition) { +/* + * Tests that the first bucket works correctly + */ +TEST(GaugeMetricProducerTest, TestFirstBucket) { GaugeMetric metric; metric.set_id(metricId); metric.set_bucket(ONE_MINUTE); @@ -59,16 +65,64 @@ TEST(GaugeMetricProducerTest, TestNoCondition) { sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); - // TODO: pending refactor of StatsPullerManager - // For now we still need this so that it doesn't do real pulling. - shared_ptr<MockStatsPullerManager> pullerManager = - make_shared<StrictMock<MockStatsPullerManager>>(); + UidMap uidMap; + SimpleAtomMatcher atomMatcher; + atomMatcher.set_atom_id(tagId); + sp<EventMatcherWizard> eventMatcherWizard = new EventMatcherWizard({ + new SimpleLogMatchingTracker(atomMatcherId, logEventMatcherIndex, atomMatcher, uidMap)}); + + sp<MockStatsPullerManager> pullerManager = new StrictMock<MockStatsPullerManager>(); + + // statsd started long ago. + // The metric starts in the middle of the bucket + GaugeMetricProducer gaugeProducer(kConfigKey, metric, -1 /*-1 meaning no condition*/, wizard, + logEventMatcherIndex, eventMatcherWizard, + -1, -1, tagId, 5, 600 * NS_PER_SEC + NS_PER_SEC / 2, + pullerManager); + + EXPECT_EQ(600500000000, gaugeProducer.mCurrentBucketStartTimeNs); + EXPECT_EQ(10, gaugeProducer.mCurrentBucketNum); + EXPECT_EQ(660000000005, gaugeProducer.getCurrentBucketEndTimeNs()); +} + +TEST(GaugeMetricProducerTest, TestPulledEventsNoCondition) { + GaugeMetric metric; + metric.set_id(metricId); + metric.set_bucket(ONE_MINUTE); + metric.mutable_gauge_fields_filter()->set_include_all(false); + auto gaugeFieldMatcher = metric.mutable_gauge_fields_filter()->mutable_fields(); + gaugeFieldMatcher->set_field(tagId); + gaugeFieldMatcher->add_child()->set_field(1); + gaugeFieldMatcher->add_child()->set_field(3); + + sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); + + UidMap uidMap; + SimpleAtomMatcher atomMatcher; + atomMatcher.set_atom_id(tagId); + sp<EventMatcherWizard> eventMatcherWizard = new EventMatcherWizard({ + new SimpleLogMatchingTracker(atomMatcherId, logEventMatcherIndex, atomMatcher, uidMap)}); + + sp<MockStatsPullerManager> pullerManager = new StrictMock<MockStatsPullerManager>(); EXPECT_CALL(*pullerManager, RegisterReceiver(tagId, _, _, _)).WillOnce(Return()); EXPECT_CALL(*pullerManager, UnRegisterReceiver(tagId, _)).WillOnce(Return()); + EXPECT_CALL(*pullerManager, Pull(tagId, _, _)) + .WillOnce(Invoke([](int tagId, int64_t timeNs, + vector<std::shared_ptr<LogEvent>>* data) { + data->clear(); + shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucketStartTimeNs + 10); + event->write(3); + event->write("some value"); + event->write(11); + event->init(); + data->push_back(event); + return true; + })); GaugeMetricProducer gaugeProducer(kConfigKey, metric, -1 /*-1 meaning no condition*/, wizard, - tagId, bucketStartTimeNs, bucketStartTimeNs, pullerManager); - gaugeProducer.setBucketSize(60 * NS_PER_SEC); + logEventMatcherIndex, eventMatcherWizard, + tagId, -1, tagId, bucketStartTimeNs, bucketStartTimeNs, + pullerManager); vector<shared_ptr<LogEvent>> allData; allData.clear(); @@ -86,11 +140,12 @@ TEST(GaugeMetricProducerTest, TestNoCondition) { EXPECT_EQ(10, it->mValue.int_value); it++; EXPECT_EQ(11, it->mValue.int_value); - EXPECT_EQ(0UL, gaugeProducer.mPastBuckets.size()); + EXPECT_EQ(1UL, gaugeProducer.mPastBuckets.size()); + EXPECT_EQ(3, gaugeProducer.mPastBuckets.begin()->second.back().mGaugeAtoms + .front().mFields->begin()->mValue.int_value); allData.clear(); - std::shared_ptr<LogEvent> event2 = - std::make_shared<LogEvent>(tagId, bucket3StartTimeNs + 10); + std::shared_ptr<LogEvent> event2 = std::make_shared<LogEvent>(tagId, bucket3StartTimeNs + 10); event2->write(24); event2->write("some value"); event2->write(25); @@ -106,7 +161,7 @@ TEST(GaugeMetricProducerTest, TestNoCondition) { EXPECT_EQ(25, it->mValue.int_value); // One dimension. EXPECT_EQ(1UL, gaugeProducer.mPastBuckets.size()); - EXPECT_EQ(1UL, gaugeProducer.mPastBuckets.begin()->second.size()); + EXPECT_EQ(2UL, gaugeProducer.mPastBuckets.begin()->second.size()); it = gaugeProducer.mPastBuckets.begin()->second.back().mGaugeAtoms.front().mFields->begin(); EXPECT_EQ(INT, it->mValue.getType()); EXPECT_EQ(10L, it->mValue.int_value); @@ -118,7 +173,7 @@ TEST(GaugeMetricProducerTest, TestNoCondition) { EXPECT_EQ(0UL, gaugeProducer.mCurrentSlicedBucket->size()); // One dimension. EXPECT_EQ(1UL, gaugeProducer.mPastBuckets.size()); - EXPECT_EQ(2UL, gaugeProducer.mPastBuckets.begin()->second.size()); + EXPECT_EQ(3UL, gaugeProducer.mPastBuckets.begin()->second.size()); it = gaugeProducer.mPastBuckets.begin()->second.back().mGaugeAtoms.front().mFields->begin(); EXPECT_EQ(INT, it->mValue.getType()); EXPECT_EQ(24L, it->mValue.int_value); @@ -140,13 +195,19 @@ TEST(GaugeMetricProducerTest, TestPushedEventsWithUpgrade) { alert.set_trigger_if_sum_gt(25); alert.set_num_buckets(100); sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); - shared_ptr<MockStatsPullerManager> pullerManager = - make_shared<StrictMock<MockStatsPullerManager>>(); + sp<MockStatsPullerManager> pullerManager = new StrictMock<MockStatsPullerManager>(); + + UidMap uidMap; + SimpleAtomMatcher atomMatcher; + atomMatcher.set_atom_id(tagId); + sp<EventMatcherWizard> eventMatcherWizard = new EventMatcherWizard({ + new SimpleLogMatchingTracker(atomMatcherId, logEventMatcherIndex, atomMatcher, uidMap)}); + GaugeMetricProducer gaugeProducer(kConfigKey, metric, -1 /*-1 meaning no condition*/, wizard, - -1 /* -1 means no pulling */, bucketStartTimeNs, + logEventMatcherIndex, eventMatcherWizard, + -1 /* -1 means no pulling */, -1, tagId, bucketStartTimeNs, bucketStartTimeNs, pullerManager); - gaugeProducer.setBucketSize(60 * NS_PER_SEC); sp<AnomalyTracker> anomalyTracker = gaugeProducer.addAnomalyTracker(alert, alarmMonitor); EXPECT_TRUE(anomalyTracker != nullptr); @@ -211,11 +272,17 @@ TEST(GaugeMetricProducerTest, TestPulledWithUpgrade) { sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); - shared_ptr<MockStatsPullerManager> pullerManager = - make_shared<StrictMock<MockStatsPullerManager>>(); + UidMap uidMap; + SimpleAtomMatcher atomMatcher; + atomMatcher.set_atom_id(tagId); + sp<EventMatcherWizard> eventMatcherWizard = new EventMatcherWizard({ + new SimpleLogMatchingTracker(atomMatcherId, logEventMatcherIndex, atomMatcher, uidMap)}); + + sp<MockStatsPullerManager> pullerManager = new StrictMock<MockStatsPullerManager>(); EXPECT_CALL(*pullerManager, RegisterReceiver(tagId, _, _, _)).WillOnce(Return()); EXPECT_CALL(*pullerManager, UnRegisterReceiver(tagId, _)).WillOnce(Return()); EXPECT_CALL(*pullerManager, Pull(tagId, _, _)) + .WillOnce(Return(false)) .WillOnce(Invoke([](int tagId, int64_t timeNs, vector<std::shared_ptr<LogEvent>>* data) { data->clear(); @@ -228,8 +295,9 @@ TEST(GaugeMetricProducerTest, TestPulledWithUpgrade) { })); GaugeMetricProducer gaugeProducer(kConfigKey, metric, -1 /*-1 meaning no condition*/, wizard, - tagId, bucketStartTimeNs, bucketStartTimeNs, pullerManager); - gaugeProducer.setBucketSize(60 * NS_PER_SEC); + logEventMatcherIndex, eventMatcherWizard, + tagId, -1, tagId, bucketStartTimeNs, bucketStartTimeNs, + pullerManager); vector<shared_ptr<LogEvent>> allData; shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucketStartTimeNs + 1); @@ -269,7 +337,7 @@ TEST(GaugeMetricProducerTest, TestPulledWithUpgrade) { ->mValue.int_value); } -TEST(GaugeMetricProducerTest, TestWithCondition) { +TEST(GaugeMetricProducerTest, TestPulledEventsWithCondition) { GaugeMetric metric; metric.set_id(metricId); metric.set_bucket(ONE_MINUTE); @@ -280,8 +348,13 @@ TEST(GaugeMetricProducerTest, TestWithCondition) { sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); - shared_ptr<MockStatsPullerManager> pullerManager = - make_shared<StrictMock<MockStatsPullerManager>>(); + UidMap uidMap; + SimpleAtomMatcher atomMatcher; + atomMatcher.set_atom_id(tagId); + sp<EventMatcherWizard> eventMatcherWizard = new EventMatcherWizard({ + new SimpleLogMatchingTracker(atomMatcherId, logEventMatcherIndex, atomMatcher, uidMap)}); + + sp<MockStatsPullerManager> pullerManager = new StrictMock<MockStatsPullerManager>(); EXPECT_CALL(*pullerManager, RegisterReceiver(tagId, _, _, _)).WillOnce(Return()); EXPECT_CALL(*pullerManager, UnRegisterReceiver(tagId, _)).WillOnce(Return()); EXPECT_CALL(*pullerManager, Pull(tagId, _, _)) @@ -296,9 +369,9 @@ TEST(GaugeMetricProducerTest, TestWithCondition) { return true; })); - GaugeMetricProducer gaugeProducer(kConfigKey, metric, 1, wizard, tagId, + GaugeMetricProducer gaugeProducer(kConfigKey, metric, 1, wizard, + logEventMatcherIndex, eventMatcherWizard, tagId, -1, tagId, bucketStartTimeNs, bucketStartTimeNs, pullerManager); - gaugeProducer.setBucketSize(60 * NS_PER_SEC); gaugeProducer.onConditionChanged(true, bucketStartTimeNs + 8); EXPECT_EQ(1UL, gaugeProducer.mCurrentSlicedBucket->size()); @@ -340,7 +413,7 @@ TEST(GaugeMetricProducerTest, TestWithCondition) { ->mValue.int_value); } -TEST(GaugeMetricProducerTest, TestWithSlicedCondition) { +TEST(GaugeMetricProducerTest, TestPulledEventsWithSlicedCondition) { const int conditionTag = 65; GaugeMetric metric; metric.set_id(1111111); @@ -355,6 +428,12 @@ TEST(GaugeMetricProducerTest, TestWithSlicedCondition) { dim->set_field(conditionTag); dim->add_child()->set_field(1); + UidMap uidMap; + SimpleAtomMatcher atomMatcher; + atomMatcher.set_atom_id(tagId); + sp<EventMatcherWizard> eventMatcherWizard = new EventMatcherWizard({ + new SimpleLogMatchingTracker(atomMatcherId, logEventMatcherIndex, atomMatcher, uidMap)}); + sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); EXPECT_CALL(*wizard, query(_, _, _, _, _, _)) .WillRepeatedly( @@ -372,8 +451,7 @@ TEST(GaugeMetricProducerTest, TestWithSlicedCondition) { return ConditionState::kTrue; })); - shared_ptr<MockStatsPullerManager> pullerManager = - make_shared<StrictMock<MockStatsPullerManager>>(); + sp<MockStatsPullerManager> pullerManager = new StrictMock<MockStatsPullerManager>(); EXPECT_CALL(*pullerManager, RegisterReceiver(tagId, _, _, _)).WillOnce(Return()); EXPECT_CALL(*pullerManager, UnRegisterReceiver(tagId, _)).WillOnce(Return()); EXPECT_CALL(*pullerManager, Pull(tagId, _, _)) @@ -388,9 +466,9 @@ TEST(GaugeMetricProducerTest, TestWithSlicedCondition) { return true; })); - GaugeMetricProducer gaugeProducer(kConfigKey, metric, 1, wizard, tagId, bucketStartTimeNs, - bucketStartTimeNs, pullerManager); - gaugeProducer.setBucketSize(60 * NS_PER_SEC); + GaugeMetricProducer gaugeProducer(kConfigKey, metric, 1, wizard, + logEventMatcherIndex, eventMatcherWizard, tagId, -1, tagId, + bucketStartTimeNs, bucketStartTimeNs, pullerManager); gaugeProducer.onSlicedConditionMayChange(true, bucketStartTimeNs + 8); @@ -417,14 +495,14 @@ TEST(GaugeMetricProducerTest, TestWithSlicedCondition) { EXPECT_EQ(1UL, gaugeProducer.mPastBuckets.size()); } -TEST(GaugeMetricProducerTest, TestAnomalyDetection) { +TEST(GaugeMetricProducerTest, TestPulledEventsAnomalyDetection) { sp<AlarmMonitor> alarmMonitor; sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); - shared_ptr<MockStatsPullerManager> pullerManager = - make_shared<StrictMock<MockStatsPullerManager>>(); + sp<MockStatsPullerManager> pullerManager = new StrictMock<MockStatsPullerManager>(); EXPECT_CALL(*pullerManager, RegisterReceiver(tagId, _, _, _)).WillOnce(Return()); EXPECT_CALL(*pullerManager, UnRegisterReceiver(tagId, _)).WillOnce(Return()); + EXPECT_CALL(*pullerManager, Pull(tagId, _, _)).WillOnce(Return(false)); GaugeMetric metric; metric.set_id(metricId); @@ -432,9 +510,17 @@ TEST(GaugeMetricProducerTest, TestAnomalyDetection) { auto gaugeFieldMatcher = metric.mutable_gauge_fields_filter()->mutable_fields(); gaugeFieldMatcher->set_field(tagId); gaugeFieldMatcher->add_child()->set_field(2); + + UidMap uidMap; + SimpleAtomMatcher atomMatcher; + atomMatcher.set_atom_id(tagId); + sp<EventMatcherWizard> eventMatcherWizard = new EventMatcherWizard({ + new SimpleLogMatchingTracker(atomMatcherId, logEventMatcherIndex, atomMatcher, uidMap)}); + GaugeMetricProducer gaugeProducer(kConfigKey, metric, -1 /*-1 meaning no condition*/, wizard, - tagId, bucketStartTimeNs, bucketStartTimeNs, pullerManager); - gaugeProducer.setBucketSize(60 * NS_PER_SEC); + logEventMatcherIndex, eventMatcherWizard, + tagId, -1, tagId, bucketStartTimeNs, bucketStartTimeNs, + pullerManager); Alert alert; alert.set_id(101); @@ -472,7 +558,7 @@ TEST(GaugeMetricProducerTest, TestAnomalyDetection) { .mFields->begin() ->mValue.int_value); EXPECT_EQ(anomalyTracker->getRefractoryPeriodEndsSec(DEFAULT_METRIC_DIMENSION_KEY), - std::ceil(1.0 * event2->GetElapsedTimestampNs() / NS_PER_SEC) + refPeriodSec); + std::ceil(1.0 * event2->GetElapsedTimestampNs() / NS_PER_SEC) + refPeriodSec); std::shared_ptr<LogEvent> event3 = std::make_shared<LogEvent>(tagId, bucketStartTimeNs + 2 * bucketSizeNs + 10); @@ -487,7 +573,7 @@ TEST(GaugeMetricProducerTest, TestAnomalyDetection) { .mFields->begin() ->mValue.int_value); EXPECT_EQ(anomalyTracker->getRefractoryPeriodEndsSec(DEFAULT_METRIC_DIMENSION_KEY), - std::ceil(1.0 * event2->GetElapsedTimestampNs() / NS_PER_SEC + refPeriodSec)); + std::ceil(1.0 * event2->GetElapsedTimestampNs() / NS_PER_SEC + refPeriodSec)); // The event4 does not have the gauge field. Thus the current bucket value is 0. std::shared_ptr<LogEvent> event4 = @@ -499,6 +585,197 @@ TEST(GaugeMetricProducerTest, TestAnomalyDetection) { EXPECT_TRUE(gaugeProducer.mCurrentSlicedBucket->begin()->second.front().mFields->empty()); } +TEST(GaugeMetricProducerTest, TestPullOnTrigger) { + GaugeMetric metric; + metric.set_id(metricId); + metric.set_bucket(ONE_MINUTE); + metric.set_sampling_type(GaugeMetric::ALL_CONDITION_CHANGES); + metric.mutable_gauge_fields_filter()->set_include_all(false); + auto gaugeFieldMatcher = metric.mutable_gauge_fields_filter()->mutable_fields(); + gaugeFieldMatcher->set_field(tagId); + gaugeFieldMatcher->add_child()->set_field(1); + + sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); + + UidMap uidMap; + SimpleAtomMatcher atomMatcher; + atomMatcher.set_atom_id(tagId); + sp<EventMatcherWizard> eventMatcherWizard = new EventMatcherWizard({ + new SimpleLogMatchingTracker(atomMatcherId, logEventMatcherIndex, atomMatcher, uidMap)}); + + sp<MockStatsPullerManager> pullerManager = new StrictMock<MockStatsPullerManager>(); + EXPECT_CALL(*pullerManager, Pull(tagId, _, _)) + .WillOnce(Invoke([](int tagId, int64_t timeNs, + vector<std::shared_ptr<LogEvent>>* data) { + data->clear(); + shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucketStartTimeNs + 3); + event->write(3); + event->init(); + data->push_back(event); + return true; + })) + .WillOnce(Invoke([](int tagId, int64_t timeNs, + vector<std::shared_ptr<LogEvent>>* data) { + data->clear(); + shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucketStartTimeNs + 10); + event->write(4); + event->init(); + data->push_back(event); + return true; + })) + .WillOnce(Invoke([](int tagId, int64_t timeNs, + vector<std::shared_ptr<LogEvent>>* data) { + data->clear(); + shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucketStartTimeNs + 20); + event->write(5); + event->init(); + data->push_back(event); + return true; + })); + + int triggerId = 5; + GaugeMetricProducer gaugeProducer(kConfigKey, metric, -1 /*-1 meaning no condition*/, wizard, + logEventMatcherIndex, eventMatcherWizard, + tagId, triggerId, tagId, bucketStartTimeNs, bucketStartTimeNs, + pullerManager); + + vector<shared_ptr<LogEvent>> allData; + allData.clear(); + + EXPECT_EQ(1UL, gaugeProducer.mCurrentSlicedBucket->size()); + LogEvent trigger(triggerId, bucketStartTimeNs + 10); + trigger.init(); + gaugeProducer.onMatchedLogEvent(1 /*log matcher index*/, trigger); + EXPECT_EQ(2UL, gaugeProducer.mCurrentSlicedBucket->begin()->second.size()); + trigger.setElapsedTimestampNs(bucketStartTimeNs + 20); + gaugeProducer.onMatchedLogEvent(1 /*log matcher index*/, trigger); + EXPECT_EQ(3UL, gaugeProducer.mCurrentSlicedBucket->begin()->second.size()); + + allData.clear(); + shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucket2StartTimeNs + 1); + event->write(10); + event->init(); + allData.push_back(event); + + gaugeProducer.onDataPulled(allData); + EXPECT_EQ(1UL, gaugeProducer.mCurrentSlicedBucket->size()); + auto it = gaugeProducer.mCurrentSlicedBucket->begin()->second.front().mFields->begin(); + EXPECT_EQ(INT, it->mValue.getType()); + EXPECT_EQ(10, it->mValue.int_value); + EXPECT_EQ(1UL, gaugeProducer.mPastBuckets.size()); + EXPECT_EQ(3UL, gaugeProducer.mPastBuckets.begin()->second.back().mGaugeAtoms.size()); + EXPECT_EQ(3, gaugeProducer.mPastBuckets.begin() + ->second.back() + .mGaugeAtoms[0] + .mFields->begin() + ->mValue.int_value); + EXPECT_EQ(4, gaugeProducer.mPastBuckets.begin() + ->second.back() + .mGaugeAtoms[1] + .mFields->begin() + ->mValue.int_value); + EXPECT_EQ(5, gaugeProducer.mPastBuckets.begin() + ->second.back() + .mGaugeAtoms[2] + .mFields->begin() + ->mValue.int_value); +} + +TEST(GaugeMetricProducerTest, TestRemoveDimensionInOutput) { + GaugeMetric metric; + metric.set_id(metricId); + metric.set_bucket(ONE_MINUTE); + metric.set_sampling_type(GaugeMetric::ALL_CONDITION_CHANGES); + metric.mutable_gauge_fields_filter()->set_include_all(true); + auto dimensionMatcher = metric.mutable_dimensions_in_what(); + // use field 1 as dimension. + dimensionMatcher->set_field(tagId); + dimensionMatcher->add_child()->set_field(1); + + sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); + + UidMap uidMap; + SimpleAtomMatcher atomMatcher; + atomMatcher.set_atom_id(tagId); + sp<EventMatcherWizard> eventMatcherWizard = new EventMatcherWizard({ + new SimpleLogMatchingTracker(atomMatcherId, logEventMatcherIndex, atomMatcher, uidMap)}); + + sp<MockStatsPullerManager> pullerManager = new StrictMock<MockStatsPullerManager>(); + EXPECT_CALL(*pullerManager, Pull(tagId, _, _)) + .WillOnce(Invoke([](int tagId, int64_t timeNs, + vector<std::shared_ptr<LogEvent>>* data) { + data->clear(); + shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucketStartTimeNs + 3); + event->write(3); + event->write(4); + event->init(); + data->push_back(event); + return true; + })) + .WillOnce(Invoke([](int tagId, int64_t timeNs, + vector<std::shared_ptr<LogEvent>>* data) { + data->clear(); + shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucketStartTimeNs + 10); + event->write(4); + event->write(5); + event->init(); + data->push_back(event); + return true; + })) + .WillOnce(Invoke([](int tagId, int64_t timeNs, + vector<std::shared_ptr<LogEvent>>* data) { + data->clear(); + shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucketStartTimeNs + 20); + event->write(4); + event->write(6); + event->init(); + data->push_back(event); + return true; + })); + + int triggerId = 5; + GaugeMetricProducer gaugeProducer(kConfigKey, metric, -1 /*-1 meaning no condition*/, wizard, + logEventMatcherIndex, eventMatcherWizard, + tagId, triggerId, tagId, bucketStartTimeNs, bucketStartTimeNs, + pullerManager); + + vector<shared_ptr<LogEvent>> allData; + allData.clear(); + + EXPECT_EQ(1UL, gaugeProducer.mCurrentSlicedBucket->size()); + LogEvent trigger(triggerId, bucketStartTimeNs + 10); + trigger.init(); + gaugeProducer.onMatchedLogEvent(1 /*log matcher index*/, trigger); + EXPECT_EQ(2UL, gaugeProducer.mCurrentSlicedBucket->size()); + EXPECT_EQ(1UL, gaugeProducer.mCurrentSlicedBucket->begin()->second.size()); + trigger.setElapsedTimestampNs(bucketStartTimeNs + 20); + gaugeProducer.onMatchedLogEvent(1 /*log matcher index*/, trigger); + EXPECT_EQ(2UL, gaugeProducer.mCurrentSlicedBucket->begin()->second.size()); + + allData.clear(); + shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucket2StartTimeNs + 1); + event->write(4); + event->write(11); + event->init(); + allData.push_back(event); + + gaugeProducer.onDataPulled(allData); + EXPECT_EQ(1UL, gaugeProducer.mCurrentSlicedBucket->size()); + auto it = gaugeProducer.mCurrentSlicedBucket->begin()->second.front().mFields->begin(); + EXPECT_EQ(INT, it->mValue.getType()); + EXPECT_EQ(11, it->mValue.int_value); + EXPECT_EQ(2UL, gaugeProducer.mPastBuckets.size()); + auto bucketIt = gaugeProducer.mPastBuckets.begin(); + EXPECT_EQ(1UL, bucketIt->second.back().mGaugeAtoms.size()); + EXPECT_EQ(3, bucketIt->first.getDimensionKeyInWhat().getValues().begin()->mValue.int_value); + EXPECT_EQ(4, bucketIt->second.back().mGaugeAtoms[0].mFields->begin()->mValue.int_value); + bucketIt++; + EXPECT_EQ(2UL, bucketIt->second.back().mGaugeAtoms.size()); + EXPECT_EQ(4, bucketIt->first.getDimensionKeyInWhat().getValues().begin()->mValue.int_value); + EXPECT_EQ(5, bucketIt->second.back().mGaugeAtoms[0].mFields->begin()->mValue.int_value); + EXPECT_EQ(6, bucketIt->second.back().mGaugeAtoms[1].mFields->begin()->mValue.int_value); +} + } // namespace statsd } // namespace os } // namespace android diff --git a/cmds/statsd/tests/metrics/ValueMetricProducer_test.cpp b/cmds/statsd/tests/metrics/ValueMetricProducer_test.cpp index e3a8a553acc9..44aa00b3046c 100644 --- a/cmds/statsd/tests/metrics/ValueMetricProducer_test.cpp +++ b/cmds/statsd/tests/metrics/ValueMetricProducer_test.cpp @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "src/matchers/SimpleLogMatchingTracker.h" #include "src/metrics/ValueMetricProducer.h" #include "src/stats_log_util.h" #include "metrics_test_helper.h" @@ -40,6 +41,8 @@ namespace statsd { const ConfigKey kConfigKey(0, 12345); const int tagId = 1; const int64_t metricId = 123; +const int64_t atomMatcherId = 678; +const int logEventMatcherIndex = 0; const int64_t bucketStartTimeNs = 10000000000; const int64_t bucketSizeNs = TimeUnitToBucketSizeInMillis(ONE_MINUTE) * 1000000LL; const int64_t bucket2StartTimeNs = bucketStartTimeNs + bucketSizeNs; @@ -47,29 +50,107 @@ const int64_t bucket3StartTimeNs = bucketStartTimeNs + 2 * bucketSizeNs; const int64_t bucket4StartTimeNs = bucketStartTimeNs + 3 * bucketSizeNs; const int64_t bucket5StartTimeNs = bucketStartTimeNs + 4 * bucketSizeNs; const int64_t bucket6StartTimeNs = bucketStartTimeNs + 5 * bucketSizeNs; -const int64_t eventUpgradeTimeNs = bucketStartTimeNs + 15 * NS_PER_SEC; +double epsilon = 0.001; + +/* + * Tests that the first bucket works correctly + */ +TEST(ValueMetricProducerTest, TestCalcPreviousBucketEndTime) { + ValueMetric metric; + metric.set_id(metricId); + metric.set_bucket(ONE_MINUTE); + metric.mutable_value_field()->set_field(tagId); + metric.mutable_value_field()->add_child()->set_field(2); + + int64_t startTimeBase = 11; + UidMap uidMap; + SimpleAtomMatcher atomMatcher; + atomMatcher.set_atom_id(tagId); + sp<EventMatcherWizard> eventMatcherWizard = + new EventMatcherWizard({new SimpleLogMatchingTracker( + atomMatcherId, logEventMatcherIndex, atomMatcher, uidMap)}); + sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); + sp<MockStatsPullerManager> pullerManager = new StrictMock<MockStatsPullerManager>(); + + // statsd started long ago. + // The metric starts in the middle of the bucket + ValueMetricProducer valueProducer(kConfigKey, metric, -1 /*-1 meaning no condition*/, wizard, + logEventMatcherIndex, eventMatcherWizard, -1, startTimeBase, + 22, pullerManager); + + EXPECT_EQ(startTimeBase, valueProducer.calcPreviousBucketEndTime(60 * NS_PER_SEC + 10)); + EXPECT_EQ(startTimeBase, valueProducer.calcPreviousBucketEndTime(60 * NS_PER_SEC + 10)); + EXPECT_EQ(60 * NS_PER_SEC + startTimeBase, + valueProducer.calcPreviousBucketEndTime(2 * 60 * NS_PER_SEC)); + EXPECT_EQ(2 * 60 * NS_PER_SEC + startTimeBase, + valueProducer.calcPreviousBucketEndTime(3 * 60 * NS_PER_SEC)); +} + +/* + * Tests that the first bucket works correctly + */ +TEST(ValueMetricProducerTest, TestFirstBucket) { + ValueMetric metric; + metric.set_id(metricId); + metric.set_bucket(ONE_MINUTE); + metric.mutable_value_field()->set_field(tagId); + metric.mutable_value_field()->add_child()->set_field(2); + + UidMap uidMap; + SimpleAtomMatcher atomMatcher; + atomMatcher.set_atom_id(tagId); + sp<EventMatcherWizard> eventMatcherWizard = + new EventMatcherWizard({new SimpleLogMatchingTracker( + atomMatcherId, logEventMatcherIndex, atomMatcher, uidMap)}); + sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); + sp<MockStatsPullerManager> pullerManager = new StrictMock<MockStatsPullerManager>(); + + // statsd started long ago. + // The metric starts in the middle of the bucket + ValueMetricProducer valueProducer(kConfigKey, metric, -1 /*-1 meaning no condition*/, wizard, + logEventMatcherIndex, eventMatcherWizard, -1, 5, + 600 * NS_PER_SEC + NS_PER_SEC / 2, pullerManager); + + EXPECT_EQ(600500000000, valueProducer.mCurrentBucketStartTimeNs); + EXPECT_EQ(10, valueProducer.mCurrentBucketNum); + EXPECT_EQ(660000000005, valueProducer.getCurrentBucketEndTimeNs()); +} /* * Tests pulled atoms with no conditions */ -TEST(ValueMetricProducerTest, TestNonDimensionalEvents) { +TEST(ValueMetricProducerTest, TestPulledEventsNoCondition) { ValueMetric metric; metric.set_id(metricId); metric.set_bucket(ONE_MINUTE); metric.mutable_value_field()->set_field(tagId); metric.mutable_value_field()->add_child()->set_field(2); + UidMap uidMap; + SimpleAtomMatcher atomMatcher; + atomMatcher.set_atom_id(tagId); + sp<EventMatcherWizard> eventMatcherWizard = + new EventMatcherWizard({new SimpleLogMatchingTracker( + atomMatcherId, logEventMatcherIndex, atomMatcher, uidMap)}); sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); - // TODO: pending refactor of StatsPullerManager - // For now we still need this so that it doesn't do real pulling. - shared_ptr<MockStatsPullerManager> pullerManager = - make_shared<StrictMock<MockStatsPullerManager>>(); + sp<MockStatsPullerManager> pullerManager = new StrictMock<MockStatsPullerManager>(); EXPECT_CALL(*pullerManager, RegisterReceiver(tagId, _, _, _)).WillOnce(Return()); EXPECT_CALL(*pullerManager, UnRegisterReceiver(tagId, _)).WillOnce(Return()); + EXPECT_CALL(*pullerManager, Pull(tagId, _, _)) + .WillOnce(Invoke([](int tagId, int64_t timeNs, + vector<std::shared_ptr<LogEvent>>* data) { + data->clear(); + shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucketStartTimeNs); + event->write(tagId); + event->write(3); + event->init(); + data->push_back(event); + return true; + })); ValueMetricProducer valueProducer(kConfigKey, metric, -1 /*-1 meaning no condition*/, wizard, - tagId, bucketStartTimeNs, bucketStartTimeNs, pullerManager); - valueProducer.setBucketSize(60 * NS_PER_SEC); + logEventMatcherIndex, eventMatcherWizard, tagId, + bucketStartTimeNs, bucketStartTimeNs, pullerManager); vector<shared_ptr<LogEvent>> allData; allData.clear(); @@ -82,14 +163,12 @@ TEST(ValueMetricProducerTest, TestNonDimensionalEvents) { valueProducer.onDataPulled(allData); // has one slice EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); - ValueMetricProducer::Interval curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - valueProducer.setBucketSize(60 * NS_PER_SEC); - - // startUpdated:true tainted:0 sum:0 start:11 - EXPECT_EQ(true, curInterval.startUpdated); - EXPECT_EQ(0, curInterval.tainted); - EXPECT_EQ(0, curInterval.sum); - EXPECT_EQ(11, curInterval.start); + ValueMetricProducer::Interval curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + + EXPECT_EQ(true, curInterval.hasBase); + EXPECT_EQ(11, curInterval.base.long_value); + EXPECT_EQ(true, curInterval.hasValue); + EXPECT_EQ(8, curInterval.value.long_value); EXPECT_EQ(0UL, valueProducer.mPastBuckets.size()); allData.clear(); @@ -101,14 +180,15 @@ TEST(ValueMetricProducerTest, TestNonDimensionalEvents) { valueProducer.onDataPulled(allData); // has one slice EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); - curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - // tartUpdated:false tainted:0 sum:12 - EXPECT_EQ(true, curInterval.startUpdated); - EXPECT_EQ(0, curInterval.tainted); - EXPECT_EQ(0, curInterval.sum); + curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + + EXPECT_EQ(true, curInterval.hasBase); + EXPECT_EQ(23, curInterval.base.long_value); + EXPECT_EQ(true, curInterval.hasValue); + EXPECT_EQ(12, curInterval.value.long_value); EXPECT_EQ(1UL, valueProducer.mPastBuckets.size()); EXPECT_EQ(1UL, valueProducer.mPastBuckets.begin()->second.size()); - EXPECT_EQ(12, valueProducer.mPastBuckets.begin()->second.back().mValue); + EXPECT_EQ(8, valueProducer.mPastBuckets.begin()->second.back().values[0].long_value); allData.clear(); event = make_shared<LogEvent>(tagId, bucket4StartTimeNs + 1); @@ -118,14 +198,111 @@ TEST(ValueMetricProducerTest, TestNonDimensionalEvents) { allData.push_back(event); valueProducer.onDataPulled(allData); EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); - curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - // startUpdated:false tainted:0 sum:12 - EXPECT_EQ(true, curInterval.startUpdated); - EXPECT_EQ(0, curInterval.tainted); - EXPECT_EQ(0, curInterval.sum); + curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + + EXPECT_EQ(true, curInterval.hasBase); + EXPECT_EQ(36, curInterval.base.long_value); + EXPECT_EQ(true, curInterval.hasValue); + EXPECT_EQ(13, curInterval.value.long_value); EXPECT_EQ(1UL, valueProducer.mPastBuckets.size()); EXPECT_EQ(2UL, valueProducer.mPastBuckets.begin()->second.size()); - EXPECT_EQ(13, valueProducer.mPastBuckets.begin()->second.back().mValue); + EXPECT_EQ(12, valueProducer.mPastBuckets.begin()->second.back().values[0].long_value); +} + +/* + * Tests pulled atoms with filtering + */ +TEST(ValueMetricProducerTest, TestPulledEventsWithFiltering) { + ValueMetric metric; + metric.set_id(metricId); + metric.set_bucket(ONE_MINUTE); + metric.mutable_value_field()->set_field(tagId); + metric.mutable_value_field()->add_child()->set_field(2); + + UidMap uidMap; + SimpleAtomMatcher atomMatcher; + atomMatcher.set_atom_id(tagId); + auto keyValue = atomMatcher.add_field_value_matcher(); + keyValue->set_field(1); + keyValue->set_eq_int(3); + sp<EventMatcherWizard> eventMatcherWizard = + new EventMatcherWizard({new SimpleLogMatchingTracker( + atomMatcherId, logEventMatcherIndex, atomMatcher, uidMap)}); + sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); + sp<MockStatsPullerManager> pullerManager = new StrictMock<MockStatsPullerManager>(); + EXPECT_CALL(*pullerManager, RegisterReceiver(tagId, _, _, _)).WillOnce(Return()); + EXPECT_CALL(*pullerManager, UnRegisterReceiver(tagId, _)).WillOnce(Return()); + EXPECT_CALL(*pullerManager, Pull(tagId, _, _)) + .WillOnce(Invoke([](int tagId, int64_t timeNs, + vector<std::shared_ptr<LogEvent>>* data) { + data->clear(); + shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucketStartTimeNs); + event->write(3); + event->write(3); + event->init(); + data->push_back(event); + return true; + })); + + ValueMetricProducer valueProducer(kConfigKey, metric, -1 /*-1 meaning no condition*/, wizard, + logEventMatcherIndex, eventMatcherWizard, tagId, + bucketStartTimeNs, bucketStartTimeNs, pullerManager); + + vector<shared_ptr<LogEvent>> allData; + allData.clear(); + shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucket2StartTimeNs + 1); + event->write(3); + event->write(11); + event->init(); + allData.push_back(event); + + valueProducer.onDataPulled(allData); + // has one slice + EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); + ValueMetricProducer::Interval curInterval = + valueProducer.mCurrentSlicedBucket.begin()->second[0]; + + EXPECT_EQ(true, curInterval.hasBase); + EXPECT_EQ(11, curInterval.base.long_value); + EXPECT_EQ(true, curInterval.hasValue); + EXPECT_EQ(8, curInterval.value.long_value); + EXPECT_EQ(0UL, valueProducer.mPastBuckets.size()); + + allData.clear(); + event = make_shared<LogEvent>(tagId, bucket3StartTimeNs + 1); + event->write(4); + event->write(23); + event->init(); + allData.push_back(event); + valueProducer.onDataPulled(allData); + // has one slice + EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); + curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + + EXPECT_EQ(true, curInterval.hasBase); + EXPECT_EQ(11, curInterval.base.long_value); + // no events caused flush of bucket + EXPECT_EQ(true, curInterval.hasValue); + EXPECT_EQ(8, curInterval.value.long_value); + EXPECT_EQ(0UL, valueProducer.mPastBuckets.size()); + + allData.clear(); + event = make_shared<LogEvent>(tagId, bucket4StartTimeNs + 1); + event->write(3); + event->write(36); + event->init(); + allData.push_back(event); + valueProducer.onDataPulled(allData); + EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); + curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + + // the base was reset + EXPECT_EQ(true, curInterval.hasBase); + EXPECT_EQ(36, curInterval.base.long_value); + EXPECT_EQ(false, curInterval.hasValue); + EXPECT_EQ(1UL, valueProducer.mPastBuckets.size()); + EXPECT_EQ(1UL, valueProducer.mPastBuckets.begin()->second.size()); + EXPECT_EQ(8, valueProducer.mPastBuckets.begin()->second.back().values[0].long_value); } /* @@ -139,15 +316,21 @@ TEST(ValueMetricProducerTest, TestPulledEventsTakeAbsoluteValueOnReset) { metric.mutable_value_field()->add_child()->set_field(2); metric.set_use_absolute_value_on_reset(true); + UidMap uidMap; + SimpleAtomMatcher atomMatcher; + atomMatcher.set_atom_id(tagId); + sp<EventMatcherWizard> eventMatcherWizard = + new EventMatcherWizard({new SimpleLogMatchingTracker( + atomMatcherId, logEventMatcherIndex, atomMatcher, uidMap)}); sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); - shared_ptr<MockStatsPullerManager> pullerManager = - make_shared<StrictMock<MockStatsPullerManager>>(); + sp<MockStatsPullerManager> pullerManager = new StrictMock<MockStatsPullerManager>(); EXPECT_CALL(*pullerManager, RegisterReceiver(tagId, _, _, _)).WillOnce(Return()); EXPECT_CALL(*pullerManager, UnRegisterReceiver(tagId, _)).WillOnce(Return()); + EXPECT_CALL(*pullerManager, Pull(tagId, _, _)).WillOnce(Return(true)); ValueMetricProducer valueProducer(kConfigKey, metric, -1 /*-1 meaning no condition*/, wizard, - tagId, bucketStartTimeNs, bucketStartTimeNs, pullerManager); - valueProducer.setBucketSize(60 * NS_PER_SEC); + logEventMatcherIndex, eventMatcherWizard, tagId, + bucketStartTimeNs, bucketStartTimeNs, pullerManager); vector<shared_ptr<LogEvent>> allData; allData.clear(); @@ -160,13 +343,11 @@ TEST(ValueMetricProducerTest, TestPulledEventsTakeAbsoluteValueOnReset) { valueProducer.onDataPulled(allData); // has one slice EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); - ValueMetricProducer::Interval curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - valueProducer.setBucketSize(60 * NS_PER_SEC); + ValueMetricProducer::Interval curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; - EXPECT_EQ(true, curInterval.startUpdated); - EXPECT_EQ(0, curInterval.tainted); - EXPECT_EQ(0, curInterval.sum); - EXPECT_EQ(11, curInterval.start); + EXPECT_EQ(true, curInterval.hasBase); + EXPECT_EQ(11, curInterval.base.long_value); + EXPECT_EQ(false, curInterval.hasValue); EXPECT_EQ(0UL, valueProducer.mPastBuckets.size()); allData.clear(); @@ -178,13 +359,12 @@ TEST(ValueMetricProducerTest, TestPulledEventsTakeAbsoluteValueOnReset) { valueProducer.onDataPulled(allData); // has one slice EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); - curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - EXPECT_EQ(true, curInterval.startUpdated); - EXPECT_EQ(0, curInterval.tainted); - EXPECT_EQ(0, curInterval.sum); - EXPECT_EQ(1UL, valueProducer.mPastBuckets.size()); - EXPECT_EQ(1UL, valueProducer.mPastBuckets.begin()->second.size()); - EXPECT_EQ(10, valueProducer.mPastBuckets.begin()->second.back().mValue); + curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + EXPECT_EQ(true, curInterval.hasBase); + EXPECT_EQ(10, curInterval.base.long_value); + EXPECT_EQ(true, curInterval.hasValue); + EXPECT_EQ(10, curInterval.value.long_value); + EXPECT_EQ(0UL, valueProducer.mPastBuckets.size()); allData.clear(); event = make_shared<LogEvent>(tagId, bucket4StartTimeNs + 1); @@ -194,13 +374,14 @@ TEST(ValueMetricProducerTest, TestPulledEventsTakeAbsoluteValueOnReset) { allData.push_back(event); valueProducer.onDataPulled(allData); EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); - curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - EXPECT_EQ(true, curInterval.startUpdated); - EXPECT_EQ(0, curInterval.tainted); - EXPECT_EQ(0, curInterval.sum); + curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + EXPECT_EQ(true, curInterval.hasBase); + EXPECT_EQ(36, curInterval.base.long_value); + EXPECT_EQ(true, curInterval.hasValue); + EXPECT_EQ(26, curInterval.value.long_value); EXPECT_EQ(1UL, valueProducer.mPastBuckets.size()); - EXPECT_EQ(2UL, valueProducer.mPastBuckets.begin()->second.size()); - EXPECT_EQ(26, valueProducer.mPastBuckets.begin()->second.back().mValue); + EXPECT_EQ(1UL, valueProducer.mPastBuckets.begin()->second.size()); + EXPECT_EQ(10, valueProducer.mPastBuckets.begin()->second.back().values[0].long_value); } /* @@ -213,15 +394,21 @@ TEST(ValueMetricProducerTest, TestPulledEventsTakeZeroOnReset) { metric.mutable_value_field()->set_field(tagId); metric.mutable_value_field()->add_child()->set_field(2); + UidMap uidMap; + SimpleAtomMatcher atomMatcher; + atomMatcher.set_atom_id(tagId); + sp<EventMatcherWizard> eventMatcherWizard = + new EventMatcherWizard({new SimpleLogMatchingTracker( + atomMatcherId, logEventMatcherIndex, atomMatcher, uidMap)}); sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); - shared_ptr<MockStatsPullerManager> pullerManager = - make_shared<StrictMock<MockStatsPullerManager>>(); + sp<MockStatsPullerManager> pullerManager = new StrictMock<MockStatsPullerManager>(); EXPECT_CALL(*pullerManager, RegisterReceiver(tagId, _, _, _)).WillOnce(Return()); EXPECT_CALL(*pullerManager, UnRegisterReceiver(tagId, _)).WillOnce(Return()); + EXPECT_CALL(*pullerManager, Pull(tagId, _, _)).WillOnce(Return(false)); ValueMetricProducer valueProducer(kConfigKey, metric, -1 /*-1 meaning no condition*/, wizard, - tagId, bucketStartTimeNs, bucketStartTimeNs, pullerManager); - valueProducer.setBucketSize(60 * NS_PER_SEC); + logEventMatcherIndex, eventMatcherWizard, tagId, + bucketStartTimeNs, bucketStartTimeNs, pullerManager); vector<shared_ptr<LogEvent>> allData; allData.clear(); @@ -234,13 +421,11 @@ TEST(ValueMetricProducerTest, TestPulledEventsTakeZeroOnReset) { valueProducer.onDataPulled(allData); // has one slice EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); - ValueMetricProducer::Interval curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - valueProducer.setBucketSize(60 * NS_PER_SEC); + ValueMetricProducer::Interval curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; - EXPECT_EQ(true, curInterval.startUpdated); - EXPECT_EQ(0, curInterval.tainted); - EXPECT_EQ(0, curInterval.sum); - EXPECT_EQ(11, curInterval.start); + EXPECT_EQ(true, curInterval.hasBase); + EXPECT_EQ(11, curInterval.base.long_value); + EXPECT_EQ(false, curInterval.hasValue); EXPECT_EQ(0UL, valueProducer.mPastBuckets.size()); allData.clear(); @@ -252,10 +437,10 @@ TEST(ValueMetricProducerTest, TestPulledEventsTakeZeroOnReset) { valueProducer.onDataPulled(allData); // has one slice EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); - curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - EXPECT_EQ(true, curInterval.startUpdated); - EXPECT_EQ(0, curInterval.tainted); - EXPECT_EQ(0, curInterval.sum); + curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + EXPECT_EQ(true, curInterval.hasBase); + EXPECT_EQ(10, curInterval.base.long_value); + EXPECT_EQ(false, curInterval.hasValue); EXPECT_EQ(0UL, valueProducer.mPastBuckets.size()); allData.clear(); @@ -266,13 +451,12 @@ TEST(ValueMetricProducerTest, TestPulledEventsTakeZeroOnReset) { allData.push_back(event); valueProducer.onDataPulled(allData); EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); - curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - EXPECT_EQ(true, curInterval.startUpdated); - EXPECT_EQ(0, curInterval.tainted); - EXPECT_EQ(0, curInterval.sum); - EXPECT_EQ(1UL, valueProducer.mPastBuckets.size()); - EXPECT_EQ(1UL, valueProducer.mPastBuckets.begin()->second.size()); - EXPECT_EQ(26, valueProducer.mPastBuckets.begin()->second.back().mValue); + curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + EXPECT_EQ(true, curInterval.hasBase); + EXPECT_EQ(36, curInterval.base.long_value); + EXPECT_EQ(true, curInterval.hasValue); + EXPECT_EQ(26, curInterval.value.long_value); + EXPECT_EQ(0UL, valueProducer.mPastBuckets.size()); } /* @@ -286,9 +470,14 @@ TEST(ValueMetricProducerTest, TestEventsWithNonSlicedCondition) { metric.mutable_value_field()->add_child()->set_field(2); metric.set_condition(StringToId("SCREEN_ON")); + UidMap uidMap; + SimpleAtomMatcher atomMatcher; + atomMatcher.set_atom_id(tagId); + sp<EventMatcherWizard> eventMatcherWizard = + new EventMatcherWizard({new SimpleLogMatchingTracker( + atomMatcherId, logEventMatcherIndex, atomMatcher, uidMap)}); sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); - shared_ptr<MockStatsPullerManager> pullerManager = - make_shared<StrictMock<MockStatsPullerManager>>(); + sp<MockStatsPullerManager> pullerManager = new StrictMock<MockStatsPullerManager>(); EXPECT_CALL(*pullerManager, RegisterReceiver(tagId, _, _, _)).WillOnce(Return()); EXPECT_CALL(*pullerManager, UnRegisterReceiver(tagId, _)).WillRepeatedly(Return()); @@ -296,7 +485,7 @@ TEST(ValueMetricProducerTest, TestEventsWithNonSlicedCondition) { .WillOnce(Invoke([](int tagId, int64_t timeNs, vector<std::shared_ptr<LogEvent>>* data) { data->clear(); - shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucketStartTimeNs + 10); + shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucketStartTimeNs + 8); event->write(tagId); event->write(100); event->init(); @@ -306,7 +495,7 @@ TEST(ValueMetricProducerTest, TestEventsWithNonSlicedCondition) { .WillOnce(Invoke([](int tagId, int64_t timeNs, vector<std::shared_ptr<LogEvent>>* data) { data->clear(); - shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucket2StartTimeNs + 10); + shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucket2StartTimeNs + 1); event->write(tagId); event->write(120); event->init(); @@ -314,19 +503,18 @@ TEST(ValueMetricProducerTest, TestEventsWithNonSlicedCondition) { return true; })); - ValueMetricProducer valueProducer(kConfigKey, metric, 1, wizard, tagId, bucketStartTimeNs, + ValueMetricProducer valueProducer(kConfigKey, metric, 1, wizard, logEventMatcherIndex, + eventMatcherWizard, tagId, bucketStartTimeNs, bucketStartTimeNs, pullerManager); - valueProducer.setBucketSize(60 * NS_PER_SEC); valueProducer.onConditionChanged(true, bucketStartTimeNs + 8); // has one slice EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); - ValueMetricProducer::Interval curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - // startUpdated:false tainted:0 sum:0 start:100 - EXPECT_EQ(100, curInterval.start); - EXPECT_EQ(true, curInterval.startUpdated); - EXPECT_EQ(0, curInterval.tainted); - EXPECT_EQ(0, curInterval.sum); + ValueMetricProducer::Interval curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + // startUpdated:false sum:0 start:100 + EXPECT_EQ(true, curInterval.hasBase); + EXPECT_EQ(100, curInterval.base.long_value); + EXPECT_EQ(false, curInterval.hasValue); EXPECT_EQ(0UL, valueProducer.mPastBuckets.size()); vector<shared_ptr<LogEvent>> allData; @@ -340,21 +528,21 @@ TEST(ValueMetricProducerTest, TestEventsWithNonSlicedCondition) { // has one slice EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); - curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - // startUpdated:false tainted:0 sum:0 start:110 - EXPECT_EQ(110, curInterval.start); - EXPECT_EQ(1UL, valueProducer.mPastBuckets.size()); - EXPECT_EQ(1UL, valueProducer.mPastBuckets.begin()->second.size()); - EXPECT_EQ(10, valueProducer.mPastBuckets.begin()->second.back().mValue); + curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + EXPECT_EQ(true, curInterval.hasBase); + EXPECT_EQ(110, curInterval.base.long_value); + EXPECT_EQ(true, curInterval.hasValue); + EXPECT_EQ(10, curInterval.value.long_value); + EXPECT_EQ(0UL, valueProducer.mPastBuckets.size()); valueProducer.onConditionChanged(false, bucket2StartTimeNs + 1); // has one slice EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); - curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - // startUpdated:false tainted:0 sum:0 start:110 - EXPECT_EQ(10, curInterval.sum); - EXPECT_EQ(false, curInterval.startUpdated); + curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + EXPECT_EQ(true, curInterval.hasValue); + EXPECT_EQ(10, curInterval.value.long_value); + EXPECT_EQ(false, curInterval.hasBase); } TEST(ValueMetricProducerTest, TestPushedEventsWithUpgrade) { @@ -364,12 +552,17 @@ TEST(ValueMetricProducerTest, TestPushedEventsWithUpgrade) { metric.mutable_value_field()->set_field(tagId); metric.mutable_value_field()->add_child()->set_field(2); + UidMap uidMap; + SimpleAtomMatcher atomMatcher; + atomMatcher.set_atom_id(tagId); + sp<EventMatcherWizard> eventMatcherWizard = + new EventMatcherWizard({new SimpleLogMatchingTracker( + atomMatcherId, logEventMatcherIndex, atomMatcher, uidMap)}); sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); - shared_ptr<MockStatsPullerManager> pullerManager = - make_shared<StrictMock<MockStatsPullerManager>>(); - ValueMetricProducer valueProducer(kConfigKey, metric, -1, wizard, -1, bucketStartTimeNs, - bucketStartTimeNs, pullerManager); - valueProducer.setBucketSize(60 * NS_PER_SEC); + sp<MockStatsPullerManager> pullerManager = new StrictMock<MockStatsPullerManager>(); + ValueMetricProducer valueProducer(kConfigKey, metric, -1, wizard, logEventMatcherIndex, + eventMatcherWizard, -1, bucketStartTimeNs, bucketStartTimeNs, + pullerManager); shared_ptr<LogEvent> event1 = make_shared<LogEvent>(tagId, bucketStartTimeNs + 10); event1->write(1); @@ -378,9 +571,9 @@ TEST(ValueMetricProducerTest, TestPushedEventsWithUpgrade) { valueProducer.onMatchedLogEvent(1 /*log matcher index*/, *event1); EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); - valueProducer.notifyAppUpgrade(eventUpgradeTimeNs, "ANY.APP", 1, 1); + valueProducer.notifyAppUpgrade(bucketStartTimeNs + 150, "ANY.APP", 1, 1); EXPECT_EQ(1UL, valueProducer.mPastBuckets[DEFAULT_METRIC_DIMENSION_KEY].size()); - EXPECT_EQ(eventUpgradeTimeNs, valueProducer.mCurrentBucketStartTimeNs); + EXPECT_EQ(bucketStartTimeNs + 150, valueProducer.mCurrentBucketStartTimeNs); shared_ptr<LogEvent> event2 = make_shared<LogEvent>(tagId, bucketStartTimeNs + 59 * NS_PER_SEC); event2->write(1); @@ -388,7 +581,7 @@ TEST(ValueMetricProducerTest, TestPushedEventsWithUpgrade) { event2->init(); valueProducer.onMatchedLogEvent(1 /*log matcher index*/, *event2); EXPECT_EQ(1UL, valueProducer.mPastBuckets[DEFAULT_METRIC_DIMENSION_KEY].size()); - EXPECT_EQ(eventUpgradeTimeNs, valueProducer.mCurrentBucketStartTimeNs); + EXPECT_EQ(bucketStartTimeNs + 150, valueProducer.mCurrentBucketStartTimeNs); // Next value should create a new bucket. shared_ptr<LogEvent> event3 = make_shared<LogEvent>(tagId, bucketStartTimeNs + 65 * NS_PER_SEC); @@ -407,29 +600,35 @@ TEST(ValueMetricProducerTest, TestPulledValueWithUpgrade) { metric.mutable_value_field()->set_field(tagId); metric.mutable_value_field()->add_child()->set_field(2); + UidMap uidMap; + SimpleAtomMatcher atomMatcher; + atomMatcher.set_atom_id(tagId); + sp<EventMatcherWizard> eventMatcherWizard = + new EventMatcherWizard({new SimpleLogMatchingTracker( + atomMatcherId, logEventMatcherIndex, atomMatcher, uidMap)}); sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); - shared_ptr<MockStatsPullerManager> pullerManager = - make_shared<StrictMock<MockStatsPullerManager>>(); + sp<MockStatsPullerManager> pullerManager = new StrictMock<MockStatsPullerManager>(); EXPECT_CALL(*pullerManager, RegisterReceiver(tagId, _, _, _)).WillOnce(Return()); EXPECT_CALL(*pullerManager, UnRegisterReceiver(tagId, _)).WillOnce(Return()); EXPECT_CALL(*pullerManager, Pull(tagId, _, _)) + .WillOnce(Return(true)) .WillOnce(Invoke([](int tagId, int64_t timeNs, vector<std::shared_ptr<LogEvent>>* data) { data->clear(); - shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucketStartTimeNs + 10); + shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucket2StartTimeNs + 149); event->write(tagId); event->write(120); event->init(); data->push_back(event); return true; })); - ValueMetricProducer valueProducer(kConfigKey, metric, -1, wizard, tagId, bucketStartTimeNs, + ValueMetricProducer valueProducer(kConfigKey, metric, -1, wizard, logEventMatcherIndex, + eventMatcherWizard, tagId, bucketStartTimeNs, bucketStartTimeNs, pullerManager); - valueProducer.setBucketSize(60 * NS_PER_SEC); vector<shared_ptr<LogEvent>> allData; allData.clear(); - shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucketStartTimeNs + 1); + shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucket2StartTimeNs + 1); event->write(tagId); event->write(100); event->init(); @@ -438,21 +637,21 @@ TEST(ValueMetricProducerTest, TestPulledValueWithUpgrade) { valueProducer.onDataPulled(allData); EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); - valueProducer.notifyAppUpgrade(eventUpgradeTimeNs, "ANY.APP", 1, 1); + valueProducer.notifyAppUpgrade(bucket2StartTimeNs + 150, "ANY.APP", 1, 1); EXPECT_EQ(1UL, valueProducer.mPastBuckets[DEFAULT_METRIC_DIMENSION_KEY].size()); - EXPECT_EQ(eventUpgradeTimeNs, valueProducer.mCurrentBucketStartTimeNs); - EXPECT_EQ(20L, valueProducer.mPastBuckets[DEFAULT_METRIC_DIMENSION_KEY][0].mValue); + EXPECT_EQ(bucket2StartTimeNs + 150, valueProducer.mCurrentBucketStartTimeNs); + EXPECT_EQ(20L, valueProducer.mPastBuckets[DEFAULT_METRIC_DIMENSION_KEY][0].values[0].long_value); allData.clear(); - event = make_shared<LogEvent>(tagId, bucket2StartTimeNs + 1); + event = make_shared<LogEvent>(tagId, bucket3StartTimeNs + 1); event->write(tagId); event->write(150); event->init(); allData.push_back(event); valueProducer.onDataPulled(allData); - EXPECT_EQ(2UL, valueProducer.mPastBuckets[DEFAULT_METRIC_DIMENSION_KEY].size()); - EXPECT_EQ(bucket2StartTimeNs, valueProducer.mCurrentBucketStartTimeNs); - EXPECT_EQ(30L, valueProducer.mPastBuckets[DEFAULT_METRIC_DIMENSION_KEY][1].mValue); + EXPECT_EQ(1UL, valueProducer.mPastBuckets[DEFAULT_METRIC_DIMENSION_KEY].size()); + EXPECT_EQ(bucket2StartTimeNs + 150, valueProducer.mCurrentBucketStartTimeNs); + EXPECT_EQ(20L, valueProducer.mPastBuckets[DEFAULT_METRIC_DIMENSION_KEY][0].values[0].long_value); } TEST(ValueMetricProducerTest, TestPulledValueWithUpgradeWhileConditionFalse) { @@ -463,16 +662,21 @@ TEST(ValueMetricProducerTest, TestPulledValueWithUpgradeWhileConditionFalse) { metric.mutable_value_field()->add_child()->set_field(2); metric.set_condition(StringToId("SCREEN_ON")); + UidMap uidMap; + SimpleAtomMatcher atomMatcher; + atomMatcher.set_atom_id(tagId); + sp<EventMatcherWizard> eventMatcherWizard = + new EventMatcherWizard({new SimpleLogMatchingTracker( + atomMatcherId, logEventMatcherIndex, atomMatcher, uidMap)}); sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); - shared_ptr<MockStatsPullerManager> pullerManager = - make_shared<StrictMock<MockStatsPullerManager>>(); + sp<MockStatsPullerManager> pullerManager = new StrictMock<MockStatsPullerManager>(); EXPECT_CALL(*pullerManager, RegisterReceiver(tagId, _, _, _)).WillOnce(Return()); EXPECT_CALL(*pullerManager, UnRegisterReceiver(tagId, _)).WillOnce(Return()); EXPECT_CALL(*pullerManager, Pull(tagId, _, _)) .WillOnce(Invoke([](int tagId, int64_t timeNs, vector<std::shared_ptr<LogEvent>>* data) { data->clear(); - shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucketStartTimeNs + 10); + shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucketStartTimeNs + 1); event->write(tagId); event->write(100); event->init(); @@ -482,16 +686,16 @@ TEST(ValueMetricProducerTest, TestPulledValueWithUpgradeWhileConditionFalse) { .WillOnce(Invoke([](int tagId, int64_t timeNs, vector<std::shared_ptr<LogEvent>>* data) { data->clear(); - shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucketStartTimeNs + 10); + shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucket2StartTimeNs - 100); event->write(tagId); event->write(120); event->init(); data->push_back(event); return true; })); - ValueMetricProducer valueProducer(kConfigKey, metric, 1, wizard, tagId, bucketStartTimeNs, + ValueMetricProducer valueProducer(kConfigKey, metric, 1, wizard, logEventMatcherIndex, + eventMatcherWizard, tagId, bucketStartTimeNs, bucketStartTimeNs, pullerManager); - valueProducer.setBucketSize(60 * NS_PER_SEC); valueProducer.onConditionChanged(true, bucketStartTimeNs + 1); valueProducer.onConditionChanged(false, bucket2StartTimeNs-100); @@ -502,7 +706,7 @@ TEST(ValueMetricProducerTest, TestPulledValueWithUpgradeWhileConditionFalse) { EXPECT_EQ(bucket2StartTimeNs-50, valueProducer.mCurrentBucketStartTimeNs); EXPECT_EQ(1UL, valueProducer.mPastBuckets[DEFAULT_METRIC_DIMENSION_KEY].size()); EXPECT_EQ(bucketStartTimeNs, valueProducer.mPastBuckets[DEFAULT_METRIC_DIMENSION_KEY][0].mBucketStartNs); - EXPECT_EQ(20L, valueProducer.mPastBuckets[DEFAULT_METRIC_DIMENSION_KEY][0].mValue); + EXPECT_EQ(20L, valueProducer.mPastBuckets[DEFAULT_METRIC_DIMENSION_KEY][0].values[0].long_value); EXPECT_FALSE(valueProducer.mCondition); } @@ -513,13 +717,18 @@ TEST(ValueMetricProducerTest, TestPushedEventsWithoutCondition) { metric.mutable_value_field()->set_field(tagId); metric.mutable_value_field()->add_child()->set_field(2); + UidMap uidMap; + SimpleAtomMatcher atomMatcher; + atomMatcher.set_atom_id(tagId); + sp<EventMatcherWizard> eventMatcherWizard = + new EventMatcherWizard({new SimpleLogMatchingTracker( + atomMatcherId, logEventMatcherIndex, atomMatcher, uidMap)}); sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); - shared_ptr<MockStatsPullerManager> pullerManager = - make_shared<StrictMock<MockStatsPullerManager>>(); + sp<MockStatsPullerManager> pullerManager = new StrictMock<MockStatsPullerManager>(); - ValueMetricProducer valueProducer(kConfigKey, metric, -1, wizard, -1, bucketStartTimeNs, - bucketStartTimeNs, pullerManager); - valueProducer.setBucketSize(60 * NS_PER_SEC); + ValueMetricProducer valueProducer(kConfigKey, metric, -1, wizard, logEventMatcherIndex, + eventMatcherWizard, -1, bucketStartTimeNs, bucketStartTimeNs, + pullerManager); shared_ptr<LogEvent> event1 = make_shared<LogEvent>(tagId, bucketStartTimeNs + 10); event1->write(1); @@ -532,20 +741,21 @@ TEST(ValueMetricProducerTest, TestPushedEventsWithoutCondition) { valueProducer.onMatchedLogEvent(1 /*log matcher index*/, *event1); // has one slice EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); - ValueMetricProducer::Interval curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - EXPECT_EQ(10, curInterval.sum); + ValueMetricProducer::Interval curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + EXPECT_EQ(10, curInterval.value.long_value); + EXPECT_EQ(true, curInterval.hasValue); valueProducer.onMatchedLogEvent(1 /*log matcher index*/, *event2); // has one slice EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); - curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - EXPECT_EQ(30, curInterval.sum); + curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + EXPECT_EQ(30, curInterval.value.long_value); valueProducer.flushIfNeededLocked(bucket3StartTimeNs); EXPECT_EQ(1UL, valueProducer.mPastBuckets.size()); EXPECT_EQ(1UL, valueProducer.mPastBuckets.begin()->second.size()); - EXPECT_EQ(30, valueProducer.mPastBuckets.begin()->second.back().mValue); + EXPECT_EQ(30, valueProducer.mPastBuckets.begin()->second.back().values[0].long_value); } TEST(ValueMetricProducerTest, TestPushedEventsWithCondition) { @@ -555,13 +765,18 @@ TEST(ValueMetricProducerTest, TestPushedEventsWithCondition) { metric.mutable_value_field()->set_field(tagId); metric.mutable_value_field()->add_child()->set_field(2); + UidMap uidMap; + SimpleAtomMatcher atomMatcher; + atomMatcher.set_atom_id(tagId); + sp<EventMatcherWizard> eventMatcherWizard = + new EventMatcherWizard({new SimpleLogMatchingTracker( + atomMatcherId, logEventMatcherIndex, atomMatcher, uidMap)}); sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); - shared_ptr<MockStatsPullerManager> pullerManager = - make_shared<StrictMock<MockStatsPullerManager>>(); + sp<MockStatsPullerManager> pullerManager = new StrictMock<MockStatsPullerManager>(); - ValueMetricProducer valueProducer(kConfigKey, metric, 1, wizard, -1, bucketStartTimeNs, - bucketStartTimeNs, pullerManager); - valueProducer.setBucketSize(60 * NS_PER_SEC); + ValueMetricProducer valueProducer(kConfigKey, metric, 1, wizard, logEventMatcherIndex, + eventMatcherWizard, -1, bucketStartTimeNs, bucketStartTimeNs, + pullerManager); shared_ptr<LogEvent> event1 = make_shared<LogEvent>(tagId, bucketStartTimeNs + 10); event1->write(1); @@ -569,9 +784,7 @@ TEST(ValueMetricProducerTest, TestPushedEventsWithCondition) { event1->init(); valueProducer.onMatchedLogEvent(1 /*log matcher index*/, *event1); // has 1 slice - EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); - ValueMetricProducer::Interval curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - EXPECT_EQ(false, curInterval.hasValue); + EXPECT_EQ(0UL, valueProducer.mCurrentSlicedBucket.size()); valueProducer.onConditionChangedLocked(true, bucketStartTimeNs + 15); shared_ptr<LogEvent> event2 = make_shared<LogEvent>(tagId, bucketStartTimeNs + 20); @@ -582,8 +795,9 @@ TEST(ValueMetricProducerTest, TestPushedEventsWithCondition) { // has one slice EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); - curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - EXPECT_EQ(20, curInterval.sum); + ValueMetricProducer::Interval curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + EXPECT_EQ(20, curInterval.value.long_value); shared_ptr<LogEvent> event3 = make_shared<LogEvent>(tagId, bucketStartTimeNs + 30); event3->write(1); @@ -593,8 +807,8 @@ TEST(ValueMetricProducerTest, TestPushedEventsWithCondition) { // has one slice EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); - curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - EXPECT_EQ(50, curInterval.sum); + curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + EXPECT_EQ(50, curInterval.value.long_value); valueProducer.onConditionChangedLocked(false, bucketStartTimeNs + 35); shared_ptr<LogEvent> event4 = make_shared<LogEvent>(tagId, bucketStartTimeNs + 40); @@ -605,13 +819,13 @@ TEST(ValueMetricProducerTest, TestPushedEventsWithCondition) { // has one slice EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); - curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - EXPECT_EQ(50, curInterval.sum); + curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + EXPECT_EQ(50, curInterval.value.long_value); valueProducer.flushIfNeededLocked(bucket3StartTimeNs); EXPECT_EQ(1UL, valueProducer.mPastBuckets.size()); EXPECT_EQ(1UL, valueProducer.mPastBuckets.begin()->second.size()); - EXPECT_EQ(50, valueProducer.mPastBuckets.begin()->second.back().mValue); + EXPECT_EQ(50, valueProducer.mPastBuckets.begin()->second.back().values[0].long_value); } TEST(ValueMetricProducerTest, TestAnomalyDetection) { @@ -630,10 +844,17 @@ TEST(ValueMetricProducerTest, TestAnomalyDetection) { metric.mutable_value_field()->set_field(tagId); metric.mutable_value_field()->add_child()->set_field(2); + UidMap uidMap; + SimpleAtomMatcher atomMatcher; + atomMatcher.set_atom_id(tagId); + sp<EventMatcherWizard> eventMatcherWizard = + new EventMatcherWizard({new SimpleLogMatchingTracker( + atomMatcherId, logEventMatcherIndex, atomMatcher, uidMap)}); sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); + sp<MockStatsPullerManager> pullerManager = new StrictMock<MockStatsPullerManager>(); ValueMetricProducer valueProducer(kConfigKey, metric, -1 /*-1 meaning no condition*/, wizard, - -1 /*not pulled*/, bucketStartTimeNs, bucketStartTimeNs); - valueProducer.setBucketSize(60 * NS_PER_SEC); + logEventMatcherIndex, eventMatcherWizard, -1 /*not pulled*/, + bucketStartTimeNs, bucketStartTimeNs, pullerManager); sp<AnomalyTracker> anomalyTracker = valueProducer.addAnomalyTracker(alert, alarmMonitor); @@ -704,15 +925,21 @@ TEST(ValueMetricProducerTest, TestBucketBoundaryNoCondition) { metric.mutable_value_field()->set_field(tagId); metric.mutable_value_field()->add_child()->set_field(2); + UidMap uidMap; + SimpleAtomMatcher atomMatcher; + atomMatcher.set_atom_id(tagId); + sp<EventMatcherWizard> eventMatcherWizard = + new EventMatcherWizard({new SimpleLogMatchingTracker( + atomMatcherId, logEventMatcherIndex, atomMatcher, uidMap)}); sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); - shared_ptr<MockStatsPullerManager> pullerManager = - make_shared<StrictMock<MockStatsPullerManager>>(); + sp<MockStatsPullerManager> pullerManager = new StrictMock<MockStatsPullerManager>(); EXPECT_CALL(*pullerManager, RegisterReceiver(tagId, _, _, _)).WillOnce(Return()); EXPECT_CALL(*pullerManager, UnRegisterReceiver(tagId, _)).WillOnce(Return()); + EXPECT_CALL(*pullerManager, Pull(tagId, _, _)).WillOnce(Return(true)); ValueMetricProducer valueProducer(kConfigKey, metric, -1 /*-1 meaning no condition*/, wizard, - tagId, bucketStartTimeNs, bucketStartTimeNs, pullerManager); - valueProducer.setBucketSize(60 * NS_PER_SEC); + logEventMatcherIndex, eventMatcherWizard, tagId, + bucketStartTimeNs, bucketStartTimeNs, pullerManager); vector<shared_ptr<LogEvent>> allData; // pull 1 @@ -726,13 +953,12 @@ TEST(ValueMetricProducerTest, TestBucketBoundaryNoCondition) { valueProducer.onDataPulled(allData); // has one slice EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); - ValueMetricProducer::Interval curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; + ValueMetricProducer::Interval curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; - // startUpdated:true tainted:0 sum:0 start:11 - EXPECT_EQ(true, curInterval.startUpdated); - EXPECT_EQ(0, curInterval.tainted); - EXPECT_EQ(0, curInterval.sum); - EXPECT_EQ(11, curInterval.start); + // startUpdated:true sum:0 start:11 + EXPECT_EQ(true, curInterval.hasBase); + EXPECT_EQ(11, curInterval.base.long_value); + EXPECT_EQ(false, curInterval.hasValue); EXPECT_EQ(0UL, valueProducer.mPastBuckets.size()); // pull 2 at correct time @@ -745,14 +971,13 @@ TEST(ValueMetricProducerTest, TestBucketBoundaryNoCondition) { valueProducer.onDataPulled(allData); // has one slice EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); - curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - // tartUpdated:false tainted:0 sum:12 - EXPECT_EQ(true, curInterval.startUpdated); - EXPECT_EQ(0, curInterval.tainted); - EXPECT_EQ(0, curInterval.sum); - EXPECT_EQ(1UL, valueProducer.mPastBuckets.size()); - EXPECT_EQ(1UL, valueProducer.mPastBuckets.begin()->second.size()); - EXPECT_EQ(12, valueProducer.mPastBuckets.begin()->second.back().mValue); + curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + // tartUpdated:false sum:12 + EXPECT_EQ(true, curInterval.hasBase); + EXPECT_EQ(23, curInterval.base.long_value); + EXPECT_EQ(true, curInterval.hasValue); + EXPECT_EQ(12, curInterval.value.long_value); + EXPECT_EQ(0UL, valueProducer.mPastBuckets.size()); // pull 3 come late. // The previous bucket gets closed with error. (Has start value 23, no ending) @@ -766,15 +991,14 @@ TEST(ValueMetricProducerTest, TestBucketBoundaryNoCondition) { allData.push_back(event); valueProducer.onDataPulled(allData); EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); - curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - // startUpdated:false tainted:0 sum:12 - EXPECT_EQ(true, curInterval.startUpdated); - EXPECT_EQ(0, curInterval.tainted); - EXPECT_EQ(36, curInterval.start); - EXPECT_EQ(0, curInterval.sum); + curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + // startUpdated:false sum:12 + EXPECT_EQ(true, curInterval.hasBase); + EXPECT_EQ(36, curInterval.base.long_value); + EXPECT_EQ(false, curInterval.hasValue); EXPECT_EQ(1UL, valueProducer.mPastBuckets.size()); EXPECT_EQ(1UL, valueProducer.mPastBuckets.begin()->second.size()); - EXPECT_EQ(12, valueProducer.mPastBuckets.begin()->second.back().mValue); + EXPECT_EQ(12, valueProducer.mPastBuckets.begin()->second.back().values[0].long_value); } /* @@ -789,9 +1013,14 @@ TEST(ValueMetricProducerTest, TestBucketBoundaryWithCondition) { metric.mutable_value_field()->add_child()->set_field(2); metric.set_condition(StringToId("SCREEN_ON")); + UidMap uidMap; + SimpleAtomMatcher atomMatcher; + atomMatcher.set_atom_id(tagId); + sp<EventMatcherWizard> eventMatcherWizard = + new EventMatcherWizard({new SimpleLogMatchingTracker( + atomMatcherId, logEventMatcherIndex, atomMatcher, uidMap)}); sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); - shared_ptr<MockStatsPullerManager> pullerManager = - make_shared<StrictMock<MockStatsPullerManager>>(); + sp<MockStatsPullerManager> pullerManager = new StrictMock<MockStatsPullerManager>(); EXPECT_CALL(*pullerManager, RegisterReceiver(tagId, _, _, _)).WillOnce(Return()); EXPECT_CALL(*pullerManager, UnRegisterReceiver(tagId, _)).WillRepeatedly(Return()); @@ -800,7 +1029,7 @@ TEST(ValueMetricProducerTest, TestBucketBoundaryWithCondition) { .WillOnce(Invoke([](int tagId, int64_t timeNs, vector<std::shared_ptr<LogEvent>>* data) { data->clear(); - shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucketStartTimeNs + 10); + shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucketStartTimeNs + 8); event->write(tagId); event->write(100); event->init(); @@ -811,7 +1040,7 @@ TEST(ValueMetricProducerTest, TestBucketBoundaryWithCondition) { .WillOnce(Invoke([](int tagId, int64_t timeNs, vector<std::shared_ptr<LogEvent>>* data) { data->clear(); - shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucket2StartTimeNs + 20); + shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucket2StartTimeNs + 1); event->write(tagId); event->write(120); event->init(); @@ -819,27 +1048,25 @@ TEST(ValueMetricProducerTest, TestBucketBoundaryWithCondition) { return true; })); - ValueMetricProducer valueProducer(kConfigKey, metric, 1, wizard, tagId, bucketStartTimeNs, + ValueMetricProducer valueProducer(kConfigKey, metric, 1, wizard, logEventMatcherIndex, + eventMatcherWizard, tagId, bucketStartTimeNs, bucketStartTimeNs, pullerManager); - valueProducer.setBucketSize(60 * NS_PER_SEC); valueProducer.onConditionChanged(true, bucketStartTimeNs + 8); // has one slice EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); - ValueMetricProducer::Interval curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - // startUpdated:false tainted:0 sum:0 start:100 - EXPECT_EQ(100, curInterval.start); - EXPECT_EQ(true, curInterval.startUpdated); - EXPECT_EQ(0, curInterval.tainted); - EXPECT_EQ(0, curInterval.sum); + ValueMetricProducer::Interval curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + EXPECT_EQ(true, curInterval.hasBase); + EXPECT_EQ(100, curInterval.base.long_value); + EXPECT_EQ(false, curInterval.hasValue); EXPECT_EQ(0UL, valueProducer.mPastBuckets.size()); // pull on bucket boundary come late, condition change happens before it valueProducer.onConditionChanged(false, bucket2StartTimeNs + 1); - curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - EXPECT_EQ(false, curInterval.startUpdated); - EXPECT_EQ(1, curInterval.tainted); - EXPECT_EQ(0, curInterval.sum); + curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + EXPECT_EQ(false, curInterval.hasBase); + EXPECT_EQ(true, curInterval.hasValue); + EXPECT_EQ(20, curInterval.value.long_value); EXPECT_EQ(0UL, valueProducer.mPastBuckets.size()); // Now the alarm is delivered. @@ -853,10 +1080,10 @@ TEST(ValueMetricProducerTest, TestBucketBoundaryWithCondition) { allData.push_back(event); valueProducer.onDataPulled(allData); - curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - EXPECT_EQ(false, curInterval.startUpdated); - EXPECT_EQ(1, curInterval.tainted); - EXPECT_EQ(0, curInterval.sum); + curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + EXPECT_EQ(false, curInterval.hasBase); + EXPECT_EQ(true, curInterval.hasValue); + EXPECT_EQ(20, curInterval.value.long_value); EXPECT_EQ(0UL, valueProducer.mPastBuckets.size()); } @@ -872,9 +1099,14 @@ TEST(ValueMetricProducerTest, TestBucketBoundaryWithCondition2) { metric.mutable_value_field()->add_child()->set_field(2); metric.set_condition(StringToId("SCREEN_ON")); + UidMap uidMap; + SimpleAtomMatcher atomMatcher; + atomMatcher.set_atom_id(tagId); + sp<EventMatcherWizard> eventMatcherWizard = + new EventMatcherWizard({new SimpleLogMatchingTracker( + atomMatcherId, logEventMatcherIndex, atomMatcher, uidMap)}); sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); - shared_ptr<MockStatsPullerManager> pullerManager = - make_shared<StrictMock<MockStatsPullerManager>>(); + sp<MockStatsPullerManager> pullerManager = new StrictMock<MockStatsPullerManager>(); EXPECT_CALL(*pullerManager, RegisterReceiver(tagId, _, _, _)).WillRepeatedly(Return()); EXPECT_CALL(*pullerManager, UnRegisterReceiver(tagId, _)).WillRepeatedly(Return()); @@ -883,7 +1115,7 @@ TEST(ValueMetricProducerTest, TestBucketBoundaryWithCondition2) { .WillOnce(Invoke([](int tagId, int64_t timeNs, vector<std::shared_ptr<LogEvent>>* data) { data->clear(); - shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucketStartTimeNs + 10); + shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucketStartTimeNs + 8); event->write(tagId); event->write(100); event->init(); @@ -894,7 +1126,7 @@ TEST(ValueMetricProducerTest, TestBucketBoundaryWithCondition2) { .WillOnce(Invoke([](int tagId, int64_t timeNs, vector<std::shared_ptr<LogEvent>>* data) { data->clear(); - shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucket2StartTimeNs + 20); + shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucket2StartTimeNs + 1); event->write(tagId); event->write(120); event->init(); @@ -905,7 +1137,7 @@ TEST(ValueMetricProducerTest, TestBucketBoundaryWithCondition2) { .WillOnce(Invoke([](int tagId, int64_t timeNs, vector<std::shared_ptr<LogEvent>>* data) { data->clear(); - shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucket2StartTimeNs + 30); + shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucket2StartTimeNs + 25); event->write(tagId); event->write(130); event->init(); @@ -913,36 +1145,35 @@ TEST(ValueMetricProducerTest, TestBucketBoundaryWithCondition2) { return true; })); - ValueMetricProducer valueProducer(kConfigKey, metric, 1, wizard, tagId, bucketStartTimeNs, + ValueMetricProducer valueProducer(kConfigKey, metric, 1, wizard, logEventMatcherIndex, + eventMatcherWizard, tagId, bucketStartTimeNs, bucketStartTimeNs, pullerManager); - valueProducer.setBucketSize(60 * NS_PER_SEC); valueProducer.onConditionChanged(true, bucketStartTimeNs + 8); // has one slice EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); - ValueMetricProducer::Interval curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - // startUpdated:false tainted:0 sum:0 start:100 - EXPECT_EQ(100, curInterval.start); - EXPECT_EQ(true, curInterval.startUpdated); - EXPECT_EQ(0, curInterval.tainted); - EXPECT_EQ(0, curInterval.sum); + ValueMetricProducer::Interval curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + // startUpdated:false sum:0 start:100 + EXPECT_EQ(true, curInterval.hasBase); + EXPECT_EQ(100, curInterval.base.long_value); + EXPECT_EQ(false, curInterval.hasValue); EXPECT_EQ(0UL, valueProducer.mPastBuckets.size()); // pull on bucket boundary come late, condition change happens before it valueProducer.onConditionChanged(false, bucket2StartTimeNs + 1); - curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - EXPECT_EQ(false, curInterval.startUpdated); - EXPECT_EQ(1, curInterval.tainted); - EXPECT_EQ(0, curInterval.sum); + curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + EXPECT_EQ(false, curInterval.hasBase); + EXPECT_EQ(true, curInterval.hasValue); + EXPECT_EQ(20, curInterval.value.long_value); EXPECT_EQ(0UL, valueProducer.mPastBuckets.size()); // condition changed to true again, before the pull alarm is delivered valueProducer.onConditionChanged(true, bucket2StartTimeNs + 25); - curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - EXPECT_EQ(true, curInterval.startUpdated); - EXPECT_EQ(130, curInterval.start); - EXPECT_EQ(1, curInterval.tainted); - EXPECT_EQ(0, curInterval.sum); + curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + EXPECT_EQ(true, curInterval.hasBase); + EXPECT_EQ(130, curInterval.base.long_value); + EXPECT_EQ(true, curInterval.hasValue); + EXPECT_EQ(20, curInterval.value.long_value); EXPECT_EQ(0UL, valueProducer.mPastBuckets.size()); // Now the alarm is delivered, but it is considered late, it has no effect @@ -955,96 +1186,286 @@ TEST(ValueMetricProducerTest, TestBucketBoundaryWithCondition2) { allData.push_back(event); valueProducer.onDataPulled(allData); - curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - EXPECT_EQ(true, curInterval.startUpdated); - EXPECT_EQ(130, curInterval.start); - EXPECT_EQ(1, curInterval.tainted); - EXPECT_EQ(0, curInterval.sum); + curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + EXPECT_EQ(true, curInterval.hasBase); + EXPECT_EQ(130, curInterval.base.long_value); + EXPECT_EQ(true, curInterval.hasValue); + EXPECT_EQ(20, curInterval.value.long_value); EXPECT_EQ(0UL, valueProducer.mPastBuckets.size()); } -/* - * Test pulled event with non sliced condition. The pull on boundary come late because the puller is - * very slow. - */ -TEST(ValueMetricProducerTest, TestBucketBoundaryWithCondition3) { +TEST(ValueMetricProducerTest, TestPushedAggregateMin) { ValueMetric metric; metric.set_id(metricId); metric.set_bucket(ONE_MINUTE); metric.mutable_value_field()->set_field(tagId); metric.mutable_value_field()->add_child()->set_field(2); - metric.set_condition(StringToId("SCREEN_ON")); + metric.set_aggregation_type(ValueMetric::MIN); + + UidMap uidMap; + SimpleAtomMatcher atomMatcher; + atomMatcher.set_atom_id(tagId); + sp<EventMatcherWizard> eventMatcherWizard = + new EventMatcherWizard({new SimpleLogMatchingTracker( + atomMatcherId, logEventMatcherIndex, atomMatcher, uidMap)}); + sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); + sp<MockStatsPullerManager> pullerManager = new StrictMock<MockStatsPullerManager>(); + + ValueMetricProducer valueProducer(kConfigKey, metric, -1, wizard, logEventMatcherIndex, + eventMatcherWizard, -1, bucketStartTimeNs, bucketStartTimeNs, + pullerManager); + + shared_ptr<LogEvent> event1 = make_shared<LogEvent>(tagId, bucketStartTimeNs + 10); + event1->write(1); + event1->write(10); + event1->init(); + shared_ptr<LogEvent> event2 = make_shared<LogEvent>(tagId, bucketStartTimeNs + 20); + event2->write(1); + event2->write(20); + event2->init(); + valueProducer.onMatchedLogEvent(1 /*log matcher index*/, *event1); + // has one slice + EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); + ValueMetricProducer::Interval curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + EXPECT_EQ(10, curInterval.value.long_value); + EXPECT_EQ(true, curInterval.hasValue); + + valueProducer.onMatchedLogEvent(1 /*log matcher index*/, *event2); + + // has one slice + EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); + curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + EXPECT_EQ(10, curInterval.value.long_value); + + valueProducer.flushIfNeededLocked(bucket3StartTimeNs); + EXPECT_EQ(1UL, valueProducer.mPastBuckets.size()); + EXPECT_EQ(1UL, valueProducer.mPastBuckets.begin()->second.size()); + EXPECT_EQ(10, valueProducer.mPastBuckets.begin()->second.back().values[0].long_value); +} +TEST(ValueMetricProducerTest, TestPushedAggregateMax) { + ValueMetric metric; + metric.set_id(metricId); + metric.set_bucket(ONE_MINUTE); + metric.mutable_value_field()->set_field(tagId); + metric.mutable_value_field()->add_child()->set_field(2); + metric.set_aggregation_type(ValueMetric::MAX); + + UidMap uidMap; + SimpleAtomMatcher atomMatcher; + atomMatcher.set_atom_id(tagId); + sp<EventMatcherWizard> eventMatcherWizard = + new EventMatcherWizard({new SimpleLogMatchingTracker( + atomMatcherId, logEventMatcherIndex, atomMatcher, uidMap)}); sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); - shared_ptr<MockStatsPullerManager> pullerManager = - make_shared<StrictMock<MockStatsPullerManager>>(); - EXPECT_CALL(*pullerManager, RegisterReceiver(tagId, _, _, _)).WillOnce(Return()); - EXPECT_CALL(*pullerManager, UnRegisterReceiver(tagId, _)).WillRepeatedly(Return()); + sp<MockStatsPullerManager> pullerManager = new StrictMock<MockStatsPullerManager>(); - EXPECT_CALL(*pullerManager, Pull(tagId, _, _)) - // condition becomes true - .WillOnce(Invoke([](int tagId, int64_t timeNs, - vector<std::shared_ptr<LogEvent>>* data) { - data->clear(); - shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucketStartTimeNs + 10); - event->write(tagId); - event->write(100); - event->init(); - data->push_back(event); - return true; - })) - // condition becomes false - .WillOnce(Invoke([](int tagId, int64_t timeNs, - vector<std::shared_ptr<LogEvent>>* data) { - data->clear(); - shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucket3StartTimeNs + 20); - event->write(tagId); - event->write(120); - event->init(); - data->push_back(event); - return true; - })); + ValueMetricProducer valueProducer(kConfigKey, metric, -1, wizard, logEventMatcherIndex, + eventMatcherWizard, -1, bucketStartTimeNs, bucketStartTimeNs, + pullerManager); - ValueMetricProducer valueProducer(kConfigKey, metric, 1, wizard, tagId, bucketStartTimeNs, - bucketStartTimeNs, pullerManager); - valueProducer.setBucketSize(60 * NS_PER_SEC); - valueProducer.onConditionChanged(true, bucketStartTimeNs + 8); + shared_ptr<LogEvent> event1 = make_shared<LogEvent>(tagId, bucketStartTimeNs + 10); + event1->write(1); + event1->write(10); + event1->init(); + shared_ptr<LogEvent> event2 = make_shared<LogEvent>(tagId, bucketStartTimeNs + 20); + event2->write(1); + event2->write(20); + event2->init(); + valueProducer.onMatchedLogEvent(1 /*log matcher index*/, *event1); + // has one slice + EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); + ValueMetricProducer::Interval curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + EXPECT_EQ(10, curInterval.value.long_value); + EXPECT_EQ(true, curInterval.hasValue); + + valueProducer.onMatchedLogEvent(1 /*log matcher index*/, *event2); // has one slice EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); - ValueMetricProducer::Interval curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - // startUpdated:false tainted:0 sum:0 start:100 - EXPECT_EQ(100, curInterval.start); - EXPECT_EQ(true, curInterval.startUpdated); - EXPECT_EQ(0, curInterval.tainted); - EXPECT_EQ(0, curInterval.sum); - EXPECT_EQ(0UL, valueProducer.mPastBuckets.size()); + curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + EXPECT_EQ(20, curInterval.value.long_value); - // pull on bucket boundary come late, condition change happens before it. - // But puller is very slow in this one, so the data come after bucket finish - valueProducer.onConditionChanged(false, bucket2StartTimeNs + 1); - curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - EXPECT_EQ(false, curInterval.startUpdated); - EXPECT_EQ(1, curInterval.tainted); - EXPECT_EQ(0, curInterval.sum); - EXPECT_EQ(0UL, valueProducer.mPastBuckets.size()); + valueProducer.flushIfNeededLocked(bucket3StartTimeNs); + EXPECT_EQ(1UL, valueProducer.mPastBuckets.size()); + EXPECT_EQ(1UL, valueProducer.mPastBuckets.begin()->second.size()); + EXPECT_EQ(20, valueProducer.mPastBuckets.begin()->second.back().values[0].long_value); +} - // Alarm is delivered in time, but the pull is very slow, and pullers are called in order, - // so this one comes even later - vector<shared_ptr<LogEvent>> allData; - allData.clear(); - shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucket3StartTimeNs + 30); - event->write(1); - event->write(110); - event->init(); - allData.push_back(event); - valueProducer.onDataPulled(allData); +TEST(ValueMetricProducerTest, TestPushedAggregateAvg) { + ValueMetric metric; + metric.set_id(metricId); + metric.set_bucket(ONE_MINUTE); + metric.mutable_value_field()->set_field(tagId); + metric.mutable_value_field()->add_child()->set_field(2); + metric.set_aggregation_type(ValueMetric::AVG); + + UidMap uidMap; + SimpleAtomMatcher atomMatcher; + atomMatcher.set_atom_id(tagId); + sp<EventMatcherWizard> eventMatcherWizard = + new EventMatcherWizard({new SimpleLogMatchingTracker( + atomMatcherId, logEventMatcherIndex, atomMatcher, uidMap)}); + sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); + sp<MockStatsPullerManager> pullerManager = new StrictMock<MockStatsPullerManager>(); - curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - EXPECT_EQ(false, curInterval.startUpdated); - EXPECT_EQ(1, curInterval.tainted); - EXPECT_EQ(0, curInterval.sum); - EXPECT_EQ(0UL, valueProducer.mPastBuckets.size()); + ValueMetricProducer valueProducer(kConfigKey, metric, -1, wizard, logEventMatcherIndex, + eventMatcherWizard, -1, bucketStartTimeNs, bucketStartTimeNs, + pullerManager); + + shared_ptr<LogEvent> event1 = make_shared<LogEvent>(tagId, bucketStartTimeNs + 10); + event1->write(1); + event1->write(10); + event1->init(); + shared_ptr<LogEvent> event2 = make_shared<LogEvent>(tagId, bucketStartTimeNs + 20); + event2->write(1); + event2->write(15); + event2->init(); + valueProducer.onMatchedLogEvent(1 /*log matcher index*/, *event1); + // has one slice + EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); + ValueMetricProducer::Interval curInterval; + curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + EXPECT_EQ(10, curInterval.value.long_value); + EXPECT_EQ(true, curInterval.hasValue); + EXPECT_EQ(1, curInterval.sampleSize); + + valueProducer.onMatchedLogEvent(1 /*log matcher index*/, *event2); + + // has one slice + EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); + curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + EXPECT_EQ(25, curInterval.value.long_value); + EXPECT_EQ(2, curInterval.sampleSize); + + valueProducer.flushIfNeededLocked(bucket3StartTimeNs); + EXPECT_EQ(1UL, valueProducer.mPastBuckets.size()); + EXPECT_EQ(1UL, valueProducer.mPastBuckets.begin()->second.size()); + EXPECT_TRUE(std::abs(valueProducer.mPastBuckets.begin()->second.back().values[0].double_value - 12.5) < epsilon); +} + +TEST(ValueMetricProducerTest, TestPushedAggregateSum) { + ValueMetric metric; + metric.set_id(metricId); + metric.set_bucket(ONE_MINUTE); + metric.mutable_value_field()->set_field(tagId); + metric.mutable_value_field()->add_child()->set_field(2); + metric.set_aggregation_type(ValueMetric::SUM); + + UidMap uidMap; + SimpleAtomMatcher atomMatcher; + atomMatcher.set_atom_id(tagId); + sp<EventMatcherWizard> eventMatcherWizard = + new EventMatcherWizard({new SimpleLogMatchingTracker( + atomMatcherId, logEventMatcherIndex, atomMatcher, uidMap)}); + sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); + sp<MockStatsPullerManager> pullerManager = new StrictMock<MockStatsPullerManager>(); + + ValueMetricProducer valueProducer(kConfigKey, metric, -1, wizard, logEventMatcherIndex, + eventMatcherWizard, -1, bucketStartTimeNs, bucketStartTimeNs, + pullerManager); + + shared_ptr<LogEvent> event1 = make_shared<LogEvent>(tagId, bucketStartTimeNs + 10); + event1->write(1); + event1->write(10); + event1->init(); + shared_ptr<LogEvent> event2 = make_shared<LogEvent>(tagId, bucketStartTimeNs + 20); + event2->write(1); + event2->write(15); + event2->init(); + valueProducer.onMatchedLogEvent(1 /*log matcher index*/, *event1); + // has one slice + EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); + ValueMetricProducer::Interval curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + EXPECT_EQ(10, curInterval.value.long_value); + EXPECT_EQ(true, curInterval.hasValue); + + valueProducer.onMatchedLogEvent(1 /*log matcher index*/, *event2); + + // has one slice + EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); + curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + EXPECT_EQ(25, curInterval.value.long_value); + + valueProducer.flushIfNeededLocked(bucket3StartTimeNs); + EXPECT_EQ(1UL, valueProducer.mPastBuckets.size()); + EXPECT_EQ(1UL, valueProducer.mPastBuckets.begin()->second.size()); + EXPECT_EQ(25, valueProducer.mPastBuckets.begin()->second.back().values[0].long_value); +} + +TEST(ValueMetricProducerTest, TestSkipZeroDiffOutput) { + ValueMetric metric; + metric.set_id(metricId); + metric.set_bucket(ONE_MINUTE); + metric.mutable_value_field()->set_field(tagId); + metric.mutable_value_field()->add_child()->set_field(2); + metric.set_aggregation_type(ValueMetric::MIN); + metric.set_use_diff(true); + + UidMap uidMap; + SimpleAtomMatcher atomMatcher; + atomMatcher.set_atom_id(tagId); + sp<EventMatcherWizard> eventMatcherWizard = + new EventMatcherWizard({new SimpleLogMatchingTracker( + atomMatcherId, logEventMatcherIndex, atomMatcher, uidMap)}); + sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); + sp<MockStatsPullerManager> pullerManager = new StrictMock<MockStatsPullerManager>(); + + ValueMetricProducer valueProducer(kConfigKey, metric, -1, wizard, logEventMatcherIndex, + eventMatcherWizard, -1, bucketStartTimeNs, bucketStartTimeNs, + pullerManager); + + shared_ptr<LogEvent> event1 = make_shared<LogEvent>(tagId, bucketStartTimeNs + 10); + event1->write(1); + event1->write(10); + event1->init(); + shared_ptr<LogEvent> event2 = make_shared<LogEvent>(tagId, bucketStartTimeNs + 15); + event2->write(1); + event2->write(15); + event2->init(); + valueProducer.onMatchedLogEvent(1 /*log matcher index*/, *event1); + // has one slice + EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); + ValueMetricProducer::Interval curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + EXPECT_EQ(true, curInterval.hasBase); + EXPECT_EQ(10, curInterval.base.long_value); + EXPECT_EQ(false, curInterval.hasValue); + + valueProducer.onMatchedLogEvent(1 /*log matcher index*/, *event2); + + // has one slice + EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); + curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + EXPECT_EQ(true, curInterval.hasValue); + EXPECT_EQ(5, curInterval.value.long_value); + + // no change in data. + shared_ptr<LogEvent> event3 = make_shared<LogEvent>(tagId, bucket2StartTimeNs + 10); + event3->write(1); + event3->write(15); + event3->init(); + valueProducer.onMatchedLogEvent(1 /*log matcher index*/, *event3); + EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); + curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + EXPECT_EQ(true, curInterval.hasBase); + EXPECT_EQ(15, curInterval.base.long_value); + EXPECT_EQ(true, curInterval.hasValue); + + shared_ptr<LogEvent> event4 = make_shared<LogEvent>(tagId, bucket2StartTimeNs + 15); + event4->write(1); + event4->write(15); + event4->init(); + valueProducer.onMatchedLogEvent(1 /*log matcher index*/, *event4); + EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); + curInterval = valueProducer.mCurrentSlicedBucket.begin()->second[0]; + EXPECT_EQ(true, curInterval.hasBase); + EXPECT_EQ(15, curInterval.base.long_value); + EXPECT_EQ(true, curInterval.hasValue); + + valueProducer.flushIfNeededLocked(bucket3StartTimeNs); + EXPECT_EQ(1UL, valueProducer.mPastBuckets.size()); + EXPECT_EQ(1UL, valueProducer.mPastBuckets.begin()->second.size()); + EXPECT_EQ(5, valueProducer.mPastBuckets.begin()->second.back().values[0].long_value); } } // namespace statsd diff --git a/cmds/statsd/tests/shell/ShellSubscriber_test.cpp b/cmds/statsd/tests/shell/ShellSubscriber_test.cpp new file mode 100644 index 000000000000..dd00561854fb --- /dev/null +++ b/cmds/statsd/tests/shell/ShellSubscriber_test.cpp @@ -0,0 +1,216 @@ +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <gtest/gtest.h> + +#include <unistd.h> +#include "frameworks/base/cmds/statsd/src/atoms.pb.h" +#include "frameworks/base/cmds/statsd/src/shell/shell_config.pb.h" +#include "frameworks/base/cmds/statsd/src/shell/shell_data.pb.h" +#include "src/shell/ShellSubscriber.h" +#include "tests/metrics/metrics_test_helper.h" + +#include <stdio.h> +#include <vector> + +using namespace android::os::statsd; +using android::sp; +using std::vector; +using testing::_; +using testing::Invoke; +using testing::NaggyMock; +using testing::StrictMock; + +#ifdef __ANDROID__ + +class MyResultReceiver : public BnResultReceiver { +public: + Mutex mMutex; + Condition mCondition; + bool mHaveResult = false; + int32_t mResult = 0; + + virtual void send(int32_t resultCode) { + AutoMutex _l(mMutex); + mResult = resultCode; + mHaveResult = true; + mCondition.signal(); + } + + int32_t waitForResult() { + AutoMutex _l(mMutex); + mCondition.waitRelative(mMutex, 1000000000); + return mResult; + } +}; + +void runShellTest(ShellSubscription config, sp<MockUidMap> uidMap, + sp<MockStatsPullerManager> pullerManager, + const vector<std::shared_ptr<LogEvent>>& pushedEvents, + const ShellData& expectedData) { + // set up 2 pipes for read/write config and data + int fds_config[2]; + ASSERT_EQ(0, pipe(fds_config)); + + int fds_data[2]; + ASSERT_EQ(0, pipe(fds_data)); + + size_t bufferSize = config.ByteSize(); + + // write the config to pipe, first write size of the config + vector<uint8_t> size_buffer(sizeof(bufferSize)); + std::memcpy(size_buffer.data(), &bufferSize, sizeof(bufferSize)); + write(fds_config[1], &bufferSize, sizeof(bufferSize)); + // then write config itself + vector<uint8_t> buffer(bufferSize); + config.SerializeToArray(&buffer[0], bufferSize); + write(fds_config[1], buffer.data(), bufferSize); + close(fds_config[1]); + + sp<ShellSubscriber> shellClient = new ShellSubscriber(uidMap, pullerManager); + sp<MyResultReceiver> resultReceiver = new MyResultReceiver(); + + // mimic a binder thread that a shell subscriber runs on. it would block. + std::thread reader([&resultReceiver, &fds_config, &fds_data, &shellClient] { + shellClient->startNewSubscription(fds_config[0], fds_data[1], resultReceiver); + }); + reader.detach(); + + // let the shell subscriber to receive the config from pipe. + std::this_thread::sleep_for(100ms); + + if (pushedEvents.size() > 0) { + // send a log event that matches the config. + std::thread log_reader([&shellClient, &pushedEvents] { + for (const auto& event : pushedEvents) { + shellClient->onLogEvent(*event); + } + }); + + log_reader.detach(); + + if (log_reader.joinable()) { + log_reader.join(); + } + } + + // wait for the data to be written. + std::this_thread::sleep_for(100ms); + + int expected_data_size = expectedData.ByteSize(); + + // now read from the pipe. firstly read the atom size. + size_t dataSize = 0; + EXPECT_EQ((int)sizeof(dataSize), read(fds_data[0], &dataSize, sizeof(dataSize))); + EXPECT_EQ(expected_data_size, (int)dataSize); + + // then read that much data which is the atom in proto binary format + vector<uint8_t> dataBuffer(dataSize); + EXPECT_EQ((int)dataSize, read(fds_data[0], dataBuffer.data(), dataSize)); + + // make sure the received bytes can be parsed to an atom + ShellData receivedAtom; + EXPECT_TRUE(receivedAtom.ParseFromArray(dataBuffer.data(), dataSize) != 0); + + // serialze the expected atom to bytes. and compare. to make sure they are the same. + vector<uint8_t> atomBuffer(expected_data_size); + expectedData.SerializeToArray(&atomBuffer[0], expected_data_size); + EXPECT_EQ(atomBuffer, dataBuffer); + close(fds_data[0]); +} + +TEST(ShellSubscriberTest, testPushedSubscription) { + sp<MockUidMap> uidMap = new NaggyMock<MockUidMap>(); + + sp<MockStatsPullerManager> pullerManager = new StrictMock<MockStatsPullerManager>(); + vector<std::shared_ptr<LogEvent>> pushedList; + + std::shared_ptr<LogEvent> event1 = + std::make_shared<LogEvent>(29 /*screen_state_atom_id*/, 1000 /*timestamp*/); + event1->write(::android::view::DisplayStateEnum::DISPLAY_STATE_ON); + event1->init(); + pushedList.push_back(event1); + + // create a simple config to get screen events + ShellSubscription config; + config.add_pushed()->set_atom_id(29); + + // this is the expected screen event atom. + ShellData shellData; + shellData.add_atom()->mutable_screen_state_changed()->set_state( + ::android::view::DisplayStateEnum::DISPLAY_STATE_ON); + + runShellTest(config, uidMap, pullerManager, pushedList, shellData); +} + +namespace { + +int kUid1 = 1000; +int kUid2 = 2000; + +int kCpuTime1 = 100; +int kCpuTime2 = 200; + +ShellData getExpectedShellData() { + ShellData shellData; + auto* atom1 = shellData.add_atom()->mutable_cpu_active_time(); + atom1->set_uid(kUid1); + atom1->set_time_millis(kCpuTime1); + + auto* atom2 = shellData.add_atom()->mutable_cpu_active_time(); + atom2->set_uid(kUid2); + atom2->set_time_millis(kCpuTime2); + + return shellData; +} + +ShellSubscription getPulledConfig() { + ShellSubscription config; + auto* pull_config = config.add_pulled(); + pull_config->mutable_matcher()->set_atom_id(10016); + pull_config->set_freq_millis(2000); + return config; +} + +} // namespace + +TEST(ShellSubscriberTest, testPulledSubscription) { + sp<MockUidMap> uidMap = new NaggyMock<MockUidMap>(); + + sp<MockStatsPullerManager> pullerManager = new StrictMock<MockStatsPullerManager>(); + EXPECT_CALL(*pullerManager, Pull(10016, _, _)) + .WillRepeatedly( + Invoke([](int tagId, int64_t timeNs, vector<std::shared_ptr<LogEvent>>* data) { + data->clear(); + shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, timeNs); + event->write(kUid1); + event->write(kCpuTime1); + event->init(); + data->push_back(event); + // another event + event = make_shared<LogEvent>(tagId, timeNs); + event->write(kUid2); + event->write(kCpuTime2); + event->init(); + data->push_back(event); + return true; + })); + + runShellTest(getPulledConfig(), uidMap, pullerManager, vector<std::shared_ptr<LogEvent>>(), + getExpectedShellData()); +} + +#else +GTEST_LOG_(INFO) << "This test does nothing.\n"; +#endif diff --git a/cmds/statsd/tests/statsd_test_util.cpp b/cmds/statsd/tests/statsd_test_util.cpp index e0c98cb9735b..b8b1a1db2c12 100644 --- a/cmds/statsd/tests/statsd_test_util.cpp +++ b/cmds/statsd/tests/statsd_test_util.cpp @@ -452,14 +452,16 @@ std::unique_ptr<LogEvent> CreateIsolatedUidChangedEvent( sp<StatsLogProcessor> CreateStatsLogProcessor(const int64_t timeBaseNs, const int64_t currentTimeNs, const StatsdConfig& config, const ConfigKey& key) { sp<UidMap> uidMap = new UidMap(); + sp<StatsPullerManager> pullerManager = new StatsPullerManager(); sp<AlarmMonitor> anomalyAlarmMonitor = new AlarmMonitor(1, [](const sp<IStatsCompanionService>&, int64_t){}, [](const sp<IStatsCompanionService>&){}); sp<AlarmMonitor> periodicAlarmMonitor = new AlarmMonitor(1, [](const sp<IStatsCompanionService>&, int64_t){}, [](const sp<IStatsCompanionService>&){}); - sp<StatsLogProcessor> processor = new StatsLogProcessor( - uidMap, anomalyAlarmMonitor, periodicAlarmMonitor, timeBaseNs, [](const ConfigKey&){return true;}); + sp<StatsLogProcessor> processor = + new StatsLogProcessor(uidMap, pullerManager, anomalyAlarmMonitor, periodicAlarmMonitor, + timeBaseNs, [](const ConfigKey&) { return true; }); processor->OnConfigUpdated(currentTimeNs, key, config); return processor; } diff --git a/cmds/statsd/tools/dogfood/src/com/android/statsd/dogfood/MainActivity.java b/cmds/statsd/tools/dogfood/src/com/android/statsd/dogfood/MainActivity.java index 2b7da6ab6d79..4f4dd011e419 100644 --- a/cmds/statsd/tools/dogfood/src/com/android/statsd/dogfood/MainActivity.java +++ b/cmds/statsd/tools/dogfood/src/com/android/statsd/dogfood/MainActivity.java @@ -319,7 +319,7 @@ public class MainActivity extends Activity { int[] uids = new int[]{mUids[id]}; String[] tags = new String[]{"acquire"}; StatsLog.write(StatsLog.WAKELOCK_STATE_CHANGED, uids, tags, - StatsLog.WAKELOCK_STATE_CHANGED__LEVEL__PARTIAL_WAKE_LOCK, name, + StatsLog.WAKELOCK_STATE_CHANGED__TYPE__PARTIAL_WAKE_LOCK, name, StatsLog.WAKELOCK_STATE_CHANGED__STATE__ACQUIRE); StringBuilder sb = new StringBuilder(); sb.append("StagsLog.write(10, ").append(mUids[id]).append(", ").append(0) @@ -335,7 +335,7 @@ public class MainActivity extends Activity { int[] uids = new int[]{mUids[id]}; String[] tags = new String[]{"release"}; StatsLog.write(StatsLog.WAKELOCK_STATE_CHANGED, uids, tags, - StatsLog.WAKELOCK_STATE_CHANGED__LEVEL__PARTIAL_WAKE_LOCK, name, + StatsLog.WAKELOCK_STATE_CHANGED__TYPE__PARTIAL_WAKE_LOCK, name, StatsLog.WAKELOCK_STATE_CHANGED__STATE__RELEASE); StringBuilder sb = new StringBuilder(); sb.append("StagsLog.write(10, ").append(mUids[id]).append(", ").append(0) diff --git a/cmds/statsd/tools/statsd-testdrive/Android.bp b/cmds/statsd/tools/statsd-testdrive/Android.bp new file mode 100644 index 000000000000..f566bc7f2a53 --- /dev/null +++ b/cmds/statsd/tools/statsd-testdrive/Android.bp @@ -0,0 +1,11 @@ +java_binary_host { + name: "statsd_testdrive", + manifest: "manifest.txt", + srcs: [ + "src/**/*.java", + ], + static_libs: [ + "platformprotos", + "guava", + ], +} diff --git a/cmds/statsd/tools/statsd-testdrive/manifest.txt b/cmds/statsd/tools/statsd-testdrive/manifest.txt new file mode 100644 index 000000000000..0266d1143245 --- /dev/null +++ b/cmds/statsd/tools/statsd-testdrive/manifest.txt @@ -0,0 +1 @@ +Main-class: com.android.statsd.testdrive.TestDrive diff --git a/cmds/statsd/tools/statsd-testdrive/src/com/android/statsd/testdrive/TestDrive.java b/cmds/statsd/tools/statsd-testdrive/src/com/android/statsd/testdrive/TestDrive.java new file mode 100644 index 000000000000..cc4e386bfdf0 --- /dev/null +++ b/cmds/statsd/tools/statsd-testdrive/src/com/android/statsd/testdrive/TestDrive.java @@ -0,0 +1,302 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.statsd.testdrive; + +import com.android.internal.os.StatsdConfigProto.AtomMatcher; +import com.android.internal.os.StatsdConfigProto.SimpleAtomMatcher; +import com.android.internal.os.StatsdConfigProto.StatsdConfig; +import com.android.os.AtomsProto.Atom; +import com.android.os.StatsLog.ConfigMetricsReport; +import com.android.os.StatsLog.ConfigMetricsReportList; + +import com.google.common.io.Files; +import com.google.protobuf.TextFormat; +import com.google.protobuf.TextFormat.ParseException; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.logging.ConsoleHandler; +import java.util.logging.Formatter; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; + +public class TestDrive { + + public static final int PULL_ATOM_START = 10000; + public static final long ATOM_MATCHER_ID = 1234567; + + public static final String UPDATE_CONFIG_CMD = "cmd stats config update"; + public static final String DUMP_REPORT_CMD = "cmd stats dump-report"; + public static final String REMOVE_CONFIG_CMD = "cmd stats config remove"; + public static final String CONFIG_UID = "2000"; // shell uid + public static final long CONFIG_ID = 54321; + + private static boolean mIsPushedAtom = false; + + private static final Logger logger = Logger.getLogger(TestDrive.class.getName()); + + public static void main(String[] args) { + if (args.length != 1) { + logger.log(Level.SEVERE, "Usage: ./test_drive <atomId>"); + return; + } + int atomId; + try { + atomId = Integer.valueOf(args[0]); + } catch (NumberFormatException e) { + logger.log(Level.SEVERE, "Bad atom id provided: " + args[0]); + return; + } + if (Atom.getDescriptor().findFieldByNumber(atomId) == null) { + logger.log(Level.SEVERE, "No such atom found: " + args[0]); + return; + } + mIsPushedAtom = atomId < PULL_ATOM_START; + + TestDrive testDrive = new TestDrive(); + TestDriveFormatter formatter = new TestDriveFormatter(); + ConsoleHandler handler = new ConsoleHandler(); + handler.setFormatter(formatter); + logger.addHandler(handler); + logger.setUseParentHandlers(false); + + try { + StatsdConfig config = testDrive.createConfig(atomId); + if (config == null) { + logger.log(Level.SEVERE, "Failed to create valid config."); + return; + } + testDrive.pushConfig(config); + logger.info("Pushed the following config to statsd:"); + logger.info(config.toString()); + if (mIsPushedAtom) { + logger.info( + "Now please play with the device to trigger the event. All events should " + + "be dumped after 1 min ..."); + Thread.sleep(60_000); + } else { + // wait for 2 min + logger.info("Now wait for 2 minutes ..."); + Thread.sleep(120_000); + } + testDrive.dumpMetrics(); + } catch (Exception e) { + logger.log(Level.SEVERE, "Failed to test drive: " + e.getMessage()); + } finally { + testDrive.removeConfig(); + } + } + + private void pushConfig(StatsdConfig config) throws IOException, InterruptedException { + File configFile = File.createTempFile("statsdconfig", ".config"); + configFile.deleteOnExit(); + Files.write(config.toByteArray(), configFile); + String remotePath = "/data/local/tmp/" + configFile.getName(); + runCommand(null, "adb", "push", configFile.getAbsolutePath(), remotePath); + runCommand( + null, "adb", "shell", "cat", remotePath, "|", UPDATE_CONFIG_CMD, + String.valueOf(CONFIG_ID)); + } + + private void removeConfig() { + try { + runCommand(null, "adb", "shell", REMOVE_CONFIG_CMD, String.valueOf(CONFIG_ID)); + } catch (Exception e) { + logger.log(Level.SEVERE, "Failed to remove config: " + e.getMessage()); + } + } + + // Runs a shell command. Output should go to outputFile. Returns error string. + private String runCommand(File outputFile, String... commands) + throws IOException, InterruptedException { + // Run macro on target + ProcessBuilder pb = new ProcessBuilder(commands); + // pb.redirectErrorStream(true); + + if (outputFile != null && outputFile.exists() && outputFile.canWrite()) { + pb.redirectOutput(outputFile); + } + Process process = pb.start(); + + // capture any errors + StringBuilder out = new StringBuilder(); + // Read output + BufferedReader br = new BufferedReader(new InputStreamReader(process.getErrorStream())); + String line = null, previous = null; + while ((line = br.readLine()) != null) { + if (!line.equals(previous)) { + previous = line; + out.append(line).append('\n'); + logger.fine(line); + } + } + + // Check result + if (process.waitFor() == 0) { + logger.fine("Success!"); + } else { + // Abnormal termination: Log command parameters and output and throw ExecutionException + logger.log(Level.SEVERE, out.toString()); + } + return out.toString(); + } + + private StatsdConfig createConfig(int atomId) { + try { + if (mIsPushedAtom) { + return createSimpleEventMetricConfig(atomId); + } else { + return createSimpleGaugeMetricConfig(atomId); + } + } catch (ParseException e) { + logger.log( + Level.SEVERE, + "Failed to parse the config! line: " + + e.getLine() + + " col: " + + e.getColumn() + + " " + + e.getMessage()); + } + return null; + } + + private StatsdConfig createSimpleEventMetricConfig(int atomId) throws ParseException { + StatsdConfig.Builder baseBuilder = getSimpleEventMetricBaseConfig(); + baseBuilder.addAtomMatcher(createAtomMatcher(atomId)); + return baseBuilder.build(); + } + + private StatsdConfig createSimpleGaugeMetricConfig(int atomId) throws ParseException { + StatsdConfig.Builder baseBuilder = getSimpleGaugeMetricBaseConfig(); + baseBuilder.addAtomMatcher(createAtomMatcher(atomId)); + return baseBuilder.build(); + } + + private AtomMatcher createAtomMatcher(int atomId) { + AtomMatcher.Builder atomMatcherBuilder = AtomMatcher.newBuilder(); + atomMatcherBuilder + .setId(ATOM_MATCHER_ID) + .setSimpleAtomMatcher(SimpleAtomMatcher.newBuilder().setAtomId(atomId)); + return atomMatcherBuilder.build(); + } + + private StatsdConfig.Builder getSimpleEventMetricBaseConfig() throws ParseException { + StatsdConfig.Builder builder = StatsdConfig.newBuilder(); + TextFormat.merge(EVENT_BASE_CONFIG_SRTR, builder); + return builder; + } + + private StatsdConfig.Builder getSimpleGaugeMetricBaseConfig() throws ParseException { + StatsdConfig.Builder builder = StatsdConfig.newBuilder(); + TextFormat.merge(GAUGE_BASE_CONFIG_STR, builder); + return builder; + } + + private ConfigMetricsReportList getReportList() throws Exception { + try { + File outputFile = File.createTempFile("statsdret", ".bin"); + outputFile.deleteOnExit(); + runCommand( + outputFile, + "adb", + "shell", + DUMP_REPORT_CMD, + String.valueOf(CONFIG_ID), + "--include_current_bucket", + "--proto"); + ConfigMetricsReportList reportList = + ConfigMetricsReportList.parseFrom(new FileInputStream(outputFile)); + return reportList; + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + logger.log( + Level.SEVERE, + "Failed to fetch and parse the statsd output report. " + + "Perhaps there is not a valid statsd config for the requested " + + "uid=" + + CONFIG_UID + + ", id=" + + CONFIG_ID + + "."); + throw (e); + } + } + + private void dumpMetrics() throws Exception { + ConfigMetricsReportList reportList = getReportList(); + // We may get multiple reports. Take the last one. + ConfigMetricsReport report = reportList.getReports(reportList.getReportsCount() - 1); + // Really should be only one metric. + if (report.getMetricsCount() != 1) { + logger.log(Level.SEVERE, + "Only one report metric expected, got " + report.getMetricsCount()); + return; + } + + logger.info("Got following metric data dump:"); + logger.info(report.getMetrics(0).toString()); + } + + private static final String EVENT_BASE_CONFIG_SRTR = + "id: 12345\n" + + "event_metric {\n" + + " id: 1111\n" + + " what: 1234567\n" + + "}\n" + + "allowed_log_source: \"AID_GRAPHICS\"\n" + + "allowed_log_source: \"AID_INCIDENTD\"\n" + + "allowed_log_source: \"AID_STATSD\"\n" + + "allowed_log_source: \"AID_RADIO\"\n" + + "allowed_log_source: \"com.android.systemui\"\n" + + "allowed_log_source: \"com.android.vending\"\n" + + "allowed_log_source: \"AID_SYSTEM\"\n" + + "allowed_log_source: \"AID_ROOT\"\n" + + "allowed_log_source: \"AID_BLUETOOTH\"\n" + + "\n" + + "hash_strings_in_metric_report: false"; + + private static final String GAUGE_BASE_CONFIG_STR = + "id: 56789\n" + + "gauge_metric {\n" + + " id: 2222\n" + + " what: 1234567\n" + + " gauge_fields_filter {\n" + + " include_all: true\n" + + " }\n" + + " bucket: ONE_MINUTE\n" + + "}\n" + + "allowed_log_source: \"AID_GRAPHICS\"\n" + + "allowed_log_source: \"AID_INCIDENTD\"\n" + + "allowed_log_source: \"AID_STATSD\"\n" + + "allowed_log_source: \"AID_RADIO\"\n" + + "allowed_log_source: \"com.android.systemui\"\n" + + "allowed_log_source: \"com.android.vending\"\n" + + "allowed_log_source: \"AID_SYSTEM\"\n" + + "allowed_log_source: \"AID_ROOT\"\n" + + "allowed_log_source: \"AID_BLUETOOTH\"\n" + + "\n" + + "hash_strings_in_metric_report: false"; + + public static class TestDriveFormatter extends Formatter { + public String format(LogRecord record) { + return record.getMessage() + "\n"; + } + } +} diff --git a/cmds/uiautomator/library/testrunner-src/com/android/uiautomator/core/ShellUiAutomatorBridge.java b/cmds/uiautomator/library/testrunner-src/com/android/uiautomator/core/ShellUiAutomatorBridge.java index 653851546d01..455e4bbc0b76 100644 --- a/cmds/uiautomator/library/testrunner-src/com/android/uiautomator/core/ShellUiAutomatorBridge.java +++ b/cmds/uiautomator/library/testrunner-src/com/android/uiautomator/core/ShellUiAutomatorBridge.java @@ -62,7 +62,7 @@ public class ShellUiAutomatorBridge extends UiAutomatorBridge { IBinder token = new Binder(); try { ContentProviderHolder holder = activityManager.getContentProviderExternal( - providerName, UserHandle.USER_SYSTEM, token); + providerName, UserHandle.USER_SYSTEM, token, "*uiautomator*"); if (holder == null) { throw new IllegalStateException("Could not find provider: " + providerName); } @@ -84,7 +84,8 @@ public class ShellUiAutomatorBridge extends UiAutomatorBridge { cursor.close(); } if (provider != null) { - activityManager.removeContentProviderExternal(providerName, token); + activityManager.removeContentProviderExternalAsUser(providerName, token, + UserHandle.USER_SYSTEM); } } } catch (RemoteException e) { |