1 files changed, 57 insertions, 49 deletions

diff --git a/cmds/statsd/src/StatsLogProcessor.cpp b/cmds/statsd/src/StatsLogProcessor.cpp
index a4066aa7bca5..87dec5d1656d 100644
--- a/cmds/statsd/src/StatsLogProcessor.cpp
+++ b/cmds/statsd/src/StatsLogProcessor.cpp
@@ -21,17 +21,17 @@
 #include <android-base/file.h>
 #include <dirent.h>
 #include "StatsLogProcessor.h"
+#include "stats_log_util.h"
 #include "android-base/stringprintf.h"
 #include "guardrail/StatsdStats.h"
 #include "metrics/CountMetricProducer.h"
 #include "external/StatsPullerManager.h"
-#include "dimension.h"
-#include "field_util.h"
 #include "stats_util.h"
 #include "storage/StorageManager.h"
 
 #include <log/log_event_list.h>
 #include <utils/Errors.h>
+#include <utils/SystemClock.h>
 
 using namespace android;
 using android::base::StringPrintf;
@@ -58,8 +58,10 @@ const int FIELD_ID_REPORTS = 2;
 const int FIELD_ID_UID = 1;
 const int FIELD_ID_ID = 2;
 // for ConfigMetricsReport
-const int FIELD_ID_METRICS = 1;
+// const int FIELD_ID_METRICS = 1; // written in MetricsManager.cpp
 const int FIELD_ID_UID_MAP = 2;
+const int FIELD_ID_LAST_REPORT_ELAPSED_NANOS = 3;
+const int FIELD_ID_CURRENT_REPORT_ELAPSED_NANOS = 4;
 
 #define STATS_DATA_DIR "/data/misc/stats-data"
 
@@ -71,9 +73,6 @@ StatsLogProcessor::StatsLogProcessor(const sp<UidMap>& uidMap,
       mAnomalyMonitor(anomalyMonitor),
       mSendBroadcast(sendBroadcast),
       mTimeBaseSec(timeBaseSec) {
-    // On each initialization of StatsLogProcessor, check stats-data directory to see if there is
-    // any left over data to be read.
-    StorageManager::sendBroadcast(STATS_DATA_DIR, mSendBroadcast);
     StatsPullerManager statsPullerManager;
     statsPullerManager.SetTimeBaseSec(mTimeBaseSec);
 }
@@ -90,27 +89,31 @@ void StatsLogProcessor::onAnomalyAlarmFired(
     }
 }
 
+void updateUid(Value* value, int hostUid) {
+    int uid = value->int_value;
+    if (uid != hostUid) {
+        value->setInt(hostUid);
+    }
+}
+
 void StatsLogProcessor::mapIsolatedUidToHostUidIfNecessaryLocked(LogEvent* event) const {
-    std::set<Field, FieldCmp> uidFields;
     if (android::util::kAtomsWithAttributionChain.find(event->GetTagId()) !=
         android::util::kAtomsWithAttributionChain.end()) {
-        FieldMatcher matcher;
-        buildAttributionUidFieldMatcher(event->GetTagId(), Position::ANY, &matcher);
-        findFields(event->getFieldValueMap(), matcher, &uidFields);
-    } else if (android::util::kAtomsWithUidField.find(event->GetTagId()) !=
-               android::util::kAtomsWithUidField.end()) {
-        FieldMatcher matcher;
-        buildSimpleAtomFieldMatcher(
-            event->GetTagId(), 1 /* uid is always the 1st field. */, &matcher);
-        findFields(event->getFieldValueMap(), matcher, &uidFields);
-    }
-
-    for (const auto& uidField : uidFields) {
-        DimensionsValue* value = event->findFieldValueOrNull(uidField);
-        if (value != nullptr && value->value_case() == DimensionsValue::ValueCase::kValueInt) {
-            const int uid = mUidMap->getHostUidOrSelf(value->value_int());
-            value->set_value_int(uid);
+        for (auto& value : *(event->getMutableValues())) {
+            if (value.mField.getPosAtDepth(0) > kAttributionField) {
+                break;
+            }
+            if (isAttributionUidField(value)) {
+                const int hostUid = mUidMap->getHostUidOrSelf(value.mValue.int_value);
+                updateUid(&value.mValue, hostUid);
+            }
         }
+    } else if (android::util::kAtomsWithUidField.find(event->GetTagId()) !=
+                       android::util::kAtomsWithUidField.end() &&
+               event->getValues().size() > 0 && (event->getValues())[0].mValue.getType() == INT) {
+        Value& value = (*event->getMutableValues())[0].mValue;
+        const int hostUid = mUidMap->getHostUidOrSelf(value.int_value);
+        updateUid(&value, hostUid);
     }
 }
 
@@ -134,7 +137,7 @@ void StatsLogProcessor::onIsolatedUidChangedEventLocked(const LogEvent& event) {
 void StatsLogProcessor::OnLogEvent(LogEvent* event) {
     std::lock_guard<std::mutex> lock(mMetricsMutex);
     StatsdStats::getInstance().noteAtomLogged(
-        event->GetTagId(), event->GetTimestampNs() / NS_PER_SEC);
+        event->GetTagId(), event->GetElapsedTimestampNs() / NS_PER_SEC);
 
     // Hard-coded logic to update the isolated uid's in the uid-map.
     // The field numbers need to be currently updated by hand with atoms.proto
@@ -146,6 +149,12 @@ void StatsLogProcessor::OnLogEvent(LogEvent* event) {
         return;
     }
 
+    uint64_t curTimeSec = getElapsedRealtimeSec();
+    if (curTimeSec - mLastPullerCacheClearTimeSec > StatsdStats::kPullerCacheClearIntervalSec) {
+        mStatsPullerManager.ClearPullerCacheIfNecessary(curTimeSec);
+        mLastPullerCacheClearTimeSec = curTimeSec;
+    }
+
     if (event->GetTagId() != android::util::ISOLATED_UID_CHANGED) {
         // Map the isolated uid to host uid if necessary.
         mapIsolatedUidToHostUidIfNecessaryLocked(event);
@@ -154,13 +163,13 @@ void StatsLogProcessor::OnLogEvent(LogEvent* event) {
     // pass the event to metrics managers.
     for (auto& pair : mMetricsManagers) {
         pair.second->onLogEvent(*event);
-        flushIfNecessaryLocked(event->GetTimestampNs(), pair.first, *(pair.second));
+        flushIfNecessaryLocked(event->GetElapsedTimestampNs(), pair.first, *(pair.second));
     }
 }
 
 void StatsLogProcessor::OnConfigUpdated(const ConfigKey& key, const StatsdConfig& config) {
     std::lock_guard<std::mutex> lock(mMetricsMutex);
-    ALOGD("Updated configuration for key %s", key.ToString().c_str());
+    VLOG("Updated configuration for key %s", key.ToString().c_str());
     sp<MetricsManager> newMetricsManager = new MetricsManager(key, config, mTimeBaseSec, mUidMap);
     auto it = mMetricsManagers.find(key);
     if (it == mMetricsManagers.end() && mMetricsManagers.size() > StatsdStats::kMaxConfigCount) {
@@ -203,27 +212,14 @@ void StatsLogProcessor::dumpStates(FILE* out, bool verbose) {
     }
 }
 
-void StatsLogProcessor::onDumpReport(const ConfigKey& key, const uint64_t& dumpTimeStampNs,
-                                     ConfigMetricsReportList* report) {
-    std::lock_guard<std::mutex> lock(mMetricsMutex);
-    auto it = mMetricsManagers.find(key);
-    if (it == mMetricsManagers.end()) {
-        ALOGW("Config source %s does not exist", key.ToString().c_str());
-        return;
-    }
-    report->mutable_config_key()->set_uid(key.GetUid());
-    report->mutable_config_key()->set_id(key.GetId());
-    ConfigMetricsReport* configMetricsReport = report->add_reports();
-    it->second->onDumpReport(dumpTimeStampNs, configMetricsReport);
-    // TODO: dump uid mapping.
-}
-
-void StatsLogProcessor::onDumpReport(const ConfigKey& key, vector<uint8_t>* outData) {
+void StatsLogProcessor::onDumpReport(const ConfigKey& key, const uint64_t dumpTimeStampNs,
+                                     vector<uint8_t>* outData) {
     std::lock_guard<std::mutex> lock(mMetricsMutex);
-    onDumpReportLocked(key, outData);
+    onDumpReportLocked(key, dumpTimeStampNs, outData);
 }
 
-void StatsLogProcessor::onDumpReportLocked(const ConfigKey& key, vector<uint8_t>* outData) {
+void StatsLogProcessor::onDumpReportLocked(const ConfigKey& key, const uint64_t dumpTimeStampNs,
+                                           vector<uint8_t>* outData) {
     auto it = mMetricsManagers.find(key);
     if (it == mMetricsManagers.end()) {
         ALOGW("Config source %s does not exist", key.ToString().c_str());
@@ -247,9 +243,10 @@ void StatsLogProcessor::onDumpReportLocked(const ConfigKey& key, vector<uint8_t>
     long long reportsToken =
             proto.start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_REPORTS);
 
+    int64_t lastReportTimeNs = it->second->getLastReportTimeNs();
     // First, fill in ConfigMetricsReport using current data on memory, which
     // starts from filling in StatsLogReport's.
-    it->second->onDumpReport(&proto);
+    it->second->onDumpReport(dumpTimeStampNs, &proto);
 
     // Fill in UidMap.
     auto uidMap = mUidMap->getOutput(key);
@@ -258,6 +255,12 @@ void StatsLogProcessor::onDumpReportLocked(const ConfigKey& key, vector<uint8_t>
     uidMap.SerializeToArray(&uidMapBuffer[0], uidMapSize);
     proto.write(FIELD_TYPE_MESSAGE | FIELD_ID_UID_MAP, uidMapBuffer, uidMapSize);
 
+    // Fill in the timestamps.
+    proto.write(FIELD_TYPE_INT64 | FIELD_ID_LAST_REPORT_ELAPSED_NANOS,
+                (long long)lastReportTimeNs);
+    proto.write(FIELD_TYPE_INT64 | FIELD_ID_CURRENT_REPORT_ELAPSED_NANOS,
+                (long long)dumpTimeStampNs);
+
     // End of ConfigMetricsReport (reports).
     proto.end(reportsToken);
 
@@ -277,6 +280,7 @@ void StatsLogProcessor::onDumpReportLocked(const ConfigKey& key, vector<uint8_t>
             iter.rp()->move(toRead);
         }
     }
+
     StatsdStats::getInstance().noteMetricsReportSent(key);
 }
 
@@ -290,6 +294,10 @@ void StatsLogProcessor::OnConfigRemoved(const ConfigKey& key) {
     StatsdStats::getInstance().noteConfigRemoved(key);
 
     mLastBroadcastTimes.erase(key);
+
+    if (mMetricsManagers.empty()) {
+        mStatsPullerManager.ForceClearPullerCache();
+    }
 }
 
 void StatsLogProcessor::flushIfNecessaryLocked(
@@ -308,7 +316,7 @@ void StatsLogProcessor::flushIfNecessaryLocked(
         StatsdStats::kMaxMetricsBytesPerConfig) {  // Too late. We need to start clearing data.
         // TODO(b/70571383): By 12/15/2017 add API to drop data directly
         ProtoOutputStream proto;
-        metricsManager.onDumpReport(&proto);
+        metricsManager.onDumpReport(time(nullptr) * NS_PER_SEC, &proto);
         StatsdStats::getInstance().noteDataDropped(key);
         VLOG("StatsD had to toss out metrics for %s", key.ToString().c_str());
     } else if (totalBytes > .9 * StatsdStats::kMaxMetricsBytesPerConfig) {
@@ -332,10 +340,10 @@ void StatsLogProcessor::WriteDataToDisk() {
     for (auto& pair : mMetricsManagers) {
         const ConfigKey& key = pair.first;
         vector<uint8_t> data;
-        onDumpReportLocked(key, &data);
+        onDumpReportLocked(key, time(nullptr) * NS_PER_SEC, &data);
         // TODO: Add a guardrail to prevent accumulation of file on disk.
-        string file_name = StringPrintf("%s/%ld_%d_%lld", STATS_DATA_DIR, time(nullptr),
-                                        key.GetUid(), (long long)key.GetId());
+        string file_name = StringPrintf("%s/%ld_%d_%lld", STATS_DATA_DIR,
+             (long)getWallClockSec(), key.GetUid(), (long long)key.GetId());
         StorageManager::writeFile(file_name.c_str(), &data[0], data.size());
     }
 }