summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--cmds/statsd/src/StatsLogProcessor.cpp123
-rw-r--r--cmds/statsd/src/StatsLogProcessor.h13
-rw-r--r--cmds/statsd/src/guardrail/StatsdStats.h3
-rw-r--r--cmds/statsd/src/metrics/MetricsManager.cpp3
-rw-r--r--cmds/statsd/src/metrics/MetricsManager.h6
-rw-r--r--cmds/statsd/src/statsd_config.proto2
-rw-r--r--cmds/statsd/src/storage/StorageManager.cpp216
-rw-r--r--cmds/statsd/src/storage/StorageManager.h34
-rw-r--r--cmds/statsd/tests/storage/StorageManager_test.cpp15
9 files changed, 258 insertions, 157 deletions
diff --git a/cmds/statsd/src/StatsLogProcessor.cpp b/cmds/statsd/src/StatsLogProcessor.cpp
index df84b6a4dc34..a9f5208ef812 100644
--- a/cmds/statsd/src/StatsLogProcessor.cpp
+++ b/cmds/statsd/src/StatsLogProcessor.cpp
@@ -77,7 +77,6 @@ const int FIELD_ID_TIME_TO_LIVE_NANOS = 2;
#define NS_PER_HOUR 3600 * NS_PER_SEC
-#define STATS_DATA_DIR "/data/misc/stats-data"
#define STATS_ACTIVE_METRIC_DIR "/data/misc/stats-active-metric"
// Cool down period for writing data to disk to avoid overwriting files.
@@ -106,6 +105,19 @@ StatsLogProcessor::StatsLogProcessor(const sp<UidMap>& uidMap,
StatsLogProcessor::~StatsLogProcessor() {
}
+static void flushProtoToBuffer(ProtoOutputStream& proto, vector<uint8_t>* outData) {
+ outData->clear();
+ outData->resize(proto.size());
+ size_t pos = 0;
+ sp<android::util::ProtoReader> reader = proto.data();
+ while (reader->readBuffer() != NULL) {
+ size_t toRead = reader->currentToRead();
+ std::memcpy(&((*outData)[pos]), reader->readBuffer(), toRead);
+ pos += toRead;
+ reader->move(toRead);
+ }
+}
+
void StatsLogProcessor::onAnomalyAlarmFired(
const int64_t& timestampNs,
unordered_set<sp<const InternalAlarm>, SpHash<InternalAlarm>> alarmSet) {
@@ -366,25 +378,29 @@ void StatsLogProcessor::onDumpReport(const ConfigKey& key, const int64_t dumpTim
proto->end(configKeyToken);
// End of ConfigKey.
+ bool keepFile = false;
+ auto it = mMetricsManagers.find(key);
+ if (it != mMetricsManagers.end() && it->second->shouldPersistLocalHistory()) {
+ keepFile = true;
+ }
+
// Then, check stats-data directory to see there's any file containing
// ConfigMetricsReport from previous shutdowns to concatenate to reports.
- StorageManager::appendConfigMetricsReport(key, proto, erase_data);
+ StorageManager::appendConfigMetricsReport(
+ key, proto, erase_data && !keepFile /* should remove file after appending it */,
+ dumpReportReason == ADB_DUMP /*if caller is adb*/);
- auto it = mMetricsManagers.find(key);
if (it != mMetricsManagers.end()) {
// This allows another broadcast to be sent within the rate-limit period if we get close to
// filling the buffer again soon.
mLastBroadcastTimes.erase(key);
- // Start of ConfigMetricsReport (reports).
- uint64_t reportsToken =
- proto->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_REPORTS);
- onConfigMetricsReportLocked(key, dumpTimeStampNs,
- include_current_partial_bucket,
- erase_data, dumpReportReason,
- dumpLatency, proto);
- proto->end(reportsToken);
- // End of ConfigMetricsReport (reports).
+ vector<uint8_t> buffer;
+ onConfigMetricsReportLocked(key, dumpTimeStampNs, include_current_partial_bucket,
+ erase_data, dumpReportReason, dumpLatency,
+ false /* is this data going to be saved on disk */, &buffer);
+ proto->write(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_REPORTS,
+ reinterpret_cast<char*>(buffer.data()), buffer.size());
} else {
ALOGW("Config source %s does not exist", key.ToString().c_str());
}
@@ -404,16 +420,8 @@ void StatsLogProcessor::onDumpReport(const ConfigKey& key, const int64_t dumpTim
dumpReportReason, dumpLatency, &proto);
if (outData != nullptr) {
- outData->clear();
- outData->resize(proto.size());
- size_t pos = 0;
- sp<android::util::ProtoReader> reader = proto.data();
- while (reader->readBuffer() != NULL) {
- size_t toRead = reader->currentToRead();
- std::memcpy(&((*outData)[pos]), reader->readBuffer(), toRead);
- pos += toRead;
- reader->move(toRead);
- }
+ flushProtoToBuffer(proto, outData);
+ VLOG("output data size %zu", outData->size());
}
StatsdStats::getInstance().noteMetricsReportSent(key, proto.size());
@@ -422,13 +430,11 @@ void StatsLogProcessor::onDumpReport(const ConfigKey& key, const int64_t dumpTim
/*
* onConfigMetricsReportLocked dumps serialized ConfigMetricsReport into outData.
*/
-void StatsLogProcessor::onConfigMetricsReportLocked(const ConfigKey& key,
- const int64_t dumpTimeStampNs,
- const bool include_current_partial_bucket,
- const bool erase_data,
- const DumpReportReason dumpReportReason,
- const DumpLatency dumpLatency,
- ProtoOutputStream* proto) {
+void StatsLogProcessor::onConfigMetricsReportLocked(
+ const ConfigKey& key, const int64_t dumpTimeStampNs,
+ const bool include_current_partial_bucket, const bool erase_data,
+ const DumpReportReason dumpReportReason, const DumpLatency dumpLatency,
+ const bool dataSavedOnDisk, vector<uint8_t>* buffer) {
// We already checked whether key exists in mMetricsManagers in
// WriteDataToDisk.
auto it = mMetricsManagers.find(key);
@@ -440,35 +446,46 @@ void StatsLogProcessor::onConfigMetricsReportLocked(const ConfigKey& key,
std::set<string> str_set;
+ ProtoOutputStream tempProto;
// First, fill in ConfigMetricsReport using current data on memory, which
// starts from filling in StatsLogReport's.
- it->second->onDumpReport(dumpTimeStampNs, include_current_partial_bucket,
- erase_data, dumpLatency, &str_set, proto);
+ it->second->onDumpReport(dumpTimeStampNs, include_current_partial_bucket, erase_data,
+ dumpLatency, &str_set, &tempProto);
// Fill in UidMap if there is at least one metric to report.
// This skips the uid map if it's an empty config.
if (it->second->getNumMetrics() > 0) {
- uint64_t uidMapToken = proto->start(FIELD_TYPE_MESSAGE | FIELD_ID_UID_MAP);
+ uint64_t uidMapToken = tempProto.start(FIELD_TYPE_MESSAGE | FIELD_ID_UID_MAP);
mUidMap->appendUidMap(
dumpTimeStampNs, key, it->second->hashStringInReport() ? &str_set : nullptr,
- it->second->versionStringsInReport(), it->second->installerInReport(), proto);
- proto->end(uidMapToken);
+ it->second->versionStringsInReport(), it->second->installerInReport(), &tempProto);
+ tempProto.end(uidMapToken);
}
// Fill in the timestamps.
- proto->write(FIELD_TYPE_INT64 | FIELD_ID_LAST_REPORT_ELAPSED_NANOS,
- (long long)lastReportTimeNs);
- proto->write(FIELD_TYPE_INT64 | FIELD_ID_CURRENT_REPORT_ELAPSED_NANOS,
- (long long)dumpTimeStampNs);
- proto->write(FIELD_TYPE_INT64 | FIELD_ID_LAST_REPORT_WALL_CLOCK_NANOS,
- (long long)lastReportWallClockNs);
- proto->write(FIELD_TYPE_INT64 | FIELD_ID_CURRENT_REPORT_WALL_CLOCK_NANOS,
- (long long)getWallClockNs());
+ tempProto.write(FIELD_TYPE_INT64 | FIELD_ID_LAST_REPORT_ELAPSED_NANOS,
+ (long long)lastReportTimeNs);
+ tempProto.write(FIELD_TYPE_INT64 | FIELD_ID_CURRENT_REPORT_ELAPSED_NANOS,
+ (long long)dumpTimeStampNs);
+ tempProto.write(FIELD_TYPE_INT64 | FIELD_ID_LAST_REPORT_WALL_CLOCK_NANOS,
+ (long long)lastReportWallClockNs);
+ tempProto.write(FIELD_TYPE_INT64 | FIELD_ID_CURRENT_REPORT_WALL_CLOCK_NANOS,
+ (long long)getWallClockNs());
// Dump report reason
- proto->write(FIELD_TYPE_INT32 | FIELD_ID_DUMP_REPORT_REASON, dumpReportReason);
+ tempProto.write(FIELD_TYPE_INT32 | FIELD_ID_DUMP_REPORT_REASON, dumpReportReason);
for (const auto& str : str_set) {
- proto->write(FIELD_TYPE_STRING | FIELD_COUNT_REPEATED | FIELD_ID_STRINGS, str);
+ tempProto.write(FIELD_TYPE_STRING | FIELD_COUNT_REPEATED | FIELD_ID_STRINGS, str);
+ }
+
+ flushProtoToBuffer(tempProto, buffer);
+
+ // save buffer to disk if needed
+ if (erase_data && !dataSavedOnDisk && it->second->shouldPersistLocalHistory()) {
+ VLOG("save history to disk");
+ string file_name = StorageManager::getDataHistoryFileName((long)getWallClockSec(),
+ key.GetUid(), key.GetId());
+ StorageManager::writeFile(file_name.c_str(), buffer->data(), buffer->size());
}
}
@@ -584,18 +601,14 @@ void StatsLogProcessor::WriteDataToDiskLocked(const ConfigKey& key,
!mMetricsManagers.find(key)->second->shouldWriteToDisk()) {
return;
}
- ProtoOutputStream proto;
+ vector<uint8_t> buffer;
onConfigMetricsReportLocked(key, timestampNs, true /* include_current_partial_bucket*/,
- true /* erase_data */, dumpReportReason, dumpLatency, &proto);
- string file_name = StringPrintf("%s/%ld_%d_%lld", STATS_DATA_DIR,
- (long)getWallClockSec(), key.GetUid(), (long long)key.GetId());
- android::base::unique_fd fd(open(file_name.c_str(),
- O_WRONLY | O_CREAT | O_CLOEXEC, S_IRUSR | S_IWUSR));
- if (fd == -1) {
- ALOGE("Attempt to write %s but failed", file_name.c_str());
- return;
- }
- proto.flush(fd.get());
+ true /* erase_data */, dumpReportReason, dumpLatency, true,
+ &buffer);
+ string file_name =
+ StorageManager::getDataFileName((long)getWallClockSec(), key.GetUid(), key.GetId());
+ StorageManager::writeFile(file_name.c_str(), buffer.data(), buffer.size());
+
// We were able to write the ConfigMetricsReport to disk, so we should trigger collection ASAP.
mOnDiskDataConfigs.insert(key);
}
diff --git a/cmds/statsd/src/StatsLogProcessor.h b/cmds/statsd/src/StatsLogProcessor.h
index 305a4ce24b49..f4db0af29cef 100644
--- a/cmds/statsd/src/StatsLogProcessor.h
+++ b/cmds/statsd/src/StatsLogProcessor.h
@@ -164,12 +164,13 @@ private:
const DumpReportReason dumpReportReason,
const DumpLatency dumpLatency);
- void onConfigMetricsReportLocked(const ConfigKey& key, const int64_t dumpTimeStampNs,
- const bool include_current_partial_bucket,
- const bool erase_data,
- const DumpReportReason dumpReportReason,
- const DumpLatency dumpLatency,
- util::ProtoOutputStream* proto);
+ void onConfigMetricsReportLocked(
+ const ConfigKey& key, const int64_t dumpTimeStampNs,
+ const bool include_current_partial_bucket, const bool erase_data,
+ const DumpReportReason dumpReportReason, const DumpLatency dumpLatency,
+ /*if dataSavedToDisk is true, it indicates the caller will write the data to disk
+ (e.g., before reboot). So no need to further persist local history.*/
+ const bool dataSavedToDisk, vector<uint8_t>* proto);
/* Check if we should send a broadcast if approaching memory limits and if we're over, we
* actually delete the data. */
diff --git a/cmds/statsd/src/guardrail/StatsdStats.h b/cmds/statsd/src/guardrail/StatsdStats.h
index 53f12acd3b84..4d21a29c1d82 100644
--- a/cmds/statsd/src/guardrail/StatsdStats.h
+++ b/cmds/statsd/src/guardrail/StatsdStats.h
@@ -145,6 +145,9 @@ public:
// Maximum age (30 days) that files on disk can exist in seconds.
static const int kMaxAgeSecond = 60 * 60 * 24 * 30;
+ // Maximum age (2 days) that local history files on disk can exist in seconds.
+ static const int kMaxLocalHistoryAgeSecond = 60 * 60 * 24 * 2;
+
// Maximum number of files (1000) that can be in stats directory on disk.
static const int kMaxFileNumber = 1000;
diff --git a/cmds/statsd/src/metrics/MetricsManager.cpp b/cmds/statsd/src/metrics/MetricsManager.cpp
index 095f9dde6129..6a55289bc8a2 100644
--- a/cmds/statsd/src/metrics/MetricsManager.cpp
+++ b/cmds/statsd/src/metrics/MetricsManager.cpp
@@ -65,7 +65,8 @@ MetricsManager::MetricsManager(const ConfigKey& key, const StatsdConfig& config,
mTtlNs(config.has_ttl_in_seconds() ? config.ttl_in_seconds() * NS_PER_SEC : -1),
mTtlEndNs(-1),
mLastReportTimeNs(currentTimeNs),
- mLastReportWallClockNs(getWallClockNs()) {
+ mLastReportWallClockNs(getWallClockNs()),
+ mShouldPersistHistory(config.persist_locally()) {
// Init the ttl end timestamp.
refreshTtl(timeBaseNs);
diff --git a/cmds/statsd/src/metrics/MetricsManager.h b/cmds/statsd/src/metrics/MetricsManager.h
index d317f8e00d82..00ae3b7028d9 100644
--- a/cmds/statsd/src/metrics/MetricsManager.h
+++ b/cmds/statsd/src/metrics/MetricsManager.h
@@ -78,6 +78,10 @@ public:
return mNoReportMetricIds.size() != mAllMetricProducers.size();
}
+ bool shouldPersistLocalHistory() const {
+ return mShouldPersistHistory;
+ }
+
void dumpStates(FILE* out, bool verbose);
inline bool isInTtl(const int64_t timestampNs) const {
@@ -184,6 +188,8 @@ private:
// Contains the annotations passed in with StatsdConfig.
std::list<std::pair<const int64_t, const int32_t>> mAnnotations;
+ const bool mShouldPersistHistory;
+
// To guard access to mAllowedLogSources
mutable std::mutex mAllowedLogSourcesMutex;
diff --git a/cmds/statsd/src/statsd_config.proto b/cmds/statsd/src/statsd_config.proto
index 257e65ee423d..2260b9b56d0b 100644
--- a/cmds/statsd/src/statsd_config.proto
+++ b/cmds/statsd/src/statsd_config.proto
@@ -439,6 +439,8 @@ message StatsdConfig {
optional bool installer_in_metric_report = 19;
+ optional bool persist_locally = 20 [default = false];
+
// Field number 1000 is reserved for later use.
reserved 1000;
}
diff --git a/cmds/statsd/src/storage/StorageManager.cpp b/cmds/statsd/src/storage/StorageManager.cpp
index cf8b97494a06..0a9161d51cfe 100644
--- a/cmds/statsd/src/storage/StorageManager.cpp
+++ b/cmds/statsd/src/storage/StorageManager.cpp
@@ -56,9 +56,31 @@ std::mutex StorageManager::sTrainInfoMutex;
using android::base::StringPrintf;
using std::unique_ptr;
-// Returns array of int64_t which contains timestamp in seconds, uid, and
-// configID.
-static void parseFileName(char* name, int64_t* result) {
+struct FileName {
+ int64_t mTimestampSec;
+ int mUid;
+ int64_t mConfigId;
+ bool mIsHistory;
+ string getFullFileName(const char* path) {
+ return StringPrintf("%s/%lld_%d_%lld%s", path, (long long)mTimestampSec, (int)mUid,
+ (long long)mConfigId, (mIsHistory ? "_history" : ""));
+ };
+};
+
+string StorageManager::getDataFileName(long wallClockSec, int uid, int64_t id) {
+ return StringPrintf("%s/%ld_%d_%lld", STATS_DATA_DIR, wallClockSec, uid,
+ (long long)id);
+}
+
+string StorageManager::getDataHistoryFileName(long wallClockSec, int uid, int64_t id) {
+ return StringPrintf("%s/%ld_%d_%lld_history", STATS_DATA_DIR, wallClockSec, uid,
+ (long long)id);
+}
+
+// Returns array of int64_t which contains timestamp in seconds, uid,
+// configID and whether the file is a local history file.
+static void parseFileName(char* name, FileName* output) {
+ int64_t result[3];
int index = 0;
char* substr = strtok(name, "_");
while (substr != nullptr && index < 3) {
@@ -72,11 +94,12 @@ static void parseFileName(char* name, int64_t* result) {
if (index < 3) {
result[0] = -1;
}
-}
-static string getFilePath(const char* path, int64_t timestamp, int64_t uid, int64_t configID) {
- return StringPrintf("%s/%lld_%d_%lld", path, (long long)timestamp, (int)uid,
- (long long)configID);
+ output->mTimestampSec = result[0];
+ output->mUid = result[1];
+ output->mConfigId = result[2];
+ // check if the file is a local history.
+ output->mIsHistory = (substr != nullptr && strcmp("history", substr) == 0);
}
void StorageManager::writeFile(const char* file, const void* buffer, int numBytes) {
@@ -88,14 +111,13 @@ void StorageManager::writeFile(const char* file, const void* buffer, int numByte
trimToFit(STATS_SERVICE_DIR);
trimToFit(STATS_DATA_DIR);
- int result = write(fd, buffer, numBytes);
- if (result == numBytes) {
+ if (android::base::WriteFully(fd, buffer, numBytes)) {
VLOG("Successfully wrote %s", file);
} else {
- VLOG("Failed to write %s", file);
+ ALOGE("Failed to write %s", file);
}
- result = fchown(fd, AID_STATSD, AID_STATSD);
+ int result = fchown(fd, AID_STATSD, AID_STATSD);
if (result) {
VLOG("Failed to chown %s to statsd", file);
}
@@ -349,13 +371,10 @@ void StorageManager::sendBroadcast(const char* path,
if (name[0] == '.') continue;
VLOG("file %s", name);
- int64_t result[3];
- parseFileName(name, result);
- if (result[0] == -1) continue;
- int64_t uid = result[1];
- int64_t configID = result[2];
-
- sendBroadcast(ConfigKey((int)uid, configID));
+ FileName output;
+ parseFileName(name, &output);
+ if (output.mTimestampSec == -1 || output.mIsHistory) continue;
+ sendBroadcast(ConfigKey((int)output.mUid, output.mConfigId));
}
}
@@ -378,55 +397,58 @@ bool StorageManager::hasConfigMetricsReport(const ConfigKey& key) {
if (suffixLen <= nameLen &&
strncmp(name + nameLen - suffixLen, suffix.c_str(), suffixLen) == 0) {
// Check again that the file name is parseable.
- int64_t result[3];
- parseFileName(name, result);
- if (result[0] == -1) continue;
+ FileName output;
+ parseFileName(name, &output);
+ if (output.mTimestampSec == -1 || output.mIsHistory) continue;
return true;
}
}
return false;
}
-void StorageManager::appendConfigMetricsReport(const ConfigKey& key,
- ProtoOutputStream* proto,
- bool erasa_data) {
+void StorageManager::appendConfigMetricsReport(const ConfigKey& key, ProtoOutputStream* proto,
+ bool erase_data, bool isAdb) {
unique_ptr<DIR, decltype(&closedir)> dir(opendir(STATS_DATA_DIR), closedir);
if (dir == NULL) {
VLOG("Path %s does not exist", STATS_DATA_DIR);
return;
}
- string suffix = StringPrintf("%d_%lld", key.GetUid(), (long long)key.GetId());
-
dirent* de;
while ((de = readdir(dir.get()))) {
char* name = de->d_name;
+ string fileName(name);
if (name[0] == '.') continue;
+ FileName output;
+ parseFileName(name, &output);
- size_t nameLen = strlen(name);
- size_t suffixLen = suffix.length();
- if (suffixLen <= nameLen &&
- strncmp(name + nameLen - suffixLen, suffix.c_str(), suffixLen) == 0) {
- int64_t result[3];
- parseFileName(name, result);
- if (result[0] == -1) continue;
- int64_t timestamp = result[0];
- int64_t uid = result[1];
- int64_t configID = result[2];
-
- string file_name = getFilePath(STATS_DATA_DIR, timestamp, uid, configID);
- int fd = open(file_name.c_str(), O_RDONLY | O_CLOEXEC);
- if (fd != -1) {
- string content;
- if (android::base::ReadFdToString(fd, &content)) {
- proto->write(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_REPORTS,
- content.c_str(), content.size());
- }
- close(fd);
+ if (output.mTimestampSec == -1 || (output.mIsHistory && !isAdb) ||
+ output.mUid != key.GetUid() || output.mConfigId != key.GetId()) {
+ continue;
+ }
+
+ auto fullPathName = StringPrintf("%s/%s", STATS_DATA_DIR, fileName.c_str());
+ int fd = open(fullPathName.c_str(), O_RDONLY | O_CLOEXEC);
+ if (fd != -1) {
+ string content;
+ if (android::base::ReadFdToString(fd, &content)) {
+ proto->write(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_REPORTS,
+ content.c_str(), content.size());
}
+ close(fd);
+ } else {
+ ALOGE("file cannot be opened");
+ }
- if (erasa_data) {
- remove(file_name.c_str());
+ if (erase_data) {
+ remove(fullPathName.c_str());
+ } else if (output.mIsHistory && !isAdb) {
+ // This means a real data owner has called to get this data. But the config says it
+ // wants to keep a local history. So now this file must be renamed as a history file.
+ // So that next time, when owner calls getData() again, this data won't be uploaded
+ // again. rename returns 0 on success
+ if (rename(fullPathName.c_str(), (fullPathName + "_history").c_str())) {
+ ALOGE("Failed to rename file %s", fullPathName.c_str());
}
}
}
@@ -458,23 +480,20 @@ void StorageManager::readConfigFromDisk(map<ConfigKey, StatsdConfig>& configsMap
while ((de = readdir(dir.get()))) {
char* name = de->d_name;
if (name[0] == '.') continue;
- VLOG("file %s", name);
- int64_t result[3];
- parseFileName(name, result);
- if (result[0] == -1) continue;
- int64_t timestamp = result[0];
- int64_t uid = result[1];
- int64_t configID = result[2];
- string file_name = getFilePath(STATS_SERVICE_DIR, timestamp, uid, configID);
+ FileName output;
+ parseFileName(name, &output);
+ if (output.mTimestampSec == -1) continue;
+ string file_name = output.getFullFileName(STATS_SERVICE_DIR);
int fd = open(file_name.c_str(), O_RDONLY | O_CLOEXEC);
if (fd != -1) {
string content;
if (android::base::ReadFdToString(fd, &content)) {
StatsdConfig config;
if (config.ParseFromString(content)) {
- configsMap[ConfigKey(uid, configID)] = config;
- VLOG("map key uid=%lld|configID=%lld", (long long)uid, (long long)configID);
+ configsMap[ConfigKey(output.mUid, output.mConfigId)] = config;
+ VLOG("map key uid=%lld|configID=%lld", (long long)output.mUid,
+ (long long)output.mConfigId);
}
}
close(fd);
@@ -533,6 +552,30 @@ bool StorageManager::hasIdenticalConfig(const ConfigKey& key,
return false;
}
+void StorageManager::sortFiles(vector<FileInfo>* fileNames) {
+ // Reverse sort to effectively remove from the back (oldest entries).
+ // This will sort files in reverse-chronological order. Local history files have lower
+ // priority than regular data files.
+ sort(fileNames->begin(), fileNames->end(), [](FileInfo& lhs, FileInfo& rhs) {
+ // first consider if the file is a local history
+ if (lhs.mIsHistory && !rhs.mIsHistory) {
+ return false;
+ } else if (rhs.mIsHistory && !lhs.mIsHistory) {
+ return true;
+ }
+
+ // then consider the age.
+ if (lhs.mFileAgeSec < rhs.mFileAgeSec) {
+ return true;
+ } else if (lhs.mFileAgeSec > rhs.mFileAgeSec) {
+ return false;
+ }
+
+ // then good luck.... use string::compare
+ return lhs.mFileName.compare(rhs.mFileName) > 0;
+ });
+}
+
void StorageManager::trimToFit(const char* path) {
unique_ptr<DIR, decltype(&closedir)> dir(opendir(path), closedir);
if (dir == NULL) {
@@ -541,55 +584,46 @@ void StorageManager::trimToFit(const char* path) {
}
dirent* de;
int totalFileSize = 0;
- vector<string> fileNames;
+ vector<FileInfo> fileNames;
+ auto nowSec = getWallClockSec();
while ((de = readdir(dir.get()))) {
char* name = de->d_name;
if (name[0] == '.') continue;
- int64_t result[3];
- parseFileName(name, result);
- if (result[0] == -1) continue;
- int64_t timestamp = result[0];
- int64_t uid = result[1];
- int64_t configID = result[2];
- string file_name = getFilePath(path, timestamp, uid, configID);
+ FileName output;
+ parseFileName(name, &output);
+ if (output.mTimestampSec == -1) continue;
+ string file_name = output.getFullFileName(path);
// Check for timestamp and delete if it's too old.
- long fileAge = getWallClockSec() - timestamp;
- if (fileAge > StatsdStats::kMaxAgeSecond) {
+ long fileAge = nowSec - output.mTimestampSec;
+ if (fileAge > StatsdStats::kMaxAgeSecond ||
+ (output.mIsHistory && fileAge > StatsdStats::kMaxLocalHistoryAgeSecond)) {
deleteFile(file_name.c_str());
+ continue;
}
- fileNames.push_back(file_name);
ifstream file(file_name.c_str(), ifstream::in | ifstream::binary);
+ int fileSize = 0;
if (file.is_open()) {
file.seekg(0, ios::end);
- int fileSize = file.tellg();
+ fileSize = file.tellg();
file.close();
totalFileSize += fileSize;
}
+ fileNames.emplace_back(file_name, output.mIsHistory, fileSize, fileAge);
}
if (fileNames.size() > StatsdStats::kMaxFileNumber ||
totalFileSize > StatsdStats::kMaxFileSize) {
- // Reverse sort to effectively remove from the back (oldest entries).
- // This will sort files in reverse-chronological order.
- sort(fileNames.begin(), fileNames.end(), std::greater<std::string>());
+ sortFiles(&fileNames);
}
// Start removing files from oldest to be under the limit.
while (fileNames.size() > 0 && (fileNames.size() > StatsdStats::kMaxFileNumber ||
totalFileSize > StatsdStats::kMaxFileSize)) {
- string file_name = fileNames.at(fileNames.size() - 1);
- ifstream file(file_name.c_str(), ifstream::in | ifstream::binary);
- if (file.is_open()) {
- file.seekg(0, ios::end);
- int fileSize = file.tellg();
- file.close();
- totalFileSize -= fileSize;
- }
-
- deleteFile(file_name.c_str());
+ totalFileSize -= fileNames.at(fileNames.size() - 1).mFileSizeBytes;
+ deleteFile(fileNames.at(fileNames.size() - 1).mFileName.c_str());
fileNames.pop_back();
}
}
@@ -614,15 +648,13 @@ void StorageManager::printDirStats(int outFd, const char* path) {
if (name[0] == '.') {
continue;
}
- int64_t result[3];
- parseFileName(name, result);
- if (result[0] == -1) continue;
- int64_t timestamp = result[0];
- int64_t uid = result[1];
- int64_t configID = result[2];
- dprintf(outFd, "\t #%d, Last updated: %lld, UID: %d, Config ID: %lld", fileCount + 1,
- (long long)timestamp, (int)uid, (long long)configID);
- string file_name = getFilePath(path, timestamp, uid, configID);
+ FileName output;
+ parseFileName(name, &output);
+ if (output.mTimestampSec == -1) continue;
+ dprintf(outFd, "\t #%d, Last updated: %lld, UID: %d, Config ID: %lld, %s", fileCount + 1,
+ (long long)output.mTimestampSec, output.mUid, (long long)output.mConfigId,
+ (output.mIsHistory ? "local history" : ""));
+ string file_name = output.getFullFileName(path);
ifstream file(file_name.c_str(), ifstream::in | ifstream::binary);
if (file.is_open()) {
file.seekg(0, ios::end);
diff --git a/cmds/statsd/src/storage/StorageManager.h b/cmds/statsd/src/storage/StorageManager.h
index dfcea65b0872..69b41c2cb974 100644
--- a/cmds/statsd/src/storage/StorageManager.h
+++ b/cmds/statsd/src/storage/StorageManager.h
@@ -31,6 +31,19 @@ using android::util::ProtoOutputStream;
class StorageManager : public virtual RefBase {
public:
+ struct FileInfo {
+ FileInfo(std::string name, bool isHistory, int fileSize, long fileAge)
+ : mFileName(name),
+ mIsHistory(isHistory),
+ mFileSizeBytes(fileSize),
+ mFileAgeSec(fileAge) {
+ }
+ std::string mFileName;
+ bool mIsHistory;
+ int mFileSizeBytes;
+ long mFileAgeSec;
+ };
+
/**
* Writes a given byte array as a file to the specified file path.
*/
@@ -81,10 +94,19 @@ public:
/**
* Appends the ConfigMetricsReport found on disk to the specifid proto
* and, if erase_data, deletes it from disk.
+ *
+ * [isAdb]: if the caller is adb dump. This includes local adb dump or dumpsys by
+ * bugreport or incidentd. When true, we will append any local history data too.
+ *
+ * When
+ * erase_data=true, isAdb=true: append history data to output, remove all data after read
+ * erase_data=false, isAdb=true: append history data to output, keep data after read
+ * erase_data=true, isAdb=false: do not append history data, and remove data after read
+ * erase_data=false, isAdb=false: do not append history data and *rename* all data files to
+ * history files.
*/
- static void appendConfigMetricsReport(const ConfigKey& key,
- ProtoOutputStream* proto,
- bool erase_data);
+ static void appendConfigMetricsReport(const ConfigKey& key, ProtoOutputStream* proto,
+ bool erase_data, bool isAdb);
/**
* Call to load the saved configs from disk.
@@ -115,6 +137,12 @@ public:
*/
static void printStats(int out);
+ static string getDataFileName(long wallClockSec, int uid, int64_t id);
+
+ static string getDataHistoryFileName(long wallClockSec, int uid, int64_t id);
+
+ static void sortFiles(vector<FileInfo>* fileNames);
+
private:
/**
* Prints disk usage statistics about a directory related to statsd.
diff --git a/cmds/statsd/tests/storage/StorageManager_test.cpp b/cmds/statsd/tests/storage/StorageManager_test.cpp
index 4564a5d058a3..cae2f3069855 100644
--- a/cmds/statsd/tests/storage/StorageManager_test.cpp
+++ b/cmds/statsd/tests/storage/StorageManager_test.cpp
@@ -110,6 +110,21 @@ TEST(StorageManagerTest, TrainInfoReadWriteTrainNameSizeOneTest) {
EXPECT_EQ(trainInfo.experimentIds, trainInfoResult.experimentIds);
}
+TEST(StorageManagerTest, SortFileTest) {
+ vector<StorageManager::FileInfo> list;
+ // assume now sec is 500
+ list.emplace_back("200_5000_123454", false, 20, 300);
+ list.emplace_back("300_2000_123454_history", true, 30, 200);
+ list.emplace_back("400_100009_123454_history", true, 40, 100);
+ list.emplace_back("100_2000_123454", false, 50, 400);
+
+ StorageManager::sortFiles(&list);
+ EXPECT_EQ("200_5000_123454", list[0].mFileName);
+ EXPECT_EQ("100_2000_123454", list[1].mFileName);
+ EXPECT_EQ("400_100009_123454_history", list[2].mFileName);
+ EXPECT_EQ("300_2000_123454_history", list[3].mFileName);
+}
+
} // namespace statsd
} // namespace os
} // namespace android
generated by cgit v1.2.3 (git 2.25.1) at 2025-10-03 07:01:35 +0000