Implement a new section to attach LAST_KMSG to incident report

This section simply gzip a large file and stores result in GZippedFileProto

This greatly improves the size, before gzip, the last kmsg size ~500KB,
after gzip the proto size is ~60KB.

Bug: 73354384
Test: atest incidentd_test and manual on device test
Change-Id: I9bfc2cf07384487671edbffb5f0bd8495608fea6

15 files changed, 376 insertions, 132 deletions

diff --git a/cmds/incidentd/Android.mk b/cmds/incidentd/Android.mk
index 6bdd9becff37..3a47fe1946c2 100644
--- a/cmds/incidentd/Android.mk
+++ b/cmds/incidentd/Android.mk
@@ -15,7 +15,8 @@
 LOCAL_PATH:= $(call my-dir)
 
 # proto files used in incidentd to generate cppstream proto headers.
-PROTO_FILES:= frameworks/base/core/proto/android/util/log.proto
+PROTO_FILES:= frameworks/base/core/proto/android/util/log.proto \
+        frameworks/base/core/proto/android/os/data.proto
 
 # ========= #
 # incidentd #
@@ -131,7 +132,7 @@ LOCAL_TEST_DATA := $(call find-test-data-in-subdirs, $(LOCAL_PATH), *, testdata)
 LOCAL_MODULE_CLASS := NATIVE_TESTS
 gen_src_dir := $(local-generated-sources-dir)
 # generate cppstream proto for testing
-GEN_PROTO := $(gen_src_dir)/log.proto.timestamp
+GEN_PROTO := $(gen_src_dir)/test.proto.timestamp
 $(GEN_PROTO): $(HOST_OUT_EXECUTABLES)/aprotoc $(HOST_OUT_EXECUTABLES)/protoc-gen-cppstream $(PROTO_FILES)
 $(GEN_PROTO): PRIVATE_GEN_SRC_DIR := $(gen_src_dir)
 $(GEN_PROTO): PRIVATE_CUSTOM_TOOL = \
diff --git a/cmds/incidentd/src/FdBuffer.cpp b/cmds/incidentd/src/FdBuffer.cpp
index db60794a7225..64da6773686a 100644
--- a/cmds/incidentd/src/FdBuffer.cpp
+++ b/cmds/incidentd/src/FdBuffer.cpp
@@ -76,6 +76,7 @@ status_t FdBuffer::read(int fd, int64_t timeout) {
                         return -errno;
                     }
                 } else if (amt == 0) {
+                    VLOG("Reached EOF of fd=%d", fd);
                     break;
                 }
                 mBuffer.wp()->move(amt);
@@ -156,10 +157,10 @@ status_t FdBuffer::readProcessedDataInStream(int fd, int toFd, int fromFd, int64
                 if (!(errno == EAGAIN || errno == EWOULDBLOCK)) {
                     VLOG("Fail to read fd %d: %s", fd, strerror(errno));
                     return -errno;
-                }                   // otherwise just continue
-            } else if (amt == 0) {  // reach EOF so don't have to poll pfds[0].
-                ::close(pfds[0].fd);
-                pfds[0].fd = -1;
+                }  // otherwise just continue
+            } else if (amt == 0) {
+                VLOG("Reached EOF of input file %d", fd);
+                pfds[0].fd = -1;  // reach EOF so don't have to poll pfds[0].
             } else {
                 rpos += amt;
                 cirSize += amt;
@@ -187,6 +188,7 @@ status_t FdBuffer::readProcessedDataInStream(int fd, int toFd, int fromFd, int64
 
         // if buffer is empty and fd is closed, close write fd.
         if (cirSize == 0 && pfds[0].fd == -1 && pfds[1].fd != -1) {
+            VLOG("Close write pipe %d", toFd);
             ::close(pfds[1].fd);
             pfds[1].fd = -1;
         }
@@ -207,6 +209,7 @@ status_t FdBuffer::readProcessedDataInStream(int fd, int toFd, int fromFd, int64
                 return -errno;
             }  // otherwise just continue
         } else if (amt == 0) {
+            VLOG("Reached EOF of fromFd %d", fromFd);
             break;
         } else {
             mBuffer.wp()->move(amt);
diff --git a/cmds/incidentd/src/FdBuffer.h b/cmds/incidentd/src/FdBuffer.h
index 5bfa0938f5e8..66a3de154c51 100644
--- a/cmds/incidentd/src/FdBuffer.h
+++ b/cmds/incidentd/src/FdBuffer.h
@@ -26,7 +26,7 @@ using namespace android::util;
 using namespace std;
 
 /**
- * Reads a file into a buffer, and then writes that data to an FdSet.
+ * Reads data from fd into a buffer, fd must be closed explicitly.
  */
 class FdBuffer {
 public:
@@ -83,6 +83,11 @@ public:
      */
     EncodedBuffer::iterator data() const;
 
+    /**
+     * Return the internal buffer, don't call unless you are familiar with EncodedBuffer.
+     */
+    EncodedBuffer* getInternalBuffer() { return &mBuffer; }
+
 private:
     EncodedBuffer mBuffer;
     int64_t mStartTime;
diff --git a/cmds/incidentd/src/Section.cpp b/cmds/incidentd/src/Section.cpp
index 64eae3acd34c..334d77c1494e 100644
--- a/cmds/incidentd/src/Section.cpp
+++ b/cmds/incidentd/src/Section.cpp
@@ -18,12 +18,8 @@
 
 #include "Section.h"
 
-#include <errno.h>
-#include <sys/prctl.h>
-#include <unistd.h>
 #include <wait.h>
 
-#include <memory>
 #include <mutex>
 
 #include <android-base/file.h>
@@ -37,6 +33,7 @@
 #include "FdBuffer.h"
 #include "Privacy.h"
 #include "PrivacyBuffer.h"
+#include "frameworks/base/core/proto/android/os/data.proto.h"
 #include "frameworks/base/core/proto/android/util/log.proto.h"
 #include "incidentd_util.h"
 
@@ -52,31 +49,11 @@ const int FIELD_ID_INCIDENT_METADATA = 2;
 const int WAIT_MAX = 5;
 const struct timespec WAIT_INTERVAL_NS = {0, 200 * 1000 * 1000};
 const char INCIDENT_HELPER[] = "/system/bin/incident_helper";
+const char GZIP[] = "/system/bin/gzip";
 
-static pid_t fork_execute_incident_helper(const int id, const char* name, Fpipe& p2cPipe,
-                                          Fpipe& c2pPipe) {
+static pid_t fork_execute_incident_helper(const int id, Fpipe* p2cPipe, Fpipe* c2pPipe) {
     const char* ihArgs[]{INCIDENT_HELPER, "-s", String8::format("%d", id).string(), NULL};
-    // fork used in multithreaded environment, avoid adding unnecessary code in child process
-    pid_t pid = fork();
-    if (pid == 0) {
-        if (TEMP_FAILURE_RETRY(dup2(p2cPipe.readFd(), STDIN_FILENO)) != 0 || !p2cPipe.close() ||
-            TEMP_FAILURE_RETRY(dup2(c2pPipe.writeFd(), STDOUT_FILENO)) != 1 || !c2pPipe.close()) {
-            ALOGW("%s can't setup stdin and stdout for incident helper", name);
-            _exit(EXIT_FAILURE);
-        }
-
-        /* make sure the child dies when incidentd dies */
-        prctl(PR_SET_PDEATHSIG, SIGKILL);
-
-        execv(INCIDENT_HELPER, const_cast<char**>(ihArgs));
-
-        ALOGW("%s failed in incident helper process: %s", name, strerror(errno));
-        _exit(EXIT_FAILURE);  // always exits with failure if any
-    }
-    // close the fds used in incident helper
-    close(p2cPipe.readFd());
-    close(c2pPipe.writeFd());
-    return pid;
+    return fork_execute_cmd(INCIDENT_HELPER, const_cast<char**>(ihArgs), p2cPipe, c2pPipe);
 }
 
 // ================================================================================
@@ -254,10 +231,12 @@ status_t MetadataSection::Execute(ReportRequestSet* requests) const {
     return NO_ERROR;
 }
 // ================================================================================
+static inline bool isSysfs(const char* filename) { return strncmp(filename, "/sys/", 5) == 0; }
+
 FileSection::FileSection(int id, const char* filename, const int64_t timeoutMs)
     : Section(id, timeoutMs), mFilename(filename) {
     name = filename;
-    mIsSysfs = strncmp(filename, "/sys/", 5) == 0;
+    mIsSysfs = isSysfs(filename);
 }
 
 FileSection::~FileSection() {}
@@ -280,7 +259,7 @@ status_t FileSection::Execute(ReportRequestSet* requests) const {
         return -errno;
     }
 
-    pid_t pid = fork_execute_incident_helper(this->id, this->name.string(), p2cPipe, c2pPipe);
+    pid_t pid = fork_execute_incident_helper(this->id, &p2cPipe, &c2pPipe);
     if (pid == -1) {
         ALOGW("FileSection '%s' failed to fork", this->name.string());
         return -errno;
@@ -289,6 +268,8 @@ status_t FileSection::Execute(ReportRequestSet* requests) const {
     // parent process
     status_t readStatus = buffer.readProcessedDataInStream(fd, p2cPipe.writeFd(), c2pPipe.readFd(),
                                                            this->timeoutMs, mIsSysfs);
+    close(fd);  // close the fd anyway.
+
     if (readStatus != NO_ERROR || buffer.timedOut()) {
         ALOGW("FileSection '%s' failed to read data from incident helper: %s, timedout: %s",
               this->name.string(), strerror(-readStatus), buffer.timedOut() ? "true" : "false");
@@ -313,7 +294,99 @@ status_t FileSection::Execute(ReportRequestSet* requests) const {
 
     return NO_ERROR;
 }
+// ================================================================================
+GZipSection::GZipSection(int id, const char* filename, ...) : Section(id) {
+    name = "gzip ";
+    name += filename;
+    va_list args;
+    va_start(args, filename);
+    mFilenames = varargs(filename, args);
+    va_end(args);
+}
+
+GZipSection::~GZipSection() {}
+
+status_t GZipSection::Execute(ReportRequestSet* requests) const {
+    // Reads the files in order, use the first available one.
+    int index = 0;
+    int fd = -1;
+    while (mFilenames[index] != NULL) {
+        fd = open(mFilenames[index], O_RDONLY | O_CLOEXEC);
+        if (fd != -1) {
+            break;
+        }
+        ALOGW("GZipSection failed to open file %s", mFilenames[index]);
+        index++;  // look at the next file.
+    }
+    VLOG("GZipSection is using file %s, fd=%d", mFilenames[index], fd);
+    if (fd == -1) return -1;
+
+    FdBuffer buffer;
+    Fpipe p2cPipe;
+    Fpipe c2pPipe;
+    // initiate pipes to pass data to/from gzip
+    if (!p2cPipe.init() || !c2pPipe.init()) {
+        ALOGW("GZipSection '%s' failed to setup pipes", this->name.string());
+        return -errno;
+    }
+
+    const char* gzipArgs[]{GZIP, NULL};
+    pid_t pid = fork_execute_cmd(GZIP, const_cast<char**>(gzipArgs), &p2cPipe, &c2pPipe);
+    if (pid == -1) {
+        ALOGW("GZipSection '%s' failed to fork", this->name.string());
+        return -errno;
+    }
+    // parent process
 
+    // construct Fdbuffer to output GZippedfileProto, the reason to do this instead of using
+    // ProtoOutputStream is to avoid allocation of another buffer inside ProtoOutputStream.
+    EncodedBuffer* internalBuffer = buffer.getInternalBuffer();
+    internalBuffer->writeHeader((uint32_t)GZippedFileProto::FILENAME, WIRE_TYPE_LENGTH_DELIMITED);
+    String8 usedFile(mFilenames[index]);
+    internalBuffer->writeRawVarint32(usedFile.size());
+    for (size_t i = 0; i < usedFile.size(); i++) {
+        internalBuffer->writeRawByte(mFilenames[index][i]);
+    }
+    internalBuffer->writeHeader((uint32_t)GZippedFileProto::GZIPPED_DATA,
+                                WIRE_TYPE_LENGTH_DELIMITED);
+    size_t editPos = internalBuffer->wp()->pos();
+    internalBuffer->wp()->move(8);  // reserve 8 bytes for the varint of the data size.
+    size_t dataBeginAt = internalBuffer->wp()->pos();
+    VLOG("GZipSection '%s' editPos=%zd, dataBeginAt=%zd", this->name.string(), editPos,
+         dataBeginAt);
+
+    status_t readStatus = buffer.readProcessedDataInStream(
+            fd, p2cPipe.writeFd(), c2pPipe.readFd(), this->timeoutMs, isSysfs(mFilenames[index]));
+    close(fd);  // close the fd anyway.
+
+    if (readStatus != NO_ERROR || buffer.timedOut()) {
+        ALOGW("GZipSection '%s' failed to read data from gzip: %s, timedout: %s",
+              this->name.string(), strerror(-readStatus), buffer.timedOut() ? "true" : "false");
+        kill_child(pid);
+        return readStatus;
+    }
+
+    status_t gzipStatus = wait_child(pid);
+    if (gzipStatus != NO_ERROR) {
+        ALOGW("GZipSection '%s' abnormal child process: %s", this->name.string(),
+              strerror(-gzipStatus));
+        return gzipStatus;
+    }
+    // Revisit the actual size from gzip result and edit the internal buffer accordingly.
+    size_t dataSize = buffer.size() - dataBeginAt;
+    internalBuffer->wp()->rewind()->move(editPos);
+    internalBuffer->writeRawVarint32(dataSize);
+    internalBuffer->copy(dataBeginAt, dataSize);
+    VLOG("GZipSection '%s' wrote %zd bytes in %d ms, dataSize=%zd", this->name.string(),
+         buffer.size(), (int)buffer.durationMs(), dataSize);
+    status_t err = write_report_requests(this->id, buffer, requests);
+    if (err != NO_ERROR) {
+        ALOGW("GZipSection '%s' failed writing: %s", this->name.string(), strerror(-err));
+        return err;
+    }
+
+    return NO_ERROR;
+}
 // ================================================================================
 struct WorkerThreadData : public virtual RefBase {
     const WorkerThreadSection* section;
@@ -457,42 +530,20 @@ status_t WorkerThreadSection::Execute(ReportRequestSet* requests) const {
 }
 
 // ================================================================================
-void CommandSection::init(const char* command, va_list args) {
-    va_list copied_args;
-    int numOfArgs = 0;
-
-    va_copy(copied_args, args);
-    while (va_arg(copied_args, const char*) != NULL) {
-        numOfArgs++;
-    }
-    va_end(copied_args);
-
-    // allocate extra 1 for command and 1 for NULL terminator
-    mCommand = (const char**)malloc(sizeof(const char*) * (numOfArgs + 2));
-
-    mCommand[0] = command;
-    name = command;
-    for (int i = 0; i < numOfArgs; i++) {
-        const char* arg = va_arg(args, const char*);
-        mCommand[i + 1] = arg;
-        name += " ";
-        name += arg;
-    }
-    mCommand[numOfArgs + 1] = NULL;
-}
-
 CommandSection::CommandSection(int id, const int64_t timeoutMs, const char* command, ...)
     : Section(id, timeoutMs) {
+    name = command;
     va_list args;
     va_start(args, command);
-    init(command, args);
+    mCommand = varargs(command, args);
     va_end(args);
 }
 
 CommandSection::CommandSection(int id, const char* command, ...) : Section(id) {
+    name = command;
     va_list args;
     va_start(args, command);
-    init(command, args);
+    mCommand = varargs(command, args);
     va_end(args);
 }
 
@@ -527,7 +578,7 @@ status_t CommandSection::Execute(ReportRequestSet* requests) const {
               strerror(errno));
         _exit(err);  // exit with command error code
     }
-    pid_t ihPid = fork_execute_incident_helper(this->id, this->name.string(), cmdPipe, ihPipe);
+    pid_t ihPid = fork_execute_incident_helper(this->id, &cmdPipe, &ihPipe);
     if (ihPid == -1) {
         ALOGW("CommandSection '%s' failed to fork", this->name.string());
         return -errno;
@@ -544,8 +595,7 @@ status_t CommandSection::Execute(ReportRequestSet* requests) const {
     }
 
     // TODO: wait for command here has one trade-off: the failed status of command won't be detected
-    // until
-    //       buffer timeout, but it has advatage on starting the data stream earlier.
+    // until buffer timeout, but it has advatage on starting the data stream earlier.
     status_t cmdStatus = wait_child(cmdPid);
     status_t ihStatus = wait_child(ihPid);
     if (cmdStatus != NO_ERROR || ihStatus != NO_ERROR) {
diff --git a/cmds/incidentd/src/Section.h b/cmds/incidentd/src/Section.h
index d6446810c40f..8294be133bcb 100644
--- a/cmds/incidentd/src/Section.h
+++ b/cmds/incidentd/src/Section.h
@@ -84,6 +84,21 @@ private:
 };
 
 /**
+ * Section that reads in a file and gzips the content.
+ */
+class GZipSection : public Section {
+public:
+    GZipSection(int id, const char* filename, ...);
+    virtual ~GZipSection();
+
+    virtual status_t Execute(ReportRequestSet* requests) const;
+
+private:
+    // It looks up the content from multiple files and stops when the first one is available.
+    const char** mFilenames;
+};
+
+/**
  * Base class for sections that call a command that might need a timeout.
  */
 class WorkerThreadSection : public Section {
@@ -111,8 +126,6 @@ public:
 
 private:
     const char** mCommand;
-
-    void init(const char* command, va_list args);
 };
 
 /**
diff --git a/cmds/incidentd/src/incidentd_util.cpp b/cmds/incidentd/src/incidentd_util.cpp
index 2415860572fb..fc7cec9dbb40 100644
--- a/cmds/incidentd/src/incidentd_util.cpp
+++ b/cmds/incidentd/src/incidentd_util.cpp
@@ -13,8 +13,13 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+#define DEBUG false
+#include "Log.h"
+
 #include "incidentd_util.h"
 
+#include <sys/prctl.h>
+
 #include "section_list.h"
 
 const Privacy* get_privacy_of_section(int id) {
@@ -50,4 +55,49 @@ bool Fpipe::init() { return Pipe(&mRead, &mWrite); }
 
 int Fpipe::readFd() const { return mRead.get(); }
 
-int Fpipe::writeFd() const { return mWrite.get(); }
\ No newline at end of file
+int Fpipe::writeFd() const { return mWrite.get(); }
+
+pid_t fork_execute_cmd(const char* cmd, char* const argv[], Fpipe* input, Fpipe* output) {
+    // fork used in multithreaded environment, avoid adding unnecessary code in child process
+    pid_t pid = fork();
+    if (pid == 0) {
+        if (TEMP_FAILURE_RETRY(dup2(input->readFd(), STDIN_FILENO)) < 0 || !input->close() ||
+            TEMP_FAILURE_RETRY(dup2(output->writeFd(), STDOUT_FILENO)) < 0 || !output->close()) {
+            ALOGW("Can't setup stdin and stdout for command %s", cmd);
+            _exit(EXIT_FAILURE);
+        }
+
+        /* make sure the child dies when incidentd dies */
+        prctl(PR_SET_PDEATHSIG, SIGKILL);
+
+        execv(cmd, argv);
+
+        ALOGW("%s failed in the child process: %s", cmd, strerror(errno));
+        _exit(EXIT_FAILURE);  // always exits with failure if any
+    }
+    // close the fds used in child process.
+    close(input->readFd());
+    close(output->writeFd());
+    return pid;
+}
+// ================================================================================
+const char** varargs(const char* first, va_list rest) {
+    va_list copied_rest;
+    int numOfArgs = 1;  // first is already count.
+
+    va_copy(copied_rest, rest);
+    while (va_arg(copied_rest, const char*) != NULL) {
+        numOfArgs++;
+    }
+    va_end(copied_rest);
+
+    // allocate extra 1 for NULL terminator
+    const char** ret = (const char**)malloc(sizeof(const char*) * (numOfArgs + 1));
+    ret[0] = first;
+    for (int i = 0; i < numOfArgs; i++) {
+        const char* arg = va_arg(rest, const char*);
+        ret[i + 1] = arg;
+    }
+    ret[numOfArgs + 1] = NULL;
+    return ret;
+}
diff --git a/cmds/incidentd/src/incidentd_util.h b/cmds/incidentd/src/incidentd_util.h
index 09aa0404277a..db7ec82d83f4 100644
--- a/cmds/incidentd/src/incidentd_util.h
+++ b/cmds/incidentd/src/incidentd_util.h
@@ -20,12 +20,20 @@
 
 #include <android-base/unique_fd.h>
 
+#include <stdarg.h>
+
 #include "Privacy.h"
 
 using namespace android::base;
 
+/**
+ * Looks up Privacy of a section in the auto-gen PRIVACY_POLICY_LIST;
+ */
 const Privacy* get_privacy_of_section(int id);
 
+/**
+ * This class wraps android::base::Pipe.
+ */
 class Fpipe {
 public:
     Fpipe();
@@ -41,4 +49,15 @@ private:
     unique_fd mWrite;
 };
 
+/**
+ * Forks and exec a command with two pipes, one connects stdin for input,
+ * one connects stdout for output. It returns the pid of the child.
+ */
+pid_t fork_execute_cmd(const char* cmd, char* const argv[], Fpipe* input, Fpipe* output);
+
+/**
+ * Grabs varargs from stack and stores them in heap with NULL-terminated array.
+ */
+const char** varargs(const char* first, va_list rest);
+
 #endif  // INCIDENTD_UTIL_H
\ No newline at end of file
diff --git a/cmds/incidentd/testdata/kmsg.txt b/cmds/incidentd/testdata/kmsg.txt
new file mode 100644
index 000000000000..a8e3c02faab5
--- /dev/null
+++ b/cmds/incidentd/testdata/kmsg.txt
@@ -0,0 +1,47 @@
+[0] bldr_log_init: bldr_log_base=0x83600000, bldr_log_size=458752
+B -    626409 - [INFO][XBL]: Bypass appsbl verification on DEVELOPMENT device
+B -    729255 - [INFO][XBL]: Bypass appsbl verification on DEVELOPMENT device
+B -    729285 - boot_elf_load_and_verify_image: boot_auth_compute_verify_hash for 9:0
+D -    104829 - APPSBL Image Loaded, Delta - (2498816 Bytes)
+B -    729468 - SBL1, End
+D -    643611 - SBL1, Delta
+S - Flash Throughput, 129000 KB/s  (4729638 Bytes,  36613 us)
+S - DDR Frequency, 1017 MHz
+0x400, 0x400
+B -    482296 - Basic DDR tests done
+B -    544638 - clock_init, Start
+D -       244 - clock_init, Delta
+B -    544913 - HTC RPM DATARAM UPDATE info: Done
+B -    545004 - Image Load, Start
+B -    548359 - boot_elf_load_and_verify_image: boot_auth_compute_verify_hash for 5:0
+D -      3386 - QSEE Dev Config Image Loaded, Delta - (46232 Bytes)
+B -    548725 - Image Load, Start
+B -    550860 - boot_elf_load_and_verify_image: boot_auth_compute_verify_hash for 512:0
+D -      2166 - APDP Image Loaded, Delta - (7696 Bytes)
+B -    550891 - Image Load, Start
+B -    601612 - boot_elf_load_and_verify_image: boot_auth_compute_verify_hash for 7:0
+D -     50782 - QSEE Image Loaded, Delta - (1648640 Bytes)
+B -    601704 - Image Load, Start
+D -       244 - SEC Image Loaded, Delta - (4096 Bytes)
+B -    602344 - 0x1310 = 0x24
+B -    602375 - is_above_vbat_weak = 1, pon_reasons (with usb_in checked) = 0x31
+B -    602466 - sbl1_efs_handle_cookies, Start
+D -        91 - sbl1_efs_handle_cookies, Delta
+B -    602558 - Image Load, Start
+B -    613446 - boot_elf_load_and_verify_image: boot_auth_compute_verify_hash for 21:0
+D -     11010 - QHEE Image Loaded, Delta - (258280 Bytes)
+B -    613568 - Image Load, Start
+B -    624274 - boot_elf_load_and_verify_image: boot_auth_compute_verify_hash for 10:0
+D -     10736 - RPM Image Loaded, Delta - (224104 Bytes)
+B -    624335 - Image Load, Start
+D -         0 - STI Image Loaded, Delta - (0 Bytes)
+B -    624548 - Image Load, Start
+m_driver_init, Delta
+B -    471804 - pm_sbl_chg
+ ******************** [ START SECOND] ******************** 
+^@B -    736605 - [INFO][XBL]: Bypass appsbl verification on DEVELOPMENT device
+B -    839451 - [INFO][XBL]: Bypass appsbl verification on DEVELOPMENT device
+B -    839482 - boot_elf_load_and_verify_image: boot_auth_compute_verify_hash for 9:0
+D -    104828 - APPSBL Image Loaded, Delta - (2498816 Bytes)
+B -    839665 - SBL1, End
+D -    753838 - SBL1, Delta
diff --git a/cmds/incidentd/testdata/kmsg.txt.gz b/cmds/incidentd/testdata/kmsg.txt.gz
new file mode 100644
index 000000000000..fba449f8922c
--- /dev/null
+++ b/cmds/incidentd/testdata/kmsg.txt.gz
diff --git a/cmds/incidentd/tests/Section_test.cpp b/cmds/incidentd/tests/Section_test.cpp
index 026bf7419eb1..1528224447af 100644
--- a/cmds/incidentd/tests/Section_test.cpp
+++ b/cmds/incidentd/tests/Section_test.cpp
@@ -39,10 +39,22 @@ using namespace android::binder;
 using namespace android::os;
 using namespace std;
 using ::testing::StrEq;
+using ::testing::Test;
 using ::testing::internal::CaptureStdout;
 using ::testing::internal::GetCapturedStdout;
 
 // NOTICE: this test requires /system/bin/incident_helper is installed.
+class SectionTest : public Test {
+public:
+    virtual void SetUp() override { ASSERT_NE(tf.fd, -1); }
+
+protected:
+    TemporaryFile tf;
+    ReportRequestSet requests;
+
+    const std::string kTestPath = GetExecutableDirectory();
+    const std::string kTestDataPath = kTestPath + "/testdata/";
+};
 
 class SimpleListener : public IIncidentReportStatusListener {
 public:
@@ -58,10 +70,8 @@ protected:
     virtual IBinder* onAsBinder() override { return nullptr; };
 };
 
-TEST(SectionTest, HeaderSection) {
-    TemporaryFile output2;
+TEST_F(SectionTest, HeaderSection) {
     HeaderSection hs;
-    ReportRequestSet requests;
 
     IncidentReportArgs args1, args2;
     args1.addSection(1);
@@ -77,7 +87,7 @@ TEST(SectionTest, HeaderSection) {
     args2.addHeader(head2);
 
     requests.add(new ReportRequest(args1, new SimpleListener(), -1));
-    requests.add(new ReportRequest(args2, new SimpleListener(), output2.fd));
+    requests.add(new ReportRequest(args2, new SimpleListener(), tf.fd));
     requests.setMainFd(STDOUT_FILENO);
 
     string content;
@@ -87,28 +97,25 @@ TEST(SectionTest, HeaderSection) {
                                            "\x12\x3"
                                            "axe\n\x05\x12\x03pup"));
 
-    EXPECT_TRUE(ReadFileToString(output2.path, &content));
+    EXPECT_TRUE(ReadFileToString(tf.path, &content));
     EXPECT_THAT(content, StrEq("\n\x05\x12\x03pup"));
 }
 
-TEST(SectionTest, MetadataSection) {
+TEST_F(SectionTest, MetadataSection) {
     MetadataSection ms;
-    ReportRequestSet requests;
 
     requests.setMainFd(STDOUT_FILENO);
     requests.sectionStats(1)->set_success(true);
 
     CaptureStdout();
     ASSERT_EQ(NO_ERROR, ms.Execute(&requests));
-    EXPECT_THAT(GetCapturedStdout(), StrEq("\x12\b\x18\x1\"\x4\b\x1\x10\x1"));
+    EXPECT_THAT(GetCapturedStdout(), StrEq("\x12\b(\x1"
+                                           "2\x4\b\x1\x10\x1"));
 }
 
-TEST(SectionTest, FileSection) {
-    TemporaryFile tf;
+TEST_F(SectionTest, FileSection) {
     FileSection fs(REVERSE_PARSER, tf.path);
-    ReportRequestSet requests;
 
-    ASSERT_TRUE(tf.fd != -1);
     ASSERT_TRUE(WriteStringToFile("iamtestdata", tf.path));
 
     requests.setMainFd(STDOUT_FILENO);
@@ -120,66 +127,79 @@ TEST(SectionTest, FileSection) {
     EXPECT_THAT(GetCapturedStdout(), StrEq("\xa\vatadtsetmai"));
 }
 
-TEST(SectionTest, FileSectionTimeout) {
-    TemporaryFile tf;
-    // id -1 is timeout parser
+TEST_F(SectionTest, FileSectionTimeout) {
     FileSection fs(TIMEOUT_PARSER, tf.path, QUICK_TIMEOUT_MS);
-    ReportRequestSet requests;
     ASSERT_EQ(NO_ERROR, fs.Execute(&requests));
 }
 
-TEST(SectionTest, CommandSectionConstructor) {
+TEST_F(SectionTest, GZipSection) {
+    const std::string testFile = kTestDataPath + "kmsg.txt";
+    const std::string testGzFile = testFile + ".gz";
+    GZipSection gs(NOOP_PARSER, "/tmp/nonexist", testFile.c_str(), NULL);
+
+    requests.setMainFd(tf.fd);
+    requests.setMainDest(android::os::DEST_LOCAL);
+
+    ASSERT_EQ(NO_ERROR, gs.Execute(&requests));
+    std::string expect, gzFile, actual;
+    ASSERT_TRUE(ReadFileToString(testGzFile, &gzFile));
+    ASSERT_TRUE(ReadFileToString(tf.path, &actual));
+    expect = "\x2\xC6\x6\n\"" + testFile + "\x12\x9F\x6" + gzFile;
+    EXPECT_THAT(actual, StrEq(expect));
+}
+
+TEST_F(SectionTest, GZipSectionNoFileFound) {
+    GZipSection gs(NOOP_PARSER, "/tmp/nonexist1", "/tmp/nonexist2", NULL);
+    requests.setMainFd(STDOUT_FILENO);
+    ASSERT_EQ(-1, gs.Execute(&requests));
+}
+
+TEST_F(SectionTest, CommandSectionConstructor) {
     CommandSection cs1(1, "echo", "\"this is a test\"", "ooo", NULL);
     CommandSection cs2(2, "single_command", NULL);
     CommandSection cs3(1, 3123, "echo", "\"this is a test\"", "ooo", NULL);
     CommandSection cs4(2, 43214, "single_command", NULL);
 
-    EXPECT_THAT(cs1.name.string(), StrEq("echo \"this is a test\" ooo"));
+    EXPECT_THAT(cs1.name.string(), StrEq("echo"));
     EXPECT_THAT(cs2.name.string(), StrEq("single_command"));
     EXPECT_EQ(3123, cs3.timeoutMs);
     EXPECT_EQ(43214, cs4.timeoutMs);
-    EXPECT_THAT(cs3.name.string(), StrEq("echo \"this is a test\" ooo"));
+    EXPECT_THAT(cs3.name.string(), StrEq("echo"));
     EXPECT_THAT(cs4.name.string(), StrEq("single_command"));
 }
 
-TEST(SectionTest, CommandSectionEcho) {
+TEST_F(SectionTest, CommandSectionEcho) {
     CommandSection cs(REVERSE_PARSER, "/system/bin/echo", "about", NULL);
-    ReportRequestSet requests;
     requests.setMainFd(STDOUT_FILENO);
     CaptureStdout();
     ASSERT_EQ(NO_ERROR, cs.Execute(&requests));
     EXPECT_THAT(GetCapturedStdout(), StrEq("\xa\x06\ntuoba"));
 }
 
-TEST(SectionTest, CommandSectionCommandTimeout) {
+TEST_F(SectionTest, CommandSectionCommandTimeout) {
     CommandSection cs(NOOP_PARSER, QUICK_TIMEOUT_MS, "/system/bin/yes", NULL);
-    ReportRequestSet requests;
     ASSERT_EQ(NO_ERROR, cs.Execute(&requests));
 }
 
-TEST(SectionTest, CommandSectionIncidentHelperTimeout) {
+TEST_F(SectionTest, CommandSectionIncidentHelperTimeout) {
     CommandSection cs(TIMEOUT_PARSER, QUICK_TIMEOUT_MS, "/system/bin/echo", "about", NULL);
-    ReportRequestSet requests;
     requests.setMainFd(STDOUT_FILENO);
     ASSERT_EQ(NO_ERROR, cs.Execute(&requests));
 }
 
-TEST(SectionTest, CommandSectionBadCommand) {
+TEST_F(SectionTest, CommandSectionBadCommand) {
     CommandSection cs(NOOP_PARSER, "echoo", "about", NULL);
-    ReportRequestSet requests;
     ASSERT_EQ(NAME_NOT_FOUND, cs.Execute(&requests));
 }
 
-TEST(SectionTest, CommandSectionBadCommandAndTimeout) {
+TEST_F(SectionTest, CommandSectionBadCommandAndTimeout) {
     CommandSection cs(TIMEOUT_PARSER, QUICK_TIMEOUT_MS, "nonexistcommand", "-opt", NULL);
-    ReportRequestSet requests;
     // timeout will return first
     ASSERT_EQ(NO_ERROR, cs.Execute(&requests));
 }
 
-TEST(SectionTest, LogSectionBinary) {
+TEST_F(SectionTest, LogSectionBinary) {
     LogSection ls(1, LOG_ID_EVENTS);
-    ReportRequestSet requests;
     requests.setMainFd(STDOUT_FILENO);
     CaptureStdout();
     ASSERT_EQ(NO_ERROR, ls.Execute(&requests));
@@ -187,9 +207,8 @@ TEST(SectionTest, LogSectionBinary) {
     EXPECT_FALSE(results.empty());
 }
 
-TEST(SectionTest, LogSectionSystem) {
+TEST_F(SectionTest, LogSectionSystem) {
     LogSection ls(1, LOG_ID_SYSTEM);
-    ReportRequestSet requests;
     requests.setMainFd(STDOUT_FILENO);
     CaptureStdout();
     ASSERT_EQ(NO_ERROR, ls.Execute(&requests));
@@ -197,12 +216,9 @@ TEST(SectionTest, LogSectionSystem) {
     EXPECT_FALSE(results.empty());
 }
 
-TEST(SectionTest, TestFilterPiiTaggedFields) {
-    TemporaryFile tf;
+TEST_F(SectionTest, TestFilterPiiTaggedFields) {
     FileSection fs(NOOP_PARSER, tf.path);
-    ReportRequestSet requests;
 
-    ASSERT_TRUE(tf.fd != -1);
     ASSERT_TRUE(WriteStringToFile(VARINT_FIELD_1 + STRING_FIELD_2 + FIX64_FIELD_3, tf.path));
 
     requests.setMainFd(STDOUT_FILENO);
@@ -212,11 +228,9 @@ TEST(SectionTest, TestFilterPiiTaggedFields) {
     EXPECT_THAT(GetCapturedStdout(), StrEq("\x02\r" + STRING_FIELD_2));
 }
 
-TEST(SectionTest, TestBadFdRequest) {
-    TemporaryFile input;
-    FileSection fs(NOOP_PARSER, input.path);
-    ReportRequestSet requests;
-    ASSERT_TRUE(WriteStringToFile(VARINT_FIELD_1 + STRING_FIELD_2 + FIX64_FIELD_3, input.path));
+TEST_F(SectionTest, TestBadFdRequest) {
+    FileSection fs(NOOP_PARSER, tf.path);
+    ASSERT_TRUE(WriteStringToFile(VARINT_FIELD_1 + STRING_FIELD_2 + FIX64_FIELD_3, tf.path));
 
     IncidentReportArgs args;
     args.setAll(true);
@@ -231,11 +245,9 @@ TEST(SectionTest, TestBadFdRequest) {
     EXPECT_EQ(badFdRequest->err, -EBADF);
 }
 
-TEST(SectionTest, TestBadRequests) {
-    TemporaryFile input;
-    FileSection fs(NOOP_PARSER, input.path);
-    ReportRequestSet requests;
-    ASSERT_TRUE(WriteStringToFile(VARINT_FIELD_1 + STRING_FIELD_2 + FIX64_FIELD_3, input.path));
+TEST_F(SectionTest, TestBadRequests) {
+    FileSection fs(NOOP_PARSER, tf.path);
+    ASSERT_TRUE(WriteStringToFile(VARINT_FIELD_1 + STRING_FIELD_2 + FIX64_FIELD_3, tf.path));
 
     IncidentReportArgs args;
     args.setAll(true);
@@ -244,16 +256,14 @@ TEST(SectionTest, TestBadRequests) {
     EXPECT_EQ(fs.Execute(&requests), -EBADF);
 }
 
-TEST(SectionTest, TestMultipleRequests) {
-    TemporaryFile input, output1, output2, output3;
-    FileSection fs(NOOP_PARSER, input.path);
-    ReportRequestSet requests;
+TEST_F(SectionTest, TestMultipleRequests) {
+    TemporaryFile output1, output2, output3;
+    FileSection fs(NOOP_PARSER, tf.path);
 
-    ASSERT_TRUE(input.fd != -1);
     ASSERT_TRUE(output1.fd != -1);
     ASSERT_TRUE(output2.fd != -1);
     ASSERT_TRUE(output3.fd != -1);
-    ASSERT_TRUE(WriteStringToFile(VARINT_FIELD_1 + STRING_FIELD_2 + FIX64_FIELD_3, input.path));
+    ASSERT_TRUE(WriteStringToFile(VARINT_FIELD_1 + STRING_FIELD_2 + FIX64_FIELD_3, tf.path));
 
     IncidentReportArgs args1, args2, args3;
     args1.setAll(true);
@@ -286,17 +296,15 @@ TEST(SectionTest, TestMultipleRequests) {
     EXPECT_THAT(content, StrEq(""));
 }
 
-TEST(SectionTest, TestMultipleRequestsBySpec) {
-    TemporaryFile input, output1, output2, output3;
-    FileSection fs(NOOP_PARSER, input.path);
-    ReportRequestSet requests;
+TEST_F(SectionTest, TestMultipleRequestsBySpec) {
+    TemporaryFile output1, output2, output3;
+    FileSection fs(NOOP_PARSER, tf.path);
 
-    ASSERT_TRUE(input.fd != -1);
     ASSERT_TRUE(output1.fd != -1);
     ASSERT_TRUE(output2.fd != -1);
     ASSERT_TRUE(output3.fd != -1);
 
-    ASSERT_TRUE(WriteStringToFile(VARINT_FIELD_1 + STRING_FIELD_2 + FIX64_FIELD_3, input.path));
+    ASSERT_TRUE(WriteStringToFile(VARINT_FIELD_1 + STRING_FIELD_2 + FIX64_FIELD_3, tf.path));
 
     IncidentReportArgs args1, args2, args3;
     args1.setAll(true);
@@ -328,4 +336,4 @@ TEST(SectionTest, TestMultipleRequestsBySpec) {
     c = (char)STRING_FIELD_2.size();
     EXPECT_TRUE(ReadFileToString(output3.path, &content));
     EXPECT_THAT(content, StrEq(string("\x02") + c + STRING_FIELD_2));
-}
\ No newline at end of file
+}
diff --git a/core/proto/android/os/data.proto b/core/proto/android/os/data.proto
new file mode 100644
index 000000000000..c06f318a7312
--- /dev/null
+++ b/core/proto/android/os/data.proto
@@ -0,0 +1,34 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+syntax = "proto2";
+option java_multiple_files = true;
+
+package android.os;
+
+// This file contains protobuf definitions used in incidentd directly.
+// The top level proto message must be used as a new SectionType in
+// incidentd.
+
+// Output of SECTION_GZIP section type, which reads a file, gzip it and attached
+// in incident report as a proto field, example is LAST_KMSG.
+// NOTE the content in the file must not contain sensitive PII otherwise
+// implement it with fine-grained proto definition.
+message GZippedFileProto {
+    optional string filename = 1;
+
+    optional bytes gzipped_data = 2;
+}
diff --git a/core/proto/android/os/incident.proto b/core/proto/android/os/incident.proto
index 9a53b89ffe30..be155977a1ea 100644
--- a/core/proto/android/os/incident.proto
+++ b/core/proto/android/os/incident.proto
@@ -20,6 +20,7 @@ option java_multiple_files = true;
 import "frameworks/base/core/proto/android/os/batterytype.proto";
 import "frameworks/base/core/proto/android/os/cpufreq.proto";
 import "frameworks/base/core/proto/android/os/cpuinfo.proto";
+import "frameworks/base/core/proto/android/os/data.proto";
 import "frameworks/base/core/proto/android/os/kernelwake.proto";
 import "frameworks/base/core/proto/android/os/pagetypeinfo.proto";
 import "frameworks/base/core/proto/android/os/procrank.proto";
@@ -52,9 +53,8 @@ import "frameworks/base/libs/incident/proto/android/section.proto";
 
 package android.os;
 
-// privacy field options must not be set at this level because all
-// the sections are able to be controlled and configured by section ids.
-// Instead privacy field options need to be configured in each section proto message.
+// Privacy tag can be marked to override UNSET messages so generic
+// message type can be handled case by case, e.g. GZippedFileProto.
 message IncidentProto {
     reserved 1001;
 
@@ -151,6 +151,12 @@ message IncidentProto {
         (section).args = "/sys/class/power_supply/bms/battery_type"
     ];
 
+    optional GZippedFileProto last_kmsg = 2007 [
+        (section).type = SECTION_GZIP,
+        (section).args = "/sys/fs/pstore/console-ramoops /sys/fs/pstore/console-ramoops-0 /proc/last_kmsg",
+        (privacy).dest = DEST_AUTOMATIC
+    ];
+
     // System Services
     optional com.android.server.fingerprint.FingerprintServiceDumpProto fingerprint = 3000 [
         (section).type = SECTION_DUMPSYS,
diff --git a/libs/incident/proto/android/section.proto b/libs/incident/proto/android/section.proto
index 49bfe1e8a598..ef6a8ff6bcea 100644
--- a/libs/incident/proto/android/section.proto
+++ b/libs/incident/proto/android/section.proto
@@ -40,6 +40,9 @@ enum SectionType {
 
     // incidentd calls logs for annotated field
     SECTION_LOG = 4;
+
+    // incidentd read file and gzip the data in bytes field
+    SECTION_GZIP = 5;
 }
 
 message SectionFlags {
diff --git a/libs/protoutil/include/android/util/EncodedBuffer.h b/libs/protoutil/include/android/util/EncodedBuffer.h
index 0a8a5aa347ba..bf698d4a8f1d 100644
--- a/libs/protoutil/include/android/util/EncodedBuffer.h
+++ b/libs/protoutil/include/android/util/EncodedBuffer.h
@@ -154,7 +154,7 @@ public:
     void editRawFixed32(size_t pos, uint32_t val);
 
     /**
-     * Copy _size_ bytes of data starting at __srcPos__ to wp.
+     * Copy _size_ bytes of data starting at __srcPos__ to wp, srcPos must be larger than wp.pos().
      */
     void copy(size_t srcPos, size_t size);
 
diff --git a/tools/incident_section_gen/main.cpp b/tools/incident_section_gen/main.cpp
index e7b269aaa9a5..9183918fcc63 100644
--- a/tools/incident_section_gen/main.cpp
+++ b/tools/incident_section_gen/main.cpp
@@ -411,6 +411,11 @@ static bool generateSectionListCpp(Descriptor const* descriptor) {
             case SECTION_LOG:
                 printf("    new LogSection(%d, %s),\n", field->number(), s.args().c_str());
                 break;
+            case SECTION_GZIP:
+                printf("    new GZipSection(%d,", field->number());
+                splitAndPrint(s.args());
+                printf(" NULL),\n");
+                break;
         }
     }
     printf("    NULL };\n");