update_engine: Add delta payload support for squashfs

This patch adds support for generating delta payloads for squashfs
images. This is needed to get delta payloads for DLC images.

In order to get the supported major and minor versions of the
update_engine that matches the squashfs image (either squashfs image
contains the entire rootfs, including the update_engine, or the image is
a DLC), we need to read /etc/update_engine inside the image.

We do this by calling unsquashfs and only unsquashing the target file
and later reading its content into a key-value store to be used for
delta payload generation.

BUG=chromium:926986
TEST=unittest
TEST=delta_generator --out_file=output --partition_names=dlc --new_partitions=dlc.img --old_partitions=dlc.img

Change-Id: Ib5599032c873223a5caca82918e138d8b4fcec43
Reviewed-on: https://chromium-review.googlesource.com/1446278
Commit-Ready: ChromeOS CL Exonerator Bot <chromiumos-cl-exonerator@appspot.gserviceaccount.com>
Tested-by: Amin Hassani <ahassani@chromium.org>
Reviewed-by: Sen Jiang <senj@chromium.org>

7 files changed, 89 insertions, 10 deletions

diff --git a/payload_generator/deflate_utils.cc b/payload_generator/deflate_utils.cc
index a7a05032..ef8d257d 100644
--- a/payload_generator/deflate_utils.cc
+++ b/payload_generator/deflate_utils.cc
@@ -273,8 +273,9 @@ bool PreprocessPartitionFiles(const PartitionConfig& part,
       TEST_AND_RETURN_FALSE(
           CopyExtentsToFile(part.path, file.extents, path.value(), kBlockSize));
       // Test if it is actually a Squashfs file.
-      auto sqfs =
-          SquashfsFilesystem::CreateFromFile(path.value(), extract_deflates);
+      auto sqfs = SquashfsFilesystem::CreateFromFile(path.value(),
+                                                     extract_deflates,
+                                                     /*load_settings=*/false);
       if (sqfs) {
         // It is an squashfs file. Get its files to replace with itself.
         vector<FilesystemInterface::File> files;
diff --git a/payload_generator/payload_generation_config.cc b/payload_generator/payload_generation_config.cc
index 694c71fa..c364797c 100644
--- a/payload_generator/payload_generation_config.cc
+++ b/payload_generator/payload_generation_config.cc
@@ -32,6 +32,7 @@
 #include "update_engine/payload_generator/ext2_filesystem.h"
 #include "update_engine/payload_generator/mapfile_filesystem.h"
 #include "update_engine/payload_generator/raw_filesystem.h"
+#include "update_engine/payload_generator/squashfs_filesystem.h"
 
 using std::string;
 
@@ -86,6 +87,14 @@ bool PartitionConfig::OpenFilesystem() {
     return true;
   }
 
+  fs_interface = SquashfsFilesystem::CreateFromFile(path,
+                                                    /*extract_deflates=*/true,
+                                                    /*load_settings=*/true);
+  if (fs_interface) {
+    TEST_AND_RETURN_FALSE(fs_interface->GetBlockSize() == kBlockSize);
+    return true;
+  }
+
   // Fall back to a RAW filesystem.
   TEST_AND_RETURN_FALSE(size % kBlockSize == 0);
   fs_interface = RawFilesystem::Create(
diff --git a/payload_generator/squashfs_filesystem.cc b/payload_generator/squashfs_filesystem.cc
index 6c892f53..c423b69c 100644
--- a/payload_generator/squashfs_filesystem.cc
+++ b/payload_generator/squashfs_filesystem.cc
@@ -23,6 +23,7 @@
 #include <utility>
 
 #include <base/files/file_util.h>
+#include <base/files/scoped_temp_dir.h>
 #include <base/logging.h>
 #include <base/strings/string_number_conversions.h>
 #include <base/strings/string_split.h>
@@ -36,6 +37,8 @@
 #include "update_engine/payload_generator/extent_utils.h"
 #include "update_engine/update_metadata.pb.h"
 
+using base::FilePath;
+using base::ScopedTempDir;
 using std::string;
 using std::unique_ptr;
 using std::vector;
@@ -49,6 +52,8 @@ constexpr size_t kSquashfsSuperBlockSize = 96;
 constexpr uint64_t kSquashfsCompressedBit = 1 << 24;
 constexpr uint32_t kSquashfsZlibCompression = 1;
 
+constexpr char kUpdateEngineConf[] = "etc/update_engine.conf";
+
 bool ReadSquashfsHeader(const brillo::Blob blob,
                         SquashfsFilesystem::SquashfsHeader* header) {
   if (blob.size() < kSquashfsSuperBlockSize) {
@@ -88,6 +93,45 @@ bool GetFileMapContent(const string& sqfs_path, string* map) {
   return true;
 }
 
+bool GetUpdateEngineConfig(const std::string& sqfs_path, string* config) {
+  ScopedTempDir unsquash_dir;
+  if (!unsquash_dir.CreateUniqueTempDir()) {
+    PLOG(ERROR) << "Failed to create a temporary directory.";
+    return false;
+  }
+
+  // Run unsquashfs to extract update_engine.conf
+  // -f: To force overriding if the target directory exists.
+  // -d: The directory to unsquash the files.
+  vector<string> cmd = {"unsquashfs",
+                        "-f",
+                        "-d",
+                        unsquash_dir.GetPath().value(),
+                        sqfs_path,
+                        kUpdateEngineConf};
+  int exit_code;
+  if (!Subprocess::SynchronousExec(cmd, &exit_code, nullptr) ||
+      exit_code != 0) {
+    PLOG(ERROR) << "Failed to unsquashfs etc/update_engine.conf: ";
+    return false;
+  }
+
+  auto config_path = unsquash_dir.GetPath().Append(kUpdateEngineConf);
+  string config_content;
+  if (!utils::ReadFile(config_path.value(), &config_content)) {
+    PLOG(ERROR) << "Failed to read " << config_path.value();
+    return false;
+  }
+
+  if (config_content.empty()) {
+    LOG(ERROR) << "update_engine config file was empty!!";
+    return false;
+  }
+
+  *config = std::move(config_content);
+  return true;
+}
+
 }  // namespace
 
 bool SquashfsFilesystem::Init(const string& map,
@@ -239,12 +283,12 @@ bool SquashfsFilesystem::Init(const string& map,
 }
 
 unique_ptr<SquashfsFilesystem> SquashfsFilesystem::CreateFromFile(
-    const string& sqfs_path, bool extract_deflates) {
+    const string& sqfs_path, bool extract_deflates, bool load_settings) {
   if (sqfs_path.empty())
     return nullptr;
 
   brillo::StreamPtr sqfs_file =
-      brillo::FileStream::Open(base::FilePath(sqfs_path),
+      brillo::FileStream::Open(FilePath(sqfs_path),
                                brillo::Stream::AccessMode::READ,
                                brillo::FileStream::Disposition::OPEN_EXISTING,
                                nullptr);
@@ -278,6 +322,12 @@ unique_ptr<SquashfsFilesystem> SquashfsFilesystem::CreateFromFile(
     return nullptr;
   }
 
+  if (load_settings) {
+    if (!GetUpdateEngineConfig(sqfs_path, &sqfs->update_engine_config_)) {
+      return nullptr;
+    }
+  }
+
   return sqfs;
 }
 
@@ -311,9 +361,12 @@ bool SquashfsFilesystem::GetFiles(vector<File>* files) const {
 }
 
 bool SquashfsFilesystem::LoadSettings(brillo::KeyValueStore* store) const {
-  // Settings not supported in squashfs.
-  LOG(ERROR) << "squashfs doesn't support LoadSettings().";
-  return false;
+  if (!store->LoadFromString(update_engine_config_)) {
+    LOG(ERROR) << "Failed to load the settings with config: "
+               << update_engine_config_;
+    return false;
+  }
+  return true;
 }
 
 bool SquashfsFilesystem::IsSquashfsImage(const brillo::Blob& blob) {
diff --git a/payload_generator/squashfs_filesystem.h b/payload_generator/squashfs_filesystem.h
index b79f8c7d..5045dfc6 100644
--- a/payload_generator/squashfs_filesystem.h
+++ b/payload_generator/squashfs_filesystem.h
@@ -59,7 +59,7 @@ class SquashfsFilesystem : public FilesystemInterface {
   // |extract_deflates| is true, it will process files to find location of all
   // deflate streams.
   static std::unique_ptr<SquashfsFilesystem> CreateFromFile(
-      const std::string& sqfs_path, bool extract_deflates);
+      const std::string& sqfs_path, bool extract_deflates, bool load_settings);
 
   // Creates the file system from a file map |filemap| which is a multi-line
   // string with each line with the following format:
@@ -113,6 +113,9 @@ class SquashfsFilesystem : public FilesystemInterface {
   // All the files in the filesystem.
   std::vector<File> files_;
 
+  // The content of /etc/update_engine.conf.
+  std::string update_engine_config_;
+
   DISALLOW_COPY_AND_ASSIGN(SquashfsFilesystem);
 };
 
diff --git a/payload_generator/squashfs_filesystem_unittest.cc b/payload_generator/squashfs_filesystem_unittest.cc
index 29fcf1cd..68ca9df2 100644
--- a/payload_generator/squashfs_filesystem_unittest.cc
+++ b/payload_generator/squashfs_filesystem_unittest.cc
@@ -112,7 +112,7 @@ class SquashfsFilesystemTest : public ::testing::Test {
 #ifdef __CHROMEOS__
 TEST_F(SquashfsFilesystemTest, EmptyFilesystemTest) {
   unique_ptr<SquashfsFilesystem> fs = SquashfsFilesystem::CreateFromFile(
-      GetBuildArtifactsPath("gen/disk_sqfs_empty.img"), true);
+      GetBuildArtifactsPath("gen/disk_sqfs_empty.img"), true, false);
   CheckSquashfs(fs);
 
   // Even an empty squashfs filesystem is rounded up to 4K.
@@ -133,7 +133,7 @@ TEST_F(SquashfsFilesystemTest, EmptyFilesystemTest) {
 
 TEST_F(SquashfsFilesystemTest, DefaultFilesystemTest) {
   unique_ptr<SquashfsFilesystem> fs = SquashfsFilesystem::CreateFromFile(
-      GetBuildArtifactsPath("gen/disk_sqfs_default.img"), true);
+      GetBuildArtifactsPath("gen/disk_sqfs_default.img"), true, false);
   CheckSquashfs(fs);
 
   vector<FilesystemInterface::File> files;
@@ -148,6 +148,18 @@ TEST_F(SquashfsFilesystemTest, DefaultFilesystemTest) {
   EXPECT_EQ(files[0].name, file.name);
   EXPECT_EQ(files[0].extents, file.extents);
 }
+
+TEST_F(SquashfsFilesystemTest, UpdateEngineConfigTest) {
+  unique_ptr<SquashfsFilesystem> fs = SquashfsFilesystem::CreateFromFile(
+      GetBuildArtifactsPath("gen/disk_sqfs_unittest.img"), true, true);
+  CheckSquashfs(fs);
+
+  brillo::KeyValueStore kvs;
+  EXPECT_TRUE(fs->LoadSettings(&kvs));
+  string minor_version;
+  EXPECT_TRUE(kvs.GetString("PAYLOAD_MINOR_VERSION", &minor_version));
+  EXPECT_EQ(minor_version, "1234");
+}
 #endif  // __CHROMEOS__
 
 TEST_F(SquashfsFilesystemTest, SimpleFileMapTest) {
diff --git a/sample_images/generate_images.sh b/sample_images/generate_images.sh
index 8478682e..e0b54ae9 100755
--- a/sample_images/generate_images.sh
+++ b/sample_images/generate_images.sh
@@ -270,6 +270,7 @@ main() {
   # Add squashfs sample images.
   generate_image disk_sqfs_empty sqfs empty $((1024 * 4096)) 4096
   generate_image disk_sqfs_default sqfs default $((1024 * 4096)) 4096
+  generate_image disk_sqfs_unittest sqfs unittest $((1024 * 4096)) 4096
 
   # Generate the tarball and delete temporary images.
   echo "Packing tar file sample_images.tar.bz2"
diff --git a/sample_images/sample_images.tar.bz2 b/sample_images/sample_images.tar.bz2
index 62154822..5c80a511 100644
--- a/sample_images/sample_images.tar.bz2
+++ b/sample_images/sample_images.tar.bz2