Merge c5dea6402ece46eb749c2e62a4738d67d46046ae on remote branch

Change-Id: Ie730c4569cbebda3ed0c85001014e6087df4d786

14 files changed, 342 insertions, 25 deletions

diff --git a/Android.bp b/Android.bp
index cc3cd092..cace5b61 100644
--- a/Android.bp
+++ b/Android.bp
@@ -902,7 +902,6 @@ cc_test_host {
         "update_engine.conf",
     ],
     static_libs: [
-        "libcurl",
         "libgmock",
         "libpayload_generator",
     ],
@@ -912,6 +911,82 @@ cc_test_host {
 // ========================================================
 // Main unittest file.
 cc_test {
+    name: "update_engine_http_unittests",
+    defaults: [
+        "ue_defaults",
+        "liblz4diff_defaults",
+        "update_metadata-protos_exports",
+    ],
+    require_root: true,
+    static_libs: [
+        "libbase",
+        "libbrillo-test-helpers",
+        "libchrome_test_helpers",
+        "libcurl",
+        "libcutils",
+        "libdm",
+        "libgmock",
+        "libz",
+    ],
+    shared_libs: [
+        "libssl",
+        "libcrypto",
+        "libziparchive",
+        "liblog",
+    ],
+
+    data: [
+        ":test_http_server",
+        ":test_subprocess",
+        ":ue_unittest_keys",
+        "otacerts.zip",
+        "unittest_key.pem",
+        "unittest_key2.pem",
+        "unittest_key_RSA4096.pem",
+        "unittest_key_EC.pem",
+    ],
+
+    // We cannot use the default generated AndroidTest.xml because of the use of helper modules
+    // (i.e. test_http_server, test_subprocess, ue_unittest_delta_generator).
+    // test_config: "test_config.xml",
+    test_suites: ["device-tests"],
+
+    srcs: [
+        "aosp/platform_constants_android.cc",
+        "certificate_checker.cc",
+        "common/action_processor.cc",
+        "common/boot_control_stub.cc",
+        "common/error_code_utils.cc",
+        "common/file_fetcher.cc",
+        "common/hash_calculator.cc",
+        "common/http_fetcher.cc",
+        "common/multi_range_http_fetcher.cc",
+        "common/http_common.cc",
+        "common/subprocess.cc",
+        "common/test_utils.cc",
+        "common/utils.cc",
+        "common/proxy_resolver.cc",
+        "libcurl_http_fetcher.cc",
+        "payload_consumer/certificate_parser_android.cc",
+        "payload_consumer/payload_verifier.cc",
+        "payload_generator/payload_signer.cc",
+        "update_status_utils.cc",
+
+        "certificate_checker_unittest.cc",
+        "common/http_fetcher_unittest.cc",
+        "common/mock_http_fetcher.cc",
+        "common/proxy_resolver_unittest.cc",
+        "common/subprocess_unittest.cc",
+        "libcurl_http_fetcher_unittest.cc",
+        "payload_consumer/certificate_parser_android_unittest.cc",
+        "update_status_utils_unittest.cc",
+    ],
+}
+
+// update_engine_unittests (type: executable)
+// ========================================================
+// Main unittest file.
+cc_test {
     name: "update_engine_unittests",
     defaults: [
         "ue_defaults",
@@ -933,8 +1008,6 @@ cc_test {
     ],
 
     data: [
-        ":test_http_server",
-        ":test_subprocess",
         ":ue_unittest_delta_generator",
         ":ue_unittest_disk_imgs",
         ":ue_unittest_erofs_imgs",
@@ -966,17 +1039,11 @@ cc_test {
         "aosp/dynamic_partition_control_android_unittest.cc",
         "aosp/update_attempter_android_integration_test.cc",
         "aosp/update_attempter_android_unittest.cc",
-        "certificate_checker_unittest.cc",
-        "common/http_fetcher_unittest.cc",
-        "common/proxy_resolver_unittest.cc",
-        "common/subprocess_unittest.cc",
         "common/utils_unittest.cc",
         "download_action_android_unittest.cc",
-        "libcurl_http_fetcher_unittest.cc",
         "payload_consumer/bzip_extent_writer_unittest.cc",
         "payload_consumer/cached_file_descriptor_unittest.cc",
         "payload_consumer/cow_writer_file_descriptor_unittest.cc",
-        "payload_consumer/certificate_parser_android_unittest.cc",
         "payload_consumer/delta_performer_integration_test.cc",
         "payload_consumer/delta_performer_unittest.cc",
         "payload_consumer/extent_reader_unittest.cc",
@@ -994,7 +1061,6 @@ cc_test {
         "payload_consumer/snapshot_extent_writer_unittest.cc",
         "payload_consumer/vabc_partition_writer_unittest.cc",
         "payload_consumer/xor_extent_writer_unittest.cc",
-        "update_status_utils_unittest.cc",
     ],
 }
 
@@ -1082,3 +1148,25 @@ cc_library_static {
         export_proto_headers: true,
     },
 }
+
+cc_binary_host {
+    name: "ota_extractor",
+    defaults: [
+        "ue_defaults",
+        "libpayload_consumer_exports",
+    ],
+    srcs: [
+        "aosp/ota_extractor.cc",
+    ],
+    static_libs: [
+        "liblog",
+        "libbrotli",
+        "libbase",
+        "libpayload_consumer",
+        "libpayload_extent_ranges",
+        "libpayload_extent_utils",
+        "libz",
+        "libgflags",
+        "update_metadata-protos",
+    ],
+}
diff --git a/TEST_MAPPING b/TEST_MAPPING
new file mode 100644
index 00000000..3a9a2389
--- /dev/null
+++ b/TEST_MAPPING
@@ -0,0 +1,10 @@
+{
+  "presubmit": [
+    {
+      "name": "update_engine_unittests"
+    },
+    {
+      "name": "update_engine_http_unittests"
+    }
+  ]
+}
diff --git a/aosp/dynamic_partition_control_android.cc b/aosp/dynamic_partition_control_android.cc
index 6d33a09a..e39e7bc7 100644
--- a/aosp/dynamic_partition_control_android.cc
+++ b/aosp/dynamic_partition_control_android.cc
@@ -465,6 +465,9 @@ bool DynamicPartitionControlAndroid::PreparePartitionsForUpdate(
   if (!SetTargetBuildVars(manifest)) {
     return false;
   }
+  for (auto& list : dynamic_partition_list_) {
+    list.clear();
+  }
 
   // Although the current build supports dynamic partitions, the given payload
   // doesn't use it for target partitions. This could happen when applying a
@@ -1280,6 +1283,9 @@ bool DynamicPartitionControlAndroid::ResetUpdate(PrefsInterface* prefs) {
   if (!GetVirtualAbFeatureFlag().IsEnabled()) {
     return true;
   }
+  for (auto& list : dynamic_partition_list_) {
+    list.clear();
+  }
 
   LOG(INFO) << __func__ << " resetting update state and deleting snapshots.";
   TEST_AND_RETURN_FALSE(prefs != nullptr);
diff --git a/aosp/dynamic_partition_control_android.h b/aosp/dynamic_partition_control_android.h
index cebca07f..457ef182 100644
--- a/aosp/dynamic_partition_control_android.h
+++ b/aosp/dynamic_partition_control_android.h
@@ -21,7 +21,7 @@
 #include <set>
 #include <string>
 #include <string_view>
-#include <vector>
+#include <array>
 
 #include <base/files/file_util.h>
 #include <libsnapshot/auto_device.h>
@@ -348,7 +348,9 @@ class DynamicPartitionControlAndroid : public DynamicPartitionControlInterface {
 
   uint32_t source_slot_ = UINT32_MAX;
   uint32_t target_slot_ = UINT32_MAX;
-  std::vector<std::vector<std::string>> dynamic_partition_list_{2UL};
+  // We assume that there's only 2 slots, A and B. This assumption is unlikely
+  // to change in the future. And certaintly won't change at runtime.
+  std::array<std::vector<std::string>, 2> dynamic_partition_list_{};
 
   DISALLOW_COPY_AND_ASSIGN(DynamicPartitionControlAndroid);
 };
diff --git a/aosp/ota_extractor.cc b/aosp/ota_extractor.cc
new file mode 100644
index 00000000..80bb74bf
--- /dev/null
+++ b/aosp/ota_extractor.cc
@@ -0,0 +1,181 @@
+//
+// Copyright (C) 2022 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include <fcntl.h>
+
+#include <cstdint>
+#include <cstdio>
+#include <iterator>
+#include <memory>
+
+#include <sys/mman.h>
+#include <sys/stat.h>
+
+#include <android-base/strings.h>
+#include <base/files/file_path.h>
+#include <gflags/gflags.h>
+#include <unistd.h>
+
+#include "update_engine/common/utils.h"
+#include "update_engine/common/hash_calculator.h"
+#include "update_engine/payload_consumer/file_descriptor.h"
+#include "update_engine/payload_consumer/install_operation_executor.h"
+#include "update_engine/payload_consumer/payload_metadata.h"
+#include "update_engine/update_metadata.pb.h"
+#include "xz.h"
+
+DEFINE_string(payload, "", "Path to payload.bin");
+DEFINE_string(output_dir, "", "Directory to put output images");
+DEFINE_int64(payload_offset,
+             0,
+             "Offset to start of payload.bin. Useful if payload path actually "
+             "points to a .zip file containing payload.bin");
+DEFINE_string(partitions,
+              "",
+              "Comma separated list of partitions to extract, leave empty for "
+              "extracting all partitions");
+
+using chromeos_update_engine::DeltaArchiveManifest;
+using chromeos_update_engine::PayloadMetadata;
+
+namespace chromeos_update_engine {
+
+bool ExtractImagesFromOTA(const DeltaArchiveManifest& manifest,
+                          const PayloadMetadata& metadata,
+                          int payload_fd,
+                          size_t payload_offset,
+                          std::string_view output_dir,
+                          const std::set<std::string>& partitions) {
+  InstallOperationExecutor executor(manifest.block_size());
+  const size_t data_begin = metadata.GetMetadataSize() +
+                            metadata.GetMetadataSignatureSize() +
+                            payload_offset;
+  const base::FilePath path(
+      base::StringPiece(output_dir.data(), output_dir.size()));
+  std::vector<unsigned char> blob;
+  for (const auto& partition : manifest.partitions()) {
+    if (!partitions.empty() &&
+        partitions.count(partition.partition_name()) == 0) {
+      continue;
+    }
+    LOG(INFO) << "Extracting partition " << partition.partition_name()
+              << " size: " << partition.new_partition_info().size();
+    const auto output_path =
+        path.Append(partition.partition_name() + ".img").value();
+    auto fd =
+        std::make_shared<chromeos_update_engine::EintrSafeFileDescriptor>();
+    TEST_AND_RETURN_FALSE_ERRNO(
+        fd->Open(output_path.c_str(), O_RDWR | O_CREAT, 0644));
+    for (const auto& op : partition.operations()) {
+      blob.resize(op.data_length());
+      const auto op_data_offset = data_begin + op.data_offset();
+      ssize_t bytes_read = 0;
+      TEST_AND_RETURN_FALSE(utils::PReadAll(
+          payload_fd, blob.data(), blob.size(), op_data_offset, &bytes_read));
+      auto direct_writer = std::make_unique<DirectExtentWriter>(fd);
+      if (op.type() == InstallOperation::ZERO) {
+        TEST_AND_RETURN_FALSE(executor.ExecuteZeroOrDiscardOperation(
+            op, std::move(direct_writer)));
+      } else if (op.type() == InstallOperation::REPLACE ||
+                 op.type() == InstallOperation::REPLACE_BZ ||
+                 op.type() == InstallOperation::REPLACE_XZ) {
+        TEST_AND_RETURN_FALSE(executor.ExecuteReplaceOperation(
+            op, std::move(direct_writer), blob.data(), blob.size()));
+      } else {
+        LOG(ERROR) << "Unsupported operation type: " << op.type() << ", "
+                   << InstallOperation::Type_Name(op.type());
+        return false;
+      }
+    }
+    int err =
+        truncate64(output_path.c_str(), partition.new_partition_info().size());
+    if (err) {
+      PLOG(ERROR) << "Failed to truncate " << output_path << " to "
+                  << partition.new_partition_info().size();
+    }
+    brillo::Blob actual_hash;
+    TEST_AND_RETURN_FALSE(
+        HashCalculator::RawHashOfFile(output_path, &actual_hash));
+    CHECK_EQ(HexEncode(ToStringView(actual_hash)),
+             HexEncode(partition.new_partition_info().hash()));
+  }
+  return true;
+}
+
+}  // namespace chromeos_update_engine
+
+int main(int argc, char* argv[]) {
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+  xz_crc32_init();
+  auto tokens = android::base::Tokenize(FLAGS_partitions, ",");
+  const std::set<std::string> partitions(
+      std::make_move_iterator(tokens.begin()),
+      std::make_move_iterator(tokens.end()));
+  if (FLAGS_payload.empty()) {
+    LOG(ERROR) << "--payload <payload path> is required";
+    return 1;
+  }
+  if (!partitions.empty()) {
+    LOG(INFO) << "Extracting " << android::base::Join(partitions, ", ");
+  }
+  int payload_fd = open(FLAGS_payload.c_str(), O_RDONLY | O_CLOEXEC);
+  if (payload_fd < 0) {
+    PLOG(ERROR) << "Failed to open payload file";
+    return 1;
+  }
+  chromeos_update_engine::ScopedFdCloser closer{&payload_fd};
+  auto payload_size = chromeos_update_engine::utils::FileSize(payload_fd);
+  if (payload_size <= 0) {
+    PLOG(ERROR)
+        << "Couldn't determine size of payload file, or payload file is empty";
+    return 1;
+  }
+
+  PayloadMetadata payload_metadata;
+  auto payload = static_cast<unsigned char*>(
+      mmap(nullptr, payload_size, PROT_READ, MAP_PRIVATE, payload_fd, 0));
+
+  if (payload == MAP_FAILED) {
+    PLOG(ERROR) << "Failed to mmap() payload file";
+    return 1;
+  }
+
+  auto munmap_deleter = [payload_size](auto payload) {
+    munmap(payload, payload_size);
+  };
+  std::unique_ptr<unsigned char, decltype(munmap_deleter)> munmapper{
+      payload, munmap_deleter};
+  if (payload_metadata.ParsePayloadHeader(payload + FLAGS_payload_offset,
+                                          payload_size - FLAGS_payload_offset,
+                                          nullptr) !=
+      chromeos_update_engine::MetadataParseResult::kSuccess) {
+    LOG(ERROR) << "Payload header parse failed!";
+    return 1;
+  }
+  DeltaArchiveManifest manifest;
+  if (!payload_metadata.GetManifest(payload + FLAGS_payload_offset,
+                                    payload_size - FLAGS_payload_offset,
+                                    &manifest)) {
+    LOG(ERROR) << "Failed to parse manifest!";
+    return 1;
+  }
+  return !ExtractImagesFromOTA(manifest,
+                               payload_metadata,
+                               payload_fd,
+                               FLAGS_payload_offset,
+                               FLAGS_output_dir,
+                               partitions);
+}
diff --git a/aosp/update_attempter_android.cc b/aosp/update_attempter_android.cc
index 4e609d4e..a3485eac 100644
--- a/aosp/update_attempter_android.cc
+++ b/aosp/update_attempter_android.cc
@@ -400,6 +400,10 @@ bool UpdateAttempterAndroid::CancelUpdate(brillo::ErrorPtr* error) {
 bool UpdateAttempterAndroid::ResetStatus(brillo::ErrorPtr* error) {
   LOG(INFO) << "Attempting to reset state from "
             << UpdateStatusToString(status_) << " to UpdateStatus::IDLE";
+  if (processor_->IsRunning()) {
+    return LogAndSetError(
+        error, FROM_HERE, "Already processing an update, cancel it first.");
+  }
 
   if (apex_handler_android_ != nullptr) {
     LOG(INFO) << "Cleaning up reserved space for compressed APEX (if any)";
@@ -416,12 +420,12 @@ bool UpdateAttempterAndroid::ResetStatus(brillo::ErrorPtr* error) {
                           "ClearUpdateCompletedMarker() failed");
   }
 
+  if (!boot_control_->GetDynamicPartitionControl()->ResetUpdate(prefs_)) {
+    LOG(WARNING) << "Failed to reset snapshots. UpdateStatus is IDLE but"
+                  << "space might not be freed.";
+  }
   switch (status_) {
     case UpdateStatus::IDLE: {
-      if (!boot_control_->GetDynamicPartitionControl()->ResetUpdate(prefs_)) {
-        LOG(WARNING) << "Failed to reset snapshots. UpdateStatus is IDLE but"
-                     << "space might not be freed.";
-      }
       return true;
     }
 
diff --git a/common/mock_http_fetcher.h b/common/mock_http_fetcher.h
index ea5b83da..3d7859ba 100644
--- a/common/mock_http_fetcher.h
+++ b/common/mock_http_fetcher.h
@@ -36,7 +36,7 @@ namespace chromeos_update_engine {
 // MockHttpFetcher will send a chunk of data down in each call to BeginTransfer
 // and Unpause. For the other chunks of data, a callback is put on the run
 // loop and when that's called, another chunk is sent down.
-const size_t kMockHttpFetcherChunkSize(65536);
+static constexpr size_t kMockHttpFetcherChunkSize(65536);
 
 class MockHttpFetcher : public HttpFetcher {
  public:
diff --git a/payload_consumer/delta_performer.cc b/payload_consumer/delta_performer.cc
index 19877db6..fc8858ff 100644
--- a/payload_consumer/delta_performer.cc
+++ b/payload_consumer/delta_performer.cc
@@ -1117,11 +1117,11 @@ ErrorCode DeltaPerformer::ValidateOperationHash(
 
   if (calculated_op_hash != expected_op_hash) {
     LOG(ERROR) << "Hash verification failed for operation "
-               << next_operation_num_ << ". Expected hash = ";
-    utils::HexDumpVector(expected_op_hash);
+               << next_operation_num_
+               << ". Expected hash = " << HexEncode(expected_op_hash);
     LOG(ERROR) << "Calculated hash over " << operation.data_length()
-               << " bytes at offset: " << operation.data_offset() << " = ";
-    utils::HexDumpVector(calculated_op_hash);
+               << " bytes at offset: " << operation.data_offset() << " = "
+               << HexEncode(calculated_op_hash);
     return ErrorCode::kDownloadOperationHashMismatch;
   }
 
diff --git a/payload_consumer/partition_writer.cc b/payload_consumer/partition_writer.cc
index b94f8c71..1fb929ea 100644
--- a/payload_consumer/partition_writer.cc
+++ b/payload_consumer/partition_writer.cc
@@ -295,6 +295,16 @@ std::unique_ptr<ExtentWriter> PartitionWriter::CreateBaseExtentWriter() {
   return std::make_unique<DirectExtentWriter>(target_fd_);
 }
 
+bool PartitionWriter::ValidateSourceHash(const InstallOperation& operation,
+                                         const FileDescriptorPtr source_fd,
+                                         size_t block_size,
+                                         ErrorCode* error) {
+  brillo::Blob source_hash;
+  TEST_AND_RETURN_FALSE_ERRNO(fd_utils::ReadAndHashExtents(
+      source_fd, operation.src_extents(), block_size, &source_hash));
+  return ValidateSourceHash(source_hash, operation, source_fd, error);
+}
+
 bool PartitionWriter::ValidateSourceHash(const brillo::Blob& calculated_hash,
                                          const InstallOperation& operation,
                                          const FileDescriptorPtr source_fd,
diff --git a/payload_consumer/partition_writer.h b/payload_consumer/partition_writer.h
index 89e58840..e620c478 100644
--- a/payload_consumer/partition_writer.h
+++ b/payload_consumer/partition_writer.h
@@ -46,6 +46,10 @@ class PartitionWriter : public PartitionWriterInterface {
                                  const InstallOperation& operation,
                                  const FileDescriptorPtr source_fd,
                                  ErrorCode* error);
+  static bool ValidateSourceHash(const InstallOperation& operation,
+                                 const FileDescriptorPtr source_fd,
+                                 size_t block_size,
+                                 ErrorCode* error);
 
   // Perform necessary initialization work before InstallOperation can be
   // applied to this partition
diff --git a/payload_consumer/vabc_partition_writer.cc b/payload_consumer/vabc_partition_writer.cc
index 9db88a91..54aea860 100644
--- a/payload_consumer/vabc_partition_writer.cc
+++ b/payload_consumer/vabc_partition_writer.cc
@@ -33,6 +33,7 @@
 #include "update_engine/payload_consumer/extent_map.h"
 #include "update_engine/payload_consumer/extent_reader.h"
 #include "update_engine/payload_consumer/file_descriptor.h"
+#include "update_engine/payload_consumer/file_descriptor_utils.h"
 #include "update_engine/payload_consumer/install_plan.h"
 #include "update_engine/payload_consumer/partition_writer.h"
 #include "update_engine/payload_consumer/snapshot_extent_writer.h"
@@ -267,9 +268,16 @@ std::unique_ptr<ExtentWriter> VABCPartitionWriter::CreateBaseExtentWriter() {
 
 [[nodiscard]] bool VABCPartitionWriter::PerformSourceCopyOperation(
     const InstallOperation& operation, ErrorCode* error) {
-  // TODO(zhangkelvin) Probably just ignore SOURCE_COPY? They should be taken
-  // care of during Init();
-  return true;
+  // COPY ops are already handled during Init(), no need to do actual work, but
+  // we still want to verify that all blocks contain expected data.
+  auto source_fd = std::make_shared<EintrSafeFileDescriptor>();
+  TEST_AND_RETURN_FALSE_ERRNO(
+      source_fd->Open(install_part_.source_path.c_str(), O_RDONLY));
+  if (!operation.has_src_sha256_hash()) {
+    return true;
+  }
+  return PartitionWriter::ValidateSourceHash(
+      operation, source_fd, block_size_, error);
 }
 
 bool VABCPartitionWriter::PerformReplaceOperation(const InstallOperation& op,
diff --git a/payload_generator/payload_generation_config.h b/payload_generator/payload_generation_config.h
index a7ddee41..7124cb0b 100644
--- a/payload_generator/payload_generation_config.h
+++ b/payload_generator/payload_generation_config.h
@@ -25,7 +25,6 @@
 
 #include <brillo/key_value_store.h>
 #include <brillo/secure_blob.h>
-#include <lz4diff/lz4diff.pb.h>
 
 #include "bsdiff/constants.h"
 #include "update_engine/payload_consumer/payload_constants.h"
diff --git a/scripts/simulate_ota.py b/scripts/simulate_ota.py
index 40f463fd..bf1fc984 100644
--- a/scripts/simulate_ota.py
+++ b/scripts/simulate_ota.py
@@ -110,6 +110,7 @@ def run_ota(source, target, payload_path, tempdir, output_dir):
   delta_generator_args.append("--partition_names=" + ":".join(partition_names))
   delta_generator_args.append("--new_partitions=" + ":".join(new_partitions))
 
+  print("Running ", " ".join(delta_generator_args))
   subprocess.check_output(delta_generator_args)
 
   valid = True
diff --git a/scripts/update_device.py b/scripts/update_device.py
index db653dc3..72cee494 100755
--- a/scripts/update_device.py
+++ b/scripts/update_device.py
@@ -442,6 +442,8 @@ def main():
                       help='Perform slot switch for this OTA package')
   parser.add_argument('--perform-reset-slot-switch', action='store_true',
                       help='Perform reset slot switch for this OTA package')
+  parser.add_argument('--wipe-user-data', action='store_true',
+                      help='Wipe userdata after installing OTA')
   args = parser.parse_args()
   logging.basicConfig(
       level=logging.WARNING if args.no_verbose else logging.INFO)
@@ -493,6 +495,8 @@ def main():
     args.extra_headers += "\nSWITCH_SLOT_ON_REBOOT=0"
   if args.no_postinstall:
     args.extra_headers += "\nRUN_POST_INSTALL=0"
+  if args.wipe_user_data:
+    args.extra_headers += "\nPOWERWASH=1"
 
   with zipfile.ZipFile(args.otafile) as zfp:
     CARE_MAP_ENTRY_NAME = "care_map.pb"