Move extent copy and hash logic to a new file.

The SOURCE_COPY operation used to copy the source blocks one by one to
the target partition. This process is sub-optimal if there are several
consecutive blocks. This patch moves this copy and hash logic to a new
file and adds several unittests for it. The new logic copies in chunks
of up to 1MiB when the source and target data is contiguous.

BUG=b:34284069
TEST=Added unittests.

Change-Id: I9ed52b429a54a2b4d6edaba051284b7dcd8a9525
(cherry picked from commit a48f630400429ca010c5462967607985f2ffa7e4)
Reviewed-on: https://chromium-review.googlesource.com/641958
Commit-Ready: Amin Hassani <ahassani@chromium.org>
Tested-by: Amin Hassani <ahassani@chromium.org>
Reviewed-by: Ben Chan <benchan@chromium.org>
Reviewed-by: Sen Jiang <senj@chromium.org>

1 files changed, 167 insertions, 0 deletions

diff --git a/payload_consumer/file_descriptor_utils_unittest.cc b/payload_consumer/file_descriptor_utils_unittest.cc
new file mode 100644
index 00000000..9910239d
--- /dev/null
+++ b/payload_consumer/file_descriptor_utils_unittest.cc
@@ -0,0 +1,167 @@
+//
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include "update_engine/payload_consumer/file_descriptor_utils.h"
+
+#include <string>
+#include <utility>
+#include <vector>
+
+#include <gtest/gtest.h>
+
+#include <brillo/data_encoding.h>
+
+#include "update_engine/common/hash_calculator.h"
+#include "update_engine/common/test_utils.h"
+#include "update_engine/common/utils.h"
+#include "update_engine/payload_consumer/fake_file_descriptor.h"
+#include "update_engine/payload_consumer/file_descriptor.h"
+#include "update_engine/payload_generator/extent_ranges.h"
+
+namespace chromeos_update_engine {
+
+namespace {
+
+::google::protobuf::RepeatedPtrField<Extent> CreateExtentList(
+    const std::vector<std::pair<uint64_t, uint64_t>>& lst) {
+  ::google::protobuf::RepeatedPtrField<Extent> result;
+  for (const auto& item : lst) {
+    *result.Add() = ExtentForRange(item.first, item.second);
+  }
+  return result;
+}
+
+}  // namespace
+
+class FileDescriptorUtilsTest : public ::testing::Test {
+ protected:
+  void SetUp() override {
+    EXPECT_TRUE(utils::MakeTempFile("fd_tgt.XXXXXX", &tgt_path_, nullptr));
+    EXPECT_TRUE(target_->Open(tgt_path_.c_str(), O_RDWR));
+  }
+
+  // Check that the |target_| file contains |expected_contents|.
+  void ExpectTarget(const std::string& expected_contents) {
+    std::string target_contents;
+    EXPECT_TRUE(utils::ReadFile(tgt_path_, &target_contents));
+    EXPECT_EQ(expected_contents.size(), target_contents.size());
+    if (target_contents != expected_contents) {
+      ADD_FAILURE() << "Contents don't match.";
+      LOG(INFO) << "Expected contents:";
+      utils::HexDumpString(expected_contents);
+      LOG(INFO) << "Actual contents:";
+      utils::HexDumpString(target_contents);
+    }
+  }
+
+  // Path to the target temporary file.
+  std::string tgt_path_;
+
+  // Source and target file descriptor used for testing the tools.
+  FakeFileDescriptor* fake_source_{new FakeFileDescriptor()};
+  FileDescriptorPtr source_{fake_source_};
+  FileDescriptorPtr target_{new EintrSafeFileDescriptor()};
+};
+
+// Source and target extents should have the same number of blocks.
+TEST_F(FileDescriptorUtilsTest, CopyAndHashExtentsMismatchBlocksTest) {
+  auto src_extents = CreateExtentList({{1, 4}});
+  auto tgt_extents = CreateExtentList({{0, 5}});
+
+  EXPECT_FALSE(fd_utils::CopyAndHashExtents(
+      source_, src_extents, target_, tgt_extents, 4, nullptr));
+}
+
+// Failing to read from the source should fail the copy.
+TEST_F(FileDescriptorUtilsTest, CopyAndHashExtentsReadFailureTest) {
+  auto extents = CreateExtentList({{0, 5}});
+  fake_source_->AddFailureRange(10, 5);
+
+  EXPECT_FALSE(fd_utils::CopyAndHashExtents(
+      source_, extents, target_, extents, 4, nullptr));
+}
+
+// Failing to write to the target should fail the copy.
+TEST_F(FileDescriptorUtilsTest, CopyAndHashExtentsWriteFailureTest) {
+  auto src_extents = CreateExtentList({{0, 2}});
+  auto tgt_extents = CreateExtentList({{5, 2}});
+  fake_source_->AddFailureRange(5 * 4, 10);
+
+  // Note that we pass |source_| as the target as well, which should fail to
+  // write.
+  EXPECT_FALSE(fd_utils::CopyAndHashExtents(
+      source_, src_extents, source_, tgt_extents, 4, nullptr));
+}
+
+// Test that we can copy extents without hashing them, allowing a nullptr
+// pointer as hash_out.
+TEST_F(FileDescriptorUtilsTest, CopyAndHashExtentsWithoutHashingTest) {
+  auto extents = CreateExtentList({{0, 5}});
+
+  EXPECT_TRUE(fd_utils::CopyAndHashExtents(
+      source_, extents, target_, extents, 4, nullptr));
+  ExpectTarget("00000001000200030004");
+}
+
+// CopyAndHash() can take different number of extents in the source and target
+// files, as long as the number of blocks is the same. Test that it handles it
+// properly.
+TEST_F(FileDescriptorUtilsTest, CopyAndHashExtentsManyToOneTest) {
+  brillo::Blob hash_out;
+  // Reorder the input as 1 4 2 3 0.
+  auto src_extents = CreateExtentList({{1, 1}, {4, 1}, {2, 2}, {0, 1}});
+  auto tgt_extents = CreateExtentList({{0, 5}});
+
+  EXPECT_TRUE(fd_utils::CopyAndHashExtents(
+      source_, src_extents, target_, tgt_extents, 4, &hash_out));
+  const char kExpectedResult[] = "00010004000200030000";
+  ExpectTarget(kExpectedResult);
+
+  brillo::Blob expected_hash;
+  EXPECT_TRUE(HashCalculator::RawHashOfBytes(
+      kExpectedResult, strlen(kExpectedResult), &expected_hash));
+  EXPECT_EQ(expected_hash, hash_out);
+}
+
+TEST_F(FileDescriptorUtilsTest, CopyAndHashExtentsManyToManyTest) {
+  brillo::Blob hash_out;
+  auto src_extents = CreateExtentList({{1, 1}, {4, 1}, {2, 2}, {0, 1}});
+  auto tgt_extents = CreateExtentList({{2, 3}, {0, 2}});
+
+  EXPECT_TRUE(fd_utils::CopyAndHashExtents(
+      source_, src_extents, target_, tgt_extents, 4, &hash_out));
+  // The reads always match the source extent list of blocks (up to the
+  // internal buffer size).
+  std::vector<std::pair<uint64_t, uint64_t>> kExpectedOps = {
+      {4, 4}, {16, 4}, {8, 8}, {0, 4}};
+  EXPECT_EQ(kExpectedOps, fake_source_->GetReadOps());
+
+  // The output here is as in the previous test but the first 3 4-byte blocks
+  // are at the end of the stream. The expected hash is as in the previous
+  // example anyway since the hash doesn't depend on the order of the target
+  // blocks.
+  const char kExpectedResult[] = "00030000000100040002";
+  ExpectTarget(kExpectedResult);
+
+  // The data in the order that the reader processes (and hashes) it.
+  const char kExpectedOrderedData[] = "00010004000200030000";
+  brillo::Blob expected_hash;
+  EXPECT_TRUE(HashCalculator::RawHashOfBytes(
+      kExpectedOrderedData, strlen(kExpectedOrderedData), &expected_hash));
+  EXPECT_EQ(expected_hash, hash_out);
+}
+
+}  // namespace chromeos_update_engine