diff options
author | adlr@google.com <adlr@google.com@06c00378-0e64-4dae-be16-12b19f9950a1> | 2009-12-04 20:57:17 +0000 |
---|---|---|
committer | adlr@google.com <adlr@google.com@06c00378-0e64-4dae-be16-12b19f9950a1> | 2009-12-04 20:57:17 +0000 |
commit | 3defe6acb3609e70e851a6eff062577d25a2af9d (patch) | |
tree | 341e979027fde117dd8906483db7a5c703a2e1cf | |
parent | c98a7edf648aad88b3f66df3b5a7d43d6a6d7fa9 (diff) |
Missed new files in last commit
Review URL: http://codereview.chromium.org/465067
git-svn-id: svn://chrome-svn/chromeos/trunk@336 06c00378-0e64-4dae-be16-12b19f9950a1
43 files changed, 6497 insertions, 0 deletions
diff --git a/delta_diff_generator.cc b/delta_diff_generator.cc new file mode 100644 index 00000000..6f352c71 --- /dev/null +++ b/delta_diff_generator.cc @@ -0,0 +1,423 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "update_engine/delta_diff_generator.h" +#include <dirent.h> +#include <endian.h> +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <algorithm> +#include <vector> +#include <tr1/memory> +#include <zlib.h> +#include "chromeos/obsolete_logging.h" +#include "base/scoped_ptr.h" +#include "update_engine/delta_diff_parser.h" +#include "update_engine/gzip.h" +#include "update_engine/subprocess.h" +#include "update_engine/utils.h" + +using std::vector; +using std::tr1::shared_ptr; +using chromeos_update_engine::DeltaArchiveManifest; + +namespace chromeos_update_engine { + +namespace { +// These structs and methods are helpers for EncodeDataToDeltaFile() + +// Before moving the data into a proto buffer, the data is stored in +// memory in these Node and Child structures. + +// Each Node struct represents a file on disk (which can be regular file, +// directory, fifo, socket, symlink, etc). Nodes that contain children +// (just directories) will have a vector of Child objects. Each child +// object has a filename and a pointer to the associated Node. Thus, +// filenames for files are stored with their parents, not as part of +// the file itself. + +// These structures are easier to work with than the protobuf format +// when adding files. When generating a delta file, we add an entry +// for each file to a root Node object. Then, we sort each Node's +// children vector so the children are stored alphabetically. Then, +// we assign an index value to the idx field of each Node by a preorder +// tree traversal. The index value assigned to a Node is the index it +// will have in the DeltaArchiveManifest protobuf. +// Finally, we add each Node to a DeltaArchiveManifest protobuf. + +struct Node; + +struct Child { + Child(const string& the_name, + Node* the_node) + : name(the_name), + node(the_node) {} + string name; + // Use shared_ptr here rather than scoped_ptr b/c this struct will be copied + // in stl containers + scoped_ptr<Node> node; +}; + +// For the C++ sort() function. +struct ChildLessThan { + bool operator()(const shared_ptr<Child>& a, const shared_ptr<Child>& b) { + return a->name < b->name; + } +}; + +struct Node { + Node() + : mode(0), + uid(0), + gid(0), + compressed(false), + offset(-1), + length(0), + idx(0) {} + + mode_t mode; + uid_t uid; + gid_t gid; + + // data + bool compressed; + int offset; // -1 means no data + int length; + + vector<shared_ptr<Child> > children; + int idx; +}; + +// This function sets *node's variables to match what's at path. +// This includes calling this function recursively on all children. Children +// not on the same device as the original node will not be considered. +// Returns true on success. +bool UpdateNodeFromPath(const string& path, Node* node) { + // Set metadata + struct stat stbuf; + TEST_AND_RETURN_FALSE_ERRNO(lstat(path.c_str(), &stbuf) == 0); + const dev_t dev = stbuf.st_dev; + node->mode = stbuf.st_mode; + node->uid = stbuf.st_uid; + node->gid = stbuf.st_gid; + if (!S_ISDIR(node->mode)) { + return true; + } + + DIR* dir = opendir(path.c_str()); + TEST_AND_RETURN_FALSE(dir); + + struct dirent entry; + struct dirent* dir_entry; + + for (;;) { + TEST_AND_RETURN_FALSE_ERRNO(readdir_r(dir, &entry, &dir_entry) == 0); + if (!dir_entry) { + // done + break; + } + if (!strcmp(".", dir_entry->d_name)) + continue; + if (!strcmp("..", dir_entry->d_name)) + continue; + + string child_path = path + "/" + dir_entry->d_name; + struct stat child_stbuf; + TEST_AND_RETURN_FALSE_ERRNO(lstat(child_path.c_str(), &child_stbuf) == 0); + // make sure it's on the same dev + if (child_stbuf.st_dev != dev) + continue; + shared_ptr<Child> child(new Child(dir_entry->d_name, new Node)); + node->children.push_back(child); + TEST_AND_RETURN_FALSE(UpdateNodeFromPath(path + "/" + child->name, + child->node.get())); + } + TEST_AND_RETURN_FALSE_ERRNO(closedir(dir) == 0); + // Done with all subdirs. sort children. + sort(node->children.begin(), node->children.end(), ChildLessThan()); + return true; +} + +// We go through n setting the index value of each Node to +// *next_index_value, then increment next_index_value. +// We then recursively assign index values to children. +// The first caller should call this with *next_index_value == 0 and +// the root Node, thus setting the root Node's index to 0. +void PopulateChildIndexes(Node* n, int* next_index_value) { + n->idx = (*next_index_value)++; + for (unsigned int i = 0; i < n->children.size(); i++) { + PopulateChildIndexes(n->children[i]->node.get(), next_index_value); + } +} + +// This converts a Node tree rooted at n into a DeltaArchiveManifest. +void NodeToDeltaArchiveManifest(Node* n, DeltaArchiveManifest* archive) { + DeltaArchiveManifest_File *f = archive->add_files(); + f->set_mode(n->mode); + f->set_uid(n->uid); + f->set_gid(n->gid); + if (!S_ISDIR(n->mode)) + return; + for (unsigned int i = 0; i < n->children.size(); i++) { + DeltaArchiveManifest_File_Child* child = f->add_children(); + child->set_name(n->children[i]->name); + child->set_index(n->children[i]->node->idx); + } + for (unsigned int i = 0; i < n->children.size(); i++) { + NodeToDeltaArchiveManifest(n->children[i]->node.get(), archive); + } +} + +} // namespace {} + +// For each file in archive, write a delta for it into out_file +// and update 'file' to refer to the delta. +// This is a recursive function. Returns true on success. +bool DeltaDiffGenerator::WriteFileDiffsToDeltaFile( + DeltaArchiveManifest* archive, + DeltaArchiveManifest_File* file, + const std::string& file_name, + const std::string& old_path, + const std::string& new_path, + FileWriter* out_file_writer, + int* out_file_length) { + TEST_AND_RETURN_FALSE(file->has_mode()); + + // Stat the actual file, too + struct stat stbuf; + TEST_AND_RETURN_FALSE_ERRNO(lstat((new_path + "/" + file_name).c_str(), + &stbuf) == 0); + TEST_AND_RETURN_FALSE(stbuf.st_mode == file->mode()); + + // See if we're a directory or not + if (S_ISDIR(file->mode())) { + for (int i = 0; i < file->children_size(); i++) { + DeltaArchiveManifest_File_Child* child = file->mutable_children(i); + DeltaArchiveManifest_File* child_file = + archive->mutable_files(child->index()); + TEST_AND_RETURN_FALSE(WriteFileDiffsToDeltaFile( + archive, + child_file, + child->name(), + old_path + "/" + file_name, + new_path + "/" + file_name, + out_file_writer, + out_file_length)); + } + return true; + } + + if (S_ISFIFO(file->mode()) || S_ISSOCK(file->mode())) { + // These don't store any additional data + return true; + } + + vector<char> data; + bool should_compress = true; + bool format_set = false; + DeltaArchiveManifest_File_DataFormat format; + if (S_ISLNK(file->mode())) { + LOG(INFO) << "link"; + TEST_AND_RETURN_FALSE(EncodeLink(new_path + "/" + file_name, &data)); + } else if (S_ISCHR(file->mode()) || S_ISBLK(file->mode())) { + LOG(INFO) << "dev"; + TEST_AND_RETURN_FALSE(EncodeDev(stbuf, &data)); + } else if (S_ISREG(file->mode())) { + LOG(INFO) << "reg"; + // regular file. We may use a delta here. + TEST_AND_RETURN_FALSE(EncodeFile(old_path, new_path, file_name, &format, + &data)); + should_compress = false; + format_set = true; + if ((format == DeltaArchiveManifest_File_DataFormat_BSDIFF) || + (format == DeltaArchiveManifest_File_DataFormat_FULL_GZ)) + TEST_AND_RETURN_FALSE(!data.empty()); + } else { + // Should never get here; unhandled mode type. + LOG(ERROR) << "Unhandled mode type: " << file->mode(); + return false; + } + LOG(INFO) << "data len: " << data.size(); + if (!format_set) { + // Pick a format now + vector<char> compressed_data; + TEST_AND_RETURN_FALSE(GzipCompress(data, &compressed_data)); + if (compressed_data.size() < data.size()) { + format = DeltaArchiveManifest_File_DataFormat_FULL_GZ; + data.swap(compressed_data); + } else { + format = DeltaArchiveManifest_File_DataFormat_FULL; + } + format_set = true; + } + + if (!data.empty()) { + TEST_AND_RETURN_FALSE(format_set); + file->set_data_format(format); + file->set_data_offset(*out_file_length); + TEST_AND_RETURN_FALSE(static_cast<ssize_t>(data.size()) == + out_file_writer->Write(&data[0], data.size())); + file->set_data_length(data.size()); + *out_file_length += data.size(); + } + return true; +} + +bool DeltaDiffGenerator::EncodeLink(const std::string& path, + std::vector<char>* out) { + // Store symlink path as file data + vector<char> link_data(4096); + int rc = readlink(path.c_str(), &link_data[0], link_data.size()); + TEST_AND_RETURN_FALSE_ERRNO(rc >= 0); + link_data.resize(rc); + out->swap(link_data); + return true; +} + +bool DeltaDiffGenerator::EncodeDev(const struct stat& stbuf, + std::vector<char>* out) { + LinuxDevice dev; + dev.set_major(major(stbuf.st_rdev)); + dev.set_minor(minor(stbuf.st_rdev)); + out->resize(dev.ByteSize()); + TEST_AND_RETURN_FALSE(dev.SerializeToArray(&(*out)[0], out->size())); + return true; +} + +// Encode the file at new_path + "/" + file_name. It may be a binary diff +// based on old_path + "/" + file_name. out_data_format will be set to +// the format used. out_data_format may not be NULL. +bool DeltaDiffGenerator::EncodeFile( + const string& old_dir, + const string& new_dir, + const string& file_name, + DeltaArchiveManifest_File_DataFormat* out_data_format, + vector<char>* out) { + TEST_AND_RETURN_FALSE(out_data_format); + // First, see the full length: + vector<char> full_data; + TEST_AND_RETURN_FALSE(utils::ReadFile(new_dir + "/" + file_name, &full_data)); + vector<char> gz_data; + if (!full_data.empty()) { + TEST_AND_RETURN_FALSE(GzipCompress(full_data, &gz_data)); + } + vector<char> *ret = NULL; + + if (gz_data.size() < full_data.size()) { + *out_data_format = DeltaArchiveManifest_File_DataFormat_FULL_GZ; + ret = &gz_data; + } else { + *out_data_format = DeltaArchiveManifest_File_DataFormat_FULL; + ret = &full_data; + } + + struct stat old_stbuf; + if ((stat((old_dir + "/" + file_name).c_str(), &old_stbuf) < 0) || + (!S_ISREG(old_stbuf.st_mode))) { + // stat() failed or old file is not a regular file. Just send back the full + // contents + *out = *ret; + return true; + } + // We have an old file. Do a binary diff. For now use bsdiff. + const string kPatchFile = "/tmp/delta.patch"; + + vector<string> cmd; + cmd.push_back("/usr/bin/bsdiff"); + cmd.push_back(old_dir + "/" + file_name); + cmd.push_back(new_dir + "/" + file_name); + cmd.push_back(kPatchFile); + + int rc = 1; + TEST_AND_RETURN_FALSE(Subprocess::SynchronousExec(cmd, &rc)); + TEST_AND_RETURN_FALSE(rc == 0); + vector<char> patch_file; + TEST_AND_RETURN_FALSE(utils::ReadFile(kPatchFile, &patch_file)); + unlink(kPatchFile.c_str()); + + if (patch_file.size() < ret->size()) { + *out_data_format = DeltaArchiveManifest_File_DataFormat_BSDIFF; + ret = &patch_file; + } + + *out = *ret; + return true; +} + +DeltaArchiveManifest* DeltaDiffGenerator::EncodeMetadataToProtoBuffer( + const char* new_path) { + Node node; + if (!UpdateNodeFromPath(new_path, &node)) + return NULL; + int index = 0; + PopulateChildIndexes(&node, &index); + DeltaArchiveManifest *ret = new DeltaArchiveManifest; + NodeToDeltaArchiveManifest(&node, ret); + return ret; +} + +bool DeltaDiffGenerator::EncodeDataToDeltaFile(DeltaArchiveManifest* archive, + const std::string& old_path, + const std::string& new_path, + const std::string& out_file) { + DirectFileWriter out_writer; + int r = out_writer.Open(out_file.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666); + TEST_AND_RETURN_FALSE_ERRNO(r >= 0); + ScopedFileWriterCloser closer(&out_writer); + TEST_AND_RETURN_FALSE(out_writer.Write(DeltaDiffParser::kFileMagic, + strlen(DeltaDiffParser::kFileMagic)) + == static_cast<ssize_t>( + strlen(DeltaDiffParser::kFileMagic))); + // Write 8 null bytes. This will be filled in w/ the offset of + // the protobuf. + TEST_AND_RETURN_FALSE(out_writer.Write("\0\0\0\0\0\0\0\0", 8) == 8); + // 8 more bytes will be filled w/ the protobuf length. + TEST_AND_RETURN_FALSE(out_writer.Write("\0\0\0\0\0\0\0\0", 8) == 8); + int out_file_length = strlen(DeltaDiffParser::kFileMagic) + 16; + + TEST_AND_RETURN_FALSE(archive->files_size() > 0); + DeltaArchiveManifest_File* file = archive->mutable_files(0); + + TEST_AND_RETURN_FALSE(WriteFileDiffsToDeltaFile(archive, + file, + "", + old_path, + new_path, + &out_writer, + &out_file_length)); + + // Finally, write the protobuf to the end of the file + string encoded_archive; + TEST_AND_RETURN_FALSE(archive->SerializeToString(&encoded_archive)); + + // Compress the protobuf (which contains filenames) + vector<char> compressed_encoded_archive; + TEST_AND_RETURN_FALSE(GzipCompressString(encoded_archive, + &compressed_encoded_archive)); + + TEST_AND_RETURN_FALSE(out_writer.Write(compressed_encoded_archive.data(), + compressed_encoded_archive.size()) == + static_cast<ssize_t>( + compressed_encoded_archive.size())); + + // write offset of protobut to just after the file magic + int64 big_endian_protobuf_offset = htobe64(out_file_length); + TEST_AND_RETURN_FALSE(pwrite(out_writer.fd(), + &big_endian_protobuf_offset, + sizeof(big_endian_protobuf_offset), + strlen(DeltaDiffParser::kFileMagic)) == + sizeof(big_endian_protobuf_offset)); + // Write the size just after the offset + int64 pb_length = htobe64(compressed_encoded_archive.size()); + TEST_AND_RETURN_FALSE(pwrite(out_writer.fd(), + &pb_length, + sizeof(pb_length), + strlen(DeltaDiffParser::kFileMagic) + + sizeof(big_endian_protobuf_offset)) == + sizeof(pb_length)); + return true; +} + +} // namespace chromeos_update_engine diff --git a/delta_diff_generator.h b/delta_diff_generator.h new file mode 100644 index 00000000..0f73d060 --- /dev/null +++ b/delta_diff_generator.h @@ -0,0 +1,72 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CHROMEOS_PLATFORM_UPDATE_ENGINE_DELTA_DIFF_GENERATOR_H__ +#define CHROMEOS_PLATFORM_UPDATE_ENGINE_DELTA_DIFF_GENERATOR_H__ + +#include <sys/types.h> +#include <sys/stat.h> +#include <string> +#include <vector> +#include "base/basictypes.h" +#include "update_engine/file_writer.h" +#include "update_engine/update_metadata.pb.h" + +namespace chromeos_update_engine { + +class DeltaDiffGenerator { + public: + // Encodes the metadata at new_path recursively into a DeltaArchiveManifest + // protobuf object. This will only read the filesystem. Children will + // be recorded recursively iff they are on the same device as their + // parent. + // This will set all fields in the DeltaArchiveManifest except for + // DeltaArchiveManifest_File_data_* as those are set only when writing + // the actual delta file to disk. + // Caller is responsible for freeing the returned value. + // Returns NULL on failure. + static DeltaArchiveManifest* EncodeMetadataToProtoBuffer( + const char* new_path); + + // Takes a DeltaArchiveManifest as given from EncodeMetadataToProtoBuffer(), + // fill in the missing fields (DeltaArchiveManifest_File_data_*), and + // write the full delta out to the output file. + // Returns true on success. + static bool EncodeDataToDeltaFile(DeltaArchiveManifest* archive, + const std::string& old_path, + const std::string& new_path, + const std::string& out_file); + private: + // These functions encode all the data about a file that's not already + // stored in the DeltaArchiveManifest message into the vector 'out'. + // They all return true on success. + + // EncodeLink stores the path the symlink points to. + static bool EncodeLink(const std::string& path, std::vector<char>* out); + // EncodeDev stores the major and minor device numbers. + // Specifically it writes a LinuxDevice message. + static bool EncodeDev(const struct stat& stbuf, std::vector<char>* out); + // EncodeFile stores the full data, gzipped data, or a binary diff from + // the old data. out_data_format will be set to the method used. + static bool EncodeFile(const std::string& old_dir, + const std::string& new_dir, + const std::string& file_name, + DeltaArchiveManifest_File_DataFormat* out_data_format, + std::vector<char>* out); + + static bool WriteFileDiffsToDeltaFile(DeltaArchiveManifest* archive, + DeltaArchiveManifest_File* file, + const std::string& file_name, + const std::string& old_path, + const std::string& new_path, + FileWriter* out_file_writer, + int* out_file_length); + + // This should never be constructed + DISALLOW_IMPLICIT_CONSTRUCTORS(DeltaDiffGenerator); +}; + +}; // namespace chromeos_update_engine + +#endif // CHROMEOS_PLATFORM_UPDATE_ENGINE_DELTA_DIFF_GENERATOR_H__ diff --git a/delta_diff_generator_unittest.cc b/delta_diff_generator_unittest.cc new file mode 100644 index 00000000..495bc3ec --- /dev/null +++ b/delta_diff_generator_unittest.cc @@ -0,0 +1,814 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <unistd.h> +#include <string> +#include <vector> +#include "base/string_util.h" +#include <gtest/gtest.h> +#include "chromeos/obsolete_logging.h" +#include "update_engine/decompressing_file_writer.h" +#include "update_engine/delta_diff_generator.h" +#include "update_engine/delta_diff_parser.h" +#include "update_engine/gzip.h" +#include "update_engine/mock_file_writer.h" +#include "update_engine/subprocess.h" +#include "update_engine/test_utils.h" +#include "update_engine/utils.h" + +namespace chromeos_update_engine { + +using std::string; +using std::vector; + +class DeltaDiffGeneratorTest : public ::testing::Test {}; + +namespace { +void DumpProto(const DeltaArchiveManifest* archive) { + for (int i = 0; i < archive->files_size(); i++) { + printf("Node %d\n", i); + const DeltaArchiveManifest_File& file = archive->files(i); + for (int j = 0; j < file.children_size(); j++) { + const DeltaArchiveManifest_File_Child& child = file.children(j); + printf(" %d %s\n", child.index(), child.name().c_str()); + } + } +} + +// The following files are generated at the path 'base': +// / +// cdev (c 2 1) +// dir/ +// bdev (b 3 1) +// emptydir/ (owner:group = 501:503) +// hello ("hello") +// newempty ("") +// subdir/ +// fifo +// link -> /target +// encoding/ +// long_new +// long_small_change +// nochange +// onebyte +// hi ("hi") +void GenerateFilesAtPath(const string& base) { + const char* base_c = base.c_str(); + EXPECT_EQ(0, System(StringPrintf("echo hi > '%s/hi'", base_c))); + EXPECT_EQ(0, System(StringPrintf("mkdir -p '%s/dir'", base_c))); + EXPECT_EQ(0, System(StringPrintf("rm -f '%s/dir/bdev'", base_c))); + EXPECT_EQ(0, System(StringPrintf("mknod '%s/dir/bdev' b 3 1", base_c))); + EXPECT_EQ(0, System(StringPrintf("rm -f '%s/cdev'", base_c))); + EXPECT_EQ(0, System(StringPrintf("mknod '%s/cdev' c 2 1", base_c))); + EXPECT_EQ(0, System(StringPrintf("mkdir -p '%s/dir/subdir'", base_c))); + EXPECT_EQ(0, System(StringPrintf("mkdir -p '%s/dir/emptydir'", base_c))); + EXPECT_EQ(0, System(StringPrintf("chown 501:503 '%s/dir/emptydir'", base_c))); + EXPECT_EQ(0, System(StringPrintf("rm -f '%s/dir/subdir/fifo'", base_c))); + EXPECT_EQ(0, System(StringPrintf("mkfifo '%s/dir/subdir/fifo'", base_c))); + EXPECT_EQ(0, System(StringPrintf("ln -f -s /target '%s/dir/subdir/link'", + base_c))); + + // Things that will encode differently: + EXPECT_EQ(0, System(StringPrintf("mkdir -p '%s/encoding'", base_c))); + EXPECT_EQ(0, System(StringPrintf("echo nochange > '%s/encoding/nochange'", + base_c))); + EXPECT_EQ(0, System(StringPrintf("echo -n > '%s/encoding/onebyte'", base_c))); + EXPECT_EQ(0, System(StringPrintf("echo -n > '%s/encoding/long_new'", + base_c))); + // Random 1 MiB byte length file + EXPECT_TRUE(WriteFile((base + + "/encoding/long_small_change").c_str(), + reinterpret_cast<const char*>(kRandomString), + sizeof(kRandomString))); +} +// base points to a folder that was passed to GenerateFilesAtPath(). +// This edits some, so that one can make a diff from the original data +// and the edited data. +void EditFilesAtPath(const string& base) { + CHECK_EQ(0, System(string("echo hello > ") + base + "/dir/hello")); + CHECK_EQ(0, System(string("echo -n > ") + base + "/dir/newempty")); + CHECK_EQ(0, System(string("echo newhi > ") + base + "/hi")); + CHECK_EQ(0, System(string("echo -n h >> ") + base + + "/encoding/onebyte")); + CHECK_EQ(0, System(string("echo -n h >> ") + base + + "/encoding/long_small_change")); + CHECK_EQ(0, System(string("echo -n This is a pice of text that should " + "compress well since it is just ascii and it " + "has repetition xxxxxxxxxxxxxxxxxxxxx" + "xxxxxxxxxxxxxxxxxxxx > ") + base + + "/encoding/long_new")); +} + +} // namespace {} + +TEST_F(DeltaDiffGeneratorTest, FakerootEncodeMetadataToProtoBufferTest) { + char cwd[1000]; + ASSERT_EQ(cwd, getcwd(cwd, sizeof(cwd))) << "cwd buf possibly too small"; + ASSERT_EQ(0, System(string("mkdir -p ") + cwd + "/diff-gen-test")); + ASSERT_EQ(0, System(string("mkdir -p ") + cwd + "/diff-gen-test/old")); + ASSERT_EQ(0, System(string("mkdir -p ") + cwd + "/diff-gen-test/new")); + GenerateFilesAtPath(string(cwd) + "/diff-gen-test/old"); + GenerateFilesAtPath(string(cwd) + "/diff-gen-test/new"); + EditFilesAtPath(string(cwd) + "/diff-gen-test/new"); + + DeltaArchiveManifest* archive = + DeltaDiffGenerator::EncodeMetadataToProtoBuffer( + (string(cwd) + "/diff-gen-test/new").c_str()); + EXPECT_TRUE(NULL != archive); + + EXPECT_EQ(16, archive->files_size()); + //DumpProto(archive); + const DeltaArchiveManifest_File& root = archive->files(0); + EXPECT_TRUE(S_ISDIR(root.mode())); + EXPECT_EQ(0, root.uid()); + EXPECT_EQ(0, root.gid()); + ASSERT_EQ(4, root.children_size()); + EXPECT_EQ("cdev", root.children(0).name()); + EXPECT_EQ("dir", root.children(1).name()); + EXPECT_EQ("encoding", root.children(2).name()); + EXPECT_EQ("hi", root.children(3).name()); + EXPECT_FALSE(root.has_data_format()); + EXPECT_FALSE(root.has_data_offset()); + EXPECT_FALSE(root.has_data_length()); + + const DeltaArchiveManifest_File& cdev = + archive->files(root.children(0).index()); + EXPECT_EQ(0, cdev.children_size()); + EXPECT_TRUE(S_ISCHR(cdev.mode())); + EXPECT_EQ(0, cdev.uid()); + EXPECT_EQ(0, cdev.gid()); + EXPECT_FALSE(cdev.has_data_format()); + EXPECT_FALSE(cdev.has_data_offset()); + EXPECT_FALSE(cdev.has_data_length()); + + const DeltaArchiveManifest_File& hi = + archive->files(root.children(3).index()); + EXPECT_EQ(0, hi.children_size()); + EXPECT_TRUE(S_ISREG(hi.mode())); + EXPECT_EQ(0, hi.uid()); + EXPECT_EQ(0, hi.gid()); + EXPECT_FALSE(hi.has_data_format()); + EXPECT_FALSE(hi.has_data_offset()); + EXPECT_FALSE(hi.has_data_length()); + + const DeltaArchiveManifest_File& encoding = + archive->files(root.children(2).index()); + EXPECT_TRUE(S_ISDIR(encoding.mode())); + EXPECT_EQ(0, encoding.uid()); + EXPECT_EQ(0, encoding.gid()); + EXPECT_EQ(4, encoding.children_size()); + EXPECT_EQ("long_new", encoding.children(0).name()); + EXPECT_EQ("long_small_change", encoding.children(1).name()); + EXPECT_EQ("nochange", encoding.children(2).name()); + EXPECT_EQ("onebyte", encoding.children(3).name()); + EXPECT_FALSE(encoding.has_data_format()); + EXPECT_FALSE(encoding.has_data_offset()); + EXPECT_FALSE(encoding.has_data_length()); + + const DeltaArchiveManifest_File& long_new = + archive->files(encoding.children(0).index()); + EXPECT_EQ(0, long_new.children_size()); + EXPECT_TRUE(S_ISREG(long_new.mode())); + EXPECT_EQ(0, long_new.uid()); + EXPECT_EQ(0, long_new.gid()); + EXPECT_FALSE(long_new.has_data_format()); + EXPECT_FALSE(long_new.has_data_offset()); + EXPECT_FALSE(long_new.has_data_length()); + + const DeltaArchiveManifest_File& long_small_change = + archive->files(encoding.children(1).index()); + EXPECT_EQ(0, long_small_change.children_size()); + EXPECT_TRUE(S_ISREG(long_small_change.mode())); + EXPECT_EQ(0, long_small_change.uid()); + EXPECT_EQ(0, long_small_change.gid()); + EXPECT_FALSE(long_small_change.has_data_format()); + EXPECT_FALSE(long_small_change.has_data_offset()); + EXPECT_FALSE(long_small_change.has_data_length()); + + const DeltaArchiveManifest_File& nochange = + archive->files(encoding.children(2).index()); + EXPECT_EQ(0, nochange.children_size()); + EXPECT_TRUE(S_ISREG(nochange.mode())); + EXPECT_EQ(0, nochange.uid()); + EXPECT_EQ(0, nochange.gid()); + EXPECT_FALSE(nochange.has_data_format()); + EXPECT_FALSE(nochange.has_data_offset()); + EXPECT_FALSE(nochange.has_data_length()); + + const DeltaArchiveManifest_File& onebyte = + archive->files(encoding.children(3).index()); + EXPECT_EQ(0, onebyte.children_size()); + EXPECT_TRUE(S_ISREG(onebyte.mode())); + EXPECT_EQ(0, onebyte.uid()); + EXPECT_EQ(0, onebyte.gid()); + EXPECT_FALSE(onebyte.has_data_format()); + EXPECT_FALSE(onebyte.has_data_offset()); + EXPECT_FALSE(onebyte.has_data_length()); + + const DeltaArchiveManifest_File& dir = + archive->files(root.children(1).index()); + EXPECT_TRUE(S_ISDIR(dir.mode())); + EXPECT_EQ(0, dir.uid()); + EXPECT_EQ(0, dir.gid()); + ASSERT_EQ(5, dir.children_size()); + EXPECT_EQ("bdev", dir.children(0).name()); + EXPECT_EQ("emptydir", dir.children(1).name()); + EXPECT_EQ("hello", dir.children(2).name()); + EXPECT_EQ("newempty", dir.children(3).name()); + EXPECT_EQ("subdir", dir.children(4).name()); + EXPECT_FALSE(dir.has_data_format()); + EXPECT_FALSE(dir.has_data_offset()); + EXPECT_FALSE(dir.has_data_length()); + + const DeltaArchiveManifest_File& bdev = + archive->files(dir.children(0).index()); + EXPECT_EQ(0, bdev.children_size()); + EXPECT_TRUE(S_ISBLK(bdev.mode())); + EXPECT_EQ(0, bdev.uid()); + EXPECT_EQ(0, bdev.gid()); + EXPECT_FALSE(bdev.has_data_format()); + EXPECT_FALSE(bdev.has_data_offset()); + EXPECT_FALSE(bdev.has_data_length()); + + const DeltaArchiveManifest_File& emptydir = + archive->files(dir.children(1).index()); + EXPECT_EQ(0, emptydir.children_size()); + EXPECT_TRUE(S_ISDIR(emptydir.mode())); + EXPECT_EQ(501, emptydir.uid()); + EXPECT_EQ(503, emptydir.gid()); + EXPECT_FALSE(emptydir.has_data_format()); + EXPECT_FALSE(emptydir.has_data_offset()); + EXPECT_FALSE(emptydir.has_data_length()); + + const DeltaArchiveManifest_File& hello = + archive->files(dir.children(2).index()); + EXPECT_EQ(0, hello.children_size()); + EXPECT_TRUE(S_ISREG(hello.mode())); + EXPECT_EQ(0, hello.uid()); + EXPECT_EQ(0, hello.gid()); + EXPECT_FALSE(hello.has_data_format()); + EXPECT_FALSE(hello.has_data_offset()); + EXPECT_FALSE(hello.has_data_length()); + + const DeltaArchiveManifest_File& newempty = + archive->files(dir.children(3).index()); + EXPECT_EQ(0, newempty.children_size()); + EXPECT_TRUE(S_ISREG(newempty.mode())); + EXPECT_EQ(0, newempty.uid()); + EXPECT_EQ(0, newempty.gid()); + EXPECT_FALSE(newempty.has_data_format()); + EXPECT_FALSE(newempty.has_data_offset()); + EXPECT_FALSE(newempty.has_data_length()); + + const DeltaArchiveManifest_File& subdir = + archive->files(dir.children(4).index()); + EXPECT_EQ(2, subdir.children_size()); + EXPECT_EQ("fifo", subdir.children(0).name()); + EXPECT_EQ("link", subdir.children(1).name()); + EXPECT_TRUE(S_ISDIR(subdir.mode())); + EXPECT_EQ(0, subdir.uid()); + EXPECT_EQ(0, subdir.gid()); + EXPECT_FALSE(subdir.has_data_format()); + EXPECT_FALSE(subdir.has_data_offset()); + EXPECT_FALSE(subdir.has_data_length()); + + const DeltaArchiveManifest_File& fifo = + archive->files(subdir.children(0).index()); + EXPECT_EQ(0, fifo.children_size()); + EXPECT_TRUE(S_ISFIFO(fifo.mode())); + EXPECT_EQ(0, fifo.uid()); + EXPECT_EQ(0, fifo.gid()); + EXPECT_FALSE(fifo.has_data_format()); + EXPECT_FALSE(fifo.has_data_offset()); + EXPECT_FALSE(fifo.has_data_length()); + + const DeltaArchiveManifest_File& link = + archive->files(subdir.children(1).index()); + EXPECT_EQ(0, link.children_size()); + EXPECT_TRUE(S_ISLNK(link.mode())); + EXPECT_EQ(0, link.uid()); + EXPECT_EQ(0, link.gid()); + EXPECT_FALSE(link.has_data_format()); + EXPECT_FALSE(link.has_data_offset()); + EXPECT_FALSE(link.has_data_length()); +} + +TEST_F(DeltaDiffGeneratorTest, FakerootEncodeDataToDeltaFileTest) { + char cwd[1000]; + ASSERT_EQ(cwd, getcwd(cwd, sizeof(cwd))) << "cwd buf possibly too small"; + ASSERT_EQ(0, System(string("mkdir -p ") + cwd + "/diff-gen-test")); + ASSERT_EQ(0, System(string("mkdir -p ") + cwd + "/diff-gen-test/old")); + ASSERT_EQ(0, System(string("mkdir -p ") + cwd + "/diff-gen-test/new")); + GenerateFilesAtPath(string(cwd) + "/diff-gen-test/old"); + GenerateFilesAtPath(string(cwd) + "/diff-gen-test/new"); + EditFilesAtPath(string(cwd) + "/diff-gen-test/new"); + + DeltaArchiveManifest* archive = + DeltaDiffGenerator::EncodeMetadataToProtoBuffer( + (string(cwd) + "/diff-gen-test/new").c_str()); + EXPECT_TRUE(NULL != archive); + + EXPECT_TRUE(DeltaDiffGenerator::EncodeDataToDeltaFile( + archive, + string(cwd) + "/diff-gen-test/old", + string(cwd) + "/diff-gen-test/new", + string(cwd) + "/diff-gen-test/out.dat")); + + EXPECT_EQ(16, archive->files_size()); + + const DeltaArchiveManifest_File& root = archive->files(0); + EXPECT_TRUE(S_ISDIR(root.mode())); + EXPECT_EQ(0, root.uid()); + EXPECT_EQ(0, root.gid()); + ASSERT_EQ(4, root.children_size()); + EXPECT_EQ("cdev", root.children(0).name()); + EXPECT_EQ("dir", root.children(1).name()); + EXPECT_EQ("encoding", root.children(2).name()); + EXPECT_EQ("hi", root.children(3).name()); + EXPECT_FALSE(root.has_data_format()); + EXPECT_FALSE(root.has_data_offset()); + EXPECT_FALSE(root.has_data_length()); + + const DeltaArchiveManifest_File& cdev = + archive->files(root.children(0).index()); + EXPECT_EQ(0, cdev.children_size()); + EXPECT_TRUE(S_ISCHR(cdev.mode())); + EXPECT_EQ(0, cdev.uid()); + EXPECT_EQ(0, cdev.gid()); + ASSERT_TRUE(cdev.has_data_format()); + EXPECT_EQ(DeltaArchiveManifest_File_DataFormat_FULL, cdev.data_format()); + EXPECT_TRUE(cdev.has_data_offset()); + ASSERT_TRUE(cdev.has_data_length()); + EXPECT_GT(cdev.data_length(), 0); + + const DeltaArchiveManifest_File& hi = + archive->files(root.children(3).index()); + EXPECT_EQ(0, hi.children_size()); + EXPECT_TRUE(S_ISREG(hi.mode())); + EXPECT_EQ(0, hi.uid()); + EXPECT_EQ(0, hi.gid()); + ASSERT_TRUE(hi.has_data_format()); + EXPECT_EQ(DeltaArchiveManifest_File_DataFormat_FULL, hi.data_format()); + EXPECT_TRUE(hi.has_data_offset()); + ASSERT_TRUE(hi.has_data_length()); + EXPECT_GT(hi.data_length(), 0); + + const DeltaArchiveManifest_File& encoding = + archive->files(root.children(2).index()); + EXPECT_TRUE(S_ISDIR(encoding.mode())); + EXPECT_EQ(0, encoding.uid()); + EXPECT_EQ(0, encoding.gid()); + EXPECT_EQ(4, encoding.children_size()); + EXPECT_EQ("long_new", encoding.children(0).name()); + EXPECT_EQ("long_small_change", encoding.children(1).name()); + EXPECT_EQ("nochange", encoding.children(2).name()); + EXPECT_EQ("onebyte", encoding.children(3).name()); + EXPECT_FALSE(encoding.has_data_format()); + EXPECT_FALSE(encoding.has_data_offset()); + EXPECT_FALSE(encoding.has_data_length()); + + const DeltaArchiveManifest_File& long_new = + archive->files(encoding.children(0).index()); + EXPECT_EQ(0, long_new.children_size()); + EXPECT_TRUE(S_ISREG(long_new.mode())); + EXPECT_EQ(0, long_new.uid()); + EXPECT_EQ(0, long_new.gid()); + EXPECT_TRUE(long_new.has_data_format()); + EXPECT_EQ(DeltaArchiveManifest_File_DataFormat_FULL_GZ, + long_new.data_format()); + EXPECT_TRUE(long_new.has_data_offset()); + EXPECT_TRUE(long_new.has_data_length()); + + const DeltaArchiveManifest_File& long_small_change = + archive->files(encoding.children(1).index()); + EXPECT_EQ(0, long_small_change.children_size()); + EXPECT_TRUE(S_ISREG(long_small_change.mode())); + EXPECT_EQ(0, long_small_change.uid()); + EXPECT_EQ(0, long_small_change.gid()); + EXPECT_TRUE(long_small_change.has_data_format()); + EXPECT_EQ(DeltaArchiveManifest_File_DataFormat_BSDIFF, + long_small_change.data_format()); + EXPECT_TRUE(long_small_change.has_data_offset()); + EXPECT_TRUE(long_small_change.has_data_length()); + + const DeltaArchiveManifest_File& nochange = + archive->files(encoding.children(2).index()); + EXPECT_EQ(0, nochange.children_size()); + EXPECT_TRUE(S_ISREG(nochange.mode())); + EXPECT_EQ(0, nochange.uid()); + EXPECT_EQ(0, nochange.gid()); + EXPECT_TRUE(nochange.has_data_format()); + EXPECT_EQ(DeltaArchiveManifest_File_DataFormat_FULL, nochange.data_format()); + EXPECT_TRUE(nochange.has_data_offset()); + EXPECT_TRUE(nochange.has_data_length()); + + const DeltaArchiveManifest_File& onebyte = + archive->files(encoding.children(3).index()); + EXPECT_EQ(0, onebyte.children_size()); + EXPECT_TRUE(S_ISREG(onebyte.mode())); + EXPECT_EQ(0, onebyte.uid()); + EXPECT_EQ(0, onebyte.gid()); + EXPECT_TRUE(onebyte.has_data_format()); + EXPECT_EQ(DeltaArchiveManifest_File_DataFormat_FULL, onebyte.data_format()); + EXPECT_TRUE(onebyte.has_data_offset()); + EXPECT_TRUE(onebyte.has_data_length()); + EXPECT_EQ(1, onebyte.data_length()); + + const DeltaArchiveManifest_File& dir = + archive->files(root.children(1).index()); + EXPECT_TRUE(S_ISDIR(dir.mode())); + EXPECT_EQ(0, dir.uid()); + EXPECT_EQ(0, dir.gid()); + ASSERT_EQ(5, dir.children_size()); + EXPECT_EQ("bdev", dir.children(0).name()); + EXPECT_EQ("emptydir", dir.children(1).name()); + EXPECT_EQ("hello", dir.children(2).name()); + EXPECT_EQ("newempty", dir.children(3).name()); + EXPECT_EQ("subdir", dir.children(4).name()); + EXPECT_FALSE(dir.has_data_format()); + EXPECT_FALSE(dir.has_data_offset()); + EXPECT_FALSE(dir.has_data_length()); + + const DeltaArchiveManifest_File& bdev = + archive->files(dir.children(0).index()); + EXPECT_EQ(0, bdev.children_size()); + EXPECT_TRUE(S_ISBLK(bdev.mode())); + EXPECT_EQ(0, bdev.uid()); + EXPECT_EQ(0, bdev.gid()); + ASSERT_TRUE(bdev.has_data_format()); + EXPECT_EQ(DeltaArchiveManifest_File_DataFormat_FULL, bdev.data_format()); + EXPECT_TRUE(bdev.has_data_offset()); + ASSERT_TRUE(bdev.has_data_length()); + EXPECT_GT(bdev.data_length(), 0); + + const DeltaArchiveManifest_File& emptydir = + archive->files(dir.children(1).index()); + EXPECT_EQ(0, emptydir.children_size()); + EXPECT_TRUE(S_ISDIR(emptydir.mode())); + EXPECT_EQ(501, emptydir.uid()); + EXPECT_EQ(503, emptydir.gid()); + EXPECT_FALSE(emptydir.has_data_format()); + EXPECT_FALSE(emptydir.has_data_offset()); + EXPECT_FALSE(emptydir.has_data_length()); + + const DeltaArchiveManifest_File& hello = + archive->files(dir.children(2).index()); + EXPECT_EQ(0, hello.children_size()); + EXPECT_TRUE(S_ISREG(hello.mode())); + EXPECT_EQ(0, hello.uid()); + EXPECT_EQ(0, hello.gid()); + ASSERT_TRUE(hello.has_data_format()); + EXPECT_EQ(DeltaArchiveManifest_File_DataFormat_FULL, hello.data_format()); + EXPECT_TRUE(hello.has_data_offset()); + ASSERT_TRUE(hello.has_data_length()); + EXPECT_GT(hello.data_length(), 0); + + const DeltaArchiveManifest_File& newempty = + archive->files(dir.children(3).index()); + EXPECT_EQ(0, newempty.children_size()); + EXPECT_TRUE(S_ISREG(newempty.mode())); + EXPECT_EQ(0, newempty.uid()); + EXPECT_EQ(0, newempty.gid()); + EXPECT_FALSE(newempty.has_data_format()); + EXPECT_FALSE(newempty.has_data_offset()); + EXPECT_FALSE(newempty.has_data_length()); + + const DeltaArchiveManifest_File& subdir = + archive->files(dir.children(4).index()); + EXPECT_EQ(2, subdir.children_size()); + EXPECT_EQ("fifo", subdir.children(0).name()); + EXPECT_EQ("link", subdir.children(1).name()); + EXPECT_TRUE(S_ISDIR(subdir.mode())); + EXPECT_EQ(0, subdir.uid()); + EXPECT_EQ(0, subdir.gid()); + EXPECT_FALSE(subdir.has_data_format()); + EXPECT_FALSE(subdir.has_data_offset()); + EXPECT_FALSE(subdir.has_data_length()); + + const DeltaArchiveManifest_File& fifo = + archive->files(subdir.children(0).index()); + EXPECT_EQ(0, fifo.children_size()); + EXPECT_TRUE(S_ISFIFO(fifo.mode())); + EXPECT_EQ(0, fifo.uid()); + EXPECT_EQ(0, fifo.gid()); + EXPECT_FALSE(fifo.has_data_format()); + EXPECT_FALSE(fifo.has_data_offset()); + EXPECT_FALSE(fifo.has_data_length()); + + const DeltaArchiveManifest_File& link = + archive->files(subdir.children(1).index()); + EXPECT_EQ(0, link.children_size()); + EXPECT_TRUE(S_ISLNK(link.mode())); + EXPECT_EQ(0, link.uid()); + EXPECT_EQ(0, link.gid()); + ASSERT_TRUE(link.has_data_format()); + EXPECT_EQ(DeltaArchiveManifest_File_DataFormat_FULL, link.data_format()); + EXPECT_TRUE(link.has_data_offset()); + ASSERT_TRUE(link.has_data_length()); + EXPECT_GT(link.data_length(), 0); +} + +class DeltaDiffParserTest : public ::testing::Test { + virtual void TearDown() { + EXPECT_EQ(0, system("rm -rf diff-gen-test")); + } +}; + +namespace { +// Reads part of a file into memory +vector<char> ReadFilePart(const string& path, off_t start, off_t size) { + vector<char> ret; + int fd = open(path.c_str(), O_RDONLY, 0); + if (fd < 0) + return ret; + ret.resize(size); + EXPECT_EQ(size, pread(fd, &ret[0], size, start)); + close(fd); + return ret; +} + +string ReadFilePartToString(const string& path, off_t start, off_t size) { + vector<char> bytes = ReadFilePart(path, start, size); + string ret; + ret.append(&bytes[0], bytes.size()); + return ret; +} + +string StringFromVectorChar(const vector<char>& in) { + return string(&in[0], in.size()); +} + +string GzipDecompressToString(const vector<char>& in) { + vector<char> out; + EXPECT_TRUE(GzipDecompress(in, &out)); + return StringFromVectorChar(out); +} + +} + +TEST_F(DeltaDiffParserTest, FakerootDecodeDataFromDeltaFileTest) { + char cwd[1000]; + ASSERT_EQ(cwd, getcwd(cwd, sizeof(cwd))) << "cwd buf possibly too small"; + ASSERT_EQ(0, System(string("mkdir -p ") + cwd + "/diff-gen-test")); + ASSERT_EQ(0, System(string("mkdir -p ") + cwd + "/diff-gen-test/old")); + ASSERT_EQ(0, System(string("mkdir -p ") + cwd + "/diff-gen-test/new")); + GenerateFilesAtPath(string(cwd) + "/diff-gen-test/old"); + GenerateFilesAtPath(string(cwd) + "/diff-gen-test/new"); + EditFilesAtPath(string(cwd) + "/diff-gen-test/new"); + + DeltaArchiveManifest* archive = + DeltaDiffGenerator::EncodeMetadataToProtoBuffer( + (string(cwd) + "/diff-gen-test/new").c_str()); + EXPECT_TRUE(NULL != archive); + + EXPECT_TRUE(DeltaDiffGenerator::EncodeDataToDeltaFile( + archive, + string(cwd) + "/diff-gen-test/old", + string(cwd) + "/diff-gen-test/new", + string(cwd) + "/diff-gen-test/out.dat")); + // parse the file + + DeltaDiffParser parser(string(cwd) + "/diff-gen-test/out.dat"); + ASSERT_TRUE(parser.valid()); + DeltaDiffParser::Iterator it = parser.Begin(); + string expected_paths[] = { + "", + "/cdev", + "/dir", + "/dir/bdev", + "/dir/emptydir", + "/dir/hello", + "/dir/newempty", + "/dir/subdir", + "/dir/subdir/fifo", + "/dir/subdir/link", + "/encoding", + "/encoding/long_new", + "/encoding/long_small_change", + "/encoding/nochange", + "/encoding/onebyte", + "/hi" + }; + for (unsigned int i = 0; + i < (sizeof(expected_paths)/sizeof(expected_paths[0])); i++) { + ASSERT_TRUE(it != parser.End()); + ASSERT_TRUE(parser.ContainsPath(expected_paths[i])); + EXPECT_EQ(expected_paths[i], it.path()); + EXPECT_EQ(expected_paths[i].substr(expected_paths[i].find_last_of('/') + 1), + it.GetName()); + DeltaArchiveManifest_File f1 = parser.GetFileAtPath(expected_paths[i]); + DeltaArchiveManifest_File f2 = it.GetFile(); + EXPECT_EQ(f1.mode(), f2.mode()) << it.path(); + EXPECT_EQ(f1.uid(), f2.uid()); + EXPECT_EQ(f1.gid(), f2.gid()); + EXPECT_EQ(f1.has_data_format(), f2.has_data_format()); + if (f1.has_data_format()) { + EXPECT_EQ(f1.data_format(), f2.data_format()); + EXPECT_TRUE(f1.has_data_offset()); + EXPECT_TRUE(f2.has_data_offset()); + EXPECT_EQ(f1.data_offset(), f2.data_offset()); + } else { + EXPECT_FALSE(f2.has_data_format()); + EXPECT_FALSE(f1.has_data_offset()); + EXPECT_FALSE(f2.has_data_offset()); + } + EXPECT_EQ(f1.children_size(), f2.children_size()); + for (int j = 0; j < f1.children_size(); j++) { + EXPECT_EQ(f1.children(j).name(), f2.children(j).name()); + EXPECT_EQ(f1.children(j).index(), f2.children(j).index()); + } + it.Increment(); + } + EXPECT_TRUE(it == parser.End()); + EXPECT_FALSE(parser.ContainsPath("/cdew")); + EXPECT_FALSE(parser.ContainsPath("/hi/hi")); + EXPECT_FALSE(parser.ContainsPath("/dir/newempty/hi")); + EXPECT_TRUE(parser.ContainsPath("/dir/")); + + // Check the data + // root + DeltaArchiveManifest_File file = parser.GetFileAtPath(""); + EXPECT_TRUE(S_ISDIR(file.mode())); + EXPECT_FALSE(file.has_data_format()); + + // cdev + file = parser.GetFileAtPath("/cdev"); + EXPECT_TRUE(S_ISCHR(file.mode())); + EXPECT_TRUE(file.has_data_format()); + vector<char> data = ReadFilePart(string(cwd) + "/diff-gen-test/out.dat", + file.data_offset(), file.data_length()); + LinuxDevice linux_device; + linux_device.ParseFromArray(&data[0], data.size()); + EXPECT_EQ(linux_device.major(), 2); + EXPECT_EQ(linux_device.minor(), 1); + + // dir + file = parser.GetFileAtPath("/dir"); + EXPECT_TRUE(S_ISDIR(file.mode())); + EXPECT_FALSE(file.has_data_format()); + + // bdev + file = parser.GetFileAtPath("/dir/bdev"); + EXPECT_TRUE(S_ISBLK(file.mode())); + EXPECT_TRUE(file.has_data_format()); + data = ReadFilePart(string(cwd) + "/diff-gen-test/out.dat", + file.data_offset(), file.data_length()); + linux_device.ParseFromArray(&data[0], data.size()); + EXPECT_EQ(linux_device.major(), 3); + EXPECT_EQ(linux_device.minor(), 1); + + // emptydir + file = parser.GetFileAtPath("/dir/emptydir"); + EXPECT_TRUE(S_ISDIR(file.mode())); + EXPECT_FALSE(file.has_data_format()); + + // hello + file = parser.GetFileAtPath("/dir/hello"); + EXPECT_TRUE(S_ISREG(file.mode())); + EXPECT_TRUE(file.has_data_format()); + EXPECT_EQ(DeltaArchiveManifest_File_DataFormat_FULL, file.data_format()); + EXPECT_EQ("hello\n", ReadFilePartToString(string(cwd) + + "/diff-gen-test/out.dat", + file.data_offset(), + file.data_length())); + + // newempty + file = parser.GetFileAtPath("/dir/newempty"); + EXPECT_TRUE(S_ISREG(file.mode())); + EXPECT_FALSE(file.has_data_format()); + + // subdir + file = parser.GetFileAtPath("/dir/subdir"); + EXPECT_TRUE(S_ISDIR(file.mode())); + EXPECT_FALSE(file.has_data_format()); + + // fifo + file = parser.GetFileAtPath("/dir/subdir/fifo"); + EXPECT_TRUE(S_ISFIFO(file.mode())); + EXPECT_FALSE(file.has_data_format()); + + // link + file = parser.GetFileAtPath("/dir/subdir/link"); + EXPECT_TRUE(S_ISLNK(file.mode())); + EXPECT_TRUE(file.has_data_format()); + EXPECT_EQ(DeltaArchiveManifest_File_DataFormat_FULL, file.data_format()); + EXPECT_EQ("/target", ReadFilePartToString(string(cwd) + + "/diff-gen-test/out.dat", + file.data_offset(), + file.data_length())); + + // encoding + file = parser.GetFileAtPath("/encoding"); + EXPECT_TRUE(S_ISDIR(file.mode())); + EXPECT_FALSE(file.has_data_format()); + + // long_new + file = parser.GetFileAtPath("/encoding/long_new"); + EXPECT_TRUE(S_ISREG(file.mode())); + EXPECT_TRUE(file.has_data_format()); + EXPECT_EQ(DeltaArchiveManifest_File_DataFormat_FULL_GZ, file.data_format()); + EXPECT_EQ("This is a pice of text that should " + "compress well since it is just ascii and it " + "has repetition xxxxxxxxxxxxxxxxxxxxx" + "xxxxxxxxxxxxxxxxxxxx", + GzipDecompressToString(ReadFilePart(string(cwd) + + "/diff-gen-test/out.dat", + file.data_offset(), + file.data_length()))); + + // long_small_change + file = parser.GetFileAtPath("/encoding/long_small_change"); + EXPECT_TRUE(S_ISREG(file.mode())); + EXPECT_TRUE(file.has_data_format()); + EXPECT_EQ(DeltaArchiveManifest_File_DataFormat_BSDIFF, file.data_format()); + data = ReadFilePart(string(cwd) + "/diff-gen-test/out.dat", + file.data_offset(), file.data_length()); + WriteFileVector(string(cwd) + "/diff-gen-test/patch", data); + int rc = 1; + vector<string> cmd; + cmd.push_back("/usr/bin/bspatch"); + cmd.push_back(string(cwd) + "/diff-gen-test/old/encoding/long_small_change"); + cmd.push_back(string(cwd) + "/diff-gen-test/patch_result"); + cmd.push_back(string(cwd) + "/diff-gen-test/patch"); + Subprocess::SynchronousExec(cmd, &rc); + EXPECT_EQ(0, rc); + vector<char> patch_result; + EXPECT_TRUE(utils::ReadFile(string(cwd) + "/diff-gen-test/patch_result", + &patch_result)); + vector<char> expected_data(sizeof(kRandomString) + 1); + memcpy(&expected_data[0], kRandomString, sizeof(kRandomString)); + expected_data[expected_data.size() - 1] = 'h'; + ExpectVectorsEq(expected_data, patch_result); + + // nochange + file = parser.GetFileAtPath("/encoding/nochange"); + EXPECT_TRUE(S_ISREG(file.mode())); + EXPECT_TRUE(file.has_data_format()); + EXPECT_EQ(DeltaArchiveManifest_File_DataFormat_FULL, file.data_format()); + EXPECT_EQ("nochange\n", ReadFilePartToString(string(cwd) + + "/diff-gen-test/out.dat", + file.data_offset(), + file.data_length())); + + // onebyte + file = parser.GetFileAtPath("/encoding/onebyte"); + EXPECT_TRUE(S_ISREG(file.mode())); + EXPECT_TRUE(file.has_data_format()); + EXPECT_EQ(DeltaArchiveManifest_File_DataFormat_FULL, file.data_format()); + EXPECT_EQ("h", ReadFilePartToString(string(cwd) + + "/diff-gen-test/out.dat", + file.data_offset(), + file.data_length())); + + // hi + file = parser.GetFileAtPath("/hi"); + EXPECT_TRUE(S_ISREG(file.mode())); + EXPECT_TRUE(file.has_data_format()); + EXPECT_EQ(DeltaArchiveManifest_File_DataFormat_FULL, file.data_format()); + EXPECT_EQ("newhi\n", ReadFilePartToString(string(cwd) + + "/diff-gen-test/out.dat", + file.data_offset(), + file.data_length())); +} + +TEST_F(DeltaDiffParserTest, FakerootInvalidTest) { + ASSERT_EQ(0, mkdir("diff-gen-test", 0777)); + { + DeltaDiffParser parser("/no/such/file"); + EXPECT_FALSE(parser.valid()); + } + { + vector<char> data(3); + memcpy(&data[0], "CrA", 3); + WriteFileVector("diff-gen-test/baddelta", data); + DeltaDiffParser parser("diff-gen-test/baddelta"); + EXPECT_FALSE(parser.valid()); + } + { + vector<char> data(5); + memcpy(&data[0], "CrAPx", 5); + WriteFileVector("diff-gen-test/baddelta", data); + DeltaDiffParser parser("diff-gen-test/baddelta"); + EXPECT_FALSE(parser.valid()); + } + { + vector<char> data(5); + memcpy(&data[0], "CrAU\0", 5); + WriteFileVector("diff-gen-test/baddelta", data); + DeltaDiffParser parser("diff-gen-test/baddelta"); + EXPECT_FALSE(parser.valid()); + } + { + vector<char> data(14); + memcpy(&data[0], "CrAU\0\0\0\0\0\0\0\x0cxx", 12); + WriteFileVector("diff-gen-test/baddelta", data); + DeltaDiffParser parser("diff-gen-test/baddelta"); + EXPECT_FALSE(parser.valid()); + } +} + +} // namespace chromeos_update_engine diff --git a/delta_diff_parser.cc b/delta_diff_parser.cc new file mode 100644 index 00000000..47c763f5 --- /dev/null +++ b/delta_diff_parser.cc @@ -0,0 +1,272 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "update_engine/delta_diff_parser.h" +#include <sys/types.h> +#include <sys/stat.h> +#include <endian.h> +#include <errno.h> +#include <fcntl.h> +#include <stdio.h> +#include <unistd.h> +#include <algorithm> +#include <string> +#include <vector> +#include <google/protobuf/io/zero_copy_stream_impl.h> +#include "base/scoped_ptr.h" +#include "update_engine/decompressing_file_writer.h" +#include "update_engine/gzip.h" +#include "update_engine/utils.h" + +using std::min; +using std::string; +using std::vector; + +namespace chromeos_update_engine { + +namespace { +const int kCopyFileBufferSize = 4096; +} + +const char* const DeltaDiffParser::kFileMagic("CrAU"); + +// The iterator returns a directory before returning its children. +// Steps taken in Increment(): +// - See if the current item has children. If so, the child becomes +// the new current item and we return. +// - If current item has no children, we loop. Each loop iteration +// considers an item (first the current item, then its parent, +// then grand parent, and so on). Each loop iteration, we see if there +// are any siblings we haven't iterated on yet. If so, we're done. +// If not, keep looping to parents. +void DeltaDiffParserIterator::Increment() { + // See if we have any children. + const DeltaArchiveManifest_File& file = GetFile(); + if (file.children_size() > 0) { + path_indices_.push_back(file.children(0).index()); + path_ += "/"; + path_ += file.children(0).name(); + child_indices_.push_back(0); + return; + } + // Look in my parent for the next child, then try grandparent, etc. + + path_indices_.pop_back(); + path_.resize(path_.rfind('/')); + + while (!child_indices_.empty()) { + // Try to bump the last entry + CHECK_EQ(path_indices_.size(), child_indices_.size()); + child_indices_.back()++; + const DeltaArchiveManifest_File& parent = + archive_->files(path_indices_.back()); + if (parent.children_size() > child_indices_.back()) { + // we found a new child! + path_indices_.push_back(parent.children(child_indices_.back()).index()); + path_ += "/"; + path_ += parent.children(child_indices_.back()).name(); + return; + } + path_indices_.pop_back(); + child_indices_.pop_back(); + if (!path_.empty()) + path_.resize(path_.rfind('/')); + } +} + +const string DeltaDiffParserIterator::GetName() const { + if (path_.empty()) + return ""; + CHECK_NE(path_.rfind('/'), string::npos); + return string(path_, path_.rfind('/') + 1); +} + +const DeltaArchiveManifest_File& DeltaDiffParserIterator::GetFile() const { + CHECK(!path_indices_.empty()); + return archive_->files(path_indices_.back()); +} + + +DeltaDiffParser::DeltaDiffParser(const string& delta_file) + : fd_(-1), + valid_(false) { + fd_ = open(delta_file.c_str(), O_RDONLY, 0); + if (fd_ < 0) { + LOG(ERROR) << "Unable to open delta file: " << delta_file; + return; + } + ScopedFdCloser fd_closer(&fd_); + scoped_array<char> magic(new char[strlen(kFileMagic)]); + if (strlen(kFileMagic) != read(fd_, magic.get(), strlen(kFileMagic))) { + LOG(ERROR) << "delta file too short"; + return; + } + if (strncmp(magic.get(), kFileMagic, strlen(kFileMagic))) { + LOG(ERROR) << "Incorrect magic at beginning of delta file"; + return; + } + + int64 proto_offset = 0; + COMPILE_ASSERT(sizeof(proto_offset) == sizeof(off_t), off_t_wrong_size); + if (sizeof(proto_offset) != read(fd_, &proto_offset, sizeof(proto_offset))) { + LOG(ERROR) << "delta file too short"; + return; + } + proto_offset = be64toh(proto_offset); // switch from big-endian to host + + int64 proto_length = 0; + if (sizeof(proto_length) != read(fd_, &proto_length, sizeof(proto_length))) { + LOG(ERROR) << "delta file too short"; + return; + } + proto_length = be64toh(proto_length); // switch from big-endian to host + + vector<char> proto(proto_length); + size_t bytes_read = 0; + while (bytes_read < proto_length) { + ssize_t r = pread(fd_, &proto[bytes_read], proto_length - bytes_read, + proto_offset + bytes_read); + TEST_AND_RETURN(r >= 0); + bytes_read += r; + } + { + vector<char> decompressed_proto; + TEST_AND_RETURN(GzipDecompress(proto, &decompressed_proto)); + proto.swap(decompressed_proto); + } + + valid_ = archive_.ParseFromArray(&proto[0], proto.size()); + if (valid_) { + fd_closer.set_should_close(false); + } else { + LOG(ERROR) << "load from file failed"; + } +} + +DeltaDiffParser::~DeltaDiffParser() { + if (fd_ >= 0) { + close(fd_); + fd_ = -1; + } +} + +bool DeltaDiffParser::ContainsPath(const string& path) const { + return GetIndexForPath(path) >= 0; +} + +const DeltaArchiveManifest_File& DeltaDiffParser::GetFileAtPath( + const string& path) const { + int idx = GetIndexForPath(path); + CHECK_GE(idx, 0) << path; + return archive_.files(idx); +} + +// Returns -1 if not found. +int DeltaDiffParser::GetIndexOfFileChild( + const DeltaArchiveManifest_File& file, const string& child_name) const { + if (file.children_size() == 0) + return -1; + int begin = 0; + int end = file.children_size(); + while (begin < end) { + int middle = (begin + end) / 2; + const string& middle_name = file.children(middle).name(); + int cmp_result = strcmp(middle_name.c_str(), child_name.c_str()); + if (cmp_result == 0) + return file.children(middle).index(); + if (cmp_result < 0) + begin = middle + 1; + else + end = middle; + } + return -1; +} + +// Converts a path to an index in archive_. It does this by separating +// the path components and going from root to leaf, finding the +// File message for each component. Index values for children are +// stored in File messages. +int DeltaDiffParser::GetIndexForPath(const string& path) const { + string cleaned_path = utils::NormalizePath(path, true); + // strip leading slash + if (cleaned_path[0] == '/') + cleaned_path = cleaned_path.c_str() + 1; + if (cleaned_path.empty()) + return 0; + string::size_type begin = 0; + string::size_type end = cleaned_path.find_first_of('/', begin + 1); + const DeltaArchiveManifest_File* file = &archive_.files(0); + int file_idx = -1; + for (;;) { + string component = cleaned_path.substr(begin, end - begin); + if (component.empty()) + break; + // search for component in 'file' + file_idx = GetIndexOfFileChild(*file, component); + if (file_idx < 0) + return file_idx; + file = &archive_.files(file_idx); + if (end == string::npos) + break; + begin = end + 1; + end = cleaned_path.find_first_of('/', begin + 1); + } + return file_idx; +} + +bool DeltaDiffParser::ReadDataVector(off_t offset, off_t length, + std::vector<char>* out) const { + out->resize(static_cast<vector<char>::size_type>(length)); + int r = pread(fd_, &((*out)[0]), length, offset); + TEST_AND_RETURN_FALSE_ERRNO(r >= 0); + return true; +} + +bool DeltaDiffParser::CopyDataToFile(off_t offset, off_t length, + bool should_decompress, + const std::string& path) const { + DirectFileWriter direct_writer; + GzipDecompressingFileWriter decompressing_writer(&direct_writer); + FileWriter* writer = NULL; // will point to one of the two writers above + + writer = (should_decompress ? + static_cast<FileWriter*>(&decompressing_writer) : + static_cast<FileWriter*>(&direct_writer)); + ScopedFileWriterCloser closer(writer); + + int r = writer->Open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0644); + TEST_AND_RETURN_FALSE(r == 0); + + off_t bytes_transferred = 0; + + while (bytes_transferred < length) { + char buf[kCopyFileBufferSize]; + size_t bytes_to_read = min(length - bytes_transferred, + static_cast<off_t>(sizeof(buf))); + ssize_t bytes_read = pread(fd_, buf, bytes_to_read, + offset + bytes_transferred); + if (bytes_read == 0) + break; // EOF + TEST_AND_RETURN_FALSE_ERRNO(bytes_read > 0); + int bytes_written = writer->Write(buf, bytes_read); + TEST_AND_RETURN_FALSE(bytes_written == bytes_read); + bytes_transferred += bytes_written; + } + TEST_AND_RETURN_FALSE(bytes_transferred == length); + LOG_IF(ERROR, bytes_transferred > length) << "Wrote too many bytes(?)"; + return true; +} + + +const DeltaDiffParser::Iterator DeltaDiffParser::Begin() { + DeltaDiffParserIterator ret(&archive_); + ret.path_indices_.push_back(0); + return ret; +} + +const DeltaDiffParser::Iterator DeltaDiffParser::End() { + return DeltaDiffParserIterator(&archive_); +} + +} // namespace chromeos_update_engine diff --git a/delta_diff_parser.h b/delta_diff_parser.h new file mode 100644 index 00000000..5c6d664f --- /dev/null +++ b/delta_diff_parser.h @@ -0,0 +1,132 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CHROMEOS_PLATFORM_UPDATE_ENGINE_DELTA_DIFF_PARSER_H__ +#define CHROMEOS_PLATFORM_UPDATE_ENGINE_DELTA_DIFF_PARSER_H__ + +#include <string> +#include <vector> +#include "chromeos/obsolete_logging.h" +#include "base/basictypes.h" +#include "update_engine/update_metadata.pb.h" + +// The DeltaDiffParser class is used to parse a delta file on disk. It will +// copy the metadata into memory, but not the file data. This class can +// also be used to copy file data out to disk. + +// The DeltaDiffParserIterator class is used to iterate through the +// metadata of a delta file. It will return directories before their +// children. + +namespace chromeos_update_engine { + +class DeltaDiffParser; + +class DeltaDiffParserIterator { + friend class DeltaDiffParser; + public: + void Increment(); + + // Returns the full path for the current file, e.g. "/bin/bash". + // Returns empty string for root. + const std::string& path() const { + return path_; + } + + // Returns the basename for the current file. If path() returns + // "/bin/bash", then GetName() returns "bash". + // Returns empty string for root + const std::string GetName() const; + + const DeltaArchiveManifest_File& GetFile() const; + bool operator==(const DeltaDiffParserIterator& that) const { + return path_indices_ == that.path_indices_ && + child_indices_ == that.child_indices_ && + path_ == that.path_ && + archive_ == that.archive_; + } + bool operator!=(const DeltaDiffParserIterator& that) const { + return !(*this == that); + } + private: + // Container of all the File messages. Each File message has an index + // in archive_. The root directory is always stored at index 0. + const DeltaArchiveManifest* archive_; + + // These variables are used to implement the common recursive depth-first + // search algorithm (which we can't use here, since we need to walk the + // tree incrementally). + + // Indices into 'archive_' of the current path components. For example, if + // the current path is "/bin/bash", 'path_stack_' will contain the archive + // indices for "/", "/bin", and "/bin/bash", in that order. This is + // analogous to the call stack of the recursive algorithm. + std::vector<int> path_indices_; + + // For each component in 'path_stack_', the currently-selected child in its + // child vector. In the previous example, if "/" has "abc" and "bin" + // subdirectories and "/bin" contains only "bash", this will contain + // [0, 1, 0], since we are using the 0th child at the root directory level + // (there's only one child there), the first of the root dir's children + // ("bin"), and the 0th child of /bin ("bash"). This is analogous to the + // state of each function (in terms of which child it's currently + // handling) in the call stack of the recursive algorithm. + std::vector<int> child_indices_; + + std::string path_; + // Instantiated by friend class DeltaDiffParser + explicit DeltaDiffParserIterator(const DeltaArchiveManifest* archive) + : archive_(archive) {} + DeltaDiffParserIterator() { + CHECK(false); // Should never be called. + } +}; + +class DeltaDiffParser { + public: + DeltaDiffParser(const std::string& delta_file); + ~DeltaDiffParser(); + bool valid() const { return valid_; } + bool ContainsPath(const std::string& path) const; + const DeltaArchiveManifest_File& GetFileAtPath(const std::string& path) const; + + // Reads length bytes at offset of the delta file into the out string + // or vector. Be careful not to call this with large length values, + // since that much memory will have to be allocated to store the output. + // Returns true on success. + bool ReadDataVector(off_t offset, off_t length, std::vector<char>* out) const; + + // Copies length bytes of data from offset into a new file at path specified. + // If should_decompress is true, will gzip decompress while writing to the + // file. Returns true on success. + bool CopyDataToFile(off_t offset, off_t length, bool should_decompress, + const std::string& path) const; + + typedef DeltaDiffParserIterator Iterator; + const Iterator Begin(); + const Iterator End(); + + // The identifier we expect at the beginning of a delta file. + static const char* const kFileMagic; + + private: + // (Binary) Searches the children of 'file' for one named child_name. + // If found, returns the index into the archive. If not found, returns -1. + int GetIndexOfFileChild(const DeltaArchiveManifest_File& file, + const std::string& child_name) const; + + // Returns -1 if not found, 0 for root + int GetIndexForPath(const std::string& path) const; + + // We keep a filedescriptor open to the delta file. + int fd_; + + DeltaArchiveManifest archive_; + bool valid_; + DISALLOW_COPY_AND_ASSIGN(DeltaDiffParser); +}; + +}; // namespace chromeos_update_engine + +#endif // CHROMEOS_PLATFORM_UPDATE_ENGINE_DELTA_DIFF_PARSER_H__ diff --git a/delta_diff_parser_unittest.cc b/delta_diff_parser_unittest.cc new file mode 100644 index 00000000..241a8349 --- /dev/null +++ b/delta_diff_parser_unittest.cc @@ -0,0 +1,2 @@ +// Due to shared code, DeltaDiffParser is tested in +// delta_diff_generator_unittest.cc diff --git a/file_writer.cc b/file_writer.cc new file mode 100644 index 00000000..07bdb464 --- /dev/null +++ b/file_writer.cc @@ -0,0 +1,47 @@ +// Copyright (c) 2009 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "update_engine/file_writer.h" +#include <errno.h> + +namespace chromeos_update_engine { + +int DirectFileWriter::Open(const char* path, int flags, mode_t mode) { + CHECK_EQ(fd_, -1); + fd_ = open(path, flags, mode); + if (fd_ < 0) + return -errno; + return 0; +} + +int DirectFileWriter::Write(const void* bytes, size_t count) { + CHECK_GE(fd_, 0); + const char* char_bytes = reinterpret_cast<const char*>(bytes); + + size_t bytes_written = 0; + while (bytes_written < count) { + ssize_t rc = write(fd_, char_bytes + bytes_written, + count - bytes_written); + if (rc < 0) + return -errno; + bytes_written += rc; + } + CHECK_EQ(bytes_written, count); + return bytes_written; +} + +int DirectFileWriter::Close() { + CHECK_GE(fd_, 0); + int rc = close(fd_); + + // This can be any negative number that's not -1. This way, this FileWriter + // won't be used again for another file. + fd_ = -2; + + if (rc < 0) + return -errno; + return rc; +} + +} // namespace chromeos_update_engine diff --git a/filesystem_copier_action.cc b/filesystem_copier_action.cc new file mode 100644 index 00000000..875df950 --- /dev/null +++ b/filesystem_copier_action.cc @@ -0,0 +1,305 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "update_engine/filesystem_copier_action.h" +#include <sys/stat.h> +#include <sys/types.h> +#include <errno.h> +#include <fcntl.h> +#include <stdlib.h> +#include <algorithm> +#include <string> +#include <vector> +#include "update_engine/filesystem_iterator.h" +#include "update_engine/subprocess.h" +#include "update_engine/utils.h" + +using std::min; +using std::string; +using std::vector; + +namespace chromeos_update_engine { + +namespace { +const char* kMountpointTemplate = "/tmp/au_dest_mnt.XXXXXX"; +const off_t kCopyFileBufferSize = 4 * 1024 * 1024; +const char* kCopyExclusionPrefix = "/lost+found"; +} // namespace {} + +void FilesystemCopierAction::PerformAction() { + if (!HasInputObject()) { + LOG(ERROR) << "No input object. Aborting."; + processor_->ActionComplete(this, false); + return; + } + install_plan_ = GetInputObject(); + + if (install_plan_.is_full_update) { + // No copy needed. + processor_->ActionComplete(this, true); + return; + } + + { + // Set up dest_path_ + char *dest_path_temp = strdup(kMountpointTemplate); + CHECK(dest_path_temp); + CHECK_EQ(mkdtemp(dest_path_temp), dest_path_temp); + CHECK_NE(dest_path_temp[0], '\0'); + dest_path_ = dest_path_temp; + free(dest_path_temp); + } + + // Make sure we're properly mounted + if (Mount(install_plan_.install_path, dest_path_)) { + bool done_early = false; + if (utils::FileExists( + (dest_path_ + + FilesystemCopierAction::kCompleteFilesystemMarker).c_str())) { + // We're done! + done_early = true; + skipped_copy_ = true; + if (HasOutputPipe()) + SetOutputObject(install_plan_); + } + if (!Unmount(dest_path_)) { + LOG(ERROR) << "Unmount failed. Aborting."; + processor_->ActionComplete(this, false); + return; + } + if (done_early) { + CHECK(!is_mounted_); + if (rmdir(dest_path_.c_str()) != 0) + LOG(ERROR) << "Unable to remove " << dest_path_; + processor_->ActionComplete(this, true); + return; + } + } + LOG(ERROR) << "not mounted; spawning thread"; + // If we get here, mount failed or we're not done yet. Reformat and copy. + CHECK_EQ(pthread_create(&helper_thread_, NULL, HelperThreadMainStatic, this), + 0); +} + +void FilesystemCopierAction::TerminateProcessing() { + if (is_mounted_) { + LOG(ERROR) << "Aborted processing, but left a filesystem mounted."; + } +} + +bool FilesystemCopierAction::Mount(const string& device, + const string& mountpoint) { + CHECK(!is_mounted_); + if(utils::MountFilesystem(device, mountpoint)) + is_mounted_ = true; + return is_mounted_; +} + +bool FilesystemCopierAction::Unmount(const string& mountpoint) { + CHECK(is_mounted_); + if (utils::UnmountFilesystem(mountpoint)) + is_mounted_ = false; + return !is_mounted_; +} + +void* FilesystemCopierAction::HelperThreadMain() { + // First, format the drive + vector<string> cmd; + cmd.push_back("/sbin/mkfs.ext3"); + cmd.push_back("-F"); + cmd.push_back(install_plan_.install_path); + int return_code = 1; + bool success = Subprocess::SynchronousExec(cmd, &return_code); + if (return_code != 0) { + LOG(INFO) << "Format of " << install_plan_.install_path + << " failed. Exit code: " << return_code; + success = false; + } + if (success) { + if (!Mount(install_plan_.install_path, dest_path_)) { + LOG(ERROR) << "Mount failed. Aborting"; + success = false; + } + } + if (success) { + success = CopySynchronously(); + } + if (success) { + // Place our marker to avoid copies again in the future + int r = open((dest_path_ + + FilesystemCopierAction::kCompleteFilesystemMarker).c_str(), + O_CREAT | O_WRONLY, 0644); + if (r >= 0) + close(r); + } + // Unmount + if (!Unmount(dest_path_)) { + LOG(ERROR) << "Unmount failed. Aborting"; + success = false; + } + if (HasOutputPipe()) + SetOutputObject(install_plan_); + + // Tell main thread that we're done + g_timeout_add(0, CollectThreadStatic, this); + return reinterpret_cast<void*>(success ? 0 : 1); +} + +void FilesystemCopierAction::CollectThread() { + void *thread_ret_value = NULL; + CHECK_EQ(pthread_join(helper_thread_, &thread_ret_value), 0); + bool success = (thread_ret_value == 0); + CHECK(!is_mounted_); + if (rmdir(dest_path_.c_str()) != 0) + LOG(INFO) << "Unable to remove " << dest_path_; + LOG(INFO) << "FilesystemCopierAction done"; + processor_->ActionComplete(this, success); +} + +bool FilesystemCopierAction::CreateDirSynchronously(const std::string& new_path, + const struct stat& stbuf) { + int r = mkdir(new_path.c_str(), stbuf.st_mode); + TEST_AND_RETURN_FALSE_ERRNO(r == 0); + return true; +} + +bool FilesystemCopierAction::CopyFileSynchronously(const std::string& old_path, + const std::string& new_path, + const struct stat& stbuf) { + int fd_out = open(new_path.c_str(), O_CREAT | O_EXCL | O_WRONLY, + stbuf.st_mode); + TEST_AND_RETURN_FALSE_ERRNO(fd_out >= 0); + ScopedFdCloser fd_out_closer(&fd_out); + int fd_in = open(old_path.c_str(), O_RDONLY, 0); + TEST_AND_RETURN_FALSE_ERRNO(fd_in >= 0); + ScopedFdCloser fd_in_closer(&fd_in); + + vector<char> buf(min(kCopyFileBufferSize, stbuf.st_size)); + off_t bytes_written = 0; + while (true) { + // Make sure we don't need to abort early: + TEST_AND_RETURN_FALSE(!g_atomic_int_get(&thread_should_exit_)); + + ssize_t read_size = read(fd_in, &buf[0], buf.size()); + TEST_AND_RETURN_FALSE_ERRNO(read_size >= 0); + if (0 == read_size) // EOF + break; + + ssize_t write_size = 0; + while (write_size < read_size) { + ssize_t r = write(fd_out, &buf[write_size], read_size - write_size); + TEST_AND_RETURN_FALSE_ERRNO(r >= 0); + write_size += r; + } + CHECK_EQ(write_size, read_size); + bytes_written += write_size; + CHECK_LE(bytes_written, stbuf.st_size); + if (bytes_written == stbuf.st_size) + break; + } + CHECK_EQ(bytes_written, stbuf.st_size); + return true; +} + +bool FilesystemCopierAction::CreateHardLinkSynchronously( + const std::string& old_path, + const std::string& new_path) { + int r = link(old_path.c_str(), new_path.c_str()); + TEST_AND_RETURN_FALSE_ERRNO(r == 0); + return true; +} + +bool FilesystemCopierAction::CopySymlinkSynchronously( + const std::string& old_path, + const std::string& new_path, + const struct stat& stbuf) { + vector<char> buf(PATH_MAX + 1); + ssize_t r = readlink(old_path.c_str(), &buf[0], buf.size()); + TEST_AND_RETURN_FALSE_ERRNO(r >= 0); + // Make sure we got the entire link + TEST_AND_RETURN_FALSE(static_cast<unsigned>(r) < buf.size()); + buf[r] = '\0'; + int rc = symlink(&buf[0], new_path.c_str()); + TEST_AND_RETURN_FALSE_ERRNO(rc == 0); + return true; +} + +bool FilesystemCopierAction::CreateNodeSynchronously( + const std::string& new_path, + const struct stat& stbuf) { + int r = mknod(new_path.c_str(), stbuf.st_mode, stbuf.st_rdev); + TEST_AND_RETURN_FALSE_ERRNO(r == 0); + return true; +} + +// Returns true on success +bool FilesystemCopierAction::CopySynchronously() { + // This map is a map from inode # to new_path. + map<ino_t, string> hard_links; + FilesystemIterator iter(copy_source_, + utils::SetWithValue<string>(kCopyExclusionPrefix)); + bool success = true; + for (; !g_atomic_int_get(&thread_should_exit_) && + !iter.IsEnd(); iter.Increment()) { + const string old_path = iter.GetFullPath(); + const string new_path = dest_path_ + iter.GetPartialPath(); + LOG(INFO) << "copying " << old_path << " to " << new_path; + const struct stat stbuf = iter.GetStat(); + success = false; + + // Skip lost+found + CHECK_NE(kCopyExclusionPrefix, iter.GetPartialPath()); + + // Directories can't be hard-linked, so check for directories first + if (iter.GetPartialPath().empty()) { + // Root has an empty path. + // We don't need to create anything for the root, which is the first + // thing we get from the iterator. + success = true; + } else if (S_ISDIR(stbuf.st_mode)) { + success = CreateDirSynchronously(new_path, stbuf); + } else { + if (stbuf.st_nlink > 1 && + utils::MapContainsKey(hard_links, stbuf.st_ino)) { + success = CreateHardLinkSynchronously(hard_links[stbuf.st_ino], + new_path); + } else { + if (stbuf.st_nlink > 1) + hard_links[stbuf.st_ino] = new_path; + if (S_ISREG(stbuf.st_mode)) { + success = CopyFileSynchronously(old_path, new_path, stbuf); + } else if (S_ISLNK(stbuf.st_mode)) { + success = CopySymlinkSynchronously(old_path, new_path, stbuf); + } else if (S_ISFIFO(stbuf.st_mode) || + S_ISCHR(stbuf.st_mode) || + S_ISBLK(stbuf.st_mode) || + S_ISSOCK(stbuf.st_mode)) { + success = CreateNodeSynchronously(new_path, stbuf); + } else { + CHECK(false) << "Unable to copy file " << old_path << " with mode " + << stbuf.st_mode; + } + } + } + TEST_AND_RETURN_FALSE(success); + + // chmod new file + if (!S_ISLNK(stbuf.st_mode)) { + int r = chmod(new_path.c_str(), stbuf.st_mode); + TEST_AND_RETURN_FALSE_ERRNO(r == 0); + } + + // Set uid/gid. + int r = lchown(new_path.c_str(), stbuf.st_uid, stbuf.st_gid); + TEST_AND_RETURN_FALSE_ERRNO(r == 0); + } + TEST_AND_RETURN_FALSE(!iter.IsErr()); + // Success! + return true; +} + +const char* FilesystemCopierAction::kCompleteFilesystemMarker( + "/update_engine_copy_success"); + +} // namespace chromeos_update_engine diff --git a/filesystem_copier_action.h b/filesystem_copier_action.h new file mode 100644 index 00000000..8f0dc06d --- /dev/null +++ b/filesystem_copier_action.h @@ -0,0 +1,147 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CHROMEOS_PLATFORM_UPDATE_ENGINE_FILESYSTEM_COPIER_ACTION_H__ +#define CHROMEOS_PLATFORM_UPDATE_ENGINE_FILESYSTEM_COPIER_ACTION_H__ + +#include <sys/stat.h> +#include <sys/types.h> +#include <string> +#include <glib.h> +#include "update_engine/action.h" +#include "update_engine/install_plan.h" + +// This action will only do real work if it's a delta update. It will +// format the install partition as ext3/4, copy the root filesystem into it, +// and then terminate. + +// Implementation notes: This action uses a helper thread, which seems to +// violate the design decision to only have a single thread and use +// asynchronous i/o. The issue is that (to the best of my knowledge), +// there are no linux APIs to crawl a filesystem's metadata asynchronously. +// The suggested way seems to be to open the raw device and parse the ext +// filesystem. That's not a good approach for a number of reasons: +// - ties us to ext filesystem +// - although this wouldn't happen at the time of writing, it may not handle +// changes to the source fs during the copy as gracefully. +// - requires us to have read-access to the source filesystem device, which +// may be a security issue. +// +// Having said this, using a helper thread is not ideal, but it's acceptable: +// we still honor the Action API. That is, all interaction between the action +// and other objects in the system (e.g. the ActionProcessor) happens on the +// main thread. The helper thread is fully encapsulated by the action. + +namespace chromeos_update_engine { + +class FilesystemCopierAction; + +template<> +class ActionTraits<FilesystemCopierAction> { + public: + // Takes the install plan as input + typedef InstallPlan InputObjectType; + // Passes the install plan as output + typedef InstallPlan OutputObjectType; +}; + +class FilesystemCopierAction : public Action<FilesystemCopierAction> { + public: + FilesystemCopierAction() + : thread_should_exit_(0), + is_mounted_(false), + copy_source_("/"), + skipped_copy_(false) {} + typedef ActionTraits<FilesystemCopierAction>::InputObjectType + InputObjectType; + typedef ActionTraits<FilesystemCopierAction>::OutputObjectType + OutputObjectType; + void PerformAction(); + void TerminateProcessing(); + + // Used for testing, so we can copy from somewhere other than root + void set_copy_source(const string& path) { + copy_source_ = path; + } + // Returns true if we detected that a copy was unneeded and thus skipped it. + bool skipped_copy() { return skipped_copy_; } + + // Debugging/logging + static std::string StaticType() { return "FilesystemCopierAction"; } + std::string Type() const { return StaticType(); } + + private: + // These synchronously mount or unmount the given mountpoint + bool Mount(const string& device, const string& mountpoint); + bool Unmount(const string& mountpoint); + + // Performs a recursive file/directory copy from copy_source_ to dest_path_. + // Doesn't return until the copy has completed. Returns true on success + // or false on error. + bool CopySynchronously(); + + // There are helper functions for CopySynchronously. They handle creating + // various types of files. They return true on success. + bool CreateDirSynchronously(const std::string& new_path, + const struct stat& stbuf); + bool CopyFileSynchronously(const std::string& old_path, + const std::string& new_path, + const struct stat& stbuf); + bool CreateHardLinkSynchronously(const std::string& old_path, + const std::string& new_path); + // Note: Here, old_path is an existing symlink that will be copied to + // new_path. Thus, old_path is *not* the same as the old_path from + // the symlink() syscall. + bool CopySymlinkSynchronously(const std::string& old_path, + const std::string& new_path, + const struct stat& stbuf); + bool CreateNodeSynchronously(const std::string& new_path, + const struct stat& stbuf); + + // Returns NULL on success + void* HelperThreadMain(); + static void* HelperThreadMainStatic(void* data) { + FilesystemCopierAction* self = + reinterpret_cast<FilesystemCopierAction*>(data); + return self->HelperThreadMain(); + } + + // Joins the thread and tells the processor that we're done + void CollectThread(); + // GMainLoop callback function: + static gboolean CollectThreadStatic(gpointer data) { + FilesystemCopierAction* self = + reinterpret_cast<FilesystemCopierAction*>(data); + self->CollectThread(); + return FALSE; + } + + pthread_t helper_thread_; + + volatile gint thread_should_exit_; + + static const char* kCompleteFilesystemMarker; + + // Whether or not the destination device is currently mounted. + bool is_mounted_; + + // Where the destination device is mounted. + string dest_path_; + + // The path to copy from. Usually left as the default "/", but tests can + // change it. + string copy_source_; + + // The install plan we're passed in via the input pipe. + InstallPlan install_plan_; + + // Set to true if we detected the copy was unneeded and thus we skipped it. + bool skipped_copy_; + + DISALLOW_COPY_AND_ASSIGN(FilesystemCopierAction); +}; + +} // namespace chromeos_update_engine + +#endif // CHROMEOS_PLATFORM_UPDATE_ENGINE_FILESYSTEM_COPIER_ACTION_H__ diff --git a/filesystem_copier_action_unittest.cc b/filesystem_copier_action_unittest.cc new file mode 100644 index 00000000..46c13a36 --- /dev/null +++ b/filesystem_copier_action_unittest.cc @@ -0,0 +1,273 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include <glib.h> +#include <set> +#include <vector> +#include <gtest/gtest.h> +#include "update_engine/filesystem_copier_action.h" +#include "update_engine/filesystem_iterator.h" +#include "update_engine/omaha_hash_calculator.h" +#include "update_engine/test_utils.h" +#include "update_engine/utils.h" + +using std::set; +using std::vector; + +namespace chromeos_update_engine { + +class FilesystemCopierActionTest : public ::testing::Test { + protected: + void DoTest(bool double_copy, bool run_out_of_space); + string TestDir() { return "./FilesystemCopierActionTestDir"; } + void SetUp() { + System(string("mkdir -p ") + TestDir()); + } + void TearDown() { + System(string("rm -rf ") + TestDir()); + } +}; + +class FilesystemCopierActionTestDelegate : public ActionProcessorDelegate { + public: + FilesystemCopierActionTestDelegate() : ran_(false), success_(false) {} + void ProcessingDone(const ActionProcessor* processor) { + g_main_loop_quit(loop_); + } + void ActionCompleted(ActionProcessor* processor, + AbstractAction* action, + bool success) { + if (action->Type() == FilesystemCopierAction::StaticType()) { + ran_ = true; + success_ = success; + } + } + void set_loop(GMainLoop* loop) { + loop_ = loop; + } + bool ran() { return ran_; } + bool success() { return success_; } + private: + GMainLoop* loop_; + bool ran_; + bool success_; +}; + +gboolean StartProcessorInRunLoop(gpointer data) { + ActionProcessor* processor = reinterpret_cast<ActionProcessor*>(data); + processor->StartProcessing(); + return FALSE; +} + +TEST_F(FilesystemCopierActionTest, RunAsRootSimpleTest) { + ASSERT_EQ(0, getuid()); + DoTest(false, false); +} +void FilesystemCopierActionTest::DoTest(bool double_copy, + bool run_out_of_space) { + GMainLoop *loop = g_main_loop_new(g_main_context_default(), FALSE); + + // make two populated ext images, mount them both (one in the other), + // and copy to a loop device setup to correspond to another file. + + const string a_image(TestDir() + "/a_image"); + const string b_image(TestDir() + "/b_image"); + const string out_image(TestDir() + "/out_image"); + + vector<string> expected_paths_vector; + CreateExtImageAtPath(a_image, &expected_paths_vector); + CreateExtImageAtPath(b_image, NULL); + + // create 5 MiB file + ASSERT_EQ(0, System(string("dd if=/dev/zero of=") + out_image + + " seek=5242879 bs=1 count=1")); + + // mount them both + System(("mkdir -p " + TestDir() + "/mnt").c_str()); + ASSERT_EQ(0, System(string("mount -o loop ") + a_image + " " + + TestDir() + "/mnt")); + ASSERT_EQ(0, + System(string("mount -o loop ") + b_image + " " + + TestDir() + "/mnt/some_dir/mnt")); + + if (run_out_of_space) + ASSERT_EQ(0, System(string("dd if=/dev/zero of=") + + TestDir() + "/mnt/big_zero bs=5M count=1")); + + string dev = GetUnusedLoopDevice(); + + EXPECT_EQ(0, System(string("losetup ") + dev + " " + out_image)); + + InstallPlan install_plan; + install_plan.is_full_update = false; + install_plan.install_path = dev; + + ActionProcessor processor; + FilesystemCopierActionTestDelegate delegate; + delegate.set_loop(loop); + processor.set_delegate(&delegate); + + ObjectFeederAction<InstallPlan> feeder_action; + FilesystemCopierAction copier_action; + FilesystemCopierAction copier_action2; + ObjectCollectorAction<InstallPlan> collector_action; + + BondActions(&feeder_action, &copier_action); + if (double_copy) { + BondActions(&copier_action, &copier_action2); + BondActions(&copier_action2, &collector_action); + } else { + BondActions(&copier_action, &collector_action); + } + + processor.EnqueueAction(&feeder_action); + processor.EnqueueAction(&copier_action); + if (double_copy) + processor.EnqueueAction(&copier_action2); + processor.EnqueueAction(&collector_action); + + copier_action.set_copy_source(TestDir() + "/mnt"); + feeder_action.set_obj(install_plan); + + g_timeout_add(0, &StartProcessorInRunLoop, &processor); + g_main_loop_run(loop); + g_main_loop_unref(loop); + + EXPECT_EQ(0, System(string("losetup -d ") + dev)); + EXPECT_EQ(0, System(string("umount ") + TestDir() + "/mnt/some_dir/mnt")); + EXPECT_EQ(0, System(string("umount ") + TestDir() + "/mnt")); + EXPECT_EQ(0, unlink(a_image.c_str())); + EXPECT_EQ(0, unlink(b_image.c_str())); + + EXPECT_TRUE(delegate.ran()); + if (run_out_of_space) { + EXPECT_FALSE(delegate.success()); + EXPECT_EQ(0, unlink(out_image.c_str())); + EXPECT_EQ(0, rmdir((TestDir() + "/mnt").c_str())); + return; + } + EXPECT_TRUE(delegate.success()); + + EXPECT_EQ(0, System(string("mount -o loop ") + out_image + " " + + TestDir() + "/mnt")); + // Make sure everything in the out_image is there + expected_paths_vector.push_back("/update_engine_copy_success"); + for (vector<string>::iterator it = expected_paths_vector.begin(); + it != expected_paths_vector.end(); ++it) { + *it = TestDir() + "/mnt" + *it; + } + set<string> expected_paths(expected_paths_vector.begin(), + expected_paths_vector.end()); + VerifyAllPaths(TestDir() + "/mnt", expected_paths); + string file_data; + EXPECT_TRUE(utils::ReadFileToString(TestDir() + "/mnt/hi", &file_data)); + EXPECT_EQ("hi\n", file_data); + EXPECT_TRUE(utils::ReadFileToString(TestDir() + "/mnt/hello", &file_data)); + EXPECT_EQ("hello\n", file_data); + EXPECT_EQ("/some/target", Readlink(TestDir() + "/mnt/sym")); + EXPECT_EQ(0, System(string("umount ") + TestDir() + "/mnt")); + + EXPECT_EQ(0, unlink(out_image.c_str())); + EXPECT_EQ(0, rmdir((TestDir() + "/mnt").c_str())); + + EXPECT_FALSE(copier_action.skipped_copy()); + LOG(INFO) << "collected plan:"; + collector_action.object().Dump(); + LOG(INFO) << "expected plan:"; + install_plan.Dump(); + EXPECT_TRUE(collector_action.object() == install_plan); +} + +class FilesystemCopierActionTest2Delegate : public ActionProcessorDelegate { + public: + void ActionCompleted(ActionProcessor* processor, + AbstractAction* action, + bool success) { + if (action->Type() == FilesystemCopierAction::StaticType()) { + ran_ = true; + success_ = success; + } + } + GMainLoop *loop_; + bool ran_; + bool success_; +}; + +TEST_F(FilesystemCopierActionTest, MissingInputObjectTest) { + ActionProcessor processor; + FilesystemCopierActionTest2Delegate delegate; + + processor.set_delegate(&delegate); + + FilesystemCopierAction copier_action; + ObjectCollectorAction<InstallPlan> collector_action; + + BondActions(&copier_action, &collector_action); + + processor.EnqueueAction(&copier_action); + processor.EnqueueAction(&collector_action); + processor.StartProcessing(); + EXPECT_FALSE(processor.IsRunning()); + EXPECT_TRUE(delegate.ran_); + EXPECT_FALSE(delegate.success_); +} + +TEST_F(FilesystemCopierActionTest, FullUpdateTest) { + ActionProcessor processor; + FilesystemCopierActionTest2Delegate delegate; + + processor.set_delegate(&delegate); + + ObjectFeederAction<InstallPlan> feeder_action; + InstallPlan install_plan(true, "", "", "", ""); + feeder_action.set_obj(install_plan); + FilesystemCopierAction copier_action; + ObjectCollectorAction<InstallPlan> collector_action; + + BondActions(&feeder_action, &copier_action); + BondActions(&copier_action, &collector_action); + + processor.EnqueueAction(&feeder_action); + processor.EnqueueAction(&copier_action); + processor.EnqueueAction(&collector_action); + processor.StartProcessing(); + EXPECT_FALSE(processor.IsRunning()); + EXPECT_TRUE(delegate.ran_); + EXPECT_TRUE(delegate.success_); +} + +TEST_F(FilesystemCopierActionTest, NonExistentDriveTest) { + ActionProcessor processor; + FilesystemCopierActionTest2Delegate delegate; + + processor.set_delegate(&delegate); + + ObjectFeederAction<InstallPlan> feeder_action; + InstallPlan install_plan(false, "", "", "", "/some/missing/file/path"); + feeder_action.set_obj(install_plan); + FilesystemCopierAction copier_action; + ObjectCollectorAction<InstallPlan> collector_action; + + BondActions(&copier_action, &collector_action); + + processor.EnqueueAction(&feeder_action); + processor.EnqueueAction(&copier_action); + processor.EnqueueAction(&collector_action); + processor.StartProcessing(); + EXPECT_FALSE(processor.IsRunning()); + EXPECT_TRUE(delegate.ran_); + EXPECT_FALSE(delegate.success_); +} + +TEST_F(FilesystemCopierActionTest, RunAsRootSkipUpdateTest) { + ASSERT_EQ(0, getuid()); + DoTest(true, false); +} + +TEST_F(FilesystemCopierActionTest, RunAsRootNoSpaceTest) { + ASSERT_EQ(0, getuid()); + DoTest(false, true); +} + +} // namespace chromeos_update_engine diff --git a/filesystem_iterator.cc b/filesystem_iterator.cc new file mode 100644 index 00000000..9d98dea9 --- /dev/null +++ b/filesystem_iterator.cc @@ -0,0 +1,147 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "update_engine/filesystem_iterator.h" +#include <sys/types.h> +#include <dirent.h> +#include <errno.h> +#include <set> +#include <string> +#include <vector> +#include "chromeos/obsolete_logging.h" +#include "update_engine/utils.h" + +using std::set; +using std::string; +using std::vector; + +namespace chromeos_update_engine { + +// We use a macro here for two reasons: +// 1. We want to be able to return from the caller's function. +// 2. We can use the #macro_arg ism to get a string of the calling statement, +// which we can log. + +#define RETURN_ERROR_IF_FALSE(_statement) \ + do { \ + bool _result = (_statement); \ + if (!_result) { \ + string _message = utils::ErrnoNumberAsString(errno); \ + LOG(INFO) << #_statement << " failed: " << _message << ". Aborting"; \ + is_end_ = true; \ + is_err_ = true; \ + return; \ + } \ + } while (0) + +FilesystemIterator::FilesystemIterator( + const std::string& path, + const std::set<std::string>& excl_prefixes) + : excl_prefixes_(excl_prefixes), + is_end_(false), + is_err_(false) { + root_path_ = utils::NormalizePath(path, true); + RETURN_ERROR_IF_FALSE(lstat(root_path_.c_str(), &stbuf_) == 0); + root_dev_ = stbuf_.st_dev; +} + +FilesystemIterator::~FilesystemIterator() { + for (vector<DIR*>::iterator it = dirs_.begin(); it != dirs_.end(); ++it) { + LOG_IF(ERROR, closedir(*it) != 0) << "closedir failed"; + } +} + +// Returns full path for current file +std::string FilesystemIterator::GetFullPath() const { + return root_path_ + GetPartialPath(); +} + +std::string FilesystemIterator::GetPartialPath() const { + std::string ret; + for (vector<string>::const_iterator it = names_.begin(); + it != names_.end(); ++it) { + ret += "/"; + ret += *it; + } + return ret; +} + +// Increments to the next file +void FilesystemIterator::Increment() { + // If we're currently on a dir, descend into children, but only if + // we're on the same device as the root device + + bool entering_dir = false; // true if we're entering into a new dir + if (S_ISDIR(stbuf_.st_mode) && (stbuf_.st_dev == root_dev_)) { + DIR* dir = opendir(GetFullPath().c_str()); + if ((!dir) && ((errno == ENOTDIR) || (errno == ENOENT))) { + // opendir failed b/c either it's not a dir or it doesn't exist. + // that's fine. let's just skip over this. + LOG(ERROR) << "Can't descend into " << GetFullPath(); + } else { + RETURN_ERROR_IF_FALSE(dir); + entering_dir = true; + dirs_.push_back(dir); + } + } + + if (!entering_dir && names_.empty()) { + // root disappeared while we tried to descend into it + is_end_ = true; + return; + } + + if (!entering_dir) + names_.pop_back(); + + IncrementInternal(); + for (set<string>::const_iterator it = excl_prefixes_.begin(); + it != excl_prefixes_.end(); ++it) { + if (utils::StringHasPrefix(GetPartialPath(), *it)) { + Increment(); + break; + } + } + return; +} + +// Assumes that we need to find the next child of dirs_.back(), or if +// there are none more, go up the chain +void FilesystemIterator::IncrementInternal() { + CHECK_EQ(dirs_.size(), names_.size() + 1); + for (;;) { + struct dirent dir_entry; + struct dirent* dir_entry_pointer; + int r; + RETURN_ERROR_IF_FALSE( + (r = readdir_r(dirs_.back(), &dir_entry, &dir_entry_pointer)) == 0); + if (dir_entry_pointer) { + // Found an entry + names_.push_back(dir_entry_pointer->d_name); + // Validate + RETURN_ERROR_IF_FALSE(lstat(GetFullPath().c_str(), &stbuf_) == 0); + if (strcmp(dir_entry_pointer->d_name, ".") && + strcmp(dir_entry_pointer->d_name, "..")) { + // Done + return; + } + // Child didn't work out. Try again + names_.pop_back(); + } else { + // No more children in this dir. Pop it and try again + RETURN_ERROR_IF_FALSE(closedir(dirs_.back()) == 0); + dirs_.pop_back(); + if (dirs_.empty()) { + CHECK(names_.empty()); + // Done with the entire iteration + is_end_ = true; + return; + } + CHECK(!names_.empty()); + names_.pop_back(); + } + } +} + +} // namespace chromeos_update_engine diff --git a/filesystem_iterator.h b/filesystem_iterator.h new file mode 100644 index 00000000..a1ba8af7 --- /dev/null +++ b/filesystem_iterator.h @@ -0,0 +1,126 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CHROMEOS_PLATFORM_UPDATE_ENGINE_FILESYSTEM_ITERATOR_H__ +#define CHROMEOS_PLATFORM_UPDATE_ENGINE_FILESYSTEM_ITERATOR_H__ + +// This class is used to walk a filesystem. It will iterate over every file +// on the same device as the file passed in the ctor. Directories will be +// visited before their children. Children will be visited in no particular +// order. + +// The iterator is a forward iterator. It's not random access nor can it be +// decremented. + +// Note: If the iterator comes across a mount point where another filesystem +// is mounted, that mount point will be present, but none of its children +// will be. Technically the mount point is on the other filesystem (and +// the Stat() call will verify that), but we return it anyway since: +// 1. Such a folder must exist in the first filesystem if it got used +// as a mount point. +// 2. You probably want to copy if it you're using the iterator to do a +// filesystem copy +// 3. If you don't want that, you can just check Stat().st_dev and skip +// foreign filesystems manually. + +#include <sys/stat.h> +#include <sys/types.h> +#include <dirent.h> +#include <unistd.h> +#include <string> +#include <set> +#include <vector> + +namespace chromeos_update_engine { + +class FilesystemIterator { + public: + FilesystemIterator(const std::string& path, + const std::set<std::string>& excl_prefixes); + + ~FilesystemIterator(); + + // Returns stat struct for the current file. + struct stat GetStat() const { + return stbuf_; + } + + // Returns full path for current file. + std::string GetFullPath() const; + + // Returns the path that's part of the iterator. For example, if + // the object were constructed by passing in "/foo/bar" and Path() + // returns "/foo/bar/baz/bat.txt", IterPath would return + // "/baz/bat.txt". When this object is on root (ie, the very first + // path), IterPath will return "", otherwise the first character of + // IterPath will be "/". + std::string GetPartialPath() const; + + // Returns name for current file. + std::string GetBasename() const { + return names_.back(); + } + + // Increments to the next file. + void Increment(); + + // If we're at the end. If at the end, do not call Stat(), Path(), etc., + // since this iterator currently isn't pointing to any file at all. + bool IsEnd() const { + return is_end_; + } + + // Returns true if the iterator is in an error state. + bool IsErr() const { + return is_err_; + } + private: + // Helper for Increment. + void IncrementInternal(); + + // Returns true if path exists and it's a directory. + bool DirectoryExists(const std::string& path); + + // In general (i.e., not midway through a call to Increment()), there is a + // relationship between dirs_ and names_: dirs[i] == names_[i - 1]. + // For example, say we are asked to iterate "/usr/local" and we're currently + // at /usr/local/share/dict/words. dirs_ contains DIR* variables for the + // dirs at: {"/usr/local", ".../share", ".../dict"} and names_ contains: + // {"share", "dict", "words"}. root_path_ contains "/usr/local". + // root_dev_ would be the dev for root_path_ + // (and /usr/local/share/dict/words). stbuf_ would be the stbuf for + // /usr/local/share/dict/words. + + // All opened directories. If this is empty, we're currently on the root, + // but not descended into the root. + // This will always contain the current directory and all it's ancestors + // in root-to-leaf order. For more details, see comment above. + std::vector<DIR*> dirs_; + + // The list of all filenames for the current path that we've descended into. + std::vector<std::string> names_; + + // The device of the root path we've been asked to iterate. + dev_t root_dev_; + + // The root path we've been asked to iteratate. + std::string root_path_; + + // Exclude items w/ this prefix. + std::set<std::string> excl_prefixes_; + + // The struct stat of the current file we're at. + struct stat stbuf_; + + // Generally false; set to true when we reach the end of files to iterate + // or error occurs. + bool is_end_; + + // Generally false; set to true if an error occurrs. + bool is_err_; +}; + +} // namespace chromeos_update_engine + +#endif // CHROMEOS_PLATFORM_UPDATE_ENGINE_FILESYSTEM_ITERATOR_H__ diff --git a/filesystem_iterator_unittest.cc b/filesystem_iterator_unittest.cc new file mode 100644 index 00000000..82b8d3f4 --- /dev/null +++ b/filesystem_iterator_unittest.cc @@ -0,0 +1,120 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> +#include <set> +#include <string> +#include <vector> +#include "base/string_util.h" +#include <gtest/gtest.h> +#include "update_engine/filesystem_iterator.h" +#include "update_engine/test_utils.h" +#include "update_engine/utils.h" + +using std::set; +using std::string; +using std::vector; + +namespace chromeos_update_engine { + +namespace { +const char* TestDir() { return "./FilesystemIteratorTest-dir"; } +} // namespace {} + +class FilesystemIteratorTest : public ::testing::Test { + protected: + virtual void SetUp() { + LOG(INFO) << "SetUp() mkdir"; + EXPECT_EQ(0, System(StringPrintf("rm -rf %s", TestDir()))); + EXPECT_EQ(0, System(StringPrintf("mkdir -p %s", TestDir()))); + } + virtual void TearDown() { + LOG(INFO) << "TearDown() rmdir"; + EXPECT_EQ(0, System(StringPrintf("rm -rf %s", TestDir()))); + } +}; + +TEST_F(FilesystemIteratorTest, RunAsRootSuccessTest) { + ASSERT_EQ(0, getuid()); + string first_image("FilesystemIteratorTest.image1"); + string sub_image("FilesystemIteratorTest.image2"); + + ASSERT_EQ(0, System(string("rm -f ") + first_image + " " + sub_image)); + vector<string> expected_paths_vector; + CreateExtImageAtPath(first_image, &expected_paths_vector); + CreateExtImageAtPath(sub_image, NULL); + ASSERT_EQ(0, System(string("mount -o loop ") + first_image + " " + + kMountPath)); + ASSERT_EQ(0, System(string("mount -o loop ") + sub_image + " " + + kMountPath + "/some_dir/mnt")); + for (vector<string>::iterator it = expected_paths_vector.begin(); + it != expected_paths_vector.end(); ++it) + *it = kMountPath + *it; + set<string> expected_paths(expected_paths_vector.begin(), + expected_paths_vector.end()); + VerifyAllPaths(kMountPath, expected_paths); + + EXPECT_EQ(0, System(string("umount ") + kMountPath + "/some_dir/mnt")); + EXPECT_EQ(0, System(string("umount ") + kMountPath)); + EXPECT_EQ(0, System(string("rm -f ") + first_image + " " + sub_image)); +} + +TEST_F(FilesystemIteratorTest, NegativeTest) { + { + FilesystemIterator iter("/non/existent/path", set<string>()); + EXPECT_TRUE(iter.IsEnd()); + EXPECT_TRUE(iter.IsErr()); + } + + { + FilesystemIterator iter(TestDir(), set<string>()); + EXPECT_FALSE(iter.IsEnd()); + EXPECT_FALSE(iter.IsErr()); + // Here I'm deleting the exact directory that iterator is point at, + // then incrementing (which normally would descend into that directory). + EXPECT_EQ(0, rmdir(TestDir())); + iter.Increment(); + EXPECT_TRUE(iter.IsEnd()); + EXPECT_FALSE(iter.IsErr()); + } +} + +TEST_F(FilesystemIteratorTest, DeleteWhileTraverseTest) { + ASSERT_EQ(0, mkdir("DeleteWhileTraverseTest", 0755)); + ASSERT_EQ(0, mkdir("DeleteWhileTraverseTest/a", 0755)); + ASSERT_EQ(0, mkdir("DeleteWhileTraverseTest/a/b", 0755)); + ASSERT_EQ(0, mkdir("DeleteWhileTraverseTest/b", 0755)); + ASSERT_EQ(0, mkdir("DeleteWhileTraverseTest/c", 0755)); + + string expected_paths_arr[] = { + "", + "/a", + "/b", + "/c" + }; + set<string> expected_paths(expected_paths_arr, + expected_paths_arr + + arraysize(expected_paths_arr)); + + FilesystemIterator iter("DeleteWhileTraverseTest", set<string>()); + while (!iter.IsEnd()) { + string path = iter.GetPartialPath(); + EXPECT_TRUE(expected_paths.find(path) != expected_paths.end()); + if (expected_paths.find(path) != expected_paths.end()) { + expected_paths.erase(path); + } + if (path == "/a") { + EXPECT_EQ(0, rmdir("DeleteWhileTraverseTest/a/b")); + EXPECT_EQ(0, rmdir("DeleteWhileTraverseTest/a")); + } + iter.Increment(); + } + EXPECT_FALSE(iter.IsErr()); + EXPECT_TRUE(expected_paths.empty()); + EXPECT_EQ(0, system("rm -rf DeleteWhileTraverseTest")); +} + +} // namespace chromeos_update_engine diff --git a/generate_delta_main.cc b/generate_delta_main.cc new file mode 100644 index 00000000..14af193a --- /dev/null +++ b/generate_delta_main.cc @@ -0,0 +1,48 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include <sys/types.h> +#include <sys/stat.h> +#include <dirent.h> +#include <fcntl.h> +#include <stdlib.h> +#include <stdio.h> +#include <unistd.h> + +#include <algorithm> +#include <string> +#include <vector> +#include <tr1/memory> + +#include <glib.h> + +#include "chromeos/obsolete_logging.h" +#include "update_engine/subprocess.h" +#include "update_engine/update_metadata.pb.h" + +using std::sort; +using std::string; +using std::vector; +using std::tr1::shared_ptr; + +// This file contains a simple program that takes an old path, a new path, +// and an output file as arguments and the path to an output file and +// generates a delta that can be sent to Chrome OS clients. + +namespace chromeos_update_engine { + +int main(int argc, char** argv) { + g_thread_init(NULL); + Subprocess::Init(); + if (argc != 4) { + usage(argv[0]); + } + const char* old_dir = argv[1]; + const char* new_dir = argv[2]; + if ((!IsDir(old_dir)) || (!IsDir(new_dir))) { + usage(argv[0]); + } + // TODO(adlr): implement using DeltaDiffGenerator + return 0; +}
\ No newline at end of file diff --git a/gzip.cc b/gzip.cc new file mode 100644 index 00000000..96437248 --- /dev/null +++ b/gzip.cc @@ -0,0 +1,185 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "update_engine/gzip.h" +#include <stdlib.h> +#include <algorithm> +#include <zlib.h> +#include "chromeos/obsolete_logging.h" +#include "update_engine/utils.h" + +using std::max; +using std::string; +using std::vector; + +namespace chromeos_update_engine { + +bool GzipDecompressData(const char* const in, const size_t in_size, + char** out, size_t* out_size) { + if (in_size == 0) { + // malloc(0) may legally return NULL, so do malloc(1) + *out = reinterpret_cast<char*>(malloc(1)); + *out_size = 0; + return true; + } + TEST_AND_RETURN_FALSE(out); + TEST_AND_RETURN_FALSE(out_size); + z_stream stream; + memset(&stream, 0, sizeof(stream)); + TEST_AND_RETURN_FALSE(inflateInit2(&stream, 16 + MAX_WBITS) == Z_OK); + + // guess that output will be roughly double the input size + *out_size = in_size * 2; + *out = reinterpret_cast<char*>(malloc(*out_size)); + TEST_AND_RETURN_FALSE(*out); + + // TODO(adlr): ensure that this const_cast is safe. + stream.next_in = const_cast<Bytef*>(reinterpret_cast<const Bytef*>(in)); + stream.avail_in = in_size; + stream.next_out = reinterpret_cast<Bytef*>(*out); + stream.avail_out = *out_size; + for (;;) { + int rc = inflate(&stream, Z_FINISH); + switch (rc) { + case Z_STREAM_END: { + *out_size = reinterpret_cast<char*>(stream.next_out) - (*out); + TEST_AND_RETURN_FALSE(inflateEnd(&stream) == Z_OK); + return true; + } + case Z_OK: // fall through + case Z_BUF_ERROR: { + // allocate more space + ptrdiff_t out_length = + reinterpret_cast<char*>(stream.next_out) - (*out); + *out_size *= 2; + char* new_out = reinterpret_cast<char*>(realloc(*out, *out_size)); + if (!new_out) { + free(*out); + return false; + } + *out = new_out; + stream.next_out = reinterpret_cast<Bytef*>((*out) + out_length); + stream.avail_out = (*out_size) - out_length; + break; + } + default: + LOG(INFO) << "Unknown inflate() return value: " << rc; + if (stream.msg) + LOG(INFO) << " message: " << stream.msg; + free(*out); + return false; + } + } +} + +bool GzipCompressData(const char* const in, const size_t in_size, + char** out, size_t* out_size) { + if (in_size == 0) { + // malloc(0) may legally return NULL, so do malloc(1) + *out = reinterpret_cast<char*>(malloc(1)); + *out_size = 0; + return true; + } + TEST_AND_RETURN_FALSE(out); + TEST_AND_RETURN_FALSE(out_size); + z_stream stream; + memset(&stream, 0, sizeof(stream)); + TEST_AND_RETURN_FALSE(deflateInit2(&stream, + Z_BEST_COMPRESSION, + Z_DEFLATED, + 16 + MAX_WBITS, + 9, // most memory used/best compression + Z_DEFAULT_STRATEGY) == Z_OK); + + // guess that output will be roughly half the input size + *out_size = max(1U, in_size / 2); + *out = reinterpret_cast<char*>(malloc(*out_size)); + TEST_AND_RETURN_FALSE(*out); + + // TODO(adlr): ensure that this const_cast is safe. + stream.next_in = const_cast<Bytef*>(reinterpret_cast<const Bytef*>(in)); + stream.avail_in = in_size; + stream.next_out = reinterpret_cast<Bytef*>(*out); + stream.avail_out = *out_size; + for (;;) { + int rc = deflate(&stream, Z_FINISH); + switch (rc) { + case Z_STREAM_END: { + *out_size = reinterpret_cast<char*>(stream.next_out) - (*out); + TEST_AND_RETURN_FALSE(deflateEnd(&stream) == Z_OK); + return true; + } + case Z_OK: // fall through + case Z_BUF_ERROR: { + // allocate more space + ptrdiff_t out_length = + reinterpret_cast<char*>(stream.next_out) - (*out); + *out_size *= 2; + char* new_out = reinterpret_cast<char*>(realloc(*out, *out_size)); + if (!new_out) { + free(*out); + return false; + } + *out = new_out; + stream.next_out = reinterpret_cast<Bytef*>((*out) + out_length); + stream.avail_out = (*out_size) - out_length; + break; + } + default: + LOG(INFO) << "Unknown defalate() return value: " << rc; + if (stream.msg) + LOG(INFO) << " message: " << stream.msg; + free(*out); + return false; + } + } +} + +bool GzipDecompress(const std::vector<char>& in, std::vector<char>* out) { + TEST_AND_RETURN_FALSE(out); + char* out_buf; + size_t out_size; + TEST_AND_RETURN_FALSE(GzipDecompressData(&in[0], in.size(), + &out_buf, &out_size)); + out->insert(out->end(), out_buf, out_buf + out_size); + free(out_buf); + return true; +} + +bool GzipCompress(const std::vector<char>& in, std::vector<char>* out) { + TEST_AND_RETURN_FALSE(out); + char* out_buf; + size_t out_size; + TEST_AND_RETURN_FALSE(GzipCompressData(&in[0], in.size(), + &out_buf, &out_size)); + out->insert(out->end(), out_buf, out_buf + out_size); + free(out_buf); + return true; +} + +bool GzipCompressString(const std::string& str, + std::vector<char>* out) { + TEST_AND_RETURN_FALSE(out); + char* out_buf; + size_t out_size; + TEST_AND_RETURN_FALSE(GzipCompressData(str.data(), str.size(), + &out_buf, &out_size)); + out->insert(out->end(), out_buf, out_buf + out_size); + free(out_buf); + return true; +} + +bool GzipDecompressString(const std::string& str, + std::vector<char>* out) { + TEST_AND_RETURN_FALSE(out); + char* out_buf; + size_t out_size; + TEST_AND_RETURN_FALSE(GzipDecompressData(str.data(), str.size(), + &out_buf, &out_size)); + out->insert(out->end(), out_buf, out_buf + out_size); + free(out_buf); + return true; +} + +} // namespace chromeos_update_engine @@ -0,0 +1,24 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include <string> +#include <vector> + +namespace chromeos_update_engine { + +// Gzip compresses or decompresses the input to the output. +// Returns true on success. If true, *out will point to a malloc()ed +// buffer, which must be free()d by the caller. +bool GzipCompressData(const char* const in, const size_t in_size, + char** out, size_t* out_size); +bool GzipDecompressData(const char* const in, const size_t in_size, + char** out, size_t* out_size); + +// Helper functions: +bool GzipDecompress(const std::vector<char>& in, std::vector<char>* out); +bool GzipCompress(const std::vector<char>& in, std::vector<char>* out); +bool GzipCompressString(const std::string& str, std::vector<char>* out); +bool GzipDecompressString(const std::string& str, std::vector<char>* out); + +} // namespace chromeos_update_engine { diff --git a/gzip_unittest.cc b/gzip_unittest.cc new file mode 100644 index 00000000..298ca9d4 --- /dev/null +++ b/gzip_unittest.cc @@ -0,0 +1,68 @@ +// Copyright (c) 2009 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include <string.h> +#include <unistd.h> +#include <string> +#include <vector> +#include <gtest/gtest.h> +#include "update_engine/gzip.h" +#include "update_engine/test_utils.h" +#include "update_engine/utils.h" + +using std::string; +using std::vector; + +namespace chromeos_update_engine { + +class GzipTest : public ::testing::Test { }; + +TEST(GzipTest, SimpleTest) { + string in("this should compress well xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"); + vector<char> out; + EXPECT_TRUE(GzipCompressString(in, &out)); + EXPECT_LT(out.size(), in.size()); + EXPECT_GT(out.size(), 0); + vector<char> decompressed; + EXPECT_TRUE(GzipDecompress(out, &decompressed)); + EXPECT_EQ(in.size(), decompressed.size()); + EXPECT_TRUE(!memcmp(in.data(), &decompressed[0], in.size())); +} + +TEST(GzipTest, PoorCompressionTest) { + string in(reinterpret_cast<const char*>(kRandomString), + sizeof(kRandomString)); + vector<char> out; + EXPECT_TRUE(GzipCompressString(in, &out)); + EXPECT_GT(out.size(), in.size()); + string out_string(&out[0], out.size()); + vector<char> decompressed; + EXPECT_TRUE(GzipDecompressString(out_string, &decompressed)); + EXPECT_EQ(in.size(), decompressed.size()); + EXPECT_TRUE(!memcmp(in.data(), &decompressed[0], in.size())); +} + +TEST(GzipTest, MalformedGzipTest) { + string in(reinterpret_cast<const char*>(kRandomString), + sizeof(kRandomString)); + vector<char> out; + EXPECT_FALSE(GzipDecompressString(in, &out)); +} + +TEST(GzipTest, EmptyInputsTest) { + string in; + vector<char> out; + EXPECT_TRUE(GzipDecompressString(in, &out)); + EXPECT_EQ(0, out.size()); + + EXPECT_TRUE(GzipCompressString(in, &out)); + EXPECT_EQ(0, out.size()); +} + +} // namespace chromeos_update_engine diff --git a/install_action.cc b/install_action.cc new file mode 100644 index 00000000..9c644df8 --- /dev/null +++ b/install_action.cc @@ -0,0 +1,249 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "update_engine/install_action.h" +#include <errno.h> +#include <vector> +#include <gflags/gflags.h> +#include "update_engine/filesystem_iterator.h" +#include "update_engine/gzip.h" +#include "update_engine/subprocess.h" +#include "update_engine/utils.h" + +DEFINE_string(mount_install_path, "", + "If set, the path to use when mounting the " + "destination device during install"); + +using std::vector; + +namespace chromeos_update_engine { + +namespace { +const string kBspatchPath = "/usr/bin/bspatch"; +} + +void InstallAction::PerformAction() { + ScopedActionCompleter completer(processor_, this); + // For now, do nothing other than pass what we need to to the output pipe + CHECK(HasInputObject()); + const InstallPlan install_plan = GetInputObject(); + if (HasOutputPipe()) + SetOutputObject(install_plan.install_path); + if (install_plan.is_full_update) { + // No need to perform an install + completer.set_success(true); + return; + } + // We have a delta update. + + // Open delta file + DeltaDiffParser parser(install_plan.download_path); + if (!parser.valid()) { + LOG(ERROR) << "Unable to open delta file"; + return; + } + + // Mount install fs + string mountpoint = FLAGS_mount_install_path; + if (mountpoint.empty()) { + // Set up dest_path_ + char *mountpoint_temp = strdup("/tmp/install_mnt.XXXXXX"); + CHECK(mountpoint_temp); + CHECK_EQ(mountpoint_temp, mkdtemp(mountpoint_temp)); + CHECK_NE('\0', mountpoint_temp[0]); + mountpoint = mountpoint_temp; + free(mountpoint_temp); + } + + TEST_AND_RETURN(utils::MountFilesystem(install_plan.install_path, + mountpoint)); + + // Automatically unmount the fs when this goes out of scope: + ScopedFilesystemUnmounter filesystem_unmounter(mountpoint); + + { + // iterate through existing fs, deleting unneeded files + FilesystemIterator iter(mountpoint, + utils::SetWithValue<string>("/lost+found")); + for (; !iter.IsEnd(); iter.Increment()) { + if (!parser.ContainsPath(iter.GetPartialPath())) { + VLOG(1) << "install removing local path: " << iter.GetFullPath(); + TEST_AND_RETURN(utils::RecursiveUnlinkDir(iter.GetFullPath())); + } + } + TEST_AND_RETURN(!iter.IsErr()); + } + + // iterate through delta metadata, writing files + DeltaDiffParserIterator iter = parser.Begin(); + for (; iter != parser.End(); iter.Increment()) { + const DeltaArchiveManifest_File& file = iter.GetFile(); + VLOG(1) << "Installing file: " << iter.path(); + TEST_AND_RETURN(InstallFile(mountpoint, file, iter.path(), parser)); + } + + completer.set_success(true); +} + +bool InstallAction::InstallFile(const std::string& mountpoint, + const DeltaArchiveManifest_File& file, + const std::string& path, + const DeltaDiffParser& parser) const { + // See what's already there + struct stat existing_stbuf; + int result = lstat((mountpoint + path).c_str(), &existing_stbuf); + TEST_AND_RETURN_FALSE_ERRNO((result == 0) || (errno == ENOENT)); + bool exists = (result == 0); + // Create the proper file + if (S_ISDIR(file.mode())) { + if (!exists) { + TEST_AND_RETURN_FALSE_ERRNO( + (mkdir((mountpoint + path).c_str(), file.mode())) == 0); + } + } else if (S_ISLNK(file.mode())) { + InstallFileSymlink(mountpoint, file, path, parser, exists); + } else if (S_ISCHR(file.mode()) || + S_ISBLK(file.mode()) || + S_ISFIFO(file.mode()) || + S_ISSOCK(file.mode())) { + InstallFileSpecialFile(mountpoint, file, path, parser, exists); + } else if (S_ISREG(file.mode())) { + InstallFileRegularFile(mountpoint, file, path, parser, exists); + } else { + // unknown mode type + TEST_AND_RETURN_FALSE(false); + } + + // chmod/chown new file + if (!S_ISLNK(file.mode())) + TEST_AND_RETURN_FALSE_ERRNO(chmod((mountpoint + path).c_str(), file.mode()) + == 0); + TEST_AND_RETURN_FALSE(file.has_uid() && file.has_gid()); + TEST_AND_RETURN_FALSE_ERRNO(lchown((mountpoint + path).c_str(), + file.uid(), file.gid()) == 0); + return true; +} + +bool InstallAction::InstallFileRegularFile( + const std::string& mountpoint, + const DeltaArchiveManifest_File& file, + const std::string& path, + const DeltaDiffParser& parser, + const bool exists) const { + if (!file.has_data_format()) + return true; + TEST_AND_RETURN_FALSE(file.has_data_offset() && file.has_data_length()); + if (file.data_format() == DeltaArchiveManifest_File_DataFormat_BSDIFF) { + // Expand with bspatch + string patch_path = utils::TempFilename(mountpoint + path + ".XXXXXX"); + TEST_AND_RETURN_FALSE(file.has_data_length()); + TEST_AND_RETURN_FALSE(parser.CopyDataToFile( + file.data_offset(), + static_cast<off_t>(file.data_length()), false, + patch_path)); + string output_path = utils::TempFilename(mountpoint + path + ".XXXXXX"); + int rc = 1; + vector<string> cmd; + cmd.push_back(kBspatchPath); + cmd.push_back(mountpoint + path); + cmd.push_back(output_path); + cmd.push_back(patch_path); + TEST_AND_RETURN_FALSE(Subprocess::SynchronousExec(cmd, &rc)); + TEST_AND_RETURN_FALSE(rc == 0); + TEST_AND_RETURN_FALSE_ERRNO(rename(output_path.c_str(), + (mountpoint + path).c_str()) == 0); + TEST_AND_RETURN_FALSE_ERRNO(unlink(patch_path.c_str()) == 0); + } else { + // Expand full data, decompressing if necessary + TEST_AND_RETURN_FALSE((file.data_format() == + DeltaArchiveManifest_File_DataFormat_FULL) || + (file.data_format() == + DeltaArchiveManifest_File_DataFormat_FULL_GZ)); + if (exists) + TEST_AND_RETURN_FALSE_ERRNO(unlink((mountpoint + path).c_str()) == 0); + TEST_AND_RETURN_FALSE(file.has_data_length()); + const bool gzipped = file.data_format() == + DeltaArchiveManifest_File_DataFormat_FULL_GZ; + bool success = + parser.CopyDataToFile(file.data_offset(), file.data_length(), + gzipped, + mountpoint + path); + TEST_AND_RETURN_FALSE(success); + } + return true; +} + +// char/block devices, fifos, and sockets: +bool InstallAction::InstallFileSpecialFile( + const std::string& mountpoint, + const DeltaArchiveManifest_File& file, + const std::string& path, + const DeltaDiffParser& parser, + const bool exists) const { + if (exists) + TEST_AND_RETURN_FALSE(unlink((mountpoint + path).c_str()) == 0); + dev_t dev = 0; + if (S_ISCHR(file.mode()) || S_ISBLK(file.mode())) { + vector<char> dev_proto; + TEST_AND_RETURN_FALSE(parser.ReadDataVector(file.data_offset(), + file.data_length(), + &dev_proto)); + if (file.data_format() == DeltaArchiveManifest_File_DataFormat_FULL_GZ) { + TEST_AND_RETURN_FALSE(file.has_data_length()); + { + vector<char> decompressed_dev_proto; + TEST_AND_RETURN_FALSE(GzipDecompress(dev_proto, + &decompressed_dev_proto)); + dev_proto = decompressed_dev_proto; + } + } else { + TEST_AND_RETURN_FALSE(file.data_format() == + DeltaArchiveManifest_File_DataFormat_FULL); + } + LinuxDevice linux_device; + utils::HexDumpVector(dev_proto); + TEST_AND_RETURN_FALSE(linux_device.ParseFromArray(&dev_proto[0], + dev_proto.size())); + dev = makedev(linux_device.major(), linux_device.minor()); + } + TEST_AND_RETURN_FALSE_ERRNO(mknod((mountpoint + path).c_str(), + file.mode(), dev) == 0); + return true; +} +// symlinks: +bool InstallAction::InstallFileSymlink(const std::string& mountpoint, + const DeltaArchiveManifest_File& file, + const std::string& path, + const DeltaDiffParser& parser, + const bool exists) const { + // If there's no data, we leave the symlink as is + if (!file.has_data_format()) + return true; // No changes needed + TEST_AND_RETURN_FALSE((file.data_format() == + DeltaArchiveManifest_File_DataFormat_FULL) || + (file.data_format() == + DeltaArchiveManifest_File_DataFormat_FULL_GZ)); + TEST_AND_RETURN_FALSE(file.has_data_offset() && file.has_data_length()); + // We have data, and thus use it to create a symlink. + // First delete any existing symlink: + if (exists) + TEST_AND_RETURN_FALSE_ERRNO(unlink((mountpoint + path).c_str()) == 0); + vector<char> symlink_data; + TEST_AND_RETURN_FALSE(parser.ReadDataVector(file.data_offset(), + file.data_length(), + &symlink_data)); + if (file.data_format() == DeltaArchiveManifest_File_DataFormat_FULL_GZ) { + vector<char> decompressed_symlink_data; + TEST_AND_RETURN_FALSE(GzipDecompress(symlink_data, + &decompressed_symlink_data)); + symlink_data = decompressed_symlink_data; + } + symlink_data.push_back('\0'); + TEST_AND_RETURN_FALSE_ERRNO(symlink(&symlink_data[0], + (mountpoint + path).c_str()) == 0); + return true; +} + + +} // namespace chromeos_update_engine diff --git a/install_action.h b/install_action.h new file mode 100644 index 00000000..8bed6322 --- /dev/null +++ b/install_action.h @@ -0,0 +1,86 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CHROMEOS_PLATFORM_UPDATE_ENGINE_INSTALL_ACTION_H__ +#define CHROMEOS_PLATFORM_UPDATE_ENGINE_INSTALL_ACTION_H__ + +#include "base/scoped_ptr.h" +#include "update_engine/action.h" +#include "update_engine/delta_diff_parser.h" +#include "update_engine/install_plan.h" +#include "update_engine/update_metadata.pb.h" + +// The Install Action is responsible for ensuring the update that's been +// downloaded has been installed. This may be a no-op in the case of a full +// update, since those will be downloaded directly into the destination +// partition. However, for a delta update some work is required. + +// An InstallPlan struct must be passed to this action before PerformAction() +// is called so that this action knows if it's a delta update, and if so, +// what the paths are. + +// TODO(adlr): At the moment, InstallAction is synchronous. It should be +// updated to be asynchronous at some point. + +namespace chromeos_update_engine { + +class InstallAction; +class NoneType; + +template<> +class ActionTraits<InstallAction> { + public: + // Takes the InstallPlan for input + typedef InstallPlan InputObjectType; + // On success, puts the output device path on output + typedef std::string OutputObjectType; +}; + +class InstallAction : public Action<InstallAction> { + public: + InstallAction() {} + typedef ActionTraits<InstallAction>::InputObjectType InputObjectType; + typedef ActionTraits<InstallAction>::OutputObjectType OutputObjectType; + void PerformAction(); + + // This action is synchronous for now. + void TerminateProcessing() { CHECK(false); } + + // Debugging/logging + static std::string StaticType() { return "InstallAction"; } + std::string Type() const { return StaticType(); } + + private: + // Installs 'file' into mountpoint. 'path' is the path that 'file' + // should have when we reboot and mountpoint is root. + bool InstallFile(const std::string& mountpoint, + const DeltaArchiveManifest_File& file, + const std::string& path, + const DeltaDiffParser& parser) const; + // These are helpers for InstallFile. They focus on specific file types: + // Regular data files: + bool InstallFileRegularFile(const std::string& mountpoint, + const DeltaArchiveManifest_File& file, + const std::string& path, + const DeltaDiffParser& parser, + const bool exists) const; + // char/block devices, fifos, and sockets: + bool InstallFileSpecialFile(const std::string& mountpoint, + const DeltaArchiveManifest_File& file, + const std::string& path, + const DeltaDiffParser& parser, + const bool exists) const; + // symlinks: + bool InstallFileSymlink(const std::string& mountpoint, + const DeltaArchiveManifest_File& file, + const std::string& path, + const DeltaDiffParser& parser, + const bool exists) const; + + DISALLOW_COPY_AND_ASSIGN(InstallAction); +}; + +} // namespace chromeos_update_engine + +#endif // CHROMEOS_PLATFORM_UPDATE_ENGINE_INSTALL_ACTION_H__ diff --git a/install_action_unittest.cc b/install_action_unittest.cc new file mode 100644 index 00000000..dde15ea5 --- /dev/null +++ b/install_action_unittest.cc @@ -0,0 +1,188 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include <sys/stat.h> +#include <sys/types.h> +#include <errno.h> +#include <set> +#include <string> +#include <vector> +#include "base/string_util.h" +#include <gtest/gtest.h> +#include "update_engine/delta_diff_generator.h" +#include "update_engine/filesystem_iterator.h" +#include "update_engine/install_action.h" +#include "update_engine/test_utils.h" +#include "update_engine/utils.h" + +using chromeos_update_engine::System; +using std::set; +using std::string; +using std::vector; + +namespace { +void GenerateFilesAtPath(const string& base) { + EXPECT_EQ(0, System(StringPrintf("echo hi > %s/hi", base.c_str()))); + EXPECT_EQ(0, System(StringPrintf("mkdir -p %s/dir", base.c_str()))); + EXPECT_EQ(0, System(StringPrintf("echo hello > %s/dir/hello", base.c_str()))); + EXPECT_EQ(0, System(StringPrintf("echo -n > %s/dir/newempty", base.c_str()))); + EXPECT_EQ(0, System(StringPrintf("rm -f %s/dir/bdev", base.c_str()))); + EXPECT_EQ(0, System(StringPrintf("mknod %s/dir/bdev b 3 1", base.c_str()))); + EXPECT_EQ(0, System(StringPrintf("rm -f %s/cdev", base.c_str()))); + EXPECT_EQ(0, System(StringPrintf("mknod %s/cdev c 2 1", base.c_str()))); + EXPECT_EQ(0, System(StringPrintf("mkdir -p %s/dir/subdir", base.c_str()))); + EXPECT_EQ(0, System(StringPrintf("mkdir -p %s/dir/emptydir", base.c_str()))); + EXPECT_EQ(0, System(StringPrintf("echo -n foo > %s/dir/bigfile", + base.c_str()))); + EXPECT_EQ(0, System(StringPrintf("chown 501:503 %s/dir/emptydir", + base.c_str()))); + EXPECT_EQ(0, System(StringPrintf("rm -f %s/dir/subdir/fifo", base.c_str()))); + EXPECT_EQ(0, System(StringPrintf("mkfifo %s/dir/subdir/fifo", base.c_str()))); + EXPECT_EQ(0, System(StringPrintf("ln -f -s /target %s/dir/subdir/link", + base.c_str()))); +} + +// Returns true if files at paths a, b are equal and there are no errors. +bool FilesEqual(const string& a, const string& b) { + struct stat a_stbuf; + struct stat b_stbuf; + + int r = lstat(a.c_str(), &a_stbuf); + TEST_AND_RETURN_FALSE_ERRNO(r == 0); + r = lstat(b.c_str(), &b_stbuf); + TEST_AND_RETURN_FALSE_ERRNO(r == 0); + + TEST_AND_RETURN_FALSE(a_stbuf.st_mode == b_stbuf.st_mode); + if (S_ISBLK(a_stbuf.st_mode) || S_ISCHR(a_stbuf.st_mode)) + TEST_AND_RETURN_FALSE(a_stbuf.st_rdev == b_stbuf.st_rdev); + if (!S_ISREG(a_stbuf.st_mode)) { + return true; + } + // Compare files + TEST_AND_RETURN_FALSE(a_stbuf.st_size == b_stbuf.st_size); + vector<char> a_data; + TEST_AND_RETURN_FALSE(chromeos_update_engine::utils::ReadFile(a, &a_data)); + vector<char> b_data; + TEST_AND_RETURN_FALSE(chromeos_update_engine::utils::ReadFile(b, &b_data)); + TEST_AND_RETURN_FALSE(a_data == b_data); + return true; +} + +class ScopedLoopDevUnmapper { + public: + explicit ScopedLoopDevUnmapper(const string& dev) : dev_(dev) {} + ~ScopedLoopDevUnmapper() { + EXPECT_EQ(0, System(string("losetup -d ") + dev_)); + } + private: + string dev_; +}; + +} + +namespace chromeos_update_engine { + +class InstallActionTest : public ::testing::Test { }; + +TEST(InstallActionTest, RunAsRootDiffTest) { + ASSERT_EQ(0, getuid()); + string loop_dev = GetUnusedLoopDevice(); + ScopedLoopDevUnmapper loop_dev_unmapper(loop_dev); + LOG(INFO) << "Using loop device: " << loop_dev; + const string original_image("orig.image"); + const string original_dir("orig"); + const string new_dir("new"); + + ASSERT_EQ(0, System(string("dd if=/dev/zero of=") + original_image + + " bs=5M count=1")); + ASSERT_EQ(0, System(string("mkfs.ext3 -F ") + original_image)); + ASSERT_EQ(0, System(string("losetup ") + loop_dev + " " + original_image)); + ASSERT_EQ(0, System(string("mkdir ") + original_dir)); + ASSERT_EQ(0, System(string("mount ") + loop_dev + " " + original_dir)); + ASSERT_EQ(0, System(string("mkdir ") + new_dir)); + + GenerateFilesAtPath(original_dir); + GenerateFilesAtPath(new_dir); + + { + // Fill bigfile w/ some data in the new folder + vector<char> buf(100 * 1024); + for (unsigned int i = 0; i < buf.size(); i++) { + buf[i] = static_cast<char>(i); + } + EXPECT_TRUE(WriteFileVector(new_dir + "/dir/bigfile", buf)); + } + // Make a diff + DeltaArchiveManifest* delta = + DeltaDiffGenerator::EncodeMetadataToProtoBuffer(new_dir.c_str()); + EXPECT_TRUE(NULL != delta); + EXPECT_TRUE(DeltaDiffGenerator::EncodeDataToDeltaFile(delta, + original_dir, + new_dir, + "delta")); + + ASSERT_EQ(0, System(string("umount ") + original_dir)); + + ObjectFeederAction<InstallPlan> feeder_action; + InstallAction install_action; + ObjectCollectorAction<string> collector_action; + + BondActions(&feeder_action, &install_action); + BondActions(&install_action, &collector_action); + + ActionProcessor processor; + processor.EnqueueAction(&feeder_action); + processor.EnqueueAction(&install_action); + processor.EnqueueAction(&collector_action); + + InstallPlan install_plan(false, "", "", "delta", loop_dev); + feeder_action.set_obj(install_plan); + + processor.StartProcessing(); + EXPECT_FALSE(processor.IsRunning()) << "Update to handle async actions"; + + EXPECT_EQ(loop_dev, collector_action.object()); + + ASSERT_EQ(0, System(string("mount ") + loop_dev + " " + original_dir)); + + // Check that original_dir and new_dir are equal + int original_count = 0; + LOG(INFO) << "checking old"; + { + FilesystemIterator iter(original_dir, + utils::SetWithValue<string>("/lost+found")); + for (; !iter.IsEnd(); iter.Increment()) { + original_count++; + LOG(INFO) << "checking path: " << iter.GetPartialPath(); + EXPECT_TRUE(FilesEqual(original_dir + iter.GetPartialPath(), + new_dir + iter.GetPartialPath())); + } + EXPECT_FALSE(iter.IsErr()); + } + LOG(INFO) << "checking new"; + int new_count = 0; + { + FilesystemIterator iter(new_dir, set<string>()); + for (; !iter.IsEnd(); iter.Increment()) { + new_count++; + LOG(INFO) << "checking path: " << iter.GetPartialPath(); + EXPECT_TRUE(FilesEqual(original_dir + iter.GetPartialPath(), + new_dir + iter.GetPartialPath())); + } + EXPECT_FALSE(iter.IsErr()); + } + LOG(INFO) << "new_count = " << new_count; + EXPECT_EQ(new_count, original_count); + EXPECT_EQ(12, original_count); // 12 files in each dir + + ASSERT_EQ(0, System(string("umount ") + original_dir)); + + // Cleanup generated files + ASSERT_EQ(0, System(string("rm -rf ") + original_dir)); + ASSERT_EQ(0, System(string("rm -rf ") + new_dir)); + EXPECT_EQ(0, System(string("rm -f ") + original_image)); + ASSERT_EQ(0, system("rm -f delta")); +} + +} // namespace chromeos_update_engine diff --git a/install_plan.h b/install_plan.h new file mode 100644 index 00000000..81893d87 --- /dev/null +++ b/install_plan.h @@ -0,0 +1,56 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CHROMEOS_PLATFORM_UPDATE_ENGINE_INSTALL_PLAN_H__ +#define CHROMEOS_PLATFORM_UPDATE_ENGINE_INSTALL_PLAN_H__ + +#include <string> +#include "chromeos/obsolete_logging.h" + +// InstallPlan is a simple struct that contains relevant info for many +// parts of the update system about the install that should happen. + +namespace chromeos_update_engine { + +struct InstallPlan { + InstallPlan(bool is_full, + const std::string& url, + const std::string& hash, + const std::string& d_path, + const std::string& i_path) + : is_full_update(is_full), + download_url(url), + download_hash(hash), + download_path(d_path), + install_path(i_path) {} + InstallPlan() : is_full_update(false) {} + + bool is_full_update; + std::string download_url; // url to download from + std::string download_hash; // hash of the data at the url + std::string download_path; // path to downloaded file from Omaha + std::string install_path; // path to install device + + bool operator==(const InstallPlan& that) const { + return (is_full_update == that.is_full_update) && + (download_url == that.download_url) && + (download_hash == that.download_hash) && + (download_path == that.download_path) && + (install_path == that.install_path); + } + bool operator!=(const InstallPlan& that) const { + return !((*this) == that); + } + void Dump() const { + LOG(INFO) << "InstallPlan: " + << (is_full_update ? "full_update" : "delta_update") + << ", url: " << download_url << ", hash: " << download_hash + << ", path: " << download_path + << ", install_path: " << install_path; + } +}; + +} // namespace chromeos_update_engine + +#endif // CHROMEOS_PLATFORM_UPDATE_ENGINE_INSTALL_PLAN_H__ diff --git a/integration_unittest.cc b/integration_unittest.cc new file mode 100644 index 00000000..671240f9 --- /dev/null +++ b/integration_unittest.cc @@ -0,0 +1,189 @@ +// Copyright (c) 2009 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include <string> +#include <vector> +#include <glib.h> +#include <pthread.h> +#include <gtest/gtest.h> +#include "update_engine/download_action.h" +#include "update_engine/install_action.h" +#include "update_engine/libcurl_http_fetcher.h" +#include "update_engine/mock_http_fetcher.h" +#include "update_engine/omaha_request_prep_action.h" +#include "update_engine/omaha_response_handler_action.h" +#include "update_engine/postinstall_runner_action.h" +#include "update_engine/set_bootable_flag_action.h" +#include "update_engine/test_utils.h" +#include "update_engine/update_check_action.h" +#include "update_engine/utils.h" + +// The tests here integrate many Action objects together. This test that +// the objects work well together, whereas most other tests focus on a single +// action at a time. + +namespace chromeos_update_engine { + +using std::string; +using std::vector; + +class IntegrationTest : public ::testing::Test { }; + +namespace { +const char* kTestDir = "/tmp/update_engine-integration-test"; + +class IntegrationTestProcessorDelegate : public ActionProcessorDelegate { + public: + IntegrationTestProcessorDelegate() + : loop_(NULL), processing_done_called_(false) {} + virtual ~IntegrationTestProcessorDelegate() { + EXPECT_TRUE(processing_done_called_); + } + virtual void ProcessingDone(const ActionProcessor* processor) { + processing_done_called_ = true; + g_main_loop_quit(loop_); + } + + virtual void ActionCompleted(ActionProcessor* processor, + AbstractAction* action, + bool success) { + // make sure actions always succeed + EXPECT_TRUE(success); + + // Swap in the device path for PostinstallRunnerAction with a loop device + if (action->Type() == InstallAction::StaticType()) { + InstallAction* install_action = static_cast<InstallAction*>(action); + old_dev_ = install_action->GetOutputObject(); + string dev = GetUnusedLoopDevice(); + string cmd = string("losetup ") + dev + " " + kTestDir + "/dev2"; + EXPECT_EQ(0, system(cmd.c_str())); + install_action->SetOutputObject(dev); + } else if (action->Type() == PostinstallRunnerAction::StaticType()) { + PostinstallRunnerAction* postinstall_runner_action = + static_cast<PostinstallRunnerAction*>(action); + string dev = postinstall_runner_action->GetOutputObject(); + EXPECT_EQ(0, system((string("losetup -d ") + dev).c_str())); + postinstall_runner_action->SetOutputObject(old_dev_); + old_dev_ = ""; + } + } + + void set_loop(GMainLoop* loop) { + loop_ = loop; + } + + private: + GMainLoop *loop_; + bool processing_done_called_; + + // We have to change the dev for the PostinstallRunnerAction action. + // Before that runs, we store the device here, and after it runs, we + // restore it. + // This is because we use a file, rather than a device, to install into, + // but the PostinstallRunnerAction requires a real device. We set up a + // loop device pointing to the file when necessary. + string old_dev_; +}; + +gboolean TestStarter(gpointer data) { + ActionProcessor *processor = reinterpret_cast<ActionProcessor*>(data); + processor->StartProcessing(); + return FALSE; +} + +} // namespace {} + +TEST(IntegrationTest, DISABLED_RunAsRootFullInstallTest) { + ASSERT_EQ(0, getuid()); + GMainLoop *loop = g_main_loop_new(g_main_context_default(), FALSE); + + ActionProcessor processor; + IntegrationTestProcessorDelegate delegate; + delegate.set_loop(loop); + processor.set_delegate(&delegate); + + // Actions: + OmahaRequestPrepAction request_prep_action(false); + UpdateCheckAction update_check_action(new LibcurlHttpFetcher); + OmahaResponseHandlerAction response_handler_action; + DownloadAction download_action(new LibcurlHttpFetcher); + InstallAction install_action; + PostinstallRunnerAction postinstall_runner_action; + SetBootableFlagAction set_bootable_flag_action; + + // Enqueue the actions + processor.EnqueueAction(&request_prep_action); + processor.EnqueueAction(&update_check_action); + processor.EnqueueAction(&response_handler_action); + processor.EnqueueAction(&download_action); + processor.EnqueueAction(&install_action); + processor.EnqueueAction(&postinstall_runner_action); + processor.EnqueueAction(&set_bootable_flag_action); + + // Bond them together + BondActions(&request_prep_action, &update_check_action); + BondActions(&update_check_action, &response_handler_action); + BondActions(&response_handler_action, &download_action); + BondActions(&download_action, &install_action); + BondActions(&install_action, &postinstall_runner_action); + BondActions(&postinstall_runner_action, &set_bootable_flag_action); + + // Set up filesystem to trick some of the actions + ASSERT_EQ(0, System(string("rm -rf ") + kTestDir)); + ASSERT_EQ(0, system("rm -f /tmp/update_engine_test_postinst_out.txt")); + ASSERT_EQ(0, System(string("mkdir -p ") + kTestDir + "/etc")); + ASSERT_TRUE(WriteFileString(string(kTestDir) + "/etc/lsb-release", + "GOOGLE_RELEASE=0.2.0.0\n" + "GOOGLE_TRACK=unittest-track")); + ASSERT_EQ(0, System(string("touch ") + kTestDir + "/dev1")); + ASSERT_EQ(0, System(string("touch ") + kTestDir + "/dev2")); + ASSERT_TRUE(WriteFileVector(string(kTestDir) + "/dev", GenerateSampleMbr())); + + request_prep_action.set_root(kTestDir); + response_handler_action.set_boot_device(string(kTestDir) + "/dev1"); + + // Run the actions + g_timeout_add(0, &TestStarter, &processor); + g_main_loop_run(loop); + g_main_loop_unref(loop); + + // Verify the results + struct stat stbuf; + ASSERT_EQ(0, lstat((string(kTestDir) + "/dev1").c_str(), &stbuf)); + EXPECT_EQ(0, stbuf.st_size); + EXPECT_TRUE(S_ISREG(stbuf.st_mode)); + ASSERT_EQ(0, lstat((string(kTestDir) + "/dev2").c_str(), &stbuf)); + EXPECT_EQ(996147200, stbuf.st_size); + EXPECT_TRUE(S_ISREG(stbuf.st_mode)); + ASSERT_EQ(0, lstat((string(kTestDir) + "/dev").c_str(), &stbuf)); + ASSERT_EQ(512, stbuf.st_size); + EXPECT_TRUE(S_ISREG(stbuf.st_mode)); + vector<char> new_mbr; + EXPECT_TRUE(utils::ReadFile((string(kTestDir) + "/dev").c_str(), &new_mbr)); + + // Check bootable flag in MBR + for (int i = 0; i < 4; i++) { + char expected_flag = '\0'; + if (i == 1) + expected_flag = 0x80; + EXPECT_EQ(expected_flag, new_mbr[446 + 16 * i]); + } + // Check MBR signature + EXPECT_EQ(static_cast<char>(0x55), new_mbr[510]); + EXPECT_EQ(static_cast<char>(0xaa), new_mbr[511]); + + ASSERT_EQ(0, lstat("/tmp/update_engine_test_postinst_out.txt", &stbuf)); + EXPECT_TRUE(S_ISREG(stbuf.st_mode)); + string file_data; + EXPECT_TRUE(utils::ReadFileToString( + "/tmp/update_engine_test_postinst_out.txt", + &file_data)); + EXPECT_EQ("POSTINST_DONE\n", file_data); + + // cleanup + ASSERT_EQ(0, System(string("rm -rf ") + kTestDir)); + ASSERT_EQ(0, system("rm -f /tmp/update_engine_test_postinst_out.txt")); +} + +} // namespace chromeos_update_engine diff --git a/local_coverage_rate.sh b/local_coverage_rate.sh new file mode 100755 index 00000000..33c06a70 --- /dev/null +++ b/local_coverage_rate.sh @@ -0,0 +1,87 @@ +#!/bin/bash + +# Copyright (c) 2009 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Calculates the test-coverage percentage for non-test files in the +# update_engine directory. Requires a file 'app.info' to contain the +# results of running the unittests while collecting coverage data. + +cat app.info | awk -F '[,:]' ' + +BEGIN { OFS = ":"; } + +/^SF:/{ FILEN = $2; } + +/^end_of_record$/{ FILEN = ""; } + +/^DA:/{ print FILEN, $2, $3; } + +' | sort | awk -F : ' +BEGIN { + OFS = ":"; + FILEN = ""; + LINE = ""; + HITS = 0; +} +{ + NEWFILEN = $1; + NEWLINE = $2; + if ((NEWFILEN == FILEN) && (NEWLINE == LINE)) { + HITS += $3 + } else { + if (FILEN != "") { + print FILEN, LINE, HITS; + } + FILEN = NEWFILEN; + LINE = NEWLINE; + HITS = $3; + } +} +' | grep '^.*\/trunk\/src\/platform\/update_engine\/' | \ +fgrep -v '_unittest.cc:' | \ +fgrep -v '/test_utils.' | \ +fgrep -v '/test_http_server.cc' | \ +fgrep -v '/testrunner.cc' | \ +fgrep -v '/mock' | \ +fgrep -v '.pb.cc' | \ +awk -F : ' + +function printfile() { + if (FNAME != "") + printf "%-40s %4d / %4d: %5.1f%%\n", FNAME, FILE_GOOD_LINES, + (FILE_BAD_LINES + FILE_GOOD_LINES), + (FILE_GOOD_LINES * 100) / (FILE_BAD_LINES + FILE_GOOD_LINES); +} + +BEGIN { + FNAME = ""; + FILE_BAD_LINES = 0; + FILE_GOOD_LINES = 0; +} +{ + // calc filename + ARR_SIZE = split($1, PARTS, "/"); + NEWFNAME = PARTS[ARR_SIZE]; + if (NEWFNAME != FNAME) { + printfile(); + FILE_BAD_LINES = 0; + FILE_GOOD_LINES = 0; + FNAME = NEWFNAME; + } + if ($3 == "0") { + BAD_LINES += 1; + FILE_BAD_LINES += 1; + } else { + GOOD_LINES += 1; + FILE_GOOD_LINES += 1; + } +} + +END { + printfile(); + print "---\nSummary: tested " GOOD_LINES " / " (BAD_LINES + GOOD_LINES); + printf "Test coverage: %.1f%%\n", ((GOOD_LINES * 100) / (BAD_LINES + GOOD_LINES)); +} +' diff --git a/omaha_request_prep_action.cc b/omaha_request_prep_action.cc new file mode 100644 index 00000000..21d57995 --- /dev/null +++ b/omaha_request_prep_action.cc @@ -0,0 +1,93 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "update_engine/omaha_request_prep_action.h" +#include <sys/utsname.h> +#include <string> +#include "update_engine/utils.h" + +using std::string; + +// This gathers local system information and prepares info used by the +// update check action. + +namespace chromeos_update_engine { + +void OmahaRequestPrepAction::PerformAction() { + // TODO(adlr): honor force_full_update_ + const string machine_id(GetMachineId()); + const string version(GetLsbValue("GOOGLE_RELEASE")); + const string sp(version + "_" + GetMachineType()); + const string track(GetLsbValue("GOOGLE_TRACK")); + + UpdateCheckParams out(machine_id, // machine_id + machine_id, // user_id (use machine_id) + UpdateCheckParams::kOsPlatform, + UpdateCheckParams::kOsVersion, + sp, // e.g. 0.2.3.3_i686 + UpdateCheckParams::kAppId, + version, // app version (from lsb-release) + "en-US", //lang + track); // track + + CHECK(HasOutputPipe()); + SetOutputObject(out); + processor_->ActionComplete(this, true); +} + +std::string OmahaRequestPrepAction::GetMachineId() const { + FILE* fp = popen("/sbin/ifconfig", "r"); + if (!fp) + return ""; + string data; + for (;;) { + char buffer[1000]; + size_t r = fread(buffer, 1, sizeof(buffer), fp); + if (r <= 0) + break; + data.insert(data.end(), buffer, buffer + r); + } + fclose(fp); + // scan data for MAC address + string::size_type pos = data.find(" HWaddr "); + if (pos == string::npos) + return ""; + // 3 * 6 - 1 is the number of bytes of the hwaddr. + return data.substr(pos + strlen(" HWaddr "), 3 * 6 - 1); +} + +std::string OmahaRequestPrepAction::GetLsbValue(const std::string& key) const { + string files[] = {utils::kStatefulPartition + "/etc/lsb-release", + "/etc/lsb-release"}; + for (unsigned int i = 0; i < arraysize(files); i++) { + string file_data; + if (!utils::ReadFileToString(root_ + files[i], &file_data)) + continue; + string::size_type pos = 0; + if (!utils::StringHasPrefix(file_data, key + "=")) { + pos = file_data.find(string("\n") + key + "="); + if (pos != string::npos) + pos++; // advance past \n + } + if (pos == string::npos) + continue; + pos += key.size() + 1; // advance past the key and the '=' + string::size_type endpos = file_data.find('\n', pos); + string::size_type length = + (endpos == string::npos ? string::npos : endpos - pos); + return file_data.substr(pos, length); + } + // not found + return ""; +} + +std::string OmahaRequestPrepAction::GetMachineType() const { + struct utsname buf; + string ret; + if (uname(&buf) == 0) + ret = buf.machine; + return ret; +} + +} // namespace chromeos_update_engine diff --git a/omaha_request_prep_action.h b/omaha_request_prep_action.h new file mode 100644 index 00000000..ca7fd178 --- /dev/null +++ b/omaha_request_prep_action.h @@ -0,0 +1,74 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CHROMEOS_PLATFORM_UPDATE_ENGINE_OMAHA_RESQUEST_PREP_ACTION_H__ +#define CHROMEOS_PLATFORM_UPDATE_ENGINE_OMAHA_RESQUEST_PREP_ACTION_H__ + +#include <string> +#include "update_engine/action.h" +#include "update_engine/update_check_action.h" + +// This gathers local system information and prepares info used by the +// update check action. + +namespace chromeos_update_engine { + +class OmahaRequestPrepAction; +class NoneType; + +template<> +class ActionTraits<OmahaRequestPrepAction> { + public: + typedef NoneType InputObjectType; + typedef UpdateCheckParams OutputObjectType; +}; + +class OmahaRequestPrepAction : public Action<OmahaRequestPrepAction> { + public: + explicit OmahaRequestPrepAction(bool force_full_update) + : force_full_update_(force_full_update) {} + typedef ActionTraits<OmahaRequestPrepAction>::InputObjectType + InputObjectType; + typedef ActionTraits<OmahaRequestPrepAction>::OutputObjectType + OutputObjectType; + void PerformAction(); + + // This is a synchronous action, and thus TerminateProcessing() should + // never be called + void TerminateProcessing() { CHECK(false); } + + // Debugging/logging + static std::string StaticType() { return "OmahaRequestPrepAction"; } + std::string Type() const { return StaticType(); } + + // For unit-tests. + void set_root(const std::string& root) { + root_ = root; + } + + private: + // Gets a machine-local ID (for now, first MAC address we find) + std::string GetMachineId() const; + + // Fetches the value for a given key from + // /mnt/stateful_partition/etc/lsb-release if possible. Failing that, + // it looks for the key in /etc/lsb-release . + std::string GetLsbValue(const std::string& key) const; + + // Gets the machine type (e.g. "i686") + std::string GetMachineType() const; + + // Set to true if this should set up the Update Check Action to do + // a full update + bool force_full_update_; + + // When reading files, prepend root_ to the paths. Useful for testing. + std::string root_; + + DISALLOW_COPY_AND_ASSIGN(OmahaRequestPrepAction); +}; + +} // namespace chromeos_update_engine + +#endif // CHROMEOS_PLATFORM_UPDATE_ENGINE_OMAHA_RESQUEST_PREP_ACTION_H__ diff --git a/omaha_request_prep_action_unittest.cc b/omaha_request_prep_action_unittest.cc new file mode 100644 index 00000000..19da5e57 --- /dev/null +++ b/omaha_request_prep_action_unittest.cc @@ -0,0 +1,175 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include <stdio.h> +#include <string> +#include <gtest/gtest.h> +#include "update_engine/install_plan.h" +#include "update_engine/omaha_request_prep_action.h" +#include "update_engine/test_utils.h" +#include "update_engine/utils.h" + +using std::string; + +namespace chromeos_update_engine { + +class OmahaRequestPrepActionTest : public ::testing::Test { + public: + // Return true iff the OmahaResponseHandlerAction succeeded. + // if out is non-NULL, it's set w/ the response from the action. + bool DoTest(bool force_full_update, UpdateCheckParams* out); + static const string kTestDir; +}; + +const string OmahaRequestPrepActionTest::kTestDir = "request_prep_action-test"; + +class OmahaRequestPrepActionProcessorDelegate + : public ActionProcessorDelegate { + public: + OmahaRequestPrepActionProcessorDelegate() + : success_(false), + success_set_(false) {} + void ActionCompleted(ActionProcessor* processor, + AbstractAction* action, + bool success) { + if (action->Type() == OmahaRequestPrepAction::StaticType()) { + success_ = success; + success_set_ = true; + } + } + bool success_; + bool success_set_; +}; + +bool OmahaRequestPrepActionTest::DoTest(bool force_full_update, + UpdateCheckParams* out) { + ActionProcessor processor; + OmahaRequestPrepActionProcessorDelegate delegate; + processor.set_delegate(&delegate); + + OmahaRequestPrepAction request_prep_action(force_full_update); + request_prep_action.set_root(string("./") + kTestDir); + ObjectCollectorAction<UpdateCheckParams> collector_action; + BondActions(&request_prep_action, &collector_action); + processor.EnqueueAction(&request_prep_action); + processor.EnqueueAction(&collector_action); + processor.StartProcessing(); + EXPECT_TRUE(!processor.IsRunning()) + << "Update test to handle non-asynch actions"; + if (out) + *out = collector_action.object(); + EXPECT_TRUE(delegate.success_set_); + return delegate.success_; +} + +namespace { +// Returns true iff str is formatted as a mac address +bool IsValidMac(const string& str) { + if (str.size() != (3 * 6 - 1)) + return false; + for (unsigned int i = 0; i < str.size(); i++) { + char c = str[i]; + switch (i % 3) { + case 0: // fall through + case 1: + if ((c >= '0') && (c <= '9')) + break; + if ((c >= 'a') && (c <= 'f')) + break; + if ((c >= 'A') && (c <= 'F')) + break; + return false; + case 2: + if (c == ':') + break; + return false; + } + } + return true; +} +string GetMachineType() { + FILE* fp = popen("uname -m", "r"); + if (!fp) + return ""; + string ret; + for (;;) { + char buffer[10]; + size_t r = fread(buffer, 1, sizeof(buffer), fp); + if (r == 0) + break; + ret.insert(ret.begin(), buffer, buffer + r); + } + // strip trailing '\n' if it exists + if ((*ret.rbegin()) == '\n') + ret.resize(ret.size() - 1); + fclose(fp); + return ret; +} +} // namespace {} + +TEST_F(OmahaRequestPrepActionTest, SimpleTest) { + ASSERT_EQ(0, System(string("mkdir -p ") + kTestDir + "/etc")); + { + ASSERT_TRUE(WriteFileString( + kTestDir + "/etc/lsb-release", + "GOOGLE_FOO=bar\nGOOGLE_RELEASE=0.2.2.3\nGOOGLE_TRACK=footrack")); + UpdateCheckParams out; + EXPECT_TRUE(DoTest(false, &out)); + EXPECT_TRUE(IsValidMac(out.machine_id)); + // for now we're just using the machine id here + EXPECT_TRUE(IsValidMac(out.user_id)); + EXPECT_EQ("Chrome OS", out.os_platform); + EXPECT_EQ(string("0.2.2.3_") + GetMachineType(), out.os_sp); + EXPECT_EQ("{87efface-864d-49a5-9bb3-4b050a7c227a}", out.app_id); + EXPECT_EQ("0.2.2.3", out.app_version); + EXPECT_EQ("en-US", out.app_lang); + EXPECT_EQ("footrack", out.app_track); + } + EXPECT_EQ(0, System(string("rm -rf ") + kTestDir)); +} + +TEST_F(OmahaRequestPrepActionTest, MissingTrackTest) { + ASSERT_EQ(0, System(string("mkdir -p ") + kTestDir + "/etc")); + { + ASSERT_TRUE(WriteFileString( + kTestDir + "/etc/lsb-release", + "GOOGLE_FOO=bar\nGOOGLE_RELEASE=0.2.2.3\nGOOGLE_TRXCK=footrack")); + UpdateCheckParams out; + EXPECT_TRUE(DoTest(false, &out)); + EXPECT_TRUE(IsValidMac(out.machine_id)); + // for now we're just using the machine id here + EXPECT_TRUE(IsValidMac(out.user_id)); + EXPECT_EQ("Chrome OS", out.os_platform); + EXPECT_EQ(string("0.2.2.3_") + GetMachineType(), out.os_sp); + EXPECT_EQ("{87efface-864d-49a5-9bb3-4b050a7c227a}", out.app_id); + EXPECT_EQ("0.2.2.3", out.app_version); + EXPECT_EQ("en-US", out.app_lang); + EXPECT_EQ("", out.app_track); + } + EXPECT_EQ(0, System(string("rm -rf ") + kTestDir)); +} + +TEST_F(OmahaRequestPrepActionTest, ConfusingReleaseTest) { + ASSERT_EQ(0, System(string("mkdir -p ") + kTestDir + "/etc")); + { + ASSERT_TRUE(WriteFileString( + kTestDir + "/etc/lsb-release", + "GOOGLE_FOO=GOOGLE_RELEASE=1.2.3.4\n" + "GOOGLE_RELEASE=0.2.2.3\nGOOGLE_TRXCK=footrack")); + UpdateCheckParams out; + EXPECT_TRUE(DoTest(false, &out)); + EXPECT_TRUE(IsValidMac(out.machine_id)); + // for now we're just using the machine id here + EXPECT_TRUE(IsValidMac(out.user_id)); + EXPECT_EQ("Chrome OS", out.os_platform); + EXPECT_EQ(string("0.2.2.3_") + GetMachineType(), out.os_sp); + EXPECT_EQ("{87efface-864d-49a5-9bb3-4b050a7c227a}", out.app_id); + EXPECT_EQ("0.2.2.3", out.app_version); + EXPECT_EQ("en-US", out.app_lang); + EXPECT_EQ("", out.app_track); + } + EXPECT_EQ(0, System(string("rm -rf ") + kTestDir)); +} + +} // namespace chromeos_update_engine diff --git a/omaha_response_handler_action.cc b/omaha_response_handler_action.cc new file mode 100644 index 00000000..0a8472ea --- /dev/null +++ b/omaha_response_handler_action.cc @@ -0,0 +1,76 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "update_engine/omaha_response_handler_action.h" +#include <string> +#include "update_engine/utils.h" + +using std::string; + +namespace chromeos_update_engine { + +namespace { +// If the file part of the download URL contains kFullUpdateTag, then and +// only then do we assume it's a full update. Otherwise, we assume it's a +// delta update. +const string kFullUpdateTag = "_FULL_"; +} // namespace + +void OmahaResponseHandlerAction::PerformAction() { + CHECK(HasInputObject()); + ScopedActionCompleter completer(processor_, this); + const UpdateCheckResponse& response = GetInputObject(); + if (!response.update_exists) { + LOG(INFO) << "There are no updates. Aborting."; + return; + } + InstallPlan install_plan; + install_plan.download_url = response.codebase; + install_plan.download_hash = response.hash; + TEST_AND_RETURN(GetInstallDev( + (!boot_device_.empty() ? boot_device_ : utils::BootDevice()), + &install_plan.install_path)); + + // Get the filename part of the url. Assume that if it has kFullUpdateTag + // in the name, it's a full update. + string::size_type last_slash = response.codebase.rfind('/'); + string filename; + if (last_slash == string::npos) + filename = response.codebase; + else + filename = response.codebase.substr(last_slash + 1); + install_plan.is_full_update = (filename.find(kFullUpdateTag) != string::npos); + + if (filename.size() > 255) { + // Very long name. Let's shorten it + filename.resize(255); + } + // TODO(adlr): come up with a better place to download to: + install_plan.download_path = utils::kStatefulPartition + "/" + filename; + if (HasOutputPipe()) + SetOutputObject(install_plan); + LOG(INFO) << "Using this install plan:"; + install_plan.Dump(); + completer.set_success(true); +} + +bool OmahaResponseHandlerAction::GetInstallDev(const std::string& boot_dev, + std::string* install_dev) { + TEST_AND_RETURN_FALSE(!boot_dev.empty()); + string ret(boot_dev); + char last_char = *ret.rbegin(); + TEST_AND_RETURN_FALSE((last_char >= '0') && (last_char <= '9')); + // Chop off last char + ret.resize(ret.size() - 1); + // If last_char is odd (1 or 3), increase it, otherwise decrease + if (last_char % 2) + last_char++; + else + last_char--; + ret += last_char; + *install_dev = ret; + return true; +} + +} // namespace chromeos_update_engine diff --git a/omaha_response_handler_action.h b/omaha_response_handler_action.h new file mode 100644 index 00000000..e25de289 --- /dev/null +++ b/omaha_response_handler_action.h @@ -0,0 +1,65 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CHROMEOS_PLATFORM_UPDATE_ENGINE_OMAHA_RESPONSE_HANDLER_ACTION_H__ +#define CHROMEOS_PLATFORM_UPDATE_ENGINE_OMAHA_RESPONSE_HANDLER_ACTION_H__ + +#include <string> +#include "update_engine/action.h" +#include "update_engine/install_plan.h" +#include "update_engine/update_check_action.h" + +// This class reads in an Omaha response and converts what it sees into +// an install plan which is passed out. + +namespace chromeos_update_engine { + +class OmahaResponseHandlerAction; + +template<> +class ActionTraits<OmahaResponseHandlerAction> { + public: + typedef UpdateCheckResponse InputObjectType; + typedef InstallPlan OutputObjectType; +}; + +class OmahaResponseHandlerAction : public Action<OmahaResponseHandlerAction> { + public: + OmahaResponseHandlerAction() {} + typedef ActionTraits<OmahaResponseHandlerAction>::InputObjectType + InputObjectType; + typedef ActionTraits<OmahaResponseHandlerAction>::OutputObjectType + OutputObjectType; + void PerformAction(); + + // This is a synchronous action, and thus TerminateProcessing() should + // never be called + void TerminateProcessing() { CHECK(false); } + + // For unit-testing + void set_boot_device(const std::string& boot_device) { + boot_device_ = boot_device; + } + + // Debugging/logging + static std::string StaticType() { return "OmahaResponseHandlerAction"; } + std::string Type() const { return StaticType(); } + + private: + // Assumes you want to install on the "other" device, where the other + // device is what you get if you swap 1 for 2 or 3 for 4 or vice versa + // for the number at the end of the boot device. E.g., /dev/sda1 -> /dev/sda2 + // or /dev/sda4 -> /dev/sda3 + static bool GetInstallDev(const std::string& boot_dev, + std::string* install_dev); + + // set to non-empty in unit tests + std::string boot_device_; + + DISALLOW_COPY_AND_ASSIGN(OmahaResponseHandlerAction); +}; + +} // namespace chromeos_update_engine + +#endif // CHROMEOS_PLATFORM_UPDATE_ENGINE_OMAHA_RESPONSE_HANDLER_ACTION_H__ diff --git a/omaha_response_handler_action_unittest.cc b/omaha_response_handler_action_unittest.cc new file mode 100644 index 00000000..0248daf9 --- /dev/null +++ b/omaha_response_handler_action_unittest.cc @@ -0,0 +1,152 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include <string> +#include <gtest/gtest.h> +#include "update_engine/omaha_response_handler_action.h" +#include "update_engine/test_utils.h" +#include "update_engine/utils.h" + +using std::string; + +namespace chromeos_update_engine { + +class OmahaResponseHandlerActionTest : public ::testing::Test { + public: + // Return true iff the OmahaResponseHandlerAction succeeded. + // If out is non-NULL, it's set w/ the response from the action. + bool DoTest(const UpdateCheckResponse& in, + const string& boot_dev, + InstallPlan* out); +}; + +class OmahaResponseHandlerActionProcessorDelegate + : public ActionProcessorDelegate { + public: + OmahaResponseHandlerActionProcessorDelegate() + : success_(false), + success_set_(false) {} + void ActionCompleted(ActionProcessor* processor, + AbstractAction* action, + bool success) { + if (action->Type() == OmahaResponseHandlerAction::StaticType()) { + success_ = success; + success_set_ = true; + } + } + bool success_; + bool success_set_; +}; + +namespace { +const string kLongName = + "very_long_name_and_no_slashes-very_long_name_and_no_slashes" + "very_long_name_and_no_slashes-very_long_name_and_no_slashes" + "very_long_name_and_no_slashes-very_long_name_and_no_slashes" + "very_long_name_and_no_slashes-very_long_name_and_no_slashes" + "very_long_name_and_no_slashes-very_long_name_and_no_slashes" + "very_long_name_and_no_slashes-very_long_name_and_no_slashes" + "very_long_name_and_no_slashes-very_long_name_and_no_slashes" + "-the_update_a.b.c.d_DELTA_.tgz"; +} // namespace {} + +bool OmahaResponseHandlerActionTest::DoTest(const UpdateCheckResponse& in, + const string& boot_dev, + InstallPlan* out) { + ActionProcessor processor; + OmahaResponseHandlerActionProcessorDelegate delegate; + processor.set_delegate(&delegate); + + ObjectFeederAction<UpdateCheckResponse> feeder_action; + feeder_action.set_obj(in); + OmahaResponseHandlerAction response_handler_action; + response_handler_action.set_boot_device(boot_dev); + BondActions(&feeder_action, &response_handler_action); + ObjectCollectorAction<InstallPlan> collector_action; + BondActions(&response_handler_action, &collector_action); + processor.EnqueueAction(&feeder_action); + processor.EnqueueAction(&response_handler_action); + processor.EnqueueAction(&collector_action); + processor.StartProcessing(); + EXPECT_TRUE(!processor.IsRunning()) + << "Update test to handle non-asynch actions"; + if (out) + *out = collector_action.object(); + EXPECT_TRUE(delegate.success_set_); + return delegate.success_; +} + +TEST_F(OmahaResponseHandlerActionTest, SimpleTest) { + { + UpdateCheckResponse in; + in.update_exists = true; + in.display_version = "a.b.c.d"; + in.codebase = "http://foo/the_update_a.b.c.d_FULL_.tgz"; + in.more_info_url = "http://more/info"; + in.hash = "HASH+"; + in.size = 12; + in.needs_admin = true; + in.prompt = false; + InstallPlan install_plan; + EXPECT_TRUE(DoTest(in, "/dev/sda1", &install_plan)); + EXPECT_TRUE(install_plan.is_full_update); + EXPECT_EQ(in.codebase, install_plan.download_url); + EXPECT_EQ(in.hash, install_plan.download_hash); + EXPECT_EQ(utils::kStatefulPartition + "/the_update_a.b.c.d_FULL_.tgz", + install_plan.download_path); + EXPECT_EQ("/dev/sda2", install_plan.install_path); + } + { + UpdateCheckResponse in; + in.update_exists = true; + in.display_version = "a.b.c.d"; + in.codebase = "http://foo/the_update_a.b.c.d_DELTA_.tgz"; + in.more_info_url = "http://more/info"; + in.hash = "HASHj+"; + in.size = 12; + in.needs_admin = true; + in.prompt = true; + InstallPlan install_plan; + EXPECT_TRUE(DoTest(in, "/dev/sda4", &install_plan)); + EXPECT_FALSE(install_plan.is_full_update); + EXPECT_EQ(in.codebase, install_plan.download_url); + EXPECT_EQ(in.hash, install_plan.download_hash); + EXPECT_EQ(utils::kStatefulPartition + "/the_update_a.b.c.d_DELTA_.tgz", + install_plan.download_path); + EXPECT_EQ("/dev/sda3", install_plan.install_path); + } + { + UpdateCheckResponse in; + in.update_exists = true; + in.display_version = "a.b.c.d"; + in.codebase = kLongName; + in.more_info_url = "http://more/info"; + in.hash = "HASHj+"; + in.size = 12; + in.needs_admin = true; + in.prompt = true; + InstallPlan install_plan; + EXPECT_TRUE(DoTest(in, "/dev/sda4", &install_plan)); + EXPECT_FALSE(install_plan.is_full_update); + EXPECT_EQ(in.codebase, install_plan.download_url); + EXPECT_EQ(in.hash, install_plan.download_hash); + EXPECT_EQ(utils::kStatefulPartition + "/" + kLongName.substr(0, 255), + install_plan.download_path); + EXPECT_EQ("/dev/sda3", install_plan.install_path); + } +} + +TEST_F(OmahaResponseHandlerActionTest, NoUpdatesTest) { + UpdateCheckResponse in; + in.update_exists = false; + InstallPlan install_plan; + EXPECT_FALSE(DoTest(in, "/dev/sda1", &install_plan)); + EXPECT_FALSE(install_plan.is_full_update); + EXPECT_EQ("", install_plan.download_url); + EXPECT_EQ("", install_plan.download_hash); + EXPECT_EQ("", install_plan.download_path); + EXPECT_EQ("", install_plan.install_path); +} + +} // namespace chromeos_update_engine diff --git a/postinstall_runner_action.cc b/postinstall_runner_action.cc new file mode 100644 index 00000000..41228607 --- /dev/null +++ b/postinstall_runner_action.cc @@ -0,0 +1,49 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "update_engine/postinstall_runner_action.h" +#include <sys/mount.h> +#include <stdlib.h> +#include "update_engine/utils.h" + +namespace chromeos_update_engine { + +using std::string; + +namespace { +const string kMountPath(utils::kStatefulPartition + "/au_destination"); +const string kPostinstallScript("/postinst"); +} + +void PostinstallRunnerAction::PerformAction() { + CHECK(HasInputObject()); + const string install_device = GetInputObject(); + + int rc = mount(install_device.c_str(), kMountPath.c_str(), "ext3", 0, NULL); + if (rc < 0) { + LOG(ERROR) << "Unable to mount destination device " << install_device + << " onto " << kMountPath; + processor_->ActionComplete(this, false); + return; + } + + // run postinstall script + rc = system((kMountPath + kPostinstallScript + " " + install_device).c_str()); + bool success = (rc == 0); + if (!success) { + LOG(ERROR) << "Postinst command failed with code: " << rc; + } + + rc = umount(kMountPath.c_str()); + if (rc < 0) { + // non-fatal + LOG(ERROR) << "Unable to umount destination device"; + } + if (success && HasOutputPipe()) { + SetOutputObject(install_device); + } + processor_->ActionComplete(this, success); +} + +} // namespace chromeos_update_engine diff --git a/postinstall_runner_action.h b/postinstall_runner_action.h new file mode 100644 index 00000000..00478cc8 --- /dev/null +++ b/postinstall_runner_action.h @@ -0,0 +1,51 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CHROMEOS_PLATFORM_UPDATE_ENGINE_POSTINSTALL_RUNNER_ACTION_H__ +#define CHROMEOS_PLATFORM_UPDATE_ENGINE_POSTINSTALL_RUNNER_ACTION_H__ + +#include <string> +#include "update_engine/action.h" + +// The Postinstall Runner Action is responsible for running the postinstall +// script of a successfully downloaded update. + +namespace chromeos_update_engine { + +class PostinstallRunnerAction; +class NoneType; + +template<> +class ActionTraits<PostinstallRunnerAction> { + public: + // Takes the device path as input + typedef std::string InputObjectType; + // Passes the device path as output + typedef std::string OutputObjectType; +}; + +class PostinstallRunnerAction : public Action<PostinstallRunnerAction> { + public: + PostinstallRunnerAction() {} + typedef ActionTraits<PostinstallRunnerAction>::InputObjectType + InputObjectType; + typedef ActionTraits<PostinstallRunnerAction>::OutputObjectType + OutputObjectType; + void PerformAction(); + + // This is a synchronous action, and thus TerminateProcessing() should + // never be called + void TerminateProcessing() { CHECK(false); } + + // Debugging/logging + static std::string StaticType() { return "PostinstallRunnerAction"; } + std::string Type() const { return StaticType(); } + + private: + DISALLOW_COPY_AND_ASSIGN(PostinstallRunnerAction); +}; + +} // namespace chromeos_update_engine + +#endif // CHROMEOS_PLATFORM_UPDATE_ENGINE_POSTINSTALL_RUNNER_ACTION_H__ diff --git a/postinstall_runner_action_unittest.cc b/postinstall_runner_action_unittest.cc new file mode 100644 index 00000000..c43cb8ac --- /dev/null +++ b/postinstall_runner_action_unittest.cc @@ -0,0 +1,155 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> +#include <string> +#include <vector> +#include <gtest/gtest.h> +#include "update_engine/postinstall_runner_action.h" +#include "update_engine/test_utils.h" +#include "update_engine/utils.h" + +using std::string; +using std::vector; + +namespace chromeos_update_engine { + +class PostinstallRunnerActionTest : public ::testing::Test { + public: + void DoTest(bool do_losetup, bool do_err_script); +}; + +class PostinstActionProcessorDelegate : public ActionProcessorDelegate { + public: + PostinstActionProcessorDelegate() : success_(false), success_set_(false) {} + void ActionCompleted(ActionProcessor* processor, + AbstractAction* action, + bool success) { + if (action->Type() == PostinstallRunnerAction::StaticType()) { + success_ = success; + success_set_ = true; + } + } + bool success_; + bool success_set_; +}; + +TEST_F(PostinstallRunnerActionTest, RunAsRootSimpleTest) { + ASSERT_EQ(0, getuid()); + DoTest(true, false); +} + +TEST_F(PostinstallRunnerActionTest, RunAsRootCantMountTest) { + ASSERT_EQ(0, getuid()); + DoTest(false, false); +} + +TEST_F(PostinstallRunnerActionTest, RunAsRootErrScriptTest) { + ASSERT_EQ(0, getuid()); + DoTest(true, true); +} + +void PostinstallRunnerActionTest::DoTest(bool do_losetup, bool do_err_script) { + ASSERT_EQ(0, getuid()) << "Run me as root. Ideally don't run other tests " + << "as root, tho."; + + const string mountpoint(utils::kStatefulPartition + "/au_destination"); + + string cwd; + { + vector<char> buf(1000); + ASSERT_EQ(&buf[0], getcwd(&buf[0], buf.size())); + cwd = string(&buf[0], strlen(&buf[0])); + } + + // create the au destination, if it doesn't exist + ASSERT_EQ(0, System(string("mkdir -p ") + mountpoint)); + + // create 10MiB sparse file + ASSERT_EQ(0, system("dd if=/dev/zero of=image.dat seek=10485759 bs=1 " + "count=1")); + + // format it as ext3 + ASSERT_EQ(0, system("mkfs.ext3 -F image.dat")); + + // mount it + ASSERT_EQ(0, System(string("mount -o loop image.dat ") + mountpoint)); + + // put a postinst script in + string script = string("#!/bin/bash\ntouch ") + cwd + "/postinst_called\n"; + if (do_err_script) { + script = "#!/bin/bash\nexit 1"; + } + ASSERT_TRUE(WriteFileString(mountpoint + "/postinst", script)); + ASSERT_EQ(0, System(string("chmod a+x ") + mountpoint + "/postinst")); + + ASSERT_EQ(0, System(string("umount ") + mountpoint)); + + ASSERT_EQ(0, System(string("rm -f ") + cwd + "/postinst_called")); + + // get a loop device we can use for the install device + FILE* find_dev_cmd = popen("losetup -f", "r"); + ASSERT_TRUE(find_dev_cmd); + + char dev[100] = {0}; + size_t r = fread(dev, 1, sizeof(dev), find_dev_cmd); + ASSERT_GT(r, 0); + ASSERT_LT(r, sizeof(dev)); + ASSERT_TRUE(feof(find_dev_cmd)); + fclose(find_dev_cmd); + + // strip trailing newline on dev + if (dev[strlen(dev) - 1] == '\n') + dev[strlen(dev) - 1] = '\0'; + + if (do_losetup) + ASSERT_EQ(0, System(string("losetup ") + dev + " " + cwd + "/image.dat")); + + ActionProcessor processor; + ObjectFeederAction<string> feeder_action; + feeder_action.set_obj(dev); + PostinstallRunnerAction runner_action; + BondActions(&feeder_action, &runner_action); + ObjectCollectorAction<string> collector_action; + BondActions(&runner_action, &collector_action); + PostinstActionProcessorDelegate delegate; + processor.EnqueueAction(&feeder_action); + processor.EnqueueAction(&runner_action); + processor.EnqueueAction(&collector_action); + processor.set_delegate(&delegate); + processor.StartProcessing(); + ASSERT_FALSE(processor.IsRunning()) + << "Update test to handle non-asynch actions"; + + EXPECT_TRUE(delegate.success_set_); + EXPECT_EQ(do_losetup && !do_err_script, delegate.success_); + EXPECT_EQ(do_losetup && !do_err_script, !collector_action.object().empty()); + if (do_losetup && !do_err_script) { + EXPECT_EQ(dev, collector_action.object()); + } + + struct stat stbuf; + int rc = lstat((string(cwd) + "/postinst_called").c_str(), &stbuf); + if (do_losetup && !do_err_script) + ASSERT_EQ(0, rc); + else + ASSERT_LT(rc, 0); + + if (do_losetup) + ASSERT_EQ(0, System(string("losetup -d ") + dev)); + ASSERT_EQ(0, System(string("rm -f ") + cwd + "/postinst_called")); + ASSERT_EQ(0, System(string("rm -f ") + cwd + "/image.dat")); +} + +// Death tests don't seem to be working on Hardy +TEST_F(PostinstallRunnerActionTest, DISABLED_RunAsRootDeathTest) { + ASSERT_EQ(0, getuid()); + PostinstallRunnerAction runner_action; + ASSERT_DEATH({ runner_action.TerminateProcessing(); }, + "postinstall_runner_action.h:.*] Check failed"); +} + +} // namespace chromeos_update_engine diff --git a/set_bootable_flag_action.cc b/set_bootable_flag_action.cc new file mode 100644 index 00000000..8075b2cb --- /dev/null +++ b/set_bootable_flag_action.cc @@ -0,0 +1,117 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "update_engine/set_bootable_flag_action.h" +#include <sys/stat.h> +#include <sys/types.h> +#include <errno.h> +#include <fcntl.h> +#include <string> +#include "update_engine/utils.h" + +using std::string; + +namespace chromeos_update_engine { + +void SetBootableFlagAction::PerformAction() { + ScopedActionCompleter completer(processor_, this); + + if (!HasInputObject() || GetInputObject().empty()) { + LOG(ERROR) << "SetBootableFlagAction: No input object."; + return; + } + string device = GetInputObject(); + + if (device.size() < 2) { + LOG(ERROR) << "Device name too short: " << device; + return; + } + + // Make sure device is valid. + const char last_char = device[device.size() - 1]; + if ((last_char < '1') || (last_char > '4')) { + LOG(ERROR) << "Bad device:" << device; + return; + } + + // We were passed the partition_number'th partition; indexed from 1, not 0 + int partition_number = last_char - '0'; + + const char second_to_last_char = device[device.size() - 2]; + if ((second_to_last_char >= '0') && (second_to_last_char <= '9')) { + LOG(ERROR) << "Bad device:" << device; + return; + } + + // Strip trailing 1-4 off end of device + device.resize(device.size() - 1); + + char mbr[512]; // MBR is the first 512 bytes of the device + if (!ReadMbr(mbr, sizeof(mbr), device.c_str())) + return; + + // Check MBR. Verify that last two byes are 0x55aa. This is the MBR signature. + if ((mbr[510] != static_cast<char>(0x55)) || + (mbr[511] != static_cast<char>(0xaa))) { + LOG(ERROR) << "Bad MBR signature"; + return; + } + + // Mark partition passed in bootable and all other partitions non bootable. + // There are 4 partition table entries, each 16 bytes, stored consecutively + // beginning at byte 446. Within each entry, the first byte is the + // bootable flag. It's set to 0x80 (bootable) or 0x00 (not bootable). + for (int i = 0; i < 4; i++) { + int offset = 446 + 16 * i; + + // partition_number is indexed from 1, not 0 + if ((i + 1) == partition_number) + mbr[offset] = 0x80; + else + mbr[offset] = '\0'; + } + + // Write MBR back to disk + bool success = WriteMbr(mbr, sizeof(mbr), device.c_str()); + if (success) { + completer.set_success(true); + if (HasOutputPipe()) { + SetOutputObject(GetInputObject()); + } + } +} + +bool SetBootableFlagAction::ReadMbr(char* buffer, + int buffer_len, + const char* device) { + int fd = open(device, O_RDONLY, 0); + TEST_AND_RETURN_FALSE(fd >= 0); + + ssize_t r = read(fd, buffer, buffer_len); + close(fd); + TEST_AND_RETURN_FALSE(r == buffer_len); + + return true; +} + +bool SetBootableFlagAction::WriteMbr(const char* buffer, + int buffer_len, + const char* device) { + int fd = open(device, O_WRONLY, 0666); + TEST_AND_RETURN_FALSE(fd >= 0); + ScopedFdCloser fd_closer(&fd); + + ssize_t bytes_written = 0; + while (bytes_written < buffer_len) { + ssize_t r = write(fd, buffer + bytes_written, buffer_len - bytes_written); + TEST_AND_RETURN_FALSE_ERRNO(r >= 0); + bytes_written += r; + } + TEST_AND_RETURN_FALSE(bytes_written == buffer_len); + + return true; +} + + +} // namespace chromeos_update_engine diff --git a/set_bootable_flag_action.h b/set_bootable_flag_action.h new file mode 100644 index 00000000..44e95556 --- /dev/null +++ b/set_bootable_flag_action.h @@ -0,0 +1,59 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CHROMEOS_PLATFORM_UPDATE_ENGINE_SET_BOOTABLE_FLAG_ACTION_H__ +#define CHROMEOS_PLATFORM_UPDATE_ENGINE_SET_BOOTABLE_FLAG_ACTION_H__ + +#include <string> +#include "update_engine/action.h" + +// This class takes in a device via the input pipe. The device is the +// partition (e.g. /dev/sda1), not the full device (e.g. /dev/sda). +// It will make that device bootable by editing the partition table +// in the root device. Currently, this class doesn't support extended +// partitions. + +namespace chromeos_update_engine { + +class SetBootableFlagAction; + +template<> +class ActionTraits<SetBootableFlagAction> { + public: + // Takes the device path as input. + typedef std::string InputObjectType; + // Passes the device path as output + typedef std::string OutputObjectType; +}; + +class SetBootableFlagAction : public Action<SetBootableFlagAction> { + public: + SetBootableFlagAction() {} + typedef ActionTraits<SetBootableFlagAction>::InputObjectType + InputObjectType; + typedef ActionTraits<SetBootableFlagAction>::OutputObjectType + OutputObjectType; + void PerformAction(); + + // This is a synchronous action, and thus TerminateProcessing() should + // never be called + void TerminateProcessing() { CHECK(false); } + + // Debugging/logging + static std::string StaticType() { return "SetBootableFlagAction"; } + std::string Type() const { return StaticType(); } + + private: + // Returns true on success + bool ReadMbr(char* buffer, int buffer_len, const char* device); + + // Returns true on success + bool WriteMbr(const char* buffer, int buffer_len, const char* device); + + DISALLOW_COPY_AND_ASSIGN(SetBootableFlagAction); +}; + +} // namespace chromeos_update_engine + +#endif // CHROMEOS_PLATFORM_UPDATE_ENGINE_SET_BOOTABLE_FLAG_ACTION_H__ diff --git a/set_bootable_flag_action_unittest.cc b/set_bootable_flag_action_unittest.cc new file mode 100644 index 00000000..215c189e --- /dev/null +++ b/set_bootable_flag_action_unittest.cc @@ -0,0 +1,153 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> +#include <string> +#include <vector> +#include <gtest/gtest.h> +#include "update_engine/set_bootable_flag_action.h" +#include "update_engine/test_utils.h" +#include "update_engine/utils.h" + +using std::string; +using std::vector; + +namespace chromeos_update_engine { + +class SetBootableFlagActionProcessorDelegate : public ActionProcessorDelegate { + public: + SetBootableFlagActionProcessorDelegate() + : success_(false), success_set_(false) {} + void ActionCompleted(ActionProcessor* processor, + AbstractAction* action, + bool success) { + if (action->Type() == SetBootableFlagAction::StaticType()) { + success_ = success; + success_set_ = true; + } + } + bool success_; + bool success_set_; +}; + +class SetBootableFlagActionTest : public ::testing::Test { + public: + SetBootableFlagActionTest(); + protected: + enum TestFlags { + EXPECT_SUCCESS = 0x01, + WRITE_FILE = 0x02, + SKIP_INPUT_OBJECT = 0x04 + }; + static const char* kTestDir; + virtual void SetUp() { + EXPECT_EQ(0, mkdir(kTestDir, 0755)); + } + virtual void TearDown() { + EXPECT_EQ(0, System(string("rm -rf ") + kTestDir)); + } + vector<char> DoTest(vector<char> mbr_in, + const string& filename, + uint32 test_flags); + // first partition bootable, no others bootable + const vector<char> sample_mbr_; +}; + +const char* SetBootableFlagActionTest::kTestDir = + "SetBootableFlagActionTestDir"; + +SetBootableFlagActionTest::SetBootableFlagActionTest() + : sample_mbr_(GenerateSampleMbr()) {} + +vector<char> SetBootableFlagActionTest::DoTest(vector<char> mbr_in, + const string& filename, + uint32 flags) { + CHECK(!filename.empty()); + const string root_filename(filename.begin(), --filename.end()); + if (flags & WRITE_FILE) + EXPECT_TRUE(WriteFileVector(root_filename, mbr_in)); + + ActionProcessor processor; + SetBootableFlagActionProcessorDelegate delegate; + processor.set_delegate(&delegate); + + ObjectFeederAction<string> feeder_action; + feeder_action.set_obj(filename); + SetBootableFlagAction set_bootable_action; + if (!(flags & SKIP_INPUT_OBJECT)) + BondActions(&feeder_action, &set_bootable_action); + ObjectCollectorAction<string> collector_action; + BondActions(&set_bootable_action, &collector_action); + processor.EnqueueAction(&feeder_action); + processor.EnqueueAction(&set_bootable_action); + processor.EnqueueAction(&collector_action); + processor.StartProcessing(); + EXPECT_TRUE(!processor.IsRunning()) + << "Update test to handle non-asynch actions"; + + EXPECT_TRUE(delegate.success_set_); + EXPECT_EQ(flags & EXPECT_SUCCESS, delegate.success_); + + vector<char> new_mbr; + if (flags & WRITE_FILE) + utils::ReadFile(root_filename, &new_mbr); + + unlink(root_filename.c_str()); + return new_mbr; +} + +TEST_F(SetBootableFlagActionTest, SimpleTest) { + for (int i = 0; i < 4; i++) { + vector<char> expected_new_mbr = sample_mbr_; + for (int j = 0; j < 4; j++) + expected_new_mbr[446 + 16 * j] = '\0'; // mark non-bootable + expected_new_mbr[446 + 16 * i] = 0x80; // mark bootable + + string filename(string(kTestDir) + "/SetBootableFlagActionTest.devX"); + filename[filename.size() - 1] = '1' + i; + + vector<char> actual_new_mbr = DoTest(expected_new_mbr, filename, + EXPECT_SUCCESS | WRITE_FILE); + + ExpectVectorsEq(expected_new_mbr, actual_new_mbr); + } +} + +TEST_F(SetBootableFlagActionTest, BadDeviceTest) { + vector<char> actual_new_mbr = DoTest(sample_mbr_, + string(kTestDir) + + "SetBootableFlagActionTest.dev5", + WRITE_FILE); + ExpectVectorsEq(sample_mbr_, actual_new_mbr); // no change + + actual_new_mbr = DoTest(sample_mbr_, + string(kTestDir) + "SetBootableFlagActionTest.dev13", + WRITE_FILE); + ExpectVectorsEq(sample_mbr_, actual_new_mbr); // no change + + actual_new_mbr = DoTest(sample_mbr_, + "/some/nonexistent/file3", + 0); + EXPECT_TRUE(actual_new_mbr.empty()); + + vector<char> bad_mbr = sample_mbr_; + bad_mbr[510] = 'x'; // makes signature invalid + + actual_new_mbr = DoTest(bad_mbr, + string(kTestDir) + "SetBootableFlagActionTest.dev2", + WRITE_FILE); + ExpectVectorsEq(bad_mbr, actual_new_mbr); // no change +} + +TEST_F(SetBootableFlagActionTest, NoInputObjectTest) { + vector<char> actual_new_mbr = DoTest(sample_mbr_, + string(kTestDir) + + "SetBootableFlagActionTest.dev5", + WRITE_FILE | SKIP_INPUT_OBJECT); + ExpectVectorsEq(sample_mbr_, actual_new_mbr); // no change +} + +} // namespace chromeos_update_engine diff --git a/subprocess.cc b/subprocess.cc new file mode 100644 index 00000000..299d7588 --- /dev/null +++ b/subprocess.cc @@ -0,0 +1,108 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "update_engine/subprocess.h" +#include <stdlib.h> +#include <string.h> +#include <string> +#include <vector> +#include "chromeos/obsolete_logging.h" +#include "base/scoped_ptr.h" + +using std::string; +using std::vector; + +namespace chromeos_update_engine { + +void Subprocess::GChildExitedCallback(GPid pid, gint status, gpointer data) { + COMPILE_ASSERT(sizeof(guint) == sizeof(uint32), + guint_uint32_size_mismatch); + guint *tag = reinterpret_cast<guint*>(data); + const SubprocessCallbackRecord& record = Get().callback_records_[*tag]; + if (record.callback) + record.callback(status, record.callback_data); + g_spawn_close_pid(pid); + Get().callback_records_.erase(*tag); + delete tag; +} + +namespace { +void FreeArgv(char** argv) { + for (int i = 0; argv[i]; i++) { + free(argv[i]); + argv[i] = NULL; + } +} +} // namespace {} + +uint32 Subprocess::Exec(const std::vector<std::string>& cmd, + ExecCallback callback, + void *p) { + GPid child_pid; + GError *err; + scoped_array<char *> argv(new char*[cmd.size() + 1]); + for (unsigned int i = 0; i < cmd.size(); i++) { + argv[i] = strdup(cmd[i].c_str()); + } + argv[cmd.size()] = NULL; + char *argp[1]; + argp[0] = NULL; + + SubprocessCallbackRecord callback_record; + callback_record.callback = callback; + callback_record.callback_data = p; + + bool success = g_spawn_async(NULL, // working directory + argv.get(), + argp, + G_SPAWN_DO_NOT_REAP_CHILD, // flags + NULL, // child setup function + NULL, // child setup data pointer + &child_pid, + &err); + FreeArgv(argv.get()); + if (!success) { + LOG(ERROR) << "g_spawn_async failed"; + return 0; + } + guint *tag = new guint; + *tag = g_child_watch_add(child_pid, GChildExitedCallback, tag); + callback_records_[*tag] = callback_record; + return *tag; +} + +void Subprocess::CancelExec(uint32 tag) { + if (callback_records_[tag].callback) { + callback_records_[tag].callback = NULL; + } +} + +bool Subprocess::SynchronousExec(const std::vector<std::string>& cmd, + int* return_code) { + GError *err; + scoped_array<char *> argv(new char*[cmd.size() + 1]); + for (unsigned int i = 0; i < cmd.size(); i++) { + argv[i] = strdup(cmd[i].c_str()); + } + argv[cmd.size()] = NULL; + char *argp[1]; + argp[0] = NULL; + + bool success = g_spawn_sync(NULL, // working directory + argv.get(), + argp, + static_cast<GSpawnFlags>(NULL), // flags + NULL, // child setup function + NULL, // data for child setup function + NULL, // return location for stdout + NULL, // return location for stderr + return_code, + &err); + FreeArgv(argv.get()); + return success; +} + +Subprocess* Subprocess::subprocess_singleton_ = NULL; + +} // namespace chromeos_update_engine diff --git a/subprocess.h b/subprocess.h new file mode 100644 index 00000000..92064a65 --- /dev/null +++ b/subprocess.h @@ -0,0 +1,79 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CHROMEOS_PLATFORM_UPDATE_ENGINE_SUBPROCESS_H__ +#define CHROMEOS_PLATFORM_UPDATE_ENGINE_SUBPROCESS_H__ + +#include <map> +#include <string> +#include <vector> +#include <glib.h> +#include "chromeos/obsolete_logging.h" +#include "base/basictypes.h" + +// The Subprocess class is a singleton. It's used to spawn off a subprocess +// and get notified when the subprocess exits. The result of Exec() can +// be saved and used to cancel the callback request. If you know you won't +// call CancelExec(), you may safely lose the return value from Exec(). + +namespace chromeos_update_engine { + +class Subprocess { + public: + static void Init() { + CHECK(!subprocess_singleton_); + subprocess_singleton_ = new Subprocess; + } + + typedef void(*ExecCallback)(int return_code, void *p); + + // Returns a tag > 0 on success. + uint32 Exec(const std::vector<std::string>& cmd, + ExecCallback callback, + void* p); + + // Used to cancel the callback. The process will still run to completion. + void CancelExec(uint32 tag); + + // Executes a command synchronously. Returns true on success. + static bool SynchronousExec(const std::vector<std::string>& cmd, + int* return_code); + + // Gets the one instance + static Subprocess& Get() { + return *subprocess_singleton_; + } + + // Returns true iff there is at least one subprocess we're waiting on. + bool SubprocessInFlight() { + for (std::map<int, SubprocessCallbackRecord>::iterator it = + callback_records_.begin(); + it != callback_records_.end(); ++it) { + if (it->second.callback) + return true; + } + return false; + } + private: + // The global instance + static Subprocess* subprocess_singleton_; + + // Callback for when any subprocess terminates. This calls the user + // requested callback. + static void GChildExitedCallback(GPid pid, gint status, gpointer data); + + struct SubprocessCallbackRecord { + ExecCallback callback; + void* callback_data; + }; + + std::map<int, SubprocessCallbackRecord> callback_records_; + + Subprocess() {} + DISALLOW_COPY_AND_ASSIGN(Subprocess); +}; + +} // namespace chromeos_update_engine + +#endif // CHROMEOS_PLATFORM_UPDATE_ENGINE_SUBPROCESS_H__ diff --git a/subprocess_unittest.cc b/subprocess_unittest.cc new file mode 100644 index 00000000..430f39b6 --- /dev/null +++ b/subprocess_unittest.cc @@ -0,0 +1,115 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> +#include <string> +#include <vector> +#include "base/string_util.h" +#include <glib.h> +#include <gtest/gtest.h> +#include "update_engine/subprocess.h" +#include "update_engine/test_utils.h" +#include "update_engine/utils.h" + +using std::string; +using std::vector; + +namespace chromeos_update_engine { + +class SubprocessTest : public ::testing::Test { + protected: + bool callback_done; +}; + +namespace { +const int kLocalHttpPort = 8080; + +void Callback(int return_code, void *p) { + EXPECT_EQ(256, return_code); + GMainLoop* loop = reinterpret_cast<GMainLoop*>(p); + g_main_loop_quit(loop); +} + +gboolean LaunchFalseInMainLoop(gpointer data) { + vector<string> cmd; + cmd.push_back("/bin/false"); + Subprocess::Get().Exec(cmd, Callback, data); + return FALSE; +} +} // namespace {} + +TEST(SubprocessTest, SimpleTest) { + GMainLoop *loop = g_main_loop_new(g_main_context_default(), FALSE); + g_timeout_add(0, &LaunchFalseInMainLoop, loop); + g_main_loop_run(loop); + g_main_loop_unref(loop); +} + +namespace { +void CallbackBad(int return_code, void *p) { + CHECK(false) << "should never be called."; +} + +struct CancelTestData { + bool spawned; + GMainLoop *loop; +}; + +gboolean StartAndCancelInRunLoop(gpointer data) { + CancelTestData* cancel_test_data = reinterpret_cast<CancelTestData*>(data); + vector<string> cmd; + cmd.push_back("./test_http_server"); + uint32 tag = Subprocess::Get().Exec(cmd, CallbackBad, NULL); + EXPECT_NE(0, tag); + cancel_test_data->spawned = true; + printf("spawned\n"); + // Wait for server to be up and running + for (;;) { + int status = + System(StringPrintf("wget -O /dev/null http://127.0.0.1:%d/foo", + kLocalHttpPort)); + EXPECT_NE(-1, status) << "system() failed"; + EXPECT_TRUE(WIFEXITED(status)) + << "command failed to run or died abnormally"; + if (0 == WEXITSTATUS(status)) + break; + usleep(100 * 1000); // 100 ms + } + Subprocess::Get().CancelExec(tag); + return FALSE; +} +} // namespace {} + +gboolean ExitWhenDone(gpointer data) { + CancelTestData* cancel_test_data = reinterpret_cast<CancelTestData*>(data); + if (cancel_test_data->spawned && !Subprocess::Get().SubprocessInFlight()) { + // tear down the sub process + printf("tear down time\n"); + int status = + System(StringPrintf("wget http://127.0.0.1:%d/quitquitquit", + kLocalHttpPort)); + EXPECT_NE(-1, status) << "system() failed"; + EXPECT_TRUE(WIFEXITED(status)) + << "command failed to run or died abnormally"; + g_main_loop_quit(cancel_test_data->loop); + return FALSE; + } + return TRUE; +} + +TEST(SubprocessTest, CancelTest) { + GMainLoop *loop = g_main_loop_new(g_main_context_default(), FALSE); + CancelTestData cancel_test_data; + cancel_test_data.spawned = false; + cancel_test_data.loop = loop; + g_timeout_add(100, &StartAndCancelInRunLoop, &cancel_test_data); + g_timeout_add(10, &ExitWhenDone, &cancel_test_data); + g_main_loop_run(loop); + g_main_loop_unref(loop); + printf("here\n"); +} + +} // namespace chromeos_update_engine diff --git a/test_http_server.cc b/test_http_server.cc new file mode 100644 index 00000000..94c88fa1 --- /dev/null +++ b/test_http_server.cc @@ -0,0 +1,240 @@ +// Copyright (c) 2009 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// This file implements a simple HTTP server. It can exhibit odd behavior +// that's useful for testing. For example, it's useful to test that +// the updater can continue a connection if it's dropped, or that it +// handles very slow data transfers. + +// To use this, simply make an HTTP connection to localhost:port and +// GET a url. + +#include <netinet/in.h> +#include <sys/socket.h> +#include <sys/types.h> +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <algorithm> +#include <string> +#include <vector> +#include "chromeos/obsolete_logging.h" + +using std::min; +using std::string; +using std::vector; + +namespace chromeos_update_engine { + +struct HttpRequest { + string url; + off_t offset; +}; + +namespace { +const int kPort = 8080; // hardcoded to 8080 for now +const int kBigLength = 100000; +} + +bool ParseRequest(int fd, HttpRequest* request) { + string headers; + while(headers.find("\r\n\r\n") == string::npos) { + vector<char> buf(1024); + memset(&buf[0], 0, buf.size()); + ssize_t r = read(fd, &buf[0], buf.size() - 1); + if (r < 0) { + perror("read"); + exit(1); + } + buf.resize(r); + + headers.insert(headers.end(), buf.begin(), buf.end()); + } + LOG(INFO) << "got headers: " << headers; + + string::size_type url_start, url_end; + CHECK_NE(headers.find("GET "), string::npos); + url_start = headers.find("GET ") + strlen("GET "); + url_end = headers.find(' ', url_start); + CHECK_NE(string::npos, url_end); + string url = headers.substr(url_start, url_end - url_start); + LOG(INFO) << "URL: " << url; + + string::size_type range_start, range_end; + if (headers.find("\r\nRange: ") == string::npos) { + request->offset = 0; + } else { + range_start = headers.find("\r\nRange: ") + strlen("\r\nRange: "); + range_end = headers.find('\r', range_start); + CHECK_NE(string::npos, range_end); + string range_header = headers.substr(range_start, range_end - range_start); + + LOG(INFO) << "Range: " << range_header; + CHECK(*range_header.rbegin() == '-'); + request->offset = atoll(range_header.c_str() + strlen("bytes=")); + LOG(INFO) << "Offset: " << request->offset; + } + request->url = url; + return true; +} + +void WriteString(int fd, const string& str) { + unsigned int bytes_written = 0; + while (bytes_written < str.size()) { + ssize_t r = write(fd, str.c_str() + bytes_written, + str.size() - bytes_written); + LOG(INFO) << "write() wrote " << r << " bytes"; + if (r < 0) { + perror("write"); + return; + } + bytes_written += r; + } + LOG(INFO) << "WriteString wrote " << bytes_written << " bytes"; +} + +string Itoa(off_t num) { + char buf[100] = {0}; + snprintf(buf, sizeof(buf), "%lld", num); + return buf; +} + +void WriteHeaders(int fd, bool support_range, off_t full_size, + off_t start_offset) { + LOG(INFO) << "writing headers"; + WriteString(fd, "HTTP/1.1 200 OK\r\n"); + WriteString(fd, "Content-Type: application/octet-stream\r\n"); + if (support_range) { + WriteString(fd, "Accept-Ranges: bytes\r\n"); + WriteString(fd, string("Content-Range: bytes ") + Itoa(start_offset) + + "-" + Itoa(full_size - 1) + "/" + Itoa(full_size) + "\r\n"); + } + off_t content_length = full_size; + if (support_range) + content_length -= start_offset; + WriteString(fd, string("Content-Length: ") + Itoa(content_length) + "\r\n"); + WriteString(fd, "\r\n"); +} + +void HandleQuitQuitQuit(int fd) { + exit(0); +} + +void HandleBig(int fd, const HttpRequest& request) { + const off_t full_length = kBigLength; + WriteHeaders(fd, true, full_length, request.offset); + const off_t content_length = full_length - request.offset; + int i = request.offset; + for (; i % 10; i++) + WriteString(fd, string(1, 'a' + (i % 10))); + CHECK_EQ(i % 10, 0); + for (; i < content_length; i += 10) + WriteString(fd, "abcdefghij"); + CHECK_EQ(i, full_length); +} + +// This is like /big, but it writes at most 9000 bytes. Also, +// half way through it sleeps for 70 seconds +// (technically, when (offset % (9000 * 7)) == 0). +void HandleFlaky(int fd, const HttpRequest& request) { + const off_t full_length = kBigLength; + WriteHeaders(fd, true, full_length, request.offset); + const off_t content_length = min(9000LL, full_length - request.offset); + const bool should_sleep = (request.offset % (9000 * 7)) == 0; + + string buf; + + for (int i = request.offset; i % 10; i++) + buf.append(1, 'a' + (i % 10)); + while (buf.size() < content_length) + buf.append("abcdefghij"); + buf.resize(content_length); + + if (!should_sleep) { + LOG(INFO) << "writing data blob of size " << buf.size(); + WriteString(fd, buf); + } else { + string::size_type half_way_point = buf.size() / 2; + LOG(INFO) << "writing small data blob of size " << half_way_point; + WriteString(fd, buf.substr(0, half_way_point)); + sleep(10); + LOG(INFO) << "writing small data blob of size " + << (buf.size() - half_way_point); + WriteString(fd, buf.substr(half_way_point, buf.size() - half_way_point)); + } +} + +void HandleDefault(int fd, const HttpRequest& request) { + const string data("unhandled path"); + WriteHeaders(fd, true, data.size(), request.offset); + const string data_to_write(data.substr(request.offset, + data.size() - request.offset)); + WriteString(fd, data_to_write); +} + +void HandleConnection(int fd) { + HttpRequest request; + ParseRequest(fd, &request); + + if (request.url == "/quitquitquit") + HandleQuitQuitQuit(fd); + else if (request.url == "/big") + HandleBig(fd, request); + else if (request.url == "/flaky") + HandleFlaky(fd, request); + else + HandleDefault(fd, request); + + close(fd); +} + +} // namespace chromeos_update_engine + +using namespace chromeos_update_engine; + +int main(int argc, char** argv) { + socklen_t clilen; + struct sockaddr_in server_addr; + struct sockaddr_in client_addr; + memset(&server_addr, 0, sizeof(server_addr)); + memset(&client_addr, 0, sizeof(client_addr)); + + int listen_fd = socket(AF_INET, SOCK_STREAM, 0); + if (listen_fd < 0) + LOG(FATAL) << "socket() failed"; + + server_addr.sin_family = AF_INET; + server_addr.sin_addr.s_addr = INADDR_ANY; + server_addr.sin_port = htons(kPort); + + { + // Get rid of "Address in use" error + int tr = 1; + if (setsockopt(listen_fd, SOL_SOCKET, SO_REUSEADDR, &tr, + sizeof(int)) == -1) { + perror("setsockopt"); + exit(1); + } + } + + if (bind(listen_fd, reinterpret_cast<struct sockaddr *>(&server_addr), + sizeof(server_addr)) < 0) { + perror("bind"); + exit(1); + } + CHECK_EQ(listen(listen_fd,5), 0); + while (1) { + clilen = sizeof(client_addr); + int client_fd = accept(listen_fd, + (struct sockaddr *) &client_addr, + &clilen); + LOG(INFO) << "got past accept"; + if (client_fd < 0) + LOG(FATAL) << "ERROR on accept"; + HandleConnection(client_fd); + } + return 0; +} diff --git a/update_metadata.proto b/update_metadata.proto new file mode 100644 index 00000000..48935d8a --- /dev/null +++ b/update_metadata.proto @@ -0,0 +1,127 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Update file format: A delta update file contains all the deltas needed +// to update a system from one specific version to another specific +// version. The update format is represented by this struct pseudocode: +// struct delta_update_file { +// char magic[4] = "CrAU"; +// uint64 bom_offset; // Offset of protobuf DeltaArchiveManifest +// uint64 bom_size; // Sise of protobuf DeltaArchiveManifest +// +// // Data blobs for files, no specific format. The specific offset +// // and length of each data blob is recorded in the DeltaArchiveManifest. +// struct { +// char data[]; +// } blobs[]; +// +// // The Gzip compressed DeltaArchiveManifest +// char bom[]; +// }; + +// The DeltaArchiveManifest protobuf is an ordered list of File objects. +// These File objects are stored in a linear array in the +// DeltaArchiveManifest, each with a specific index. Each File object +// can contain children in its children list. Each child in the list +// has a name and an index. The index refers to the index within +// DeltaArchiveManifest.files. Thus, the DeltaArchiveManifest.files +// can be seen as a tree structure that mimicks the filesystem. +// The root object (the object an index 0) has no name, since names +// for children are stored in the parent. + +// The DeltaArchiveManifest will contain one File entry for each +// file that will be on the resultant filesystem. Because we have +// a tree structure, and children are ordered alphabetically within +// a parent, we can do log-time˜path lookup on a DeltaArchiveManifest +// object. We can also iterate through a DeltaArchiveManifest object +// using a preorder tree traversal to see each file in the +// DeltaArchiveManifest, seeing each directory before any of its children; +// this takes linear time. + +// Here's an example from Dan Erat showing DeltaArchiveManifest +// for a filesystem with files /bin/cat and /bin/ls.: + +// files[0] { // "/" directory +// children[0] { +// name "bin" +// index 1 +// } +// } +// files[1] { // "/bin" directory +// children[0] { +// name "cat" +// index 2 +// } +// children[1] { +// name "ls" +// index 3 +// } +// } +// files[2] { // "/bin/cat" +// } +// files[3] { // "/bin/ls" +// } + +// If a file has a data_format set, it should also have data_offset and +// data_length set. data_offset and data_length refer to a range of bytes +// in the delta update file itself which have the format specified by +// data_format. FULL and FULL_GZ mean the entire file is present (FULL_GZ, +// gzip compressed). BSDIFF means the old file with the same path should be +// patched with 'bspatch' to produce the desired output file. COURGETTE +// is not yet used, but it will be another binary diff format. + +// Directories should not have any data. + +// There are other types of files, too: symlinks, block and character devices, +// fifos, and sockets. Fifos and sockets contain no data. Block and +// character devices have data. It must be the format FULL or FULL_GZ, and +// the contents are a serialized LinuxDevice protobuf. Symlinks must either +// be FULL, FULL_GZ, or have no data. A symlink with no data is unchanged, +// and with data it's set to that data. + +// TODO(adlr): Add support for hard links; CL is prepared already. +// Extended attributes are unsupported at this time. + +package chromeos_update_engine; + +message DeltaArchiveManifest { + message File { + // This is st_mode from struct stat. It includes file type and permission + // bits. + optional uint32 mode = 1; + optional uint32 uid = 2; + optional uint32 gid = 3; + + // File Data, not for directories + enum DataFormat { + FULL = 0; // The data is the complete file + FULL_GZ = 1; // The data is the complete file gzipped + BSDIFF = 2; // The data is a bsdiff binary diff + COURGETTE = 3; // The data is a courgette binary diff + } + // If present, there is data associated with this File object and + // data_offset and data_size must be set. + optional DataFormat data_format = 4; + // The offset into the delta file where the data (if any) is stored + optional uint32 data_offset = 5; + // The length of the data in the delta file + optional uint32 data_length = 6; + + message Child { + // A File that's a directory (and only those types of File objects) + // will have exactly one Child submessage per child. + required string name = 1; // File name of child + + // Index into DeltaArchiveManifest.files for the File object of the child. + required uint32 index = 2; + } + repeated Child children = 7; + } + repeated File files = 1; +} + +message LinuxDevice { + required int32 major = 1; + required int32 minor = 2; +}
\ No newline at end of file diff --git a/utils.cc b/utils.cc new file mode 100644 index 00000000..693062df --- /dev/null +++ b/utils.cc @@ -0,0 +1,253 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "update_engine/utils.h" +#include <sys/mount.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <dirent.h> +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <algorithm> +#include "chromeos/obsolete_logging.h" + +using std::min; +using std::string; +using std::vector; + +namespace chromeos_update_engine { + +namespace utils { + +bool ReadFile(const std::string& path, std::vector<char>* out) { + CHECK(out); + FILE* fp = fopen(path.c_str(), "r"); + if (!fp) + return false; + const size_t kChunkSize = 1024; + size_t read_size; + do { + char buf[kChunkSize]; + read_size = fread(buf, 1, kChunkSize, fp); + if (read_size == 0) + break; + out->insert(out->end(), buf, buf + read_size); + } while (read_size == kChunkSize); + bool success = !ferror(fp); + TEST_AND_RETURN_FALSE_ERRNO(fclose(fp) == 0); + return success; +} + +bool ReadFileToString(const std::string& path, std::string* out) { + vector<char> data; + bool success = ReadFile(path, &data); + if (!success) { + return false; + } + (*out) = string(&data[0], data.size()); + return true; +} + +void HexDumpArray(const unsigned char* const arr, const size_t length) { + const unsigned char* const char_arr = + reinterpret_cast<const unsigned char* const>(arr); + LOG(INFO) << "Logging array of length: " << length; + const unsigned int bytes_per_line = 16; + for (size_t i = 0; i < length; i += bytes_per_line) { + const unsigned int bytes_remaining = length - i; + const unsigned int bytes_per_this_line = min(bytes_per_line, + bytes_remaining); + char header[100]; + int r = snprintf(header, sizeof(header), "0x%08x : ", i); + TEST_AND_RETURN(r == 13); + string line = header; + for (unsigned int j = 0; j < bytes_per_this_line; j++) { + char buf[20]; + unsigned char c = char_arr[i + j]; + r = snprintf(buf, sizeof(buf), "%02x ", static_cast<unsigned int>(c)); + TEST_AND_RETURN(r == 3); + line += buf; + } + LOG(INFO) << line; + } +} + +namespace { +class ScopedDirCloser { + public: + explicit ScopedDirCloser(DIR** dir) : dir_(dir) {} + ~ScopedDirCloser() { + if (dir_ && *dir_) { + int r = closedir(*dir_); + TEST_AND_RETURN_ERRNO(r == 0); + *dir_ = NULL; + dir_ = NULL; + } + } + private: + DIR** dir_; +}; +} // namespace {} + +bool RecursiveUnlinkDir(const std::string& path) { + struct stat stbuf; + int r = lstat(path.c_str(), &stbuf); + TEST_AND_RETURN_FALSE_ERRNO((r == 0) || (errno == ENOENT)); + if ((r < 0) && (errno == ENOENT)) + // path request is missing. that's fine. + return true; + if (!S_ISDIR(stbuf.st_mode)) { + TEST_AND_RETURN_FALSE_ERRNO((unlink(path.c_str()) == 0) || + (errno == ENOENT)); + // success or path disappeared before we could unlink. + return true; + } + { + // We have a dir, unlink all children, then delete dir + DIR *dir = opendir(path.c_str()); + TEST_AND_RETURN_FALSE_ERRNO(dir); + ScopedDirCloser dir_closer(&dir); + struct dirent dir_entry; + struct dirent *dir_entry_p; + int err = 0; + while ((err = readdir_r(dir, &dir_entry, &dir_entry_p)) == 0) { + if (dir_entry_p == NULL) { + // end of stream reached + break; + } + // Skip . and .. + if (!strcmp(dir_entry_p->d_name, ".") || + !strcmp(dir_entry_p->d_name, "..")) + continue; + TEST_AND_RETURN_FALSE(RecursiveUnlinkDir(path + "/" + + dir_entry_p->d_name)); + } + TEST_AND_RETURN_FALSE(err == 0); + } + // unlink dir + TEST_AND_RETURN_FALSE_ERRNO((rmdir(path.c_str()) == 0) || (errno == ENOENT)); + return true; +} + +std::string ErrnoNumberAsString(int err) { + char buf[100]; + buf[0] = '\0'; + return strerror_r(err, buf, sizeof(buf)); +} + +std::string NormalizePath(const std::string& path, bool strip_trailing_slash) { + string ret; + bool last_insert_was_slash = false; + for (string::const_iterator it = path.begin(); it != path.end(); ++it) { + if (*it == '/') { + if (last_insert_was_slash) + continue; + last_insert_was_slash = true; + } else { + last_insert_was_slash = false; + } + ret.push_back(*it); + } + if (strip_trailing_slash && last_insert_was_slash) { + string::size_type last_non_slash = ret.find_last_not_of('/'); + if (last_non_slash != string::npos) { + ret.resize(last_non_slash + 1); + } else { + ret = ""; + } + } + return ret; +} + +bool FileExists(const char* path) { + struct stat stbuf; + return 0 == lstat(path, &stbuf); +} + +std::string TempFilename(string path) { + static const string suffix("XXXXXX"); + CHECK(StringHasSuffix(path, suffix)); + do { + string new_suffix; + for (unsigned int i = 0; i < suffix.size(); i++) { + int r = rand() % (26 * 2 + 10); // [a-zA-Z0-9] + if (r < 26) + new_suffix.append(1, 'a' + r); + else if (r < (26 * 2)) + new_suffix.append(1, 'A' + r - 26); + else + new_suffix.append(1, '0' + r - (26 * 2)); + } + CHECK_EQ(new_suffix.size(), suffix.size()); + path.resize(path.size() - new_suffix.size()); + path.append(new_suffix); + } while (FileExists(path.c_str())); + return path; +} + +bool StringHasSuffix(const std::string& str, const std::string& suffix) { + if (suffix.size() > str.size()) + return false; + return 0 == str.compare(str.size() - suffix.size(), suffix.size(), suffix); +} + +bool StringHasPrefix(const std::string& str, const std::string& prefix) { + if (prefix.size() > str.size()) + return false; + return 0 == str.compare(0, prefix.size(), prefix); +} + +const std::string BootDevice() { + string proc_cmdline; + if (!ReadFileToString("/proc/cmdline", &proc_cmdline)) + return ""; + // look for "root=" in the command line + string::size_type pos = 0; + if (!StringHasPrefix(proc_cmdline, "root=")) { + pos = proc_cmdline.find(" root=") + 1; + } + if (pos == string::npos) { + // can't find root= + return ""; + } + // at this point, pos is the point in the string where "root=" starts + string ret; + pos += strlen("root="); // advance to the device name itself + while (pos < proc_cmdline.size()) { + char c = proc_cmdline[pos]; + if (c == ' ') + break; + ret += c; + pos++; + } + return ret; + // TODO(adlr): use findfs to figure out UUID= or LABEL= filesystems +} + +bool MountFilesystem(const string& device, + const string& mountpoint) { + int rc = mount(device.c_str(), mountpoint.c_str(), "ext3", 0, NULL); + if (rc < 0) { + string msg = ErrnoNumberAsString(errno); + LOG(ERROR) << "Unable to mount destination device: " << msg << ". " + << device << " on " << mountpoint; + return false; + } + return true; +} + +bool UnmountFilesystem(const string& mountpoint) { + TEST_AND_RETURN_FALSE_ERRNO(umount(mountpoint.c_str()) == 0); + return true; +} + +const string kStatefulPartition = "/mnt/stateful_partition"; + +} // namespace utils + +} // namespace chromeos_update_engine + diff --git a/utils.h b/utils.h new file mode 100644 index 00000000..e4757832 --- /dev/null +++ b/utils.h @@ -0,0 +1,184 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CHROMEOS_PLATFORM_UPDATE_ENGINE_UTILS_H__ +#define CHROMEOS_PLATFORM_UPDATE_ENGINE_UTILS_H__ + +#include <set> +#include <string> +#include <vector> +#include "update_engine/action.h" +#include "update_engine/action_processor.h" + +namespace chromeos_update_engine { + +namespace utils { + +// Returns the entire contents of the file at path. Returns true on success. +bool ReadFile(const std::string& path, std::vector<char>* out); +bool ReadFileToString(const std::string& path, std::string* out); + +std::string ErrnoNumberAsString(int err); + +// Strips duplicate slashes, and optionally removes all trailing slashes. +// Does not compact /./ or /../. +std::string NormalizePath(const std::string& path, bool strip_trailing_slash); + +// Returns true if the file exists for sure. Returns false if it doesn't exist, +// or an error occurs. +bool FileExists(const char* path); + +// The last 6 chars of path must be XXXXXX. They will be randomly changed +// and a non-existent path will be returned. Intentionally makes a copy +// of the string passed in. +// NEVER CALL THIS FUNCTION UNLESS YOU ARE SURE +// THAT YOUR PROCESS WILL BE THE ONLY THING WRITING FILES IN THIS DIRECTORY. +std::string TempFilename(string path); + +// Deletes a directory and all its contents synchronously. Returns true +// on success. This may be called with a regular file--it will just unlink it. +// This WILL cross filesystem boundaries. +bool RecursiveUnlinkDir(const std::string& path); + +// Synchronously mount or unmount a filesystem. Return true on success. +// Mounts as ext3 with default options. +bool MountFilesystem(const string& device, const string& mountpoint); +bool UnmountFilesystem(const string& mountpoint); + +// Log a string in hex to LOG(INFO). Useful for debugging. +void HexDumpArray(const unsigned char* const arr, const size_t length); +inline void HexDumpString(const std::string& str) { + HexDumpArray(reinterpret_cast<const unsigned char*>(str.data()), str.size()); +} +inline void HexDumpVector(const std::vector<char>& vect) { + HexDumpArray(reinterpret_cast<const unsigned char*>(&vect[0]), vect.size()); +} + +extern const string kStatefulPartition; + +bool StringHasSuffix(const std::string& str, const std::string& suffix); +bool StringHasPrefix(const std::string& str, const std::string& prefix); + +template<typename KeyType, typename ValueType> +bool MapContainsKey(const std::map<KeyType, ValueType>& m, const KeyType& k) { + return m.find(k) != m.end(); +} + +template<typename ValueType> +std::set<ValueType> SetWithValue(const ValueType& value) { + std::set<ValueType> ret; + ret.insert(value); + return ret; +} + +// Returns the currently booted device. "/dev/sda1", for example. +// This will not interpret LABEL= or UUID=. You'll need to use findfs +// or something with equivalent funcionality to interpret those. +const std::string BootDevice(); + +} // namespace utils + +// Class to unmount FS when object goes out of scope +class ScopedFilesystemUnmounter { + public: + explicit ScopedFilesystemUnmounter(const std::string& mountpoint) + : mountpoint_(mountpoint) {} + ~ScopedFilesystemUnmounter() { + utils::UnmountFilesystem(mountpoint_); + } + private: + const std::string mountpoint_; +}; + +// Utility class to close a file descriptor +class ScopedFdCloser { + public: + explicit ScopedFdCloser(int* fd) : fd_(fd), should_close_(true) {} + void set_should_close(bool should_close) { should_close_ = should_close; } + ~ScopedFdCloser() { + if (!should_close_) + return; + if (fd_ && (*fd_ >= 0)) { + close(*fd_); + *fd_ = -1; + } + } + private: + int* fd_; + bool should_close_; +}; + +// A little object to call ActionComplete on the ActionProcessor when +// it's destructed. +class ScopedActionCompleter { + public: + explicit ScopedActionCompleter(ActionProcessor* processor, + AbstractAction* action) + : processor_(processor), + action_(action), + success_(false), + should_complete_(true) {} + ~ScopedActionCompleter() { + if (should_complete_) + processor_->ActionComplete(action_, success_); + } + void set_success(bool success) { + success_ = success; + } + void set_should_complete(bool should_complete) { + should_complete_ = should_complete; + } + private: + ActionProcessor* processor_; + AbstractAction* action_; + bool success_; + bool should_complete_; + DISALLOW_COPY_AND_ASSIGN(ScopedActionCompleter); +}; + +} // namespace chromeos_update_engine + +#define TEST_AND_RETURN_FALSE_ERRNO(_x) \ + do { \ + bool _success = (_x); \ + if (!_success) { \ + std::string _msg = \ + chromeos_update_engine::utils::ErrnoNumberAsString(errno); \ + LOG(ERROR) << #_x " failed: " << _msg; \ + return false; \ + } \ + } while (0) + +#define TEST_AND_RETURN_FALSE(_x) \ + do { \ + bool _success = (_x); \ + if (!_success) { \ + LOG(ERROR) << #_x " failed."; \ + return false; \ + } \ + } while (0) + +#define TEST_AND_RETURN_ERRNO(_x) \ + do { \ + bool _success = (_x); \ + if (!_success) { \ + std::string _msg = \ + chromeos_update_engine::utils::ErrnoNumberAsString(errno); \ + LOG(ERROR) << #_x " failed: " << _msg; \ + return; \ + } \ + } while (0) + +#define TEST_AND_RETURN(_x) \ + do { \ + bool _success = (_x); \ + if (!_success) { \ + LOG(ERROR) << #_x " failed."; \ + return; \ + } \ + } while (0) + + + +#endif // CHROMEOS_PLATFORM_UPDATE_ENGINE_UTILS_H__ diff --git a/utils_unittest.cc b/utils_unittest.cc new file mode 100644 index 00000000..4fee5448 --- /dev/null +++ b/utils_unittest.cc @@ -0,0 +1,112 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include <sys/stat.h> +#include <sys/types.h> +#include <errno.h> +#include <vector> +#include <gtest/gtest.h> +#include "update_engine/utils.h" + +using std::vector; + +namespace chromeos_update_engine { + +class UtilsTest : public ::testing::Test { }; + +TEST(UtilsTest, NormalizePathTest) { + EXPECT_EQ("", utils::NormalizePath("", false)); + EXPECT_EQ("", utils::NormalizePath("", true)); + EXPECT_EQ("/", utils::NormalizePath("/", false)); + EXPECT_EQ("", utils::NormalizePath("/", true)); + EXPECT_EQ("/", utils::NormalizePath("//", false)); + EXPECT_EQ("", utils::NormalizePath("//", true)); + EXPECT_EQ("foo", utils::NormalizePath("foo", false)); + EXPECT_EQ("foo", utils::NormalizePath("foo", true)); + EXPECT_EQ("/foo/", utils::NormalizePath("/foo//", false)); + EXPECT_EQ("/foo", utils::NormalizePath("/foo//", true)); + EXPECT_EQ("bar/baz/foo/adlr", utils::NormalizePath("bar/baz//foo/adlr", + false)); + EXPECT_EQ("bar/baz/foo/adlr", utils::NormalizePath("bar/baz//foo/adlr", + true)); + EXPECT_EQ("/bar/baz/foo/adlr/", utils::NormalizePath("/bar/baz//foo/adlr/", + false)); + EXPECT_EQ("/bar/baz/foo/adlr", utils::NormalizePath("/bar/baz//foo/adlr/", + true)); + EXPECT_EQ("\\\\", utils::NormalizePath("\\\\", false)); + EXPECT_EQ("\\\\", utils::NormalizePath("\\\\", true)); + EXPECT_EQ("\\:/;$PATH\n\\", utils::NormalizePath("\\://;$PATH\n\\", false)); + EXPECT_EQ("\\:/;$PATH\n\\", utils::NormalizePath("\\://;$PATH\n\\", true)); + EXPECT_EQ("/spaces s/ ok/s / / /", + utils::NormalizePath("/spaces s/ ok/s / / /", false)); + EXPECT_EQ("/spaces s/ ok/s / / ", + utils::NormalizePath("/spaces s/ ok/s / / /", true)); +} + +TEST(UtilsTest, ReadFileFailure) { + vector<char> empty; + EXPECT_FALSE(utils::ReadFile("/this/doesn't/exist", &empty)); +} + +TEST(UtilsTest, ErrnoNumberAsStringTest) { + EXPECT_EQ("No such file or directory", utils::ErrnoNumberAsString(ENOENT)); +} + +TEST(UtilsTest, StringHasSuffixTest) { + EXPECT_TRUE(utils::StringHasSuffix("foo", "foo")); + EXPECT_TRUE(utils::StringHasSuffix("foo", "o")); + EXPECT_TRUE(utils::StringHasSuffix("", "")); + EXPECT_TRUE(utils::StringHasSuffix("abcabc", "abc")); + EXPECT_TRUE(utils::StringHasSuffix("adlrwashere", "ere")); + EXPECT_TRUE(utils::StringHasSuffix("abcdefgh", "gh")); + EXPECT_TRUE(utils::StringHasSuffix("abcdefgh", "")); + EXPECT_FALSE(utils::StringHasSuffix("foo", "afoo")); + EXPECT_FALSE(utils::StringHasSuffix("", "x")); + EXPECT_FALSE(utils::StringHasSuffix("abcdefgh", "fg")); + EXPECT_FALSE(utils::StringHasSuffix("abcdefgh", "ab")); +} + +TEST(UtilsTest, StringHasPrefixTest) { + EXPECT_TRUE(utils::StringHasPrefix("foo", "foo")); + EXPECT_TRUE(utils::StringHasPrefix("foo", "f")); + EXPECT_TRUE(utils::StringHasPrefix("", "")); + EXPECT_TRUE(utils::StringHasPrefix("abcabc", "abc")); + EXPECT_TRUE(utils::StringHasPrefix("adlrwashere", "adl")); + EXPECT_TRUE(utils::StringHasPrefix("abcdefgh", "ab")); + EXPECT_TRUE(utils::StringHasPrefix("abcdefgh", "")); + EXPECT_FALSE(utils::StringHasPrefix("foo", "fooa")); + EXPECT_FALSE(utils::StringHasPrefix("", "x")); + EXPECT_FALSE(utils::StringHasPrefix("abcdefgh", "bc")); + EXPECT_FALSE(utils::StringHasPrefix("abcdefgh", "gh")); +} + +TEST(UtilsTest, BootDeviceTest) { + // Pretty lame test... + EXPECT_FALSE(utils::BootDevice().empty()); +} + +TEST(UtilsTest, RecursiveUnlinkDirTest) { + EXPECT_EQ(0, mkdir("RecursiveUnlinkDirTest-a", 0755)); + EXPECT_EQ(0, mkdir("RecursiveUnlinkDirTest-b", 0755)); + EXPECT_EQ(0, symlink("../RecursiveUnlinkDirTest-a", + "RecursiveUnlinkDirTest-b/link")); + EXPECT_EQ(0, system("echo hi > RecursiveUnlinkDirTest-b/file")); + EXPECT_EQ(0, mkdir("RecursiveUnlinkDirTest-b/dir", 0755)); + EXPECT_EQ(0, system("echo ok > RecursiveUnlinkDirTest-b/dir/subfile")); + EXPECT_TRUE(utils::RecursiveUnlinkDir("RecursiveUnlinkDirTest-b")); + EXPECT_TRUE(utils::FileExists("RecursiveUnlinkDirTest-a")); + EXPECT_EQ(0, system("rm -rf RecursiveUnlinkDirTest-a")); + EXPECT_FALSE(utils::FileExists("RecursiveUnlinkDirTest-b")); + EXPECT_TRUE(utils::RecursiveUnlinkDir("/something/that/doesnt/exist")); +} + +TEST(UtilsTest, TempFilenameTest) { + const string original = "/foo.XXXXXX"; + const string result = utils::TempFilename(original); + EXPECT_EQ(original.size(), result.size()); + EXPECT_TRUE(utils::StringHasPrefix(result, "/foo.")); + EXPECT_FALSE(utils::StringHasSuffix(result, "XXXXXX")); +} + +} // namespace chromeos_update_engine |