diff options
71 files changed, 2467 insertions, 1488 deletions
diff --git a/adb/Android.bp b/adb/Android.bp index 3dc70b5208..2f9c8fcfbb 100644 --- a/adb/Android.bp +++ b/adb/Android.bp @@ -27,7 +27,6 @@ cc_defaults { "-DADB_HOST=1", // overridden by adbd_defaults "-DALLOW_ADBD_ROOT=0", // overridden by adbd_defaults "-DANDROID_BASE_UNIQUE_FD_DISABLE_IMPLICIT_CONVERSION=1", - "-DENABLE_FASTDEPLOY=1", // enable fast deploy ], cpp_std: "experimental", @@ -691,6 +690,7 @@ cc_library_host_static { name: "libfastdeploy_host", defaults: ["adb_defaults"], srcs: [ + "fastdeploy/deploypatchgenerator/apk_archive.cpp", "fastdeploy/deploypatchgenerator/deploy_patch_generator.cpp", "fastdeploy/deploypatchgenerator/patch_utils.cpp", "fastdeploy/proto/ApkEntry.proto", @@ -727,6 +727,7 @@ cc_test_host { name: "fastdeploy_test", defaults: ["adb_defaults"], srcs: [ + "fastdeploy/deploypatchgenerator/apk_archive_test.cpp", "fastdeploy/deploypatchgenerator/deploy_patch_generator_test.cpp", "fastdeploy/deploypatchgenerator/patch_utils_test.cpp", ], @@ -754,6 +755,9 @@ cc_test_host { }, }, data: [ + "fastdeploy/testdata/rotating_cube-metadata-release.data", "fastdeploy/testdata/rotating_cube-release.apk", + "fastdeploy/testdata/sample.apk", + "fastdeploy/testdata/sample.cd", ], } diff --git a/adb/client/adb_install.cpp b/adb/client/adb_install.cpp index 042a2d6c4b..73dcde1fe3 100644 --- a/adb/client/adb_install.cpp +++ b/adb/client/adb_install.cpp @@ -37,9 +37,7 @@ #include "commandline.h" #include "fastdeploy.h" -#if defined(ENABLE_FASTDEPLOY) static constexpr int kFastDeployMinApi = 24; -#endif namespace { @@ -146,15 +144,7 @@ static void read_status_line(int fd, char* buf, size_t count) { *buf = '\0'; } -#if defined(ENABLE_FASTDEPLOY) -static int delete_device_patch_file(const char* apkPath) { - std::string patchDevicePath = get_patch_path(apkPath); - return delete_device_file(patchDevicePath); -} -#endif - -static int install_app_streamed(int argc, const char** argv, bool use_fastdeploy, - bool use_localagent) { +static int install_app_streamed(int argc, const char** argv, bool use_fastdeploy) { printf("Performing Streamed Install\n"); // The last argument must be the APK file @@ -176,31 +166,15 @@ static int install_app_streamed(int argc, const char** argv, bool use_fastdeploy error_exit("--fastdeploy doesn't support .apex files"); } - if (use_fastdeploy == true) { -#if defined(ENABLE_FASTDEPLOY) - TemporaryFile metadataTmpFile; - std::string patchTmpFilePath; - { - TemporaryFile patchTmpFile; - patchTmpFile.DoNotRemove(); - patchTmpFilePath = patchTmpFile.path; + if (use_fastdeploy) { + auto metadata = extract_metadata(file); + if (metadata.has_value()) { + // pass all but 1st (command) and last (apk path) parameters through to pm for + // session creation + std::vector<const char*> pm_args{argv + 1, argv + argc - 1}; + auto patchFd = install_patch(pm_args.size(), pm_args.data()); + return stream_patch(file, std::move(metadata.value()), std::move(patchFd)); } - - FILE* metadataFile = fopen(metadataTmpFile.path, "wb"); - extract_metadata(file, metadataFile); - fclose(metadataFile); - - create_patch(file, metadataTmpFile.path, patchTmpFilePath.c_str()); - // pass all but 1st (command) and last (apk path) parameters through to pm for - // session creation - std::vector<const char*> pm_args{argv + 1, argv + argc - 1}; - install_patch(file, patchTmpFilePath.c_str(), pm_args.size(), pm_args.data()); - adb_unlink(patchTmpFilePath.c_str()); - delete_device_patch_file(file); - return 0; -#else - error_exit("fastdeploy is disabled"); -#endif } struct stat sb; @@ -265,8 +239,7 @@ static int install_app_streamed(int argc, const char** argv, bool use_fastdeploy return 1; } -static int install_app_legacy(int argc, const char** argv, bool use_fastdeploy, - bool use_localagent) { +static int install_app_legacy(int argc, const char** argv, bool use_fastdeploy) { printf("Performing Push Install\n"); // Find last APK argument. @@ -287,35 +260,26 @@ static int install_app_legacy(int argc, const char** argv, bool use_fastdeploy, int result = -1; std::vector<const char*> apk_file = {argv[last_apk]}; std::string apk_dest = "/data/local/tmp/" + android::base::Basename(argv[last_apk]); + argv[last_apk] = apk_dest.c_str(); /* destination name, not source location */ - if (use_fastdeploy == true) { -#if defined(ENABLE_FASTDEPLOY) - TemporaryFile metadataTmpFile; - TemporaryFile patchTmpFile; + if (use_fastdeploy) { + auto metadata = extract_metadata(apk_file[0]); + if (metadata.has_value()) { + auto patchFd = apply_patch_on_device(apk_dest.c_str()); + int status = stream_patch(apk_file[0], std::move(metadata.value()), std::move(patchFd)); - FILE* metadataFile = fopen(metadataTmpFile.path, "wb"); - extract_metadata(apk_file[0], metadataFile); - fclose(metadataFile); + result = pm_command(argc, argv); + delete_device_file(apk_dest); - create_patch(apk_file[0], metadataTmpFile.path, patchTmpFile.path); - apply_patch_on_device(apk_file[0], patchTmpFile.path, apk_dest.c_str()); -#else - error_exit("fastdeploy is disabled"); -#endif - } else { - if (!do_sync_push(apk_file, apk_dest.c_str(), false)) goto cleanup_apk; + return status; + } } - argv[last_apk] = apk_dest.c_str(); /* destination name, not source location */ - result = pm_command(argc, argv); - -cleanup_apk: - if (use_fastdeploy == true) { -#if defined(ENABLE_FASTDEPLOY) - delete_device_patch_file(apk_file[0]); -#endif + if (do_sync_push(apk_file, apk_dest.c_str(), false)) { + result = pm_command(argc, argv); + delete_device_file(apk_dest); } - delete_device_file(apk_dest); + return result; } @@ -324,7 +288,6 @@ int install_app(int argc, const char** argv) { InstallMode installMode = INSTALL_DEFAULT; bool use_fastdeploy = false; bool is_reinstall = false; - bool use_localagent = false; FastDeploy_AgentUpdateStrategy agent_update_strategy = FastDeploy_AgentUpdateDifferentVersion; for (int i = 1; i < argc; i++) { @@ -353,11 +316,6 @@ int install_app(int argc, const char** argv) { } else if (!strcmp(argv[i], "--version-check-agent")) { processedArgIndicies.push_back(i); agent_update_strategy = FastDeploy_AgentUpdateDifferentVersion; -#ifndef _WIN32 - } else if (!strcmp(argv[i], "--local-agent")) { - processedArgIndicies.push_back(i); - use_localagent = true; -#endif } } @@ -369,14 +327,13 @@ int install_app(int argc, const char** argv) { error_exit("Attempting to use streaming install on unsupported device"); } -#if defined(ENABLE_FASTDEPLOY) - if (use_fastdeploy == true && get_device_api_level() < kFastDeployMinApi) { + if (use_fastdeploy && get_device_api_level() < kFastDeployMinApi) { printf("Fast Deploy is only compatible with devices of API version %d or higher, " "ignoring.\n", kFastDeployMinApi); use_fastdeploy = false; } -#endif + fastdeploy_set_agent_update_strategy(agent_update_strategy); std::vector<const char*> passthrough_argv; for (int i = 0; i < argc; i++) { @@ -389,26 +346,13 @@ int install_app(int argc, const char** argv) { error_exit("install requires an apk argument"); } - if (use_fastdeploy == true) { -#if defined(ENABLE_FASTDEPLOY) - fastdeploy_set_local_agent(use_localagent); - update_agent(agent_update_strategy); - - // The last argument must be the APK file - const char* file = passthrough_argv.back(); - use_fastdeploy = find_package(file); -#else - error_exit("fastdeploy is disabled"); -#endif - } - switch (installMode) { case INSTALL_PUSH: return install_app_legacy(passthrough_argv.size(), passthrough_argv.data(), - use_fastdeploy, use_localagent); + use_fastdeploy); case INSTALL_STREAM: return install_app_streamed(passthrough_argv.size(), passthrough_argv.data(), - use_fastdeploy, use_localagent); + use_fastdeploy); case INSTALL_DEFAULT: default: return 1; diff --git a/adb/client/commandline.cpp b/adb/client/commandline.cpp index a0818d3ed8..0ffdbc28b7 100644 --- a/adb/client/commandline.cpp +++ b/adb/client/commandline.cpp @@ -255,13 +255,8 @@ static void stdin_raw_restore() { } #endif -// Reads from |fd| and prints received data. If |use_shell_protocol| is true -// this expects that incoming data will use the shell protocol, in which case -// stdout/stderr are routed independently and the remote exit code will be -// returned. -// if |callback| is non-null, stdout/stderr output will be handled by it. -int read_and_dump(borrowed_fd fd, bool use_shell_protocol = false, - StandardStreamsCallbackInterface* callback = &DEFAULT_STANDARD_STREAMS_CALLBACK) { +int read_and_dump(borrowed_fd fd, bool use_shell_protocol, + StandardStreamsCallbackInterface* callback) { int exit_code = 0; if (fd < 0) return exit_code; diff --git a/adb/client/commandline.h b/adb/client/commandline.h index cd5933a16c..ab77b29b3c 100644 --- a/adb/client/commandline.h +++ b/adb/client/commandline.h @@ -109,6 +109,14 @@ int send_shell_command( const std::string& command, bool disable_shell_protocol = false, StandardStreamsCallbackInterface* callback = &DEFAULT_STANDARD_STREAMS_CALLBACK); +// Reads from |fd| and prints received data. If |use_shell_protocol| is true +// this expects that incoming data will use the shell protocol, in which case +// stdout/stderr are routed independently and the remote exit code will be +// returned. +// if |callback| is non-null, stdout/stderr output will be handled by it. +int read_and_dump(borrowed_fd fd, bool use_shell_protocol = false, + StandardStreamsCallbackInterface* callback = &DEFAULT_STANDARD_STREAMS_CALLBACK); + // Connects to the device "abb" service with |command| and returns the fd. template <typename ContainerT> unique_fd send_abb_exec_command(const ContainerT& command_args, std::string* error) { diff --git a/adb/client/fastdeploy.cpp b/adb/client/fastdeploy.cpp index fbae2190d7..bdc9e5660f 100644 --- a/adb/client/fastdeploy.cpp +++ b/adb/client/fastdeploy.cpp @@ -30,112 +30,114 @@ #include "deployagent.inc" // Generated include via build rule. #include "deployagentscript.inc" // Generated include via build rule. #include "fastdeploy/deploypatchgenerator/deploy_patch_generator.h" +#include "fastdeploy/deploypatchgenerator/patch_utils.h" +#include "fastdeploy/proto/ApkEntry.pb.h" #include "fastdeploycallbacks.h" #include "sysdeps.h" #include "adb_utils.h" -static constexpr long kRequiredAgentVersion = 0x00000002; +static constexpr long kRequiredAgentVersion = 0x00000003; + +static constexpr int kPackageMissing = 3; +static constexpr int kInvalidAgentVersion = 4; -static constexpr const char* kDeviceAgentPath = "/data/local/tmp/"; static constexpr const char* kDeviceAgentFile = "/data/local/tmp/deployagent.jar"; static constexpr const char* kDeviceAgentScript = "/data/local/tmp/deployagent"; -static bool g_use_localagent = false; +static constexpr bool g_verbose_timings = false; +static FastDeploy_AgentUpdateStrategy g_agent_update_strategy = + FastDeploy_AgentUpdateDifferentVersion; -long get_agent_version() { - std::vector<char> versionOutputBuffer; - std::vector<char> versionErrorBuffer; +using APKMetaData = com::android::fastdeploy::APKMetaData; - int statusCode = capture_shell_command("/data/local/tmp/deployagent version", - &versionOutputBuffer, &versionErrorBuffer); - long version = -1; +namespace { - if (statusCode == 0 && versionOutputBuffer.size() > 0) { - version = strtol((char*)versionOutputBuffer.data(), NULL, 16); +struct TimeReporter { + TimeReporter(const char* label) : label_(label) {} + ~TimeReporter() { + if (g_verbose_timings) { + auto duration = std::chrono::duration_cast<std::chrono::milliseconds>( + std::chrono::steady_clock::now() - start_); + fprintf(stderr, "%s finished in %lldms\n", label_, + static_cast<long long>(duration.count())); + } } - return version; -} + private: + const char* label_; + std::chrono::steady_clock::time_point start_ = std::chrono::steady_clock::now(); +}; +#define REPORT_FUNC_TIME() TimeReporter reporter(__func__) + +struct FileDeleter { + FileDeleter(const char* path) : path_(path) {} + ~FileDeleter() { adb_unlink(path_); } + + private: + const char* const path_; +}; + +} // namespace int get_device_api_level() { - std::vector<char> sdkVersionOutputBuffer; - std::vector<char> sdkVersionErrorBuffer; + REPORT_FUNC_TIME(); + std::vector<char> sdk_version_output_buffer; + std::vector<char> sdk_version_error_buffer; int api_level = -1; - int statusCode = capture_shell_command("getprop ro.build.version.sdk", &sdkVersionOutputBuffer, - &sdkVersionErrorBuffer); - if (statusCode == 0 && sdkVersionOutputBuffer.size() > 0) { - api_level = strtol((char*)sdkVersionOutputBuffer.data(), NULL, 10); + int statusCode = capture_shell_command("getprop ro.build.version.sdk", + &sdk_version_output_buffer, &sdk_version_error_buffer); + if (statusCode == 0 && sdk_version_output_buffer.size() > 0) { + api_level = strtol((char*)sdk_version_output_buffer.data(), NULL, 10); } return api_level; } -void fastdeploy_set_local_agent(bool use_localagent) { - g_use_localagent = use_localagent; +void fastdeploy_set_agent_update_strategy(FastDeploy_AgentUpdateStrategy agent_update_strategy) { + g_agent_update_strategy = agent_update_strategy; } -static bool deploy_agent(bool checkTimeStamps) { +static void push_to_device(const void* data, size_t byte_count, const char* dst, bool sync) { std::vector<const char*> srcs; - // TODO: Deploy agent from bin2c directly instead of writing to disk first. - TemporaryFile tempAgent; - android::base::WriteFully(tempAgent.fd, kDeployAgent, sizeof(kDeployAgent)); - srcs.push_back(tempAgent.path); - if (!do_sync_push(srcs, kDeviceAgentFile, checkTimeStamps)) { - error_exit("Failed to push fastdeploy agent to device."); + { + TemporaryFile temp; + android::base::WriteFully(temp.fd, data, byte_count); + srcs.push_back(temp.path); + + // On Windows, the file needs to be flushed before pushing to device. + // closing the file flushes its content, but we still need to remove it after push. + // FileDeleter does exactly that. + temp.DoNotRemove(); } - srcs.clear(); - // TODO: Deploy agent from bin2c directly instead of writing to disk first. - TemporaryFile tempAgentScript; - android::base::WriteFully(tempAgentScript.fd, kDeployAgentScript, sizeof(kDeployAgentScript)); - srcs.push_back(tempAgentScript.path); - if (!do_sync_push(srcs, kDeviceAgentScript, checkTimeStamps)) { - error_exit("Failed to push fastdeploy agent script to device."); + FileDeleter temp_deleter(srcs.back()); + + if (!do_sync_push(srcs, dst, sync)) { + error_exit("Failed to push fastdeploy agent to device."); } - srcs.clear(); +} + +static bool deploy_agent(bool check_time_stamps) { + REPORT_FUNC_TIME(); + + push_to_device(kDeployAgent, sizeof(kDeployAgent), kDeviceAgentFile, check_time_stamps); + push_to_device(kDeployAgentScript, sizeof(kDeployAgentScript), kDeviceAgentScript, + check_time_stamps); + // on windows the shell script might have lost execute permission // so need to set this explicitly const char* kChmodCommandPattern = "chmod 777 %s"; - std::string chmodCommand = + std::string chmod_command = android::base::StringPrintf(kChmodCommandPattern, kDeviceAgentScript); - int ret = send_shell_command(chmodCommand); + int ret = send_shell_command(chmod_command); if (ret != 0) { - error_exit("Error executing %s returncode: %d", chmodCommand.c_str(), ret); + error_exit("Error executing %s returncode: %d", chmod_command.c_str(), ret); } return true; } -void update_agent(FastDeploy_AgentUpdateStrategy agentUpdateStrategy) { - long agent_version = get_agent_version(); - switch (agentUpdateStrategy) { - case FastDeploy_AgentUpdateAlways: - deploy_agent(false); - break; - case FastDeploy_AgentUpdateNewerTimeStamp: - deploy_agent(true); - break; - case FastDeploy_AgentUpdateDifferentVersion: - if (agent_version != kRequiredAgentVersion) { - if (agent_version < 0) { - printf("Could not detect agent on device, deploying\n"); - } else { - printf("Device agent version is (%ld), (%ld) is required, re-deploying\n", - agent_version, kRequiredAgentVersion); - } - deploy_agent(false); - } - break; - } - - agent_version = get_agent_version(); - if (agent_version != kRequiredAgentVersion) { - error_exit("After update agent version remains incorrect! Expected %ld but version is %ld", - kRequiredAgentVersion, agent_version); - } -} - static std::string get_string_from_utf16(const char16_t* input, int input_len) { ssize_t utf8_length = utf16_to_utf8_length(input, input_len); if (utf8_length <= 0) { @@ -147,29 +149,29 @@ static std::string get_string_from_utf16(const char16_t* input, int input_len) { return utf8; } -static std::string get_packagename_from_apk(const char* apkPath) { +static std::string get_package_name_from_apk(const char* apk_path) { #undef open - std::unique_ptr<android::ZipFileRO> zipFile(android::ZipFileRO::open(apkPath)); + std::unique_ptr<android::ZipFileRO> zip_file((android::ZipFileRO::open)(apk_path)); #define open ___xxx_unix_open - if (zipFile == nullptr) { - perror_exit("Could not open %s", apkPath); + if (zip_file == nullptr) { + perror_exit("Could not open %s", apk_path); } - android::ZipEntryRO entry = zipFile->findEntryByName("AndroidManifest.xml"); + android::ZipEntryRO entry = zip_file->findEntryByName("AndroidManifest.xml"); if (entry == nullptr) { - error_exit("Could not find AndroidManifest.xml inside %s", apkPath); + error_exit("Could not find AndroidManifest.xml inside %s", apk_path); } uint32_t manifest_len = 0; - if (!zipFile->getEntryInfo(entry, NULL, &manifest_len, NULL, NULL, NULL, NULL)) { - error_exit("Could not read AndroidManifest.xml inside %s", apkPath); + if (!zip_file->getEntryInfo(entry, NULL, &manifest_len, NULL, NULL, NULL, NULL)) { + error_exit("Could not read AndroidManifest.xml inside %s", apk_path); } std::vector<char> manifest_data(manifest_len); - if (!zipFile->uncompressEntry(entry, manifest_data.data(), manifest_len)) { - error_exit("Could not uncompress AndroidManifest.xml inside %s", apkPath); + if (!zip_file->uncompressEntry(entry, manifest_data.data(), manifest_len)) { + error_exit("Could not uncompress AndroidManifest.xml inside %s", apk_path); } android::ResXMLTree tree; android::status_t setto_status = tree.setTo(manifest_data.data(), manifest_len, true); if (setto_status != android::OK) { - error_exit("Could not parse AndroidManifest.xml inside %s", apkPath); + error_exit("Could not parse AndroidManifest.xml inside %s", apk_path); } android::ResXMLParser::event_code_t code; while ((code = tree.next()) != android::ResXMLParser::BAD_DOCUMENT && @@ -210,80 +212,97 @@ static std::string get_packagename_from_apk(const char* apkPath) { break; } } - error_exit("Could not find package name tag in AndroidManifest.xml inside %s", apkPath); + error_exit("Could not find package name tag in AndroidManifest.xml inside %s", apk_path); } -void extract_metadata(const char* apkPath, FILE* outputFp) { - std::string packageName = get_packagename_from_apk(apkPath); - const char* kAgentExtractCommandPattern = "/data/local/tmp/deployagent extract %s"; - std::string extractCommand = - android::base::StringPrintf(kAgentExtractCommandPattern, packageName.c_str()); - - std::vector<char> extractErrorBuffer; - DeployAgentFileCallback cb(outputFp, &extractErrorBuffer); - int returnCode = send_shell_command(extractCommand, false, &cb); - if (returnCode != 0) { - fprintf(stderr, "Executing %s returned %d\n", extractCommand.c_str(), returnCode); - fprintf(stderr, "%*s\n", int(extractErrorBuffer.size()), extractErrorBuffer.data()); - error_exit("Aborting"); +static long parse_agent_version(const std::vector<char>& version_buffer) { + long version = -1; + if (!version_buffer.empty()) { + version = strtol((char*)version_buffer.data(), NULL, 16); } + return version; } -void create_patch(const char* apkPath, const char* metadataPath, const char* patchPath) { - DeployPatchGenerator generator(false); - unique_fd patchFd(adb_open(patchPath, O_WRONLY | O_CREAT | O_CLOEXEC)); - if (patchFd < 0) { - perror_exit("adb: failed to create %s", patchPath); - } - bool success = generator.CreatePatch(apkPath, metadataPath, patchFd); - if (!success) { - error_exit("Failed to create patch for %s", apkPath); +static void update_agent_if_necessary() { + switch (g_agent_update_strategy) { + case FastDeploy_AgentUpdateAlways: + deploy_agent(/*check_time_stamps=*/false); + break; + case FastDeploy_AgentUpdateNewerTimeStamp: + deploy_agent(/*check_time_stamps=*/true); + break; + default: + break; } } -std::string get_patch_path(const char* apkPath) { - std::string packageName = get_packagename_from_apk(apkPath); - std::string patchDevicePath = - android::base::StringPrintf("%s%s.patch", kDeviceAgentPath, packageName.c_str()); - return patchDevicePath; -} - -void apply_patch_on_device(const char* apkPath, const char* patchPath, const char* outputPath) { - const std::string kAgentApplyCommandPattern = "/data/local/tmp/deployagent apply %s %s -o %s"; - std::string packageName = get_packagename_from_apk(apkPath); - std::string patchDevicePath = get_patch_path(apkPath); +std::optional<APKMetaData> extract_metadata(const char* apk_path) { + // Update agent if there is a command line argument forcing to do so. + update_agent_if_necessary(); + + REPORT_FUNC_TIME(); + + std::string package_name = get_package_name_from_apk(apk_path); + + // Dump apk command checks the required vs current agent version and if they match then returns + // the APK dump for package. Doing this in a single call saves round-trip and agent launch time. + constexpr const char* kAgentDumpCommandPattern = "/data/local/tmp/deployagent dump %ld %s"; + std::string dump_command = android::base::StringPrintf( + kAgentDumpCommandPattern, kRequiredAgentVersion, package_name.c_str()); + + std::vector<char> dump_out_buffer; + std::vector<char> dump_error_buffer; + int returnCode = + capture_shell_command(dump_command.c_str(), &dump_out_buffer, &dump_error_buffer); + if (returnCode >= kInvalidAgentVersion) { + // Agent has wrong version or missing. + long agent_version = parse_agent_version(dump_out_buffer); + if (agent_version < 0) { + printf("Could not detect agent on device, deploying\n"); + } else { + printf("Device agent version is (%ld), (%ld) is required, re-deploying\n", + agent_version, kRequiredAgentVersion); + } + deploy_agent(/*check_time_stamps=*/false); - std::vector<const char*> srcs = {patchPath}; - bool push_ok = do_sync_push(srcs, patchDevicePath.c_str(), false); - if (!push_ok) { - error_exit("Error pushing %s to %s returned", patchPath, patchDevicePath.c_str()); + // Retry with new agent. + dump_out_buffer.clear(); + dump_error_buffer.clear(); + returnCode = + capture_shell_command(dump_command.c_str(), &dump_out_buffer, &dump_error_buffer); } - - std::string applyPatchCommand = - android::base::StringPrintf(kAgentApplyCommandPattern.c_str(), packageName.c_str(), - patchDevicePath.c_str(), outputPath); - - int returnCode = send_shell_command(applyPatchCommand); if (returnCode != 0) { - error_exit("Executing %s returned %d", applyPatchCommand.c_str(), returnCode); + if (returnCode == kInvalidAgentVersion) { + long agent_version = parse_agent_version(dump_out_buffer); + error_exit( + "After update agent version remains incorrect! Expected %ld but version is %ld", + kRequiredAgentVersion, agent_version); + } + if (returnCode == kPackageMissing) { + fprintf(stderr, "Package %s not found, falling back to install\n", + package_name.c_str()); + return {}; + } + fprintf(stderr, "Executing %s returned %d\n", dump_command.c_str(), returnCode); + fprintf(stderr, "%*s\n", int(dump_error_buffer.size()), dump_error_buffer.data()); + error_exit("Aborting"); } -} -void install_patch(const char* apkPath, const char* patchPath, int argc, const char** argv) { - const std::string kAgentApplyCommandPattern = "/data/local/tmp/deployagent apply %s %s -pm %s"; - std::string packageName = get_packagename_from_apk(apkPath); + com::android::fastdeploy::APKDump dump; + if (!dump.ParseFromArray(dump_out_buffer.data(), dump_out_buffer.size())) { + fprintf(stderr, "Can't parse output of %s\n", dump_command.c_str()); + error_exit("Aborting"); + } - std::string patchDevicePath = - android::base::StringPrintf("%s%s.patch", kDeviceAgentPath, packageName.c_str()); + return PatchUtils::GetDeviceAPKMetaData(dump); +} - std::vector<const char*> srcs{patchPath}; - bool push_ok = do_sync_push(srcs, patchDevicePath.c_str(), false); - if (!push_ok) { - error_exit("Error pushing %s to %s returned", patchPath, patchDevicePath.c_str()); - } +unique_fd install_patch(int argc, const char** argv) { + REPORT_FUNC_TIME(); + constexpr char kAgentApplyServicePattern[] = "shell:/data/local/tmp/deployagent apply - -pm %s"; - std::vector<unsigned char> applyOutputBuffer; - std::vector<unsigned char> applyErrorBuffer; + std::vector<unsigned char> apply_output_buffer; + std::vector<unsigned char> apply_error_buffer; std::string argsString; bool rSwitchPresent = false; @@ -298,17 +317,42 @@ void install_patch(const char* apkPath, const char* patchPath, int argc, const c argsString.append("-r"); } - std::string applyPatchCommand = - android::base::StringPrintf(kAgentApplyCommandPattern.c_str(), packageName.c_str(), - patchDevicePath.c_str(), argsString.c_str()); - int returnCode = send_shell_command(applyPatchCommand); - if (returnCode != 0) { - error_exit("Executing %s returned %d", applyPatchCommand.c_str(), returnCode); + std::string error; + std::string apply_patch_service_string = + android::base::StringPrintf(kAgentApplyServicePattern, argsString.c_str()); + unique_fd fd{adb_connect(apply_patch_service_string, &error)}; + if (fd < 0) { + error_exit("Executing %s returned %s", apply_patch_service_string.c_str(), error.c_str()); } + return fd; } -bool find_package(const char* apkPath) { - const std::string findCommand = - "/data/local/tmp/deployagent find " + get_packagename_from_apk(apkPath); - return !send_shell_command(findCommand); +unique_fd apply_patch_on_device(const char* output_path) { + REPORT_FUNC_TIME(); + constexpr char kAgentApplyServicePattern[] = "shell:/data/local/tmp/deployagent apply - -o %s"; + + std::string error; + std::string apply_patch_service_string = + android::base::StringPrintf(kAgentApplyServicePattern, output_path); + unique_fd fd{adb_connect(apply_patch_service_string, &error)}; + if (fd < 0) { + error_exit("Executing %s returned %s", apply_patch_service_string.c_str(), error.c_str()); + } + return fd; +} + +static void create_patch(const char* apk_path, APKMetaData metadata, borrowed_fd patch_fd) { + REPORT_FUNC_TIME(); + DeployPatchGenerator generator(/*is_verbose=*/false); + bool success = generator.CreatePatch(apk_path, std::move(metadata), patch_fd); + if (!success) { + error_exit("Failed to create patch for %s", apk_path); + } +} + +int stream_patch(const char* apk_path, APKMetaData metadata, unique_fd patch_fd) { + create_patch(apk_path, std::move(metadata), patch_fd); + + REPORT_FUNC_TIME(); + return read_and_dump(patch_fd.get()); } diff --git a/adb/client/fastdeploy.h b/adb/client/fastdeploy.h index 7b7f2ec186..830aeb2ca9 100644 --- a/adb/client/fastdeploy.h +++ b/adb/client/fastdeploy.h @@ -16,6 +16,11 @@ #pragma once +#include "adb_unique_fd.h" + +#include "fastdeploy/proto/ApkEntry.pb.h" + +#include <optional> #include <string> enum FastDeploy_AgentUpdateStrategy { @@ -24,12 +29,11 @@ enum FastDeploy_AgentUpdateStrategy { FastDeploy_AgentUpdateDifferentVersion }; -void fastdeploy_set_local_agent(bool use_localagent); +void fastdeploy_set_agent_update_strategy(FastDeploy_AgentUpdateStrategy agent_update_strategy); int get_device_api_level(); -void update_agent(FastDeploy_AgentUpdateStrategy agentUpdateStrategy); -void extract_metadata(const char* apkPath, FILE* outputFp); -void create_patch(const char* apkPath, const char* metadataPath, const char* patchPath); -void apply_patch_on_device(const char* apkPath, const char* patchPath, const char* outputPath); -void install_patch(const char* apkPath, const char* patchPath, int argc, const char** argv); -std::string get_patch_path(const char* apkPath); -bool find_package(const char* apkPath); + +std::optional<com::android::fastdeploy::APKMetaData> extract_metadata(const char* apk_path); +unique_fd install_patch(int argc, const char** argv); +unique_fd apply_patch_on_device(const char* output_path); +int stream_patch(const char* apk_path, com::android::fastdeploy::APKMetaData metadata, + unique_fd patch_fd); diff --git a/adb/client/fastdeploycallbacks.cpp b/adb/client/fastdeploycallbacks.cpp index 23a0aca59d..86ceaa1378 100644 --- a/adb/client/fastdeploycallbacks.cpp +++ b/adb/client/fastdeploycallbacks.cpp @@ -49,35 +49,7 @@ class DeployAgentBufferCallback : public StandardStreamsCallbackInterface { int capture_shell_command(const char* command, std::vector<char>* outBuffer, std::vector<char>* errBuffer) { DeployAgentBufferCallback cb(outBuffer, errBuffer); - return send_shell_command(command, false, &cb); -} - -DeployAgentFileCallback::DeployAgentFileCallback(FILE* outputFile, std::vector<char>* errBuffer) { - mpOutFile = outputFile; - mpErrBuffer = errBuffer; - mBytesWritten = 0; -} - -void DeployAgentFileCallback::OnStdout(const char* buffer, int length) { - if (mpOutFile != NULL) { - int bytes_written = fwrite(buffer, 1, length, mpOutFile); - if (bytes_written != length) { - printf("Write error %d\n", bytes_written); - } - mBytesWritten += bytes_written; - } -} - -void DeployAgentFileCallback::OnStderr(const char* buffer, int length) { - appendBuffer(mpErrBuffer, buffer, length); -} - -int DeployAgentFileCallback::Done(int status) { - return status; -} - -int DeployAgentFileCallback::getBytesWritten() { - return mBytesWritten; + return send_shell_command(command, /*disable_shell_protocol=*/false, &cb); } DeployAgentBufferCallback::DeployAgentBufferCallback(std::vector<char>* outBuffer, diff --git a/adb/client/fastdeploycallbacks.h b/adb/client/fastdeploycallbacks.h index 7e049c5ae1..4a6fb99152 100644 --- a/adb/client/fastdeploycallbacks.h +++ b/adb/client/fastdeploycallbacks.h @@ -17,23 +17,6 @@ #pragma once #include <vector> -#include "commandline.h" - -class DeployAgentFileCallback : public StandardStreamsCallbackInterface { - public: - DeployAgentFileCallback(FILE* outputFile, std::vector<char>* errBuffer); - - virtual void OnStdout(const char* buffer, int length); - virtual void OnStderr(const char* buffer, int length); - virtual int Done(int status); - - int getBytesWritten(); - - private: - FILE* mpOutFile; - std::vector<char>* mpErrBuffer; - int mBytesWritten; -}; int capture_shell_command(const char* command, std::vector<char>* outBuffer, std::vector<char>* errBuffer); diff --git a/adb/fastdeploy/Android.bp b/adb/fastdeploy/Android.bp index 95e1a28d59..245d52a78e 100644 --- a/adb/fastdeploy/Android.bp +++ b/adb/fastdeploy/Android.bp @@ -13,15 +13,76 @@ // See the License for the specific language governing permissions and // limitations under the License. // -java_binary { - name: "deployagent", +java_library { + name: "deployagent_lib", sdk_version: "24", - srcs: ["deployagent/src/**/*.java", "deploylib/src/**/*.java", "proto/**/*.proto"], - static_libs: ["apkzlib_zip"], + srcs: [ + "deployagent/src/**/*.java", + "proto/**/*.proto", + ], proto: { type: "lite", }, +} + +java_binary { + name: "deployagent", + static_libs: [ + "deployagent_lib", + ], dex_preopt: { enabled: false, } -}
\ No newline at end of file +} + +android_test { + name: "FastDeployTests", + + manifest: "AndroidManifest.xml", + + srcs: [ + "deployagent/test/com/android/fastdeploy/ApkArchiveTest.java", + ], + + static_libs: [ + "androidx.test.core", + "androidx.test.runner", + "androidx.test.rules", + "deployagent_lib", + "mockito-target-inline-minus-junit4", + ], + + libs: [ + "android.test.runner", + "android.test.base", + "android.test.mock", + ], + + data: [ + "testdata/sample.apk", + "testdata/sample.cd", + ], + + optimize: { + enabled: false, + }, +} + +java_test_host { + name: "FastDeployHostTests", + srcs: [ + "deployagent/test/com/android/fastdeploy/FastDeployTest.java", + ], + data: [ + "testdata/helloworld5.apk", + "testdata/helloworld7.apk", + ], + libs: [ + "compatibility-host-util", + "cts-tradefed", + "tradefed", + ], + test_suites: [ + "general-tests", + ], +} diff --git a/adb/fastdeploy/AndroidManifest.xml b/adb/fastdeploy/AndroidManifest.xml new file mode 100644 index 0000000000..89dc7451bd --- /dev/null +++ b/adb/fastdeploy/AndroidManifest.xml @@ -0,0 +1,29 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2019 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<manifest xmlns:android="http://schemas.android.com/apk/res/android" + package="com.android.fastdeploytests"> + + <application android:testOnly="true" + android:debuggable="true"> + <uses-library android:name="android.test.runner" /> + </application> + + <instrumentation + android:name="androidx.test.runner.AndroidJUnitRunner" + android:targetPackage="com.android.fastdeploytests" + android:label="FastDeploy Tests" /> +</manifest>
\ No newline at end of file diff --git a/adb/fastdeploy/AndroidTest.xml b/adb/fastdeploy/AndroidTest.xml new file mode 100644 index 0000000000..24a72bc85a --- /dev/null +++ b/adb/fastdeploy/AndroidTest.xml @@ -0,0 +1,40 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- + ~ Copyright (C) 2019 The Android Open Source Project + ~ + ~ Licensed under the Apache License, Version 2.0 (the "License"); + ~ you may not use this file except in compliance with the License. + ~ You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License + --> +<configuration description="Runs Device Tests for FastDeploy."> + <option name="test-suite-tag" value="FastDeployTests"/> + + <target_preparer class="com.android.tradefed.targetprep.suite.SuiteApkInstaller"> + <option name="cleanup-apks" value="true"/> + <option name="install-arg" value="-t"/> + <option name="test-file-name" value="FastDeployTests.apk"/> + </target_preparer> + + <target_preparer class="com.android.tradefed.targetprep.PushFilePreparer"> + <option name="cleanup" value="false" /> + <option name="push-file" key="sample.apk" value="/data/local/tmp/FastDeployTests/sample.apk" /> + <option name="push-file" key="sample.cd" value="/data/local/tmp/FastDeployTests/sample.cd" /> + </target_preparer> + + <test class="com.android.tradefed.testtype.AndroidJUnitTest"> + <option name="package" value="com.android.fastdeploytests"/> + <option name="runner" value="androidx.test.runner.AndroidJUnitRunner"/> + </test> + + <test class="com.android.compatibility.common.tradefed.testtype.JarHostTest" > + <option name="jar" value="FastDeployHostTests.jar" /> + </test> +</configuration> diff --git a/adb/fastdeploy/deployagent/src/com/android/fastdeploy/ApkArchive.java b/adb/fastdeploy/deployagent/src/com/android/fastdeploy/ApkArchive.java new file mode 100644 index 0000000000..31e05023ca --- /dev/null +++ b/adb/fastdeploy/deployagent/src/com/android/fastdeploy/ApkArchive.java @@ -0,0 +1,193 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.fastdeploy; + +import android.util.Log; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.FileChannel; + +/** + * Extremely light-weight APK parser class. + * Aware of Central Directory, Local File Headers and Signature. + * No Zip64 support yet. + */ +public final class ApkArchive { + private static final String TAG = "ApkArchive"; + + // Central Directory constants. + private static final int EOCD_SIGNATURE = 0x06054b50; + private static final int EOCD_MIN_SIZE = 22; + private static final long EOCD_MAX_SIZE = 65_535L + EOCD_MIN_SIZE; + + private static final int CD_ENTRY_HEADER_SIZE_BYTES = 22; + private static final int CD_LOCAL_FILE_HEADER_SIZE_OFFSET = 12; + + // Signature constants. + private static final int EOSIGNATURE_SIZE = 24; + + public final static class Dump { + final byte[] cd; + final byte[] signature; + + Dump(byte[] cd, byte[] signature) { + this.cd = cd; + this.signature = signature; + } + } + + final static class Location { + final long offset; + final long size; + + public Location(long offset, long size) { + this.offset = offset; + this.size = size; + } + } + + private final RandomAccessFile mFile; + private final FileChannel mChannel; + + public ApkArchive(File apk) throws IOException { + mFile = new RandomAccessFile(apk, "r"); + mChannel = mFile.getChannel(); + } + + /** + * Extract the APK metadata: content of Central Directory and Signature. + * + * @return raw content from APK representing CD and Signature data. + */ + public Dump extractMetadata() throws IOException { + Location cdLoc = getCDLocation(); + byte[] cd = readMetadata(cdLoc); + + byte[] signature = null; + Location sigLoc = getSignatureLocation(cdLoc.offset); + if (sigLoc != null) { + signature = readMetadata(sigLoc); + long size = ByteBuffer.wrap(signature).order(ByteOrder.LITTLE_ENDIAN).getLong(); + if (sigLoc.size != size) { + Log.e(TAG, "Mismatching signature sizes: " + sigLoc.size + " != " + size); + signature = null; + } + } + + return new Dump(cd, signature); + } + + private long findEndOfCDRecord() throws IOException { + final long fileSize = mChannel.size(); + int sizeToRead = Math.toIntExact(Math.min(fileSize, EOCD_MAX_SIZE)); + final long readOffset = fileSize - sizeToRead; + ByteBuffer buffer = mChannel.map(FileChannel.MapMode.READ_ONLY, readOffset, + sizeToRead).order(ByteOrder.LITTLE_ENDIAN); + + buffer.position(sizeToRead - EOCD_MIN_SIZE); + while (true) { + int signature = buffer.getInt(); // Read 4 bytes. + if (signature == EOCD_SIGNATURE) { + return readOffset + buffer.position() - 4; + } + if (buffer.position() == 4) { + break; + } + buffer.position(buffer.position() - Integer.BYTES - 1); // Backtrack 5 bytes. + } + + return -1L; + } + + private Location findCDRecord(ByteBuffer buf) { + if (buf.order() != ByteOrder.LITTLE_ENDIAN) { + throw new IllegalArgumentException("ByteBuffer byte order must be little endian"); + } + if (buf.remaining() < CD_ENTRY_HEADER_SIZE_BYTES) { + throw new IllegalArgumentException( + "Input too short. Need at least " + CD_ENTRY_HEADER_SIZE_BYTES + + " bytes, available: " + buf.remaining() + "bytes."); + } + + int originalPosition = buf.position(); + int recordSignature = buf.getInt(); + if (recordSignature != EOCD_SIGNATURE) { + throw new IllegalArgumentException( + "Not a Central Directory record. Signature: 0x" + + Long.toHexString(recordSignature & 0xffffffffL)); + } + + buf.position(originalPosition + CD_LOCAL_FILE_HEADER_SIZE_OFFSET); + long size = buf.getInt() & 0xffffffffL; + long offset = buf.getInt() & 0xffffffffL; + return new Location(offset, size); + } + + // Retrieve the location of the Central Directory Record. + Location getCDLocation() throws IOException { + long eocdRecord = findEndOfCDRecord(); + if (eocdRecord < 0) { + throw new IllegalArgumentException("Unable to find End of Central Directory record."); + } + + Location location = findCDRecord(mChannel.map(FileChannel.MapMode.READ_ONLY, eocdRecord, + CD_ENTRY_HEADER_SIZE_BYTES).order(ByteOrder.LITTLE_ENDIAN)); + if (location == null) { + throw new IllegalArgumentException("Unable to find Central Directory File Header."); + } + + return location; + } + + // Retrieve the location of the signature block starting from Central + // Directory Record or null if signature is not found. + Location getSignatureLocation(long cdRecordOffset) throws IOException { + long signatureOffset = cdRecordOffset - EOSIGNATURE_SIZE; + if (signatureOffset < 0) { + Log.e(TAG, "Unable to find Signature."); + return null; + } + + ByteBuffer signature = mChannel.map(FileChannel.MapMode.READ_ONLY, signatureOffset, + EOSIGNATURE_SIZE).order(ByteOrder.LITTLE_ENDIAN); + + long size = signature.getLong(); + + byte[] sign = new byte[16]; + signature.get(sign); + String signAsString = new String(sign); + if (!"APK Sig Block 42".equals(signAsString)) { + Log.e(TAG, "Signature magic does not match: " + signAsString); + return null; + } + + long offset = cdRecordOffset - size - 8; + + return new Location(offset, size); + } + + private byte[] readMetadata(Location loc) throws IOException { + byte[] payload = new byte[(int) loc.size]; + ByteBuffer buffer = mChannel.map(FileChannel.MapMode.READ_ONLY, loc.offset, loc.size); + buffer.get(payload); + return payload; + } +} diff --git a/adb/fastdeploy/deployagent/src/com/android/fastdeploy/DeployAgent.java b/adb/fastdeploy/deployagent/src/com/android/fastdeploy/DeployAgent.java index a8103c4d25..3812307cca 100644 --- a/adb/fastdeploy/deployagent/src/com/android/fastdeploy/DeployAgent.java +++ b/adb/fastdeploy/deployagent/src/com/android/fastdeploy/DeployAgent.java @@ -24,18 +24,22 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.TimeUnit; -import java.io.OutputStream; import java.io.RandomAccessFile; -import java.util.Set; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.nio.channels.WritableByteChannel; +import com.android.fastdeploy.PatchFormatException; +import com.android.fastdeploy.ApkArchive; +import com.android.fastdeploy.APKDump; import com.android.fastdeploy.APKMetaData; import com.android.fastdeploy.PatchUtils; +import com.google.protobuf.ByteString; + public final class DeployAgent { private static final int BUFFER_SIZE = 128 * 1024; - private static final int AGENT_VERSION = 0x00000002; + private static final int AGENT_VERSION = 0x00000003; public static void main(String[] args) { int exitCode = 0; @@ -45,68 +49,70 @@ public final class DeployAgent { } String commandString = args[0]; + switch (commandString) { + case "dump": { + if (args.length != 3) { + showUsage(1); + } - if (commandString.equals("extract")) { - if (args.length != 2) { - showUsage(1); - } - - String packageName = args[1]; - extractMetaData(packageName); - } else if (commandString.equals("find")) { - if (args.length != 2) { - showUsage(1); - } - - String packageName = args[1]; - if (getFilenameFromPackageName(packageName) == null) { - exitCode = 3; - } - } else if (commandString.equals("apply")) { - if (args.length < 4) { - showUsage(1); + String requiredVersion = args[1]; + if (AGENT_VERSION == Integer.parseInt(requiredVersion)) { + String packageName = args[2]; + String packagePath = getFilenameFromPackageName(packageName); + if (packagePath != null) { + dumpApk(packageName, packagePath); + } else { + exitCode = 3; + } + } else { + System.out.printf("0x%08X\n", AGENT_VERSION); + exitCode = 4; + } + break; } + case "apply": { + if (args.length < 3) { + showUsage(1); + } - String packageName = args[1]; - String patchPath = args[2]; - String outputParam = args[3]; + String patchPath = args[1]; + String outputParam = args[2]; - InputStream deltaInputStream = null; - if (patchPath.compareTo("-") == 0) { - deltaInputStream = System.in; - } else { - deltaInputStream = new FileInputStream(patchPath); - } + InputStream deltaInputStream = null; + if (patchPath.compareTo("-") == 0) { + deltaInputStream = System.in; + } else { + deltaInputStream = new FileInputStream(patchPath); + } - if (outputParam.equals("-o")) { - OutputStream outputStream = null; - if (args.length > 4) { - String outputPath = args[4]; - if (!outputPath.equals("-")) { - outputStream = new FileOutputStream(outputPath); + if (outputParam.equals("-o")) { + OutputStream outputStream = null; + if (args.length > 3) { + String outputPath = args[3]; + if (!outputPath.equals("-")) { + outputStream = new FileOutputStream(outputPath); + } } - } - if (outputStream == null) { - outputStream = System.out; - } - File deviceFile = getFileFromPackageName(packageName); - writePatchToStream( - new RandomAccessFile(deviceFile, "r"), deltaInputStream, outputStream); - } else if (outputParam.equals("-pm")) { - String[] sessionArgs = null; - if (args.length > 4) { - int numSessionArgs = args.length-4; - sessionArgs = new String[numSessionArgs]; - for (int i=0 ; i<numSessionArgs ; i++) { - sessionArgs[i] = args[i+4]; + if (outputStream == null) { + outputStream = System.out; + } + writePatchToStream(deltaInputStream, outputStream); + } else if (outputParam.equals("-pm")) { + String[] sessionArgs = null; + if (args.length > 3) { + int numSessionArgs = args.length - 3; + sessionArgs = new String[numSessionArgs]; + for (int i = 0; i < numSessionArgs; i++) { + sessionArgs[i] = args[i + 3]; + } } + exitCode = applyPatch(deltaInputStream, sessionArgs); } - exitCode = applyPatch(packageName, deltaInputStream, sessionArgs); + break; } - } else if (commandString.equals("version")) { - System.out.printf("0x%08X\n", AGENT_VERSION); - } else { - showUsage(1); + default: + showUsage(1); + break; } } catch (Exception e) { System.err.println("Error: " + e); @@ -118,16 +124,16 @@ public final class DeployAgent { private static void showUsage(int exitCode) { System.err.println( - "usage: deployagent <command> [<args>]\n\n" + - "commands:\n" + - "version get the version\n" + - "find PKGNAME return zero if package found, else non-zero\n" + - "extract PKGNAME extract an installed package's metadata\n" + - "apply PKGNAME PATCHFILE [-o|-pm] apply a patch from PATCHFILE (- for stdin) to an installed package\n" + - " -o <FILE> directs output to FILE, default or - for stdout\n" + - " -pm <ARGS> directs output to package manager, passes <ARGS> to 'pm install-create'\n" - ); - + "usage: deployagent <command> [<args>]\n\n" + + "commands:\n" + + "dump VERSION PKGNAME dump info for an installed package given that " + + "VERSION equals current agent's version\n" + + "apply PATCHFILE [-o|-pm] apply a patch from PATCHFILE " + + "(- for stdin) to an installed package\n" + + " -o <FILE> directs output to FILE, default or - for stdout\n" + + " -pm <ARGS> directs output to package manager, passes <ARGS> to " + + "'pm install-create'\n" + ); System.exit(exitCode); } @@ -162,32 +168,34 @@ public final class DeployAgent { } int equalsIndex = line.lastIndexOf(packageSuffix); String fileName = - line.substring(packageIndex + packagePrefix.length(), equalsIndex); + line.substring(packageIndex + packagePrefix.length(), equalsIndex); return fileName; } } return null; } - private static File getFileFromPackageName(String packageName) throws IOException { - String filename = getFilenameFromPackageName(packageName); - if (filename == null) { - // Should not happen (function is only called when we know the package exists) - throw new IOException("package not found"); + private static void dumpApk(String packageName, String packagePath) throws IOException { + File apk = new File(packagePath); + ApkArchive.Dump dump = new ApkArchive(apk).extractMetadata(); + + APKDump.Builder apkDumpBuilder = APKDump.newBuilder(); + apkDumpBuilder.setName(packageName); + if (dump.cd != null) { + apkDumpBuilder.setCd(ByteString.copyFrom(dump.cd)); } - return new File(filename); - } + if (dump.signature != null) { + apkDumpBuilder.setSignature(ByteString.copyFrom(dump.signature)); + } + apkDumpBuilder.setAbsolutePath(apk.getAbsolutePath()); - private static void extractMetaData(String packageName) throws IOException { - File apkFile = getFileFromPackageName(packageName); - APKMetaData apkMetaData = PatchUtils.getAPKMetaData(apkFile); - apkMetaData.writeTo(System.out); + apkDumpBuilder.build().writeTo(System.out); } private static int createInstallSession(String[] args) throws IOException { StringBuilder commandBuilder = new StringBuilder(); commandBuilder.append("pm install-create "); - for (int i=0 ; args != null && i<args.length ; i++) { + for (int i = 0; args != null && i < args.length; i++) { commandBuilder.append(args[i] + " "); } @@ -199,7 +207,8 @@ public final class DeployAgent { String successLineEnd = "]"; while ((line = reader.readLine()) != null) { if (line.startsWith(successLineStart) && line.endsWith(successLineEnd)) { - return Integer.parseInt(line.substring(successLineStart.length(), line.lastIndexOf(successLineEnd))); + return Integer.parseInt(line.substring(successLineStart.length(), + line.lastIndexOf(successLineEnd))); } } @@ -213,16 +222,15 @@ public final class DeployAgent { return p.exitValue(); } - private static int applyPatch(String packageName, InputStream deltaStream, String[] sessionArgs) + private static int applyPatch(InputStream deltaStream, String[] sessionArgs) throws IOException, PatchFormatException { - File deviceFile = getFileFromPackageName(packageName); int sessionId = createInstallSession(sessionArgs); if (sessionId < 0) { System.err.println("PM Create Session Failed"); return -1; } - int writeExitCode = writePatchedDataToSession(new RandomAccessFile(deviceFile, "r"), deltaStream, sessionId); + int writeExitCode = writePatchedDataToSession(deltaStream, sessionId); if (writeExitCode == 0) { return commitInstallSession(sessionId); } else { @@ -230,84 +238,94 @@ public final class DeployAgent { } } - private static long writePatchToStream(RandomAccessFile oldData, InputStream patchData, - OutputStream outputStream) throws IOException, PatchFormatException { + private static long writePatchToStream(InputStream patchData, + OutputStream outputStream) throws IOException, PatchFormatException { long newSize = readPatchHeader(patchData); - long bytesWritten = writePatchedDataToStream(oldData, newSize, patchData, outputStream); + long bytesWritten = writePatchedDataToStream(newSize, patchData, outputStream); outputStream.flush(); if (bytesWritten != newSize) { throw new PatchFormatException(String.format( - "output size mismatch (expected %ld but wrote %ld)", newSize, bytesWritten)); + "output size mismatch (expected %ld but wrote %ld)", newSize, bytesWritten)); } return bytesWritten; } private static long readPatchHeader(InputStream patchData) - throws IOException, PatchFormatException { + throws IOException, PatchFormatException { byte[] signatureBuffer = new byte[PatchUtils.SIGNATURE.length()]; try { - PatchUtils.readFully(patchData, signatureBuffer, 0, signatureBuffer.length); + PatchUtils.readFully(patchData, signatureBuffer); } catch (IOException e) { throw new PatchFormatException("truncated signature"); } - String signature = new String(signatureBuffer, 0, signatureBuffer.length, "US-ASCII"); + String signature = new String(signatureBuffer); if (!PatchUtils.SIGNATURE.equals(signature)) { throw new PatchFormatException("bad signature"); } - long newSize = PatchUtils.readBsdiffLong(patchData); - if (newSize < 0 || newSize > Integer.MAX_VALUE) { - throw new PatchFormatException("bad newSize"); + long newSize = PatchUtils.readLELong(patchData); + if (newSize < 0) { + throw new PatchFormatException("bad newSize: " + newSize); } return newSize; } // Note that this function assumes patchData has been seek'ed to the start of the delta stream - // (i.e. the signature has already been read by readPatchHeader). For a stream that points to the - // start of a patch file call writePatchToStream - private static long writePatchedDataToStream(RandomAccessFile oldData, long newSize, - InputStream patchData, OutputStream outputStream) throws IOException { + // (i.e. the signature has already been read by readPatchHeader). For a stream that points to + // the start of a patch file call writePatchToStream + private static long writePatchedDataToStream(long newSize, InputStream patchData, + OutputStream outputStream) throws IOException { + String deviceFile = PatchUtils.readString(patchData); + RandomAccessFile oldDataFile = new RandomAccessFile(deviceFile, "r"); + FileChannel oldData = oldDataFile.getChannel(); + + WritableByteChannel newData = Channels.newChannel(outputStream); + long newDataBytesWritten = 0; byte[] buffer = new byte[BUFFER_SIZE]; while (newDataBytesWritten < newSize) { - long copyLen = PatchUtils.readFormattedLong(patchData); - if (copyLen > 0) { - PatchUtils.pipe(patchData, outputStream, buffer, (int) copyLen); + long newDataLen = PatchUtils.readLELong(patchData); + if (newDataLen > 0) { + PatchUtils.pipe(patchData, outputStream, buffer, newDataLen); } - long oldDataOffset = PatchUtils.readFormattedLong(patchData); - long oldDataLen = PatchUtils.readFormattedLong(patchData); - oldData.seek(oldDataOffset); - if (oldDataLen > 0) { - PatchUtils.pipe(oldData, outputStream, buffer, (int) oldDataLen); + long oldDataOffset = PatchUtils.readLELong(patchData); + long oldDataLen = PatchUtils.readLELong(patchData); + if (oldDataLen >= 0) { + long offset = oldDataOffset; + long len = oldDataLen; + while (len > 0) { + long chunkLen = Math.min(len, 1024*1024*1024); + oldData.transferTo(offset, chunkLen, newData); + offset += chunkLen; + len -= chunkLen; + } } - newDataBytesWritten += copyLen + oldDataLen; + newDataBytesWritten += newDataLen + oldDataLen; } return newDataBytesWritten; } - private static int writePatchedDataToSession(RandomAccessFile oldData, InputStream patchData, int sessionId) + private static int writePatchedDataToSession(InputStream patchData, int sessionId) throws IOException, PatchFormatException { try { Process p; long newSize = readPatchHeader(patchData); - StringBuilder commandBuilder = new StringBuilder(); - commandBuilder.append(String.format("pm install-write -S %d %d -- -", newSize, sessionId)); - - String command = commandBuilder.toString(); + String command = String.format("pm install-write -S %d %d -- -", newSize, sessionId); p = Runtime.getRuntime().exec(command); OutputStream sessionOutputStream = p.getOutputStream(); - long bytesWritten = writePatchedDataToStream(oldData, newSize, patchData, sessionOutputStream); + long bytesWritten = writePatchedDataToStream(newSize, patchData, sessionOutputStream); sessionOutputStream.flush(); p.waitFor(); if (bytesWritten != newSize) { throw new PatchFormatException( - String.format("output size mismatch (expected %d but wrote %)", newSize, bytesWritten)); + String.format("output size mismatch (expected %d but wrote %)", newSize, + bytesWritten)); } return p.exitValue(); } catch (InterruptedException e) { diff --git a/adb/fastdeploy/deploylib/src/com/android/fastdeploy/PatchFormatException.java b/adb/fastdeploy/deployagent/src/com/android/fastdeploy/PatchFormatException.java index f0655f325c..f0655f325c 100644 --- a/adb/fastdeploy/deploylib/src/com/android/fastdeploy/PatchFormatException.java +++ b/adb/fastdeploy/deployagent/src/com/android/fastdeploy/PatchFormatException.java diff --git a/adb/fastdeploy/deployagent/src/com/android/fastdeploy/PatchUtils.java b/adb/fastdeploy/deployagent/src/com/android/fastdeploy/PatchUtils.java new file mode 100644 index 0000000000..54be26f651 --- /dev/null +++ b/adb/fastdeploy/deployagent/src/com/android/fastdeploy/PatchUtils.java @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.fastdeploy; + +import java.io.DataInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +class PatchUtils { + public static final String SIGNATURE = "FASTDEPLOY"; + + /** + * Reads a 64-bit signed integer in Little Endian format from the specified {@link + * DataInputStream}. + * + * @param in the stream to read from. + */ + static long readLELong(InputStream in) throws IOException { + byte[] buffer = new byte[Long.BYTES]; + readFully(in, buffer); + ByteBuffer buf = ByteBuffer.wrap(buffer).order(ByteOrder.LITTLE_ENDIAN); + return buf.getLong(); + } + + static String readString(InputStream in) throws IOException { + int size = (int) readLELong(in); + byte[] buffer = new byte[size]; + readFully(in, buffer); + return new String(buffer); + } + + static void readFully(final InputStream in, final byte[] destination, final int startAt, + final int numBytes) throws IOException { + int numRead = 0; + while (numRead < numBytes) { + int readNow = in.read(destination, startAt + numRead, numBytes - numRead); + if (readNow == -1) { + throw new IOException("truncated input stream"); + } + numRead += readNow; + } + } + + static void readFully(final InputStream in, final byte[] destination) throws IOException { + readFully(in, destination, 0, destination.length); + } + + static void pipe(final InputStream in, final OutputStream out, final byte[] buffer, + long copyLength) throws IOException { + while (copyLength > 0) { + int maxCopy = (int) Math.min(buffer.length, copyLength); + readFully(in, buffer, 0, maxCopy); + out.write(buffer, 0, maxCopy); + copyLength -= maxCopy; + } + } +} diff --git a/adb/fastdeploy/deployagent/test/com/android/fastdeploy/ApkArchiveTest.java b/adb/fastdeploy/deployagent/test/com/android/fastdeploy/ApkArchiveTest.java new file mode 100644 index 0000000000..7c2468f524 --- /dev/null +++ b/adb/fastdeploy/deployagent/test/com/android/fastdeploy/ApkArchiveTest.java @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.fastdeploy; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + +import androidx.test.filters.SmallTest; +import androidx.test.runner.AndroidJUnit4; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; + +import com.android.fastdeploy.ApkArchive; + +import java.io.File; +import java.io.IOException; + +@SmallTest +@RunWith(AndroidJUnit4.class) +public class ApkArchiveTest { + private static final File SAMPLE_APK = new File("/data/local/tmp/FastDeployTests/sample.apk"); + private static final File WRONG_APK = new File("/data/local/tmp/FastDeployTests/sample.cd"); + + @Test + public void testApkArchiveSizes() throws IOException { + ApkArchive archive = new ApkArchive(SAMPLE_APK); + + ApkArchive.Location cdLoc = archive.getCDLocation(); + assertNotEquals(cdLoc, null); + assertEquals(cdLoc.offset, 2044145); + assertEquals(cdLoc.size, 49390); + + // Check that block can be retrieved + ApkArchive.Location sigLoc = archive.getSignatureLocation(cdLoc.offset); + assertNotEquals(sigLoc, null); + assertEquals(sigLoc.offset, 2040049); + assertEquals(sigLoc.size, 4088); + } + + @Test + public void testApkArchiveDump() throws IOException { + ApkArchive archive = new ApkArchive(SAMPLE_APK); + + ApkArchive.Dump dump = archive.extractMetadata(); + assertNotEquals(dump, null); + assertNotEquals(dump.cd, null); + assertNotEquals(dump.signature, null); + assertEquals(dump.cd.length, 49390); + assertEquals(dump.signature.length, 4088); + } + + @Test(expected = IllegalArgumentException.class) + public void testApkArchiveDumpWrongApk() throws IOException { + ApkArchive archive = new ApkArchive(WRONG_APK); + + archive.extractMetadata(); + } +} diff --git a/adb/fastdeploy/deployagent/test/com/android/fastdeploy/FastDeployTest.java b/adb/fastdeploy/deployagent/test/com/android/fastdeploy/FastDeployTest.java new file mode 100644 index 0000000000..ef6ccaea6b --- /dev/null +++ b/adb/fastdeploy/deployagent/test/com/android/fastdeploy/FastDeployTest.java @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.fastdeploy; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.testtype.DeviceJUnit4ClassRunner; +import com.android.tradefed.testtype.junit4.BaseHostJUnit4Test; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.Arrays; + +@RunWith(DeviceJUnit4ClassRunner.class) +public class FastDeployTest extends BaseHostJUnit4Test { + + private static final String TEST_APP_PACKAGE = "com.example.helloworld"; + private static final String TEST_APK5_NAME = "helloworld5.apk"; + private static final String TEST_APK7_NAME = "helloworld7.apk"; + + private String mTestApk5Path; + private String mTestApk7Path; + + @Before + public void setUp() throws Exception { + CompatibilityBuildHelper buildHelper = new CompatibilityBuildHelper(getBuild()); + getDevice().uninstallPackage(TEST_APP_PACKAGE); + mTestApk5Path = buildHelper.getTestFile(TEST_APK5_NAME).getAbsolutePath(); + mTestApk7Path = buildHelper.getTestFile(TEST_APK7_NAME).getAbsolutePath(); + } + + @Test + public void testAppInstalls() throws Exception { + fastInstallPackage(mTestApk5Path); + assertTrue(isAppInstalled(TEST_APP_PACKAGE)); + getDevice().uninstallPackage(TEST_APP_PACKAGE); + assertFalse(isAppInstalled(TEST_APP_PACKAGE)); + } + + @Test + public void testAppPatch() throws Exception { + fastInstallPackage(mTestApk5Path); + assertTrue(isAppInstalled(TEST_APP_PACKAGE)); + fastInstallPackage(mTestApk7Path); + assertTrue(isAppInstalled(TEST_APP_PACKAGE)); + getDevice().uninstallPackage(TEST_APP_PACKAGE); + assertFalse(isAppInstalled(TEST_APP_PACKAGE)); + } + + private boolean isAppInstalled(String packageName) throws DeviceNotAvailableException { + final String commandResult = getDevice().executeShellCommand("pm list packages"); + final int prefixLength = "package:".length(); + return Arrays.stream(commandResult.split("\\r?\\n")) + .anyMatch(line -> line.substring(prefixLength).equals(packageName)); + } + + // Mostly copied from PkgInstallSignatureVerificationTest.java. + private String fastInstallPackage(String apkPath) + throws IOException, DeviceNotAvailableException { + return getDevice().executeAdbCommand("install", "-t", "--fastdeploy", "--force-agent", + apkPath); + } +} + + diff --git a/adb/fastdeploy/deploylib/src/com/android/fastdeploy/PatchUtils.java b/adb/fastdeploy/deploylib/src/com/android/fastdeploy/PatchUtils.java deleted file mode 100644 index c60f9a613e..0000000000 --- a/adb/fastdeploy/deploylib/src/com/android/fastdeploy/PatchUtils.java +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.fastdeploy; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.RandomAccessFile; -import java.util.Arrays; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; - -import com.android.tools.build.apkzlib.zip.ZFile; -import com.android.tools.build.apkzlib.zip.ZFileOptions; -import com.android.tools.build.apkzlib.zip.StoredEntry; -import com.android.tools.build.apkzlib.zip.StoredEntryType; -import com.android.tools.build.apkzlib.zip.CentralDirectoryHeaderCompressInfo; -import com.android.tools.build.apkzlib.zip.CentralDirectoryHeader; - -import com.android.fastdeploy.APKMetaData; -import com.android.fastdeploy.APKEntry; - -class PatchUtils { - private static final long NEGATIVE_MASK = 1L << 63; - private static final long NEGATIVE_LONG_SIGN_MASK = 1L << 63; - public static final String SIGNATURE = "FASTDEPLOY"; - - private static long getOffsetFromEntry(StoredEntry entry) { - return entry.getCentralDirectoryHeader().getOffset() + entry.getLocalHeaderSize(); - } - - public static APKMetaData getAPKMetaData(File apkFile) throws IOException { - APKMetaData.Builder apkEntriesBuilder = APKMetaData.newBuilder(); - ZFileOptions options = new ZFileOptions(); - ZFile zFile = new ZFile(apkFile, options); - - ArrayList<StoredEntry> metaDataEntries = new ArrayList<StoredEntry>(); - - for (StoredEntry entry : zFile.entries()) { - if (entry.getType() != StoredEntryType.FILE) { - continue; - } - metaDataEntries.add(entry); - } - - Collections.sort(metaDataEntries, new Comparator<StoredEntry>() { - private long getOffsetFromEntry(StoredEntry entry) { - return PatchUtils.getOffsetFromEntry(entry); - } - - @Override - public int compare(StoredEntry lhs, StoredEntry rhs) { - // -1 - less than, 1 - greater than, 0 - equal, all inversed for descending - return Long.compare(getOffsetFromEntry(lhs), getOffsetFromEntry(rhs)); - } - }); - - for (StoredEntry entry : metaDataEntries) { - CentralDirectoryHeader cdh = entry.getCentralDirectoryHeader(); - CentralDirectoryHeaderCompressInfo cdhci = cdh.getCompressionInfoWithWait(); - - APKEntry.Builder entryBuilder = APKEntry.newBuilder(); - entryBuilder.setCrc32(cdh.getCrc32()); - entryBuilder.setFileName(cdh.getName()); - entryBuilder.setCompressedSize(cdhci.getCompressedSize()); - entryBuilder.setUncompressedSize(cdh.getUncompressedSize()); - entryBuilder.setDataOffset(getOffsetFromEntry(entry)); - - apkEntriesBuilder.addEntries(entryBuilder); - apkEntriesBuilder.build(); - } - return apkEntriesBuilder.build(); - } - - /** - * Writes a 64-bit signed integer to the specified {@link OutputStream}. The least significant - * byte is written first and the most significant byte is written last. - * @param value the value to write - * @param outputStream the stream to write to - */ - static void writeFormattedLong(final long value, OutputStream outputStream) throws IOException { - long y = value; - if (y < 0) { - y = (-y) | NEGATIVE_MASK; - } - - for (int i = 0; i < 8; ++i) { - outputStream.write((byte) (y & 0xff)); - y >>>= 8; - } - } - - /** - * Reads a 64-bit signed integer written by {@link #writeFormattedLong(long, OutputStream)} from - * the specified {@link InputStream}. - * @param inputStream the stream to read from - */ - static long readFormattedLong(InputStream inputStream) throws IOException { - long result = 0; - for (int bitshift = 0; bitshift < 64; bitshift += 8) { - result |= ((long) inputStream.read()) << bitshift; - } - - if ((result - NEGATIVE_MASK) > 0) { - result = (result & ~NEGATIVE_MASK) * -1; - } - return result; - } - - static final long readBsdiffLong(InputStream in) throws PatchFormatException, IOException { - long result = 0; - for (int bitshift = 0; bitshift < 64; bitshift += 8) { - result |= ((long) in.read()) << bitshift; - } - - if (result == NEGATIVE_LONG_SIGN_MASK) { - // "Negative zero", which is valid in signed-magnitude format. - // NB: No sane patch generator should ever produce such a value. - throw new PatchFormatException("read negative zero"); - } - - if ((result & NEGATIVE_LONG_SIGN_MASK) != 0) { - result = -(result & ~NEGATIVE_LONG_SIGN_MASK); - } - - return result; - } - - static void readFully(final InputStream in, final byte[] destination, final int startAt, - final int numBytes) throws IOException { - int numRead = 0; - while (numRead < numBytes) { - int readNow = in.read(destination, startAt + numRead, numBytes - numRead); - if (readNow == -1) { - throw new IOException("truncated input stream"); - } - numRead += readNow; - } - } - - static void pipe(final InputStream in, final OutputStream out, final byte[] buffer, - long copyLength) throws IOException { - while (copyLength > 0) { - int maxCopy = Math.min(buffer.length, (int) copyLength); - readFully(in, buffer, 0, maxCopy); - out.write(buffer, 0, maxCopy); - copyLength -= maxCopy; - } - } - - static void pipe(final RandomAccessFile in, final OutputStream out, final byte[] buffer, - long copyLength) throws IOException { - while (copyLength > 0) { - int maxCopy = Math.min(buffer.length, (int) copyLength); - in.readFully(buffer, 0, maxCopy); - out.write(buffer, 0, maxCopy); - copyLength -= maxCopy; - } - } - - static void fill(byte value, final OutputStream out, final byte[] buffer, long fillLength) - throws IOException { - while (fillLength > 0) { - int maxCopy = Math.min(buffer.length, (int) fillLength); - Arrays.fill(buffer, 0, maxCopy, value); - out.write(buffer, 0, maxCopy); - fillLength -= maxCopy; - } - } -} diff --git a/adb/fastdeploy/deploypatchgenerator/apk_archive.cpp b/adb/fastdeploy/deploypatchgenerator/apk_archive.cpp new file mode 100644 index 0000000000..3dc5e506e8 --- /dev/null +++ b/adb/fastdeploy/deploypatchgenerator/apk_archive.cpp @@ -0,0 +1,415 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define TRACE_TAG ADB + +#include "apk_archive.h" + +#include "adb_trace.h" +#include "sysdeps.h" + +#include <android-base/endian.h> +#include <android-base/mapped_file.h> + +#include <openssl/md5.h> + +constexpr uint16_t kCompressStored = 0; + +// mask value that signifies that the entry has a DD +static const uint32_t kGPBDDFlagMask = 0x0008; + +namespace { +struct FileRegion { + FileRegion(borrowed_fd fd, off64_t offset, size_t length) + : mapped_(android::base::MappedFile::FromOsHandle(adb_get_os_handle(fd), offset, length, + PROT_READ)) { + if (mapped_.data() != nullptr) { + return; + } + + // Mapped file failed, falling back to pread. + buffer_.resize(length); + if (auto err = adb_pread(fd.get(), buffer_.data(), length, offset); size_t(err) != length) { + fprintf(stderr, "Unable to read %lld bytes at offset %" PRId64 " \n", + static_cast<long long>(length), offset); + buffer_.clear(); + return; + } + } + + const char* data() const { return mapped_.data() ? mapped_.data() : buffer_.data(); } + size_t size() const { return mapped_.data() ? mapped_.size() : buffer_.size(); } + + private: + FileRegion() = default; + DISALLOW_COPY_AND_ASSIGN(FileRegion); + + android::base::MappedFile mapped_; + std::string buffer_; +}; +} // namespace + +using com::android::fastdeploy::APKDump; + +ApkArchive::ApkArchive(const std::string& path) : path_(path), size_(0) { + fd_.reset(adb_open(path_.c_str(), O_RDONLY)); + if (fd_ == -1) { + fprintf(stderr, "Unable to open file '%s'\n", path_.c_str()); + return; + } + + struct stat st; + if (stat(path_.c_str(), &st) == -1) { + fprintf(stderr, "Unable to stat file '%s'\n", path_.c_str()); + return; + } + size_ = st.st_size; +} + +ApkArchive::~ApkArchive() {} + +APKDump ApkArchive::ExtractMetadata() { + D("ExtractMetadata"); + if (!ready()) { + return {}; + } + + Location cdLoc = GetCDLocation(); + if (!cdLoc.valid) { + return {}; + } + + APKDump dump; + dump.set_absolute_path(path_); + dump.set_cd(ReadMetadata(cdLoc)); + + Location sigLoc = GetSignatureLocation(cdLoc.offset); + if (sigLoc.valid) { + dump.set_signature(ReadMetadata(sigLoc)); + } + return dump; +} + +off_t ApkArchive::FindEndOfCDRecord() const { + constexpr int endOfCDSignature = 0x06054b50; + constexpr off_t endOfCDMinSize = 22; + constexpr off_t endOfCDMaxSize = 65535 + endOfCDMinSize; + + auto sizeToRead = std::min(size_, endOfCDMaxSize); + auto readOffset = size_ - sizeToRead; + FileRegion mapped(fd_, readOffset, sizeToRead); + + // Start scanning from the end + auto* start = mapped.data(); + auto* cursor = start + mapped.size() - sizeof(endOfCDSignature); + + // Search for End of Central Directory record signature. + while (cursor >= start) { + if (*(int32_t*)cursor == endOfCDSignature) { + return readOffset + (cursor - start); + } + cursor--; + } + return -1; +} + +ApkArchive::Location ApkArchive::FindCDRecord(const char* cursor) { + struct ecdr_t { + int32_t signature; + uint16_t diskNumber; + uint16_t numDisk; + uint16_t diskEntries; + uint16_t numEntries; + uint32_t crSize; + uint32_t offsetToCdHeader; + uint16_t commentSize; + uint8_t comment[0]; + } __attribute__((packed)); + ecdr_t* header = (ecdr_t*)cursor; + + Location location; + location.offset = header->offsetToCdHeader; + location.size = header->crSize; + location.valid = true; + return location; +} + +ApkArchive::Location ApkArchive::GetCDLocation() { + constexpr off_t cdEntryHeaderSizeBytes = 22; + Location location; + + // Find End of Central Directory Record + off_t eocdRecord = FindEndOfCDRecord(); + if (eocdRecord < 0) { + fprintf(stderr, "Unable to find End of Central Directory record in file '%s'\n", + path_.c_str()); + return location; + } + + // Find Central Directory Record + FileRegion mapped(fd_, eocdRecord, cdEntryHeaderSizeBytes); + location = FindCDRecord(mapped.data()); + if (!location.valid) { + fprintf(stderr, "Unable to find Central Directory File Header in file '%s'\n", + path_.c_str()); + return location; + } + + return location; +} + +ApkArchive::Location ApkArchive::GetSignatureLocation(off_t cdRecordOffset) { + Location location; + + // Signature constants. + constexpr off_t endOfSignatureSize = 24; + off_t signatureOffset = cdRecordOffset - endOfSignatureSize; + if (signatureOffset < 0) { + fprintf(stderr, "Unable to find signature in file '%s'\n", path_.c_str()); + return location; + } + + FileRegion mapped(fd_, signatureOffset, endOfSignatureSize); + + uint64_t signatureSize = *(uint64_t*)mapped.data(); + auto* signature = mapped.data() + sizeof(signatureSize); + // Check if there is a v2/v3 Signature block here. + if (memcmp(signature, "APK Sig Block 42", 16)) { + return location; + } + + // This is likely a signature block. + location.size = signatureSize; + location.offset = cdRecordOffset - location.size - 8; + location.valid = true; + + return location; +} + +std::string ApkArchive::ReadMetadata(Location loc) const { + FileRegion mapped(fd_, loc.offset, loc.size); + return {mapped.data(), mapped.size()}; +} + +size_t ApkArchive::ParseCentralDirectoryRecord(const char* input, size_t size, std::string* md5Hash, + int64_t* localFileHeaderOffset, int64_t* dataSize) { + // A structure representing the fixed length fields for a single + // record in the central directory of the archive. In addition to + // the fixed length fields listed here, each central directory + // record contains a variable length "file_name" and "extra_field" + // whose lengths are given by |file_name_length| and |extra_field_length| + // respectively. + static constexpr int kCDFileHeaderMagic = 0x02014b50; + struct CentralDirectoryRecord { + // The start of record signature. Must be |kSignature|. + uint32_t record_signature; + // Source tool version. Top byte gives source OS. + uint16_t version_made_by; + // Tool version. Ignored by this implementation. + uint16_t version_needed; + // The "general purpose bit flags" for this entry. The only + // flag value that we currently check for is the "data descriptor" + // flag. + uint16_t gpb_flags; + // The compression method for this entry, one of |kCompressStored| + // and |kCompressDeflated|. + uint16_t compression_method; + // The file modification time and date for this entry. + uint16_t last_mod_time; + uint16_t last_mod_date; + // The CRC-32 checksum for this entry. + uint32_t crc32; + // The compressed size (in bytes) of this entry. + uint32_t compressed_size; + // The uncompressed size (in bytes) of this entry. + uint32_t uncompressed_size; + // The length of the entry file name in bytes. The file name + // will appear immediately after this record. + uint16_t file_name_length; + // The length of the extra field info (in bytes). This data + // will appear immediately after the entry file name. + uint16_t extra_field_length; + // The length of the entry comment (in bytes). This data will + // appear immediately after the extra field. + uint16_t comment_length; + // The start disk for this entry. Ignored by this implementation). + uint16_t file_start_disk; + // File attributes. Ignored by this implementation. + uint16_t internal_file_attributes; + // File attributes. For archives created on Unix, the top bits are the + // mode. + uint32_t external_file_attributes; + // The offset to the local file header for this entry, from the + // beginning of this archive. + uint32_t local_file_header_offset; + + private: + CentralDirectoryRecord() = default; + DISALLOW_COPY_AND_ASSIGN(CentralDirectoryRecord); + } __attribute__((packed)); + + const CentralDirectoryRecord* cdr; + if (size < sizeof(*cdr)) { + return 0; + } + + auto begin = input; + cdr = reinterpret_cast<const CentralDirectoryRecord*>(begin); + if (cdr->record_signature != kCDFileHeaderMagic) { + fprintf(stderr, "Invalid Central Directory Record signature\n"); + return 0; + } + auto end = begin + sizeof(*cdr) + cdr->file_name_length + cdr->extra_field_length + + cdr->comment_length; + + uint8_t md5Digest[MD5_DIGEST_LENGTH]; + MD5((const unsigned char*)begin, end - begin, md5Digest); + md5Hash->assign((const char*)md5Digest, sizeof(md5Digest)); + + *localFileHeaderOffset = cdr->local_file_header_offset; + *dataSize = (cdr->compression_method == kCompressStored) ? cdr->uncompressed_size + : cdr->compressed_size; + + return end - begin; +} + +size_t ApkArchive::CalculateLocalFileEntrySize(int64_t localFileHeaderOffset, + int64_t dataSize) const { + // The local file header for a given entry. This duplicates information + // present in the central directory of the archive. It is an error for + // the information here to be different from the central directory + // information for a given entry. + static constexpr int kLocalFileHeaderMagic = 0x04034b50; + struct LocalFileHeader { + // The local file header signature, must be |kSignature|. + uint32_t lfh_signature; + // Tool version. Ignored by this implementation. + uint16_t version_needed; + // The "general purpose bit flags" for this entry. The only + // flag value that we currently check for is the "data descriptor" + // flag. + uint16_t gpb_flags; + // The compression method for this entry, one of |kCompressStored| + // and |kCompressDeflated|. + uint16_t compression_method; + // The file modification time and date for this entry. + uint16_t last_mod_time; + uint16_t last_mod_date; + // The CRC-32 checksum for this entry. + uint32_t crc32; + // The compressed size (in bytes) of this entry. + uint32_t compressed_size; + // The uncompressed size (in bytes) of this entry. + uint32_t uncompressed_size; + // The length of the entry file name in bytes. The file name + // will appear immediately after this record. + uint16_t file_name_length; + // The length of the extra field info (in bytes). This data + // will appear immediately after the entry file name. + uint16_t extra_field_length; + + private: + LocalFileHeader() = default; + DISALLOW_COPY_AND_ASSIGN(LocalFileHeader); + } __attribute__((packed)); + static constexpr int kLocalFileHeaderSize = sizeof(LocalFileHeader); + CHECK(ready()) << path_; + + const LocalFileHeader* lfh; + if (localFileHeaderOffset + kLocalFileHeaderSize > size_) { + fprintf(stderr, + "Invalid Local File Header offset in file '%s' at offset %lld, file size %lld\n", + path_.c_str(), static_cast<long long>(localFileHeaderOffset), + static_cast<long long>(size_)); + return 0; + } + + FileRegion lfhMapped(fd_, localFileHeaderOffset, sizeof(LocalFileHeader)); + lfh = reinterpret_cast<const LocalFileHeader*>(lfhMapped.data()); + if (lfh->lfh_signature != kLocalFileHeaderMagic) { + fprintf(stderr, "Invalid Local File Header signature in file '%s' at offset %lld\n", + path_.c_str(), static_cast<long long>(localFileHeaderOffset)); + return 0; + } + + // The *optional* data descriptor start signature. + static constexpr int kOptionalDataDescriptorMagic = 0x08074b50; + struct DataDescriptor { + // CRC-32 checksum of the entry. + uint32_t crc32; + // Compressed size of the entry. + uint32_t compressed_size; + // Uncompressed size of the entry. + uint32_t uncompressed_size; + + private: + DataDescriptor() = default; + DISALLOW_COPY_AND_ASSIGN(DataDescriptor); + } __attribute__((packed)); + static constexpr int kDataDescriptorSize = sizeof(DataDescriptor); + + off_t ddOffset = localFileHeaderOffset + kLocalFileHeaderSize + lfh->file_name_length + + lfh->extra_field_length + dataSize; + int64_t ddSize = 0; + + int64_t localDataSize; + if (lfh->gpb_flags & kGPBDDFlagMask) { + // There is trailing data descriptor. + const DataDescriptor* dd; + + if (ddOffset + int(sizeof(uint32_t)) > size_) { + fprintf(stderr, + "Error reading trailing data descriptor signature in file '%s' at offset %lld, " + "file size %lld\n", + path_.c_str(), static_cast<long long>(ddOffset), static_cast<long long>(size_)); + return 0; + } + + FileRegion ddMapped(fd_, ddOffset, sizeof(uint32_t) + sizeof(DataDescriptor)); + + off_t localDDOffset = 0; + if (kOptionalDataDescriptorMagic == *(uint32_t*)ddMapped.data()) { + ddOffset += sizeof(uint32_t); + localDDOffset += sizeof(uint32_t); + ddSize += sizeof(uint32_t); + } + if (ddOffset + kDataDescriptorSize > size_) { + fprintf(stderr, + "Error reading trailing data descriptor in file '%s' at offset %lld, file size " + "%lld\n", + path_.c_str(), static_cast<long long>(ddOffset), static_cast<long long>(size_)); + return 0; + } + + dd = reinterpret_cast<const DataDescriptor*>(ddMapped.data() + localDDOffset); + localDataSize = (lfh->compression_method == kCompressStored) ? dd->uncompressed_size + : dd->compressed_size; + ddSize += sizeof(*dd); + } else { + localDataSize = (lfh->compression_method == kCompressStored) ? lfh->uncompressed_size + : lfh->compressed_size; + } + if (localDataSize != dataSize) { + fprintf(stderr, + "Data sizes mismatch in file '%s' at offset %lld, CDr: %lld vs LHR/DD: %lld\n", + path_.c_str(), static_cast<long long>(localFileHeaderOffset), + static_cast<long long>(dataSize), static_cast<long long>(localDataSize)); + return 0; + } + + return kLocalFileHeaderSize + lfh->file_name_length + lfh->extra_field_length + dataSize + + ddSize; +} diff --git a/adb/fastdeploy/deploypatchgenerator/apk_archive.h b/adb/fastdeploy/deploypatchgenerator/apk_archive.h new file mode 100644 index 0000000000..7127800a93 --- /dev/null +++ b/adb/fastdeploy/deploypatchgenerator/apk_archive.h @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <memory> +#include <string> +#include <vector> + +#include <adb_unique_fd.h> + +#include "fastdeploy/proto/ApkEntry.pb.h" + +class ApkArchiveTester; + +// Manipulates an APK archive. Process it by mmaping it in order to minimize +// I/Os. +class ApkArchive { + public: + friend ApkArchiveTester; + + // A convenience struct to store the result of search operation when + // locating the EoCDr, CDr, and Signature Block. + struct Location { + off_t offset = 0; + off_t size = 0; + bool valid = false; + }; + + ApkArchive(const std::string& path); + ~ApkArchive(); + + com::android::fastdeploy::APKDump ExtractMetadata(); + + // Parses the CDr starting from |input| and returns number of bytes consumed. + // Extracts local file header offset, data size and calculates MD5 hash of the record. + // 0 indicates invalid CDr. + static size_t ParseCentralDirectoryRecord(const char* input, size_t size, std::string* md5Hash, + int64_t* localFileHeaderOffset, int64_t* dataSize); + // Calculates Local File Entry size including header using offset and data size from CDr. + // 0 indicates invalid Local File Entry. + size_t CalculateLocalFileEntrySize(int64_t localFileHeaderOffset, int64_t dataSize) const; + + private: + std::string ReadMetadata(Location loc) const; + + // Retrieve the location of the Central Directory Record. + Location GetCDLocation(); + + // Retrieve the location of the signature block starting from Central + // Directory Record + Location GetSignatureLocation(off_t cdRecordOffset); + + // Find the End of Central Directory Record, starting from the end of the + // file. + off_t FindEndOfCDRecord() const; + + // Find Central Directory Record, starting from the end of the file. + Location FindCDRecord(const char* cursor); + + // Checks if the archive can be used. + bool ready() const { return fd_ >= 0; } + + std::string path_; + off_t size_; + unique_fd fd_; +}; diff --git a/adb/fastdeploy/deploypatchgenerator/apk_archive_test.cpp b/adb/fastdeploy/deploypatchgenerator/apk_archive_test.cpp new file mode 100644 index 0000000000..554cb570ee --- /dev/null +++ b/adb/fastdeploy/deploypatchgenerator/apk_archive_test.cpp @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <iostream> + +#include <gtest/gtest.h> + +#include "apk_archive.h" + +// Friend test to get around private scope of ApkArchive private functions. +class ApkArchiveTester { + public: + ApkArchiveTester(const std::string& path) : archive_(path) {} + + bool ready() { return archive_.ready(); } + + auto ExtractMetadata() { return archive_.ExtractMetadata(); } + + ApkArchive::Location GetCDLocation() { return archive_.GetCDLocation(); } + ApkArchive::Location GetSignatureLocation(size_t start) { + return archive_.GetSignatureLocation(start); + } + + private: + ApkArchive archive_; +}; + +TEST(ApkArchiveTest, TestApkArchiveSizes) { + ApkArchiveTester archiveTester("fastdeploy/testdata/sample.apk"); + EXPECT_TRUE(archiveTester.ready()); + + ApkArchive::Location cdLoc = archiveTester.GetCDLocation(); + EXPECT_TRUE(cdLoc.valid); + ASSERT_EQ(cdLoc.offset, 2044145u); + ASSERT_EQ(cdLoc.size, 49390u); + + // Check that block can be retrieved + ApkArchive::Location sigLoc = archiveTester.GetSignatureLocation(cdLoc.offset); + EXPECT_TRUE(sigLoc.valid); + ASSERT_EQ(sigLoc.offset, 2040049u); + ASSERT_EQ(sigLoc.size, 4088u); +} + +TEST(ApkArchiveTest, TestApkArchiveDump) { + ApkArchiveTester archiveTester("fastdeploy/testdata/sample.apk"); + EXPECT_TRUE(archiveTester.ready()); + + auto dump = archiveTester.ExtractMetadata(); + ASSERT_EQ(dump.cd().size(), 49390u); + ASSERT_EQ(dump.signature().size(), 4088u); +} + +TEST(ApkArchiveTest, WrongApk) { + ApkArchiveTester archiveTester("fastdeploy/testdata/sample.cd"); + EXPECT_TRUE(archiveTester.ready()); + + auto dump = archiveTester.ExtractMetadata(); + ASSERT_EQ(dump.cd().size(), 0u); + ASSERT_EQ(dump.signature().size(), 0u); +} diff --git a/adb/fastdeploy/deploypatchgenerator/deploy_patch_generator.cpp b/adb/fastdeploy/deploypatchgenerator/deploy_patch_generator.cpp index 154c9b9085..8aa7da72fc 100644 --- a/adb/fastdeploy/deploypatchgenerator/deploy_patch_generator.cpp +++ b/adb/fastdeploy/deploypatchgenerator/deploy_patch_generator.cpp @@ -25,8 +25,12 @@ #include <iostream> #include <sstream> #include <string> +#include <unordered_map> + +#include <openssl/md5.h> #include "adb_unique_fd.h" +#include "adb_utils.h" #include "android-base/file.h" #include "patch_utils.h" #include "sysdeps.h" @@ -34,9 +38,6 @@ using namespace com::android::fastdeploy; void DeployPatchGenerator::Log(const char* fmt, ...) { - if (!is_verbose_) { - return; - } va_list ap; va_start(ap, fmt); vprintf(fmt, ap); @@ -44,19 +45,34 @@ void DeployPatchGenerator::Log(const char* fmt, ...) { va_end(ap); } +static std::string HexEncode(const void* in_buffer, unsigned int size) { + static const char kHexChars[] = "0123456789ABCDEF"; + + // Each input byte creates two output hex characters. + std::string out_buffer(size * 2, '\0'); + + for (unsigned int i = 0; i < size; ++i) { + char byte = ((const uint8_t*)in_buffer)[i]; + out_buffer[(i << 1)] = kHexChars[(byte >> 4) & 0xf]; + out_buffer[(i << 1) + 1] = kHexChars[byte & 0xf]; + } + return out_buffer; +} + void DeployPatchGenerator::APKEntryToLog(const APKEntry& entry) { - Log("Filename: %s", entry.filename().c_str()); - Log("CRC32: 0x%08" PRIX64, entry.crc32()); + if (!is_verbose_) { + return; + } + Log("MD5: %s", HexEncode(entry.md5().data(), entry.md5().size()).c_str()); Log("Data Offset: %" PRId64, entry.dataoffset()); - Log("Compressed Size: %" PRId64, entry.compressedsize()); - Log("Uncompressed Size: %" PRId64, entry.uncompressedsize()); + Log("Data Size: %" PRId64, entry.datasize()); } -void DeployPatchGenerator::APKMetaDataToLog(const char* file, const APKMetaData& metadata) { +void DeployPatchGenerator::APKMetaDataToLog(const APKMetaData& metadata) { if (!is_verbose_) { return; } - Log("APK Metadata: %s", file); + Log("APK Metadata: %s", metadata.absolute_path().c_str()); for (int i = 0; i < metadata.entries_size(); i++) { const APKEntry& entry = metadata.entries(i); APKEntryToLog(entry); @@ -65,49 +81,93 @@ void DeployPatchGenerator::APKMetaDataToLog(const char* file, const APKMetaData& void DeployPatchGenerator::ReportSavings(const std::vector<SimpleEntry>& identicalEntries, uint64_t totalSize) { - long totalEqualBytes = 0; - int totalEqualFiles = 0; + uint64_t totalEqualBytes = 0; + uint64_t totalEqualFiles = 0; for (size_t i = 0; i < identicalEntries.size(); i++) { if (identicalEntries[i].deviceEntry != nullptr) { - totalEqualBytes += identicalEntries[i].localEntry->compressedsize(); + totalEqualBytes += identicalEntries[i].localEntry->datasize(); totalEqualFiles++; } } - float savingPercent = (totalEqualBytes * 100.0f) / totalSize; - fprintf(stderr, "Detected %d equal APK entries\n", totalEqualFiles); - fprintf(stderr, "%ld bytes are equal out of %" PRIu64 " (%.2f%%)\n", totalEqualBytes, totalSize, - savingPercent); + double savingPercent = (totalEqualBytes * 100.0f) / totalSize; + fprintf(stderr, "Detected %" PRIu64 " equal APK entries\n", totalEqualFiles); + fprintf(stderr, "%" PRIu64 " bytes are equal out of %" PRIu64 " (%.2f%%)\n", totalEqualBytes, + totalSize, savingPercent); +} + +struct PatchEntry { + int64_t deltaFromDeviceDataStart = 0; + int64_t deviceDataOffset = 0; + int64_t deviceDataLength = 0; +}; +static void WritePatchEntry(const PatchEntry& patchEntry, borrowed_fd input, borrowed_fd output, + size_t* realSizeOut) { + if (!(patchEntry.deltaFromDeviceDataStart | patchEntry.deviceDataOffset | + patchEntry.deviceDataLength)) { + return; + } + + PatchUtils::WriteLong(patchEntry.deltaFromDeviceDataStart, output); + if (patchEntry.deltaFromDeviceDataStart > 0) { + PatchUtils::Pipe(input, output, patchEntry.deltaFromDeviceDataStart); + } + auto hostDataLength = patchEntry.deviceDataLength; + adb_lseek(input, hostDataLength, SEEK_CUR); + + PatchUtils::WriteLong(patchEntry.deviceDataOffset, output); + PatchUtils::WriteLong(patchEntry.deviceDataLength, output); + + *realSizeOut += patchEntry.deltaFromDeviceDataStart + hostDataLength; } void DeployPatchGenerator::GeneratePatch(const std::vector<SimpleEntry>& entriesToUseOnDevice, - const char* localApkPath, borrowed_fd output) { - unique_fd input(adb_open(localApkPath, O_RDONLY | O_CLOEXEC)); + const std::string& localApkPath, + const std::string& deviceApkPath, borrowed_fd output) { + unique_fd input(adb_open(localApkPath.c_str(), O_RDONLY | O_CLOEXEC)); size_t newApkSize = adb_lseek(input, 0L, SEEK_END); adb_lseek(input, 0L, SEEK_SET); + // Header. PatchUtils::WriteSignature(output); PatchUtils::WriteLong(newApkSize, output); + PatchUtils::WriteString(deviceApkPath, output); + size_t currentSizeOut = 0; + size_t realSizeOut = 0; // Write data from the host upto the first entry we have that matches a device entry. Then write // the metadata about the device entry and repeat for all entries that match on device. Finally // write out any data left. If the device and host APKs are exactly the same this ends up // writing out zip metadata from the local APK followed by offsets to the data to use from the // device APK. - for (auto&& entry : entriesToUseOnDevice) { - int64_t deviceDataOffset = entry.deviceEntry->dataoffset(); + PatchEntry patchEntry; + for (size_t i = 0, size = entriesToUseOnDevice.size(); i < size; ++i) { + auto&& entry = entriesToUseOnDevice[i]; int64_t hostDataOffset = entry.localEntry->dataoffset(); - int64_t deviceDataLength = entry.deviceEntry->compressedsize(); + int64_t hostDataLength = entry.localEntry->datasize(); + int64_t deviceDataOffset = entry.deviceEntry->dataoffset(); + // Both entries are the same, using host data length. + int64_t deviceDataLength = hostDataLength; + int64_t deltaFromDeviceDataStart = hostDataOffset - currentSizeOut; - PatchUtils::WriteLong(deltaFromDeviceDataStart, output); if (deltaFromDeviceDataStart > 0) { - PatchUtils::Pipe(input, output, deltaFromDeviceDataStart); + WritePatchEntry(patchEntry, input, output, &realSizeOut); + patchEntry.deltaFromDeviceDataStart = deltaFromDeviceDataStart; + patchEntry.deviceDataOffset = deviceDataOffset; + patchEntry.deviceDataLength = deviceDataLength; + } else { + patchEntry.deviceDataLength += deviceDataLength; } - PatchUtils::WriteLong(deviceDataOffset, output); - PatchUtils::WriteLong(deviceDataLength, output); - adb_lseek(input, deviceDataLength, SEEK_CUR); - currentSizeOut += deltaFromDeviceDataStart + deviceDataLength; + + currentSizeOut += deltaFromDeviceDataStart + hostDataLength; } - if (currentSizeOut != newApkSize) { + WritePatchEntry(patchEntry, input, output, &realSizeOut); + if (realSizeOut != currentSizeOut) { + fprintf(stderr, "Size mismatch current %lld vs real %lld\n", + static_cast<long long>(currentSizeOut), static_cast<long long>(realSizeOut)); + error_exit("Aborting"); + } + + if (newApkSize > currentSizeOut) { PatchUtils::WriteLong(newApkSize - currentSizeOut, output); PatchUtils::Pipe(input, output, newApkSize - currentSizeOut); PatchUtils::WriteLong(0, output); @@ -115,44 +175,72 @@ void DeployPatchGenerator::GeneratePatch(const std::vector<SimpleEntry>& entries } } -bool DeployPatchGenerator::CreatePatch(const char* localApkPath, const char* deviceApkMetadataPath, +bool DeployPatchGenerator::CreatePatch(const char* localApkPath, APKMetaData deviceApkMetadata, + android::base::borrowed_fd output) { + return CreatePatch(PatchUtils::GetHostAPKMetaData(localApkPath), std::move(deviceApkMetadata), + output); +} + +bool DeployPatchGenerator::CreatePatch(APKMetaData localApkMetadata, APKMetaData deviceApkMetadata, borrowed_fd output) { - std::string content; - APKMetaData deviceApkMetadata; - if (android::base::ReadFileToString(deviceApkMetadataPath, &content)) { - deviceApkMetadata.ParsePartialFromString(content); - } else { - // TODO: What do we want to do if we don't find any metadata. - // The current fallback behavior is to build a patch with the contents of |localApkPath|. - } + // Log metadata info. + APKMetaDataToLog(deviceApkMetadata); + APKMetaDataToLog(localApkMetadata); - APKMetaData localApkMetadata = PatchUtils::GetAPKMetaData(localApkPath); - // Log gathered metadata info. - APKMetaDataToLog(deviceApkMetadataPath, deviceApkMetadata); - APKMetaDataToLog(localApkPath, localApkMetadata); + const std::string localApkPath = localApkMetadata.absolute_path(); + const std::string deviceApkPath = deviceApkMetadata.absolute_path(); std::vector<SimpleEntry> identicalEntries; uint64_t totalSize = BuildIdenticalEntries(identicalEntries, localApkMetadata, deviceApkMetadata); ReportSavings(identicalEntries, totalSize); - GeneratePatch(identicalEntries, localApkPath, output); + GeneratePatch(identicalEntries, localApkPath, deviceApkPath, output); + return true; } uint64_t DeployPatchGenerator::BuildIdenticalEntries(std::vector<SimpleEntry>& outIdenticalEntries, const APKMetaData& localApkMetadata, const APKMetaData& deviceApkMetadata) { + outIdenticalEntries.reserve( + std::min(localApkMetadata.entries_size(), deviceApkMetadata.entries_size())); + + using md5Digest = std::pair<uint64_t, uint64_t>; + struct md5Hash { + size_t operator()(const md5Digest& digest) const { + std::hash<uint64_t> hasher; + size_t seed = 0; + seed ^= hasher(digest.first) + 0x9e3779b9 + (seed << 6) + (seed >> 2); + seed ^= hasher(digest.second) + 0x9e3779b9 + (seed << 6) + (seed >> 2); + return seed; + } + }; + static_assert(sizeof(md5Digest) == MD5_DIGEST_LENGTH); + std::unordered_map<md5Digest, std::vector<const APKEntry*>, md5Hash> deviceEntries; + for (const auto& deviceEntry : deviceApkMetadata.entries()) { + md5Digest md5; + memcpy(&md5, deviceEntry.md5().data(), deviceEntry.md5().size()); + + deviceEntries[md5].push_back(&deviceEntry); + } + uint64_t totalSize = 0; - for (int i = 0; i < localApkMetadata.entries_size(); i++) { - const APKEntry& localEntry = localApkMetadata.entries(i); - totalSize += localEntry.compressedsize(); - for (int j = 0; j < deviceApkMetadata.entries_size(); j++) { - const APKEntry& deviceEntry = deviceApkMetadata.entries(j); - if (deviceEntry.crc32() == localEntry.crc32() && - deviceEntry.filename().compare(localEntry.filename()) == 0) { + for (const auto& localEntry : localApkMetadata.entries()) { + totalSize += localEntry.datasize(); + + md5Digest md5; + memcpy(&md5, localEntry.md5().data(), localEntry.md5().size()); + + auto deviceEntriesIt = deviceEntries.find(md5); + if (deviceEntriesIt == deviceEntries.end()) { + continue; + } + + for (const auto* deviceEntry : deviceEntriesIt->second) { + if (deviceEntry->md5() == localEntry.md5()) { SimpleEntry simpleEntry; - simpleEntry.localEntry = const_cast<APKEntry*>(&localEntry); - simpleEntry.deviceEntry = const_cast<APKEntry*>(&deviceEntry); + simpleEntry.localEntry = &localEntry; + simpleEntry.deviceEntry = deviceEntry; APKEntryToLog(localEntry); outIdenticalEntries.push_back(simpleEntry); break; diff --git a/adb/fastdeploy/deploypatchgenerator/deploy_patch_generator.h b/adb/fastdeploy/deploypatchgenerator/deploy_patch_generator.h index 30e41a542d..fd7eaeee9b 100644 --- a/adb/fastdeploy/deploypatchgenerator/deploy_patch_generator.h +++ b/adb/fastdeploy/deploypatchgenerator/deploy_patch_generator.h @@ -27,12 +27,15 @@ */ class DeployPatchGenerator { public: + using APKEntry = com::android::fastdeploy::APKEntry; + using APKMetaData = com::android::fastdeploy::APKMetaData; + /** * Simple struct to hold mapping between local metadata and device metadata. */ struct SimpleEntry { - com::android::fastdeploy::APKEntry* localEntry; - com::android::fastdeploy::APKEntry* deviceEntry; + const APKEntry* localEntry; + const APKEntry* deviceEntry; }; /** @@ -41,10 +44,10 @@ class DeployPatchGenerator { */ explicit DeployPatchGenerator(bool is_verbose) : is_verbose_(is_verbose) {} /** - * Given a |localApkPath|, and the |deviceApkMetadataPath| from an installed APK this function + * Given a |localApkPath|, and the |deviceApkMetadata| from an installed APK this function * writes a patch to the given |output|. */ - bool CreatePatch(const char* localApkPath, const char* deviceApkMetadataPath, + bool CreatePatch(const char* localApkPath, APKMetaData deviceApkMetadata, android::base::borrowed_fd output); private: @@ -57,14 +60,20 @@ class DeployPatchGenerator { /** * Helper function to log the APKMetaData structure. If |is_verbose_| is false this function - * early outs. |file| is the path to the file represented by |metadata|. This function is used - * for debugging / information. + * early outs. This function is used for debugging / information. */ - void APKMetaDataToLog(const char* file, const com::android::fastdeploy::APKMetaData& metadata); + void APKMetaDataToLog(const APKMetaData& metadata); /** * Helper function to log APKEntry. */ - void APKEntryToLog(const com::android::fastdeploy::APKEntry& entry); + void APKEntryToLog(const APKEntry& entry); + + /** + * Given the |localApkMetadata| metadata, and the |deviceApkMetadata| from an installed APK this + * function writes a patch to the given |output|. + */ + bool CreatePatch(APKMetaData localApkMetadata, APKMetaData deviceApkMetadata, + android::base::borrowed_fd output); /** * Helper function to report savings by fastdeploy. This function prints out savings even with @@ -92,11 +101,11 @@ class DeployPatchGenerator { * highest. */ void GeneratePatch(const std::vector<SimpleEntry>& entriesToUseOnDevice, - const char* localApkPath, android::base::borrowed_fd output); + const std::string& localApkPath, const std::string& deviceApkPath, + android::base::borrowed_fd output); protected: - uint64_t BuildIdenticalEntries( - std::vector<SimpleEntry>& outIdenticalEntries, - const com::android::fastdeploy::APKMetaData& localApkMetadata, - const com::android::fastdeploy::APKMetaData& deviceApkMetadataPath); -};
\ No newline at end of file + uint64_t BuildIdenticalEntries(std::vector<SimpleEntry>& outIdenticalEntries, + const APKMetaData& localApkMetadata, + const APKMetaData& deviceApkMetadata); +}; diff --git a/adb/fastdeploy/deploypatchgenerator/deploy_patch_generator_test.cpp b/adb/fastdeploy/deploypatchgenerator/deploy_patch_generator_test.cpp index 9cdc44ef14..e4c96eaffa 100644 --- a/adb/fastdeploy/deploypatchgenerator/deploy_patch_generator_test.cpp +++ b/adb/fastdeploy/deploypatchgenerator/deploy_patch_generator_test.cpp @@ -15,6 +15,7 @@ */ #include "deploy_patch_generator.h" +#include "apk_archive.h" #include "patch_utils.h" #include <android-base/file.h> @@ -31,21 +32,17 @@ static std::string GetTestFile(const std::string& name) { return "fastdeploy/testdata/" + name; } -class TestPatchGenerator : DeployPatchGenerator { - public: - TestPatchGenerator() : DeployPatchGenerator(false) {} - void GatherIdenticalEntries(std::vector<DeployPatchGenerator::SimpleEntry>& outIdenticalEntries, - const APKMetaData& metadataA, const APKMetaData& metadataB) { - BuildIdenticalEntries(outIdenticalEntries, metadataA, metadataB); - } +struct TestPatchGenerator : DeployPatchGenerator { + using DeployPatchGenerator::BuildIdenticalEntries; + using DeployPatchGenerator::DeployPatchGenerator; }; TEST(DeployPatchGeneratorTest, IdenticalFileEntries) { std::string apkPath = GetTestFile("rotating_cube-release.apk"); - APKMetaData metadataA = PatchUtils::GetAPKMetaData(apkPath.c_str()); - TestPatchGenerator generator; + APKMetaData metadataA = PatchUtils::GetHostAPKMetaData(apkPath.c_str()); + TestPatchGenerator generator(false); std::vector<DeployPatchGenerator::SimpleEntry> entries; - generator.GatherIdenticalEntries(entries, metadataA, metadataA); + generator.BuildIdenticalEntries(entries, metadataA, metadataA); // Expect the entry count to match the number of entries in the metadata. const uint32_t identicalCount = entries.size(); const uint32_t entriesCount = metadataA.entries_size(); @@ -64,9 +61,28 @@ TEST(DeployPatchGeneratorTest, NoDeviceMetadata) { // Create a patch that is 100% different. TemporaryFile output; DeployPatchGenerator generator(true); - generator.CreatePatch(apkPath.c_str(), "", output.fd); + generator.CreatePatch(apkPath.c_str(), {}, output.fd); // Expect a patch file that has a size at least the size of our initial APK. long patchSize = adb_lseek(output.fd, 0L, SEEK_END); EXPECT_GT(patchSize, apkSize); -}
\ No newline at end of file +} + +TEST(DeployPatchGeneratorTest, ZeroSizePatch) { + std::string apkPath = GetTestFile("rotating_cube-release.apk"); + ApkArchive archive(apkPath); + auto dump = archive.ExtractMetadata(); + EXPECT_NE(dump.cd().size(), 0u); + + APKMetaData metadata = PatchUtils::GetDeviceAPKMetaData(dump); + + // Create a patch that is 100% the same. + TemporaryFile output; + output.DoNotRemove(); + DeployPatchGenerator generator(true); + generator.CreatePatch(apkPath.c_str(), metadata, output.fd); + + // Expect a patch file that is smaller than 0.5K. + int64_t patchSize = adb_lseek(output.fd, 0L, SEEK_END); + EXPECT_LE(patchSize, 512); +} diff --git a/adb/fastdeploy/deploypatchgenerator/patch_utils.cpp b/adb/fastdeploy/deploypatchgenerator/patch_utils.cpp index f11ddd11f4..2b00c80164 100644 --- a/adb/fastdeploy/deploypatchgenerator/patch_utils.cpp +++ b/adb/fastdeploy/deploypatchgenerator/patch_utils.cpp @@ -16,51 +16,65 @@ #include "patch_utils.h" -#include <androidfw/ZipFileRO.h> #include <stdio.h> #include "adb_io.h" +#include "adb_utils.h" #include "android-base/endian.h" #include "sysdeps.h" +#include "apk_archive.h" + using namespace com::android; using namespace com::android::fastdeploy; using namespace android::base; static constexpr char kSignature[] = "FASTDEPLOY"; -APKMetaData PatchUtils::GetAPKMetaData(const char* apkPath) { +APKMetaData PatchUtils::GetDeviceAPKMetaData(const APKDump& apk_dump) { APKMetaData apkMetaData; -#undef open - std::unique_ptr<android::ZipFileRO> zipFile(android::ZipFileRO::open(apkPath)); -#define open ___xxx_unix_open - if (zipFile == nullptr) { - printf("Could not open %s", apkPath); - exit(1); + apkMetaData.set_absolute_path(apk_dump.absolute_path()); + + std::string md5Hash; + int64_t localFileHeaderOffset; + int64_t dataSize; + + const auto& cd = apk_dump.cd(); + auto cur = cd.data(); + int64_t size = cd.size(); + while (auto consumed = ApkArchive::ParseCentralDirectoryRecord( + cur, size, &md5Hash, &localFileHeaderOffset, &dataSize)) { + cur += consumed; + size -= consumed; + + auto apkEntry = apkMetaData.add_entries(); + apkEntry->set_md5(md5Hash); + apkEntry->set_dataoffset(localFileHeaderOffset); + apkEntry->set_datasize(dataSize); + } + return apkMetaData; +} + +APKMetaData PatchUtils::GetHostAPKMetaData(const char* apkPath) { + ApkArchive archive(apkPath); + auto dump = archive.ExtractMetadata(); + if (dump.cd().empty()) { + fprintf(stderr, "adb: Could not extract Central Directory from %s\n", apkPath); + error_exit("Aborting"); } - void* cookie; - if (zipFile->startIteration(&cookie)) { - android::ZipEntryRO entry; - while ((entry = zipFile->nextEntry(cookie)) != NULL) { - char fileName[256]; - // Make sure we have a file name. - // TODO: Handle filenames longer than 256. - if (zipFile->getEntryFileName(entry, fileName, sizeof(fileName))) { - continue; - } - - uint32_t uncompressedSize, compressedSize, crc32; - int64_t dataOffset; - zipFile->getEntryInfo(entry, nullptr, &uncompressedSize, &compressedSize, &dataOffset, - nullptr, &crc32); - APKEntry* apkEntry = apkMetaData.add_entries(); - apkEntry->set_crc32(crc32); - apkEntry->set_filename(fileName); - apkEntry->set_compressedsize(compressedSize); - apkEntry->set_uncompressedsize(uncompressedSize); - apkEntry->set_dataoffset(dataOffset); + + auto apkMetaData = GetDeviceAPKMetaData(dump); + + // Now let's set data sizes. + for (auto& apkEntry : *apkMetaData.mutable_entries()) { + auto dataSize = + archive.CalculateLocalFileEntrySize(apkEntry.dataoffset(), apkEntry.datasize()); + if (dataSize == 0) { + error_exit("Aborting"); } + apkEntry.set_datasize(dataSize); } + return apkMetaData; } @@ -69,19 +83,27 @@ void PatchUtils::WriteSignature(borrowed_fd output) { } void PatchUtils::WriteLong(int64_t value, borrowed_fd output) { - int64_t toLittleEndian = htole64(value); - WriteFdExactly(output, &toLittleEndian, sizeof(int64_t)); + int64_t littleEndian = htole64(value); + WriteFdExactly(output, &littleEndian, sizeof(littleEndian)); +} + +void PatchUtils::WriteString(const std::string& value, android::base::borrowed_fd output) { + WriteLong(value.size(), output); + WriteFdExactly(output, value); } void PatchUtils::Pipe(borrowed_fd input, borrowed_fd output, size_t amount) { - constexpr static int BUFFER_SIZE = 128 * 1024; + constexpr static size_t BUFFER_SIZE = 128 * 1024; char buffer[BUFFER_SIZE]; size_t transferAmount = 0; while (transferAmount != amount) { - long chunkAmount = - amount - transferAmount > BUFFER_SIZE ? BUFFER_SIZE : amount - transferAmount; - long readAmount = adb_read(input, buffer, chunkAmount); + auto chunkAmount = std::min(amount - transferAmount, BUFFER_SIZE); + auto readAmount = adb_read(input, buffer, chunkAmount); + if (readAmount < 0) { + fprintf(stderr, "adb: failed to read from input: %s\n", strerror(errno)); + error_exit("Aborting"); + } WriteFdExactly(output, buffer, readAmount); transferAmount += readAmount; } -}
\ No newline at end of file +} diff --git a/adb/fastdeploy/deploypatchgenerator/patch_utils.h b/adb/fastdeploy/deploypatchgenerator/patch_utils.h index 0ebfe8fe34..8dc9b9cb2a 100644 --- a/adb/fastdeploy/deploypatchgenerator/patch_utils.h +++ b/adb/fastdeploy/deploypatchgenerator/patch_utils.h @@ -25,11 +25,18 @@ class PatchUtils { public: /** + * This function takes the dump of Central Directly and builds the APKMetaData required by the + * patching algorithm. The if this function has an error a string is printed to the terminal and + * exit(1) is called. + */ + static com::android::fastdeploy::APKMetaData GetDeviceAPKMetaData( + const com::android::fastdeploy::APKDump& apk_dump); + /** * This function takes a local APK file and builds the APKMetaData required by the patching * algorithm. The if this function has an error a string is printed to the terminal and exit(1) * is called. */ - static com::android::fastdeploy::APKMetaData GetAPKMetaData(const char* file); + static com::android::fastdeploy::APKMetaData GetHostAPKMetaData(const char* file); /** * Writes a fixed signature string to the header of the patch. */ @@ -39,8 +46,12 @@ class PatchUtils { */ static void WriteLong(int64_t value, android::base::borrowed_fd output); /** + * Writes string to the |output|. + */ + static void WriteString(const std::string& value, android::base::borrowed_fd output); + /** * Copy |amount| of data from |input| to |output|. */ static void Pipe(android::base::borrowed_fd input, android::base::borrowed_fd output, size_t amount); -};
\ No newline at end of file +}; diff --git a/adb/fastdeploy/deploypatchgenerator/patch_utils_test.cpp b/adb/fastdeploy/deploypatchgenerator/patch_utils_test.cpp index a7eeebfc78..3ec5ab3e0c 100644 --- a/adb/fastdeploy/deploypatchgenerator/patch_utils_test.cpp +++ b/adb/fastdeploy/deploypatchgenerator/patch_utils_test.cpp @@ -23,10 +23,13 @@ #include <sstream> #include <string> +#include <google/protobuf/util/message_differencer.h> + #include "adb_io.h" #include "sysdeps.h" using namespace com::android::fastdeploy; +using google::protobuf::util::MessageDifferencer; static std::string GetTestFile(const std::string& name) { return "fastdeploy/testdata/" + name; @@ -86,11 +89,56 @@ TEST(PatchUtilsTest, SignatureConstMatches) { TEST(PatchUtilsTest, GatherMetadata) { std::string apkFile = GetTestFile("rotating_cube-release.apk"); - APKMetaData metadata = PatchUtils::GetAPKMetaData(apkFile.c_str()); + APKMetaData actual = PatchUtils::GetHostAPKMetaData(apkFile.c_str()); + std::string expectedMetadata; android::base::ReadFileToString(GetTestFile("rotating_cube-metadata-release.data"), &expectedMetadata); + APKMetaData expected; + EXPECT_TRUE(expected.ParseFromString(expectedMetadata)); + + // Test paths might vary. + expected.set_absolute_path(actual.absolute_path()); + + std::string actualMetadata; + actual.SerializeToString(&actualMetadata); + + expected.SerializeToString(&expectedMetadata); + + EXPECT_EQ(expectedMetadata, actualMetadata); +} + +static inline void sanitize(APKMetaData& metadata) { + metadata.clear_absolute_path(); + for (auto&& entry : *metadata.mutable_entries()) { + entry.clear_datasize(); + } +} + +TEST(PatchUtilsTest, GatherDumpMetadata) { + APKMetaData hostMetadata; + APKMetaData deviceMetadata; + + hostMetadata = PatchUtils::GetHostAPKMetaData(GetTestFile("sample.apk").c_str()); + + { + std::string cd; + android::base::ReadFileToString(GetTestFile("sample.cd"), &cd); + + APKDump dump; + dump.set_cd(std::move(cd)); + + deviceMetadata = PatchUtils::GetDeviceAPKMetaData(dump); + } + + sanitize(hostMetadata); + sanitize(deviceMetadata); + + std::string expectedMetadata; + hostMetadata.SerializeToString(&expectedMetadata); + std::string actualMetadata; - metadata.SerializeToString(&actualMetadata); + deviceMetadata.SerializeToString(&actualMetadata); + EXPECT_EQ(expectedMetadata, actualMetadata); -}
\ No newline at end of file +} diff --git a/adb/fastdeploy/proto/ApkEntry.proto b/adb/fastdeploy/proto/ApkEntry.proto index 9460d1510a..d84c5a54e0 100644 --- a/adb/fastdeploy/proto/ApkEntry.proto +++ b/adb/fastdeploy/proto/ApkEntry.proto @@ -1,18 +1,26 @@ -syntax = "proto2"; +syntax = "proto3"; package com.android.fastdeploy; option java_package = "com.android.fastdeploy"; +option java_outer_classname = "ApkEntryProto"; option java_multiple_files = true; +option optimize_for = LITE_RUNTIME; + +message APKDump { + string name = 1; + bytes cd = 2; + bytes signature = 3; + string absolute_path = 4; +} message APKEntry { - required int64 crc32 = 1; - required string fileName = 2; - required int64 dataOffset = 3; - required int64 compressedSize = 4; - required int64 uncompressedSize = 5; + bytes md5 = 1; + int64 dataOffset = 2; + int64 dataSize = 3; } message APKMetaData { - repeated APKEntry entries = 1; + string absolute_path = 1; + repeated APKEntry entries = 2; } diff --git a/adb/fastdeploy/testdata/helloworld5.apk b/adb/fastdeploy/testdata/helloworld5.apk Binary files differnew file mode 100644 index 0000000000..4a1539ecdc --- /dev/null +++ b/adb/fastdeploy/testdata/helloworld5.apk diff --git a/adb/fastdeploy/testdata/helloworld7.apk b/adb/fastdeploy/testdata/helloworld7.apk Binary files differnew file mode 100644 index 0000000000..82c46df80e --- /dev/null +++ b/adb/fastdeploy/testdata/helloworld7.apk diff --git a/adb/fastdeploy/testdata/rotating_cube-metadata-release.data b/adb/fastdeploy/testdata/rotating_cube-metadata-release.data index 0671bf3521..52352ff712 100644 --- a/adb/fastdeploy/testdata/rotating_cube-metadata-release.data +++ b/adb/fastdeploy/testdata/rotating_cube-metadata-release.data @@ -1,6 +1,8 @@ -#Ç‚Ï« META-INF/MANIFEST.MFÇ Q(W -#ƒ•ŽAndroidManifest.xml1 ä(è -6¦µ€>#lib/armeabi-v7a/libvulkan_sample.so ÀÒQ(Œ²ì - —ã—‘resources.arscôàQ ´(´ - ‹œÂÉ classes.dexÁ ÿ(ô +-fastdeploy/testdata/rotating_cube-release.apk +ìv4@obO#&kýn• +K›• 3Qcp^<Ð̽sF0•ƒ +xB ™a©–2áÒ_O'˜¨ +¶ÅhsêÃDÍY +ª^"cvÀ
ÓQ +qžp¶Îó{2 ÐÄÒÙ«ÂÁàQç
\ No newline at end of file diff --git a/adb/fastdeploy/testdata/sample.apk b/adb/fastdeploy/testdata/sample.apk Binary files differnew file mode 100644 index 0000000000..c31620572b --- /dev/null +++ b/adb/fastdeploy/testdata/sample.apk diff --git a/adb/fastdeploy/testdata/sample.cd b/adb/fastdeploy/testdata/sample.cd Binary files differnew file mode 100644 index 0000000000..5e5b4d4c83 --- /dev/null +++ b/adb/fastdeploy/testdata/sample.cd diff --git a/adb/sysdeps.h b/adb/sysdeps.h index 987f9942bd..466c2cedbc 100644 --- a/adb/sysdeps.h +++ b/adb/sysdeps.h @@ -601,6 +601,10 @@ static __inline__ int adb_is_absolute_host_path(const char* path) { return path[0] == '/'; } +static __inline__ int adb_get_os_handle(borrowed_fd fd) { + return fd.get(); +} + #endif /* !_WIN32 */ static inline void disable_tcp_nagle(borrowed_fd fd) { diff --git a/base/file.cpp b/base/file.cpp index 3dfcfbb7ec..6321fc6246 100644 --- a/base/file.cpp +++ b/base/file.cpp @@ -49,29 +49,54 @@ #include "android-base/unique_fd.h" #include "android-base/utf8.h" +namespace { + #ifdef _WIN32 -int mkstemp(char* template_name) { - if (_mktemp(template_name) == nullptr) { +static int mkstemp(char* name_template, size_t size_in_chars) { + std::wstring path; + CHECK(android::base::UTF8ToWide(name_template, &path)) + << "path can't be converted to wchar: " << name_template; + if (_wmktemp_s(path.data(), path.size() + 1) != 0) { return -1; } + // Use open() to match the close() that TemporaryFile's destructor does. // Use O_BINARY to match base file APIs. - return open(template_name, O_CREAT | O_EXCL | O_RDWR | O_BINARY, S_IRUSR | S_IWUSR); + int fd = _wopen(path.c_str(), O_CREAT | O_EXCL | O_RDWR | O_BINARY, S_IRUSR | S_IWUSR); + if (fd < 0) { + return -1; + } + + std::string path_utf8; + CHECK(android::base::WideToUTF8(path, &path_utf8)) << "path can't be converted to utf8"; + CHECK(strcpy_s(name_template, size_in_chars, path_utf8.c_str()) == 0) + << "utf8 path can't be assigned back to name_template"; + + return fd; } -char* mkdtemp(char* template_name) { - if (_mktemp(template_name) == nullptr) { +static char* mkdtemp(char* name_template, size_t size_in_chars) { + std::wstring path; + CHECK(android::base::UTF8ToWide(name_template, &path)) + << "path can't be converted to wchar: " << name_template; + + if (_wmktemp_s(path.data(), path.size() + 1) != 0) { return nullptr; } - if (_mkdir(template_name) == -1) { + + if (_wmkdir(path.c_str()) != 0) { return nullptr; } - return template_name; + + std::string path_utf8; + CHECK(android::base::WideToUTF8(path, &path_utf8)) << "path can't be converted to utf8"; + CHECK(strcpy_s(name_template, size_in_chars, path_utf8.c_str()) == 0) + << "utf8 path can't be assigned back to name_template"; + + return name_template; } #endif -namespace { - std::string GetSystemTempDir() { #if defined(__ANDROID__) const auto* tmpdir = getenv("TMPDIR"); @@ -83,15 +108,20 @@ std::string GetSystemTempDir() { // so try current directory if /data/local/tmp is not accessible. return "."; #elif defined(_WIN32) - char tmp_dir[MAX_PATH]; - DWORD result = GetTempPathA(sizeof(tmp_dir), tmp_dir); // checks TMP env - CHECK_NE(result, 0ul) << "GetTempPathA failed, error: " << GetLastError(); - CHECK_LT(result, sizeof(tmp_dir)) << "path truncated to: " << result; + wchar_t tmp_dir_w[MAX_PATH]; + DWORD result = GetTempPathW(std::size(tmp_dir_w), tmp_dir_w); // checks TMP env + CHECK_NE(result, 0ul) << "GetTempPathW failed, error: " << GetLastError(); + CHECK_LT(result, std::size(tmp_dir_w)) << "path truncated to: " << result; // GetTempPath() returns a path with a trailing slash, but init() // does not expect that, so remove it. - CHECK_EQ(tmp_dir[result - 1], '\\'); - tmp_dir[result - 1] = '\0'; + if (tmp_dir_w[result - 1] == L'\\') { + tmp_dir_w[result - 1] = L'\0'; + } + + std::string tmp_dir; + CHECK(android::base::WideToUTF8(tmp_dir_w, &tmp_dir)) << "path can't be converted to utf8"; + return tmp_dir; #else const auto* tmpdir = getenv("TMPDIR"); @@ -127,7 +157,11 @@ int TemporaryFile::release() { void TemporaryFile::init(const std::string& tmp_dir) { snprintf(path, sizeof(path), "%s%cTemporaryFile-XXXXXX", tmp_dir.c_str(), OS_PATH_SEPARATOR); +#if defined(_WIN32) + fd = mkstemp(path, sizeof(path)); +#else fd = mkstemp(path); +#endif } TemporaryDir::TemporaryDir() { @@ -167,7 +201,11 @@ TemporaryDir::~TemporaryDir() { bool TemporaryDir::init(const std::string& tmp_dir) { snprintf(path, sizeof(path), "%s%cTemporaryDir-XXXXXX", tmp_dir.c_str(), OS_PATH_SEPARATOR); +#if defined(_WIN32) + return (mkdtemp(path, sizeof(path)) != nullptr); +#else return (mkdtemp(path) != nullptr); +#endif } namespace android { diff --git a/base/file_test.cpp b/base/file_test.cpp index f64e81c0db..120228d947 100644 --- a/base/file_test.cpp +++ b/base/file_test.cpp @@ -16,18 +16,25 @@ #include "android-base/file.h" +#include "android-base/utf8.h" + #include <gtest/gtest.h> #include <errno.h> #include <fcntl.h> #include <unistd.h> +#include <wchar.h> #include <string> #if !defined(_WIN32) #include <pwd.h> +#else +#include <windows.h> #endif +#include "android-base/logging.h" // and must be after windows.h for ERROR + TEST(file, ReadFileToString_ENOENT) { std::string s("hello"); errno = 0; @@ -38,7 +45,7 @@ TEST(file, ReadFileToString_ENOENT) { TEST(file, ReadFileToString_WriteStringToFile) { TemporaryFile tf; - ASSERT_TRUE(tf.fd != -1); + ASSERT_NE(tf.fd, -1) << tf.path; ASSERT_TRUE(android::base::WriteStringToFile("abc", tf.path)) << strerror(errno); std::string s; @@ -70,7 +77,7 @@ TEST(file, ReadFileToString_WriteStringToFile_symlink) { #if !defined(_WIN32) TEST(file, WriteStringToFile2) { TemporaryFile tf; - ASSERT_TRUE(tf.fd != -1); + ASSERT_NE(tf.fd, -1) << tf.path; ASSERT_TRUE(android::base::WriteStringToFile("abc", tf.path, 0660, getuid(), getgid())) << strerror(errno); @@ -86,9 +93,92 @@ TEST(file, WriteStringToFile2) { } #endif +#if defined(_WIN32) +TEST(file, NonUnicodeCharsWindows) { + constexpr auto kMaxEnvVariableValueSize = 32767; + std::wstring old_tmp; + old_tmp.resize(kMaxEnvVariableValueSize); + old_tmp.resize(GetEnvironmentVariableW(L"TMP", old_tmp.data(), old_tmp.size())); + if (old_tmp.empty()) { + // Can't continue with empty TMP folder. + return; + } + + std::wstring new_tmp = old_tmp; + if (new_tmp.back() != L'\\') { + new_tmp.push_back(L'\\'); + } + + { + auto path(new_tmp + L"锦绣æˆéƒ½\\"); + _wmkdir(path.c_str()); + ASSERT_TRUE(SetEnvironmentVariableW(L"TMP", path.c_str())); + + TemporaryFile tf; + ASSERT_NE(tf.fd, -1) << tf.path; + ASSERT_TRUE(android::base::WriteStringToFd("abc", tf.fd)); + + ASSERT_EQ(0, lseek(tf.fd, 0, SEEK_SET)) << strerror(errno); + + std::string s; + ASSERT_TRUE(android::base::ReadFdToString(tf.fd, &s)) << strerror(errno); + EXPECT_EQ("abc", s); + } + { + auto path(new_tmp + L"Ð´Ð¸Ñ€ÐµÐºÑ‚Ð¾Ñ€Ð¸Ñ Ñ Ð´Ð»Ð¸Ð½Ð½Ñ‹Ð¼ именем\\"); + _wmkdir(path.c_str()); + ASSERT_TRUE(SetEnvironmentVariableW(L"TMP", path.c_str())); + + TemporaryFile tf; + ASSERT_NE(tf.fd, -1) << tf.path; + ASSERT_TRUE(android::base::WriteStringToFd("abc", tf.fd)); + + ASSERT_EQ(0, lseek(tf.fd, 0, SEEK_SET)) << strerror(errno); + + std::string s; + ASSERT_TRUE(android::base::ReadFdToString(tf.fd, &s)) << strerror(errno); + EXPECT_EQ("abc", s); + } + { + auto path(new_tmp + L"äüöß weiß\\"); + _wmkdir(path.c_str()); + ASSERT_TRUE(SetEnvironmentVariableW(L"TMP", path.c_str())); + + TemporaryFile tf; + ASSERT_NE(tf.fd, -1) << tf.path; + ASSERT_TRUE(android::base::WriteStringToFd("abc", tf.fd)); + + ASSERT_EQ(0, lseek(tf.fd, 0, SEEK_SET)) << strerror(errno); + + std::string s; + ASSERT_TRUE(android::base::ReadFdToString(tf.fd, &s)) << strerror(errno); + EXPECT_EQ("abc", s); + } + + SetEnvironmentVariableW(L"TMP", old_tmp.c_str()); +} + +TEST(file, RootDirectoryWindows) { + constexpr auto kMaxEnvVariableValueSize = 32767; + std::wstring old_tmp; + bool tmp_is_empty = false; + old_tmp.resize(kMaxEnvVariableValueSize); + old_tmp.resize(GetEnvironmentVariableW(L"TMP", old_tmp.data(), old_tmp.size())); + if (old_tmp.empty()) { + tmp_is_empty = (GetLastError() == ERROR_ENVVAR_NOT_FOUND); + } + SetEnvironmentVariableW(L"TMP", L"C:"); + + TemporaryFile tf; + ASSERT_NE(tf.fd, -1) << tf.path; + + SetEnvironmentVariableW(L"TMP", tmp_is_empty ? nullptr : old_tmp.c_str()); +} +#endif + TEST(file, WriteStringToFd) { TemporaryFile tf; - ASSERT_TRUE(tf.fd != -1); + ASSERT_NE(tf.fd, -1) << tf.path; ASSERT_TRUE(android::base::WriteStringToFd("abc", tf.fd)); ASSERT_EQ(0, lseek(tf.fd, 0, SEEK_SET)) << strerror(errno); @@ -100,7 +190,7 @@ TEST(file, WriteStringToFd) { TEST(file, WriteFully) { TemporaryFile tf; - ASSERT_TRUE(tf.fd != -1); + ASSERT_NE(tf.fd, -1) << tf.path; ASSERT_TRUE(android::base::WriteFully(tf.fd, "abc", 3)); ASSERT_EQ(0, lseek(tf.fd, 0, SEEK_SET)) << strerror(errno); @@ -119,7 +209,7 @@ TEST(file, WriteFully) { TEST(file, RemoveFileIfExists) { TemporaryFile tf; - ASSERT_TRUE(tf.fd != -1); + ASSERT_NE(tf.fd, -1) << tf.path; close(tf.fd); tf.fd = -1; std::string err; @@ -253,7 +343,7 @@ TEST(file, Dirname) { TEST(file, ReadFileToString_capacity) { TemporaryFile tf; - ASSERT_TRUE(tf.fd != -1); + ASSERT_NE(tf.fd, -1) << tf.path; // For a huge file, the overhead should still be small. std::string s; @@ -280,7 +370,7 @@ TEST(file, ReadFileToString_capacity) { TEST(file, ReadFileToString_capacity_0) { TemporaryFile tf; - ASSERT_TRUE(tf.fd != -1); + ASSERT_NE(tf.fd, -1) << tf.path; // Because /proc reports its files as zero-length, we don't actually trust // any file that claims to be zero-length. Rather than add increasingly diff --git a/fs_mgr/libsnapshot/snapshot.cpp b/fs_mgr/libsnapshot/snapshot.cpp index 354aafa904..02c7de6bf7 100644 --- a/fs_mgr/libsnapshot/snapshot.cpp +++ b/fs_mgr/libsnapshot/snapshot.cpp @@ -1307,8 +1307,8 @@ bool SnapshotManager::MapPartitionWithSnapshot(LockedFile* lock, // device itself. This device consists of the real blocks in the super // partition that this logical partition occupies. auto& dm = DeviceMapper::Instance(); - std::string ignore_path; - if (!CreateLogicalPartition(params, &ignore_path)) { + std::string base_path; + if (!CreateLogicalPartition(params, &base_path)) { LOG(ERROR) << "Could not create logical partition " << params.GetPartitionName() << " as device " << params.GetDeviceName(); return false; @@ -1316,6 +1316,7 @@ bool SnapshotManager::MapPartitionWithSnapshot(LockedFile* lock, created_devices.EmplaceBack<AutoUnmapDevice>(&dm, params.GetDeviceName()); if (!live_snapshot_status.has_value()) { + *path = base_path; created_devices.Release(); return true; } diff --git a/init/property_service.cpp b/init/property_service.cpp index f5d1143caa..d7e4021758 100644 --- a/init/property_service.cpp +++ b/init/property_service.cpp @@ -963,6 +963,10 @@ void CreateSerializedPropertyInfo() { // Don't check for failure here, so we always have a sane list of properties. // E.g. In case of recovery, the vendor partition will not have mounted and we // still need the system / platform properties to function. + if (access("/system_ext/etc/selinux/system_ext_property_contexts", R_OK) != -1) { + LoadPropertyInfoFromFile("/system_ext/etc/selinux/system_ext_property_contexts", + &property_infos); + } if (!LoadPropertyInfoFromFile("/vendor/etc/selinux/vendor_property_contexts", &property_infos)) { // Fallback to nonplat_* if vendor_* doesn't exist. @@ -980,6 +984,7 @@ void CreateSerializedPropertyInfo() { if (!LoadPropertyInfoFromFile("/plat_property_contexts", &property_infos)) { return; } + LoadPropertyInfoFromFile("/system_ext_property_contexts", &property_infos); if (!LoadPropertyInfoFromFile("/vendor_property_contexts", &property_infos)) { // Fallback to nonplat_* if vendor_* doesn't exist. LoadPropertyInfoFromFile("/nonplat_property_contexts", &property_infos); diff --git a/libkeyutils/Android.bp b/libkeyutils/Android.bp index dda491a08f..b388e95559 100644 --- a/libkeyutils/Android.bp +++ b/libkeyutils/Android.bp @@ -16,17 +16,3 @@ cc_test { srcs: ["keyutils_test.cpp"], test_suites: ["device-tests"], } - -cc_binary { - name: "mini-keyctl", - srcs: [ - "mini_keyctl.cpp", - "mini_keyctl_utils.cpp" - ], - shared_libs: [ - "libbase", - "libkeyutils", - "liblog", - ], - cflags: ["-Werror", "-Wall", "-Wextra", "-fexceptions"], -} diff --git a/libkeyutils/mini_keyctl.cpp b/libkeyutils/mini_keyctl.cpp deleted file mode 100644 index fe89e62abc..0000000000 --- a/libkeyutils/mini_keyctl.cpp +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * A tool loads keys to keyring. - */ - -#include "mini_keyctl_utils.h" - -#include <error.h> -#include <stdio.h> -#include <unistd.h> - -#include <android-base/parseint.h> - -static void Usage(int exit_code) { - fprintf(stderr, "usage: mini-keyctl <action> [args,]\n"); - fprintf(stderr, " mini-keyctl add <type> <desc> <data> <keyring>\n"); - fprintf(stderr, " mini-keyctl padd <type> <desc> <keyring>\n"); - fprintf(stderr, " mini-keyctl unlink <key> <keyring>\n"); - fprintf(stderr, " mini-keyctl restrict_keyring <keyring>\n"); - fprintf(stderr, " mini-keyctl security <key>\n"); - _exit(exit_code); -} - -static key_serial_t parseKeyOrDie(const char* str) { - key_serial_t key; - if (!android::base::ParseInt(str, &key)) { - error(1 /* exit code */, 0 /* errno */, "Unparsable key: '%s'\n", str); - } - return key; -} - -int main(int argc, const char** argv) { - if (argc < 2) Usage(1); - const std::string action = argv[1]; - - if (action == "add") { - if (argc != 6) Usage(1); - std::string type = argv[2]; - std::string desc = argv[3]; - std::string data = argv[4]; - std::string keyring = argv[5]; - return Add(type, desc, data, keyring); - } else if (action == "padd") { - if (argc != 5) Usage(1); - std::string type = argv[2]; - std::string desc = argv[3]; - std::string keyring = argv[4]; - return Padd(type, desc, keyring); - } else if (action == "restrict_keyring") { - if (argc != 3) Usage(1); - std::string keyring = argv[2]; - return RestrictKeyring(keyring); - } else if (action == "unlink") { - if (argc != 4) Usage(1); - key_serial_t key = parseKeyOrDie(argv[2]); - const std::string keyring = argv[3]; - return Unlink(key, keyring); - } else if (action == "security") { - if (argc != 3) Usage(1); - const char* key_str = argv[2]; - key_serial_t key = parseKeyOrDie(key_str); - std::string context = RetrieveSecurityContext(key); - if (context.empty()) { - perror(key_str); - return 1; - } - fprintf(stderr, "%s\n", context.c_str()); - return 0; - } else { - fprintf(stderr, "Unrecognized action: %s\n", action.c_str()); - Usage(1); - } - - return 0; -} diff --git a/libkeyutils/mini_keyctl/Android.bp b/libkeyutils/mini_keyctl/Android.bp new file mode 100644 index 0000000000..a04a3db240 --- /dev/null +++ b/libkeyutils/mini_keyctl/Android.bp @@ -0,0 +1,27 @@ +cc_library_static { + name: "libmini_keyctl_static", + srcs: [ + "mini_keyctl_utils.cpp" + ], + shared_libs: [ + "libbase", + "libkeyutils", + ], + cflags: ["-Werror", "-Wall", "-Wextra"], + export_include_dirs: ["."], +} + +cc_binary { + name: "mini-keyctl", + srcs: [ + "mini_keyctl.cpp", + ], + static_libs: [ + "libmini_keyctl_static", + ], + shared_libs: [ + "libbase", + "libkeyutils", + ], + cflags: ["-Werror", "-Wall", "-Wextra"], +} diff --git a/libkeyutils/mini_keyctl_utils.cpp b/libkeyutils/mini_keyctl/mini_keyctl.cpp index 79b468087b..8aace9adbb 100644 --- a/libkeyutils/mini_keyctl_utils.cpp +++ b/libkeyutils/mini_keyctl/mini_keyctl.cpp @@ -14,79 +14,48 @@ * limitations under the License. */ -#include <mini_keyctl_utils.h> +/* + * A tool loads keys to keyring. + */ #include <dirent.h> #include <errno.h> #include <error.h> +#include <stdio.h> #include <sys/types.h> #include <unistd.h> -#include <fstream> #include <iostream> #include <iterator> -#include <sstream> #include <string> -#include <vector> #include <android-base/file.h> #include <android-base/parseint.h> -#include <android-base/properties.h> -#include <android-base/strings.h> #include <keyutils.h> +#include <mini_keyctl_utils.h> -static constexpr int kMaxCertSize = 4096; +constexpr int kMaxCertSize = 4096; -static std::vector<std::string> SplitBySpace(const std::string& s) { - std::istringstream iss(s); - return std::vector<std::string>{std::istream_iterator<std::string>{iss}, - std::istream_iterator<std::string>{}}; +static void Usage(int exit_code) { + fprintf(stderr, "usage: mini-keyctl <action> [args,]\n"); + fprintf(stderr, " mini-keyctl add <type> <desc> <data> <keyring>\n"); + fprintf(stderr, " mini-keyctl padd <type> <desc> <keyring>\n"); + fprintf(stderr, " mini-keyctl unlink <key> <keyring>\n"); + fprintf(stderr, " mini-keyctl restrict_keyring <keyring>\n"); + fprintf(stderr, " mini-keyctl security <key>\n"); + _exit(exit_code); } -// Find the keyring id. Because request_key(2) syscall is not available or the key is -// kernel keyring, the id is looked up from /proc/keys. The keyring description may contain other -// information in the descritption section depending on the key type, only the first word in the -// keyring description is used for searching. -static key_serial_t GetKeyringIdOrDie(const std::string& keyring_desc) { - // If the keyring id is already a hex number, directly convert it to keyring id - key_serial_t keyring_id; - if (android::base::ParseInt(keyring_desc.c_str(), &keyring_id)) { - return keyring_id; +static key_serial_t parseKeyOrDie(const char* str) { + key_serial_t key; + if (!android::base::ParseInt(str, &key)) { + error(1 /* exit code */, 0 /* errno */, "Unparsable key: '%s'\n", str); } - - // Only keys allowed by SELinux rules will be shown here. - std::ifstream proc_keys_file("/proc/keys"); - if (!proc_keys_file.is_open()) { - error(1, errno, "Failed to open /proc/keys"); - return -1; - } - - std::string line; - while (getline(proc_keys_file, line)) { - std::vector<std::string> tokens = SplitBySpace(line); - if (tokens.size() < 9) { - continue; - } - std::string key_id = "0x" + tokens[0]; - std::string key_type = tokens[7]; - // The key description may contain space. - std::string key_desc_prefix = tokens[8]; - // The prefix has a ":" at the end - std::string key_desc_pattern = keyring_desc + ":"; - if (key_type != "keyring" || key_desc_prefix != key_desc_pattern) { - continue; - } - if (!android::base::ParseInt(key_id.c_str(), &keyring_id)) { - error(1, 0, "Unexpected key format in /proc/keys: %s", key_id.c_str()); - return -1; - } - return keyring_id; - } - return -1; + return key; } int Unlink(key_serial_t key, const std::string& keyring) { - key_serial_t keyring_id = GetKeyringIdOrDie(keyring); + key_serial_t keyring_id = android::GetKeyringId(keyring); if (keyctl_unlink(key, keyring_id) < 0) { error(1, errno, "Failed to unlink key %x from keyring %s", key, keyring.c_str()); return 1; @@ -101,7 +70,7 @@ int Add(const std::string& type, const std::string& desc, const std::string& dat return 1; } - key_serial_t keyring_id = GetKeyringIdOrDie(keyring); + key_serial_t keyring_id = android::GetKeyringId(keyring); key_serial_t key = add_key(type.c_str(), desc.c_str(), data.c_str(), data.size(), keyring_id); if (key < 0) { @@ -114,7 +83,7 @@ int Add(const std::string& type, const std::string& desc, const std::string& dat } int Padd(const std::string& type, const std::string& desc, const std::string& keyring) { - key_serial_t keyring_id = GetKeyringIdOrDie(keyring); + key_serial_t keyring_id = android::GetKeyringId(keyring); // read from stdin to get the certificates std::istreambuf_iterator<char> begin(std::cin), end; @@ -137,7 +106,7 @@ int Padd(const std::string& type, const std::string& desc, const std::string& ke } int RestrictKeyring(const std::string& keyring) { - key_serial_t keyring_id = GetKeyringIdOrDie(keyring); + key_serial_t keyring_id = android::GetKeyringId(keyring); if (keyctl_restrict_keyring(keyring_id, nullptr, nullptr) < 0) { error(1, errno, "Cannot restrict keyring '%s'", keyring.c_str()); return 1; @@ -162,3 +131,48 @@ std::string RetrieveSecurityContext(key_serial_t key) { context.resize(retval); return context; } + +int main(int argc, const char** argv) { + if (argc < 2) Usage(1); + const std::string action = argv[1]; + + if (action == "add") { + if (argc != 6) Usage(1); + std::string type = argv[2]; + std::string desc = argv[3]; + std::string data = argv[4]; + std::string keyring = argv[5]; + return Add(type, desc, data, keyring); + } else if (action == "padd") { + if (argc != 5) Usage(1); + std::string type = argv[2]; + std::string desc = argv[3]; + std::string keyring = argv[4]; + return Padd(type, desc, keyring); + } else if (action == "restrict_keyring") { + if (argc != 3) Usage(1); + std::string keyring = argv[2]; + return RestrictKeyring(keyring); + } else if (action == "unlink") { + if (argc != 4) Usage(1); + key_serial_t key = parseKeyOrDie(argv[2]); + const std::string keyring = argv[3]; + return Unlink(key, keyring); + } else if (action == "security") { + if (argc != 3) Usage(1); + const char* key_str = argv[2]; + key_serial_t key = parseKeyOrDie(key_str); + std::string context = RetrieveSecurityContext(key); + if (context.empty()) { + perror(key_str); + return 1; + } + fprintf(stderr, "%s\n", context.c_str()); + return 0; + } else { + fprintf(stderr, "Unrecognized action: %s\n", action.c_str()); + Usage(1); + } + + return 0; +} diff --git a/libkeyutils/mini_keyctl/mini_keyctl_utils.cpp b/libkeyutils/mini_keyctl/mini_keyctl_utils.cpp new file mode 100644 index 0000000000..fb9503f14b --- /dev/null +++ b/libkeyutils/mini_keyctl/mini_keyctl_utils.cpp @@ -0,0 +1,83 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <mini_keyctl_utils.h> + +#include <fstream> +#include <iterator> +#include <sstream> +#include <string> +#include <vector> + +#include <android-base/logging.h> +#include <android-base/parseint.h> + +namespace android { + +namespace { + +std::vector<std::string> SplitBySpace(const std::string& s) { + std::istringstream iss(s); + return std::vector<std::string>{std::istream_iterator<std::string>{iss}, + std::istream_iterator<std::string>{}}; +} + +} // namespace + +// Find the keyring id. request_key(2) only finds keys in the process, session or thread keyring +// hierarchy, but not internal keyring of a kernel subsystem (e.g. .fs-verity). To support all +// cases, this function looks up a keyring's ID by parsing /proc/keys. The keyring description may +// contain other information in the descritption section depending on the key type, only the first +// word in the keyring description is used for searching. +key_serial_t GetKeyringId(const std::string& keyring_desc) { + // If the keyring id is already a hex number, directly convert it to keyring id + key_serial_t keyring_id; + if (android::base::ParseInt(keyring_desc.c_str(), &keyring_id)) { + return keyring_id; + } + + // Only keys allowed by SELinux rules will be shown here. + std::ifstream proc_keys_file("/proc/keys"); + if (!proc_keys_file.is_open()) { + PLOG(ERROR) << "Failed to open /proc/keys"; + return -1; + } + + std::string line; + while (getline(proc_keys_file, line)) { + std::vector<std::string> tokens = SplitBySpace(line); + if (tokens.size() < 9) { + continue; + } + std::string key_id = "0x" + tokens[0]; + std::string key_type = tokens[7]; + // The key description may contain space. + std::string key_desc_prefix = tokens[8]; + // The prefix has a ":" at the end + std::string key_desc_pattern = keyring_desc + ":"; + if (key_type != "keyring" || key_desc_prefix != key_desc_pattern) { + continue; + } + if (!android::base::ParseInt(key_id.c_str(), &keyring_id)) { + LOG(ERROR) << "Unexpected key format in /proc/keys: " << key_id; + return -1; + } + return keyring_id; + } + return -1; +} + +} // namespace android diff --git a/libkeyutils/mini_keyctl/mini_keyctl_utils.h b/libkeyutils/mini_keyctl/mini_keyctl_utils.h new file mode 100644 index 0000000000..cc31d29ce1 --- /dev/null +++ b/libkeyutils/mini_keyctl/mini_keyctl_utils.h @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _MINI_KEYCTL_MINI_KEYCTL_UTILS_H_ +#define _MINI_KEYCTL_MINI_KEYCTL_UTILS_H_ + +#include <string> + +#include <keyutils.h> + +namespace android { +key_serial_t GetKeyringId(const std::string& keyring_desc); +} // namespace android + +#endif // _MINI_KEYCTL_MINI_KEYCTL_UTILS_H_ diff --git a/libkeyutils/mini_keyctl_utils.h b/libkeyutils/mini_keyctl_utils.h deleted file mode 100644 index 3616831afe..0000000000 --- a/libkeyutils/mini_keyctl_utils.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "include/keyutils.h" - -#include <string> - -// Add key to a keyring. Returns non-zero if error happens. -int Add(const std::string& type, const std::string& desc, const std::string& data, - const std::string& keyring); - -// Add key from stdin to a keyring. Returns non-zero if error happens. -int Padd(const std::string& type, const std::string& desc, const std::string& keyring); - -// Removes the link from a keyring to a key if exists. Return non-zero if error happens. -int Unlink(key_serial_t key, const std::string& keyring); - -// Apply key-linking to a keyring. Return non-zero if error happens. -int RestrictKeyring(const std::string& keyring); - -// Retrieves a key's security context. Return the context string, or empty string on error. -std::string RetrieveSecurityContext(key_serial_t key); diff --git a/liblog/Android.bp b/liblog/Android.bp index 8a63007dc7..c40c5ef8b9 100644 --- a/liblog/Android.bp +++ b/liblog/Android.bp @@ -15,7 +15,6 @@ // liblog_sources = [ - "config_write.cpp", "log_event_list.cpp", "log_event_write.cpp", "logger_lock.cpp", diff --git a/liblog/README.md b/liblog/README.md index 98bee9f7dd..871399a57d 100644 --- a/liblog/README.md +++ b/liblog/README.md @@ -96,11 +96,6 @@ Public Functions and Macros int android_log_destroy(android_log_context *ctx) - #include <log/log_transport.h> - - int android_set_log_transport(int transport_flag) - int android_get_log_transport() - Description ----------- @@ -144,11 +139,6 @@ size and log buffer format protocol version respectively. `android_logger_get_i that was used when opening the sub-log. It is recommended to open the log `ANDROID_LOG_RDONLY` in these cases. -`android_set_log_transport()` selects transport filters. Argument is either `LOGGER_DEFAULT`, -`LOGGER_LOGD`, or `LOGGER_NULL`. Log to logger daemon for default or logd, or drop contents on floor -respectively. `Both android_set_log_transport()` and `android_get_log_transport()` return the -current transport mask, or a negative errno for any problems. - Errors ------ diff --git a/liblog/config_write.cpp b/liblog/config_write.cpp deleted file mode 100644 index 6ed893dbd6..0000000000 --- a/liblog/config_write.cpp +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright (C) 2016 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <log/log_transport.h> - -#include "config_write.h" -#include "logger.h" - -struct listnode __android_log_transport_write = {&__android_log_transport_write, - &__android_log_transport_write}; -struct listnode __android_log_persist_write = {&__android_log_persist_write, - &__android_log_persist_write}; - -static void __android_log_add_transport(struct listnode* list, - struct android_log_transport_write* transport) { - uint32_t i; - - /* Try to keep one functioning transport for each log buffer id */ - for (i = LOG_ID_MIN; i < LOG_ID_MAX; i++) { - struct android_log_transport_write* transp; - - if (list_empty(list)) { - if (!transport->available || ((*transport->available)(static_cast<log_id_t>(i)) >= 0)) { - list_add_tail(list, &transport->node); - return; - } - } else { - write_transport_for_each(transp, list) { - if (!transp->available) { - return; - } - if (((*transp->available)(static_cast<log_id_t>(i)) < 0) && - (!transport->available || ((*transport->available)(static_cast<log_id_t>(i)) >= 0))) { - list_add_tail(list, &transport->node); - return; - } - } - } - } -} - -void __android_log_config_write() { - if ((__android_log_transport == LOGGER_DEFAULT) || (__android_log_transport & LOGGER_LOGD)) { -#if (FAKE_LOG_DEVICE == 0) - extern struct android_log_transport_write logdLoggerWrite; - extern struct android_log_transport_write pmsgLoggerWrite; - - __android_log_add_transport(&__android_log_transport_write, &logdLoggerWrite); - __android_log_add_transport(&__android_log_persist_write, &pmsgLoggerWrite); -#else - extern struct android_log_transport_write fakeLoggerWrite; - - __android_log_add_transport(&__android_log_transport_write, &fakeLoggerWrite); -#endif - } -} - -void __android_log_config_write_close() { - struct android_log_transport_write* transport; - struct listnode* n; - - write_transport_for_each_safe(transport, n, &__android_log_transport_write) { - transport->logMask = 0; - list_remove(&transport->node); - } - write_transport_for_each_safe(transport, n, &__android_log_persist_write) { - transport->logMask = 0; - list_remove(&transport->node); - } -} diff --git a/liblog/config_write.h b/liblog/config_write.h deleted file mode 100644 index a901f1352c..0000000000 --- a/liblog/config_write.h +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (C) 2016 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include <cutils/list.h> - -#include "log_portability.h" - -__BEGIN_DECLS - -extern struct listnode __android_log_transport_write; -extern struct listnode __android_log_persist_write; - -#define write_transport_for_each(transp, transports) \ - for ((transp) = node_to_item((transports)->next, \ - struct android_log_transport_write, node); \ - ((transp) != node_to_item((transports), \ - struct android_log_transport_write, node)) && \ - ((transp) != node_to_item((transp)->node.next, \ - struct android_log_transport_write, node)); \ - (transp) = node_to_item((transp)->node.next, \ - struct android_log_transport_write, node)) - -#define write_transport_for_each_safe(transp, n, transports) \ - for ((transp) = node_to_item((transports)->next, \ - struct android_log_transport_write, node), \ - (n) = (transp)->node.next; \ - ((transp) != node_to_item((transports), \ - struct android_log_transport_write, node)) && \ - ((transp) != \ - node_to_item((n), struct android_log_transport_write, node)); \ - (transp) = node_to_item((n), struct android_log_transport_write, node), \ - (n) = (transp)->node.next) - -void __android_log_config_write(); -void __android_log_config_write_close(); - -__END_DECLS diff --git a/liblog/fake_writer.cpp b/liblog/fake_writer.cpp index c0b0e69a50..f1ddff17b0 100644 --- a/liblog/fake_writer.cpp +++ b/liblog/fake_writer.cpp @@ -20,11 +20,11 @@ #include <log/log.h> -#include "config_write.h" #include "fake_log_device.h" #include "log_portability.h" #include "logger.h" +static int fakeAvailable(log_id_t); static int fakeOpen(); static void fakeClose(); static int fakeWrite(log_id_t log_id, struct timespec* ts, struct iovec* vec, size_t nr); @@ -32,15 +32,19 @@ static int fakeWrite(log_id_t log_id, struct timespec* ts, struct iovec* vec, si static int logFds[(int)LOG_ID_MAX] = {-1, -1, -1, -1, -1, -1}; struct android_log_transport_write fakeLoggerWrite = { - .node = {&fakeLoggerWrite.node, &fakeLoggerWrite.node}, - .context.priv = &logFds, .name = "fake", - .available = NULL, + .logMask = 0, + .context.priv = &logFds, + .available = fakeAvailable, .open = fakeOpen, .close = fakeClose, .write = fakeWrite, }; +static int fakeAvailable(log_id_t) { + return 0; +} + static int fakeOpen() { int i; diff --git a/liblog/include/log/log_transport.h b/liblog/include/log/log_transport.h deleted file mode 100644 index bda7c2579b..0000000000 --- a/liblog/include/log/log_transport.h +++ /dev/null @@ -1,34 +0,0 @@ -/* -** -** Copyright 2017, The Android Open Source Project -** -** This file is dual licensed. It may be redistributed and/or modified -** under the terms of the Apache 2.0 License OR version 2 of the GNU -** General Public License. -*/ - -#pragma once - -#ifdef __cplusplus -extern "C" { -#endif - -/* - * Logging transports, bit mask to select features. Function returns selection. - */ -/* clang-format off */ -#define LOGGER_DEFAULT 0x00 -#define LOGGER_LOGD 0x01 -#define LOGGER_KERNEL 0x02 /* Reserved/Deprecated */ -#define LOGGER_NULL 0x04 /* Does not release resources of other selections */ -#define LOGGER_RESERVED 0x08 /* Reserved, previously for logging to local memory */ -#define LOGGER_RESERVED2 0x10 /* Reserved, previously for logs sent to stderr */ -/* clang-format on */ - -/* Both return the selected transport flag mask, or negative errno */ -int android_set_log_transport(int transport_flag); -int android_get_log_transport(); - -#ifdef __cplusplus -} -#endif diff --git a/liblog/log_event_list.cpp b/liblog/log_event_list.cpp index f148ef345f..18ea930569 100644 --- a/liblog/log_event_list.cpp +++ b/liblog/log_event_list.cpp @@ -176,13 +176,6 @@ int android_log_write_list_begin(android_log_context ctx) { return 0; } -static inline void copy4LE(uint8_t* buf, uint32_t val) { - buf[0] = val & 0xFF; - buf[1] = (val >> 8) & 0xFF; - buf[2] = (val >> 16) & 0xFF; - buf[3] = (val >> 24) & 0xFF; -} - int android_log_write_int32(android_log_context ctx, int32_t value) { size_t needed; android_log_context_internal* context; @@ -201,22 +194,11 @@ int android_log_write_int32(android_log_context ctx, int32_t value) { } context->count[context->list_nest_depth]++; context->storage[context->pos + 0] = EVENT_TYPE_INT; - copy4LE(&context->storage[context->pos + 1], value); + *reinterpret_cast<int32_t*>(&context->storage[context->pos + 1]) = value; context->pos += needed; return 0; } -static inline void copy8LE(uint8_t* buf, uint64_t val) { - buf[0] = val & 0xFF; - buf[1] = (val >> 8) & 0xFF; - buf[2] = (val >> 16) & 0xFF; - buf[3] = (val >> 24) & 0xFF; - buf[4] = (val >> 32) & 0xFF; - buf[5] = (val >> 40) & 0xFF; - buf[6] = (val >> 48) & 0xFF; - buf[7] = (val >> 56) & 0xFF; -} - int android_log_write_int64(android_log_context ctx, int64_t value) { size_t needed; android_log_context_internal* context; @@ -235,7 +217,7 @@ int android_log_write_int64(android_log_context ctx, int64_t value) { } context->count[context->list_nest_depth]++; context->storage[context->pos + 0] = EVENT_TYPE_LONG; - copy8LE(&context->storage[context->pos + 1], value); + *reinterpret_cast<int64_t*>(&context->storage[context->pos + 1]) = value; context->pos += needed; return 0; } @@ -267,7 +249,7 @@ int android_log_write_string8_len(android_log_context ctx, const char* value, si } context->count[context->list_nest_depth]++; context->storage[context->pos + 0] = EVENT_TYPE_STRING; - copy4LE(&context->storage[context->pos + 1], len); + *reinterpret_cast<ssize_t*>(&context->storage[context->pos + 1]) = len; if (len) { memcpy(&context->storage[context->pos + 5], value, len); } @@ -299,7 +281,7 @@ int android_log_write_float32(android_log_context ctx, float value) { ivalue = *(uint32_t*)&value; context->count[context->list_nest_depth]++; context->storage[context->pos + 0] = EVENT_TYPE_FLOAT; - copy4LE(&context->storage[context->pos + 1], ivalue); + *reinterpret_cast<uint32_t*>(&context->storage[context->pos + 1]) = ivalue; context->pos += needed; return 0; } diff --git a/liblog/logd_reader.cpp b/liblog/logd_reader.cpp index d5bf84405a..916a428195 100644 --- a/liblog/logd_reader.cpp +++ b/liblog/logd_reader.cpp @@ -23,6 +23,7 @@ #include <stdio.h> #include <stdlib.h> #include <string.h> +#include <sys/param.h> #include <sys/socket.h> #include <sys/stat.h> #include <sys/types.h> @@ -38,9 +39,6 @@ #include "logd_reader.h" #include "logger.h" -/* branchless on many architectures. */ -#define min(x, y) ((y) ^ (((x) ^ (y)) & -((x) < (y)))) - static int logdAvailable(log_id_t LogId); static int logdVersion(struct android_log_logger* logger, struct android_log_transport_context* transp); @@ -66,7 +64,6 @@ static ssize_t logdGetStats(struct android_log_logger_list* logger, struct android_log_transport_context* transp, char* buf, size_t len); struct android_log_transport_read logdLoggerRead = { - .node = {&logdLoggerRead.node, &logdLoggerRead.node}, .name = "logd", .available = logdAvailable, .version = logdVersion, @@ -279,13 +276,13 @@ static ssize_t logdGetStats(struct android_log_logger_list* logger_list, size_t n; n = snprintf(cp, remaining, "getStatistics"); - n = min(n, remaining); + n = MIN(n, remaining); remaining -= n; cp += n; logger_for_each(logger, logger_list) { n = snprintf(cp, remaining, " %d", logger->logId); - n = min(n, remaining); + n = MIN(n, remaining); remaining -= n; cp += n; } @@ -362,7 +359,7 @@ static int logdOpen(struct android_log_logger_list* logger_list, remaining = sizeof(buffer) - (cp - buffer); logger_for_each(logger, logger_list) { ret = snprintf(cp, remaining, "%c%u", c, logger->logId); - ret = min(ret, remaining); + ret = MIN(ret, remaining); remaining -= ret; cp += ret; c = ','; @@ -370,7 +367,7 @@ static int logdOpen(struct android_log_logger_list* logger_list, if (logger_list->tail) { ret = snprintf(cp, remaining, " tail=%u", logger_list->tail); - ret = min(ret, remaining); + ret = MIN(ret, remaining); remaining -= ret; cp += ret; } @@ -379,20 +376,20 @@ static int logdOpen(struct android_log_logger_list* logger_list, if (logger_list->mode & ANDROID_LOG_WRAP) { // ToDo: alternate API to allow timeout to be adjusted. ret = snprintf(cp, remaining, " timeout=%u", ANDROID_LOG_WRAP_DEFAULT_TIMEOUT); - ret = min(ret, remaining); + ret = MIN(ret, remaining); remaining -= ret; cp += ret; } ret = snprintf(cp, remaining, " start=%" PRIu32 ".%09" PRIu32, logger_list->start.tv_sec, logger_list->start.tv_nsec); - ret = min(ret, remaining); + ret = MIN(ret, remaining); remaining -= ret; cp += ret; } if (logger_list->pid) { ret = snprintf(cp, remaining, " pid=%u", logger_list->pid); - ret = min(ret, remaining); + ret = MIN(ret, remaining); cp += ret; } diff --git a/liblog/logd_writer.cpp b/liblog/logd_writer.cpp index 2c64b0b899..06a2bafc08 100644 --- a/liblog/logd_writer.cpp +++ b/liblog/logd_writer.cpp @@ -34,23 +34,19 @@ #include <private/android_filesystem_config.h> #include <private/android_logger.h> -#include "config_write.h" #include "log_portability.h" #include "logger.h" #include "uio.h" -/* branchless on many architectures. */ -#define min(x, y) ((y) ^ (((x) ^ (y)) & -((x) < (y)))) - static int logdAvailable(log_id_t LogId); static int logdOpen(); static void logdClose(); static int logdWrite(log_id_t logId, struct timespec* ts, struct iovec* vec, size_t nr); struct android_log_transport_write logdLoggerWrite = { - .node = {&logdLoggerWrite.node, &logdLoggerWrite.node}, - .context.sock = -EBADF, .name = "logd", + .logMask = 0, + .context.sock = -EBADF, .available = logdAvailable, .open = logdOpen, .close = logdClose, diff --git a/liblog/logger.h b/liblog/logger.h index 4b4ef5fc35..8cae66c88b 100644 --- a/liblog/logger.h +++ b/liblog/logger.h @@ -31,12 +31,9 @@ union android_log_context_union { void* priv; atomic_int sock; atomic_int fd; - struct listnode* node; - atomic_uintptr_t atomic_pointer; }; struct android_log_transport_write { - struct listnode node; const char* name; /* human name to describe the transport */ unsigned logMask; /* mask cache of available() success */ union android_log_context_union context; /* Initialized by static allocation */ @@ -54,7 +51,6 @@ struct android_log_transport_context; struct android_log_logger; struct android_log_transport_read { - struct listnode node; const char* name; /* human name to describe the transport */ /* Does not cause resources to be taken */ @@ -149,6 +145,4 @@ void __android_log_lock(); int __android_log_trylock(); void __android_log_unlock(); -extern int __android_log_transport; - __END_DECLS diff --git a/liblog/logger_write.cpp b/liblog/logger_write.cpp index 7fc87477e7..e1772f1909 100644 --- a/liblog/logger_write.cpp +++ b/liblog/logger_write.cpp @@ -25,17 +25,28 @@ #endif #include <log/event_tag_map.h> -#include <log/log_transport.h> #include <private/android_filesystem_config.h> #include <private/android_logger.h> -#include "config_write.h" #include "log_portability.h" #include "logger.h" #include "uio.h" #define LOG_BUF_SIZE 1024 +#if (FAKE_LOG_DEVICE == 0) +extern struct android_log_transport_write logdLoggerWrite; +extern struct android_log_transport_write pmsgLoggerWrite; + +android_log_transport_write* android_log_write = &logdLoggerWrite; +android_log_transport_write* android_log_persist_write = &pmsgLoggerWrite; +#else +extern android_log_transport_write fakeLoggerWrite; + +android_log_transport_write* android_log_write = &fakeLoggerWrite; +android_log_transport_write* android_log_persist_write = nullptr; +#endif + static int __write_to_log_init(log_id_t, struct iovec* vec, size_t nr); static int (*write_to_log)(log_id_t, struct iovec* vec, size_t nr) = __write_to_log_init; @@ -89,9 +100,8 @@ static void __android_log_cache_available(struct android_log_transport_write* no } for (i = LOG_ID_MIN; i < LOG_ID_MAX; ++i) { - if (node->write && (i != LOG_ID_KERNEL) && - ((i != LOG_ID_SECURITY) || (check_log_uid_permissions() == 0)) && - (!node->available || ((*node->available)(static_cast<log_id_t>(i)) >= 0))) { + if (i != LOG_ID_KERNEL && (i != LOG_ID_SECURITY || check_log_uid_permissions() == 0) && + (*node->available)(static_cast<log_id_t>(i)) >= 0) { node->logMask |= 1 << i; } } @@ -105,7 +115,6 @@ static atomic_uintptr_t tagMap; * Release any logger resources. A new log write will immediately re-acquire. */ void __android_log_close() { - struct android_log_transport_write* transport; #if defined(__ANDROID__) EventTagMap* m; #endif @@ -124,20 +133,14 @@ void __android_log_close() { * disengenuous use of this function. */ - write_transport_for_each(transport, &__android_log_persist_write) { - if (transport->close) { - (*transport->close)(); - } + if (android_log_write != nullptr) { + android_log_write->close(); } - write_transport_for_each(transport, &__android_log_transport_write) { - if (transport->close) { - (*transport->close)(); - } + if (android_log_persist_write != nullptr) { + android_log_persist_write->close(); } - __android_log_config_write_close(); - #if defined(__ANDROID__) /* * Additional risk here somewhat mitigated by immediately unlock flushing @@ -161,62 +164,39 @@ void __android_log_close() { #endif } -/* log_init_lock assumed */ -static int __write_to_log_initialize() { - struct android_log_transport_write* transport; - struct listnode* n; - int i = 0, ret = 0; - - __android_log_config_write(); - write_transport_for_each_safe(transport, n, &__android_log_transport_write) { - __android_log_cache_available(transport); - if (!transport->logMask) { - list_remove(&transport->node); - continue; - } - if (!transport->open || ((*transport->open)() < 0)) { - if (transport->close) { - (*transport->close)(); - } - list_remove(&transport->node); - continue; - } - ++ret; +static bool transport_initialize(android_log_transport_write* transport) { + if (transport == nullptr) { + return false; } - write_transport_for_each_safe(transport, n, &__android_log_persist_write) { - __android_log_cache_available(transport); - if (!transport->logMask) { - list_remove(&transport->node); - continue; - } - if (!transport->open || ((*transport->open)() < 0)) { - if (transport->close) { - (*transport->close)(); - } - list_remove(&transport->node); - continue; - } - ++i; + + __android_log_cache_available(transport); + if (!transport->logMask) { + return false; } - if (!ret && !i) { + + // TODO: Do we actually need to call close() if open() fails? + if (transport->open() < 0) { + transport->close(); + return false; + } + + return true; +} + +/* log_init_lock assumed */ +static int __write_to_log_initialize() { + if (!transport_initialize(android_log_write)) { return -ENODEV; } - return ret; + transport_initialize(android_log_persist_write); + + return 1; } static int __write_to_log_daemon(log_id_t log_id, struct iovec* vec, size_t nr) { - struct android_log_transport_write* node; int ret, save_errno; struct timespec ts; - size_t len, i; - - for (len = i = 0; i < nr; ++i) { - len += vec[i].iov_len; - } - if (!len) { - return -EINVAL; - } save_errno = errno; #if defined(__ANDROID__) @@ -283,30 +263,9 @@ static int __write_to_log_daemon(log_id_t log_id, struct iovec* vec, size_t nr) return -EPERM; } } else { - /* Validate the incoming tag, tag content can not split across iovec */ - char prio = ANDROID_LOG_VERBOSE; - const char* tag = static_cast<const char*>(vec[0].iov_base); - size_t len = vec[0].iov_len; - if (!tag) { - len = 0; - } - if (len > 0) { - prio = *tag; - if (len > 1) { - --len; - ++tag; - } else { - len = vec[1].iov_len; - tag = ((const char*)vec[1].iov_base); - if (!tag) { - len = 0; - } - } - } - /* tag must be nul terminated */ - if (tag && strnlen(tag, len) >= len) { - tag = NULL; - } + int prio = *static_cast<int*>(vec[0].iov_base); + const char* tag = static_cast<const char*>(vec[1].iov_base); + size_t len = vec[1].iov_len; if (!__android_log_is_loggable_len(prio, tag, len - 1, ANDROID_LOG_VERBOSE)) { errno = save_errno; @@ -324,21 +283,18 @@ static int __write_to_log_daemon(log_id_t log_id, struct iovec* vec, size_t nr) #endif ret = 0; - i = 1 << log_id; - write_transport_for_each(node, &__android_log_transport_write) { - if (node->logMask & i) { - ssize_t retval; - retval = (*node->write)(log_id, &ts, vec, nr); - if (ret >= 0) { - ret = retval; - } + size_t i = 1 << log_id; + + if (android_log_write != nullptr && (android_log_write->logMask & i)) { + ssize_t retval; + retval = android_log_write->write(log_id, &ts, vec, nr); + if (ret >= 0) { + ret = retval; } } - write_transport_for_each(node, &__android_log_persist_write) { - if (node->logMask & i) { - (void)(*node->write)(log_id, &ts, vec, nr); - } + if (android_log_persist_write != nullptr && (android_log_persist_write->logMask & i)) { + android_log_persist_write->write(log_id, &ts, vec, nr); } errno = save_errno; @@ -354,9 +310,6 @@ static int __write_to_log_init(log_id_t log_id, struct iovec* vec, size_t nr) { ret = __write_to_log_initialize(); if (ret < 0) { __android_log_unlock(); - if (!list_empty(&__android_log_persist_write)) { - __write_to_log_daemon(log_id, vec, nr); - } errno = save_errno; return ret; } @@ -546,81 +499,3 @@ int __android_log_security_bswrite(int32_t tag, const char* payload) { return write_to_log(LOG_ID_SECURITY, vec, 4); } - -static int __write_to_log_null(log_id_t log_id, struct iovec* vec, size_t nr) { - size_t len, i; - - if ((log_id < LOG_ID_MIN) || (log_id >= LOG_ID_MAX)) { - return -EINVAL; - } - - for (len = i = 0; i < nr; ++i) { - len += vec[i].iov_len; - } - if (!len) { - return -EINVAL; - } - return len; -} - -/* Following functions need access to our internal write_to_log status */ - -int __android_log_transport; - -int android_set_log_transport(int transport_flag) { - int retval; - - if (transport_flag < 0) { - return -EINVAL; - } - - retval = LOGGER_NULL; - - __android_log_lock(); - - if (transport_flag & LOGGER_NULL) { - write_to_log = __write_to_log_null; - - __android_log_unlock(); - - return retval; - } - - __android_log_transport &= LOGGER_LOGD; - - transport_flag &= LOGGER_LOGD; - - if (__android_log_transport != transport_flag) { - __android_log_transport = transport_flag; - __android_log_config_write_close(); - - write_to_log = __write_to_log_init; - /* generically we only expect these two values for write_to_log */ - } else if ((write_to_log != __write_to_log_init) && (write_to_log != __write_to_log_daemon)) { - write_to_log = __write_to_log_init; - } - - retval = __android_log_transport; - - __android_log_unlock(); - - return retval; -} - -int android_get_log_transport() { - int ret = LOGGER_DEFAULT; - - __android_log_lock(); - if (write_to_log == __write_to_log_null) { - ret = LOGGER_NULL; - } else { - __android_log_transport &= LOGGER_LOGD; - ret = __android_log_transport; - if ((write_to_log != __write_to_log_init) && (write_to_log != __write_to_log_daemon)) { - ret = -EINVAL; - } - } - __android_log_unlock(); - - return ret; -} diff --git a/liblog/pmsg_reader.cpp b/liblog/pmsg_reader.cpp index 005fec8027..2db45a1442 100644 --- a/liblog/pmsg_reader.cpp +++ b/liblog/pmsg_reader.cpp @@ -37,7 +37,6 @@ static int pmsgClear(struct android_log_logger* logger, struct android_log_transport_context* transp); struct android_log_transport_read pmsgLoggerRead = { - .node = {&pmsgLoggerRead.node, &pmsgLoggerRead.node}, .name = "pmsg", .available = pmsgAvailable, .version = pmsgVersion, diff --git a/liblog/pmsg_writer.cpp b/liblog/pmsg_writer.cpp index 69720ed65a..54980d9fea 100644 --- a/liblog/pmsg_writer.cpp +++ b/liblog/pmsg_writer.cpp @@ -29,7 +29,6 @@ #include <private/android_filesystem_config.h> #include <private/android_logger.h> -#include "config_write.h" #include "log_portability.h" #include "logger.h" #include "uio.h" @@ -40,9 +39,9 @@ static int pmsgAvailable(log_id_t logId); static int pmsgWrite(log_id_t logId, struct timespec* ts, struct iovec* vec, size_t nr); struct android_log_transport_write pmsgLoggerWrite = { - .node = {&pmsgLoggerWrite.node, &pmsgLoggerWrite.node}, - .context.fd = -1, .name = "pmsg", + .logMask = 0, + .context.fd = -1, .available = pmsgAvailable, .open = pmsgOpen, .close = pmsgClose, @@ -100,7 +99,7 @@ static int pmsgWrite(log_id_t logId, struct timespec* ts, struct iovec* vec, siz return -EINVAL; } - if (SNET_EVENT_LOG_TAG != *static_cast<uint8_t*>(vec[0].iov_base)) { + if (SNET_EVENT_LOG_TAG != *static_cast<uint32_t*>(vec[0].iov_base)) { return -EPERM; } } diff --git a/liblog/tests/liblog_benchmark.cpp b/liblog/tests/liblog_benchmark.cpp index 71637434df..4642b9bf41 100644 --- a/liblog/tests/liblog_benchmark.cpp +++ b/liblog/tests/liblog_benchmark.cpp @@ -29,7 +29,6 @@ #include <benchmark/benchmark.h> #include <cutils/sockets.h> #include <log/event_tag_map.h> -#include <log/log_transport.h> #include <private/android_logger.h> BENCHMARK_MAIN(); @@ -73,21 +72,6 @@ static void BM_log_maximum(benchmark::State& state) { } BENCHMARK(BM_log_maximum); -static void set_log_null() { - android_set_log_transport(LOGGER_NULL); -} - -static void set_log_default() { - android_set_log_transport(LOGGER_DEFAULT); -} - -static void BM_log_maximum_null(benchmark::State& state) { - set_log_null(); - BM_log_maximum(state); - set_log_default(); -} -BENCHMARK(BM_log_maximum_null); - /* * Measure the time it takes to collect the time using * discrete acquisition (state.PauseTiming() to state.ResumeTiming()) @@ -618,13 +602,6 @@ static void BM_log_event_overhead_42(benchmark::State& state) { } BENCHMARK(BM_log_event_overhead_42); -static void BM_log_event_overhead_null(benchmark::State& state) { - set_log_null(); - BM_log_event_overhead(state); - set_log_default(); -} -BENCHMARK(BM_log_event_overhead_null); - /* * Measure the time it takes to submit the android event logging call * using discrete acquisition under very-light load (<1% CPU utilization). @@ -639,15 +616,6 @@ static void BM_log_light_overhead(benchmark::State& state) { } BENCHMARK(BM_log_light_overhead); -static void BM_log_light_overhead_null(benchmark::State& state) { - set_log_null(); - BM_log_light_overhead(state); - set_log_default(); -} -// Default gets out of hand for this test, so we set a reasonable number of -// iterations for a timely result. -BENCHMARK(BM_log_light_overhead_null)->Iterations(500); - static void caught_latency(int /*signum*/) { unsigned long long v = 0xDEADBEEFA55A5AA5ULL; diff --git a/liblog/tests/liblog_test.cpp b/liblog/tests/liblog_test.cpp index 34fb909eb9..d8b0ced4f1 100644 --- a/liblog/tests/liblog_test.cpp +++ b/liblog/tests/liblog_test.cpp @@ -2664,8 +2664,7 @@ static void create_android_logger(const char* (*fn)(uint32_t tag, print_barrier(); } EXPECT_EQ(0, buffer_to_string); - EXPECT_EQ(strlen(expected_string), strlen(msgBuf)); - EXPECT_EQ(0, strcmp(expected_string, msgBuf)); + EXPECT_STREQ(expected_string, msgBuf); } EXPECT_EQ(1, count); @@ -2858,7 +2857,7 @@ static ssize_t __pmsg_fn(log_id_t logId, char prio, const char* filename, EXPECT_EQ(ANDROID_LOG_VERBOSE, prio); EXPECT_FALSE(NULL == strstr(__pmsg_file, filename)); EXPECT_EQ(len, sizeof(max_payload_buf)); - EXPECT_EQ(0, strcmp(max_payload_buf, buf)); + EXPECT_STREQ(max_payload_buf, buf); ++signaled; if ((len != sizeof(max_payload_buf)) || strcmp(max_payload_buf, buf)) { diff --git a/liblog/tests/log_wrap_test.cpp b/liblog/tests/log_wrap_test.cpp index ebf0b15058..c7dd8e8122 100644 --- a/liblog/tests/log_wrap_test.cpp +++ b/liblog/tests/log_wrap_test.cpp @@ -27,12 +27,9 @@ #include <log/log_properties.h> #include <log/log_read.h> #include <log/log_time.h> -#include <log/log_transport.h> #ifdef __ANDROID__ static void read_with_wrap() { - android_set_log_transport(LOGGER_LOGD); - // Read the last line in the log to get a starting timestamp. We're assuming // the log is not empty. const int mode = ANDROID_LOG_RDONLY | ANDROID_LOG_NONBLOCK; diff --git a/libnativeloader/public_libraries.cpp b/libnativeloader/public_libraries.cpp index 93df1d05ad..010e8cc3bb 100644 --- a/libnativeloader/public_libraries.cpp +++ b/libnativeloader/public_libraries.cpp @@ -176,6 +176,11 @@ static std::string InitDefaultPublicLibraries(bool for_preload) { std::copy(vec.begin(), vec.end(), std::back_inserter(*sonames)); } + // If this is for preloading libs, don't remove the libs from APEXes. + if (for_preload) { + return android::base::Join(*sonames, ':'); + } + // Remove the public libs in the runtime namespace. // These libs are listed in public.android.txt, but we don't want the rest of android // in default namespace to dlopen the libs. diff --git a/libstats/statsd_writer.c b/libstats/statsd_writer.c index b1c05eac95..073b67fa0e 100644 --- a/libstats/statsd_writer.c +++ b/libstats/statsd_writer.c @@ -36,25 +36,6 @@ #include <time.h> #include <unistd.h> -/* branchless on many architectures. */ -#define min(x, y) ((y) ^ (((x) ^ (y)) & -((x) < (y)))) - -#ifndef htole32 -#if __BYTE_ORDER == __LITTLE_ENDIAN -#define htole32(x) (x) -#else -#define htole32(x) __bswap_32(x) -#endif -#endif - -#ifndef htole64 -#if __BYTE_ORDER == __LITTLE_ENDIAN -#define htole64(x) (x) -#else -#define htole64(x) __bswap_64(x) -#endif -#endif - static pthread_mutex_t log_init_lock = PTHREAD_MUTEX_INITIALIZER; static atomic_int dropped = 0; static atomic_int log_error = 0; @@ -221,14 +202,14 @@ static int statsdWrite(struct timespec* ts, struct iovec* vec, size_t nr) { android_log_event_long_t buffer; header.id = LOG_ID_STATS; // store the last log error in the tag field. This tag field is not used by statsd. - buffer.header.tag = htole32(atomic_load(&log_error)); + buffer.header.tag = atomic_load(&log_error); buffer.payload.type = EVENT_TYPE_LONG; // format: // |atom_tag|dropped_count| int64_t composed_long = atomic_load(&atom_tag); // Send 2 int32's via an int64. composed_long = ((composed_long << 32) | ((int64_t)snapshot)); - buffer.payload.data = htole64(composed_long); + buffer.payload.data = composed_long; newVec[headerLength].iov_base = &buffer; newVec[headerLength].iov_len = sizeof(buffer); diff --git a/libunwindstack/ElfInterface.cpp b/libunwindstack/ElfInterface.cpp index bdfee0153e..be1f092607 100644 --- a/libunwindstack/ElfInterface.cpp +++ b/libunwindstack/ElfInterface.cpp @@ -187,8 +187,13 @@ uint64_t ElfInterface::GetLoadBias(Memory* memory) { if (!memory->ReadFully(offset, &phdr, sizeof(phdr))) { return 0; } - if (phdr.p_type == PT_LOAD && phdr.p_offset == 0) { - return phdr.p_vaddr; + + // Find the first executable load when looking for the load bias. + if (phdr.p_type == PT_LOAD && (phdr.p_flags & PF_X)) { + if (phdr.p_vaddr > phdr.p_offset) { + return phdr.p_vaddr - phdr.p_offset; + } + break; } } return 0; diff --git a/libunwindstack/tests/ElfInterfaceTest.cpp b/libunwindstack/tests/ElfInterfaceTest.cpp index f9ee9eb936..5b2036bdb8 100644 --- a/libunwindstack/tests/ElfInterfaceTest.cpp +++ b/libunwindstack/tests/ElfInterfaceTest.cpp @@ -131,6 +131,12 @@ class ElfInterfaceTest : public ::testing::Test { template <typename Ehdr, typename Shdr, typename Nhdr, typename ElfInterfaceType> void BuildIDSectionTooSmallForHeader(); + template <typename Ehdr, typename Phdr, typename ElfInterfaceType> + void CheckLoadBiasInFirstPhdr(uint64_t load_bias); + + template <typename Ehdr, typename Phdr, typename ElfInterfaceType> + void CheckLoadBiasInFirstExecPhdr(uint64_t offset, uint64_t vaddr, uint64_t load_bias); + MemoryFake memory_; }; @@ -1495,4 +1501,122 @@ TEST_F(ElfInterfaceTest, build_id_section_too_small_for_header64) { BuildIDSectionTooSmallForHeader<Elf64_Ehdr, Elf64_Shdr, Elf64_Nhdr, ElfInterface64>(); } +template <typename Ehdr, typename Phdr, typename ElfInterfaceType> +void ElfInterfaceTest::CheckLoadBiasInFirstPhdr(uint64_t load_bias) { + Ehdr ehdr = {}; + ehdr.e_phoff = 0x100; + ehdr.e_phnum = 2; + ehdr.e_phentsize = sizeof(Phdr); + memory_.SetMemory(0, &ehdr, sizeof(ehdr)); + + Phdr phdr = {}; + phdr.p_type = PT_LOAD; + phdr.p_offset = 0; + phdr.p_vaddr = load_bias; + phdr.p_memsz = 0x10000; + phdr.p_flags = PF_R | PF_X; + phdr.p_align = 0x1000; + memory_.SetMemory(0x100, &phdr, sizeof(phdr)); + + memset(&phdr, 0, sizeof(phdr)); + phdr.p_type = PT_LOAD; + phdr.p_offset = 0x1000; + phdr.p_memsz = 0x2000; + phdr.p_flags = PF_R | PF_X; + phdr.p_align = 0x1000; + memory_.SetMemory(0x100 + sizeof(phdr), &phdr, sizeof(phdr)); + + uint64_t static_load_bias = ElfInterface::GetLoadBias<Ehdr, Phdr>(&memory_); + ASSERT_EQ(load_bias, static_load_bias); + + std::unique_ptr<ElfInterfaceType> elf(new ElfInterfaceType(&memory_)); + uint64_t init_load_bias = 0; + ASSERT_TRUE(elf->Init(&init_load_bias)); + ASSERT_EQ(init_load_bias, static_load_bias); +} + +TEST_F(ElfInterfaceTest, get_load_bias_zero_32) { + CheckLoadBiasInFirstPhdr<Elf32_Ehdr, Elf32_Phdr, ElfInterface32>(0); +} + +TEST_F(ElfInterfaceTest, get_load_bias_zero_64) { + CheckLoadBiasInFirstPhdr<Elf64_Ehdr, Elf64_Phdr, ElfInterface64>(0); +} + +TEST_F(ElfInterfaceTest, get_load_bias_non_zero_32) { + CheckLoadBiasInFirstPhdr<Elf32_Ehdr, Elf32_Phdr, ElfInterface32>(0x1000); +} + +TEST_F(ElfInterfaceTest, get_load_bias_non_zero_64) { + CheckLoadBiasInFirstPhdr<Elf64_Ehdr, Elf64_Phdr, ElfInterface64>(0x1000); +} + +template <typename Ehdr, typename Phdr, typename ElfInterfaceType> +void ElfInterfaceTest::CheckLoadBiasInFirstExecPhdr(uint64_t offset, uint64_t vaddr, + uint64_t load_bias) { + Ehdr ehdr = {}; + ehdr.e_phoff = 0x100; + ehdr.e_phnum = 3; + ehdr.e_phentsize = sizeof(Phdr); + memory_.SetMemory(0, &ehdr, sizeof(ehdr)); + + Phdr phdr = {}; + phdr.p_type = PT_LOAD; + phdr.p_memsz = 0x10000; + phdr.p_flags = PF_R; + phdr.p_align = 0x1000; + memory_.SetMemory(0x100, &phdr, sizeof(phdr)); + + memset(&phdr, 0, sizeof(phdr)); + phdr.p_type = PT_LOAD; + phdr.p_offset = offset; + phdr.p_vaddr = vaddr; + phdr.p_memsz = 0x2000; + phdr.p_flags = PF_R | PF_X; + phdr.p_align = 0x1000; + memory_.SetMemory(0x100 + sizeof(phdr), &phdr, sizeof(phdr)); + + // Second executable load should be ignored for load bias computation. + memset(&phdr, 0, sizeof(phdr)); + phdr.p_type = PT_LOAD; + phdr.p_offset = 0x1234; + phdr.p_vaddr = 0x2000; + phdr.p_memsz = 0x2000; + phdr.p_flags = PF_R | PF_X; + phdr.p_align = 0x1000; + memory_.SetMemory(0x200 + sizeof(phdr), &phdr, sizeof(phdr)); + + uint64_t static_load_bias = ElfInterface::GetLoadBias<Ehdr, Phdr>(&memory_); + ASSERT_EQ(load_bias, static_load_bias); + + std::unique_ptr<ElfInterfaceType> elf(new ElfInterfaceType(&memory_)); + uint64_t init_load_bias = 0; + ASSERT_TRUE(elf->Init(&init_load_bias)); + ASSERT_EQ(init_load_bias, static_load_bias); +} + +TEST_F(ElfInterfaceTest, get_load_bias_exec_zero_32) { + CheckLoadBiasInFirstExecPhdr<Elf32_Ehdr, Elf32_Phdr, ElfInterface32>(0x1000, 0x1000, 0); +} + +TEST_F(ElfInterfaceTest, get_load_bias_exec_zero_64) { + CheckLoadBiasInFirstExecPhdr<Elf64_Ehdr, Elf64_Phdr, ElfInterface64>(0x1000, 0x1000, 0); +} + +TEST_F(ElfInterfaceTest, get_load_bias_exec_non_zero_32) { + CheckLoadBiasInFirstExecPhdr<Elf32_Ehdr, Elf32_Phdr, ElfInterface32>(0x1000, 0x4000, 0x3000); +} + +TEST_F(ElfInterfaceTest, get_load_bias_exec_non_zero_64) { + CheckLoadBiasInFirstExecPhdr<Elf64_Ehdr, Elf64_Phdr, ElfInterface64>(0x1000, 0x4000, 0x3000); +} + +TEST_F(ElfInterfaceTest, get_load_bias_exec_zero_from_error_32) { + CheckLoadBiasInFirstExecPhdr<Elf32_Ehdr, Elf32_Phdr, ElfInterface32>(0x5000, 0x1000, 0); +} + +TEST_F(ElfInterfaceTest, get_load_bias_exec_zero_from_error_64) { + CheckLoadBiasInFirstExecPhdr<Elf64_Ehdr, Elf64_Phdr, ElfInterface64>(0x5000, 0x1000, 0); +} + } // namespace unwindstack diff --git a/libunwindstack/tests/MapInfoGetLoadBiasTest.cpp b/libunwindstack/tests/MapInfoGetLoadBiasTest.cpp index f5ac6cb444..2c98928473 100644 --- a/libunwindstack/tests/MapInfoGetLoadBiasTest.cpp +++ b/libunwindstack/tests/MapInfoGetLoadBiasTest.cpp @@ -141,6 +141,7 @@ static void InitElfData(MemoryFake* memory, uint64_t offset) { phdr.p_type = PT_NULL; memory->SetMemory(offset + 0x5000, &phdr, sizeof(phdr)); phdr.p_type = PT_LOAD; + phdr.p_flags = PF_X; phdr.p_offset = 0; phdr.p_vaddr = 0xe000; memory->SetMemory(offset + 0x5000 + sizeof(phdr), &phdr, sizeof(phdr)); diff --git a/lmkd/event.logtags b/lmkd/event.logtags index 065c6db6d7..452f4119c9 100644 --- a/lmkd/event.logtags +++ b/lmkd/event.logtags @@ -17,8 +17,8 @@ # Multiple values are separated by commas. # # The data type is a number from the following values: -# 1: int -# 2: long +# 1: int32_t +# 2: int64_t # 3: string # 4: list # @@ -34,5 +34,5 @@ # # TODO: generate ".java" and ".h" files with integer constants from this file. -# for meminfo logs -10195355 meminfo (MemFree|1),(Cached|1),(SwapCached|1),(Buffers|1),(Shmem|1),(Unevictable|1),(SwapTotal|1),(SwapFree|1),(ActiveAnon|1),(InactiveAnon|1),(ActiveFile|1),(InactiveFile|1),(SReclaimable|1),(SUnreclaim|1),(KernelStack|1),(PageTables|1),(ION_heap|1),(ION_heap_pool|1),(CmaFree|1) +# for killinfo logs +10195355 killinfo (Pid|1|5),(Uid|1|5),(OomAdj|1),(MinOomAdj|1),(TaskSize|1),(enum kill_reasons|1|5),(MemFree|1),(Cached|1),(SwapCached|1),(Buffers|1),(Shmem|1),(Unevictable|1),(SwapTotal|1),(SwapFree|1),(ActiveAnon|1),(InactiveAnon|1),(ActiveFile|1),(InactiveFile|1),(SReclaimable|1),(SUnreclaim|1),(KernelStack|1),(PageTables|1),(IonHeap|1),(IonHeapPool|1),(CmaFree|1) diff --git a/lmkd/lmkd.c b/lmkd/lmkd.c index 449088afbe..372e10f1a3 100644 --- a/lmkd/lmkd.c +++ b/lmkd/lmkd.c @@ -84,7 +84,7 @@ #define PERCEPTIBLE_APP_ADJ 200 /* Android Logger event logtags (see event.logtags) */ -#define MEMINFO_LOG_TAG 10195355 +#define KILLINFO_LOG_TAG 10195355 /* gid containing AID_SYSTEM required */ #define INKERNEL_MINFREE_PATH "/sys/module/lowmemorykiller/parameters/minfree" @@ -926,12 +926,12 @@ static void cmd_procprio(LMKD_CTRL_PACKET packet) { static void cmd_procremove(LMKD_CTRL_PACKET packet) { struct lmk_procremove params; + lmkd_pack_get_procremove(packet, ¶ms); if (use_inkernel_interface) { stats_remove_taskname(params.pid, kpoll_info.poll_fd); return; } - lmkd_pack_get_procremove(packet, ¶ms); /* * WARNING: After pid_remove() procp is freed and can't be used! * Therefore placed at the end of the function. @@ -1394,7 +1394,7 @@ static int zoneinfo_parse(struct zoneinfo *zi) { char *buf; char *save_ptr; char *line; - char zone_name[LINE_MAX]; + char zone_name[LINE_MAX + 1]; struct zoneinfo_node *node = NULL; int node_idx = 0; int zone_idx = 0; @@ -1566,7 +1566,17 @@ static int vmstat_parse(union vmstat *vs) { return 0; } -static void meminfo_log(union meminfo *mi) { +static void killinfo_log(struct proc* procp, int min_oom_score, int tasksize, + int kill_reason, union meminfo *mi) { + /* log process information */ + android_log_write_int32(ctx, procp->pid); + android_log_write_int32(ctx, procp->uid); + android_log_write_int32(ctx, procp->oomadj); + android_log_write_int32(ctx, min_oom_score); + android_log_write_int32(ctx, (int32_t)min(tasksize * page_k, INT32_MAX)); + android_log_write_int32(ctx, kill_reason); + + /* log meminfo fields */ for (int field_idx = 0; field_idx < MI_FIELD_COUNT; field_idx++) { android_log_write_int32(ctx, (int32_t)min(mi->arr[field_idx] * page_k, INT32_MAX)); } @@ -1640,7 +1650,8 @@ static void set_process_group_and_prio(int pid, SchedPolicy sp, int prio) { static int last_killed_pid = -1; /* Kill one process specified by procp. Returns the size of the process killed */ -static int kill_one_process(struct proc* procp, int min_oom_score, const char *reason) { +static int kill_one_process(struct proc* procp, int min_oom_score, int kill_reason, + const char *kill_desc, union meminfo *mi) { int pid = procp->pid; uid_t uid = procp->uid; int tgid; @@ -1684,9 +1695,12 @@ static int kill_one_process(struct proc* procp, int min_oom_score, const char *r set_process_group_and_prio(pid, SP_FOREGROUND, ANDROID_PRIORITY_HIGHEST); inc_killcnt(procp->oomadj); - if (reason) { + + killinfo_log(procp, min_oom_score, tasksize, kill_reason, mi); + + if (kill_desc) { ALOGI("Kill '%s' (%d), uid %d, oom_adj %d to free %ldkB; reason: %s", taskname, pid, - uid, procp->oomadj, tasksize * page_k, reason); + uid, procp->oomadj, tasksize * page_k, kill_desc); } else { ALOGI("Kill '%s' (%d), uid %d, oom_adj %d to free %ldkB", taskname, pid, uid, procp->oomadj, tasksize * page_k); @@ -1712,7 +1726,8 @@ out: * Find one process to kill at or above the given oom_adj level. * Returns size of the killed process. */ -static int find_and_kill_process(int min_score_adj, const char *reason) { +static int find_and_kill_process(int min_score_adj, int kill_reason, const char *kill_desc, + union meminfo *mi) { int i; int killed_size = 0; bool lmk_state_change_start = false; @@ -1727,7 +1742,7 @@ static int find_and_kill_process(int min_score_adj, const char *reason) { if (!procp) break; - killed_size = kill_one_process(procp, min_score_adj, reason); + killed_size = kill_one_process(procp, min_score_adj, kill_reason, kill_desc, mi); if (killed_size >= 0) { if (!lmk_state_change_start) { lmk_state_change_start = true; @@ -2051,7 +2066,7 @@ static void mp_event_psi(int data, uint32_t events, struct polling_params *poll_ /* Kill a process if necessary */ if (kill_reason != NONE) { - int pages_freed = find_and_kill_process(min_score_adj, kill_desc); + int pages_freed = find_and_kill_process(min_score_adj, kill_reason, kill_desc, &mi); if (pages_freed > 0) { killing = true; if (cut_thrashing_limit) { @@ -2062,7 +2077,6 @@ static void mp_event_psi(int data, uint32_t events, struct polling_params *poll_ thrashing_limit = (thrashing_limit * (100 - thrashing_limit_decay_pct)) / 100; } } - meminfo_log(&mi); } no_kill: @@ -2251,12 +2265,10 @@ static void mp_event_common(int data, uint32_t events, struct polling_params *po do_kill: if (low_ram_device) { /* For Go devices kill only one task */ - if (find_and_kill_process(level_oomadj[level], NULL) == 0) { + if (find_and_kill_process(level_oomadj[level], -1, NULL, &mi) == 0) { if (debug_process_killing) { ALOGI("Nothing to kill"); } - } else { - meminfo_log(&mi); } } else { int pages_freed; @@ -2276,7 +2288,7 @@ do_kill: min_score_adj = level_oomadj[level]; } - pages_freed = find_and_kill_process(min_score_adj, NULL); + pages_freed = find_and_kill_process(min_score_adj, -1, NULL, &mi); if (pages_freed == 0) { /* Rate limit kill reports when nothing was reclaimed */ @@ -2289,9 +2301,7 @@ do_kill: last_kill_tm = curr_tm; } - /* Log meminfo whenever we kill or when report rate limit allows */ - meminfo_log(&mi); - + /* Log whenever we kill or when report rate limit allows */ if (use_minfree_levels) { ALOGI("Reclaimed %ldkB, cache(%ldkB) and " "free(%" PRId64 "kB)-reserved(%" PRId64 "kB) below min(%ldkB) for oom_adj %d", @@ -2714,7 +2724,7 @@ int main(int argc __unused, char **argv __unused) { thrashing_limit_decay_pct = clamp(0, 100, property_get_int32("ro.lmk.thrashing_limit_decay", low_ram_device ? DEF_THRASHING_DECAY_LOWRAM : DEF_THRASHING_DECAY)); - ctx = create_android_logger(MEMINFO_LOG_TAG); + ctx = create_android_logger(KILLINFO_LOG_TAG); statslog_init(); diff --git a/rootdir/Android.mk b/rootdir/Android.mk index a0059dbe3b..c8f0a8b735 100644 --- a/rootdir/Android.mk +++ b/rootdir/Android.mk @@ -58,15 +58,6 @@ endif endif ####################################### -# fsverity_init - -include $(CLEAR_VARS) -LOCAL_MODULE:= fsverity_init -LOCAL_MODULE_CLASS := EXECUTABLES -LOCAL_SRC_FILES := fsverity_init.sh -include $(BUILD_PREBUILT) - -####################################### # init.environ.rc include $(CLEAR_VARS) diff --git a/rootdir/fsverity_init.sh b/rootdir/fsverity_init.sh deleted file mode 100644 index 4fee15fb33..0000000000 --- a/rootdir/fsverity_init.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/system/bin/sh -# -# Copyright (C) 2019 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Enforce fsverity signature checking -echo 1 > /proc/sys/fs/verity/require_signatures - -# Load all keys -for cert in /product/etc/security/fsverity/*.der; do - /system/bin/mini-keyctl padd asymmetric fsv_product .fs-verity < "$cert" || - log -p e -t fsverity_init "Failed to load $cert" -done - -DEBUGGABLE=$(getprop ro.debuggable) -if [ $DEBUGGABLE != "1" ]; then - # Prevent future key links to .fs-verity keyring - /system/bin/mini-keyctl restrict_keyring .fs-verity || - log -p e -t fsverity_init "Failed to restrict .fs-verity keyring" -fi diff --git a/rootdir/init.rc b/rootdir/init.rc index 918390102f..bad06425ce 100644 --- a/rootdir/init.rc +++ b/rootdir/init.rc @@ -489,10 +489,6 @@ on post-fs-data mkdir /data/bootchart 0755 shell shell bootchart start - # Load fsverity keys. This needs to happen before apexd, as post-install of - # APEXes may rely on keys. - exec -- /system/bin/fsverity_init - # Make sure that apexd is started in the default namespace enter_default_mount_ns @@ -823,6 +819,9 @@ on boot class_start core + # Requires keystore (currently a core service) to be ready first. + exec -- /system/bin/fsverity_init + on nonencrypted class_start main class_start late_start |