diff options
135 files changed, 4981 insertions, 1995 deletions
diff --git a/TEST_MAPPING b/TEST_MAPPING index e3a86756c..9b6213a9b 100644 --- a/TEST_MAPPING +++ b/TEST_MAPPING @@ -28,6 +28,9 @@ "name": "fs_mgr_vendor_overlay_test" }, { + "name": "init_kill_services_test" + }, + { "name": "libbase_test" }, { @@ -64,5 +67,11 @@ { "name": "ziparchive-tests" } + ], + + "postsubmit": [ + { + "name": "ziparchive_tests_large" + } ] } diff --git a/adb/Android.bp b/adb/Android.bp index 1b7a26633..81d20c1e7 100644 --- a/adb/Android.bp +++ b/adb/Android.bp @@ -298,6 +298,7 @@ cc_binary_host { "client/fastdeploycallbacks.cpp", "client/incremental.cpp", "client/incremental_server.cpp", + "client/incremental_utils.cpp", "shell_service_protocol.cpp", ], @@ -316,6 +317,7 @@ cc_binary_host { "libandroidfw", "libapp_processes_protos_full", "libbase", + "libbrotli", "libcutils", "libcrypto_utils", "libcrypto", @@ -387,17 +389,17 @@ cc_library_static { generated_headers: ["platform_tools_version"], static_libs: [ - "libadbconnection_server", - "libapp_processes_protos_lite", "libdiagnose_usb", ], shared_libs: [ + "libadbconnection_server", "libadb_crypto", "libadb_pairing_connection", "libadb_protos", "libadb_tls_connection", "libadbd_auth", + "libapp_processes_protos_lite", "libasyncio", "libbase", "libcrypto", @@ -432,9 +434,19 @@ cc_library_static { exclude_shared_libs: [ "libadb_pairing_auth", "libadb_pairing_connection", + "libapp_processes_protos_lite", ], } }, + + apex_available: [ + "//apex_available:platform", + "com.android.adbd", + ], + visibility: [ + "//bootable/recovery/minadbd", + "//system/core/adb", + ], } cc_library { @@ -458,9 +470,8 @@ cc_library { static_libs: [ "libadbconnection_server", "libadbd_core", - "libapp_processes_protos_lite", + "libbrotli", "libdiagnose_usb", - "libprotobuf-cpp-lite", ], shared_libs: [ @@ -470,12 +481,14 @@ cc_library { "libadb_tls_connection", "libadbd_auth", "libadbd_fs", + "libapp_processes_protos_lite", "libasyncio", "libbase", "libcrypto", "libcrypto_utils", "libcutils_sockets", "liblog", + "libprotobuf-cpp-lite", ], target: { @@ -501,12 +514,22 @@ cc_library { ], }, }, + + apex_available: [ + "//apex_available:platform", + "com.android.adbd", + ], + visibility: [ + "//system/core/adb", + ], + } cc_library { name: "libadbd", defaults: ["adbd_defaults", "host_adbd_supported"], recovery_available: true, + apex_available: ["com.android.adbd"], // avoid getting duplicate symbol of android::build::getbuildnumber(). use_version_lib: false, @@ -514,15 +537,14 @@ cc_library { // libminadbd wants both, as it's used to build native tests. compile_multilib: "both", - // libadbd doesn't build any additional source, but to expose libadbd_core as a shared library. whole_static_libs: [ - "libadbconnection_server", "libadbd_core", - "libapp_processes_protos_lite", - "libprotobuf-cpp-lite", ], shared_libs: [ + "libadbconnection_server", + "libapp_processes_protos_lite", + "libprotobuf-cpp-lite", "libadb_crypto", "libadb_pairing_connection", "libadb_tls_connection", @@ -547,6 +569,7 @@ cc_library { }, static_libs: [ + "libbrotli", "libcutils_sockets", "libdiagnose_usb", "libmdnssd", @@ -562,6 +585,7 @@ cc_binary { defaults: ["adbd_defaults", "host_adbd_supported"], stl: "libc++_static", recovery_available: true, + apex_available: ["com.android.adbd"], srcs: [ "daemon/main.cpp", @@ -585,6 +609,7 @@ cc_binary { "libapp_processes_protos_lite", "libasyncio", "libbase", + "libbrotli", "libcap", "libcrypto_utils", "libcutils_sockets", diff --git a/adb/adb.cpp b/adb/adb.cpp index 98db19123..b5143685a 100644 --- a/adb/adb.cpp +++ b/adb/adb.cpp @@ -1016,8 +1016,12 @@ bool handle_forward_request(const char* service, if (kill_forward) { r = remove_listener(pieces[0].c_str(), transport); } else { - r = install_listener(pieces[0], pieces[1].c_str(), transport, no_rebind, - &resolved_tcp_port, &error); + int flags = 0; + if (no_rebind) { + flags |= INSTALL_LISTENER_NO_REBIND; + } + r = install_listener(pieces[0], pieces[1].c_str(), transport, flags, &resolved_tcp_port, + &error); } if (r == INSTALL_STATUS_OK) { #if ADB_HOST diff --git a/adb/adb_listeners.cpp b/adb/adb_listeners.cpp index 29909a555..43a925282 100644 --- a/adb/adb_listeners.cpp +++ b/adb/adb_listeners.cpp @@ -164,6 +164,15 @@ void remove_all_listeners() EXCLUDES(listener_list_mutex) { } } +void enable_daemon_sockets() EXCLUDES(listener_list_mutex) { + std::lock_guard<std::mutex> lock(listener_list_mutex); + for (auto& l : listener_list) { + if (l->connect_to == "*smartsocket*") { + fdevent_set(l->fde, FDE_READ); + } + } +} + void close_smartsockets() EXCLUDES(listener_list_mutex) { std::lock_guard<std::mutex> lock(listener_list_mutex); auto pred = [](const std::unique_ptr<alistener>& listener) { @@ -173,7 +182,7 @@ void close_smartsockets() EXCLUDES(listener_list_mutex) { } InstallStatus install_listener(const std::string& local_name, const char* connect_to, - atransport* transport, int no_rebind, int* resolved_tcp_port, + atransport* transport, int flags, int* resolved_tcp_port, std::string* error) EXCLUDES(listener_list_mutex) { std::lock_guard<std::mutex> lock(listener_list_mutex); for (auto& l : listener_list) { @@ -184,8 +193,8 @@ InstallStatus install_listener(const std::string& local_name, const char* connec return INSTALL_STATUS_INTERNAL_ERROR; } - // Can't repurpose a listener if 'no_rebind' is true. - if (no_rebind) { + // Can't repurpose a listener if INSTALL_LISTENER_NO_REBIND is set + if (flags & INSTALL_LISTENER_NO_REBIND) { *error = "cannot rebind"; return INSTALL_STATUS_CANNOT_REBIND; } @@ -222,7 +231,9 @@ InstallStatus install_listener(const std::string& local_name, const char* connec } else { listener->fde = fdevent_create(listener->fd, listener_event_func, listener.get()); } - fdevent_set(listener->fde, FDE_READ); + if ((flags & INSTALL_LISTENER_DISABLED) == 0) { + fdevent_set(listener->fde, FDE_READ); + } listener->transport = transport; diff --git a/adb/adb_listeners.h b/adb/adb_listeners.h index 70a2ee121..354dcc5c8 100644 --- a/adb/adb_listeners.h +++ b/adb/adb_listeners.h @@ -32,8 +32,11 @@ enum InstallStatus { INSTALL_STATUS_LISTENER_NOT_FOUND = -4, }; +inline constexpr int INSTALL_LISTENER_NO_REBIND = 1 << 0; +inline constexpr int INSTALL_LISTENER_DISABLED = 1 << 1; + InstallStatus install_listener(const std::string& local_name, const char* connect_to, - atransport* transport, int no_rebind, int* resolved_tcp_port, + atransport* transport, int flags, int* resolved_tcp_port, std::string* error); std::string format_listeners(); @@ -41,6 +44,7 @@ std::string format_listeners(); InstallStatus remove_listener(const char* local_name, atransport* transport); void remove_all_listeners(void); +void enable_daemon_sockets(); void close_smartsockets(); #endif /* __ADB_LISTENERS_H */ diff --git a/adb/brotli_utils.h b/adb/brotli_utils.h new file mode 100644 index 000000000..c5be73d0c --- /dev/null +++ b/adb/brotli_utils.h @@ -0,0 +1,144 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <span> + +#include <brotli/decode.h> +#include <brotli/encode.h> + +#include "types.h" + +enum class BrotliDecodeResult { + Error, + Done, + NeedInput, + MoreOutput, +}; + +struct BrotliDecoder { + explicit BrotliDecoder(std::span<char> output_buffer) + : output_buffer_(output_buffer), + decoder_(BrotliDecoderCreateInstance(nullptr, nullptr, nullptr), + BrotliDecoderDestroyInstance) {} + + void Append(Block&& block) { input_buffer_.append(std::move(block)); } + + BrotliDecodeResult Decode(std::span<char>* output) { + size_t available_in = input_buffer_.front_size(); + const uint8_t* next_in = reinterpret_cast<const uint8_t*>(input_buffer_.front_data()); + + size_t available_out = output_buffer_.size(); + uint8_t* next_out = reinterpret_cast<uint8_t*>(output_buffer_.data()); + + BrotliDecoderResult r = BrotliDecoderDecompressStream( + decoder_.get(), &available_in, &next_in, &available_out, &next_out, nullptr); + + size_t bytes_consumed = input_buffer_.front_size() - available_in; + input_buffer_.drop_front(bytes_consumed); + + size_t bytes_emitted = output_buffer_.size() - available_out; + *output = std::span<char>(output_buffer_.data(), bytes_emitted); + + switch (r) { + case BROTLI_DECODER_RESULT_SUCCESS: + return BrotliDecodeResult::Done; + case BROTLI_DECODER_RESULT_ERROR: + return BrotliDecodeResult::Error; + case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT: + // Brotli guarantees as one of its invariants that if it returns NEEDS_MORE_INPUT, + // it will consume the entire input buffer passed in, so we don't have to worry + // about bytes left over in the front block with more input remaining. + return BrotliDecodeResult::NeedInput; + case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT: + return BrotliDecodeResult::MoreOutput; + } + } + + private: + IOVector input_buffer_; + std::span<char> output_buffer_; + std::unique_ptr<BrotliDecoderState, void (*)(BrotliDecoderState*)> decoder_; +}; + +enum class BrotliEncodeResult { + Error, + Done, + NeedInput, + MoreOutput, +}; + +template <size_t OutputBlockSize> +struct BrotliEncoder { + explicit BrotliEncoder() + : output_block_(OutputBlockSize), + output_bytes_left_(OutputBlockSize), + encoder_(BrotliEncoderCreateInstance(nullptr, nullptr, nullptr), + BrotliEncoderDestroyInstance) { + BrotliEncoderSetParameter(encoder_.get(), BROTLI_PARAM_QUALITY, 1); + } + + void Append(Block input) { input_buffer_.append(std::move(input)); } + void Finish() { finished_ = true; } + + BrotliEncodeResult Encode(Block* output) { + output->clear(); + while (true) { + size_t available_in = input_buffer_.front_size(); + const uint8_t* next_in = reinterpret_cast<const uint8_t*>(input_buffer_.front_data()); + + size_t available_out = output_bytes_left_; + uint8_t* next_out = reinterpret_cast<uint8_t*>(output_block_.data() + + (OutputBlockSize - output_bytes_left_)); + + BrotliEncoderOperation op = BROTLI_OPERATION_PROCESS; + if (finished_) { + op = BROTLI_OPERATION_FINISH; + } + + if (!BrotliEncoderCompressStream(encoder_.get(), op, &available_in, &next_in, + &available_out, &next_out, nullptr)) { + return BrotliEncodeResult::Error; + } + + size_t bytes_consumed = input_buffer_.front_size() - available_in; + input_buffer_.drop_front(bytes_consumed); + + output_bytes_left_ = available_out; + + if (BrotliEncoderIsFinished(encoder_.get())) { + output_block_.resize(OutputBlockSize - output_bytes_left_); + *output = std::move(output_block_); + return BrotliEncodeResult::Done; + } else if (output_bytes_left_ == 0) { + *output = std::move(output_block_); + output_block_.resize(OutputBlockSize); + output_bytes_left_ = OutputBlockSize; + return BrotliEncodeResult::MoreOutput; + } else if (input_buffer_.empty()) { + return BrotliEncodeResult::NeedInput; + } + } + } + + private: + bool finished_ = false; + IOVector input_buffer_; + Block output_block_; + size_t output_bytes_left_; + std::unique_ptr<BrotliEncoderState, void (*)(BrotliEncoderState*)> encoder_; +}; diff --git a/adb/client/adb_install.cpp b/adb/client/adb_install.cpp index 21b8f4999..e4d010c71 100644 --- a/adb/client/adb_install.cpp +++ b/adb/client/adb_install.cpp @@ -22,11 +22,12 @@ #include <stdlib.h> #include <unistd.h> #include <algorithm> -#include <iostream> #include <string> +#include <string_view> #include <vector> #include <android-base/file.h> +#include <android-base/parsebool.h> #include <android-base/stringprintf.h> #include <android-base/strings.h> @@ -39,8 +40,9 @@ #include "fastdeploy.h" #include "incremental.h" +using namespace std::literals; + static constexpr int kFastDeployMinApi = 24; -static constexpr int kIncrementalMinApi = 29; namespace { @@ -50,6 +52,8 @@ enum InstallMode { INSTALL_STREAM, INSTALL_INCREMENTAL, }; + +enum class CmdlineOption { None, Enable, Disable }; } static bool can_use_feature(const char* feature) { @@ -286,7 +290,7 @@ static int install_app_legacy(int argc, const char** argv, bool use_fastdeploy) } } - if (do_sync_push(apk_file, apk_dest.c_str(), false)) { + if (do_sync_push(apk_file, apk_dest.c_str(), false, true)) { result = pm_command(argc, argv); delete_device_file(apk_dest); } @@ -299,28 +303,23 @@ static int msBetween(TimePoint start, TimePoint end) { return std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count(); } -static int install_app_incremental(int argc, const char** argv) { - printf("Performing Incremental Install\n"); +static int install_app_incremental(int argc, const char** argv, bool wait, bool silent) { using clock = std::chrono::high_resolution_clock; const auto start = clock::now(); int first_apk = -1; int last_apk = -1; - std::string cert_path; - bool wait = false; - std::vector<std::string_view> args = {"package"}; + std::vector<std::string_view> args = {"package"sv}; for (int i = 0; i < argc; ++i) { const auto arg = std::string_view(argv[i]); - if (android::base::EndsWithIgnoreCase(arg, ".apk")) { + if (android::base::EndsWithIgnoreCase(arg, ".apk"sv)) { last_apk = i; if (first_apk == -1) { first_apk = i; } - } else if (arg == "--wait") { - wait = true; - } else if (arg.starts_with("install-")) { + } else if (arg.starts_with("install-"sv)) { // incremental installation command on the device is the same for all its variations in // the adb, e.g. install-multiple or install-multi-package - args.push_back("install"); + args.push_back("install"sv); } else { args.push_back(arg); } @@ -328,16 +327,23 @@ static int install_app_incremental(int argc, const char** argv) { if (first_apk == -1) error_exit("Need at least one APK file on command line"); - const auto afterApk = clock::now(); + auto files = incremental::Files{argv + first_apk, argv + last_apk + 1}; + if (silent) { + // For a silent installation we want to do the lightweight check first and bail early and + // quietly if it fails. + if (!incremental::can_install(files)) { + return -1; + } + } - auto server_process = incremental::install({argv + first_apk, argv + last_apk + 1}); + printf("Performing Incremental Install\n"); + auto server_process = incremental::install(files, silent); if (!server_process) { return -1; } const auto end = clock::now(); - printf("Install command complete (ms: %d total, %d apk prep, %d install)\n", - msBetween(start, end), msBetween(start, afterApk), msBetween(afterApk, end)); + printf("Install command complete in %d ms\n", msBetween(start, end)); if (wait) { (*server_process).wait(); @@ -346,66 +352,134 @@ static int install_app_incremental(int argc, const char** argv) { return 0; } +static std::pair<InstallMode, std::optional<InstallMode>> calculateInstallMode( + InstallMode modeFromArgs, bool fastdeploy, CmdlineOption incrementalRequest) { + if (incrementalRequest == CmdlineOption::Enable) { + if (fastdeploy) { + error_exit( + "--incremental and --fast-deploy options are incompatible. " + "Please choose one"); + } + } + + if (modeFromArgs != INSTALL_DEFAULT) { + if (incrementalRequest == CmdlineOption::Enable) { + error_exit("--incremental is not compatible with other installation modes"); + } + return {modeFromArgs, std::nullopt}; + } + + if (incrementalRequest != CmdlineOption::Disable && !is_abb_exec_supported()) { + if (incrementalRequest == CmdlineOption::None) { + incrementalRequest = CmdlineOption::Disable; + } else { + error_exit("Device doesn't support incremental installations"); + } + } + if (incrementalRequest == CmdlineOption::None) { + // check if the host is ok with incremental by default + if (const char* incrementalFromEnv = getenv("ADB_INSTALL_DEFAULT_INCREMENTAL")) { + using namespace android::base; + auto val = ParseBool(incrementalFromEnv); + if (val == ParseBoolResult::kFalse) { + incrementalRequest = CmdlineOption::Disable; + } + } + } + if (incrementalRequest == CmdlineOption::None) { + // still ok: let's see if the device allows using incremental by default + // it starts feeling like we're looking for an excuse to not to use incremental... + std::string error; + std::vector<std::string> args = {"settings", "get", + "enable_adb_incremental_install_default"}; + auto fd = send_abb_exec_command(args, &error); + if (!fd.ok()) { + fprintf(stderr, "adb: retrieving the default device installation mode failed: %s", + error.c_str()); + } else { + char buf[BUFSIZ] = {}; + read_status_line(fd.get(), buf, sizeof(buf)); + using namespace android::base; + auto val = ParseBool(buf); + if (val == ParseBoolResult::kFalse) { + incrementalRequest = CmdlineOption::Disable; + } + } + } + + if (incrementalRequest == CmdlineOption::Enable) { + // explicitly requested - no fallback + return {INSTALL_INCREMENTAL, std::nullopt}; + } + const auto bestMode = best_install_mode(); + if (incrementalRequest == CmdlineOption::None) { + // no opinion - use incremental, fallback to regular on a failure. + return {INSTALL_INCREMENTAL, bestMode}; + } + // incremental turned off - use the regular best mode without a fallback. + return {bestMode, std::nullopt}; +} + int install_app(int argc, const char** argv) { std::vector<int> processedArgIndices; InstallMode installMode = INSTALL_DEFAULT; bool use_fastdeploy = false; bool is_reinstall = false; + bool wait = false; + auto incremental_request = CmdlineOption::None; FastDeploy_AgentUpdateStrategy agent_update_strategy = FastDeploy_AgentUpdateDifferentVersion; for (int i = 1; i < argc; i++) { - if (!strcmp(argv[i], "--streaming")) { + if (argv[i] == "--streaming"sv) { processedArgIndices.push_back(i); installMode = INSTALL_STREAM; - } else if (!strcmp(argv[i], "--no-streaming")) { + } else if (argv[i] == "--no-streaming"sv) { processedArgIndices.push_back(i); installMode = INSTALL_PUSH; - } else if (!strcmp(argv[i], "-r")) { + } else if (argv[i] == "-r"sv) { // Note that this argument is not added to processedArgIndices because it // must be passed through to pm is_reinstall = true; - } else if (!strcmp(argv[i], "--fastdeploy")) { + } else if (argv[i] == "--fastdeploy"sv) { processedArgIndices.push_back(i); use_fastdeploy = true; - } else if (!strcmp(argv[i], "--no-fastdeploy")) { + } else if (argv[i] == "--no-fastdeploy"sv) { processedArgIndices.push_back(i); use_fastdeploy = false; - } else if (!strcmp(argv[i], "--force-agent")) { + } else if (argv[i] == "--force-agent"sv) { processedArgIndices.push_back(i); agent_update_strategy = FastDeploy_AgentUpdateAlways; - } else if (!strcmp(argv[i], "--date-check-agent")) { + } else if (argv[i] == "--date-check-agent"sv) { processedArgIndices.push_back(i); agent_update_strategy = FastDeploy_AgentUpdateNewerTimeStamp; - } else if (!strcmp(argv[i], "--version-check-agent")) { + } else if (argv[i] == "--version-check-agent"sv) { processedArgIndices.push_back(i); agent_update_strategy = FastDeploy_AgentUpdateDifferentVersion; - } else if (!strcmp(argv[i], "--incremental")) { + } else if (strlen(argv[i]) >= "--incr"sv.size() && "--incremental"sv.starts_with(argv[i])) { processedArgIndices.push_back(i); - installMode = INSTALL_INCREMENTAL; - } else if (!strcmp(argv[i], "--no-incremental")) { + incremental_request = CmdlineOption::Enable; + } else if (strlen(argv[i]) >= "--no-incr"sv.size() && + "--no-incremental"sv.starts_with(argv[i])) { processedArgIndices.push_back(i); - installMode = INSTALL_DEFAULT; - } - } - - if (installMode == INSTALL_INCREMENTAL) { - if (get_device_api_level() < kIncrementalMinApi || !is_abb_exec_supported()) { - error_exit("Attempting to use incremental install on unsupported device"); + incremental_request = CmdlineOption::Disable; + } else if (argv[i] == "--wait"sv) { + processedArgIndices.push_back(i); + wait = true; } } - if (installMode == INSTALL_DEFAULT) { - installMode = best_install_mode(); - } - - if (installMode == INSTALL_STREAM && best_install_mode() == INSTALL_PUSH) { + auto [primaryMode, fallbackMode] = + calculateInstallMode(installMode, use_fastdeploy, incremental_request); + if ((primaryMode == INSTALL_STREAM || fallbackMode.value_or(INSTALL_PUSH) == INSTALL_STREAM) && + best_install_mode() == INSTALL_PUSH) { error_exit("Attempting to use streaming install on unsupported device"); } if (use_fastdeploy && get_device_api_level() < kFastDeployMinApi) { - printf("Fast Deploy is only compatible with devices of API version %d or higher, " - "ignoring.\n", - kFastDeployMinApi); + fprintf(stderr, + "Fast Deploy is only compatible with devices of API version %d or higher, " + "ignoring.\n", + kFastDeployMinApi); use_fastdeploy = false; } fastdeploy_set_agent_update_strategy(agent_update_strategy); @@ -421,19 +495,27 @@ int install_app(int argc, const char** argv) { error_exit("install requires an apk argument"); } - switch (installMode) { - case INSTALL_PUSH: - return install_app_legacy(passthrough_argv.size(), passthrough_argv.data(), - use_fastdeploy); - case INSTALL_STREAM: - return install_app_streamed(passthrough_argv.size(), passthrough_argv.data(), - use_fastdeploy); - case INSTALL_INCREMENTAL: - return install_app_incremental(passthrough_argv.size(), passthrough_argv.data()); - case INSTALL_DEFAULT: - default: - return 1; - } + auto runInstallMode = [&](InstallMode installMode, bool silent) { + switch (installMode) { + case INSTALL_PUSH: + return install_app_legacy(passthrough_argv.size(), passthrough_argv.data(), + use_fastdeploy); + case INSTALL_STREAM: + return install_app_streamed(passthrough_argv.size(), passthrough_argv.data(), + use_fastdeploy); + case INSTALL_INCREMENTAL: + return install_app_incremental(passthrough_argv.size(), passthrough_argv.data(), + wait, silent); + case INSTALL_DEFAULT: + default: + return 1; + } + }; + auto res = runInstallMode(primaryMode, fallbackMode.has_value()); + if (res && fallbackMode.value_or(primaryMode) != primaryMode) { + res = runInstallMode(*fallbackMode, false); + } + return res; } int install_multiple_app(int argc, const char** argv) { diff --git a/adb/client/bugreport.cpp b/adb/client/bugreport.cpp index 8ca44e8b5..ab93f7d87 100644 --- a/adb/client/bugreport.cpp +++ b/adb/client/bugreport.cpp @@ -282,5 +282,5 @@ int Bugreport::SendShellCommand(const std::string& command, bool disable_shell_p bool Bugreport::DoSyncPull(const std::vector<const char*>& srcs, const char* dst, bool copy_attrs, const char* name) { - return do_sync_pull(srcs, dst, copy_attrs, name); + return do_sync_pull(srcs, dst, copy_attrs, false, name); } diff --git a/adb/client/commandline.cpp b/adb/client/commandline.cpp index 7c7da0873..04b250df2 100644 --- a/adb/client/commandline.cpp +++ b/adb/client/commandline.cpp @@ -129,15 +129,21 @@ static void help() { " reverse --remove-all remove all reverse socket connections from device\n" "\n" "file transfer:\n" - " push [--sync] LOCAL... REMOTE\n" + " push [--sync] [-zZ] LOCAL... REMOTE\n" " copy local files/directories to device\n" " --sync: only push files that are newer on the host than the device\n" - " pull [-a] REMOTE... LOCAL\n" + " -z: enable compression\n" + " -Z: disable compression\n" + " pull [-azZ] REMOTE... LOCAL\n" " copy files/dirs from device\n" " -a: preserve file timestamp and mode\n" - " sync [all|data|odm|oem|product|system|system_ext|vendor]\n" + " -z: enable compression\n" + " -Z: disable compression\n" + " sync [-lzZ] [all|data|odm|oem|product|system|system_ext|vendor]\n" " sync a local build from $ANDROID_PRODUCT_OUT to the device (default all)\n" " -l: list files that would be copied, but don't copy them\n" + " -z: enable compression\n" + " -Z: disable compression\n" "\n" "shell:\n" " shell [-e ESCAPE] [-n] [-Tt] [-x] [COMMAND...]\n" @@ -1309,8 +1315,12 @@ static int restore(int argc, const char** argv) { } static void parse_push_pull_args(const char** arg, int narg, std::vector<const char*>* srcs, - const char** dst, bool* copy_attrs, bool* sync) { + const char** dst, bool* copy_attrs, bool* sync, bool* compressed) { *copy_attrs = false; + const char* adb_compression = getenv("ADB_COMPRESSION"); + if (adb_compression && strcmp(adb_compression, "0") == 0) { + *compressed = false; + } srcs->clear(); bool ignore_flags = false; @@ -1322,6 +1332,14 @@ static void parse_push_pull_args(const char** arg, int narg, std::vector<const c // Silently ignore for backwards compatibility. } else if (!strcmp(*arg, "-a")) { *copy_attrs = true; + } else if (!strcmp(*arg, "-z")) { + if (compressed != nullptr) { + *compressed = true; + } + } else if (!strcmp(*arg, "-Z")) { + if (compressed != nullptr) { + *compressed = false; + } } else if (!strcmp(*arg, "--sync")) { if (sync != nullptr) { *sync = true; @@ -1443,6 +1461,26 @@ static bool _is_valid_ack_reply_fd(const int ack_reply_fd) { #endif } +static bool _is_valid_os_fd(int fd) { + // Disallow invalid FDs and stdin/out/err as well. + if (fd < 3) { + return false; + } +#ifdef _WIN32 + auto handle = (HANDLE)fd; + DWORD info = 0; + if (GetHandleInformation(handle, &info) == 0) { + return false; + } +#else + int flags = fcntl(fd, F_GETFD); + if (flags == -1) { + return false; + } +#endif + return true; +} + int adb_commandline(int argc, const char** argv) { bool no_daemon = false; bool is_daemon = false; @@ -1856,20 +1894,22 @@ int adb_commandline(int argc, const char** argv) { } else if (!strcmp(argv[0], "push")) { bool copy_attrs = false; bool sync = false; + bool compressed = true; std::vector<const char*> srcs; const char* dst = nullptr; - parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, ©_attrs, &sync); + parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, ©_attrs, &sync, &compressed); if (srcs.empty() || !dst) error_exit("push requires an argument"); - return do_sync_push(srcs, dst, sync) ? 0 : 1; + return do_sync_push(srcs, dst, sync, compressed) ? 0 : 1; } else if (!strcmp(argv[0], "pull")) { bool copy_attrs = false; + bool compressed = true; std::vector<const char*> srcs; const char* dst = "."; - parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, ©_attrs, nullptr); + parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, ©_attrs, nullptr, &compressed); if (srcs.empty()) error_exit("pull requires an argument"); - return do_sync_pull(srcs, dst, copy_attrs) ? 0 : 1; + return do_sync_pull(srcs, dst, copy_attrs, compressed) ? 0 : 1; } else if (!strcmp(argv[0], "install")) { if (argc < 2) error_exit("install requires an argument"); return install_app(argc, argv); @@ -1885,18 +1925,38 @@ int adb_commandline(int argc, const char** argv) { } else if (!strcmp(argv[0], "sync")) { std::string src; bool list_only = false; - if (argc < 2) { - // No partition specified: sync all of them. - } else if (argc >= 2 && strcmp(argv[1], "-l") == 0) { - list_only = true; - if (argc == 3) src = argv[2]; - } else if (argc == 2) { - src = argv[1]; + bool compressed = true; + + const char* adb_compression = getenv("ADB_COMPRESSION"); + if (adb_compression && strcmp(adb_compression, "0") == 0) { + compressed = false; + } + + int opt; + while ((opt = getopt(argc, const_cast<char**>(argv), "lzZ")) != -1) { + switch (opt) { + case 'l': + list_only = true; + break; + case 'z': + compressed = true; + break; + case 'Z': + compressed = false; + break; + default: + error_exit("usage: adb sync [-lzZ] [PARTITION]"); + } + } + + if (optind == argc) { + src = "all"; + } else if (optind + 1 == argc) { + src = argv[optind]; } else { - error_exit("usage: adb sync [-l] [PARTITION]"); + error_exit("usage: adb sync [-lzZ] [PARTITION]"); } - if (src.empty()) src = "all"; std::vector<std::string> partitions{"data", "odm", "oem", "product", "system", "system_ext", "vendor"}; bool found = false; @@ -1905,7 +1965,7 @@ int adb_commandline(int argc, const char** argv) { std::string src_dir{product_file(partition)}; if (!directory_exists(src_dir)) continue; found = true; - if (!do_sync_sync(src_dir, "/" + partition, list_only)) return 1; + if (!do_sync_sync(src_dir, "/" + partition, list_only, compressed)) return 1; } } if (!found) error_exit("don't know how to sync %s partition", src.c_str()); @@ -2009,17 +2069,28 @@ int adb_commandline(int argc, const char** argv) { } } } else if (!strcmp(argv[0], "inc-server")) { - if (argc < 3) { - error_exit("usage: adb inc-server FD FILE1 FILE2 ..."); + if (argc < 4) { +#ifdef _WIN32 + error_exit("usage: adb inc-server CONNECTION_HANDLE OUTPUT_HANDLE FILE1 FILE2 ..."); +#else + error_exit("usage: adb inc-server CONNECTION_FD OUTPUT_FD FILE1 FILE2 ..."); +#endif + } + int connection_fd = atoi(argv[1]); + if (!_is_valid_os_fd(connection_fd)) { + error_exit("Invalid connection_fd number given: %d", connection_fd); } - int fd = atoi(argv[1]); - if (fd < 3) { - // Disallow invalid FDs and stdin/out/err as well. - error_exit("Invalid fd number given: %d", fd); + + connection_fd = adb_register_socket(connection_fd); + close_on_exec(connection_fd); + + int output_fd = atoi(argv[2]); + if (!_is_valid_os_fd(output_fd)) { + error_exit("Invalid output_fd number given: %d", output_fd); } - fd = adb_register_socket(fd); - close_on_exec(fd); - return incremental::serve(fd, argc - 2, argv + 2); + output_fd = adb_register_socket(output_fd); + close_on_exec(output_fd); + return incremental::serve(connection_fd, output_fd, argc - 3, argv + 3); } error_exit("unknown command %s", argv[0]); diff --git a/adb/client/fastdeploy.cpp b/adb/client/fastdeploy.cpp index c5fc12f0a..de82e14e5 100644 --- a/adb/client/fastdeploy.cpp +++ b/adb/client/fastdeploy.cpp @@ -112,7 +112,7 @@ static void push_to_device(const void* data, size_t byte_count, const char* dst, // but can't be removed until after the push. unix_close(tf.release()); - if (!do_sync_push(srcs, dst, sync)) { + if (!do_sync_push(srcs, dst, sync, true)) { error_exit("Failed to push fastdeploy agent to device."); } } diff --git a/adb/client/file_sync_client.cpp b/adb/client/file_sync_client.cpp index 04ad5363d..08444281d 100644 --- a/adb/client/file_sync_client.cpp +++ b/adb/client/file_sync_client.cpp @@ -42,6 +42,7 @@ #include "adb_client.h" #include "adb_io.h" #include "adb_utils.h" +#include "brotli_utils.h" #include "file_sync_protocol.h" #include "line_printer.h" #include "sysdeps/errno.h" @@ -53,6 +54,8 @@ #include <android-base/strings.h> #include <android-base/stringprintf.h> +using namespace std::literals; + typedef void(sync_ls_cb)(unsigned mode, uint64_t size, uint64_t time, const char* name); struct syncsendbuf { @@ -113,6 +116,11 @@ struct TransferLedger { uint64_t bytes_expected; bool expect_multiple_files; + private: + std::string last_progress_str; + std::chrono::steady_clock::time_point last_progress_time; + + public: TransferLedger() { Reset(); } @@ -132,6 +140,8 @@ struct TransferLedger { files_skipped = 0; bytes_transferred = 0; bytes_expected = 0; + last_progress_str.clear(); + last_progress_time = {}; } std::string TransferRate() { @@ -151,6 +161,12 @@ struct TransferLedger { void ReportProgress(LinePrinter& lp, const std::string& file, uint64_t file_copied_bytes, uint64_t file_total_bytes) { + static constexpr auto kProgressReportInterval = 100ms; + + auto now = std::chrono::steady_clock::now(); + if (now < last_progress_time + kProgressReportInterval) { + return; + } char overall_percentage_str[5] = "?"; if (bytes_expected != 0 && bytes_transferred <= bytes_expected) { int overall_percentage = static_cast<int>(bytes_transferred * 100 / bytes_expected); @@ -181,7 +197,11 @@ struct TransferLedger { android::base::StringPrintf("[%4s] %s", overall_percentage_str, file.c_str()); } } - lp.Print(output, LinePrinter::LineType::INFO); + if (output != last_progress_str) { + lp.Print(output, LinePrinter::LineType::INFO); + last_progress_str = std::move(output); + last_progress_time = now; + } } void ReportTransferRate(LinePrinter& lp, const std::string& name, TransferDirection direction) { @@ -204,7 +224,8 @@ struct TransferLedger { class SyncConnection { public: - SyncConnection() { + SyncConnection() : acknowledgement_buffer_(sizeof(sync_status) + SYNC_DATA_MAX) { + acknowledgement_buffer_.resize(0); max = SYNC_DATA_MAX; // TODO: decide at runtime. std::string error; @@ -213,6 +234,8 @@ class SyncConnection { } else { have_stat_v2_ = CanUseFeature(features_, kFeatureStat2); have_ls_v2_ = CanUseFeature(features_, kFeatureLs2); + have_sendrecv_v2_ = CanUseFeature(features_, kFeatureSendRecv2); + have_sendrecv_v2_brotli_ = CanUseFeature(features_, kFeatureSendRecv2Brotli); fd.reset(adb_connect("sync:", &error)); if (fd < 0) { Error("connect failed: %s", error.c_str()); @@ -236,6 +259,9 @@ class SyncConnection { line_printer_.KeepInfoLine(); } + bool HaveSendRecv2() const { return have_sendrecv_v2_; } + bool HaveSendRecv2Brotli() const { return have_sendrecv_v2_brotli_; } + const FeatureSet& Features() const { return features_; } bool IsValid() { return fd >= 0; } @@ -294,6 +320,62 @@ class SyncConnection { req->path_length = path.length(); char* data = reinterpret_cast<char*>(req + 1); memcpy(data, path.data(), path.length()); + return WriteFdExactly(fd, buf.data(), buf.size()); + } + + bool SendSend2(std::string_view path, mode_t mode, bool compressed) { + if (path.length() > 1024) { + Error("SendRequest failed: path too long: %zu", path.length()); + errno = ENAMETOOLONG; + return false; + } + + Block buf; + + SyncRequest req; + req.id = ID_SEND_V2; + req.path_length = path.length(); + + syncmsg msg; + msg.send_v2_setup.id = ID_SEND_V2; + msg.send_v2_setup.mode = mode; + msg.send_v2_setup.flags = compressed ? kSyncFlagBrotli : kSyncFlagNone; + + buf.resize(sizeof(SyncRequest) + path.length() + sizeof(msg.send_v2_setup)); + + void* p = buf.data(); + + p = mempcpy(p, &req, sizeof(SyncRequest)); + p = mempcpy(p, path.data(), path.length()); + p = mempcpy(p, &msg.send_v2_setup, sizeof(msg.send_v2_setup)); + + return WriteFdExactly(fd, buf.data(), buf.size()); + } + + bool SendRecv2(const std::string& path) { + if (path.length() > 1024) { + Error("SendRequest failed: path too long: %zu", path.length()); + errno = ENAMETOOLONG; + return false; + } + + Block buf; + + SyncRequest req; + req.id = ID_RECV_V2; + req.path_length = path.length(); + + syncmsg msg; + msg.recv_v2_setup.id = ID_RECV_V2; + msg.recv_v2_setup.flags = kSyncFlagBrotli; + + buf.resize(sizeof(SyncRequest) + path.length() + sizeof(msg.recv_v2_setup)); + + void* p = buf.data(); + + p = mempcpy(p, &req, sizeof(SyncRequest)); + p = mempcpy(p, path.data(), path.length()); + p = mempcpy(p, &msg.recv_v2_setup, sizeof(msg.recv_v2_setup)); return WriteFdExactly(fd, buf.data(), buf.size()); } @@ -350,8 +432,8 @@ class SyncConnection { } if (msg.stat_v1.id != ID_LSTAT_V1) { - PLOG(FATAL) << "protocol fault: stat response has wrong message id: " - << msg.stat_v1.id; + LOG(FATAL) << "protocol fault: stat response has wrong message id: " + << msg.stat_v1.id; } if (msg.stat_v1.mode == 0 && msg.stat_v1.size == 0 && msg.stat_v1.mtime == 0) { @@ -425,7 +507,7 @@ class SyncConnection { char* p = &buf[0]; SyncRequest* req_send = reinterpret_cast<SyncRequest*>(p); - req_send->id = ID_SEND; + req_send->id = ID_SEND_V1; req_send->path_length = path_and_mode.length(); p += sizeof(SyncRequest); memcpy(p, path_and_mode.data(), path_and_mode.size()); @@ -451,11 +533,92 @@ class SyncConnection { return true; } + bool SendLargeFileCompressed(const std::string& path, mode_t mode, const std::string& lpath, + const std::string& rpath, unsigned mtime) { + if (!SendSend2(path, mode, true)) { + Error("failed to send ID_SEND_V2 message '%s': %s", path.c_str(), strerror(errno)); + return false; + } + + struct stat st; + if (stat(lpath.c_str(), &st) == -1) { + Error("cannot stat '%s': %s", lpath.c_str(), strerror(errno)); + return false; + } + + uint64_t total_size = st.st_size; + uint64_t bytes_copied = 0; + + unique_fd lfd(adb_open(lpath.c_str(), O_RDONLY | O_CLOEXEC)); + if (lfd < 0) { + Error("opening '%s' locally failed: %s", lpath.c_str(), strerror(errno)); + return false; + } + + syncsendbuf sbuf; + sbuf.id = ID_DATA; + + BrotliEncoder<SYNC_DATA_MAX> encoder; + bool sending = true; + while (sending) { + Block input(SYNC_DATA_MAX); + int r = adb_read(lfd.get(), input.data(), input.size()); + if (r < 0) { + Error("reading '%s' locally failed: %s", lpath.c_str(), strerror(errno)); + return false; + } + + if (r == 0) { + encoder.Finish(); + } else { + input.resize(r); + encoder.Append(std::move(input)); + RecordBytesTransferred(r); + bytes_copied += r; + ReportProgress(rpath, bytes_copied, total_size); + } + + while (true) { + Block output; + BrotliEncodeResult result = encoder.Encode(&output); + if (result == BrotliEncodeResult::Error) { + Error("compressing '%s' locally failed", lpath.c_str()); + return false; + } + + if (!output.empty()) { + sbuf.size = output.size(); + memcpy(sbuf.data, output.data(), output.size()); + WriteOrDie(lpath, rpath, &sbuf, sizeof(SyncRequest) + output.size()); + } + + if (result == BrotliEncodeResult::Done) { + sending = false; + break; + } else if (result == BrotliEncodeResult::NeedInput) { + break; + } else if (result == BrotliEncodeResult::MoreOutput) { + continue; + } + } + } + + syncmsg msg; + msg.data.id = ID_DONE; + msg.data.size = mtime; + RecordFileSent(lpath, rpath); + return WriteOrDie(lpath, rpath, &msg.data, sizeof(msg.data)); + } + bool SendLargeFile(const std::string& path, mode_t mode, const std::string& lpath, - const std::string& rpath, unsigned mtime) { + const std::string& rpath, unsigned mtime, bool compressed) { + if (compressed && HaveSendRecv2Brotli()) { + return SendLargeFileCompressed(path, mode, lpath, rpath, mtime); + } + std::string path_and_mode = android::base::StringPrintf("%s,%d", path.c_str(), mode); - if (!SendRequest(ID_SEND, path_and_mode)) { - Error("failed to send ID_SEND message '%s': %s", path_and_mode.c_str(), + if (!SendRequest(ID_SEND_V1, path_and_mode)) { + Error("failed to send ID_SEND_V1 message '%s': %s", path_and_mode.c_str(), strerror(errno)); return false; } @@ -469,7 +632,7 @@ class SyncConnection { uint64_t total_size = st.st_size; uint64_t bytes_copied = 0; - unique_fd lfd(adb_open(lpath.c_str(), O_RDONLY)); + unique_fd lfd(adb_open(lpath.c_str(), O_RDONLY | O_CLOEXEC)); if (lfd < 0) { Error("opening '%s' locally failed: %s", lpath.c_str(), strerror(errno)); return false; @@ -477,8 +640,9 @@ class SyncConnection { syncsendbuf sbuf; sbuf.id = ID_DATA; + while (true) { - int bytes_read = adb_read(lfd, sbuf.data, max - sizeof(SyncRequest)); + int bytes_read = adb_read(lfd, sbuf.data, max); if (bytes_read == -1) { Error("reading '%s' locally failed: %s", lpath.c_str(), strerror(errno)); return false; @@ -491,7 +655,6 @@ class SyncConnection { RecordBytesTransferred(bytes_read); bytes_copied += bytes_read; - ReportProgress(rpath, bytes_copied, total_size); } @@ -502,34 +665,6 @@ class SyncConnection { return WriteOrDie(lpath, rpath, &msg.data, sizeof(msg.data)); } - bool ReadAcknowledgments() { - bool result = true; - while (!deferred_acknowledgements_.empty()) { - auto [from, to] = std::move(deferred_acknowledgements_.front()); - deferred_acknowledgements_.pop_front(); - result &= CopyDone(from, to); - } - return result; - } - - bool CopyDone(const std::string& from, const std::string& to) { - syncmsg msg; - if (!ReadFdExactly(fd, &msg.status, sizeof(msg.status))) { - Error("failed to copy '%s' to '%s': couldn't read from device", from.c_str(), - to.c_str()); - return false; - } - if (msg.status.id == ID_OKAY) { - return true; - } - if (msg.status.id != ID_FAIL) { - Error("failed to copy '%s' to '%s': unknown reason %d", from.c_str(), to.c_str(), - msg.status.id); - return false; - } - return ReportCopyFailure(from, to, msg); - } - bool ReportCopyFailure(const std::string& from, const std::string& to, const syncmsg& msg) { std::vector<char> buf(msg.status.msglen + 1); if (!ReadFdExactly(fd, &buf[0], msg.status.msglen)) { @@ -542,6 +677,97 @@ class SyncConnection { return false; } + void CopyDone() { deferred_acknowledgements_.pop_front(); } + + void ReportDeferredCopyFailure(const std::string& msg) { + auto& [from, to] = deferred_acknowledgements_.front(); + Error("failed to copy '%s' to '%s': remote %s", from.c_str(), to.c_str(), msg.c_str()); + deferred_acknowledgements_.pop_front(); + } + + bool ReadAcknowledgements(bool read_all = false) { + // We need to read enough such that adbd's intermediate socket's write buffer can't be + // full. The default buffer on Linux is 212992 bytes, but there's 576 bytes of bookkeeping + // overhead per write. The worst case scenario is a continuous string of failures, since + // each logical packet is divided into two writes. If our packet size if conservatively 512 + // bytes long, this leaves us with space for 128 responses. + constexpr size_t max_deferred_acks = 128; + auto& buf = acknowledgement_buffer_; + adb_pollfd pfd = {.fd = fd.get(), .events = POLLIN}; + while (!deferred_acknowledgements_.empty()) { + bool should_block = read_all || deferred_acknowledgements_.size() >= max_deferred_acks; + + ssize_t rc = adb_poll(&pfd, 1, should_block ? -1 : 0); + if (rc == 0) { + CHECK(!should_block); + return true; + } + + if (acknowledgement_buffer_.size() < sizeof(sync_status)) { + const ssize_t header_bytes_left = sizeof(sync_status) - buf.size(); + ssize_t rc = adb_read(fd, buf.end(), header_bytes_left); + if (rc <= 0) { + Error("failed to read copy response"); + return false; + } + + buf.resize(buf.size() + rc); + if (rc != header_bytes_left) { + // Early exit if we run out of data in the socket. + return true; + } + + if (!should_block) { + // We don't want to read again yet, because the socket might be empty. + continue; + } + } + + auto* hdr = reinterpret_cast<sync_status*>(buf.data()); + if (hdr->id == ID_OKAY) { + buf.resize(0); + if (hdr->msglen != 0) { + Error("received ID_OKAY with msg_len (%" PRIu32 " != 0", hdr->msglen); + return false; + } + CopyDone(); + continue; + } else if (hdr->id != ID_FAIL) { + Error("unexpected response from daemon: id = %#" PRIx32, hdr->id); + return false; + } else if (hdr->msglen > SYNC_DATA_MAX) { + Error("too-long message length from daemon: msglen = %" PRIu32, hdr->msglen); + return false; + } + + const ssize_t msg_bytes_left = hdr->msglen + sizeof(sync_status) - buf.size(); + CHECK_GE(msg_bytes_left, 0); + if (msg_bytes_left > 0) { + ssize_t rc = adb_read(fd, buf.end(), msg_bytes_left); + if (rc <= 0) { + Error("failed to read copy failure message"); + return false; + } + + buf.resize(buf.size() + rc); + if (rc != msg_bytes_left) { + if (should_block) { + continue; + } else { + return true; + } + } + + std::string msg(buf.begin() + sizeof(sync_status), buf.end()); + ReportDeferredCopyFailure(msg); + buf.resize(0); + return false; + } + } + + return true; + } + void Printf(const char* fmt, ...) __attribute__((__format__(__printf__, 2, 3))) { std::string s; @@ -608,9 +834,12 @@ class SyncConnection { private: std::deque<std::pair<std::string, std::string>> deferred_acknowledgements_; + Block acknowledgement_buffer_; FeatureSet features_; bool have_stat_v2_; bool have_ls_v2_; + bool have_sendrecv_v2_; + bool have_sendrecv_v2_brotli_; TransferLedger global_ledger_; TransferLedger current_ledger_; @@ -692,7 +921,7 @@ static bool sync_stat_fallback(SyncConnection& sc, const std::string& path, stru } static bool sync_send(SyncConnection& sc, const std::string& lpath, const std::string& rpath, - unsigned mtime, mode_t mode, bool sync) { + unsigned mtime, mode_t mode, bool sync, bool compressed) { if (sync) { struct stat st; if (sync_lstat(sc, rpath, &st)) { @@ -716,7 +945,7 @@ static bool sync_send(SyncConnection& sc, const std::string& lpath, const std::s if (!sc.SendSmallFile(rpath, mode, lpath, rpath, mtime, buf, data_length)) { return false; } - return true; + return sc.ReadAcknowledgements(); #endif } @@ -735,16 +964,16 @@ static bool sync_send(SyncConnection& sc, const std::string& lpath, const std::s return false; } } else { - if (!sc.SendLargeFile(rpath, mode, lpath, rpath, mtime)) { + if (!sc.SendLargeFile(rpath, mode, lpath, rpath, mtime, compressed)) { return false; } } - return true; + return sc.ReadAcknowledgements(); } -static bool sync_recv(SyncConnection& sc, const char* rpath, const char* lpath, - const char* name, uint64_t expected_size) { - if (!sc.SendRequest(ID_RECV, rpath)) return false; +static bool sync_recv_v1(SyncConnection& sc, const char* rpath, const char* lpath, const char* name, + uint64_t expected_size) { + if (!sc.SendRequest(ID_RECV_V1, rpath)) return false; adb_unlink(lpath); unique_fd lfd(adb_creat(lpath, 0644)); @@ -797,6 +1026,114 @@ static bool sync_recv(SyncConnection& sc, const char* rpath, const char* lpath, return true; } +static bool sync_recv_v2(SyncConnection& sc, const char* rpath, const char* lpath, const char* name, + uint64_t expected_size) { + if (!sc.SendRecv2(rpath)) return false; + + adb_unlink(lpath); + unique_fd lfd(adb_creat(lpath, 0644)); + if (lfd < 0) { + sc.Error("cannot create '%s': %s", lpath, strerror(errno)); + return false; + } + + uint64_t bytes_copied = 0; + + Block buffer(SYNC_DATA_MAX); + BrotliDecoder decoder(std::span(buffer.data(), buffer.size())); + bool reading = true; + while (reading) { + syncmsg msg; + if (!ReadFdExactly(sc.fd, &msg.data, sizeof(msg.data))) { + adb_unlink(lpath); + return false; + } + + if (msg.data.id == ID_DONE) { + adb_unlink(lpath); + sc.Error("unexpected ID_DONE"); + return false; + } + + if (msg.data.id != ID_DATA) { + adb_unlink(lpath); + sc.ReportCopyFailure(rpath, lpath, msg); + return false; + } + + if (msg.data.size > sc.max) { + sc.Error("msg.data.size too large: %u (max %zu)", msg.data.size, sc.max); + adb_unlink(lpath); + return false; + } + + Block block(msg.data.size); + if (!ReadFdExactly(sc.fd, block.data(), msg.data.size)) { + adb_unlink(lpath); + return false; + } + decoder.Append(std::move(block)); + + while (true) { + std::span<char> output; + BrotliDecodeResult result = decoder.Decode(&output); + + if (result == BrotliDecodeResult::Error) { + sc.Error("decompress failed"); + adb_unlink(lpath); + return false; + } + + if (!output.empty()) { + if (!WriteFdExactly(lfd, output.data(), output.size())) { + sc.Error("cannot write '%s': %s", lpath, strerror(errno)); + adb_unlink(lpath); + return false; + } + } + + bytes_copied += output.size(); + + sc.RecordBytesTransferred(msg.data.size); + sc.ReportProgress(name != nullptr ? name : rpath, bytes_copied, expected_size); + + if (result == BrotliDecodeResult::NeedInput) { + break; + } else if (result == BrotliDecodeResult::MoreOutput) { + continue; + } else if (result == BrotliDecodeResult::Done) { + reading = false; + break; + } else { + LOG(FATAL) << "invalid BrotliDecodeResult: " << static_cast<int>(result); + } + } + } + + syncmsg msg; + if (!ReadFdExactly(sc.fd, &msg.data, sizeof(msg.data))) { + sc.Error("failed to read ID_DONE"); + return false; + } + + if (msg.data.id != ID_DONE) { + sc.Error("unexpected message after transfer: id = %d (expected ID_DONE)", msg.data.id); + return false; + } + + sc.RecordFilesTransferred(1); + return true; +} + +static bool sync_recv(SyncConnection& sc, const char* rpath, const char* lpath, const char* name, + uint64_t expected_size, bool compressed) { + if (sc.HaveSendRecv2() && compressed) { + return sync_recv_v2(sc, rpath, lpath, name, expected_size); + } else { + return sync_recv_v1(sc, rpath, lpath, name, expected_size); + } +} + bool do_sync_ls(const char* path) { SyncConnection sc; if (!sc.IsValid()) return false; @@ -872,9 +1209,8 @@ static bool is_root_dir(std::string_view path) { return true; } -static bool copy_local_dir_remote(SyncConnection& sc, std::string lpath, - std::string rpath, bool check_timestamps, - bool list_only) { +static bool copy_local_dir_remote(SyncConnection& sc, std::string lpath, std::string rpath, + bool check_timestamps, bool list_only, bool compressed) { sc.NewTransfer(); // Make sure that both directory paths end in a slash. @@ -956,7 +1292,7 @@ static bool copy_local_dir_remote(SyncConnection& sc, std::string lpath, if (list_only) { sc.Println("would push: %s -> %s", ci.lpath.c_str(), ci.rpath.c_str()); } else { - if (!sync_send(sc, ci.lpath, ci.rpath, ci.time, ci.mode, false)) { + if (!sync_send(sc, ci.lpath, ci.rpath, ci.time, ci.mode, false, compressed)) { return false; } } @@ -966,11 +1302,13 @@ static bool copy_local_dir_remote(SyncConnection& sc, std::string lpath, } sc.RecordFilesSkipped(skipped); + bool success = sc.ReadAcknowledgements(true); sc.ReportTransferRate(lpath, TransferDirection::push); - return true; + return success; } -bool do_sync_push(const std::vector<const char*>& srcs, const char* dst, bool sync) { +bool do_sync_push(const std::vector<const char*>& srcs, const char* dst, bool sync, + bool compressed) { SyncConnection sc; if (!sc.IsValid()) return false; @@ -1035,7 +1373,7 @@ bool do_sync_push(const std::vector<const char*>& srcs, const char* dst, bool sy dst_dir.append(android::base::Basename(src_path)); } - success &= copy_local_dir_remote(sc, src_path, dst_dir, sync, false); + success &= copy_local_dir_remote(sc, src_path, dst_dir, sync, false, compressed); continue; } else if (!should_push_file(st.st_mode)) { sc.Warning("skipping special file '%s' (mode = 0o%o)", src_path, st.st_mode); @@ -1056,11 +1394,11 @@ bool do_sync_push(const std::vector<const char*>& srcs, const char* dst, bool sy sc.NewTransfer(); sc.SetExpectedTotalBytes(st.st_size); - success &= sync_send(sc, src_path, dst_path, st.st_mtime, st.st_mode, sync); + success &= sync_send(sc, src_path, dst_path, st.st_mtime, st.st_mode, sync, compressed); sc.ReportTransferRate(src_path, TransferDirection::push); } - success &= sc.ReadAcknowledgments(); + success &= sc.ReadAcknowledgements(true); sc.ReportOverallTransferRate(TransferDirection::push); return success; } @@ -1141,8 +1479,8 @@ static int set_time_and_mode(const std::string& lpath, time_t time, return r1 ? r1 : r2; } -static bool copy_remote_dir_local(SyncConnection& sc, std::string rpath, - std::string lpath, bool copy_attrs) { +static bool copy_remote_dir_local(SyncConnection& sc, std::string rpath, std::string lpath, + bool copy_attrs, bool compressed) { sc.NewTransfer(); // Make sure that both directory paths end in a slash. @@ -1172,7 +1510,7 @@ static bool copy_remote_dir_local(SyncConnection& sc, std::string rpath, continue; } - if (!sync_recv(sc, ci.rpath.c_str(), ci.lpath.c_str(), nullptr, ci.size)) { + if (!sync_recv(sc, ci.rpath.c_str(), ci.lpath.c_str(), nullptr, ci.size, compressed)) { return false; } @@ -1189,8 +1527,8 @@ static bool copy_remote_dir_local(SyncConnection& sc, std::string rpath, return true; } -bool do_sync_pull(const std::vector<const char*>& srcs, const char* dst, - bool copy_attrs, const char* name) { +bool do_sync_pull(const std::vector<const char*>& srcs, const char* dst, bool copy_attrs, + bool compressed, const char* name) { SyncConnection sc; if (!sc.IsValid()) return false; @@ -1264,7 +1602,7 @@ bool do_sync_pull(const std::vector<const char*>& srcs, const char* dst, dst_dir.append(android::base::Basename(src_path)); } - success &= copy_remote_dir_local(sc, src_path, dst_dir, copy_attrs); + success &= copy_remote_dir_local(sc, src_path, dst_dir, copy_attrs, compressed); continue; } else if (!should_pull_file(src_st.st_mode)) { sc.Warning("skipping special file '%s' (mode = 0o%o)", src_path, src_st.st_mode); @@ -1283,7 +1621,7 @@ bool do_sync_pull(const std::vector<const char*>& srcs, const char* dst, sc.NewTransfer(); sc.SetExpectedTotalBytes(src_st.st_size); - if (!sync_recv(sc, src_path, dst_path, name, src_st.st_size)) { + if (!sync_recv(sc, src_path, dst_path, name, src_st.st_size, compressed)) { success = false; continue; } @@ -1299,11 +1637,12 @@ bool do_sync_pull(const std::vector<const char*>& srcs, const char* dst, return success; } -bool do_sync_sync(const std::string& lpath, const std::string& rpath, bool list_only) { +bool do_sync_sync(const std::string& lpath, const std::string& rpath, bool list_only, + bool compressed) { SyncConnection sc; if (!sc.IsValid()) return false; - bool success = copy_local_dir_remote(sc, lpath, rpath, true, list_only); + bool success = copy_local_dir_remote(sc, lpath, rpath, true, list_only, compressed); if (!list_only) { sc.ReportOverallTransferRate(TransferDirection::push); } diff --git a/adb/client/file_sync_client.h b/adb/client/file_sync_client.h index df7f14ce4..de3f19245 100644 --- a/adb/client/file_sync_client.h +++ b/adb/client/file_sync_client.h @@ -20,8 +20,10 @@ #include <vector> bool do_sync_ls(const char* path); -bool do_sync_push(const std::vector<const char*>& srcs, const char* dst, bool sync); +bool do_sync_push(const std::vector<const char*>& srcs, const char* dst, bool sync, + bool compressed); bool do_sync_pull(const std::vector<const char*>& srcs, const char* dst, bool copy_attrs, - const char* name = nullptr); + bool compressed, const char* name = nullptr); -bool do_sync_sync(const std::string& lpath, const std::string& rpath, bool list_only); +bool do_sync_sync(const std::string& lpath, const std::string& rpath, bool list_only, + bool compressed); diff --git a/adb/client/incremental.cpp b/adb/client/incremental.cpp index 6499d4603..9765292c5 100644 --- a/adb/client/incremental.cpp +++ b/adb/client/incremental.cpp @@ -41,37 +41,35 @@ using Size = int64_t; static inline int32_t read_int32(borrowed_fd fd) { int32_t result; - ReadFully(fd, &result, sizeof(result)); - return result; -} - -static inline int32_t read_be_int32(borrowed_fd fd) { - return int32_t(be32toh(read_int32(fd))); + return ReadFdExactly(fd, &result, sizeof(result)) ? result : -1; } static inline void append_int(borrowed_fd fd, std::vector<char>* bytes) { - int32_t be_val = read_int32(fd); + int32_t le_val = read_int32(fd); auto old_size = bytes->size(); - bytes->resize(old_size + sizeof(be_val)); - memcpy(bytes->data() + old_size, &be_val, sizeof(be_val)); + bytes->resize(old_size + sizeof(le_val)); + memcpy(bytes->data() + old_size, &le_val, sizeof(le_val)); } static inline void append_bytes_with_size(borrowed_fd fd, std::vector<char>* bytes) { - int32_t be_size = read_int32(fd); - int32_t size = int32_t(be32toh(be_size)); + int32_t le_size = read_int32(fd); + if (le_size < 0) { + return; + } + int32_t size = int32_t(le32toh(le_size)); auto old_size = bytes->size(); - bytes->resize(old_size + sizeof(be_size) + size); - memcpy(bytes->data() + old_size, &be_size, sizeof(be_size)); - ReadFully(fd, bytes->data() + old_size + sizeof(be_size), size); + bytes->resize(old_size + sizeof(le_size) + size); + memcpy(bytes->data() + old_size, &le_size, sizeof(le_size)); + ReadFdExactly(fd, bytes->data() + old_size + sizeof(le_size), size); } static inline std::pair<std::vector<char>, int32_t> read_id_sig_headers(borrowed_fd fd) { std::vector<char> result; append_int(fd, &result); // version - append_bytes_with_size(fd, &result); // verityRootHash - append_bytes_with_size(fd, &result); // v3Digest - append_bytes_with_size(fd, &result); // pkcs7SignatureBlock - auto tree_size = read_be_int32(fd); // size of the verity tree + append_bytes_with_size(fd, &result); // hashingInfo + append_bytes_with_size(fd, &result); // signingInfo + auto le_tree_size = read_int32(fd); + auto tree_size = int32_t(le32toh(le_tree_size)); // size of the verity tree return {std::move(result), tree_size}; } @@ -92,38 +90,58 @@ static inline Size verity_tree_size_for_file(Size fileSize) { return total_tree_block_count * INCFS_DATA_FILE_BLOCK_SIZE; } -// Base64-encode signature bytes. Keeping fd at the position of start of verity tree. -static std::pair<unique_fd, std::string> read_and_encode_signature(Size file_size, - std::string signature_file) { +// Read, verify and return the signature bytes. Keeping fd at the position of start of verity tree. +static std::pair<unique_fd, std::vector<char>> read_signature(Size file_size, + std::string signature_file, + bool silent) { signature_file += IDSIG; struct stat st; if (stat(signature_file.c_str(), &st)) { - fprintf(stderr, "Failed to stat signature file %s. Abort.\n", signature_file.c_str()); + if (!silent) { + fprintf(stderr, "Failed to stat signature file %s. Abort.\n", signature_file.c_str()); + } return {}; } unique_fd fd(adb_open(signature_file.c_str(), O_RDONLY | O_CLOEXEC)); if (fd < 0) { - fprintf(stderr, "Failed to open signature file: %s. Abort.\n", signature_file.c_str()); + if (!silent) { + fprintf(stderr, "Failed to open signature file: %s. Abort.\n", signature_file.c_str()); + } return {}; } auto [signature, tree_size] = read_id_sig_headers(fd); if (auto expected = verity_tree_size_for_file(file_size); tree_size != expected) { - fprintf(stderr, - "Verity tree size mismatch in signature file: %s [was %lld, expected %lld].\n", - signature_file.c_str(), (long long)tree_size, (long long)expected); + if (!silent) { + fprintf(stderr, + "Verity tree size mismatch in signature file: %s [was %lld, expected %lld].\n", + signature_file.c_str(), (long long)tree_size, (long long)expected); + } + return {}; + } + + return {std::move(fd), std::move(signature)}; +} + +// Base64-encode signature bytes. Keeping fd at the position of start of verity tree. +static std::pair<unique_fd, std::string> read_and_encode_signature(Size file_size, + std::string signature_file, + bool silent) { + auto [fd, signature] = read_signature(file_size, std::move(signature_file), silent); + if (!fd.ok()) { return {}; } size_t base64_len = 0; if (!EVP_EncodedLength(&base64_len, signature.size())) { - fprintf(stderr, "Fail to estimate base64 encoded length. Abort.\n"); + if (!silent) { + fprintf(stderr, "Fail to estimate base64 encoded length. Abort.\n"); + } return {}; } - std::string encoded_signature; - encoded_signature.resize(base64_len); + std::string encoded_signature(base64_len, '\0'); encoded_signature.resize(EVP_EncodeBlock((uint8_t*)encoded_signature.data(), (const uint8_t*)signature.data(), signature.size())); @@ -132,7 +150,7 @@ static std::pair<unique_fd, std::string> read_and_encode_signature(Size file_siz // Send install-incremental to the device along with properly configured file descriptors in // streaming format. Once connection established, send all fs-verity tree bytes. -static unique_fd start_install(const std::vector<std::string>& files) { +static unique_fd start_install(const Files& files, bool silent) { std::vector<std::string> command_args{"package", "install-incremental"}; // fd's with positions at the beginning of fs-verity @@ -143,11 +161,13 @@ static unique_fd start_install(const std::vector<std::string>& files) { struct stat st; if (stat(file.c_str(), &st)) { - fprintf(stderr, "Failed to stat input file %s. Abort.\n", file.c_str()); + if (!silent) { + fprintf(stderr, "Failed to stat input file %s. Abort.\n", file.c_str()); + } return {}; } - auto [signature_fd, signature] = read_and_encode_signature(st.st_size, file); + auto [signature_fd, signature] = read_and_encode_signature(st.st_size, file, silent); if (!signature_fd.ok()) { return {}; } @@ -163,15 +183,19 @@ static unique_fd start_install(const std::vector<std::string>& files) { std::string error; auto connection_fd = unique_fd(send_abb_exec_command(command_args, &error)); if (connection_fd < 0) { - fprintf(stderr, "Failed to run: %s, error: %s\n", - android::base::Join(command_args, " ").c_str(), error.c_str()); + if (!silent) { + fprintf(stderr, "Failed to run: %s, error: %s\n", + android::base::Join(command_args, " ").c_str(), error.c_str()); + } return {}; } // Pushing verity trees for all installation files. for (auto&& local_fd : signature_fds) { if (!copy_to_file(local_fd.get(), connection_fd.get())) { - fprintf(stderr, "Failed to stream tree bytes: %s. Abort.\n", strerror(errno)); + if (!silent) { + fprintf(stderr, "Failed to stream tree bytes: %s. Abort.\n", strerror(errno)); + } return {}; } } @@ -181,36 +205,105 @@ static unique_fd start_install(const std::vector<std::string>& files) { } // namespace -std::optional<Process> install(std::vector<std::string> files) { - auto connection_fd = start_install(files); +bool can_install(const Files& files) { + for (const auto& file : files) { + struct stat st; + if (stat(file.c_str(), &st)) { + return false; + } + + auto [fd, _] = read_signature(st.st_size, file, true); + if (!fd.ok()) { + return false; + } + } + return true; +} + +std::optional<Process> install(const Files& files, bool silent) { + auto connection_fd = start_install(files, silent); if (connection_fd < 0) { - fprintf(stderr, "adb: failed to initiate installation on device.\n"); + if (!silent) { + fprintf(stderr, "adb: failed to initiate installation on device.\n"); + } return {}; } std::string adb_path = android::base::GetExecutablePath(); - auto osh = adb_get_os_handle(connection_fd.get()); -#ifdef _WIN32 - auto fd_param = std::to_string(reinterpret_cast<intptr_t>(osh)); -#else /* !_WIN32 a.k.a. Unix */ + auto osh = cast_handle_to_int(adb_get_os_handle(connection_fd.get())); auto fd_param = std::to_string(osh); -#endif + + // pipe for child process to write output + int print_fds[2]; + if (adb_socketpair(print_fds) != 0) { + if (!silent) { + fprintf(stderr, "Failed to create socket pair for child to print to parent\n"); + } + return {}; + } + auto [pipe_read_fd, pipe_write_fd] = print_fds; + auto pipe_write_fd_param = std::to_string(cast_handle_to_int(adb_get_os_handle(pipe_write_fd))); + close_on_exec(pipe_read_fd); std::vector<std::string> args(std::move(files)); - args.insert(args.begin(), {"inc-server", fd_param}); - auto child = adb_launch_process(adb_path, std::move(args), {connection_fd.get()}); + args.insert(args.begin(), {"inc-server", fd_param, pipe_write_fd_param}); + auto child = + adb_launch_process(adb_path, std::move(args), {connection_fd.get(), pipe_write_fd}); if (!child) { - fprintf(stderr, "adb: failed to fork: %s\n", strerror(errno)); + if (!silent) { + fprintf(stderr, "adb: failed to fork: %s\n", strerror(errno)); + } return {}; } + adb_close(pipe_write_fd); + auto killOnExit = [](Process* p) { p->kill(); }; std::unique_ptr<Process, decltype(killOnExit)> serverKiller(&child, killOnExit); - // TODO: Terminate server process if installation fails. - serverKiller.release(); + Result result = wait_for_installation(pipe_read_fd); + adb_close(pipe_read_fd); + + if (result == Result::Success) { + // adb client exits now but inc-server can continue + serverKiller.release(); + } return child; } +Result wait_for_installation(int read_fd) { + static constexpr int maxMessageSize = 256; + std::vector<char> child_stdout(CHUNK_SIZE); + int bytes_read; + int buf_size = 0; + // TODO(b/150865433): optimize child's output parsing + while ((bytes_read = adb_read(read_fd, child_stdout.data() + buf_size, + child_stdout.size() - buf_size)) > 0) { + // print to parent's stdout + fprintf(stdout, "%.*s", bytes_read, child_stdout.data() + buf_size); + + buf_size += bytes_read; + const std::string_view stdout_str(child_stdout.data(), buf_size); + // wait till installation either succeeds or fails + if (stdout_str.find("Success") != std::string::npos) { + return Result::Success; + } + // on failure, wait for full message + static constexpr auto failure_msg_head = "Failure ["sv; + if (const auto begin_itr = stdout_str.find(failure_msg_head); + begin_itr != std::string::npos) { + if (buf_size >= maxMessageSize) { + return Result::Failure; + } + const auto end_itr = stdout_str.rfind("]"); + if (end_itr != std::string::npos && end_itr >= begin_itr + failure_msg_head.size()) { + return Result::Failure; + } + } + child_stdout.resize(buf_size + CHUNK_SIZE); + } + return Result::None; +} + } // namespace incremental diff --git a/adb/client/incremental.h b/adb/client/incremental.h index 4b9f6bde0..1fb1e0b19 100644 --- a/adb/client/incremental.h +++ b/adb/client/incremental.h @@ -25,6 +25,12 @@ namespace incremental { -std::optional<Process> install(std::vector<std::string> files); +using Files = std::vector<std::string>; + +bool can_install(const Files& files); +std::optional<Process> install(const Files& files, bool silent); + +enum class Result { Success, Failure, None }; +Result wait_for_installation(int read_fd); } // namespace incremental diff --git a/adb/client/incremental_server.cpp b/adb/client/incremental_server.cpp index 2512d0562..4b87d0a7b 100644 --- a/adb/client/incremental_server.cpp +++ b/adb/client/incremental_server.cpp @@ -18,13 +18,6 @@ #include "incremental_server.h" -#include "adb.h" -#include "adb_io.h" -#include "adb_trace.h" -#include "adb_unique_fd.h" -#include "adb_utils.h" -#include "sysdeps.h" - #include <android-base/endian.h> #include <android-base/strings.h> #include <inttypes.h> @@ -41,29 +34,41 @@ #include <type_traits> #include <unordered_set> +#include "adb.h" +#include "adb_io.h" +#include "adb_trace.h" +#include "adb_unique_fd.h" +#include "adb_utils.h" +#include "incremental_utils.h" +#include "sysdeps.h" + namespace incremental { static constexpr int kBlockSize = 4096; static constexpr int kCompressedSizeMax = kBlockSize * 0.95; -static constexpr short kCompressionNone = 0; -static constexpr short kCompressionLZ4 = 1; +static constexpr int8_t kTypeData = 0; +static constexpr int8_t kCompressionNone = 0; +static constexpr int8_t kCompressionLZ4 = 1; static constexpr int kCompressBound = std::max(kBlockSize, LZ4_COMPRESSBOUND(kBlockSize)); static constexpr auto kReadBufferSize = 128 * 1024; +static constexpr int kPollTimeoutMillis = 300000; // 5 minutes using BlockSize = int16_t; using FileId = int16_t; using BlockIdx = int32_t; using NumBlocks = int32_t; -using CompressionType = int16_t; +using BlockType = int8_t; +using CompressionType = int8_t; using RequestType = int16_t; using ChunkHeader = int32_t; using MagicType = uint32_t; static constexpr MagicType INCR = 0x494e4352; // LE INCR -static constexpr RequestType EXIT = 0; +static constexpr RequestType SERVING_COMPLETE = 0; static constexpr RequestType BLOCK_MISSING = 1; static constexpr RequestType PREFETCH = 2; +static constexpr RequestType DESTROY = 3; static constexpr inline int64_t roundDownToBlockOffset(int64_t val) { return val & ~(kBlockSize - 1); @@ -123,7 +128,8 @@ struct RequestCommand { // Placed before actual data bytes of each block struct ResponseHeader { FileId file_id; // 2 bytes - CompressionType compression_type; // 2 bytes + BlockType block_type; // 1 byte + CompressionType compression_type; // 1 byte BlockIdx block_idx; // 4 bytes BlockSize block_size; // 2 bytes } __attribute__((packed)); @@ -134,6 +140,7 @@ class File { // Plain file File(const char* filepath, FileId id, int64_t size, unique_fd fd) : File(filepath, id, size) { this->fd_ = std::move(fd); + priority_blocks_ = PriorityBlocksForFile(filepath, fd_.get(), size); } int64_t ReadBlock(BlockIdx block_idx, void* buf, bool* is_zip_compressed, std::string* error) const { @@ -145,6 +152,7 @@ class File { } const unique_fd& RawFd() const { return fd_; } + const std::vector<BlockIdx>& PriorityBlocks() const { return priority_blocks_; } std::vector<bool> sentBlocks; NumBlocks sentBlocksCount = 0; @@ -158,12 +166,13 @@ class File { sentBlocks.resize(numBytesToNumBlocks(size)); } unique_fd fd_; + std::vector<BlockIdx> priority_blocks_; }; class IncrementalServer { public: - IncrementalServer(unique_fd fd, std::vector<File> files) - : adb_fd_(std::move(fd)), files_(std::move(files)) { + IncrementalServer(unique_fd adb_fd, unique_fd output_fd, std::vector<File> files) + : adb_fd_(std::move(adb_fd)), output_fd_(std::move(output_fd)), files_(std::move(files)) { buffer_.reserve(kReadBufferSize); } @@ -174,14 +183,23 @@ class IncrementalServer { const File* file; BlockIdx overallIndex = 0; BlockIdx overallEnd = 0; + BlockIdx priorityIndex = 0; - PrefetchState(const File& f) : file(&f), overallEnd((BlockIdx)f.sentBlocks.size()) {} - PrefetchState(const File& f, BlockIdx start, int count) + explicit PrefetchState(const File& f, BlockIdx start, int count) : file(&f), overallIndex(start), overallEnd(std::min<BlockIdx>(start + count, f.sentBlocks.size())) {} - bool done() const { return overallIndex >= overallEnd; } + explicit PrefetchState(const File& f) + : PrefetchState(f, 0, (BlockIdx)f.sentBlocks.size()) {} + + bool done() const { + const bool overallSent = (overallIndex >= overallEnd); + if (file->PriorityBlocks().empty()) { + return overallSent; + } + return overallSent && (priorityIndex >= (BlockIdx)file->PriorityBlocks().size()); + } }; bool SkipToRequest(void* buffer, size_t* size, bool blocking); @@ -197,9 +215,10 @@ class IncrementalServer { void Send(const void* data, size_t size, bool flush); void Flush(); using TimePoint = decltype(std::chrono::high_resolution_clock::now()); - bool Exit(std::optional<TimePoint> startTime, int missesCount, int missesSent); + bool ServingComplete(std::optional<TimePoint> startTime, int missesCount, int missesSent); unique_fd const adb_fd_; + unique_fd const output_fd_; std::vector<File> files_; // Incoming data buffer. @@ -210,6 +229,9 @@ class IncrementalServer { long long sentSize_ = 0; std::vector<char> pendingBlocks_; + + // True when client notifies that all the data has been received + bool servingComplete_; }; bool IncrementalServer::SkipToRequest(void* buffer, size_t* size, bool blocking) { @@ -217,7 +239,8 @@ bool IncrementalServer::SkipToRequest(void* buffer, size_t* size, bool blocking) // Looking for INCR magic. bool magic_found = false; int bcur = 0; - for (int bsize = buffer_.size(); bcur + 4 < bsize; ++bcur) { + int bsize = buffer_.size(); + for (bcur = 0; bcur + 4 < bsize; ++bcur) { uint32_t magic = be32toh(*(uint32_t*)(buffer_.data() + bcur)); if (magic == INCR) { magic_found = true; @@ -226,8 +249,8 @@ bool IncrementalServer::SkipToRequest(void* buffer, size_t* size, bool blocking) } if (bcur > 0) { - // Stream the rest to stderr. - fprintf(stderr, "%.*s", bcur, buffer_.data()); + // output the rest. + WriteFdExactly(output_fd_, buffer_.data(), bcur); erase_buffer_head(bcur); } @@ -239,17 +262,26 @@ bool IncrementalServer::SkipToRequest(void* buffer, size_t* size, bool blocking) } adb_pollfd pfd = {adb_fd_.get(), POLLIN, 0}; - auto res = adb_poll(&pfd, 1, blocking ? -1 : 0); + auto res = adb_poll(&pfd, 1, blocking ? kPollTimeoutMillis : 0); + if (res != 1) { + WriteFdExactly(output_fd_, buffer_.data(), buffer_.size()); if (res < 0) { - fprintf(stderr, "Failed to poll: %s\n", strerror(errno)); + D("Failed to poll: %s\n", strerror(errno)); + return false; + } + if (blocking) { + fprintf(stderr, "Timed out waiting for data from device.\n"); + } + if (blocking && servingComplete_) { + // timeout waiting from client. Serving is complete, so quit. return false; } *size = 0; return true; } - auto bsize = buffer_.size(); + bsize = buffer_.size(); buffer_.resize(kReadBufferSize); int r = adb_read(adb_fd_, buffer_.data() + bsize, kReadBufferSize - bsize); if (r > 0) { @@ -257,21 +289,19 @@ bool IncrementalServer::SkipToRequest(void* buffer, size_t* size, bool blocking) continue; } - if (r == -1) { - fprintf(stderr, "Failed to read from fd %d: %d. Exit\n", adb_fd_.get(), errno); - return false; - } - - // socket is closed - return false; + D("Failed to read from fd %d: %d. Exit\n", adb_fd_.get(), errno); + break; } + // socket is closed. print remaining messages + WriteFdExactly(output_fd_, buffer_.data(), buffer_.size()); + return false; } std::optional<RequestCommand> IncrementalServer::ReadRequest(bool blocking) { uint8_t commandBuf[sizeof(RequestCommand)]; auto size = sizeof(commandBuf); if (!SkipToRequest(&commandBuf, &size, blocking)) { - return {{EXIT}}; + return {{DESTROY}}; } if (size < sizeof(RequestCommand)) { return {}; @@ -316,14 +346,16 @@ auto IncrementalServer::SendBlock(FileId fileId, BlockIdx blockIdx, bool flush) ++compressed_; blockSize = compressedSize; header = reinterpret_cast<ResponseHeader*>(data); - header->compression_type = toBigEndian(kCompressionLZ4); + header->compression_type = kCompressionLZ4; } else { ++uncompressed_; blockSize = bytesRead; header = reinterpret_cast<ResponseHeader*>(raw); - header->compression_type = toBigEndian(kCompressionNone); + header->compression_type = kCompressionNone; } + header->block_type = kTypeData; + header->file_id = toBigEndian(fileId); header->block_size = toBigEndian(blockSize); header->block_idx = toBigEndian(blockIdx); @@ -337,6 +369,7 @@ auto IncrementalServer::SendBlock(FileId fileId, BlockIdx blockIdx, bool flush) bool IncrementalServer::SendDone() { ResponseHeader header; header.file_id = -1; + header.block_type = 0; header.compression_type = 0; header.block_idx = 0; header.block_size = 0; @@ -351,6 +384,17 @@ void IncrementalServer::RunPrefetching() { while (!prefetches_.empty() && blocksToSend > 0) { auto& prefetch = prefetches_.front(); const auto& file = *prefetch.file; + const auto& priority_blocks = file.PriorityBlocks(); + if (!priority_blocks.empty()) { + for (auto& i = prefetch.priorityIndex; + blocksToSend > 0 && i < (BlockIdx)priority_blocks.size(); ++i) { + if (auto res = SendBlock(file.id, priority_blocks[i]); res == SendResult::Sent) { + --blocksToSend; + } else if (res == SendResult::Error) { + fprintf(stderr, "Failed to send priority block %" PRId32 "\n", i); + } + } + } for (auto& i = prefetch.overallIndex; blocksToSend > 0 && i < prefetch.overallEnd; ++i) { if (auto res = SendBlock(file.id, i); res == SendResult::Sent) { --blocksToSend; @@ -391,17 +435,17 @@ void IncrementalServer::Flush() { pendingBlocks_.clear(); } -bool IncrementalServer::Exit(std::optional<TimePoint> startTime, int missesCount, int missesSent) { +bool IncrementalServer::ServingComplete(std::optional<TimePoint> startTime, int missesCount, + int missesSent) { + servingComplete_ = true; using namespace std::chrono; auto endTime = high_resolution_clock::now(); - fprintf(stderr, - "Connection failed or received exit command. Exit.\n" - "Misses: %d, of those unique: %d; sent compressed: %d, uncompressed: " - "%d, mb: %.3f\n" - "Total time taken: %.3fms\n", - missesCount, missesSent, compressed_, uncompressed_, sentSize_ / 1024.0 / 1024.0, - duration_cast<microseconds>(endTime - (startTime ? *startTime : endTime)).count() / - 1000.0); + D("Streaming completed.\n" + "Misses: %d, of those unique: %d; sent compressed: %d, uncompressed: " + "%d, mb: %.3f\n" + "Total time taken: %.3fms\n", + missesCount, missesSent, compressed_, uncompressed_, sentSize_ / 1024.0 / 1024.0, + duration_cast<microseconds>(endTime - (startTime ? *startTime : endTime)).count() / 1000.0); return true; } @@ -425,7 +469,7 @@ bool IncrementalServer::Serve() { std::all_of(files_.begin(), files_.end(), [](const File& f) { return f.sentBlocksCount == NumBlocks(f.sentBlocks.size()); })) { - fprintf(stdout, "All files should be loaded. Notifying the device.\n"); + fprintf(stderr, "All files should be loaded. Notifying the device.\n"); SendDone(); doneSent = true; } @@ -446,9 +490,14 @@ bool IncrementalServer::Serve() { BlockIdx blockIdx = request->block_idx; switch (request->request_type) { - case EXIT: { + case DESTROY: { // Stop everything. - return Exit(startTime, missesCount, missesSent); + return true; + } + case SERVING_COMPLETE: { + // Not stopping the server here. + ServingComplete(startTime, missesCount, missesSent); + break; } case BLOCK_MISSING: { ++missesCount; @@ -502,8 +551,9 @@ bool IncrementalServer::Serve() { } } -bool serve(int adb_fd, int argc, const char** argv) { - auto connection_fd = unique_fd(adb_fd); +bool serve(int connection_fd, int output_fd, int argc, const char** argv) { + auto connection_ufd = unique_fd(connection_fd); + auto output_ufd = unique_fd(output_fd); if (argc <= 0) { error_exit("inc-server: must specify at least one file."); } @@ -526,7 +576,7 @@ bool serve(int adb_fd, int argc, const char** argv) { files.emplace_back(filepath, i, st.st_size, std::move(fd)); } - IncrementalServer server(std::move(connection_fd), std::move(files)); + IncrementalServer server(std::move(connection_ufd), std::move(output_ufd), std::move(files)); printf("Serving...\n"); fclose(stdin); fclose(stdout); diff --git a/adb/client/incremental_server.h b/adb/client/incremental_server.h index 53f011eb3..55b8215b8 100644 --- a/adb/client/incremental_server.h +++ b/adb/client/incremental_server.h @@ -21,6 +21,6 @@ namespace incremental { // Expecting arguments like: // {FILE1 FILE2 ...} // Where FILE* are files to serve. -bool serve(int adbFd, int argc, const char** argv); +bool serve(int connection_fd, int output_fd, int argc, const char** argv); } // namespace incremental diff --git a/adb/client/incremental_utils.cpp b/adb/client/incremental_utils.cpp new file mode 100644 index 000000000..fa501e49d --- /dev/null +++ b/adb/client/incremental_utils.cpp @@ -0,0 +1,294 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define TRACE_TAG INCREMENTAL + +#include "incremental_utils.h" + +#include <android-base/mapped_file.h> +#include <android-base/strings.h> +#include <ziparchive/zip_archive.h> +#include <ziparchive/zip_writer.h> + +#include <array> +#include <cinttypes> +#include <numeric> +#include <unordered_set> + +#include "adb_trace.h" +#include "sysdeps.h" + +using namespace std::literals; + +static constexpr int kBlockSize = 4096; + +static constexpr inline int32_t offsetToBlockIndex(int64_t offset) { + return (offset & ~(kBlockSize - 1)) >> 12; +} + +template <class T> +T valueAt(int fd, off64_t offset) { + T t; + memset(&t, 0, sizeof(T)); + if (adb_pread(fd, &t, sizeof(T), offset) != sizeof(T)) { + memset(&t, -1, sizeof(T)); + } + + return t; +} + +static void appendBlocks(int32_t start, int count, std::vector<int32_t>* blocks) { + if (count == 1) { + blocks->push_back(start); + } else { + auto oldSize = blocks->size(); + blocks->resize(oldSize + count); + std::iota(blocks->begin() + oldSize, blocks->end(), start); + } +} + +template <class T> +static void unduplicate(std::vector<T>& v) { + std::unordered_set<T> uniques(v.size()); + v.erase(std::remove_if(v.begin(), v.end(), + [&uniques](T t) { return !uniques.insert(t).second; }), + v.end()); +} + +static off64_t CentralDirOffset(int fd, int64_t fileSize) { + static constexpr int kZipEocdRecMinSize = 22; + static constexpr int32_t kZipEocdRecSig = 0x06054b50; + static constexpr int kZipEocdCentralDirSizeFieldOffset = 12; + static constexpr int kZipEocdCommentLengthFieldOffset = 20; + + int32_t sigBuf = 0; + off64_t eocdOffset = -1; + off64_t maxEocdOffset = fileSize - kZipEocdRecMinSize; + int16_t commentLenBuf = 0; + + // Search from the end of zip, backward to find beginning of EOCD + for (int16_t commentLen = 0; commentLen < fileSize; ++commentLen) { + sigBuf = valueAt<int32_t>(fd, maxEocdOffset - commentLen); + if (sigBuf == kZipEocdRecSig) { + commentLenBuf = valueAt<int16_t>( + fd, maxEocdOffset - commentLen + kZipEocdCommentLengthFieldOffset); + if (commentLenBuf == commentLen) { + eocdOffset = maxEocdOffset - commentLen; + break; + } + } + } + + if (eocdOffset < 0) { + return -1; + } + + off64_t cdLen = static_cast<int64_t>( + valueAt<int32_t>(fd, eocdOffset + kZipEocdCentralDirSizeFieldOffset)); + + return eocdOffset - cdLen; +} + +// Does not support APKs larger than 4GB +static off64_t SignerBlockOffset(int fd, int64_t fileSize) { + static constexpr int kApkSigBlockMinSize = 32; + static constexpr int kApkSigBlockFooterSize = 24; + static constexpr int64_t APK_SIG_BLOCK_MAGIC_HI = 0x3234206b636f6c42l; + static constexpr int64_t APK_SIG_BLOCK_MAGIC_LO = 0x20676953204b5041l; + + off64_t cdOffset = CentralDirOffset(fd, fileSize); + if (cdOffset < 0) { + return -1; + } + // CD offset is where original signer block ends. Search backwards for magic and footer. + if (cdOffset < kApkSigBlockMinSize || + valueAt<int64_t>(fd, cdOffset - 2 * sizeof(int64_t)) != APK_SIG_BLOCK_MAGIC_LO || + valueAt<int64_t>(fd, cdOffset - sizeof(int64_t)) != APK_SIG_BLOCK_MAGIC_HI) { + return -1; + } + int32_t signerSizeInFooter = valueAt<int32_t>(fd, cdOffset - kApkSigBlockFooterSize); + off64_t signerBlockOffset = cdOffset - signerSizeInFooter - sizeof(int64_t); + if (signerBlockOffset < 0) { + return -1; + } + int32_t signerSizeInHeader = valueAt<int32_t>(fd, signerBlockOffset); + if (signerSizeInFooter != signerSizeInHeader) { + return -1; + } + + return signerBlockOffset; +} + +static std::vector<int32_t> ZipPriorityBlocks(off64_t signerBlockOffset, int64_t fileSize) { + int32_t signerBlockIndex = offsetToBlockIndex(signerBlockOffset); + int32_t lastBlockIndex = offsetToBlockIndex(fileSize); + const auto numPriorityBlocks = lastBlockIndex - signerBlockIndex + 1; + + std::vector<int32_t> zipPriorityBlocks; + + // Some magic here: most of zip libraries perform a scan for EOCD record starting at the offset + // of a maximum comment size from the end of the file. This means the last 65-ish KBs will be + // accessed first, followed by the rest of the central directory blocks. Make sure we + // send the data in the proper order, as central directory can be quite big by itself. + static constexpr auto kMaxZipCommentSize = 64 * 1024; + static constexpr auto kNumBlocksInEocdSearch = kMaxZipCommentSize / kBlockSize + 1; + if (numPriorityBlocks > kNumBlocksInEocdSearch) { + appendBlocks(lastBlockIndex - kNumBlocksInEocdSearch + 1, kNumBlocksInEocdSearch, + &zipPriorityBlocks); + appendBlocks(signerBlockIndex, numPriorityBlocks - kNumBlocksInEocdSearch, + &zipPriorityBlocks); + } else { + appendBlocks(signerBlockIndex, numPriorityBlocks, &zipPriorityBlocks); + } + + // Somehow someone keeps accessing the start of the archive, even if there's nothing really + // interesting there... + appendBlocks(0, 1, &zipPriorityBlocks); + return zipPriorityBlocks; +} + +[[maybe_unused]] static ZipArchiveHandle openZipArchiveFd(int fd) { + bool transferFdOwnership = false; +#ifdef _WIN32 + // + // Need to create a special CRT FD here as the current one is not compatible with + // normal read()/write() calls that libziparchive uses. + // To make this work we have to create a copy of the file handle, as CRT doesn't care + // and closes it together with the new descriptor. + // + // Note: don't move this into a helper function, it's better to be hard to reuse because + // the code is ugly and won't work unless it's a last resort. + // + auto handle = adb_get_os_handle(fd); + HANDLE dupedHandle; + if (!::DuplicateHandle(::GetCurrentProcess(), handle, ::GetCurrentProcess(), &dupedHandle, 0, + false, DUPLICATE_SAME_ACCESS)) { + D("%s failed at DuplicateHandle: %d", __func__, (int)::GetLastError()); + return {}; + } + fd = _open_osfhandle((intptr_t)dupedHandle, _O_RDONLY | _O_BINARY); + if (fd < 0) { + D("%s failed at _open_osfhandle: %d", __func__, errno); + ::CloseHandle(handle); + return {}; + } + transferFdOwnership = true; +#endif + ZipArchiveHandle zip; + if (OpenArchiveFd(fd, "apk_fd", &zip, transferFdOwnership) != 0) { + D("%s failed at OpenArchiveFd: %d", __func__, errno); +#ifdef _WIN32 + // "_close()" is a secret WinCRT name for the regular close() function. + _close(fd); +#endif + return {}; + } + return zip; +} + +static std::pair<ZipArchiveHandle, std::unique_ptr<android::base::MappedFile>> openZipArchive( + int fd, int64_t fileSize) { +#ifndef __LP64__ + if (fileSize >= INT_MAX) { + return {openZipArchiveFd(fd), nullptr}; + } +#endif + auto mapping = + android::base::MappedFile::FromOsHandle(adb_get_os_handle(fd), 0, fileSize, PROT_READ); + if (!mapping) { + D("%s failed at FromOsHandle: %d", __func__, errno); + return {}; + } + ZipArchiveHandle zip; + if (OpenArchiveFromMemory(mapping->data(), mapping->size(), "apk_mapping", &zip) != 0) { + D("%s failed at OpenArchiveFromMemory: %d", __func__, errno); + return {}; + } + return {zip, std::move(mapping)}; +} + +static std::vector<int32_t> InstallationPriorityBlocks(int fd, int64_t fileSize) { + static constexpr std::array<std::string_view, 3> additional_matches = { + "resources.arsc"sv, "AndroidManifest.xml"sv, "classes.dex"sv}; + auto [zip, _] = openZipArchive(fd, fileSize); + if (!zip) { + return {}; + } + + auto matcher = [](std::string_view entry_name) { + if (entry_name.starts_with("lib/"sv) && entry_name.ends_with(".so"sv)) { + return true; + } + return std::any_of(additional_matches.begin(), additional_matches.end(), + [entry_name](std::string_view i) { return i == entry_name; }); + }; + + void* cookie = nullptr; + if (StartIteration(zip, &cookie, std::move(matcher)) != 0) { + D("%s failed at StartIteration: %d", __func__, errno); + return {}; + } + + std::vector<int32_t> installationPriorityBlocks; + ZipEntry entry; + std::string_view entryName; + while (Next(cookie, &entry, &entryName) == 0) { + if (entryName == "classes.dex"sv) { + // Only the head is needed for installation + int32_t startBlockIndex = offsetToBlockIndex(entry.offset); + appendBlocks(startBlockIndex, 1, &installationPriorityBlocks); + D("\tadding to priority blocks: '%.*s' 1", (int)entryName.size(), entryName.data()); + } else { + // Full entries are needed for installation + off64_t entryStartOffset = entry.offset; + off64_t entryEndOffset = + entryStartOffset + + (entry.method == kCompressStored ? entry.uncompressed_length + : entry.compressed_length) + + (entry.has_data_descriptor ? 16 /* sizeof(DataDescriptor) */ : 0); + int32_t startBlockIndex = offsetToBlockIndex(entryStartOffset); + int32_t endBlockIndex = offsetToBlockIndex(entryEndOffset); + int32_t numNewBlocks = endBlockIndex - startBlockIndex + 1; + appendBlocks(startBlockIndex, numNewBlocks, &installationPriorityBlocks); + D("\tadding to priority blocks: '%.*s' (%d)", (int)entryName.size(), entryName.data(), + numNewBlocks); + } + } + + EndIteration(cookie); + CloseArchive(zip); + return installationPriorityBlocks; +} + +namespace incremental { +std::vector<int32_t> PriorityBlocksForFile(const std::string& filepath, int fd, int64_t fileSize) { + if (!android::base::EndsWithIgnoreCase(filepath, ".apk")) { + return {}; + } + off64_t signerOffset = SignerBlockOffset(fd, fileSize); + if (signerOffset < 0) { + // No signer block? not a valid APK + return {}; + } + std::vector<int32_t> priorityBlocks = ZipPriorityBlocks(signerOffset, fileSize); + std::vector<int32_t> installationPriorityBlocks = InstallationPriorityBlocks(fd, fileSize); + + priorityBlocks.insert(priorityBlocks.end(), installationPriorityBlocks.begin(), + installationPriorityBlocks.end()); + unduplicate(priorityBlocks); + return priorityBlocks; +} +} // namespace incremental diff --git a/adb/client/incremental_utils.h b/adb/client/incremental_utils.h new file mode 100644 index 000000000..8bcf6c081 --- /dev/null +++ b/adb/client/incremental_utils.h @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <stdint.h> + +#include <string> +#include <vector> + +namespace incremental { +std::vector<int32_t> PriorityBlocksForFile(const std::string& filepath, int fd, int64_t fileSize); +} // namespace incremental
\ No newline at end of file diff --git a/adb/client/main.cpp b/adb/client/main.cpp index a85a18c4e..33e0716ab 100644 --- a/adb/client/main.cpp +++ b/adb/client/main.cpp @@ -139,9 +139,10 @@ int adb_server_main(int is_daemon, const std::string& socket_spec, int ack_reply auto start = std::chrono::steady_clock::now(); // If we told a previous adb server to quit because of version mismatch, we can get to this - // point before it's finished exiting. Retry for a while to give it some time. - while (install_listener(socket_spec, "*smartsocket*", nullptr, 0, nullptr, &error) != - INSTALL_STATUS_OK) { + // point before it's finished exiting. Retry for a while to give it some time. Don't actually + // accept any connections until adb_wait_for_device_initialization finishes below. + while (install_listener(socket_spec, "*smartsocket*", nullptr, INSTALL_LISTENER_DISABLED, + nullptr, &error) != INSTALL_STATUS_OK) { if (std::chrono::steady_clock::now() - start > 0.5s) { LOG(FATAL) << "could not install *smartsocket* listener: " << error; } @@ -162,12 +163,14 @@ int adb_server_main(int is_daemon, const std::string& socket_spec, int ack_reply PLOG(FATAL) << "setsid() failed"; } #endif + } - // Wait for the USB scan to complete before notifying the parent that we're up. - // We need to perform this in a thread, because we would otherwise block the event loop. - std::thread notify_thread([ack_reply_fd]() { - adb_wait_for_device_initialization(); + // Wait for the USB scan to complete before notifying the parent that we're up. + // We need to perform this in a thread, because we would otherwise block the event loop. + std::thread notify_thread([ack_reply_fd]() { + adb_wait_for_device_initialization(); + if (ack_reply_fd >= 0) { // Any error output written to stderr now goes to adb.log. We could // keep around a copy of the stderr fd and use that to write any errors // encountered by the following code, but that is probably overkill. @@ -193,9 +196,13 @@ int adb_server_main(int is_daemon, const std::string& socket_spec, int ack_reply } unix_close(ack_reply_fd); #endif - }); - notify_thread.detach(); - } + } + // We don't accept() client connections until this point: this way, clients + // can't see wonky state early in startup even if they're connecting directly + // to the server instead of going through the adb program. + fdevent_run_on_main_thread([] { enable_daemon_sockets(); }); + }); + notify_thread.detach(); #if defined(__linux__) // Write our location to .android/adb.$PORT, so that older clients can exec us. diff --git a/adb/client/pairing/pairing_client.cpp b/adb/client/pairing/pairing_client.cpp index 2f878bf04..937a5bd4c 100644 --- a/adb/client/pairing/pairing_client.cpp +++ b/adb/client/pairing/pairing_client.cpp @@ -141,7 +141,8 @@ bool PairingClientImpl::StartConnection() { cert_.size(), priv_key_.data(), priv_key_.size())); CHECK(connection_); - if (!pairing_connection_start(connection_.get(), fd.release(), OnPairingResult, this)) { + int osh = cast_handle_to_int(adb_get_os_handle(fd.release())); + if (!pairing_connection_start(connection_.get(), osh, OnPairingResult, this)) { LOG(ERROR) << "PairingClient failed to start the PairingConnection"; state_ = State::Stopped; return false; diff --git a/adb/daemon/file_sync_service.cpp b/adb/daemon/file_sync_service.cpp index edf5683d3..07f6e65e8 100644 --- a/adb/daemon/file_sync_service.cpp +++ b/adb/daemon/file_sync_service.cpp @@ -32,6 +32,8 @@ #include <utime.h> #include <memory> +#include <optional> +#include <span> #include <string> #include <vector> @@ -55,10 +57,12 @@ #include "adb_io.h" #include "adb_trace.h" #include "adb_utils.h" +#include "brotli_utils.h" #include "file_sync_protocol.h" #include "security_log_tags.h" #include "sysdeps/errno.h" +using android::base::borrowed_fd; using android::base::Dirname; using android::base::StringPrintf; @@ -249,7 +253,7 @@ static bool do_list_v2(int s, const char* path) { // Make sure that SendFail from adb_io.cpp isn't accidentally used in this file. #pragma GCC poison SendFail -static bool SendSyncFail(int fd, const std::string& reason) { +static bool SendSyncFail(borrowed_fd fd, const std::string& reason) { D("sync: failure: %s", reason.c_str()); syncmsg msg; @@ -258,13 +262,89 @@ static bool SendSyncFail(int fd, const std::string& reason) { return WriteFdExactly(fd, &msg.data, sizeof(msg.data)) && WriteFdExactly(fd, reason); } -static bool SendSyncFailErrno(int fd, const std::string& reason) { +static bool SendSyncFailErrno(borrowed_fd fd, const std::string& reason) { return SendSyncFail(fd, StringPrintf("%s: %s", reason.c_str(), strerror(errno))); } -static bool handle_send_file(int s, const char* path, uint32_t* timestamp, uid_t uid, gid_t gid, - uint64_t capabilities, mode_t mode, std::vector<char>& buffer, - bool do_unlink) { +static bool handle_send_file_compressed(borrowed_fd s, unique_fd fd, uint32_t* timestamp) { + syncmsg msg; + Block decode_buffer(SYNC_DATA_MAX); + BrotliDecoder decoder(std::span(decode_buffer.data(), decode_buffer.size())); + while (true) { + if (!ReadFdExactly(s, &msg.data, sizeof(msg.data))) return false; + + if (msg.data.id != ID_DATA) { + if (msg.data.id == ID_DONE) { + *timestamp = msg.data.size; + return true; + } + SendSyncFail(s, "invalid data message"); + return false; + } + + Block block(msg.data.size); + if (!ReadFdExactly(s, block.data(), msg.data.size)) return false; + decoder.Append(std::move(block)); + + while (true) { + std::span<char> output; + BrotliDecodeResult result = decoder.Decode(&output); + if (result == BrotliDecodeResult::Error) { + SendSyncFailErrno(s, "decompress failed"); + return false; + } + + if (!WriteFdExactly(fd, output.data(), output.size())) { + SendSyncFailErrno(s, "write failed"); + return false; + } + + if (result == BrotliDecodeResult::NeedInput) { + break; + } else if (result == BrotliDecodeResult::MoreOutput) { + continue; + } else if (result == BrotliDecodeResult::Done) { + break; + } else { + LOG(FATAL) << "invalid BrotliDecodeResult: " << static_cast<int>(result); + } + } + } + + __builtin_unreachable(); +} + +static bool handle_send_file_uncompressed(borrowed_fd s, unique_fd fd, uint32_t* timestamp, + std::vector<char>& buffer) { + syncmsg msg; + + while (true) { + if (!ReadFdExactly(s, &msg.data, sizeof(msg.data))) return false; + + if (msg.data.id != ID_DATA) { + if (msg.data.id == ID_DONE) { + *timestamp = msg.data.size; + return true; + } + SendSyncFail(s, "invalid data message"); + return false; + } + + if (msg.data.size > buffer.size()) { // TODO: resize buffer? + SendSyncFail(s, "oversize data message"); + return false; + } + if (!ReadFdExactly(s, &buffer[0], msg.data.size)) return false; + if (!WriteFdExactly(fd, &buffer[0], msg.data.size)) { + SendSyncFailErrno(s, "write failed"); + return false; + } + } +} + +static bool handle_send_file(borrowed_fd s, const char* path, uint32_t* timestamp, uid_t uid, + gid_t gid, uint64_t capabilities, mode_t mode, bool compressed, + std::vector<char>& buffer, bool do_unlink) { int rc; syncmsg msg; @@ -302,46 +382,34 @@ static bool handle_send_file(int s, const char* path, uint32_t* timestamp, uid_t fchmod(fd.get(), mode); } - rc = posix_fadvise(fd.get(), 0, 0, - POSIX_FADV_SEQUENTIAL | POSIX_FADV_NOREUSE | POSIX_FADV_WILLNEED); - if (rc != 0) { - D("[ Failed to fadvise: %s ]", strerror(rc)); - } - - while (true) { - if (!ReadFdExactly(s, &msg.data, sizeof(msg.data))) goto fail; - - if (msg.data.id != ID_DATA) { - if (msg.data.id == ID_DONE) { - *timestamp = msg.data.size; - break; - } - SendSyncFail(s, "invalid data message"); - goto abort; + { + rc = posix_fadvise(fd.get(), 0, 0, + POSIX_FADV_SEQUENTIAL | POSIX_FADV_NOREUSE | POSIX_FADV_WILLNEED); + if (rc != 0) { + D("[ Failed to fadvise: %s ]", strerror(rc)); } - if (msg.data.size > buffer.size()) { // TODO: resize buffer? - SendSyncFail(s, "oversize data message"); - goto abort; + bool result; + if (compressed) { + result = handle_send_file_compressed(s, std::move(fd), timestamp); + } else { + result = handle_send_file_uncompressed(s, std::move(fd), timestamp, buffer); } - if (!ReadFdExactly(s, &buffer[0], msg.data.size)) goto abort; + if (!result) { + goto fail; + } - if (!WriteFdExactly(fd.get(), &buffer[0], msg.data.size)) { - SendSyncFailErrno(s, "write failed"); + if (!update_capabilities(path, capabilities)) { + SendSyncFailErrno(s, "update_capabilities failed"); goto fail; } - } - if (!update_capabilities(path, capabilities)) { - SendSyncFailErrno(s, "update_capabilities failed"); - goto fail; + msg.status.id = ID_OKAY; + msg.status.msglen = 0; + return WriteFdExactly(s, &msg.status, sizeof(msg.status)); } - msg.status.id = ID_OKAY; - msg.status.msglen = 0; - return WriteFdExactly(s, &msg.status, sizeof(msg.status)); - fail: // If there's a problem on the device, we'll send an ID_FAIL message and // close the socket. Unfortunately the kernel will sometimes throw that @@ -371,7 +439,6 @@ fail: if (!ReadFdExactly(s, &buffer[0], msg.data.size)) break; } -abort: if (do_unlink) adb_unlink(path); return false; } @@ -432,23 +499,8 @@ static bool handle_send_link(int s, const std::string& path, uint32_t* timestamp } #endif -static bool do_send(int s, const std::string& spec, std::vector<char>& buffer) { - // 'spec' is of the form "/some/path,0755". Break it up. - size_t comma = spec.find_last_of(','); - if (comma == std::string::npos) { - SendSyncFail(s, "missing , in ID_SEND"); - return false; - } - - std::string path = spec.substr(0, comma); - - errno = 0; - mode_t mode = strtoul(spec.substr(comma + 1).c_str(), nullptr, 0); - if (errno != 0) { - SendSyncFail(s, "bad mode"); - return false; - } - +static bool send_impl(int s, const std::string& path, mode_t mode, bool compressed, + std::vector<char>& buffer) { // Don't delete files before copying if they are not "regular" or symlinks. struct stat st; bool do_unlink = (lstat(path.c_str(), &st) == -1) || S_ISREG(st.st_mode) || @@ -474,8 +526,8 @@ static bool do_send(int s, const std::string& spec, std::vector<char>& buffer) { adbd_fs_config(path.c_str(), 0, nullptr, &uid, &gid, &mode, &capabilities); } - result = handle_send_file(s, path.c_str(), ×tamp, uid, gid, capabilities, mode, buffer, - do_unlink); + result = handle_send_file(s, path.c_str(), ×tamp, uid, gid, capabilities, mode, + compressed, buffer, do_unlink); } if (!result) { @@ -491,22 +543,55 @@ static bool do_send(int s, const std::string& spec, std::vector<char>& buffer) { return true; } -static bool do_recv(int s, const char* path, std::vector<char>& buffer) { - __android_log_security_bswrite(SEC_TAG_ADB_RECV_FILE, path); +static bool do_send_v1(int s, const std::string& spec, std::vector<char>& buffer) { + // 'spec' is of the form "/some/path,0755". Break it up. + size_t comma = spec.find_last_of(','); + if (comma == std::string::npos) { + SendSyncFail(s, "missing , in ID_SEND_V1"); + return false; + } - unique_fd fd(adb_open(path, O_RDONLY | O_CLOEXEC)); - if (fd < 0) { - SendSyncFailErrno(s, "open failed"); + std::string path = spec.substr(0, comma); + + errno = 0; + mode_t mode = strtoul(spec.substr(comma + 1).c_str(), nullptr, 0); + if (errno != 0) { + SendSyncFail(s, "bad mode"); return false; } - int rc = posix_fadvise(fd.get(), 0, 0, POSIX_FADV_SEQUENTIAL | POSIX_FADV_NOREUSE); - if (rc != 0) { - D("[ Failed to fadvise: %s ]", strerror(rc)); + return send_impl(s, path, mode, false, buffer); +} + +static bool do_send_v2(int s, const std::string& path, std::vector<char>& buffer) { + // Read the setup packet. + syncmsg msg; + int rc = ReadFdExactly(s, &msg.send_v2_setup, sizeof(msg.send_v2_setup)); + if (rc == 0) { + LOG(ERROR) << "failed to read send_v2 setup packet: EOF"; + return false; + } else if (rc < 0) { + PLOG(ERROR) << "failed to read send_v2 setup packet"; } + bool compressed = false; + if (msg.send_v2_setup.flags & kSyncFlagBrotli) { + msg.send_v2_setup.flags &= ~kSyncFlagBrotli; + compressed = true; + } + if (msg.send_v2_setup.flags) { + SendSyncFail(s, android::base::StringPrintf("unknown flags: %d", msg.send_v2_setup.flags)); + return false; + } + + errno = 0; + return send_impl(s, path, msg.send_v2_setup.mode, compressed, buffer); +} + +static bool recv_uncompressed(borrowed_fd s, unique_fd fd, std::vector<char>& buffer) { syncmsg msg; msg.data.id = ID_DATA; + std::optional<BrotliEncoder<SYNC_DATA_MAX>> encoder; while (true) { int r = adb_read(fd.get(), &buffer[0], buffer.size() - sizeof(msg.data)); if (r <= 0) { @@ -515,16 +600,126 @@ static bool do_recv(int s, const char* path, std::vector<char>& buffer) { return false; } msg.data.size = r; + if (!WriteFdExactly(s, &msg.data, sizeof(msg.data)) || !WriteFdExactly(s, &buffer[0], r)) { return false; } } + return true; +} + +static bool recv_compressed(borrowed_fd s, unique_fd fd) { + syncmsg msg; + msg.data.id = ID_DATA; + + BrotliEncoder<SYNC_DATA_MAX> encoder; + + bool sending = true; + while (sending) { + Block input(SYNC_DATA_MAX); + int r = adb_read(fd.get(), input.data(), input.size()); + if (r < 0) { + SendSyncFailErrno(s, "read failed"); + return false; + } + + if (r == 0) { + encoder.Finish(); + } else { + input.resize(r); + encoder.Append(std::move(input)); + } + + while (true) { + Block output; + BrotliEncodeResult result = encoder.Encode(&output); + if (result == BrotliEncodeResult::Error) { + SendSyncFailErrno(s, "compress failed"); + return false; + } + + if (!output.empty()) { + msg.data.size = output.size(); + if (!WriteFdExactly(s, &msg.data, sizeof(msg.data)) || + !WriteFdExactly(s, output.data(), output.size())) { + return false; + } + } + + if (result == BrotliEncodeResult::Done) { + sending = false; + break; + } else if (result == BrotliEncodeResult::NeedInput) { + break; + } else if (result == BrotliEncodeResult::MoreOutput) { + continue; + } + } + } + + return true; +} + +static bool recv_impl(borrowed_fd s, const char* path, bool compressed, std::vector<char>& buffer) { + __android_log_security_bswrite(SEC_TAG_ADB_RECV_FILE, path); + + unique_fd fd(adb_open(path, O_RDONLY | O_CLOEXEC)); + if (fd < 0) { + SendSyncFailErrno(s, "open failed"); + return false; + } + + int rc = posix_fadvise(fd.get(), 0, 0, POSIX_FADV_SEQUENTIAL | POSIX_FADV_NOREUSE); + if (rc != 0) { + D("[ Failed to fadvise: %s ]", strerror(rc)); + } + + bool result; + if (compressed) { + result = recv_compressed(s, std::move(fd)); + } else { + result = recv_uncompressed(s, std::move(fd), buffer); + } + + if (!result) { + return false; + } + + syncmsg msg; msg.data.id = ID_DONE; msg.data.size = 0; return WriteFdExactly(s, &msg.data, sizeof(msg.data)); } +static bool do_recv_v1(borrowed_fd s, const char* path, std::vector<char>& buffer) { + return recv_impl(s, path, false, buffer); +} + +static bool do_recv_v2(borrowed_fd s, const char* path, std::vector<char>& buffer) { + syncmsg msg; + // Read the setup packet. + int rc = ReadFdExactly(s, &msg.recv_v2_setup, sizeof(msg.recv_v2_setup)); + if (rc == 0) { + LOG(ERROR) << "failed to read recv_v2 setup packet: EOF"; + return false; + } else if (rc < 0) { + PLOG(ERROR) << "failed to read recv_v2 setup packet"; + } + + bool compressed = false; + if (msg.recv_v2_setup.flags & kSyncFlagBrotli) { + msg.recv_v2_setup.flags &= ~kSyncFlagBrotli; + compressed = true; + } + if (msg.recv_v2_setup.flags) { + SendSyncFail(s, android::base::StringPrintf("unknown flags: %d", msg.recv_v2_setup.flags)); + return false; + } + + return recv_impl(s, path, compressed, buffer); +} + static const char* sync_id_to_name(uint32_t id) { switch (id) { case ID_LSTAT_V1: @@ -537,10 +732,14 @@ static const char* sync_id_to_name(uint32_t id) { return "list_v1"; case ID_LIST_V2: return "list_v2"; - case ID_SEND: - return "send"; - case ID_RECV: - return "recv"; + case ID_SEND_V1: + return "send_v1"; + case ID_SEND_V2: + return "send_v2"; + case ID_RECV_V1: + return "recv_v1"; + case ID_RECV_V2: + return "recv_v2"; case ID_QUIT: return "quit"; default: @@ -585,11 +784,17 @@ static bool handle_sync_command(int fd, std::vector<char>& buffer) { case ID_LIST_V2: if (!do_list_v2(fd, name)) return false; break; - case ID_SEND: - if (!do_send(fd, name, buffer)) return false; + case ID_SEND_V1: + if (!do_send_v1(fd, name, buffer)) return false; + break; + case ID_SEND_V2: + if (!do_send_v2(fd, name, buffer)) return false; + break; + case ID_RECV_V1: + if (!do_recv_v1(fd, name, buffer)) return false; break; - case ID_RECV: - if (!do_recv(fd, name, buffer)) return false; + case ID_RECV_V2: + if (!do_recv_v2(fd, name, buffer)) return false; break; case ID_QUIT: return false; diff --git a/adb/daemon/jdwp_service.cpp b/adb/daemon/jdwp_service.cpp index c99aead9c..adae9f7f4 100644 --- a/adb/daemon/jdwp_service.cpp +++ b/adb/daemon/jdwp_service.cpp @@ -16,6 +16,7 @@ #if !ADB_HOST +#if !defined(__ANDROID_RECOVERY__) #define TRACE_TAG JDWP #include "sysdeps.h" @@ -459,7 +460,7 @@ static int jdwp_tracker_enqueue(asocket* s, apacket::payload_type) { return -1; } -asocket* create_process_tracker_service_socket(TrackerKind kind) { +static asocket* create_process_tracker_service_socket(TrackerKind kind) { auto t = std::make_unique<JdwpTracker>(kind, true); if (!t) { LOG(FATAL) << "failed to allocate JdwpTracker"; @@ -509,4 +510,28 @@ int init_jdwp(void) { return 0; } +#else // !defined(__ANDROID_RECOVERY) +#include "adb.h" + +asocket* create_jdwp_service_socket(void) { + return nullptr; +} + +unique_fd create_jdwp_connection_fd(int pid) { + return {}; +} + +asocket* create_app_tracker_service_socket() { + return nullptr; +} + +asocket* create_jdwp_tracker_service_socket() { + return nullptr; +} + +int init_jdwp() { + return 0; +} + +#endif /* defined(__ANDROID_RECOVERY__) */ #endif /* !ADB_HOST */ diff --git a/adb/fdevent/fdevent.cpp b/adb/fdevent/fdevent.cpp index 562f5872d..fd550200f 100644 --- a/adb/fdevent/fdevent.cpp +++ b/adb/fdevent/fdevent.cpp @@ -63,7 +63,10 @@ fdevent* fdevent_context::Create(unique_fd fd, std::variant<fd_func, fd_func2> f int fd_num = fd.get(); - fdevent* fde = new fdevent(); + auto [it, inserted] = this->installed_fdevents_.emplace(fd_num, fdevent{}); + CHECK(inserted); + + fdevent* fde = &it->second; fde->id = fdevent_id_++; fde->state = 0; fde->fd = std::move(fd); @@ -76,10 +79,6 @@ fdevent* fdevent_context::Create(unique_fd fd, std::variant<fd_func, fd_func2> f LOG(ERROR) << "failed to set non-blocking mode for fd " << fde->fd.get(); } - auto [it, inserted] = this->installed_fdevents_.emplace(fd_num, fde); - CHECK(inserted); - UNUSED(it); - this->Register(fde); return fde; } @@ -92,12 +91,12 @@ unique_fd fdevent_context::Destroy(fdevent* fde) { this->Unregister(fde); - auto erased = this->installed_fdevents_.erase(fde->fd.get()); + unique_fd fd = std::move(fde->fd); + + auto erased = this->installed_fdevents_.erase(fd.get()); CHECK_EQ(1UL, erased); - unique_fd result = std::move(fde->fd); - delete fde; - return result; + return fd; } void fdevent_context::Add(fdevent* fde, unsigned events) { @@ -123,9 +122,9 @@ std::optional<std::chrono::milliseconds> fdevent_context::CalculatePollDuration( for (const auto& [fd, fde] : this->installed_fdevents_) { UNUSED(fd); - auto timeout_opt = fde->timeout; + auto timeout_opt = fde.timeout; if (timeout_opt) { - auto deadline = fde->last_active + *timeout_opt; + auto deadline = fde.last_active + *timeout_opt; auto time_left = duration_cast<std::chrono::milliseconds>(deadline - now); if (time_left < 0ms) { time_left = 0ms; @@ -194,11 +193,13 @@ static std::unique_ptr<fdevent_context> fdevent_create_context() { #endif } -static auto& g_ambient_fdevent_context = - *new std::unique_ptr<fdevent_context>(fdevent_create_context()); +static auto& g_ambient_fdevent_context() { + static auto context = fdevent_create_context().release(); + return context; +} static fdevent_context* fdevent_get_ambient() { - return g_ambient_fdevent_context.get(); + return g_ambient_fdevent_context(); } fdevent* fdevent_create(int fd, fd_func func, void* arg) { @@ -256,5 +257,6 @@ size_t fdevent_installed_count() { } void fdevent_reset() { - g_ambient_fdevent_context = fdevent_create_context(); + auto old = std::exchange(g_ambient_fdevent_context(), fdevent_create_context().release()); + delete old; } diff --git a/adb/fdevent/fdevent.h b/adb/fdevent/fdevent.h index 86814d7c5..9fc3b2c0c 100644 --- a/adb/fdevent/fdevent.h +++ b/adb/fdevent/fdevent.h @@ -52,6 +52,20 @@ struct fdevent_event { unsigned events; }; +struct fdevent final { + uint64_t id; + + unique_fd fd; + int force_eof = 0; + + uint16_t state = 0; + std::optional<std::chrono::milliseconds> timeout; + std::chrono::steady_clock::time_point last_active; + + std::variant<fd_func, fd_func2> func; + void* arg = nullptr; +}; + struct fdevent_context { public: virtual ~fdevent_context() = default; @@ -113,7 +127,7 @@ struct fdevent_context { std::atomic<bool> terminate_loop_ = false; protected: - std::unordered_map<int, fdevent*> installed_fdevents_; + std::unordered_map<int, fdevent> installed_fdevents_; private: uint64_t fdevent_id_ = 0; @@ -121,20 +135,6 @@ struct fdevent_context { std::deque<std::function<void()>> run_queue_ GUARDED_BY(run_queue_mutex_); }; -struct fdevent { - uint64_t id; - - unique_fd fd; - int force_eof = 0; - - uint16_t state = 0; - std::optional<std::chrono::milliseconds> timeout; - std::chrono::steady_clock::time_point last_active; - - std::variant<fd_func, fd_func2> func; - void* arg = nullptr; -}; - // Backwards compatibility shims that forward to the global fdevent_context. fdevent* fdevent_create(int fd, fd_func func, void* arg); fdevent* fdevent_create(int fd, fd_func2 func, void* arg); diff --git a/adb/fdevent/fdevent_epoll.cpp b/adb/fdevent/fdevent_epoll.cpp index e3d167424..4ef41d1ee 100644 --- a/adb/fdevent/fdevent_epoll.cpp +++ b/adb/fdevent/fdevent_epoll.cpp @@ -155,15 +155,15 @@ void fdevent_context_epoll::Loop() { event_map[fde] = events; } - for (const auto& [fd, fde] : installed_fdevents_) { + for (auto& [fd, fde] : installed_fdevents_) { unsigned events = 0; - if (auto it = event_map.find(fde); it != event_map.end()) { + if (auto it = event_map.find(&fde); it != event_map.end()) { events = it->second; } if (events == 0) { - if (fde->timeout) { - auto deadline = fde->last_active + *fde->timeout; + if (fde.timeout) { + auto deadline = fde.last_active + *fde.timeout; if (deadline < post_poll) { events |= FDE_TIMEOUT; } @@ -171,13 +171,13 @@ void fdevent_context_epoll::Loop() { } if (events != 0) { - LOG(DEBUG) << dump_fde(fde) << " got events " << std::hex << std::showbase + LOG(DEBUG) << dump_fde(&fde) << " got events " << std::hex << std::showbase << events; - fde_events.push_back({fde, events}); - fde->last_active = post_poll; + fde_events.push_back({&fde, events}); + fde.last_active = post_poll; } } - this->HandleEvents(std::move(fde_events)); + this->HandleEvents(fde_events); fde_events.clear(); } diff --git a/adb/fdevent/fdevent_epoll.h b/adb/fdevent/fdevent_epoll.h index 684fa32bc..6214d2e27 100644 --- a/adb/fdevent/fdevent_epoll.h +++ b/adb/fdevent/fdevent_epoll.h @@ -47,12 +47,7 @@ struct fdevent_context_epoll final : public fdevent_context { protected: virtual void Interrupt() final; - public: - // All operations to fdevent should happen only in the main thread. - // That's why we don't need a lock for fdevent. - std::unordered_map<int, fdevent*> epoll_node_map_; - std::list<fdevent*> pending_list_; - + private: unique_fd epoll_fd_; unique_fd interrupt_fd_; fdevent* interrupt_fde_ = nullptr; diff --git a/adb/fdevent/fdevent_poll.cpp b/adb/fdevent/fdevent_poll.cpp index cc4a7a151..ac86c08e1 100644 --- a/adb/fdevent/fdevent_poll.cpp +++ b/adb/fdevent/fdevent_poll.cpp @@ -103,24 +103,27 @@ static std::string dump_pollfds(const std::vector<adb_pollfd>& pollfds) { void fdevent_context_poll::Loop() { main_thread_id_ = android::base::GetThreadId(); + std::vector<adb_pollfd> pollfds; + std::vector<fdevent_event> poll_events; + while (true) { if (terminate_loop_) { break; } D("--- --- waiting for events"); - std::vector<adb_pollfd> pollfds; + pollfds.clear(); for (const auto& [fd, fde] : this->installed_fdevents_) { adb_pollfd pfd; pfd.fd = fd; pfd.events = 0; - if (fde->state & FDE_READ) { + if (fde.state & FDE_READ) { pfd.events |= POLLIN; } - if (fde->state & FDE_WRITE) { + if (fde.state & FDE_WRITE) { pfd.events |= POLLOUT; } - if (fde->state & FDE_ERROR) { + if (fde.state & FDE_ERROR) { pfd.events |= POLLERR; } #if defined(__linux__) @@ -147,7 +150,6 @@ void fdevent_context_poll::Loop() { } auto post_poll = std::chrono::steady_clock::now(); - std::vector<fdevent_event> poll_events; for (const auto& pollfd : pollfds) { unsigned events = 0; @@ -170,7 +172,7 @@ void fdevent_context_poll::Loop() { auto it = this->installed_fdevents_.find(pollfd.fd); CHECK(it != this->installed_fdevents_.end()); - fdevent* fde = it->second; + fdevent* fde = &it->second; if (events == 0) { if (fde->timeout) { @@ -187,7 +189,8 @@ void fdevent_context_poll::Loop() { fde->last_active = post_poll; } } - this->HandleEvents(std::move(poll_events)); + this->HandleEvents(poll_events); + poll_events.clear(); } main_thread_id_.reset(); diff --git a/adb/file_sync_protocol.h b/adb/file_sync_protocol.h index 508c13851..fd9a5169e 100644 --- a/adb/file_sync_protocol.h +++ b/adb/file_sync_protocol.h @@ -27,8 +27,10 @@ #define ID_DENT_V1 MKID('D', 'E', 'N', 'T') #define ID_DENT_V2 MKID('D', 'N', 'T', '2') -#define ID_SEND MKID('S', 'E', 'N', 'D') -#define ID_RECV MKID('R', 'E', 'C', 'V') +#define ID_SEND_V1 MKID('S', 'E', 'N', 'D') +#define ID_SEND_V2 MKID('S', 'N', 'D', '2') +#define ID_RECV_V1 MKID('R', 'E', 'C', 'V') +#define ID_RECV_V2 MKID('R', 'C', 'V', '2') #define ID_DONE MKID('D', 'O', 'N', 'E') #define ID_DATA MKID('D', 'A', 'T', 'A') #define ID_OKAY MKID('O', 'K', 'A', 'Y') @@ -41,57 +43,91 @@ struct SyncRequest { // Followed by 'path_length' bytes of path (not NUL-terminated). } __attribute__((packed)); +struct __attribute__((packed)) sync_stat_v1 { + uint32_t id; + uint32_t mode; + uint32_t size; + uint32_t mtime; +}; + +struct __attribute__((packed)) sync_stat_v2 { + uint32_t id; + uint32_t error; + uint64_t dev; + uint64_t ino; + uint32_t mode; + uint32_t nlink; + uint32_t uid; + uint32_t gid; + uint64_t size; + int64_t atime; + int64_t mtime; + int64_t ctime; +}; + +struct __attribute__((packed)) sync_dent_v1 { + uint32_t id; + uint32_t mode; + uint32_t size; + uint32_t mtime; + uint32_t namelen; +}; // followed by `namelen` bytes of the name. + +struct __attribute__((packed)) sync_dent_v2 { + uint32_t id; + uint32_t error; + uint64_t dev; + uint64_t ino; + uint32_t mode; + uint32_t nlink; + uint32_t uid; + uint32_t gid; + uint64_t size; + int64_t atime; + int64_t mtime; + int64_t ctime; + uint32_t namelen; +}; // followed by `namelen` bytes of the name. + +enum SyncFlag : uint32_t { + kSyncFlagNone = 0, + kSyncFlagBrotli = 1, +}; + +// send_v1 sent the path in a buffer, followed by a comma and the mode as a string. +// send_v2 sends just the path in the first request, and then sends another syncmsg (with the +// same ID!) with details. +struct __attribute__((packed)) sync_send_v2 { + uint32_t id; + uint32_t mode; + uint32_t flags; +}; + +// Likewise, recv_v1 just sent the path without any accompanying data. +struct __attribute__((packed)) sync_recv_v2 { + uint32_t id; + uint32_t flags; +}; + +struct __attribute__((packed)) sync_data { + uint32_t id; + uint32_t size; +}; // followed by `size` bytes of data. + +struct __attribute__((packed)) sync_status { + uint32_t id; + uint32_t msglen; +}; // followed by `msglen` bytes of error message, if id == ID_FAIL. + union syncmsg { - struct __attribute__((packed)) { - uint32_t id; - uint32_t mode; - uint32_t size; - uint32_t mtime; - } stat_v1; - struct __attribute__((packed)) { - uint32_t id; - uint32_t error; - uint64_t dev; - uint64_t ino; - uint32_t mode; - uint32_t nlink; - uint32_t uid; - uint32_t gid; - uint64_t size; - int64_t atime; - int64_t mtime; - int64_t ctime; - } stat_v2; - struct __attribute__((packed)) { - uint32_t id; - uint32_t mode; - uint32_t size; - uint32_t mtime; - uint32_t namelen; - } dent_v1; // followed by `namelen` bytes of the name. - struct __attribute__((packed)) { - uint32_t id; - uint32_t error; - uint64_t dev; - uint64_t ino; - uint32_t mode; - uint32_t nlink; - uint32_t uid; - uint32_t gid; - uint64_t size; - int64_t atime; - int64_t mtime; - int64_t ctime; - uint32_t namelen; - } dent_v2; // followed by `namelen` bytes of the name. - struct __attribute__((packed)) { - uint32_t id; - uint32_t size; - } data; // followed by `size` bytes of data. - struct __attribute__((packed)) { - uint32_t id; - uint32_t msglen; - } status; // followed by `msglen` bytes of error message, if id == ID_FAIL. + sync_stat_v1 stat_v1; + sync_stat_v2 stat_v2; + sync_dent_v1 dent_v1; + sync_dent_v2 dent_v2; + sync_data data; + sync_status status; + sync_send_v2 send_v2_setup; + sync_recv_v2 recv_v2_setup; }; #define SYNC_DATA_MAX (64 * 1024) diff --git a/adb/libs/adbconnection/Android.bp b/adb/libs/adbconnection/Android.bp index f6b0a4239..ce2ab51ef 100644 --- a/adb/libs/adbconnection/Android.bp +++ b/adb/libs/adbconnection/Android.bp @@ -18,6 +18,11 @@ cc_library { use_version_lib: false, recovery_available: true, + apex_available: [ + "com.android.adbd", + // TODO(b/151398197) remove the below + "//apex_available:platform", + ], compile_multilib: "both", } diff --git a/adb/pairing_auth/aes_128_gcm.cpp b/adb/pairing_auth/aes_128_gcm.cpp index 2978834d8..51520d814 100644 --- a/adb/pairing_auth/aes_128_gcm.cpp +++ b/adb/pairing_auth/aes_128_gcm.cpp @@ -19,7 +19,6 @@ #include <android-base/endian.h> #include <android-base/logging.h> -#include <openssl/crypto.h> #include <openssl/evp.h> #include <openssl/hkdf.h> #include <openssl/rand.h> @@ -28,155 +27,64 @@ namespace adb { namespace pairing { namespace { -static const size_t kHkdfKeyLength = 256; - -struct Header { - uint32_t payload; - uint8_t iv[AES_128_GCM_IV_SIZE]; - uint8_t tag[AES_128_GCM_TAG_SIZE]; -} __attribute__((packed)); +// Size of AES-128-GCM key, in bytes +static constexpr size_t kHkdfKeyLength = 16; } // namespace -// static -const EVP_CIPHER* Aes128Gcm::cipher_ = EVP_aes_128_gcm(); - Aes128Gcm::Aes128Gcm(const uint8_t* key_material, size_t key_material_len) { CHECK(key_material); CHECK_NE(key_material_len, 0ul); - context_.reset(EVP_CIPHER_CTX_new()); - CHECK(context_.get()); - - // Start with a random number for our counter - CHECK_EQ(RAND_bytes(counter_.data(), counter_.size()), 1); - uint8_t key[kHkdfKeyLength] = {}; - uint8_t salt[64] = "this is the salt"; - uint8_t info[64] = "this is the info"; - CHECK_EQ(HKDF(key, sizeof(key), EVP_sha256(), key_material, key_material_len, salt, - sizeof(salt), info, sizeof(info)), + uint8_t key[kHkdfKeyLength]; + uint8_t info[] = "adb pairing_auth aes-128-gcm key"; + CHECK_EQ(HKDF(key, sizeof(key), EVP_sha256(), key_material, key_material_len, nullptr, 0, info, + sizeof(info) - 1), 1); - CHECK_EQ(AES_set_encrypt_key(key, sizeof(key), &aes_key_), 0); + CHECK(EVP_AEAD_CTX_init(context_.get(), EVP_aead_aes_128_gcm(), key, sizeof(key), + EVP_AEAD_DEFAULT_TAG_LENGTH, nullptr)); } -int Aes128Gcm::Encrypt(const uint8_t* in, size_t in_len, uint8_t* out, size_t out_len) { - if (out_len < EncryptedSize(in_len)) { - LOG(ERROR) << "out buffer size (sz=" << out_len - << ") not big enough (sz=" << EncryptedSize(in_len) << ")"; - return -1; - } - auto& header = *reinterpret_cast<Header*>(out); - // Place the IV in the header - memcpy(header.iv, counter_.data(), counter_.size()); - int status = EVP_EncryptInit_ex(context_.get(), cipher_, nullptr, - reinterpret_cast<const uint8_t*>(&aes_key_), counter_.data()); - counter_.Increase(); - if (status != 1) { - return -1; - } - - int cipherLen = 0; - out += sizeof(header); - status = EVP_EncryptUpdate(context_.get(), out, &cipherLen, in, in_len); - if (status != 1 || cipherLen < 0) { - return -1; +std::optional<size_t> Aes128Gcm::Encrypt(const uint8_t* in, size_t in_len, uint8_t* out, + size_t out_len) { + std::vector<uint8_t> nonce(EVP_AEAD_nonce_length(EVP_AEAD_CTX_aead(context_.get())), 0); + memcpy(nonce.data(), &enc_sequence_, sizeof(enc_sequence_)); + size_t written_sz; + if (!EVP_AEAD_CTX_seal(context_.get(), out, &written_sz, out_len, nonce.data(), nonce.size(), + in, in_len, nullptr, 0)) { + LOG(ERROR) << "Failed to encrypt (in_len=" << in_len << ", out_len=" << out_len + << ", out_len_needed=" << EncryptedSize(in_len) << ")"; + return std::nullopt; } - // Padding is enabled by default, so EVP_EncryptFinal_ex will pad any - // remaining partial data up to the block size. - int padding = 0; - status = EVP_EncryptFinal_ex(context_.get(), out + cipherLen, &padding); - if (status != 1 || padding < 0) { - return -1; - } - - // Place the tag in the header - status = EVP_CIPHER_CTX_ctrl(context_.get(), EVP_CTRL_GCM_GET_TAG, sizeof(header.tag), - header.tag); - if (status != 1) { - return -1; - } - // Place the payload size in the header - uint32_t totalLen = sizeof(header) + cipherLen + padding; - header.payload = htonl(static_cast<uint32_t>(cipherLen) + static_cast<uint32_t>(padding)); - return totalLen; + ++enc_sequence_; + return written_sz; } -int Aes128Gcm::Decrypt(const uint8_t* in, size_t in_len, uint8_t* out, size_t out_len) { - if (in_len < sizeof(Header)) { - return 0; - } - if (out_len < DecryptedSize(in, in_len)) { - return 0; - } - const auto& header = *reinterpret_cast<const Header*>(in); - uint32_t payload = ntohl(header.payload); - uint32_t expected_inlen = sizeof(Header) + payload; - if (in_len < expected_inlen) { - // Not enough data available - return 0; - } - // Initialized with expected IV from header - int status = EVP_DecryptInit_ex(context_.get(), cipher_, nullptr, - reinterpret_cast<const uint8_t*>(&aes_key_), header.iv); - if (status != 1) { - return -1; - } - - int decrypted_len = 0; - status = EVP_DecryptUpdate(context_.get(), out, &decrypted_len, in + sizeof(header), payload); - if (status != 1 || decrypted_len < 0) { - return -1; - } - - // Set expected tag from header - status = EVP_CIPHER_CTX_ctrl(context_.get(), EVP_CTRL_GCM_SET_TAG, sizeof(header.tag), - const_cast<uint8_t*>(header.tag)); - if (status != 1) { - return -1; +std::optional<size_t> Aes128Gcm::Decrypt(const uint8_t* in, size_t in_len, uint8_t* out, + size_t out_len) { + std::vector<uint8_t> nonce(EVP_AEAD_nonce_length(EVP_AEAD_CTX_aead(context_.get())), 0); + memcpy(nonce.data(), &dec_sequence_, sizeof(dec_sequence_)); + size_t written_sz; + if (!EVP_AEAD_CTX_open(context_.get(), out, &written_sz, out_len, nonce.data(), nonce.size(), + in, in_len, nullptr, 0)) { + LOG(ERROR) << "Failed to decrypt (in_len=" << in_len << ", out_len=" << out_len + << ", out_len_needed=" << DecryptedSize(in_len) << ")"; + return std::nullopt; } - // This is the padding. It can be ignored. - int len = 0; - status = EVP_DecryptFinal_ex(context_.get(), out + decrypted_len, &len); - if (status != 1) { - LOG(ERROR) << "EVP_DecryptFinal_ex failed. Tag mismatch"; - return -1; - } - - // Return the length without the padding. - return decrypted_len; + ++dec_sequence_; + return written_sz; } size_t Aes128Gcm::EncryptedSize(size_t size) { - // We need to account for block alignment of the encrypted data. - // According to openssl.org/docs/man1.0.2/man3/EVP_EncryptUpdate.html, - // "The amount of data written depends on the block alignment of the - // encrypted data ..." - // ".. the amount of data written may be anything from zero bytes to - // (inl + cipher_block_size - 1) ..." - const size_t cipher_block_size = EVP_CIPHER_block_size(cipher_); - size_t padding = cipher_block_size - (size % cipher_block_size); - if (padding != cipher_block_size) { - size += padding; - } - return size + sizeof(Header); + // https://commondatastorage.googleapis.com/chromium-boringssl-docs/aead.h.html#EVP_AEAD_CTX_seal + return size + EVP_AEAD_max_overhead(EVP_AEAD_CTX_aead(context_.get())); } -size_t Aes128Gcm::DecryptedSize(const uint8_t* encrypted_data, size_t encrypted_size) { - if (encrypted_size < sizeof(Header)) { - // Not enough data yet - return 0; - } - auto header = reinterpret_cast<const Header*>(encrypted_data); - uint32_t payload = ntohl(header->payload); - size_t total_size = payload + sizeof(Header); - if (encrypted_size < total_size) { - // There's enough data for the header but not enough data for the - // payload. Indicate that there's not enough data for now. - return 0; - } - return payload; +size_t Aes128Gcm::DecryptedSize(size_t size) { + // https://commondatastorage.googleapis.com/chromium-boringssl-docs/aead.h.html#EVP_AEAD_CTX_open + return size; } } // namespace pairing diff --git a/adb/pairing_auth/include/adb/pairing/aes_128_gcm.h b/adb/pairing_auth/include/adb/pairing/aes_128_gcm.h index 490dd12a3..6be585690 100644 --- a/adb/pairing_auth/include/adb/pairing/aes_128_gcm.h +++ b/adb/pairing_auth/include/adb/pairing/aes_128_gcm.h @@ -16,17 +16,12 @@ #pragma once -#include <openssl/aes.h> -#include <openssl/cipher.h> - #include <stdint.h> -#include "adb/pairing/counter.h" +#include <optional> +#include <vector> -// This is the default size of the initialization vector (iv) for AES-128-GCM -#define AES_128_GCM_IV_SIZE 12 -// This is the full tag size for AES-128-GCM -#define AES_128_GCM_TAG_SIZE 16 +#include <openssl/aead.h> namespace adb { namespace pairing { @@ -42,7 +37,7 @@ class Aes128Gcm { // suitable for decryption with this class. // The method returns the number of bytes placed in |out| on success and a // negative value if an error occurs. - int Encrypt(const uint8_t* in, size_t in_len, uint8_t* out, size_t out_len); + std::optional<size_t> Encrypt(const uint8_t* in, size_t in_len, uint8_t* out, size_t out_len); // Decrypt a block of data in |in| of length |in_len|, this consumes all data // in |in_len| bytes of data. The decrypted output is placed in the |out| // buffer of length |out_len|. On successful decryption the number of bytes in @@ -50,22 +45,18 @@ class Aes128Gcm { // The method returns the number of bytes consumed from the |in| buffer. If // there is not enough data available in |in| the method returns zero. If // an error occurs the method returns a negative value. - int Decrypt(const uint8_t* in, size_t in_len, uint8_t* out, size_t out_len); + std::optional<size_t> Decrypt(const uint8_t* in, size_t in_len, uint8_t* out, size_t out_len); // Return a safe amount of buffer storage needed to encrypt |size| bytes. size_t EncryptedSize(size_t size); - // Return a safe amount of buffer storage needed to decrypt the encrypted - // data in |encrypted_data| which is of length |encrypted_size|. Returns 0 if - // there is not enough data available to determine the required size. - size_t DecryptedSize(const uint8_t* encrypted_data, size_t encrypted_size); - - static const EVP_CIPHER* cipher_; + // Return a safe amount of buffer storage needed to decrypt |size| bytes. + size_t DecryptedSize(size_t size); private: - bssl::UniquePtr<EVP_CIPHER_CTX> context_; - AES_KEY aes_key_; - // We're going to use this counter for our iv so that it never repeats - Counter<AES_128_GCM_IV_SIZE> counter_; + bssl::ScopedEVP_AEAD_CTX context_; + // Sequence numbers to use as nonces in the encryption scheme + uint64_t dec_sequence_ = 0; + uint64_t enc_sequence_ = 0; }; } // namespace pairing diff --git a/adb/pairing_auth/include/adb/pairing/counter.h b/adb/pairing_auth/include/adb/pairing/counter.h deleted file mode 100644 index 263ceb780..000000000 --- a/adb/pairing_auth/include/adb/pairing/counter.h +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (C) 2020 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include <stddef.h> -#include <stdint.h> - -namespace adb { -namespace pairing { - -template <size_t N> -class Counter { - public: - void Increase() { - for (size_t i = sizeof(counter_) - 1; i < sizeof(counter_); --i) { - if (++counter_[i] != 0) { - break; - } - } - } - - uint8_t* data() { return counter_; } - const uint8_t* data() const { return counter_; } - - constexpr size_t size() const { return sizeof(counter_); } - - uint8_t& operator[](size_t index) { return counter_[index]; } - const uint8_t& operator[](size_t index) const { return counter_[index]; } - - private: - uint8_t counter_[N]; -}; - -} // namespace pairing -} // namespace adb diff --git a/adb/pairing_auth/pairing_auth.cpp b/adb/pairing_auth/pairing_auth.cpp index 96bc11067..0ac04e691 100644 --- a/adb/pairing_auth/pairing_auth.cpp +++ b/adb/pairing_auth/pairing_auth.cpp @@ -75,8 +75,8 @@ struct PairingAuthCtx { // Returns a safe buffer size for encrypting a buffer of size |len|. size_t SafeEncryptedSize(size_t len); - // Returns a safe buffer size for decrypting a buffer |buf|. - size_t SafeDecryptedSize(const Data& buf); + // Returns a safe buffer size for decrypting a buffer of size |len|. + size_t SafeDecryptedSize(size_t len); private: Data our_msg_; @@ -167,12 +167,12 @@ PairingAuthCtx::Data PairingAuthCtx::Encrypt(const PairingAuthCtx::Data& data) { // Determine the size for the encrypted data based on the raw data. Data encrypted(cipher_->EncryptedSize(data.size())); - int bytes = cipher_->Encrypt(data.data(), data.size(), encrypted.data(), encrypted.size()); - if (bytes < 0) { + auto out_size = cipher_->Encrypt(data.data(), data.size(), encrypted.data(), encrypted.size()); + if (!out_size.has_value() || *out_size == 0) { LOG(ERROR) << "Unable to encrypt data"; return Data(); } - encrypted.resize(bytes); + encrypted.resize(*out_size); return encrypted; } @@ -182,14 +182,14 @@ PairingAuthCtx::Data PairingAuthCtx::Decrypt(const PairingAuthCtx::Data& data) { CHECK(!data.empty()); // Determine the size for the decrypted data based on the raw data. - Data decrypted(cipher_->DecryptedSize(data.data(), data.size())); + Data decrypted(cipher_->DecryptedSize(data.size())); size_t decrypted_size = decrypted.size(); - int bytes = cipher_->Decrypt(data.data(), data.size(), decrypted.data(), decrypted_size); - if (bytes <= 0) { + auto out_size = cipher_->Decrypt(data.data(), data.size(), decrypted.data(), decrypted_size); + if (!out_size.has_value() || *out_size == 0) { LOG(ERROR) << "Unable to decrypt data"; return Data(); } - decrypted.resize(bytes); + decrypted.resize(*out_size); return decrypted; } @@ -199,9 +199,9 @@ size_t PairingAuthCtx::SafeEncryptedSize(size_t len) { return cipher_->EncryptedSize(len); } -size_t PairingAuthCtx::SafeDecryptedSize(const PairingAuthCtx::Data& buf) { +size_t PairingAuthCtx::SafeDecryptedSize(size_t len) { CHECK(cipher_); - return cipher_->DecryptedSize(buf.data(), buf.size()); + return cipher_->DecryptedSize(len); } PairingAuthCtx* pairing_auth_server_new(const uint8_t* pswd, size_t len) { @@ -271,8 +271,8 @@ size_t pairing_auth_safe_decrypted_size(PairingAuthCtx* ctx, const uint8_t* buf, CHECK(ctx); CHECK(buf); CHECK_GT(len, 0U); - std::vector<uint8_t> p(buf, buf + len); - return ctx->SafeDecryptedSize(p); + // We no longer need buf for EVP_AEAD + return ctx->SafeDecryptedSize(len); } bool pairing_auth_decrypt(PairingAuthCtx* ctx, const uint8_t* inbuf, size_t inlen, uint8_t* outbuf, diff --git a/adb/pairing_auth/tests/Android.bp b/adb/pairing_auth/tests/Android.bp index 292fff527..213123d59 100644 --- a/adb/pairing_auth/tests/Android.bp +++ b/adb/pairing_auth/tests/Android.bp @@ -18,7 +18,6 @@ cc_test { name: "adb_pairing_auth_test", srcs: [ "aes_128_gcm_test.cpp", - "counter_test.cpp", "pairing_auth_test.cpp", ], diff --git a/adb/pairing_auth/tests/aes_128_gcm_test.cpp b/adb/pairing_auth/tests/aes_128_gcm_test.cpp index e1a20e80d..55689d67a 100644 --- a/adb/pairing_auth/tests/aes_128_gcm_test.cpp +++ b/adb/pairing_auth/tests/aes_128_gcm_test.cpp @@ -39,7 +39,7 @@ TEST(Aes128GcmTest, encrypt_decrypt) { const uint8_t msg[] = "alice and bob, sitting in a binary tree"; uint8_t material[256]; uint8_t encrypted[1024]; - uint8_t out_buf[1024]; + uint8_t out_buf[1024] = {}; RAND_bytes(material, sizeof(material)); Aes128Gcm alice(material, sizeof(material)); @@ -47,82 +47,16 @@ TEST(Aes128GcmTest, encrypt_decrypt) { ; ASSERT_GE(alice.EncryptedSize(sizeof(msg)), sizeof(msg)); - int encrypted_size = alice.Encrypt(msg, sizeof(msg), encrypted, sizeof(encrypted)); - ASSERT_GT(encrypted_size, 0); + auto encrypted_size = alice.Encrypt(msg, sizeof(msg), encrypted, sizeof(encrypted)); + ASSERT_TRUE(encrypted_size.has_value()); + ASSERT_GT(*encrypted_size, 0); size_t out_size = sizeof(out_buf); - ASSERT_GE(bob.DecryptedSize(encrypted, sizeof(encrypted)), sizeof(msg)); - int decrypted_size = bob.Decrypt(encrypted, sizeof(encrypted), out_buf, out_size); - ASSERT_EQ(sizeof(msg), decrypted_size); - memset(out_buf + decrypted_size, 0, sizeof(out_buf) - decrypted_size); + ASSERT_GE(bob.DecryptedSize(*encrypted_size), sizeof(msg)); + auto decrypted_size = bob.Decrypt(encrypted, *encrypted_size, out_buf, out_size); + ASSERT_TRUE(decrypted_size.has_value()); + ASSERT_EQ(sizeof(msg), *decrypted_size); ASSERT_STREQ(reinterpret_cast<const char*>(msg), reinterpret_cast<const char*>(out_buf)); } -TEST(Aes128GcmTest, padding) { - // Test with block-align data as well as unaligned data. - const size_t cipher_block_size = EVP_CIPHER_block_size(Aes128Gcm::cipher_); - uint8_t material[256]; - RAND_bytes(material, sizeof(material)); - Aes128Gcm alice(material, sizeof(material)); - Aes128Gcm bob(material, sizeof(material)); - ; - std::vector<uint8_t> msg; - std::vector<uint8_t> encrypted; - std::vector<uint8_t> decrypted; - - // Test with aligned data - { - msg.resize(cipher_block_size); - RAND_bytes(msg.data(), msg.size()); - - // encrypt - size_t safe_encrypted_sz = alice.EncryptedSize(msg.size()); - ASSERT_GE(safe_encrypted_sz, msg.size()); - encrypted.resize(safe_encrypted_sz); - int encrypted_size = - alice.Encrypt(msg.data(), msg.size(), encrypted.data(), encrypted.size()); - ASSERT_GT(encrypted_size, 0); - ASSERT_LE(encrypted_size, safe_encrypted_sz); - encrypted.resize(encrypted_size); - - // decrypt - size_t safe_decrypted_size = bob.DecryptedSize(encrypted.data(), encrypted.size()); - ASSERT_GE(safe_decrypted_size, msg.size()); - decrypted.resize(safe_decrypted_size); - int decrypted_size = - bob.Decrypt(encrypted.data(), encrypted.size(), decrypted.data(), decrypted.size()); - ASSERT_GT(decrypted_size, 0); - ASSERT_LE(decrypted_size, safe_decrypted_size); - ASSERT_EQ(msg.size(), decrypted_size); - ASSERT_EQ(memcmp(msg.data(), decrypted.data(), decrypted.size()), 0); - } - - // Test with unaligned data - { - msg.resize(cipher_block_size + 1); - RAND_bytes(msg.data(), msg.size()); - - // encrypt - size_t safe_encrypted_sz = alice.EncryptedSize(msg.size()); - ASSERT_GE(safe_encrypted_sz, msg.size()); - encrypted.resize(safe_encrypted_sz); - int encrypted_size = - alice.Encrypt(msg.data(), msg.size(), encrypted.data(), encrypted.size()); - ASSERT_GT(encrypted_size, 0); - ASSERT_LE(encrypted_size, safe_encrypted_sz); - encrypted.resize(encrypted_size); - - // decrypt - size_t safe_decrypted_size = bob.DecryptedSize(encrypted.data(), encrypted.size()); - ASSERT_GE(safe_decrypted_size, msg.size()); - decrypted.resize(safe_decrypted_size); - int decrypted_size = - bob.Decrypt(encrypted.data(), encrypted.size(), decrypted.data(), decrypted.size()); - ASSERT_GT(decrypted_size, 0); - ASSERT_LE(decrypted_size, safe_decrypted_size); - ASSERT_EQ(msg.size(), decrypted_size); - ASSERT_EQ(memcmp(msg.data(), decrypted.data(), decrypted.size()), 0); - } -} - } // namespace pairing } // namespace adb diff --git a/adb/pairing_auth/tests/counter_test.cpp b/adb/pairing_auth/tests/counter_test.cpp deleted file mode 100644 index b3385518c..000000000 --- a/adb/pairing_auth/tests/counter_test.cpp +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (C) 2020 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <gtest/gtest.h> - -#include <adb/pairing/counter.h> - -namespace adb { -namespace pairing { - -static constexpr size_t kTestCounterSize = 13; -static const uint8_t kZeroes[64] = {0}; - -TEST(AdbCounterTest, size_match) { - Counter<kTestCounterSize> counter; - ASSERT_EQ(kTestCounterSize, counter.size()); -} - -TEST(AdbCounterTest, Increase) { - Counter<kTestCounterSize> counter; - memset(counter.data(), 0, counter.size()); - counter.Increase(); - EXPECT_EQ(1, counter[counter.size() - 1]); - EXPECT_EQ(0, memcmp(counter.data(), kZeroes, counter.size() - 1)); -} - -TEST(AdbCounterTest, rollover_first_byte) { - Counter<kTestCounterSize> counter; - memset(counter.data(), 0, counter.size()); - counter[counter.size() - 1] = 0xFF; - counter.Increase(); - EXPECT_EQ(0, counter[counter.size() - 1]); - EXPECT_EQ(1, counter[counter.size() - 2]); - EXPECT_EQ(0, memcmp(counter.data(), kZeroes, counter.size() - 2)); -} - -TEST(AdbCounterTest, multiple_rollover) { - Counter<kTestCounterSize> counter; - memset(counter.data(), 0xFF, counter.size()); - memset(counter.data(), 0, counter.size() - 3); - counter.Increase(); - EXPECT_EQ(0, counter[counter.size() - 5]); - EXPECT_EQ(1, counter[counter.size() - 4]); - EXPECT_EQ(0, counter[counter.size() - 3]); - EXPECT_EQ(0, counter[counter.size() - 2]); - EXPECT_EQ(0, counter[counter.size() - 1]); -} - -TEST(AdbCounterTest, full_rollover) { - Counter<kTestCounterSize> counter; - memset(counter.data(), 0xFF, counter.size()); - counter.Increase(); - EXPECT_EQ(0, memcmp(counter.data(), kZeroes, counter.size())); -} - -} // namespace pairing -} // namespace adb diff --git a/adb/pairing_connection/pairing_connection.cpp b/adb/pairing_connection/pairing_connection.cpp index a26a6b4d2..ffe49a91e 100644 --- a/adb/pairing_connection/pairing_connection.cpp +++ b/adb/pairing_connection/pairing_connection.cpp @@ -278,13 +278,13 @@ bool PairingConnectionCtx::Start(int fd, ResultCallback cb, void* opaque) { if (fd < 0) { return false; } + fd_.reset(fd); State expected = State::Ready; if (!state_.compare_exchange_strong(expected, State::ExchangingMsgs)) { return false; } - fd_.reset(fd); cb_ = cb; opaque_ = opaque; diff --git a/adb/sysdeps.h b/adb/sysdeps.h index 4efbc02c3..9a879b57b 100644 --- a/adb/sysdeps.h +++ b/adb/sysdeps.h @@ -276,6 +276,7 @@ inline void seekdir(DIR*, long) { class Process { public: constexpr explicit Process(HANDLE h = nullptr) : h_(h) {} + constexpr Process(Process&& other) : h_(std::exchange(other.h_, nullptr)) {} ~Process() { close(); } constexpr explicit operator bool() const { return h_ != nullptr; } @@ -292,6 +293,8 @@ class Process { } private: + DISALLOW_COPY_AND_ASSIGN(Process); + void close() { if (*this) { ::CloseHandle(h_); @@ -662,10 +665,16 @@ static __inline__ int adb_get_os_handle(borrowed_fd fd) { return fd.get(); } +static __inline__ int cast_handle_to_int(int fd) { + return fd; +} + // A very simple wrapper over a launched child process class Process { public: constexpr explicit Process(pid_t pid) : pid_(pid) {} + constexpr Process(Process&& other) : pid_(std::exchange(other.pid_, -1)) {} + constexpr explicit operator bool() const { return pid_ >= 0; } void wait() { @@ -682,6 +691,8 @@ class Process { } private: + DISALLOW_COPY_AND_ASSIGN(Process); + pid_t pid_; }; diff --git a/adb/transport.cpp b/adb/transport.cpp index 447a8fedf..6dccb7f68 100644 --- a/adb/transport.cpp +++ b/adb/transport.cpp @@ -82,6 +82,8 @@ const char* const kFeatureFixedPushSymlinkTimestamp = "fixed_push_symlink_timest const char* const kFeatureAbbExec = "abb_exec"; const char* const kFeatureRemountShell = "remount_shell"; const char* const kFeatureTrackApp = "track_app"; +const char* const kFeatureSendRecv2 = "sendrecv_v2"; +const char* const kFeatureSendRecv2Brotli = "sendrecv_v2_brotli"; namespace { @@ -498,12 +500,14 @@ bool FdConnection::DoTlsHandshake(RSA* key, std::string* auth_key) { auto x509 = GenerateX509Certificate(evp_pkey.get()); auto x509_str = X509ToPEMString(x509.get()); auto evp_str = Key::ToPEMString(evp_pkey.get()); + + int osh = cast_handle_to_int(adb_get_os_handle(fd_)); #if ADB_HOST tls_ = TlsConnection::Create(TlsConnection::Role::Client, #else tls_ = TlsConnection::Create(TlsConnection::Role::Server, #endif - x509_str, evp_str, fd_); + x509_str, evp_str, osh); CHECK(tls_); #if ADB_HOST // TLS 1.3 gives the client no message if the server rejected the @@ -1177,6 +1181,8 @@ const FeatureSet& supported_features() { kFeatureAbbExec, kFeatureRemountShell, kFeatureTrackApp, + kFeatureSendRecv2, + kFeatureSendRecv2Brotli, // Increment ADB_SERVER_VERSION when adding a feature that adbd needs // to know about. Otherwise, the client can be stuck running an old // version of the server even after upgrading their copy of adb. diff --git a/adb/transport.h b/adb/transport.h index a62349e63..3a77cf6d5 100644 --- a/adb/transport.h +++ b/adb/transport.h @@ -81,9 +81,14 @@ extern const char* const kFeatureAbb; extern const char* const kFeatureAbbExec; // adbd properly updates symlink timestamps on push. extern const char* const kFeatureFixedPushSymlinkTimestamp; +// Implement `adb remount` via shelling out to /system/bin/remount. extern const char* const kFeatureRemountShell; // adbd supports `track-app` service reporting debuggable/profileable apps. extern const char* const kFeatureTrackApp; +// adbd supports version 2 of send/recv. +extern const char* const kFeatureSendRecv2; +// adbd supports brotli for send/recv v2. +extern const char* const kFeatureSendRecv2Brotli; TransportId NextTransportId(); diff --git a/adb/types.h b/adb/types.h index c619fffcf..deca7eafb 100644 --- a/adb/types.h +++ b/adb/types.h @@ -150,6 +150,22 @@ struct IOVector { IOVector& operator=(const IOVector& copy) = delete; IOVector& operator=(IOVector&& move) noexcept; + const value_type* front_data() const { + if (chain_.empty()) { + return nullptr; + } + + return chain_.front().data() + begin_offset_; + } + + size_type front_size() const { + if (chain_.empty()) { + return 0; + } + + return chain_.front().size() - begin_offset_; + } + size_type size() const { return chain_length_ - begin_offset_; } bool empty() const { return size() == 0; } diff --git a/base/Android.bp b/base/Android.bp index 12de3b22c..894ad6c50 100644 --- a/base/Android.bp +++ b/base/Android.bp @@ -74,10 +74,6 @@ cc_defaults { "test_utils.cpp", ], - static: { - cflags: ["-DNO_LIBLOG_DLSYM"], - }, - cppflags: ["-Wexit-time-destructors"], shared_libs: ["liblog"], target: { diff --git a/base/liblog_symbols.cpp b/base/liblog_symbols.cpp index 8d5917907..1f4b69b53 100644 --- a/base/liblog_symbols.cpp +++ b/base/liblog_symbols.cpp @@ -16,11 +16,9 @@ #include "liblog_symbols.h" -#if defined(__ANDROID__) -#if !defined(NO_LIBLOG_DLSYM) || defined(__ANDROID_APEX__) +#if defined(__ANDROID_SDK_VERSION__) && (__ANDROID_SDK_VERSION__ <= 29) #define USE_DLSYM #endif -#endif #ifdef USE_DLSYM #include <dlfcn.h> @@ -48,7 +46,7 @@ const std::optional<LibLogFunctions>& GetLibLogFunctions() { } DLSYM(__android_log_set_logger) - DLSYM(__android_log_write_logger_data) + DLSYM(__android_log_write_log_message) DLSYM(__android_log_logd_logger) DLSYM(__android_log_stderr_logger) DLSYM(__android_log_set_aborter) @@ -71,7 +69,7 @@ const std::optional<LibLogFunctions>& GetLibLogFunctions() { static std::optional<LibLogFunctions> liblog_functions = []() -> std::optional<LibLogFunctions> { return LibLogFunctions{ .__android_log_set_logger = __android_log_set_logger, - .__android_log_write_logger_data = __android_log_write_logger_data, + .__android_log_write_log_message = __android_log_write_log_message, .__android_log_logd_logger = __android_log_logd_logger, .__android_log_stderr_logger = __android_log_stderr_logger, .__android_log_set_aborter = __android_log_set_aborter, diff --git a/base/liblog_symbols.h b/base/liblog_symbols.h index b4ab06a88..2e6b47f7d 100644 --- a/base/liblog_symbols.h +++ b/base/liblog_symbols.h @@ -25,13 +25,10 @@ namespace base { struct LibLogFunctions { void (*__android_log_set_logger)(__android_logger_function logger); - void (*__android_log_write_logger_data)(struct __android_logger_data* logger_data, - const char* msg); + void (*__android_log_write_log_message)(struct __android_log_message* log_message); - void (*__android_log_logd_logger)(const struct __android_logger_data* logger_data, - const char* msg); - void (*__android_log_stderr_logger)(const struct __android_logger_data* logger_data, - const char* message); + void (*__android_log_logd_logger)(const struct __android_log_message* log_message); + void (*__android_log_stderr_logger)(const struct __android_log_message* log_message); void (*__android_log_set_aborter)(__android_aborter_function aborter); void (*__android_log_call_aborter)(const char* abort_message); diff --git a/base/logging.cpp b/base/logging.cpp index 9a6e0fb7e..cd460eb46 100644 --- a/base/logging.cpp +++ b/base/logging.cpp @@ -349,9 +349,9 @@ void LogdLogger::operator()(LogId id, LogSeverity severity, const char* tag, static auto& liblog_functions = GetLibLogFunctions(); if (liblog_functions) { - __android_logger_data logger_data = {sizeof(__android_logger_data), lg_id, priority, tag, - static_cast<const char*>(nullptr), 0}; - liblog_functions->__android_log_logd_logger(&logger_data, message); + __android_log_message log_message = {sizeof(__android_log_message), lg_id, priority, tag, + static_cast<const char*>(nullptr), 0, message}; + liblog_functions->__android_log_logd_logger(&log_message); } else { __android_log_buf_print(lg_id, priority, tag, "%s", message); } @@ -426,13 +426,13 @@ void SetLogger(LogFunction&& logger) { // std::function<>, which is the not-thread-safe alternative. static std::atomic<LogFunction*> logger_function(nullptr); auto* old_logger_function = logger_function.exchange(new LogFunction(logger)); - liblog_functions->__android_log_set_logger([](const struct __android_logger_data* logger_data, - const char* message) { - auto log_id = log_id_tToLogId(logger_data->buffer_id); - auto severity = PriorityToLogSeverity(logger_data->priority); + liblog_functions->__android_log_set_logger([](const struct __android_log_message* log_message) { + auto log_id = log_id_tToLogId(log_message->buffer_id); + auto severity = PriorityToLogSeverity(log_message->priority); auto& function = *logger_function.load(std::memory_order_acquire); - function(log_id, severity, logger_data->tag, logger_data->file, logger_data->line, message); + function(log_id, severity, log_message->tag, log_message->file, log_message->line, + log_message->message); }); delete old_logger_function; } else { @@ -576,9 +576,9 @@ void LogMessage::LogLine(const char* file, unsigned int line, LogSeverity severi static auto& liblog_functions = GetLibLogFunctions(); int32_t priority = LogSeverityToPriority(severity); if (liblog_functions) { - __android_logger_data logger_data = { - sizeof(__android_logger_data), LOG_ID_DEFAULT, priority, tag, file, line}; - liblog_functions->__android_log_write_logger_data(&logger_data, message); + __android_log_message log_message = { + sizeof(__android_log_message), LOG_ID_DEFAULT, priority, tag, file, line, message}; + liblog_functions->__android_log_write_log_message(&log_message); } else { if (tag == nullptr) { std::lock_guard<std::recursive_mutex> lock(TagLock()); diff --git a/debuggerd/Android.bp b/debuggerd/Android.bp index f28c778fe..d67b52277 100644 --- a/debuggerd/Android.bp +++ b/debuggerd/Android.bp @@ -103,9 +103,14 @@ cc_library_static { export_include_dirs: ["include"], } -// Fallback implementation. +// Fallback implementation, for use in the Bionic linker only. cc_library_static { name: "libdebuggerd_handler_fallback", + visibility: ["//bionic/linker"], + apex_available: [ + "com.android.runtime", + "//apex_available:platform", + ], defaults: ["debuggerd_defaults"], recovery_available: true, srcs: [ @@ -118,8 +123,7 @@ cc_library_static { "libasync_safe", "libbase", "libdebuggerd", - "libunwindstack", - "libdexfile_support_static", // libunwindstack dependency + "libunwindstack_no_dex", "liblzma", "libcutils", ], @@ -127,14 +131,6 @@ cc_library_static { header_libs: ["bionic_libc_platform_headers"], export_header_lib_headers: ["bionic_libc_platform_headers"], - target: { - recovery: { - exclude_static_libs: [ - "libdexfile_support_static", - ], - }, - }, - export_include_dirs: ["include"], } @@ -188,7 +184,7 @@ cc_library_static { ], static_libs: [ - "libdexfile_support_static", // libunwindstack dependency + "libdexfile_support", // libunwindstack dependency "libunwindstack", "liblzma", "libbase", @@ -201,7 +197,7 @@ cc_library_static { target: { recovery: { exclude_static_libs: [ - "libdexfile_support_static", + "libdexfile_support", ], }, }, @@ -321,6 +317,10 @@ cc_binary { "libprocinfo", "libunwindstack", ], + + apex_available: [ + "com.android.runtime", + ], } cc_binary { diff --git a/debuggerd/handler/debuggerd_handler.cpp b/debuggerd/handler/debuggerd_handler.cpp index 003c3d9a6..ac28fe907 100644 --- a/debuggerd/handler/debuggerd_handler.cpp +++ b/debuggerd/handler/debuggerd_handler.cpp @@ -83,7 +83,7 @@ using unique_fd = android::base::unique_fd_impl<FdsanBypassCloser>; #define CRASH_DUMP_NAME "crash_dump32" #endif -#define CRASH_DUMP_PATH "/system/bin/" CRASH_DUMP_NAME +#define CRASH_DUMP_PATH "/apex/com.android.runtime/bin/" CRASH_DUMP_NAME // Wrappers that directly invoke the respective syscalls, in case the cached values are invalid. #pragma GCC poison getpid gettid diff --git a/debuggerd/libdebuggerd/gwp_asan.cpp b/debuggerd/libdebuggerd/gwp_asan.cpp index fe3a1730d..f27136558 100644 --- a/debuggerd/libdebuggerd/gwp_asan.cpp +++ b/debuggerd/libdebuggerd/gwp_asan.cpp @@ -157,63 +157,6 @@ void GwpAsanCrashData::DumpCause(log_t* log) const { error_string_, diff, byte_suffix, location_str, alloc_size, alloc_address); } -// Build a frame for symbolization using the maps from the provided unwinder. -// The constructed frame contains just enough information to be used to -// symbolize a GWP-ASan stack trace. -static unwindstack::FrameData BuildFrame(unwindstack::Unwinder* unwinder, uintptr_t pc, - size_t frame_num) { - unwindstack::FrameData frame; - frame.num = frame_num; - - unwindstack::Maps* maps = unwinder->GetMaps(); - unwindstack::MapInfo* map_info = maps->Find(pc); - if (!map_info) { - frame.rel_pc = pc; - return frame; - } - - unwindstack::Elf* elf = - map_info->GetElf(unwinder->GetProcessMemory(), unwindstack::Regs::CurrentArch()); - - uint64_t relative_pc = elf->GetRelPc(pc, map_info); - - // Create registers just to get PC adjustment. Doesn't matter what they point - // to. - unwindstack::Regs* regs = unwindstack::Regs::CreateFromLocal(); - uint64_t pc_adjustment = regs->GetPcAdjustment(relative_pc, elf); - relative_pc -= pc_adjustment; - // The debug PC may be different if the PC comes from the JIT. - uint64_t debug_pc = relative_pc; - - // If we don't have a valid ELF file, check the JIT. - if (!elf->valid()) { - unwindstack::JitDebug jit_debug(unwinder->GetProcessMemory()); - uint64_t jit_pc = pc - pc_adjustment; - unwindstack::Elf* jit_elf = jit_debug.GetElf(maps, jit_pc); - if (jit_elf != nullptr) { - debug_pc = jit_pc; - elf = jit_elf; - } - } - - // Copy all the things we need into the frame for symbolization. - frame.rel_pc = relative_pc; - frame.pc = pc - pc_adjustment; - frame.map_name = map_info->name; - frame.map_elf_start_offset = map_info->elf_start_offset; - frame.map_exact_offset = map_info->offset; - frame.map_start = map_info->start; - frame.map_end = map_info->end; - frame.map_flags = map_info->flags; - frame.map_load_bias = elf->GetLoadBias(); - - if (!elf->GetFunctionName(relative_pc, &frame.function_name, &frame.function_offset)) { - frame.function_name = ""; - frame.function_offset = 0; - } - return frame; -} - constexpr size_t kMaxTraceLength = gwp_asan::AllocationMetadata::kMaxTraceLengthToCollect; bool GwpAsanCrashData::HasDeallocationTrace() const { @@ -240,7 +183,8 @@ void GwpAsanCrashData::DumpDeallocationTrace(log_t* log, unwindstack::Unwinder* unwinder->SetDisplayBuildID(true); for (size_t i = 0; i < num_frames; ++i) { - unwindstack::FrameData frame_data = BuildFrame(unwinder, frames.get()[i], i); + unwindstack::FrameData frame_data = unwinder->BuildFrameFromPcOnly(frames.get()[i]); + frame_data.num = i; _LOG(log, logtype::BACKTRACE, " %s\n", unwinder->FormatFrame(frame_data).c_str()); } } @@ -266,7 +210,8 @@ void GwpAsanCrashData::DumpAllocationTrace(log_t* log, unwindstack::Unwinder* un unwinder->SetDisplayBuildID(true); for (size_t i = 0; i < num_frames; ++i) { - unwindstack::FrameData frame_data = BuildFrame(unwinder, frames.get()[i], i); + unwindstack::FrameData frame_data = unwinder->BuildFrameFromPcOnly(frames.get()[i]); + frame_data.num = i; _LOG(log, logtype::BACKTRACE, " %s\n", unwinder->FormatFrame(frame_data).c_str()); } } diff --git a/debuggerd/libdebuggerd/tombstone.cpp b/debuggerd/libdebuggerd/tombstone.cpp index 70b8817be..e0168d539 100644 --- a/debuggerd/libdebuggerd/tombstone.cpp +++ b/debuggerd/libdebuggerd/tombstone.cpp @@ -447,8 +447,6 @@ static bool dump_thread(log_t* log, unwindstack::Unwinder* unwinder, const Threa // that don't match the specified pid, and writes them to the tombstone file. // // If "tail" is non-zero, log the last "tail" number of lines. -static EventTagMap* g_eventTagMap = NULL; - static void dump_log_file(log_t* log, pid_t pid, const char* filename, unsigned int tail) { bool first = true; logger_list* logger_list; @@ -457,8 +455,8 @@ static void dump_log_file(log_t* log, pid_t pid, const char* filename, unsigned return; } - logger_list = android_logger_list_open( - android_name_to_log_id(filename), ANDROID_LOG_RDONLY | ANDROID_LOG_NONBLOCK, tail, pid); + logger_list = + android_logger_list_open(android_name_to_log_id(filename), ANDROID_LOG_NONBLOCK, tail, pid); if (!logger_list) { ALOGE("Unable to open %s: %s\n", filename, strerror(errno)); @@ -507,21 +505,6 @@ static void dump_log_file(log_t* log, pid_t pid, const char* filename, unsigned ptm = localtime_r(&sec, &tmBuf); strftime(timeBuf, sizeof(timeBuf), "%m-%d %H:%M:%S", ptm); - if (log_entry.id() == LOG_ID_EVENTS) { - if (!g_eventTagMap) { - g_eventTagMap = android_openEventTagMap(nullptr); - } - AndroidLogEntry e; - char buf[512]; - if (android_log_processBinaryLogBuffer(&log_entry.entry, &e, g_eventTagMap, buf, - sizeof(buf)) == 0) { - _LOG(log, logtype::LOGS, "%s.%03d %5d %5d %c %-8.*s: %s\n", timeBuf, - log_entry.entry.nsec / 1000000, log_entry.entry.pid, log_entry.entry.tid, 'I', - (int)e.tagLen, e.tag, e.message); - } - continue; - } - char* msg = log_entry.msg(); if (msg == nullptr) { continue; diff --git a/diagnose_usb/Android.bp b/diagnose_usb/Android.bp index 6bee28c10..93d13bd56 100644 --- a/diagnose_usb/Android.bp +++ b/diagnose_usb/Android.bp @@ -3,6 +3,11 @@ cc_library_static { cflags: ["-Wall", "-Wextra", "-Werror"], host_supported: true, recovery_available: true, + apex_available: [ + "com.android.adbd", + // TODO(b/151398197) remove the below + "//apex_available:platform", + ], target: { windows: { enabled: true, diff --git a/fastboot/bootimg_utils.cpp b/fastboot/bootimg_utils.cpp index 46d4bd30e..2c0989ed1 100644 --- a/fastboot/bootimg_utils.cpp +++ b/fastboot/bootimg_utils.cpp @@ -34,14 +34,54 @@ #include <stdlib.h> #include <string.h> +static void bootimg_set_cmdline_v3(boot_img_hdr_v3* h, const std::string& cmdline) { + if (cmdline.size() >= sizeof(h->cmdline)) die("command line too large: %zu", cmdline.size()); + strcpy(reinterpret_cast<char*>(h->cmdline), cmdline.c_str()); +} + void bootimg_set_cmdline(boot_img_hdr_v2* h, const std::string& cmdline) { + if (h->header_version == 3) { + return bootimg_set_cmdline_v3(reinterpret_cast<boot_img_hdr_v3*>(h), cmdline); + } if (cmdline.size() >= sizeof(h->cmdline)) die("command line too large: %zu", cmdline.size()); strcpy(reinterpret_cast<char*>(h->cmdline), cmdline.c_str()); } +static boot_img_hdr_v3* mkbootimg_v3(const std::vector<char>& kernel, + const std::vector<char>& ramdisk, const boot_img_hdr_v2& src, + std::vector<char>* out) { +#define V3_PAGE_SIZE 4096 + const size_t page_mask = V3_PAGE_SIZE - 1; + int64_t kernel_actual = (kernel.size() + page_mask) & (~page_mask); + int64_t ramdisk_actual = (ramdisk.size() + page_mask) & (~page_mask); + + int64_t bootimg_size = V3_PAGE_SIZE + kernel_actual + ramdisk_actual; + out->resize(bootimg_size); + + boot_img_hdr_v3* hdr = reinterpret_cast<boot_img_hdr_v3*>(out->data()); + + memcpy(hdr->magic, BOOT_MAGIC, BOOT_MAGIC_SIZE); + hdr->kernel_size = kernel.size(); + hdr->ramdisk_size = ramdisk.size(); + hdr->os_version = src.os_version; + hdr->header_size = sizeof(boot_img_hdr_v3); + hdr->header_version = 3; + + memcpy(hdr->magic + V3_PAGE_SIZE, kernel.data(), kernel.size()); + memcpy(hdr->magic + V3_PAGE_SIZE + kernel_actual, ramdisk.data(), ramdisk.size()); + + return hdr; +} + boot_img_hdr_v2* mkbootimg(const std::vector<char>& kernel, const std::vector<char>& ramdisk, const std::vector<char>& second, const std::vector<char>& dtb, size_t base, const boot_img_hdr_v2& src, std::vector<char>* out) { + if (src.header_version == 3) { + if (!second.empty() || !dtb.empty()) { + die("Second stage bootloader and dtb not supported in v3 boot image\n"); + } + return reinterpret_cast<boot_img_hdr_v2*>(mkbootimg_v3(kernel, ramdisk, src, out)); + } const size_t page_mask = src.page_size - 1; int64_t header_actual = (sizeof(boot_img_hdr_v1) + page_mask) & (~page_mask); diff --git a/fastboot/device/commands.cpp b/fastboot/device/commands.cpp index ca120c62a..b8eee4ac1 100644 --- a/fastboot/device/commands.cpp +++ b/fastboot/device/commands.cpp @@ -31,6 +31,7 @@ #include <cutils/android_reboot.h> #include <ext4_utils/wipe.h> #include <fs_mgr.h> +#include <fs_mgr/roots.h> #include <libgsi/libgsi.h> #include <liblp/builder.h> #include <liblp/liblp.h> diff --git a/fastboot/device/fastboot_device.cpp b/fastboot/device/fastboot_device.cpp index 31fc35937..bb085c536 100644 --- a/fastboot/device/fastboot_device.cpp +++ b/fastboot/device/fastboot_device.cpp @@ -16,18 +16,22 @@ #include "fastboot_device.h" +#include <algorithm> + #include <android-base/logging.h> #include <android-base/strings.h> #include <android/hardware/boot/1.0/IBootControl.h> #include <android/hardware/fastboot/1.0/IFastboot.h> +#include <fs_mgr.h> +#include <fs_mgr/roots.h> #include <healthhalutils/HealthHalUtils.h> -#include <algorithm> - #include "constants.h" #include "flashing.h" #include "usb_client.h" +using android::fs_mgr::EnsurePathUnmounted; +using android::fs_mgr::Fstab; using ::android::hardware::hidl_string; using ::android::hardware::boot::V1_0::IBootControl; using ::android::hardware::boot::V1_0::Slot; @@ -64,6 +68,13 @@ FastbootDevice::FastbootDevice() if (boot_control_hal_) { boot1_1_ = android::hardware::boot::V1_1::IBootControl::castFrom(boot_control_hal_); } + + // Make sure cache is unmounted, since recovery will have mounted it for + // logging. + Fstab fstab; + if (ReadDefaultFstab(&fstab)) { + EnsurePathUnmounted(&fstab, "/cache"); + } } FastbootDevice::~FastbootDevice() { diff --git a/fastboot/fastboot.cpp b/fastboot/fastboot.cpp index 7fdc28b3d..7f6e7230f 100644 --- a/fastboot/fastboot.cpp +++ b/fastboot/fastboot.cpp @@ -464,7 +464,7 @@ static std::vector<char> LoadBootableImage(const std::string& kernel, const std: } // Is this actually a boot image? - if (kernel_data.size() < sizeof(boot_img_hdr_v2)) { + if (kernel_data.size() < sizeof(boot_img_hdr_v3)) { die("cannot load '%s': too short", kernel.c_str()); } if (!memcmp(kernel_data.data(), BOOT_MAGIC, BOOT_MAGIC_SIZE)) { @@ -493,7 +493,7 @@ static std::vector<char> LoadBootableImage(const std::string& kernel, const std: std::vector<char> dtb_data; if (!g_dtb_path.empty()) { - if (g_boot_img_hdr.header_version < 2) { + if (g_boot_img_hdr.header_version != 2) { die("Argument dtb not supported for boot image header version %d\n", g_boot_img_hdr.header_version); } diff --git a/fastboot/fuzzy_fastboot/main.cpp b/fastboot/fuzzy_fastboot/main.cpp index d9167e75b..e7f785b83 100644 --- a/fastboot/fuzzy_fastboot/main.cpp +++ b/fastboot/fuzzy_fastboot/main.cpp @@ -261,6 +261,10 @@ TEST_F(LogicalPartitionCompliance, CreateResizeDeleteLP) { GTEST_LOG_(INFO) << "Flashing a logical partition.."; EXPECT_EQ(fb->FlashPartition(test_partition_name, buf), SUCCESS) << "flash logical -partition failed"; + + GTEST_LOG_(INFO) << "Testing 'fastboot delete-logical-partition' command"; + EXPECT_EQ(fb->DeletePartition(test_partition_name), SUCCESS) + << "delete logical-partition failed"; } // Conformance tests diff --git a/fs_mgr/fs_mgr.cpp b/fs_mgr/fs_mgr.cpp index c0c0e99a8..5475caebf 100644 --- a/fs_mgr/fs_mgr.cpp +++ b/fs_mgr/fs_mgr.cpp @@ -96,6 +96,7 @@ using android::base::Basename; using android::base::GetBoolProperty; +using android::base::GetUintProperty; using android::base::Readlink; using android::base::Realpath; using android::base::SetProperty; @@ -1545,11 +1546,16 @@ int fs_mgr_umount_all(android::fs_mgr::Fstab* fstab) { return ret; } +static std::chrono::milliseconds GetMillisProperty(const std::string& name, + std::chrono::milliseconds default_value) { + auto value = GetUintProperty(name, static_cast<uint64_t>(default_value.count())); + return std::chrono::milliseconds(std::move(value)); +} + static bool fs_mgr_unmount_all_data_mounts(const std::string& block_device) { LINFO << __FUNCTION__ << "(): about to umount everything on top of " << block_device; Timer t; - // TODO(b/135984674): should be configured via a read-only property. - std::chrono::milliseconds timeout = 5s; + auto timeout = GetMillisProperty("init.userspace_reboot.userdata_remount.timeoutmillis", 5s); while (true) { bool umount_done = true; Fstab proc_mounts; diff --git a/fs_mgr/libdm/Android.bp b/fs_mgr/libdm/Android.bp index c2c2cc4da..0499e8d51 100644 --- a/fs_mgr/libdm/Android.bp +++ b/fs_mgr/libdm/Android.bp @@ -58,11 +58,11 @@ cc_defaults { defaults: ["fs_mgr_defaults"], static_libs: [ "libdm", + "libext2_uuid", + "libfs_mgr", ], shared_libs: [ "libbase", - "libext2_uuid", - "libfs_mgr", "liblog", ], srcs: [":libdm_test_srcs"], diff --git a/fs_mgr/libsnapshot/include/libsnapshot/snapshot.h b/fs_mgr/libsnapshot/include/libsnapshot/snapshot.h index 957c26c10..1daa83b8a 100644 --- a/fs_mgr/libsnapshot/include/libsnapshot/snapshot.h +++ b/fs_mgr/libsnapshot/include/libsnapshot/snapshot.h @@ -142,7 +142,9 @@ class SnapshotManager final { // be created, and the device must either cancel the OTA (either before // rebooting or after rolling back), or merge the OTA. // Before calling this function, all snapshots must be mapped. - bool FinishedSnapshotWrites(); + // If |wipe| is set to true, wipe is scheduled after reboot, and snapshots + // may need to be merged before wiping. + bool FinishedSnapshotWrites(bool wipe); // Initiate a merge on all snapshot devices. This should only be used after an // update has been marked successful after booting. @@ -230,8 +232,14 @@ class SnapshotManager final { // devices; // - CreateResult::ERROR if a fatal error occurred, mounting /system should // be aborted. + // This function mounts /metadata when called, and unmounts /metadata upon + // return. CreateResult RecoveryCreateSnapshotDevices(); + // Same as RecoveryCreateSnapshotDevices(), but does not auto mount/umount + // /metadata. + CreateResult RecoveryCreateSnapshotDevices(const std::unique_ptr<AutoDevice>& metadata_device); + // Dump debug information. bool Dump(std::ostream& os); @@ -446,6 +454,7 @@ class SnapshotManager final { std::string GetSnapshotBootIndicatorPath(); std::string GetRollbackIndicatorPath(); + std::string GetForwardMergeIndicatorPath(); // Return the name of the device holding the "snapshot" or "snapshot-merge" // target. This may not be the final device presented via MapSnapshot(), if @@ -516,6 +525,17 @@ class SnapshotManager final { bool ShouldDeleteSnapshot(LockedFile* lock, const std::map<std::string, bool>& flashing_status, Slot current_slot, const std::string& name); + // Create or delete forward merge indicator given |wipe|. Iff wipe is scheduled, + // allow forward merge on FDR. + bool UpdateForwardMergeIndicator(bool wipe); + + // Helper for HandleImminentDataWipe. + // Call ProcessUpdateState and handle states with special rules before data wipe. Specifically, + // if |allow_forward_merge| and allow-forward-merge indicator exists, initiate merge if + // necessary. + bool ProcessUpdateStateOnDataWipe(bool allow_forward_merge, + const std::function<bool()>& callback); + std::string gsid_dir_; std::string metadata_dir_; std::unique_ptr<IDeviceInfo> device_; diff --git a/fs_mgr/libsnapshot/snapshot.cpp b/fs_mgr/libsnapshot/snapshot.cpp index ed052b15b..c9fa28e9f 100644 --- a/fs_mgr/libsnapshot/snapshot.cpp +++ b/fs_mgr/libsnapshot/snapshot.cpp @@ -228,10 +228,17 @@ bool SnapshotManager::RemoveAllUpdateState(LockedFile* lock, const std::function return false; } - // It's okay if these fail - first-stage init performs a deeper check after + // It's okay if these fail: + // - For SnapshotBoot and Rollback, first-stage init performs a deeper check after // reading the indicator file, so it's not a problem if it still exists // after the update completes. - std::vector<std::string> files = {GetSnapshotBootIndicatorPath(), GetRollbackIndicatorPath()}; + // - For ForwardMerge, FinishedSnapshotWrites asserts that the existence of the indicator + // matches the incoming update. + std::vector<std::string> files = { + GetSnapshotBootIndicatorPath(), + GetRollbackIndicatorPath(), + GetForwardMergeIndicatorPath(), + }; for (const auto& file : files) { RemoveFileIfExists(file); } @@ -241,7 +248,7 @@ bool SnapshotManager::RemoveAllUpdateState(LockedFile* lock, const std::function return WriteUpdateState(lock, UpdateState::None); } -bool SnapshotManager::FinishedSnapshotWrites() { +bool SnapshotManager::FinishedSnapshotWrites(bool wipe) { auto lock = LockExclusive(); if (!lock) return false; @@ -261,6 +268,10 @@ bool SnapshotManager::FinishedSnapshotWrites() { return false; } + if (!UpdateForwardMergeIndicator(wipe)) { + return false; + } + // This file is written on boot to detect whether a rollback occurred. It // MUST NOT exist before rebooting, otherwise, we're at risk of deleting // snapshots too early. @@ -992,6 +1003,10 @@ std::string SnapshotManager::GetRollbackIndicatorPath() { return metadata_dir_ + "/" + android::base::Basename(kRollbackIndicatorPath); } +std::string SnapshotManager::GetForwardMergeIndicatorPath() { + return metadata_dir_ + "/allow-forward-merge"; +} + void SnapshotManager::AcknowledgeMergeSuccess(LockedFile* lock) { RemoveAllUpdateState(lock); } @@ -2438,6 +2453,9 @@ bool SnapshotManager::Dump(std::ostream& os) { ss << "Rollback indicator: " << (access(GetRollbackIndicatorPath().c_str(), F_OK) == 0 ? "exists" : strerror(errno)) << std::endl; + ss << "Forward merge indicator: " + << (access(GetForwardMergeIndicatorPath().c_str(), F_OK) == 0 ? "exists" : strerror(errno)) + << std::endl; bool ok = true; std::vector<std::string> snapshots; @@ -2504,17 +2522,39 @@ bool SnapshotManager::HandleImminentDataWipe(const std::function<void()>& callba return false; } - UpdateState state = ProcessUpdateState([&]() -> bool { - callback(); + auto process_callback = [&]() -> bool { + if (callback) { + callback(); + } return true; - }); + }; + if (!ProcessUpdateStateOnDataWipe(true /* allow_forward_merge */, process_callback)) { + return false; + } + + // Nothing should be depending on partitions now, so unmap them all. + if (!UnmapAllPartitions()) { + LOG(ERROR) << "Unable to unmap all partitions; fastboot may fail to flash."; + } + return true; +} + +bool SnapshotManager::ProcessUpdateStateOnDataWipe(bool allow_forward_merge, + const std::function<bool()>& callback) { + auto slot_number = SlotNumberForSlotSuffix(device_->GetSlotSuffix()); + UpdateState state = ProcessUpdateState(callback); LOG(INFO) << "Update state in recovery: " << state; switch (state) { case UpdateState::MergeFailed: LOG(ERROR) << "Unrecoverable merge failure detected."; return false; case UpdateState::Unverified: { - // If an OTA was just applied but has not yet started merging, we + // If an OTA was just applied but has not yet started merging: + // + // - if forward merge is allowed, initiate merge and call + // ProcessUpdateState again. + // + // - if forward merge is not allowed, we // have no choice but to revert slots, because the current slot will // immediately become unbootable. Rather than wait for the device // to reboot N times until a rollback, we proactively disable the @@ -2524,8 +2564,17 @@ bool SnapshotManager::HandleImminentDataWipe(const std::function<void()>& callba // as an error here. auto slot = GetCurrentSlot(); if (slot == Slot::Target) { + if (allow_forward_merge && + access(GetForwardMergeIndicatorPath().c_str(), F_OK) == 0) { + LOG(INFO) << "Forward merge allowed, initiating merge now."; + return InitiateMerge() && + ProcessUpdateStateOnDataWipe(false /* allow_forward_merge */, callback); + } + LOG(ERROR) << "Reverting to old slot since update will be deleted."; device_->SetSlotAsUnbootable(slot_number); + } else { + LOG(INFO) << "Booting from " << slot << " slot, no action is taken."; } break; } @@ -2537,11 +2586,6 @@ bool SnapshotManager::HandleImminentDataWipe(const std::function<void()>& callba default: break; } - - // Nothing should be depending on partitions now, so unmap them all. - if (!UnmapAllPartitions()) { - LOG(ERROR) << "Unable to unmap all partitions; fastboot may fail to flash."; - } return true; } @@ -2587,6 +2631,20 @@ CreateResult SnapshotManager::RecoveryCreateSnapshotDevices() { LOG(ERROR) << "Couldn't mount Metadata."; return CreateResult::NOT_CREATED; } + return RecoveryCreateSnapshotDevices(mount); +} + +CreateResult SnapshotManager::RecoveryCreateSnapshotDevices( + const std::unique_ptr<AutoDevice>& metadata_device) { + if (!device_->IsRecovery()) { + LOG(ERROR) << __func__ << " is only allowed in recovery."; + return CreateResult::NOT_CREATED; + } + + if (metadata_device == nullptr || !metadata_device->HasDevice()) { + LOG(ERROR) << "Metadata not mounted."; + return CreateResult::NOT_CREATED; + } auto state_file = GetStateFilePath(); if (access(state_file.c_str(), F_OK) != 0 && errno == ENOENT) { @@ -2608,5 +2666,24 @@ CreateResult SnapshotManager::RecoveryCreateSnapshotDevices() { return CreateResult::CREATED; } +bool SnapshotManager::UpdateForwardMergeIndicator(bool wipe) { + auto path = GetForwardMergeIndicatorPath(); + + if (!wipe) { + LOG(INFO) << "Wipe is not scheduled. Deleting forward merge indicator."; + return RemoveFileIfExists(path); + } + + // TODO(b/152094219): Don't forward merge if no CoW file is allocated. + + LOG(INFO) << "Wipe will be scheduled. Allowing forward merge of snapshots."; + if (!android::base::WriteStringToFile("1", path)) { + PLOG(ERROR) << "Unable to write forward merge indicator: " << path; + return false; + } + + return true; +} + } // namespace snapshot } // namespace android diff --git a/fs_mgr/libsnapshot/snapshot_test.cpp b/fs_mgr/libsnapshot/snapshot_test.cpp index 855451d82..f82c082bf 100644 --- a/fs_mgr/libsnapshot/snapshot_test.cpp +++ b/fs_mgr/libsnapshot/snapshot_test.cpp @@ -320,7 +320,7 @@ class SnapshotTest : public ::testing::Test { // Simulate a reboot into the new slot. AssertionResult SimulateReboot() { lock_ = nullptr; - if (!sm->FinishedSnapshotWrites()) { + if (!sm->FinishedSnapshotWrites(false)) { return AssertionFailure(); } if (!dm_.DeleteDevice("test_partition_b")) { @@ -424,7 +424,7 @@ TEST_F(SnapshotTest, MapPartialSnapshot) { } TEST_F(SnapshotTest, NoMergeBeforeReboot) { - ASSERT_TRUE(sm->FinishedSnapshotWrites()); + ASSERT_TRUE(sm->FinishedSnapshotWrites(false)); // Merge should fail, since the slot hasn't changed. ASSERT_FALSE(sm->InitiateMerge()); @@ -440,7 +440,7 @@ TEST_F(SnapshotTest, CleanFirstStageMount) { } TEST_F(SnapshotTest, FirstStageMountAfterRollback) { - ASSERT_TRUE(sm->FinishedSnapshotWrites()); + ASSERT_TRUE(sm->FinishedSnapshotWrites(false)); // We didn't change the slot, so we shouldn't need snapshots. TestDeviceInfo* info = new TestDeviceInfo(fake_super); @@ -476,7 +476,7 @@ TEST_F(SnapshotTest, Merge) { lock_ = nullptr; // Done updating. - ASSERT_TRUE(sm->FinishedSnapshotWrites()); + ASSERT_TRUE(sm->FinishedSnapshotWrites(false)); test_device->set_slot_suffix("_b"); ASSERT_TRUE(sm->InitiateMerge()); @@ -1007,7 +1007,7 @@ TEST_F(SnapshotUpdateTest, FullUpdateFlow) { ASSERT_TRUE(IsPartitionUnchanged(name)); } - ASSERT_TRUE(sm->FinishedSnapshotWrites()); + ASSERT_TRUE(sm->FinishedSnapshotWrites(false)); // Simulate shutting down the device. ASSERT_TRUE(UnmapAll()); @@ -1139,7 +1139,7 @@ TEST_F(SnapshotUpdateTest, TestRollback) { ASSERT_TRUE(IsPartitionUnchanged(name)); } - ASSERT_TRUE(sm->FinishedSnapshotWrites()); + ASSERT_TRUE(sm->FinishedSnapshotWrites(false)); // Simulate shutting down the device. ASSERT_TRUE(UnmapAll()); @@ -1171,7 +1171,7 @@ TEST_F(SnapshotUpdateTest, TestRollback) { // Test that if an update is applied but not booted into, it can be canceled. TEST_F(SnapshotUpdateTest, CancelAfterApply) { ASSERT_TRUE(sm->BeginUpdate()); - ASSERT_TRUE(sm->FinishedSnapshotWrites()); + ASSERT_TRUE(sm->FinishedSnapshotWrites(false)); ASSERT_TRUE(sm->CancelUpdate()); } @@ -1188,7 +1188,7 @@ TEST_F(SnapshotUpdateTest, ReclaimCow) { ASSERT_TRUE(sm->BeginUpdate()); ASSERT_TRUE(sm->CreateUpdateSnapshots(manifest_)); ASSERT_TRUE(MapUpdateSnapshots()); - ASSERT_TRUE(sm->FinishedSnapshotWrites()); + ASSERT_TRUE(sm->FinishedSnapshotWrites(false)); // Simulate shutting down the device. ASSERT_TRUE(UnmapAll()); @@ -1295,7 +1295,7 @@ TEST_F(SnapshotUpdateTest, RetrofitAfterRegularAb) { ASSERT_TRUE(IsPartitionUnchanged(name)); } - ASSERT_TRUE(sm->FinishedSnapshotWrites()); + ASSERT_TRUE(sm->FinishedSnapshotWrites(false)); } TEST_F(SnapshotUpdateTest, MergeCannotRemoveCow) { @@ -1324,7 +1324,7 @@ TEST_F(SnapshotUpdateTest, MergeCannotRemoveCow) { ASSERT_TRUE(sm->BeginUpdate()); ASSERT_TRUE(sm->CreateUpdateSnapshots(manifest_)); ASSERT_TRUE(MapUpdateSnapshots()); - ASSERT_TRUE(sm->FinishedSnapshotWrites()); + ASSERT_TRUE(sm->FinishedSnapshotWrites(false)); // Simulate shutting down the device. ASSERT_TRUE(UnmapAll()); @@ -1428,7 +1428,7 @@ TEST_F(SnapshotUpdateTest, MergeInRecovery) { ASSERT_TRUE(sm->BeginUpdate()); ASSERT_TRUE(sm->CreateUpdateSnapshots(manifest_)); ASSERT_TRUE(MapUpdateSnapshots()); - ASSERT_TRUE(sm->FinishedSnapshotWrites()); + ASSERT_TRUE(sm->FinishedSnapshotWrites(false)); // Simulate shutting down the device. ASSERT_TRUE(UnmapAll()); @@ -1460,7 +1460,7 @@ TEST_F(SnapshotUpdateTest, DataWipeRollbackInRecovery) { ASSERT_TRUE(sm->BeginUpdate()); ASSERT_TRUE(sm->CreateUpdateSnapshots(manifest_)); ASSERT_TRUE(MapUpdateSnapshots()); - ASSERT_TRUE(sm->FinishedSnapshotWrites()); + ASSERT_TRUE(sm->FinishedSnapshotWrites(false)); // Simulate shutting down the device. ASSERT_TRUE(UnmapAll()); @@ -1485,7 +1485,7 @@ TEST_F(SnapshotUpdateTest, DataWipeAfterRollback) { ASSERT_TRUE(sm->BeginUpdate()); ASSERT_TRUE(sm->CreateUpdateSnapshots(manifest_)); ASSERT_TRUE(MapUpdateSnapshots()); - ASSERT_TRUE(sm->FinishedSnapshotWrites()); + ASSERT_TRUE(sm->FinishedSnapshotWrites(false)); // Simulate shutting down the device. ASSERT_TRUE(UnmapAll()); @@ -1498,7 +1498,46 @@ TEST_F(SnapshotUpdateTest, DataWipeAfterRollback) { ASSERT_TRUE(new_sm->HandleImminentDataWipe()); EXPECT_EQ(new_sm->GetUpdateState(), UpdateState::None); EXPECT_FALSE(test_device->IsSlotUnbootable(0)); - EXPECT_FALSE(test_device->IsSlotUnbootable(0)); + EXPECT_FALSE(test_device->IsSlotUnbootable(1)); +} + +// Test update package that requests data wipe. +TEST_F(SnapshotUpdateTest, DataWipeRequiredInPackage) { + AddOperationForPartitions(); + // Execute the update. + ASSERT_TRUE(sm->BeginUpdate()); + ASSERT_TRUE(sm->CreateUpdateSnapshots(manifest_)); + + // Write some data to target partitions. + for (const auto& name : {"sys_b", "vnd_b", "prd_b"}) { + ASSERT_TRUE(WriteSnapshotAndHash(name)) << name; + } + + ASSERT_TRUE(sm->FinishedSnapshotWrites(true /* wipe */)); + + // Simulate shutting down the device. + ASSERT_TRUE(UnmapAll()); + + // Simulate a reboot into recovery. + auto test_device = new TestDeviceInfo(fake_super, "_b"); + test_device->set_recovery(true); + auto new_sm = SnapshotManager::NewForFirstStageMount(test_device); + + ASSERT_TRUE(new_sm->HandleImminentDataWipe()); + // Manually mount metadata so that we can call GetUpdateState() below. + MountMetadata(); + EXPECT_EQ(new_sm->GetUpdateState(), UpdateState::None); + ASSERT_FALSE(test_device->IsSlotUnbootable(1)); + ASSERT_FALSE(test_device->IsSlotUnbootable(0)); + + // Now reboot into new slot. + test_device = new TestDeviceInfo(fake_super, "_b"); + auto init = SnapshotManager::NewForFirstStageMount(test_device); + ASSERT_TRUE(init->CreateLogicalAndSnapshotPartitions("super", snapshot_timeout_)); + // Verify that we are on the downgraded build. + for (const auto& name : {"sys_b", "vnd_b", "prd_b"}) { + ASSERT_TRUE(IsPartitionUnchanged(name)) << name; + } } TEST_F(SnapshotUpdateTest, Hashtree) { @@ -1533,7 +1572,7 @@ TEST_F(SnapshotUpdateTest, Hashtree) { ASSERT_TRUE(WriteSnapshotAndHash("sys_b", partition_size)); // Finish update. - ASSERT_TRUE(sm->FinishedSnapshotWrites()); + ASSERT_TRUE(sm->FinishedSnapshotWrites(false)); // Simulate shutting down the device. ASSERT_TRUE(UnmapAll()); @@ -1569,7 +1608,7 @@ TEST_F(SnapshotUpdateTest, Overflow) { ASSERT_EQ(1u, table.size()); EXPECT_TRUE(table[0].IsOverflowSnapshot()); - ASSERT_FALSE(sm->FinishedSnapshotWrites()) + ASSERT_FALSE(sm->FinishedSnapshotWrites(false)) << "FinishedSnapshotWrites should detect overflow of CoW device."; } @@ -1623,7 +1662,7 @@ TEST_P(FlashAfterUpdateTest, FlashSlotAfterUpdate) { ASSERT_TRUE(sm->BeginUpdate()); ASSERT_TRUE(sm->CreateUpdateSnapshots(manifest_)); ASSERT_TRUE(MapUpdateSnapshots()); - ASSERT_TRUE(sm->FinishedSnapshotWrites()); + ASSERT_TRUE(sm->FinishedSnapshotWrites(false)); // Simulate shutting down the device. ASSERT_TRUE(UnmapAll()); diff --git a/fs_mgr/tests/Android.bp b/fs_mgr/tests/Android.bp index 4f6ec5a24..28dee88fe 100644 --- a/fs_mgr/tests/Android.bp +++ b/fs_mgr/tests/Android.bp @@ -18,6 +18,7 @@ cc_test { "cts", "device-tests", "vts", + "vts10", ], compile_multilib: "both", multilib: { diff --git a/healthd/BatteryMonitor.cpp b/healthd/BatteryMonitor.cpp index 8e9e074ab..8b67e2268 100644 --- a/healthd/BatteryMonitor.cpp +++ b/healthd/BatteryMonitor.cpp @@ -233,6 +233,15 @@ int BatteryMonitor::getIntField(const String8& path) { return value; } +bool BatteryMonitor::isScopedPowerSupply(const char* name) { + constexpr char kScopeDevice[] = "Device"; + + String8 path; + path.appendFormat("%s/%s/scope", POWER_SUPPLY_SYSFS_PATH, name); + std::string scope; + return (readFromFile(path, &scope) > 0 && scope == kScopeDevice); +} + void BatteryMonitor::updateValues(void) { initHealthInfo(mHealthInfo.get()); @@ -547,6 +556,11 @@ void BatteryMonitor::init(struct healthd_config *hc) { break; case ANDROID_POWER_SUPPLY_TYPE_BATTERY: + // Some devices expose the battery status of sub-component like + // stylus. Such a device-scoped battery info needs to be skipped + // in BatteryMonitor, which is intended to report the status of + // the battery supplying the power to the whole system. + if (isScopedPowerSupply(name)) continue; mBatteryDevicePresent = true; if (mHealthdConfig->batteryStatusPath.isEmpty()) { diff --git a/healthd/include/healthd/BatteryMonitor.h b/healthd/include/healthd/BatteryMonitor.h index d41a374a6..fadb5a58c 100644 --- a/healthd/include/healthd/BatteryMonitor.h +++ b/healthd/include/healthd/BatteryMonitor.h @@ -78,6 +78,7 @@ class BatteryMonitor { PowerSupplyType readPowerSupplyType(const String8& path); bool getBooleanField(const String8& path); int getIntField(const String8& path); + bool isScopedPowerSupply(const char* name); }; }; // namespace android diff --git a/init/Android.bp b/init/Android.bp index 72a7bfed4..d512a4e81 100644 --- a/init/Android.bp +++ b/init/Android.bp @@ -36,6 +36,7 @@ init_common_sources = [ "util.cpp", ] init_device_sources = [ + "block_dev_initializer.cpp", "bootchart.cpp", "builtins.cpp", "devices.cpp", @@ -240,6 +241,7 @@ cc_test { "firmware_handler_test.cpp", "init_test.cpp", "keychords_test.cpp", + "oneshot_on_test.cpp", "persistent_properties_test.cpp", "property_service_test.cpp", "property_type_test.cpp", @@ -257,6 +259,7 @@ cc_test { "cts", "device-tests", "vts", + "vts10", ], } diff --git a/init/Android.mk b/init/Android.mk index 07b0f950f..b49fb3b8b 100644 --- a/init/Android.mk +++ b/init/Android.mk @@ -48,6 +48,7 @@ ifneq ($(BOARD_BUILD_SYSTEM_ROOT_IMAGE),true) include $(CLEAR_VARS) LOCAL_CPPFLAGS := $(init_cflags) LOCAL_SRC_FILES := \ + block_dev_initializer.cpp \ devices.cpp \ first_stage_init.cpp \ first_stage_main.cpp \ @@ -105,9 +106,8 @@ LOCAL_STATIC_LIBRARIES := \ libgsi \ libcom.android.sysprop.apex \ liblzma \ - libdexfile_support_static \ - libunwindstack \ - libbacktrace \ + libunwindstack_no_dex \ + libbacktrace_no_dex \ libmodprobe \ libext2_uuid \ libprotobuf-cpp-lite \ diff --git a/init/README.md b/init/README.md index 4f0a7eced..13f1bac65 100644 --- a/init/README.md +++ b/init/README.md @@ -720,23 +720,35 @@ Init provides state information with the following properties. characteristics in a device agnostic manner. Init responds to properties that begin with `ctl.`. These properties take the format of -`ctl.<command>` and the _value_ of the system property is used as a parameter, for example: -`SetProperty("ctl.start", "logd")` will run the `start` command on `logd`. Note that these +`ctl.[<target>_]<command>` and the _value_ of the system property is used as a parameter. The +_target_ is optional and specifies the service option that _value_ is meant to match with. There is +only one option for _target_, `interface` which indicates that _value_ will refer to an interface +that a service provides and not the service name itself. + +For example: + +`SetProperty("ctl.start", "logd")` will run the `start` command on `logd`. + +`SetProperty("ctl.interface_start", "aidl/aidl_lazy_test_1")` will run the `start` command on the +service that exposes the `aidl aidl_lazy_test_1` interface. + +Note that these properties are only settable; they will have no value when read. -`ctl.start` \ -`ctl.restart` \ -`ctl.stop` -> These are equivalent to using the `start`, `restart`, and `stop` commands on the service specified +The _commands_ are listed below. + +`start` \ +`restart` \ +`stop` \ +These are equivalent to using the `start`, `restart`, and `stop` commands on the service specified by the _value_ of the property. -`ctl.interface_start` \ -`ctl.interface_restart` \ -`ctl.interface_stop` -> These are equivalent to using the `interface_start`, `interface_restart`, and `interface_stop` -commands on the interface specified by the _value_ of the property. +`oneshot_one` and `oneshot_off` will turn on or off the _oneshot_ +flag for the service specified by the _value_ of the property. This is +particularly intended for services that are conditionally lazy HALs. When +they are lazy HALs, oneshot must be on, otherwise oneshot should be off. -`ctl.sigstop_on` and `ctl.sigstop_off` will turn on or off the _sigstop_ feature for the service +`sigstop_on` and `sigstop_off` will turn on or off the _sigstop_ feature for the service specified by the _value_ of the property. See the _Debugging init_ section below for more details about this feature. diff --git a/init/block_dev_initializer.cpp b/init/block_dev_initializer.cpp new file mode 100644 index 000000000..b423f864e --- /dev/null +++ b/init/block_dev_initializer.cpp @@ -0,0 +1,148 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <chrono> +#include <string_view> +#include <vector> + +#include <android-base/chrono_utils.h> +#include <android-base/logging.h> +#include <android-base/strings.h> +#include <fs_mgr.h> + +#include "block_dev_initializer.h" + +namespace android { +namespace init { + +using android::base::Timer; +using namespace std::chrono_literals; + +BlockDevInitializer::BlockDevInitializer() : uevent_listener_(16 * 1024 * 1024) { + auto boot_devices = android::fs_mgr::GetBootDevices(); + device_handler_ = std::make_unique<DeviceHandler>( + std::vector<Permissions>{}, std::vector<SysfsPermissions>{}, std::vector<Subsystem>{}, + std::move(boot_devices), false); +} + +bool BlockDevInitializer::InitDeviceMapper() { + const std::string dm_path = "/devices/virtual/misc/device-mapper"; + bool found = false; + auto dm_callback = [this, &dm_path, &found](const Uevent& uevent) { + if (uevent.path == dm_path) { + device_handler_->HandleUevent(uevent); + found = true; + return ListenerAction::kStop; + } + return ListenerAction::kContinue; + }; + uevent_listener_.RegenerateUeventsForPath("/sys" + dm_path, dm_callback); + if (!found) { + LOG(INFO) << "device-mapper device not found in /sys, waiting for its uevent"; + Timer t; + uevent_listener_.Poll(dm_callback, 10s); + LOG(INFO) << "Wait for device-mapper returned after " << t; + } + if (!found) { + LOG(ERROR) << "device-mapper device not found after polling timeout"; + return false; + } + return true; +} + +ListenerAction BlockDevInitializer::HandleUevent(const Uevent& uevent, + std::set<std::string>* devices) { + // Ignore everything that is not a block device. + if (uevent.subsystem != "block") { + return ListenerAction::kContinue; + } + + auto name = uevent.partition_name; + if (name.empty()) { + size_t base_idx = uevent.path.rfind('/'); + if (base_idx == std::string::npos) { + return ListenerAction::kContinue; + } + name = uevent.path.substr(base_idx + 1); + } + + auto iter = devices->find(name); + if (iter == devices->end()) { + return ListenerAction::kContinue; + } + + LOG(VERBOSE) << __PRETTY_FUNCTION__ << ": found partition: " << name; + + devices->erase(iter); + device_handler_->HandleUevent(uevent); + return devices->empty() ? ListenerAction::kStop : ListenerAction::kContinue; +} + +bool BlockDevInitializer::InitDevices(std::set<std::string> devices) { + auto uevent_callback = [&, this](const Uevent& uevent) -> ListenerAction { + return HandleUevent(uevent, &devices); + }; + uevent_listener_.RegenerateUevents(uevent_callback); + + // UeventCallback() will remove found partitions from |devices|. So if it + // isn't empty here, it means some partitions are not found. + if (!devices.empty()) { + LOG(INFO) << __PRETTY_FUNCTION__ + << ": partition(s) not found in /sys, waiting for their uevent(s): " + << android::base::Join(devices, ", "); + Timer t; + uevent_listener_.Poll(uevent_callback, 10s); + LOG(INFO) << "Wait for partitions returned after " << t; + } + + if (!devices.empty()) { + LOG(ERROR) << __PRETTY_FUNCTION__ << ": partition(s) not found after polling timeout: " + << android::base::Join(devices, ", "); + return false; + } + return true; +} + +// Creates "/dev/block/dm-XX" for dm nodes by running coldboot on /sys/block/dm-XX. +bool BlockDevInitializer::InitDmDevice(const std::string& device) { + const std::string device_name(basename(device.c_str())); + const std::string syspath = "/sys/block/" + device_name; + bool found = false; + + auto uevent_callback = [&device_name, &device, this, &found](const Uevent& uevent) { + if (uevent.device_name == device_name) { + LOG(VERBOSE) << "Creating device-mapper device : " << device; + device_handler_->HandleUevent(uevent); + found = true; + return ListenerAction::kStop; + } + return ListenerAction::kContinue; + }; + + uevent_listener_.RegenerateUeventsForPath(syspath, uevent_callback); + if (!found) { + LOG(INFO) << "dm device '" << device << "' not found in /sys, waiting for its uevent"; + Timer t; + uevent_listener_.Poll(uevent_callback, 10s); + LOG(INFO) << "wait for dm device '" << device << "' returned after " << t; + } + if (!found) { + LOG(ERROR) << "dm device '" << device << "' not found after polling timeout"; + return false; + } + return true; +} + +} // namespace init +} // namespace android diff --git a/init/block_dev_initializer.h b/init/block_dev_initializer.h new file mode 100644 index 000000000..0d4c6e9fc --- /dev/null +++ b/init/block_dev_initializer.h @@ -0,0 +1,41 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <memory> +#include <set> +#include <string> + +#include "devices.h" +#include "uevent_listener.h" + +namespace android { +namespace init { + +class BlockDevInitializer final { + public: + BlockDevInitializer(); + + bool InitDeviceMapper(); + bool InitDevices(std::set<std::string> devices); + bool InitDmDevice(const std::string& device); + + private: + ListenerAction HandleUevent(const Uevent& uevent, std::set<std::string>* devices); + + std::unique_ptr<DeviceHandler> device_handler_; + UeventListener uevent_listener_; +}; + +} // namespace init +} // namespace android diff --git a/init/first_stage_mount.cpp b/init/first_stage_mount.cpp index 622e45716..8eb2f9748 100644 --- a/init/first_stage_mount.cpp +++ b/init/first_stage_mount.cpp @@ -42,6 +42,7 @@ #include <liblp/liblp.h> #include <libsnapshot/snapshot.h> +#include "block_dev_initializer.h" #include "devices.h" #include "switch_root.h" #include "uevent.h" @@ -84,11 +85,7 @@ class FirstStageMount { bool InitDevices(); protected: - ListenerAction HandleBlockDevice(const std::string& name, const Uevent&, - std::set<std::string>* required_devices); bool InitRequiredDevices(std::set<std::string> devices); - bool InitMappedDevice(const std::string& verity_device); - bool InitDeviceMapper(); bool CreateLogicalPartitions(); bool MountPartition(const Fstab::iterator& begin, bool erase_same_mounts, Fstab::iterator* end = nullptr); @@ -97,7 +94,7 @@ class FirstStageMount { bool TrySwitchSystemAsRoot(); bool TrySkipMountingPartitions(); bool IsDmLinearEnabled(); - void GetDmLinearMetadataDevice(std::set<std::string>* devices); + void GetSuperDeviceName(std::set<std::string>* devices); bool InitDmLinearBackingDevices(const android::fs_mgr::LpMetadata& metadata); void UseDsuIfPresent(); // Reads all fstab.avb_keys from the ramdisk for first-stage mount. @@ -106,8 +103,6 @@ class FirstStageMount { // revocation check by DSU installation service. void CopyDsuAvbKeys(); - ListenerAction UeventCallback(const Uevent& uevent, std::set<std::string>* required_devices); - // Pure virtual functions. virtual bool GetDmVerityDevices(std::set<std::string>* devices) = 0; virtual bool SetUpDmVerity(FstabEntry* fstab_entry) = 0; @@ -116,10 +111,10 @@ class FirstStageMount { bool dsu_not_on_userdata_ = false; Fstab fstab_; - std::string lp_metadata_partition_; + // The super path is only set after InitDevices, and is invalid before. + std::string super_path_; std::string super_partition_name_; - std::unique_ptr<DeviceHandler> device_handler_; - UeventListener uevent_listener_; + BlockDevInitializer block_dev_init_; // Reads all AVB keys before chroot into /system, as they might be used // later when mounting other partitions, e.g., /vendor and /product. std::map<std::string, std::vector<std::string>> preload_avb_key_blobs_; @@ -233,13 +228,7 @@ static bool IsStandaloneImageRollback(const AvbHandle& builtin_vbmeta, // Class Definitions // ----------------- -FirstStageMount::FirstStageMount(Fstab fstab) - : need_dm_verity_(false), fstab_(std::move(fstab)), uevent_listener_(16 * 1024 * 1024) { - auto boot_devices = android::fs_mgr::GetBootDevices(); - device_handler_ = std::make_unique<DeviceHandler>( - std::vector<Permissions>{}, std::vector<SysfsPermissions>{}, std::vector<Subsystem>{}, - std::move(boot_devices), false); - +FirstStageMount::FirstStageMount(Fstab fstab) : need_dm_verity_(false), fstab_(std::move(fstab)) { super_partition_name_ = fs_mgr_get_super_partition_name(); } @@ -268,12 +257,23 @@ bool FirstStageMount::DoFirstStageMount() { bool FirstStageMount::InitDevices() { std::set<std::string> devices; - GetDmLinearMetadataDevice(&devices); + GetSuperDeviceName(&devices); if (!GetDmVerityDevices(&devices)) { return false; } - return InitRequiredDevices(std::move(devices)); + if (!InitRequiredDevices(std::move(devices))) { + return false; + } + + if (IsDmLinearEnabled()) { + auto super_symlink = "/dev/block/by-name/"s + super_partition_name_; + if (!android::base::Realpath(super_symlink, &super_path_)) { + PLOG(ERROR) << "realpath failed: " << super_symlink; + return false; + } + } + return true; } bool FirstStageMount::IsDmLinearEnabled() { @@ -283,7 +283,7 @@ bool FirstStageMount::IsDmLinearEnabled() { return false; } -void FirstStageMount::GetDmLinearMetadataDevice(std::set<std::string>* devices) { +void FirstStageMount::GetSuperDeviceName(std::set<std::string>* devices) { // Add any additional devices required for dm-linear mappings. if (!IsDmLinearEnabled()) { return; @@ -296,62 +296,13 @@ void FirstStageMount::GetDmLinearMetadataDevice(std::set<std::string>* devices) // Found partitions will then be removed from it for the subsequent member // function to check which devices are NOT created. bool FirstStageMount::InitRequiredDevices(std::set<std::string> devices) { - if (!InitDeviceMapper()) { + if (!block_dev_init_.InitDeviceMapper()) { return false; } - if (devices.empty()) { return true; } - - auto uevent_callback = [&, this](const Uevent& uevent) { - return UeventCallback(uevent, &devices); - }; - uevent_listener_.RegenerateUevents(uevent_callback); - - // UeventCallback() will remove found partitions from |devices|. So if it - // isn't empty here, it means some partitions are not found. - if (!devices.empty()) { - LOG(INFO) << __PRETTY_FUNCTION__ - << ": partition(s) not found in /sys, waiting for their uevent(s): " - << android::base::Join(devices, ", "); - Timer t; - uevent_listener_.Poll(uevent_callback, 10s); - LOG(INFO) << "Wait for partitions returned after " << t; - } - - if (!devices.empty()) { - LOG(ERROR) << __PRETTY_FUNCTION__ << ": partition(s) not found after polling timeout: " - << android::base::Join(devices, ", "); - return false; - } - - return true; -} - -bool FirstStageMount::InitDeviceMapper() { - const std::string dm_path = "/devices/virtual/misc/device-mapper"; - bool found = false; - auto dm_callback = [this, &dm_path, &found](const Uevent& uevent) { - if (uevent.path == dm_path) { - device_handler_->HandleUevent(uevent); - found = true; - return ListenerAction::kStop; - } - return ListenerAction::kContinue; - }; - uevent_listener_.RegenerateUeventsForPath("/sys" + dm_path, dm_callback); - if (!found) { - LOG(INFO) << "device-mapper device not found in /sys, waiting for its uevent"; - Timer t; - uevent_listener_.Poll(dm_callback, 10s); - LOG(INFO) << "Wait for device-mapper returned after " << t; - } - if (!found) { - LOG(ERROR) << "device-mapper device not found after polling timeout"; - return false; - } - return true; + return block_dev_init_.InitDevices(std::move(devices)); } bool FirstStageMount::InitDmLinearBackingDevices(const android::fs_mgr::LpMetadata& metadata) { @@ -375,7 +326,7 @@ bool FirstStageMount::CreateLogicalPartitions() { if (!IsDmLinearEnabled()) { return true; } - if (lp_metadata_partition_.empty()) { + if (super_path_.empty()) { LOG(ERROR) << "Could not locate logical partition tables in partition " << super_partition_name_; return false; @@ -392,92 +343,19 @@ bool FirstStageMount::CreateLogicalPartitions() { if (!InitRequiredDevices({"userdata"})) { return false; } - return sm->CreateLogicalAndSnapshotPartitions(lp_metadata_partition_); + return sm->CreateLogicalAndSnapshotPartitions(super_path_); } } - auto metadata = android::fs_mgr::ReadCurrentMetadata(lp_metadata_partition_); + auto metadata = android::fs_mgr::ReadCurrentMetadata(super_path_); if (!metadata) { - LOG(ERROR) << "Could not read logical partition metadata from " << lp_metadata_partition_; + LOG(ERROR) << "Could not read logical partition metadata from " << super_path_; return false; } if (!InitDmLinearBackingDevices(*metadata.get())) { return false; } - return android::fs_mgr::CreateLogicalPartitions(*metadata.get(), lp_metadata_partition_); -} - -ListenerAction FirstStageMount::HandleBlockDevice(const std::string& name, const Uevent& uevent, - std::set<std::string>* required_devices) { - // Matches partition name to create device nodes. - // Both required_devices_partition_names_ and uevent->partition_name have A/B - // suffix when A/B is used. - auto iter = required_devices->find(name); - if (iter != required_devices->end()) { - LOG(VERBOSE) << __PRETTY_FUNCTION__ << ": found partition: " << *iter; - if (IsDmLinearEnabled() && name == super_partition_name_) { - std::vector<std::string> links = device_handler_->GetBlockDeviceSymlinks(uevent); - lp_metadata_partition_ = links[0]; - } - required_devices->erase(iter); - device_handler_->HandleUevent(uevent); - if (required_devices->empty()) { - return ListenerAction::kStop; - } else { - return ListenerAction::kContinue; - } - } - return ListenerAction::kContinue; -} - -ListenerAction FirstStageMount::UeventCallback(const Uevent& uevent, - std::set<std::string>* required_devices) { - // Ignores everything that is not a block device. - if (uevent.subsystem != "block") { - return ListenerAction::kContinue; - } - - if (!uevent.partition_name.empty()) { - return HandleBlockDevice(uevent.partition_name, uevent, required_devices); - } else { - size_t base_idx = uevent.path.rfind('/'); - if (base_idx != std::string::npos) { - return HandleBlockDevice(uevent.path.substr(base_idx + 1), uevent, required_devices); - } - } - // Not found a partition or find an unneeded partition, continue to find others. - return ListenerAction::kContinue; -} - -// Creates "/dev/block/dm-XX" for dm-verity by running coldboot on /sys/block/dm-XX. -bool FirstStageMount::InitMappedDevice(const std::string& dm_device) { - const std::string device_name(basename(dm_device.c_str())); - const std::string syspath = "/sys/block/" + device_name; - bool found = false; - - auto verity_callback = [&device_name, &dm_device, this, &found](const Uevent& uevent) { - if (uevent.device_name == device_name) { - LOG(VERBOSE) << "Creating device-mapper device : " << dm_device; - device_handler_->HandleUevent(uevent); - found = true; - return ListenerAction::kStop; - } - return ListenerAction::kContinue; - }; - - uevent_listener_.RegenerateUeventsForPath(syspath, verity_callback); - if (!found) { - LOG(INFO) << "dm device '" << dm_device << "' not found in /sys, waiting for its uevent"; - Timer t; - uevent_listener_.Poll(verity_callback, 10s); - LOG(INFO) << "wait for dm device '" << dm_device << "' returned after " << t; - } - if (!found) { - LOG(ERROR) << "dm device '" << dm_device << "' not found after polling timeout"; - return false; - } - - return true; + return android::fs_mgr::CreateLogicalPartitions(*metadata.get(), super_path_); } bool FirstStageMount::MountPartition(const Fstab::iterator& begin, bool erase_same_mounts, @@ -491,7 +369,7 @@ bool FirstStageMount::MountPartition(const Fstab::iterator& begin, bool erase_sa if (!fs_mgr_update_logical_partition(&(*begin))) { return false; } - if (!InitMappedDevice(begin->blk_device)) { + if (!block_dev_init_.InitDmDevice(begin->blk_device)) { return false; } } @@ -658,7 +536,9 @@ bool FirstStageMount::MountPartitions() { auto init_devices = [this](std::set<std::string> devices) -> bool { for (auto iter = devices.begin(); iter != devices.end();) { if (android::base::StartsWith(*iter, "/dev/block/dm-")) { - if (!InitMappedDevice(*iter)) return false; + if (!block_dev_init_.InitDmDevice(*iter)) { + return false; + } iter = devices.erase(iter); } else { iter++; @@ -774,7 +654,7 @@ bool FirstStageMountVBootV1::SetUpDmVerity(FstabEntry* fstab_entry) { // The exact block device name (fstab_rec->blk_device) is changed to // "/dev/block/dm-XX". Needs to create it because ueventd isn't started in init // first stage. - return InitMappedDevice(fstab_entry->blk_device); + return block_dev_init_.InitDmDevice(fstab_entry->blk_device); default: return false; } @@ -894,7 +774,7 @@ bool FirstStageMountVBootV2::SetUpDmVerity(FstabEntry* fstab_entry) { // The exact block device name (fstab_rec->blk_device) is changed to // "/dev/block/dm-XX". Needs to create it because ueventd isn't started in init // first stage. - return InitMappedDevice(fstab_entry->blk_device); + return block_dev_init_.InitDmDevice(fstab_entry->blk_device); default: return false; } diff --git a/init/init.cpp b/init/init.cpp index b29dfa3f6..a9d63013f 100644 --- a/init/init.cpp +++ b/init/init.cpp @@ -22,6 +22,7 @@ #include <signal.h> #include <stdlib.h> #include <string.h> +#include <sys/eventfd.h> #include <sys/mount.h> #include <sys/signalfd.h> #include <sys/types.h> @@ -45,6 +46,7 @@ #include <android-base/properties.h> #include <android-base/stringprintf.h> #include <android-base/strings.h> +#include <backtrace/Backtrace.h> #include <fs_avb/fs_avb.h> #include <fs_mgr_vendor_overlay.h> #include <keyutils.h> @@ -81,6 +83,7 @@ using namespace std::chrono_literals; using namespace std::string_literals; using android::base::boot_clock; +using android::base::ConsumePrefix; using android::base::GetProperty; using android::base::ReadFileToString; using android::base::SetProperty; @@ -113,30 +116,26 @@ static std::queue<PendingControlMessage> pending_control_messages; // to fill that socket and deadlock the system. Now we use locks to handle the property changes // directly in the property thread, however we still must wake the epoll to inform init that there // is a change to process, so we use this FD. It is non-blocking, since we do not care how many -// times WakeEpoll() is called, only that the epoll will wake. -static int wake_epoll_fd = -1; +// times WakeMainInitThread() is called, only that the epoll will wake. +static int wake_main_thread_fd = -1; static void InstallInitNotifier(Epoll* epoll) { - int sockets[2]; - if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC | SOCK_NONBLOCK, 0, sockets) != 0) { - PLOG(FATAL) << "Failed to socketpair() between property_service and init"; + wake_main_thread_fd = eventfd(0, EFD_CLOEXEC); + if (wake_main_thread_fd == -1) { + PLOG(FATAL) << "Failed to create eventfd for waking init"; } - int epoll_fd = sockets[0]; - wake_epoll_fd = sockets[1]; - - auto drain_socket = [epoll_fd] { - char buf[512]; - while (read(epoll_fd, buf, sizeof(buf)) > 0) { - } + auto clear_eventfd = [] { + uint64_t counter; + TEMP_FAILURE_RETRY(read(wake_main_thread_fd, &counter, sizeof(counter))); }; - if (auto result = epoll->RegisterHandler(epoll_fd, drain_socket); !result.ok()) { + if (auto result = epoll->RegisterHandler(wake_main_thread_fd, clear_eventfd); !result.ok()) { LOG(FATAL) << result.error(); } } -static void WakeEpoll() { - constexpr char value[] = "1"; - write(wake_epoll_fd, value, sizeof(value)); +static void WakeMainInitThread() { + uint64_t counter = 1; + TEMP_FAILURE_RETRY(write(wake_main_thread_fd, &counter, sizeof(counter))); } static class PropWaiterState { @@ -180,7 +179,7 @@ static class PropWaiterState { LOG(INFO) << "Wait for property '" << wait_prop_name_ << "=" << wait_prop_value_ << "' took " << *waiting_for_prop_; ResetWaitForPropLocked(); - WakeEpoll(); + WakeMainInitThread(); } } } @@ -217,6 +216,16 @@ void ResetWaitForProp() { prop_waiter_state.ResetWaitForProp(); } +static void UnwindMainThreadStack() { + std::unique_ptr<Backtrace> backtrace(Backtrace::Create(BACKTRACE_CURRENT_PROCESS, 1)); + if (!backtrace->Unwind(0)) { + LOG(ERROR) << __FUNCTION__ << ": Failed to unwind callstack."; + } + for (size_t i = 0; i < backtrace->NumFrames(); i++) { + LOG(ERROR) << backtrace->FormatFrameData(i); + } +} + static class ShutdownState { public: void TriggerShutdown(const std::string& command) { @@ -228,7 +237,7 @@ static class ShutdownState { auto lock = std::lock_guard{shutdown_command_lock_}; shutdown_command_ = command; do_shutdown_ = true; - WakeEpoll(); + WakeMainInitThread(); } std::optional<std::string> CheckShutdown() { @@ -240,12 +249,27 @@ static class ShutdownState { return {}; } + bool do_shutdown() const { return do_shutdown_; } + private: std::mutex shutdown_command_lock_; std::string shutdown_command_; bool do_shutdown_ = false; } shutdown_state; +void DebugRebootLogging() { + LOG(INFO) << "do_shutdown: " << shutdown_state.do_shutdown() + << " IsShuttingDown: " << IsShuttingDown(); + if (shutdown_state.do_shutdown()) { + LOG(ERROR) << "sys.powerctl set while a previous shutdown command has not been handled"; + UnwindMainThreadStack(); + } + if (IsShuttingDown()) { + LOG(ERROR) << "sys.powerctl set while init is already shutting down"; + UnwindMainThreadStack(); + } +} + void DumpState() { ServiceList::GetInstance().DumpState(); ActionManager::GetInstance().DumpState(); @@ -312,7 +336,7 @@ void PropertyChanged(const std::string& name, const std::string& value) { if (property_triggers_enabled) { ActionManager::GetInstance().QueuePropertyChange(name, value); - WakeEpoll(); + WakeMainInitThread(); } prop_waiter_state.CheckAndResetWait(name, value); @@ -367,40 +391,27 @@ enum class ControlTarget { INTERFACE, // action gets called for every service that holds this interface }; -struct ControlMessageFunction { - ControlTarget target; - std::function<Result<void>(Service*)> action; -}; +using ControlMessageFunction = std::function<Result<void>(Service*)>; -static const std::map<std::string, ControlMessageFunction>& get_control_message_map() { +static const std::map<std::string, ControlMessageFunction, std::less<>>& GetControlMessageMap() { // clang-format off - static const std::map<std::string, ControlMessageFunction> control_message_functions = { - {"sigstop_on", {ControlTarget::SERVICE, - [](auto* service) { service->set_sigstop(true); return Result<void>{}; }}}, - {"sigstop_off", {ControlTarget::SERVICE, - [](auto* service) { service->set_sigstop(false); return Result<void>{}; }}}, - {"start", {ControlTarget::SERVICE, DoControlStart}}, - {"stop", {ControlTarget::SERVICE, DoControlStop}}, - {"restart", {ControlTarget::SERVICE, DoControlRestart}}, - {"interface_start", {ControlTarget::INTERFACE, DoControlStart}}, - {"interface_stop", {ControlTarget::INTERFACE, DoControlStop}}, - {"interface_restart", {ControlTarget::INTERFACE, DoControlRestart}}, + static const std::map<std::string, ControlMessageFunction, std::less<>> control_message_functions = { + {"sigstop_on", [](auto* service) { service->set_sigstop(true); return Result<void>{}; }}, + {"sigstop_off", [](auto* service) { service->set_sigstop(false); return Result<void>{}; }}, + {"oneshot_on", [](auto* service) { service->set_oneshot(true); return Result<void>{}; }}, + {"oneshot_off", [](auto* service) { service->set_oneshot(false); return Result<void>{}; }}, + {"start", DoControlStart}, + {"stop", DoControlStop}, + {"restart", DoControlRestart}, }; // clang-format on return control_message_functions; } -bool HandleControlMessage(const std::string& msg, const std::string& name, pid_t pid) { - const auto& map = get_control_message_map(); - const auto it = map.find(msg); - - if (it == map.end()) { - LOG(ERROR) << "Unknown control msg '" << msg << "'"; - return false; - } - - std::string cmdline_path = StringPrintf("proc/%d/cmdline", pid); +static bool HandleControlMessage(std::string_view message, const std::string& name, + pid_t from_pid) { + std::string cmdline_path = StringPrintf("proc/%d/cmdline", from_pid); std::string process_cmdline; if (ReadFileToString(cmdline_path, &process_cmdline)) { std::replace(process_cmdline.begin(), process_cmdline.end(), '\0', ' '); @@ -409,37 +420,37 @@ bool HandleControlMessage(const std::string& msg, const std::string& name, pid_t process_cmdline = "unknown process"; } - const ControlMessageFunction& function = it->second; - - Service* svc = nullptr; + Service* service = nullptr; + auto action = message; + if (ConsumePrefix(&action, "interface_")) { + service = ServiceList::GetInstance().FindInterface(name); + } else { + service = ServiceList::GetInstance().FindService(name); + } - switch (function.target) { - case ControlTarget::SERVICE: - svc = ServiceList::GetInstance().FindService(name); - break; - case ControlTarget::INTERFACE: - svc = ServiceList::GetInstance().FindInterface(name); - break; - default: - LOG(ERROR) << "Invalid function target from static map key ctl." << msg << ": " - << static_cast<std::underlying_type<ControlTarget>::type>(function.target); - return false; + if (service == nullptr) { + LOG(ERROR) << "Control message: Could not find '" << name << "' for ctl." << message + << " from pid: " << from_pid << " (" << process_cmdline << ")"; + return false; } - if (svc == nullptr) { - LOG(ERROR) << "Control message: Could not find '" << name << "' for ctl." << msg - << " from pid: " << pid << " (" << process_cmdline << ")"; + const auto& map = GetControlMessageMap(); + const auto it = map.find(action); + if (it == map.end()) { + LOG(ERROR) << "Unknown control msg '" << message << "'"; return false; } + const auto& function = it->second; - if (auto result = function.action(svc); !result.ok()) { - LOG(ERROR) << "Control message: Could not ctl." << msg << " for '" << name - << "' from pid: " << pid << " (" << process_cmdline << "): " << result.error(); + if (auto result = function(service); !result.ok()) { + LOG(ERROR) << "Control message: Could not ctl." << message << " for '" << name + << "' from pid: " << from_pid << " (" << process_cmdline + << "): " << result.error(); return false; } - LOG(INFO) << "Control message: Processed ctl." << msg << " for '" << name - << "' from pid: " << pid << " (" << process_cmdline << ")"; + LOG(INFO) << "Control message: Processed ctl." << message << " for '" << name + << "' from pid: " << from_pid << " (" << process_cmdline << ")"; return true; } @@ -451,7 +462,7 @@ bool QueueControlMessage(const std::string& message, const std::string& name, pi return false; } pending_control_messages.push({message, name, pid, fd}); - WakeEpoll(); + WakeMainInitThread(); return true; } @@ -477,7 +488,7 @@ static void HandleControlMessages() { } // If we still have items to process, make sure we wake back up to do so. if (!pending_control_messages.empty()) { - WakeEpoll(); + WakeMainInitThread(); } } @@ -713,8 +724,15 @@ int SecondStageMain(int argc, char** argv) { InitKernelLogging(argv); LOG(INFO) << "init second stage started!"; - // Will handle EPIPE at the time of write by checking the errno - signal(SIGPIPE, SIG_IGN); + // Init should not crash because of a dependence on any other process, therefore we ignore + // SIGPIPE and handle EPIPE at the call site directly. Note that setting a signal to SIG_IGN + // is inherited across exec, but custom signal handlers are not. Since we do not want to + // ignore SIGPIPE for child processes, we set a no-op function for the signal handler instead. + { + struct sigaction action = {.sa_flags = SA_RESTART}; + action.sa_handler = [](int) {}; + sigaction(SIGPIPE, &action, nullptr); + } // Set init and its forked children's oom_adj. if (auto result = @@ -853,6 +871,8 @@ int SecondStageMain(int argc, char** argv) { auto shutdown_command = shutdown_state.CheckShutdown(); if (shutdown_command) { + LOG(INFO) << "Got shutdown_command '" << *shutdown_command + << "' Calling HandlePowerctlMessage()"; HandlePowerctlMessage(*shutdown_command); } @@ -887,7 +907,9 @@ int SecondStageMain(int argc, char** argv) { (*function)(); } } - HandleControlMessages(); + if (!IsShuttingDown()) { + HandleControlMessages(); + } } return 0; diff --git a/init/init.h b/init/init.h index 27f64e297..4f686cb5b 100644 --- a/init/init.h +++ b/init/init.h @@ -42,6 +42,8 @@ void SendLoadPersistentPropertiesMessage(); void PropertyChanged(const std::string& name, const std::string& value); bool QueueControlMessage(const std::string& message, const std::string& name, pid_t pid, int fd); +void DebugRebootLogging(); + int SecondStageMain(int argc, char** argv); } // namespace init diff --git a/init/oneshot_on_test.cpp b/init/oneshot_on_test.cpp new file mode 100644 index 000000000..650f0650b --- /dev/null +++ b/init/oneshot_on_test.cpp @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <gtest/gtest.h> + +#include <chrono> + +#include <android-base/properties.h> + +using android::base::GetProperty; +using android::base::SetProperty; +using android::base::WaitForProperty; +using namespace std::literals; + +TEST(init, oneshot_on) { + if (getuid() != 0) { + GTEST_SKIP() << "Skipping test, must be run as root."; + return; + } + + // Bootanim shouldn't be running once the device has booted. + ASSERT_EQ("stopped", GetProperty("init.svc.bootanim", "")); + + SetProperty("ctl.oneshot_off", "bootanim"); + SetProperty("ctl.start", "bootanim"); + + // Bootanim exits quickly when the device is fully booted, so check that it goes back to the + // 'restarting' state that non-oneshot services enter once they've restarted. + EXPECT_TRUE(WaitForProperty("init.svc.bootanim", "restarting", 10s)); + + SetProperty("ctl.oneshot_on", "bootanim"); + SetProperty("ctl.start", "bootanim"); + + // Now that oneshot is enabled again, bootanim should transition into the 'stopped' state. + EXPECT_TRUE(WaitForProperty("init.svc.bootanim", "stopped", 10s)); +} diff --git a/init/property_service.cpp b/init/property_service.cpp index 820652249..1e4e127f9 100644 --- a/init/property_service.cpp +++ b/init/property_service.cpp @@ -125,7 +125,7 @@ void StartSendingMessages() { void StopSendingMessages() { auto lock = std::lock_guard{accept_messages_lock}; - accept_messages = true; + accept_messages = false; } bool CanReadProperty(const std::string& source_context, const std::string& name) { @@ -489,6 +489,7 @@ uint32_t HandlePropertySet(const std::string& name, const std::string& value, } LOG(INFO) << "Received sys.powerctl='" << value << "' from pid: " << cr.pid << process_log_string; + DebugRebootLogging(); } // If a process other than init is writing a non-empty value, it means that process is diff --git a/init/reboot.cpp b/init/reboot.cpp index f006df3a3..081f6953a 100644 --- a/init/reboot.cpp +++ b/init/reboot.cpp @@ -72,6 +72,7 @@ using namespace std::literals; using android::base::boot_clock; using android::base::GetBoolProperty; +using android::base::GetUintProperty; using android::base::SetProperty; using android::base::Split; using android::base::Timer; @@ -732,6 +733,12 @@ static Result<void> UnmountAllApexes() { return Error() << "'/system/bin/apexd --unmount-all' failed : " << status; } +static std::chrono::milliseconds GetMillisProperty(const std::string& name, + std::chrono::milliseconds default_value) { + auto value = GetUintProperty(name, static_cast<uint64_t>(default_value.count())); + return std::chrono::milliseconds(std::move(value)); +} + static Result<void> DoUserspaceReboot() { LOG(INFO) << "Userspace reboot initiated"; auto guard = android::base::make_scope_guard([] { @@ -769,10 +776,13 @@ static Result<void> DoUserspaceReboot() { sync(); LOG(INFO) << "sync() took " << sync_timer; } - // TODO(b/135984674): do we need shutdown animation for userspace reboot? - // TODO(b/135984674): control userspace timeout via read-only property? - StopServicesAndLogViolations(stop_first, 10s, true /* SIGTERM */); - if (int r = StopServicesAndLogViolations(stop_first, 20s, false /* SIGKILL */); r > 0) { + auto sigterm_timeout = GetMillisProperty("init.userspace_reboot.sigterm.timeoutmillis", 5s); + auto sigkill_timeout = GetMillisProperty("init.userspace_reboot.sigkill.timeoutmillis", 10s); + LOG(INFO) << "Timeout to terminate services : " << sigterm_timeout.count() << "ms" + << "Timeout to kill services: " << sigkill_timeout.count() << "ms"; + StopServicesAndLogViolations(stop_first, sigterm_timeout, true /* SIGTERM */); + if (int r = StopServicesAndLogViolations(stop_first, sigkill_timeout, false /* SIGKILL */); + r > 0) { // TODO(b/135984674): store information about offending services for debugging. return Error() << r << " post-data services are still running"; } @@ -782,8 +792,8 @@ static Result<void> DoUserspaceReboot() { if (auto result = CallVdc("volume", "reset"); !result.ok()) { return result; } - if (int r = StopServicesAndLogViolations(GetDebuggingServices(true /* only_post_data */), 5s, - false /* SIGKILL */); + if (int r = StopServicesAndLogViolations(GetDebuggingServices(true /* only_post_data */), + sigkill_timeout, false /* SIGKILL */); r > 0) { // TODO(b/135984674): store information about offending services for debugging. return Error() << r << " debugging services are still running"; @@ -827,8 +837,8 @@ static void UserspaceRebootWatchdogThread() { return; } LOG(INFO) << "Starting userspace reboot watchdog"; - // TODO(b/135984674): this should be configured via a read-only sysprop. - std::chrono::milliseconds timeout = 60s; + auto timeout = GetMillisProperty("init.userspace_reboot.watchdog.timeoutmillis", 5min); + LOG(INFO) << "UserspaceRebootWatchdog timeout: " << timeout.count() << "ms"; if (!WaitForProperty("sys.boot_completed", "1", timeout)) { LOG(ERROR) << "Failed to boot in " << timeout.count() << "ms. Switching to full reboot"; // In this case device is in a boot loop. Only way to recover is to do dirty reboot. diff --git a/init/selinux.cpp b/init/selinux.cpp index acbcbd647..5a0255acd 100644 --- a/init/selinux.cpp +++ b/init/selinux.cpp @@ -63,12 +63,15 @@ #include <android-base/file.h> #include <android-base/logging.h> #include <android-base/parseint.h> +#include <android-base/strings.h> #include <android-base/unique_fd.h> #include <fs_avb/fs_avb.h> +#include <fs_mgr.h> #include <libgsi/libgsi.h> #include <libsnapshot/snapshot.h> #include <selinux/android.h> +#include "block_dev_initializer.h" #include "debug_ramdisk.h" #include "reboot_utils.h" #include "util.h" @@ -539,9 +542,9 @@ void SelinuxRestoreContext() { // adb remount, snapshot-based updates, and DSUs all create files during // first-stage init. - selinux_android_restorecon("/metadata", SELINUX_ANDROID_RESTORECON_RECURSE); - selinux_android_restorecon(SnapshotManager::GetGlobalRollbackIndicatorPath().c_str(), 0); + selinux_android_restorecon("/metadata/gsi", SELINUX_ANDROID_RESTORECON_RECURSE | + SELINUX_ANDROID_RESTORECON_SKIP_SEHASH); } int SelinuxKlogCallback(int type, const char* fmt, ...) { @@ -598,6 +601,74 @@ int SelinuxGetVendorAndroidVersion() { return vendor_android_version; } +// This is for R system.img/system_ext.img to work on old vendor.img as system_ext.img +// is introduced in R. We mount system_ext in second stage init because the first-stage +// init in boot.img won't be updated in the system-only OTA scenario. +void MountMissingSystemPartitions() { + android::fs_mgr::Fstab fstab; + if (!ReadDefaultFstab(&fstab)) { + LOG(ERROR) << "Could not read default fstab"; + } + + android::fs_mgr::Fstab mounts; + if (!ReadFstabFromFile("/proc/mounts", &mounts)) { + LOG(ERROR) << "Could not read /proc/mounts"; + } + + static const std::vector<std::string> kPartitionNames = {"system_ext", "product"}; + + android::fs_mgr::Fstab extra_fstab; + for (const auto& name : kPartitionNames) { + if (GetEntryForMountPoint(&mounts, "/"s + name)) { + // The partition is already mounted. + continue; + } + + auto system_entry = GetEntryForMountPoint(&fstab, "/system"); + if (!system_entry) { + LOG(ERROR) << "Could not find mount entry for /system"; + break; + } + if (!system_entry->fs_mgr_flags.logical) { + LOG(INFO) << "Skipping mount of " << name << ", system is not dynamic."; + break; + } + + auto entry = *system_entry; + auto partition_name = name + fs_mgr_get_slot_suffix(); + auto replace_name = "system"s + fs_mgr_get_slot_suffix(); + + entry.mount_point = "/"s + name; + entry.blk_device = + android::base::StringReplace(entry.blk_device, replace_name, partition_name, false); + if (!fs_mgr_update_logical_partition(&entry)) { + LOG(ERROR) << "Could not update logical partition"; + continue; + } + + extra_fstab.emplace_back(std::move(entry)); + } + + SkipMountingPartitions(&extra_fstab); + if (extra_fstab.empty()) { + return; + } + + BlockDevInitializer block_dev_init; + for (auto& entry : extra_fstab) { + if (access(entry.blk_device.c_str(), F_OK) != 0) { + auto block_dev = android::base::Basename(entry.blk_device); + if (!block_dev_init.InitDmDevice(block_dev)) { + LOG(ERROR) << "Failed to find device-mapper node: " << block_dev; + continue; + } + } + if (fs_mgr_do_mount_one(entry)) { + LOG(ERROR) << "Could not mount " << entry.mount_point; + } + } +} + int SetupSelinux(char** argv) { SetStdioToDevNull(argv); InitKernelLogging(argv); @@ -608,6 +679,8 @@ int SetupSelinux(char** argv) { boot_clock::time_point start_time = boot_clock::now(); + MountMissingSystemPartitions(); + // Set up SELinux, loading the SELinux policy. SelinuxSetupKernelLogging(); SelinuxInitialize(); diff --git a/init/service.h b/init/service.h index cf3f0c290..9f1d697f0 100644 --- a/init/service.h +++ b/init/service.h @@ -130,6 +130,13 @@ class Service { bool is_updatable() const { return updatable_; } bool is_post_data() const { return post_data_; } bool is_from_apex() const { return from_apex_; } + void set_oneshot(bool value) { + if (value) { + flags_ |= SVC_ONESHOT; + } else { + flags_ &= ~SVC_ONESHOT; + } + } private: void NotifyStateChange(const std::string& new_state) const; diff --git a/init/test_kill_services/Android.bp b/init/test_kill_services/Android.bp new file mode 100644 index 000000000..f6e85e289 --- /dev/null +++ b/init/test_kill_services/Android.bp @@ -0,0 +1,7 @@ +cc_test { + name: "init_kill_services_test", + srcs: ["init_kill_services_test.cpp"], + shared_libs: ["libbase"], + test_suites: ["general-tests"], + require_root: true, +} diff --git a/init/test_kill_services/init_kill_services_test.cpp b/init/test_kill_services/init_kill_services_test.cpp new file mode 100644 index 000000000..7e543f232 --- /dev/null +++ b/init/test_kill_services/init_kill_services_test.cpp @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <gtest/gtest.h> + +#include <android-base/properties.h> + +#include <iostream> + +using ::android::base::GetProperty; +using ::android::base::SetProperty; + +void ExpectKillingServiceRecovers(const std::string& service_name) { + const std::string status_prop = "init.svc." + service_name; + const std::string pid_prop = "init.svc_debug_pid." + service_name; + + const std::string initial_pid = GetProperty(pid_prop, ""); + + EXPECT_EQ("running", GetProperty(status_prop, "")) << status_prop; + EXPECT_NE("", initial_pid) << pid_prop; + + EXPECT_EQ(0, system(("kill -9 " + initial_pid).c_str())); + + constexpr size_t kMaxWaitMilliseconds = 10000; + constexpr size_t kRetryWaitMilliseconds = 100; + + constexpr size_t kRetryTimes = kMaxWaitMilliseconds / kRetryWaitMilliseconds; + + for (size_t retry = 0; retry < kRetryTimes; retry++) { + const std::string& pid = GetProperty(pid_prop, ""); + if (pid != initial_pid && pid != "") break; + usleep(kRetryWaitMilliseconds * 1000); + } + + // svc_debug_pid is set after svc property + EXPECT_EQ("running", GetProperty(status_prop, "")); +} + +class InitKillServicesTest : public ::testing::TestWithParam<std::string> {}; + +TEST_P(InitKillServicesTest, KillCriticalProcesses) { + ExpectKillingServiceRecovers(GetParam()); + + // sanity check init is still responding + EXPECT_TRUE(SetProperty("test.death.test", "asdf")); + EXPECT_EQ(GetProperty("test.death.test", ""), "asdf"); + EXPECT_TRUE(SetProperty("test.death.test", "")); +} + +static inline std::string PrintName(const testing::TestParamInfo<std::string>& info) { + return info.param; +} + +INSTANTIATE_TEST_CASE_P(DeathTest, InitKillServicesTest, + ::testing::Values("lmkd", "ueventd", "hwservicemanager", "servicemanager"), + PrintName); diff --git a/libasyncio/Android.bp b/libasyncio/Android.bp index 4ab439d57..44e7933c3 100644 --- a/libasyncio/Android.bp +++ b/libasyncio/Android.bp @@ -28,6 +28,10 @@ cc_library { defaults: ["libasyncio_defaults"], vendor_available: true, recovery_available: true, + apex_available: [ + "//apex_available:platform", + "com.android.adbd", + ], host_supported: true, srcs: [ "AsyncIO.cpp", diff --git a/libbacktrace/Android.bp b/libbacktrace/Android.bp index 76caadcca..dc989a0a8 100644 --- a/libbacktrace/Android.bp +++ b/libbacktrace/Android.bp @@ -50,20 +50,9 @@ cc_library_headers { ], } -cc_library { - name: "libbacktrace", - vendor_available: false, - recovery_available: true, - apex_available: [ - "//apex_available:platform", - "//apex_available:anyapex", - ], - vndk: { - enabled: true, - support_system_process: true, - }, +cc_defaults { + name: "libbacktrace_defaults", defaults: ["libbacktrace_common"], - host_supported: true, cflags: [ "-Wexit-time-destructors", @@ -88,7 +77,6 @@ cc_library { shared_libs: [ "libbase", "liblog", - "libunwindstack", ], static_libs: [ @@ -101,6 +89,30 @@ cc_library { whole_static_libs: ["libasync_safe"], }, }, + }, +} + +cc_library { + name: "libbacktrace", + vendor_available: false, + recovery_available: true, + apex_available: [ + "//apex_available:platform", + "//apex_available:anyapex", + ], + vndk: { + enabled: true, + support_system_process: true, + }, + host_supported: true, + defaults: ["libbacktrace_defaults"], + + target: { + linux: { + shared_libs: [ + "libunwindstack", + ], + }, vendor: { cflags: ["-DNO_LIBDEXFILE_SUPPORT"], }, @@ -110,6 +122,21 @@ cc_library { }, } +// Static library without DEX support to avoid dependencies on the ART APEX. +cc_library_static { + name: "libbacktrace_no_dex", + visibility: ["//system/core/debuggerd"], + defaults: ["libbacktrace_defaults"], + cflags: ["-DNO_LIBDEXFILE_SUPPORT"], + target: { + linux: { + static_libs: [ + "libunwindstack_no_dex", + ], + }, + }, +} + cc_test_library { name: "libbacktrace_test", defaults: ["libbacktrace_common"], diff --git a/liblog/README.md b/liblog/README.md index 871399a57..f64f37624 100644 --- a/liblog/README.md +++ b/liblog/README.md @@ -118,10 +118,9 @@ at a time in time sorted order, optionally limited to a specific pid and tail of finally a call closing the logs. A single log can be opened with `android_logger_list_open()`; or multiple logs can be opened with `android_logger_list_alloc()`, calling in turn the `android_logger_open()` for each log id. Each entry can be retrieved with -`android_logger_list_read()`. The log(s) can be closed with `android_logger_list_free()`. The logs -should be opened with an `ANDROID_LOG_RDONLY` mode. `ANDROID_LOG_NONBLOCK` mode will report when -the log reading is done with an `EAGAIN` error return code, otherwise the -`android_logger_list_read()` call will block for new entries. +`android_logger_list_read()`. The log(s) can be closed with `android_logger_list_free()`. +`ANDROID_LOG_NONBLOCK` mode will report when the log reading is done with an `EAGAIN` error return +code, otherwise the `android_logger_list_read()` call will block for new entries. The `ANDROID_LOG_WRAP` mode flag to the `android_logger_list_alloc_time()` signals logd to quiesce the reader until the buffer is about to prune at the start time then proceed to dumping content. @@ -130,14 +129,12 @@ The `ANDROID_LOG_PSTORE` mode flag to the `android_logger_open()` is used to swi logs to the persistent logs from before the last reboot. The value returned by `android_logger_open()` can be used as a parameter to the -`android_logger_clear()` function to empty the sub-log. It is recommended to only open log -`ANDROID_LOG_WRONLY` in that case. +`android_logger_clear()` function to empty the sub-log. The value returned by `android_logger_open()` can be used as a parameter to the `android_logger_get_log_(size|readable_size|version)` to retrieve the sub-log maximum size, readable size and log buffer format protocol version respectively. `android_logger_get_id()` returns the id -that was used when opening the sub-log. It is recommended to open the log `ANDROID_LOG_RDONLY` in -these cases. +that was used when opening the sub-log. Errors ------ diff --git a/liblog/include/android/log.h b/liblog/include/android/log.h index 43a91ab5b..512c7cd7b 100644 --- a/liblog/include/android/log.h +++ b/liblog/include/android/log.h @@ -184,20 +184,21 @@ int __android_log_buf_print(int bufID, int prio, const char* tag, const char* fm * Logger data struct used for writing log messages to liblog via __android_log_write_logger_data() * and sending log messages to user defined loggers specified in __android_log_set_logger(). */ -struct __android_logger_data { - size_t struct_size; /* Must be set to sizeof(__android_logger_data) and is used for versioning. */ - int32_t buffer_id; /* log_id_t or -1 to represent 'default'. */ - int32_t priority; /* android_LogPriority values. */ - const char* tag; - const char* file; /* Optional file name, may be set to nullptr. */ - uint32_t line; /* Optional line number, ignore if file is nullptr. */ +struct __android_log_message { + size_t + struct_size; /** Must be set to sizeof(__android_log_message) and is used for versioning. */ + int32_t buffer_id; /** {@link log_id_t} values. */ + int32_t priority; /** {@link android_LogPriority} values. */ + const char* tag; /** The tag for the log message. */ + const char* file; /** Optional file name, may be set to nullptr. */ + uint32_t line; /** Optional line number, ignore if file is nullptr. */ + const char* message; /** The log message itself. */ }; /** * Prototype for the 'logger' function that is called for every log message. */ -typedef void (*__android_logger_function)(const struct __android_logger_data* logger_data, - const char* message); +typedef void (*__android_logger_function)(const struct __android_log_message* log_message); /** * Prototype for the 'abort' function that is called when liblog will abort due to * __android_log_assert() failures. @@ -206,52 +207,85 @@ typedef void (*__android_aborter_function)(const char* abort_message); #if !defined(__ANDROID__) || __ANDROID_API__ >= 30 /** - * Writes the log message specified with logger_data and msg to the log. logger_data includes - * additional file name and line number information that a logger may use. logger_data is versioned - * for backwards compatibility. + * Writes the log message specified by log_message. log_message includes additional file name and + * line number information that a logger may use. log_message is versioned for backwards + * compatibility. * This assumes that loggability has already been checked through __android_log_is_loggable(). * Higher level logging libraries, such as libbase, first check loggability, then format their * buffers, then pass the message to liblog via this function, and therefore we do not want to * duplicate the loggability check here. + * + * @param log_message the log message itself, see {@link __android_log_message}. + * + * Available since API level 30. */ -void __android_log_write_logger_data(struct __android_logger_data* logger_data, const char* msg) - __INTRODUCED_IN(30); +void __android_log_write_log_message(struct __android_log_message* log_message) __INTRODUCED_IN(30); /** * Sets a user defined logger function. All log messages sent to liblog will be set to the - * function pointer specified by logger for processing. + * function pointer specified by logger for processing. It is not expected that log messages are + * already terminated with a new line. This function should add new lines if required for line + * separation. + * + * @param logger the new function that will handle log messages. + * + * Available since API level 30. */ void __android_log_set_logger(__android_logger_function logger) __INTRODUCED_IN(30); /** * Writes the log message to logd. This is an __android_logger_function and can be provided to * __android_log_set_logger(). It is the default logger when running liblog on a device. + * + * @param log_message the log message to write, see {@link __android_log_message}. + * + * Available since API level 30. */ -void __android_log_logd_logger(const struct __android_logger_data* logger_data, const char* msg) - __INTRODUCED_IN(30); +void __android_log_logd_logger(const struct __android_log_message* log_message) __INTRODUCED_IN(30); /** * Writes the log message to stderr. This is an __android_logger_function and can be provided to * __android_log_set_logger(). It is the default logger when running liblog on host. + * + * @param log_message the log message to write, see {@link __android_log_message}. + * + * Available since API level 30. */ -void __android_log_stderr_logger(const struct __android_logger_data* logger_data, - const char* message) __INTRODUCED_IN(30); +void __android_log_stderr_logger(const struct __android_log_message* log_message) + __INTRODUCED_IN(30); /** - * Sets a user defined aborter function that is called for __android_log_assert() failures. + * Sets a user defined aborter function that is called for __android_log_assert() failures. This + * user defined aborter function is highly recommended to abort and be noreturn, but is not strictly + * required to. + * + * @param aborter the new aborter function, see {@link __android_aborter_function}. + * + * Available since API level 30. */ void __android_log_set_aborter(__android_aborter_function aborter) __INTRODUCED_IN(30); /** * Calls the stored aborter function. This allows for other logging libraries to use the same * aborter function by calling this function in liblog. + * + * @param abort_message an additional message supplied when aborting, for example this is used to + * call android_set_abort_message() in __android_log_default_aborter(). + * + * Available since API level 30. */ void __android_log_call_aborter(const char* abort_message) __INTRODUCED_IN(30); /** * Sets android_set_abort_message() on device then aborts(). This is the default aborter. + * + * @param abort_message an additional message supplied when aborting. This functions calls + * android_set_abort_message() with its contents. + * + * Available since API level 30. */ -void __android_log_default_aborter(const char* abort_message) __INTRODUCED_IN(30); +void __android_log_default_aborter(const char* abort_message) __attribute__((noreturn)) +__INTRODUCED_IN(30); /** * Use the per-tag properties "log.tag.<tagname>" along with the minimum priority from @@ -263,7 +297,13 @@ void __android_log_default_aborter(const char* abort_message) __INTRODUCED_IN(30 * minimum priority needed to log. If only one is set, then that value is used to determine the * minimum priority needed. If none are set, then default_priority is used. * - * prio is ANDROID_LOG_VERBOSE to ANDROID_LOG_FATAL. + * @param prio the priority to test, takes {@link android_LogPriority} values. + * @param tag the tag to test. + * @param len the length of the tag. + * @param default_prio the default priority to use if no properties or minimum priority are set. + * @return an integer where 1 indicates that the message is loggable and 0 indicates that it is not. + * + * Available since API level 30. */ int __android_log_is_loggable(int prio, const char* tag, int default_prio) __INTRODUCED_IN(30); int __android_log_is_loggable_len(int prio, const char* tag, size_t len, int default_prio) @@ -272,13 +312,22 @@ int __android_log_is_loggable_len(int prio, const char* tag, size_t len, int def /** * Sets the minimum priority that will be logged for this process. * - * This returns the previous set minimum priority, or ANDROID_LOG_DEFAULT if none was set. + * @param priority the new minimum priority to set, takes @{link android_LogPriority} values. + * @return the previous set minimum priority as @{link android_LogPriority} values, or + * ANDROID_LOG_DEFAULT if none was set. + * + * Available since API level 30. */ int32_t __android_log_set_minimum_priority(int32_t priority) __INTRODUCED_IN(30); /** * Gets the minimum priority that will be logged for this process. If none has been set by a * previous __android_log_set_minimum_priority() call, this returns ANDROID_LOG_DEFAULT. + * + * @return the current minimum priority as @{link android_LogPriority} values, or + * ANDROID_LOG_DEFAULT if none is set. + * + * Available since API level 30. */ int32_t __android_log_get_minimum_priority(void) __INTRODUCED_IN(30); @@ -286,6 +335,10 @@ int32_t __android_log_get_minimum_priority(void) __INTRODUCED_IN(30); * Sets the default tag if no tag is provided when writing a log message. Defaults to * getprogname(). This truncates tag to the maximum log message size, though appropriate tags * should be much smaller. + * + * @param tag the new log tag. + * + * Available since API level 30. */ void __android_log_set_default_tag(const char* tag) __INTRODUCED_IN(30); #endif diff --git a/liblog/include/log/log_read.h b/liblog/include/log/log_read.h index 18c1c33e9..05ad25f5e 100644 --- a/liblog/include/log/log_read.h +++ b/liblog/include/log/log_read.h @@ -141,10 +141,6 @@ ssize_t android_logger_get_prune_list(struct logger_list* logger_list, char* buf, size_t len); int android_logger_set_prune_list(struct logger_list* logger_list, const char* buf, size_t len); -#define ANDROID_LOG_RDONLY O_RDONLY -#define ANDROID_LOG_WRONLY O_WRONLY -#define ANDROID_LOG_RDWR O_RDWR -#define ANDROID_LOG_ACCMODE O_ACCMODE #ifndef O_NONBLOCK #define ANDROID_LOG_NONBLOCK 0x00000800 #else diff --git a/liblog/liblog.map.txt b/liblog/liblog.map.txt index 198cdae21..161fcf1a9 100644 --- a/liblog/liblog.map.txt +++ b/liblog/liblog.map.txt @@ -1,6 +1,6 @@ LIBLOG { global: - android_name_to_log_id; # llndk + android_name_to_log_id; # apex llndk android_log_id_to_name; # llndk __android_log_assert; __android_log_buf_print; @@ -22,7 +22,7 @@ LIBLOG_L { android_logger_list_alloc; # apex llndk android_logger_list_alloc_time; # apex llndk android_logger_list_free; # apex llndk - android_logger_list_open; # llndk + android_logger_list_open; # apex llndk android_logger_list_read; # apex llndk android_logger_open; # apex llndk android_logger_set_log_size; # llndk @@ -65,7 +65,7 @@ LIBLOG_Q { # introduced=29 android_log_parser_reset; # llndk }; -LIGLOG_R { # introduced=30 +LIBLOG_R { # introduced=30 global: __android_log_call_aborter; __android_log_default_aborter; @@ -77,7 +77,7 @@ LIGLOG_R { # introduced=30 __android_log_set_logger; __android_log_set_minimum_priority; __android_log_stderr_logger; - __android_log_write_logger_data; + __android_log_write_log_message; }; LIBLOG_PRIVATE { diff --git a/liblog/logger_write.cpp b/liblog/logger_write.cpp index a8620a06c..c174b8511 100644 --- a/liblog/logger_write.cpp +++ b/liblog/logger_write.cpp @@ -250,8 +250,7 @@ static uint64_t GetThreadId() { #endif } -void __android_log_stderr_logger(const struct __android_logger_data* logger_data, - const char* message) { +void __android_log_stderr_logger(const struct __android_log_message* log_message) { struct tm now; time_t t = time(nullptr); @@ -268,33 +267,32 @@ void __android_log_stderr_logger(const struct __android_logger_data* logger_data static_assert(arraysize(log_characters) - 1 == ANDROID_LOG_SILENT, "Mismatch in size of log_characters and values in android_LogPriority"); int32_t priority = - logger_data->priority > ANDROID_LOG_SILENT ? ANDROID_LOG_FATAL : logger_data->priority; + log_message->priority > ANDROID_LOG_SILENT ? ANDROID_LOG_FATAL : log_message->priority; char priority_char = log_characters[priority]; uint64_t tid = GetThreadId(); - if (logger_data->file != nullptr) { + if (log_message->file != nullptr) { fprintf(stderr, "%s %c %s %5d %5" PRIu64 " %s:%u] %s\n", - logger_data->tag ? logger_data->tag : "nullptr", priority_char, timestamp, getpid(), - tid, logger_data->file, logger_data->line, message); + log_message->tag ? log_message->tag : "nullptr", priority_char, timestamp, getpid(), + tid, log_message->file, log_message->line, log_message->message); } else { fprintf(stderr, "%s %c %s %5d %5" PRIu64 " %s\n", - logger_data->tag ? logger_data->tag : "nullptr", priority_char, timestamp, getpid(), - tid, message); + log_message->tag ? log_message->tag : "nullptr", priority_char, timestamp, getpid(), + tid, log_message->message); } } -void __android_log_logd_logger(const struct __android_logger_data* logger_data, - const char* message) { - int buffer_id = logger_data->buffer_id == LOG_ID_DEFAULT ? LOG_ID_MAIN : logger_data->buffer_id; +void __android_log_logd_logger(const struct __android_log_message* log_message) { + int buffer_id = log_message->buffer_id == LOG_ID_DEFAULT ? LOG_ID_MAIN : log_message->buffer_id; struct iovec vec[3]; vec[0].iov_base = - const_cast<unsigned char*>(reinterpret_cast<const unsigned char*>(&logger_data->priority)); + const_cast<unsigned char*>(reinterpret_cast<const unsigned char*>(&log_message->priority)); vec[0].iov_len = 1; - vec[1].iov_base = const_cast<void*>(static_cast<const void*>(logger_data->tag)); - vec[1].iov_len = strlen(logger_data->tag) + 1; - vec[2].iov_base = const_cast<void*>(static_cast<const void*>(message)); - vec[2].iov_len = strlen(message) + 1; + vec[1].iov_base = const_cast<void*>(static_cast<const void*>(log_message->tag)); + vec[1].iov_len = strlen(log_message->tag) + 1; + vec[2].iov_base = const_cast<void*>(static_cast<const void*>(log_message->message)); + vec[2].iov_len = strlen(log_message->message) + 1; write_to_log(static_cast<log_id_t>(buffer_id), vec, 3); } @@ -303,29 +301,29 @@ int __android_log_write(int prio, const char* tag, const char* msg) { return __android_log_buf_write(LOG_ID_MAIN, prio, tag, msg); } -void __android_log_write_logger_data(__android_logger_data* logger_data, const char* msg) { +void __android_log_write_log_message(__android_log_message* log_message) { ErrnoRestorer errno_restorer; - if (logger_data->buffer_id != LOG_ID_DEFAULT && logger_data->buffer_id != LOG_ID_MAIN && - logger_data->buffer_id != LOG_ID_SYSTEM && logger_data->buffer_id != LOG_ID_RADIO && - logger_data->buffer_id != LOG_ID_CRASH) { + if (log_message->buffer_id != LOG_ID_DEFAULT && log_message->buffer_id != LOG_ID_MAIN && + log_message->buffer_id != LOG_ID_SYSTEM && log_message->buffer_id != LOG_ID_RADIO && + log_message->buffer_id != LOG_ID_CRASH) { return; } auto tag_lock = std::shared_lock{default_tag_lock, std::defer_lock}; - if (logger_data->tag == nullptr) { + if (log_message->tag == nullptr) { tag_lock.lock(); - logger_data->tag = GetDefaultTag().c_str(); + log_message->tag = GetDefaultTag().c_str(); } #if __BIONIC__ - if (logger_data->priority == ANDROID_LOG_FATAL) { - android_set_abort_message(msg); + if (log_message->priority == ANDROID_LOG_FATAL) { + android_set_abort_message(log_message->message); } #endif auto lock = std::shared_lock{logger_function_lock}; - logger_function(logger_data, msg); + logger_function(log_message); } int __android_log_buf_write(int bufID, int prio, const char* tag, const char* msg) { @@ -335,8 +333,9 @@ int __android_log_buf_write(int bufID, int prio, const char* tag, const char* ms return 0; } - __android_logger_data logger_data = {sizeof(__android_logger_data), bufID, prio, tag, nullptr, 0}; - __android_log_write_logger_data(&logger_data, msg); + __android_log_message log_message = { + sizeof(__android_log_message), bufID, prio, tag, nullptr, 0, msg}; + __android_log_write_log_message(&log_message); return 1; } @@ -351,9 +350,9 @@ int __android_log_vprint(int prio, const char* tag, const char* fmt, va_list ap) vsnprintf(buf, LOG_BUF_SIZE, fmt, ap); - __android_logger_data logger_data = { - sizeof(__android_logger_data), LOG_ID_MAIN, prio, tag, nullptr, 0}; - __android_log_write_logger_data(&logger_data, buf); + __android_log_message log_message = { + sizeof(__android_log_message), LOG_ID_MAIN, prio, tag, nullptr, 0, buf}; + __android_log_write_log_message(&log_message); return 1; } @@ -371,9 +370,9 @@ int __android_log_print(int prio, const char* tag, const char* fmt, ...) { vsnprintf(buf, LOG_BUF_SIZE, fmt, ap); va_end(ap); - __android_logger_data logger_data = { - sizeof(__android_logger_data), LOG_ID_MAIN, prio, tag, nullptr, 0}; - __android_log_write_logger_data(&logger_data, buf); + __android_log_message log_message = { + sizeof(__android_log_message), LOG_ID_MAIN, prio, tag, nullptr, 0, buf}; + __android_log_write_log_message(&log_message); return 1; } @@ -391,8 +390,9 @@ int __android_log_buf_print(int bufID, int prio, const char* tag, const char* fm vsnprintf(buf, LOG_BUF_SIZE, fmt, ap); va_end(ap); - __android_logger_data logger_data = {sizeof(__android_logger_data), bufID, prio, tag, nullptr, 0}; - __android_log_write_logger_data(&logger_data, buf); + __android_log_message log_message = { + sizeof(__android_log_message), bufID, prio, tag, nullptr, 0, buf}; + __android_log_write_log_message(&log_message); return 1; } diff --git a/liblog/pmsg_reader.cpp b/liblog/pmsg_reader.cpp index 64a92b7ce..129d76721 100644 --- a/liblog/pmsg_reader.cpp +++ b/liblog/pmsg_reader.cpp @@ -185,7 +185,7 @@ ssize_t __android_log_pmsg_file_read(log_id_t logId, char prio, const char* pref /* Add just enough clues in logger_list and transp to make API function */ memset(&logger_list, 0, sizeof(logger_list)); - logger_list.mode = ANDROID_LOG_PSTORE | ANDROID_LOG_NONBLOCK | ANDROID_LOG_RDONLY; + logger_list.mode = ANDROID_LOG_PSTORE | ANDROID_LOG_NONBLOCK; logger_list.log_mask = (unsigned)-1; if (logId != LOG_ID_ANY) { logger_list.log_mask = (1 << logId); diff --git a/liblog/tests/Android.bp b/liblog/tests/Android.bp index b4bb77f66..fffb809a7 100644 --- a/liblog/tests/Android.bp +++ b/liblog/tests/Android.bp @@ -97,6 +97,7 @@ cc_test { test_suites: [ "cts", "vts", + "vts10", ], } diff --git a/liblog/tests/libc_test.cpp b/liblog/tests/libc_test.cpp index 3534eb8d3..1f2626390 100644 --- a/liblog/tests/libc_test.cpp +++ b/liblog/tests/libc_test.cpp @@ -22,6 +22,10 @@ TEST(libc, __pstore_append) { #ifdef __ANDROID__ #ifndef NO_PSTORE + if (access("/dev/pmsg0", W_OK) != 0) { + GTEST_SKIP() << "pmsg0 not found, skipping test"; + } + FILE* fp; ASSERT_TRUE(NULL != (fp = fopen("/dev/pmsg0", "ae"))); static const char message[] = "libc.__pstore_append\n"; diff --git a/liblog/tests/liblog_benchmark.cpp b/liblog/tests/liblog_benchmark.cpp index 4366f3d3b..3a6ed90d4 100644 --- a/liblog/tests/liblog_benchmark.cpp +++ b/liblog/tests/liblog_benchmark.cpp @@ -648,8 +648,7 @@ static const int alarm_time = 3; static void BM_log_latency(benchmark::State& state) { pid_t pid = getpid(); - struct logger_list* logger_list = - android_logger_list_open(LOG_ID_EVENTS, ANDROID_LOG_RDONLY, 0, pid); + struct logger_list* logger_list = android_logger_list_open(LOG_ID_EVENTS, 0, 0, pid); if (!logger_list) { fprintf(stderr, "Unable to open events log: %s\n", strerror(errno)); @@ -723,8 +722,7 @@ static void caught_delay(int /*signum*/) { static void BM_log_delay(benchmark::State& state) { pid_t pid = getpid(); - struct logger_list* logger_list = - android_logger_list_open(LOG_ID_EVENTS, ANDROID_LOG_RDONLY, 0, pid); + struct logger_list* logger_list = android_logger_list_open(LOG_ID_EVENTS, 0, 0, pid); if (!logger_list) { fprintf(stderr, "Unable to open events log: %s\n", strerror(errno)); diff --git a/liblog/tests/liblog_global_state.cpp b/liblog/tests/liblog_global_state.cpp index 9a181ef62..3508818f2 100644 --- a/liblog/tests/liblog_global_state.cpp +++ b/liblog/tests/liblog_global_state.cpp @@ -59,16 +59,15 @@ TEST(liblog_global_state, libbase_logs_with_liblog_set_logger) { static unsigned int expected_line; static std::string expected_message = "libbase test message"; - auto liblog_logger_function = [](const struct __android_logger_data* logger_data, - const char* message) { + auto liblog_logger_function = [](const struct __android_log_message* log_message) { message_seen = true; - EXPECT_EQ(sizeof(__android_logger_data), logger_data->struct_size); - EXPECT_EQ(LOG_ID_DEFAULT, logger_data->buffer_id); - EXPECT_EQ(ANDROID_LOG_WARN, logger_data->priority); - EXPECT_STREQ(LOG_TAG, logger_data->tag); - EXPECT_EQ(expected_file, logger_data->file); - EXPECT_EQ(expected_line, logger_data->line); - EXPECT_EQ(expected_message, message); + EXPECT_EQ(sizeof(__android_log_message), log_message->struct_size); + EXPECT_EQ(LOG_ID_DEFAULT, log_message->buffer_id); + EXPECT_EQ(ANDROID_LOG_WARN, log_message->priority); + EXPECT_STREQ(LOG_TAG, log_message->tag); + EXPECT_EQ(expected_file, log_message->file); + EXPECT_EQ(expected_line, log_message->line); + EXPECT_EQ(expected_message, log_message->message); }; __android_log_set_logger(liblog_logger_function); @@ -111,16 +110,15 @@ TEST(liblog_global_state, liblog_logs_with_liblog_set_logger) { static int expected_priority = ANDROID_LOG_WARN; static std::string expected_message = "libbase test message"; - auto liblog_logger_function = [](const struct __android_logger_data* logger_data, - const char* message) { + auto liblog_logger_function = [](const struct __android_log_message* log_message) { message_seen = true; - EXPECT_EQ(sizeof(__android_logger_data), logger_data->struct_size); - EXPECT_EQ(expected_buffer_id, logger_data->buffer_id); - EXPECT_EQ(expected_priority, logger_data->priority); - EXPECT_STREQ(LOG_TAG, logger_data->tag); - EXPECT_STREQ(nullptr, logger_data->file); - EXPECT_EQ(0U, logger_data->line); - EXPECT_EQ(expected_message, message); + EXPECT_EQ(sizeof(__android_log_message), log_message->struct_size); + EXPECT_EQ(expected_buffer_id, log_message->buffer_id); + EXPECT_EQ(expected_priority, log_message->priority); + EXPECT_STREQ(LOG_TAG, log_message->tag); + EXPECT_STREQ(nullptr, log_message->file); + EXPECT_EQ(0U, log_message->line); + EXPECT_EQ(expected_message, log_message->message); }; __android_log_set_logger(liblog_logger_function); diff --git a/liblog/tests/liblog_test.cpp b/liblog/tests/liblog_test.cpp index a60d2df31..a031531cb 100644 --- a/liblog/tests/liblog_test.cpp +++ b/liblog/tests/liblog_test.cpp @@ -82,7 +82,7 @@ static void RunLogTests(log_id_t log_buffer, FWrite write_messages, FCheck check pid_t pid = getpid(); auto logger_list = std::unique_ptr<struct logger_list, ListCloser>{ - android_logger_list_open(log_buffer, ANDROID_LOG_RDONLY, 1000, pid)}; + android_logger_list_open(log_buffer, 0, 1000, pid)}; ASSERT_TRUE(logger_list); write_messages(); @@ -106,7 +106,7 @@ static void RunLogTests(log_id_t log_buffer, FWrite write_messages, FCheck check } auto logger_list_non_block = std::unique_ptr<struct logger_list, ListCloser>{ - android_logger_list_open(log_buffer, ANDROID_LOG_RDONLY | ANDROID_LOG_NONBLOCK, 1000, pid)}; + android_logger_list_open(log_buffer, ANDROID_LOG_NONBLOCK, 1000, pid)}; ASSERT_TRUE(logger_list_non_block); size_t count = 0; @@ -160,7 +160,6 @@ static std::string popenToString(const std::string& command) { return ret; } -#ifndef NO_PSTORE static bool isPmsgActive() { pid_t pid = getpid(); @@ -170,7 +169,6 @@ static bool isPmsgActive() { return std::string::npos != myPidFds.find(" -> /dev/pmsg0"); } -#endif /* NO_PSTORE */ static bool isLogdwActive() { std::string logdwSignature = @@ -222,16 +220,18 @@ TEST(liblog, __android_log_btwrite__android_logger_list_read) { log_time ts(CLOCK_MONOTONIC); log_time ts1(ts); + bool has_pstore = access("/dev/pmsg0", W_OK) == 0; + auto write_function = [&] { EXPECT_LT(0, __android_log_btwrite(0, EVENT_TYPE_LONG, &ts, sizeof(ts))); // Check that we can close and reopen the logger bool logdwActiveAfter__android_log_btwrite; if (getuid() == AID_ROOT) { tested__android_log_close = true; -#ifndef NO_PSTORE - bool pmsgActiveAfter__android_log_btwrite = isPmsgActive(); - EXPECT_TRUE(pmsgActiveAfter__android_log_btwrite); -#endif /* NO_PSTORE */ + if (has_pstore) { + bool pmsgActiveAfter__android_log_btwrite = isPmsgActive(); + EXPECT_TRUE(pmsgActiveAfter__android_log_btwrite); + } logdwActiveAfter__android_log_btwrite = isLogdwActive(); EXPECT_TRUE(logdwActiveAfter__android_log_btwrite); } else if (!tested__android_log_close) { @@ -239,10 +239,10 @@ TEST(liblog, __android_log_btwrite__android_logger_list_read) { } __android_log_close(); if (getuid() == AID_ROOT) { -#ifndef NO_PSTORE - bool pmsgActiveAfter__android_log_close = isPmsgActive(); - EXPECT_FALSE(pmsgActiveAfter__android_log_close); -#endif /* NO_PSTORE */ + if (has_pstore) { + bool pmsgActiveAfter__android_log_close = isPmsgActive(); + EXPECT_FALSE(pmsgActiveAfter__android_log_close); + } bool logdwActiveAfter__android_log_close = isLogdwActive(); EXPECT_FALSE(logdwActiveAfter__android_log_close); } @@ -250,10 +250,10 @@ TEST(liblog, __android_log_btwrite__android_logger_list_read) { ts1 = log_time(CLOCK_MONOTONIC); EXPECT_LT(0, __android_log_btwrite(0, EVENT_TYPE_LONG, &ts1, sizeof(ts1))); if (getuid() == AID_ROOT) { -#ifndef NO_PSTORE - bool pmsgActiveAfter__android_log_btwrite = isPmsgActive(); - EXPECT_TRUE(pmsgActiveAfter__android_log_btwrite); -#endif /* NO_PSTORE */ + if (has_pstore) { + bool pmsgActiveAfter__android_log_btwrite = isPmsgActive(); + EXPECT_TRUE(pmsgActiveAfter__android_log_btwrite); + } logdwActiveAfter__android_log_btwrite = isLogdwActive(); EXPECT_TRUE(logdwActiveAfter__android_log_btwrite); } @@ -572,8 +572,7 @@ TEST(liblog, android_logger_list_read__cpu_signal) { v += pid & 0xFFFF; - ASSERT_TRUE(NULL != (logger_list = android_logger_list_open( - LOG_ID_EVENTS, ANDROID_LOG_RDONLY, 1000, pid))); + ASSERT_TRUE(NULL != (logger_list = android_logger_list_open(LOG_ID_EVENTS, 0, 1000, pid))); int count = 0; @@ -728,8 +727,7 @@ TEST(liblog, android_logger_list_read__cpu_thread) { v += pid & 0xFFFF; - ASSERT_TRUE(NULL != (logger_list = android_logger_list_open( - LOG_ID_EVENTS, ANDROID_LOG_RDONLY, 1000, pid))); + ASSERT_TRUE(NULL != (logger_list = android_logger_list_open(LOG_ID_EVENTS, 0, 1000, pid))); int count = 0; @@ -1093,11 +1091,11 @@ TEST(liblog, dual_reader) { pid_t pid = getpid(); auto logger_list1 = std::unique_ptr<struct logger_list, ListCloser>{ - android_logger_list_open(LOG_ID_MAIN, ANDROID_LOG_RDONLY, expected_count1, pid)}; + android_logger_list_open(LOG_ID_MAIN, 0, expected_count1, pid)}; ASSERT_TRUE(logger_list1); auto logger_list2 = std::unique_ptr<struct logger_list, ListCloser>{ - android_logger_list_open(LOG_ID_MAIN, ANDROID_LOG_RDONLY, expected_count2, pid)}; + android_logger_list_open(LOG_ID_MAIN, 0, expected_count2, pid)}; ASSERT_TRUE(logger_list2); for (int i = 25; i > 0; --i) { @@ -1128,14 +1126,12 @@ TEST(liblog, dual_reader) { } // Test again with the nonblocking reader. - auto logger_list_non_block1 = - std::unique_ptr<struct logger_list, ListCloser>{android_logger_list_open( - LOG_ID_MAIN, ANDROID_LOG_RDONLY | ANDROID_LOG_NONBLOCK, expected_count1, pid)}; + auto logger_list_non_block1 = std::unique_ptr<struct logger_list, ListCloser>{ + android_logger_list_open(LOG_ID_MAIN, ANDROID_LOG_NONBLOCK, expected_count1, pid)}; ASSERT_TRUE(logger_list_non_block1); - auto logger_list_non_block2 = - std::unique_ptr<struct logger_list, ListCloser>{android_logger_list_open( - LOG_ID_MAIN, ANDROID_LOG_RDONLY | ANDROID_LOG_NONBLOCK, expected_count2, pid)}; + auto logger_list_non_block2 = std::unique_ptr<struct logger_list, ListCloser>{ + android_logger_list_open(LOG_ID_MAIN, ANDROID_LOG_NONBLOCK, expected_count2, pid)}; ASSERT_TRUE(logger_list_non_block2); count1 = 0; count2 = 0; @@ -1542,8 +1538,8 @@ static int count_matching_ts(log_time ts) { pid_t pid = getpid(); - struct logger_list* logger_list = android_logger_list_open( - LOG_ID_EVENTS, ANDROID_LOG_RDONLY | ANDROID_LOG_NONBLOCK, 1000, pid); + struct logger_list* logger_list = + android_logger_list_open(LOG_ID_EVENTS, ANDROID_LOG_NONBLOCK, 1000, pid); int count = 0; if (logger_list == NULL) return count; @@ -1832,10 +1828,8 @@ TEST(liblog, __security_buffer) { gid = getgid(); pid_t pid = getpid(); - ASSERT_TRUE(NULL != - (logger_list = android_logger_list_open( - LOG_ID_SECURITY, ANDROID_LOG_RDONLY | ANDROID_LOG_NONBLOCK, - 1000, pid))); + ASSERT_TRUE(NULL != (logger_list = android_logger_list_open(LOG_ID_SECURITY, ANDROID_LOG_NONBLOCK, + 1000, pid))); log_time ts(CLOCK_MONOTONIC); diff --git a/liblog/tests/log_read_test.cpp b/liblog/tests/log_read_test.cpp index 1be99aa80..3e0961749 100644 --- a/liblog/tests/log_read_test.cpp +++ b/liblog/tests/log_read_test.cpp @@ -34,8 +34,7 @@ TEST(liblog, android_logger_get_) { // This test assumes the log buffers are filled with noise from // normal operations. It will fail if done immediately after a // logcat -c. - struct logger_list* logger_list = - android_logger_list_alloc(ANDROID_LOG_WRONLY, 0, 0); + struct logger_list* logger_list = android_logger_list_alloc(0, 0, 0); for (int i = LOG_ID_MIN; i < LOG_ID_MAX; ++i) { log_id_t id = static_cast<log_id_t>(i); diff --git a/liblog/tests/log_wrap_test.cpp b/liblog/tests/log_wrap_test.cpp index e06964f72..755898a41 100644 --- a/liblog/tests/log_wrap_test.cpp +++ b/liblog/tests/log_wrap_test.cpp @@ -32,7 +32,7 @@ static void read_with_wrap() { // Read the last line in the log to get a starting timestamp. We're assuming // the log is not empty. - const int mode = ANDROID_LOG_RDONLY | ANDROID_LOG_NONBLOCK; + const int mode = ANDROID_LOG_NONBLOCK; struct logger_list* logger_list = android_logger_list_open(LOG_ID_MAIN, mode, 1000, 0); diff --git a/libunwindstack/Android.bp b/libunwindstack/Android.bp index 36449ebb8..9afc9a326 100644 --- a/libunwindstack/Android.bp +++ b/libunwindstack/Android.bp @@ -35,20 +35,13 @@ cc_defaults { }, } -cc_library { - name: "libunwindstack", - vendor_available: true, - recovery_available: true, - vndk: { - enabled: true, - support_system_process: true, - }, +cc_defaults { + name: "libunwindstack_defaults", defaults: ["libunwindstack_flags"], export_include_dirs: ["include"], srcs: [ "ArmExidx.cpp", - "DexFile.cpp", "DexFiles.cpp", "DwarfCfa.cpp", "DwarfEhFrameWithHdr.cpp", @@ -77,7 +70,6 @@ cc_library { ], cflags: [ - "-DDEXFILE_SUPPORT", "-Wexit-time-destructors", ], @@ -89,24 +81,6 @@ cc_library { "-g", ], }, - vendor: { - cflags: ["-UDEXFILE_SUPPORT"], - exclude_srcs: [ - "DexFile.cpp", - ], - exclude_shared_libs: [ - "libdexfile_support", - ], - }, - recovery: { - cflags: ["-UDEXFILE_SUPPORT"], - exclude_srcs: [ - "DexFile.cpp", - ], - exclude_shared_libs: [ - "libdexfile_support", - ], - }, }, arch: { @@ -124,12 +98,56 @@ cc_library { shared_libs: [ "libbase", - "libdexfile_support", "liblog", "liblzma", ], } +cc_library { + name: "libunwindstack", + vendor_available: true, + recovery_available: true, + vndk: { + enabled: true, + support_system_process: true, + }, + defaults: ["libunwindstack_defaults"], + + srcs: ["DexFile.cpp"], + cflags: ["-DDEXFILE_SUPPORT"], + shared_libs: ["libdexfile_support"], + + target: { + vendor: { + cflags: ["-UDEXFILE_SUPPORT"], + exclude_srcs: ["DexFile.cpp"], + exclude_shared_libs: ["libdexfile_support"], + }, + recovery: { + cflags: ["-UDEXFILE_SUPPORT"], + exclude_srcs: ["DexFile.cpp"], + exclude_shared_libs: ["libdexfile_support"], + }, + }, +} + +// Static library without DEX support to avoid dependencies on the ART APEX. +cc_library_static { + name: "libunwindstack_no_dex", + recovery_available: true, + defaults: ["libunwindstack_defaults"], + + visibility: [ + "//system/core/debuggerd", + "//system/core/init", + "//system/core/libbacktrace", + ], + apex_available: [ + "//apex_available:platform", + "com.android.runtime", + ], +} + //------------------------------------------------------------------------- // Unit Tests //------------------------------------------------------------------------- diff --git a/libunwindstack/DexFile.cpp b/libunwindstack/DexFile.cpp index dff7a8b3c..bf63abf1c 100644 --- a/libunwindstack/DexFile.cpp +++ b/libunwindstack/DexFile.cpp @@ -89,7 +89,7 @@ std::unique_ptr<DexFileFromFile> DexFileFromFile::Create(uint64_t dex_file_offse return nullptr; } - return std::unique_ptr<DexFileFromFile>(new DexFileFromFile(std::move(*art_dex_file.release()))); + return std::unique_ptr<DexFileFromFile>(new DexFileFromFile(art_dex_file)); } std::unique_ptr<DexFileFromMemory> DexFileFromMemory::Create(uint64_t dex_file_offset_in_memory, @@ -108,7 +108,7 @@ std::unique_ptr<DexFileFromMemory> DexFileFromMemory::Create(uint64_t dex_file_o if (art_dex_file != nullptr) { return std::unique_ptr<DexFileFromMemory>( - new DexFileFromMemory(std::move(*art_dex_file.release()), std::move(backing_memory))); + new DexFileFromMemory(art_dex_file, std::move(backing_memory))); } if (!error_msg.empty()) { diff --git a/libunwindstack/DexFile.h b/libunwindstack/DexFile.h index ca658e688..4e8369f84 100644 --- a/libunwindstack/DexFile.h +++ b/libunwindstack/DexFile.h @@ -39,7 +39,8 @@ class DexFile : protected art_api::dex::DexFile { MapInfo* info); protected: - DexFile(art_api::dex::DexFile&& art_dex_file) : art_api::dex::DexFile(std::move(art_dex_file)) {} + DexFile(std::unique_ptr<art_api::dex::DexFile>& art_dex_file) + : art_api::dex::DexFile(art_dex_file) {} }; class DexFileFromFile : public DexFile { @@ -48,7 +49,7 @@ class DexFileFromFile : public DexFile { const std::string& file); private: - DexFileFromFile(art_api::dex::DexFile&& art_dex_file) : DexFile(std::move(art_dex_file)) {} + DexFileFromFile(std::unique_ptr<art_api::dex::DexFile>& art_dex_file) : DexFile(art_dex_file) {} }; class DexFileFromMemory : public DexFile { @@ -57,8 +58,9 @@ class DexFileFromMemory : public DexFile { Memory* memory, const std::string& name); private: - DexFileFromMemory(art_api::dex::DexFile&& art_dex_file, std::vector<uint8_t>&& memory) - : DexFile(std::move(art_dex_file)), memory_(std::move(memory)) {} + DexFileFromMemory(std::unique_ptr<art_api::dex::DexFile>& art_dex_file, + std::vector<uint8_t>&& memory) + : DexFile(art_dex_file), memory_(std::move(memory)) {} std::vector<uint8_t> memory_; }; diff --git a/libunwindstack/LocalUnwinder.cpp b/libunwindstack/LocalUnwinder.cpp index 5d8120036..05650fb2e 100644 --- a/libunwindstack/LocalUnwinder.cpp +++ b/libunwindstack/LocalUnwinder.cpp @@ -106,7 +106,7 @@ bool LocalUnwinder::Unwind(std::vector<LocalFrameData>* frame_info, size_t max_f uint64_t step_pc = rel_pc; uint64_t pc_adjustment; if (adjust_pc) { - pc_adjustment = regs->GetPcAdjustment(rel_pc, elf); + pc_adjustment = GetPcAdjustment(rel_pc, elf, arch); } else { pc_adjustment = 0; } diff --git a/libunwindstack/Regs.cpp b/libunwindstack/Regs.cpp index e0a785b3f..03aa6c205 100644 --- a/libunwindstack/Regs.cpp +++ b/libunwindstack/Regs.cpp @@ -121,4 +121,62 @@ Regs* Regs::CreateFromLocal() { return regs; } +uint64_t GetPcAdjustment(uint64_t rel_pc, Elf* elf, ArchEnum arch) { + switch (arch) { + case ARCH_ARM: { + if (!elf->valid()) { + return 2; + } + + uint64_t load_bias = elf->GetLoadBias(); + if (rel_pc < load_bias) { + if (rel_pc < 2) { + return 0; + } + return 2; + } + uint64_t adjusted_rel_pc = rel_pc - load_bias; + if (adjusted_rel_pc < 5) { + if (adjusted_rel_pc < 2) { + return 0; + } + return 2; + } + + if (adjusted_rel_pc & 1) { + // This is a thumb instruction, it could be 2 or 4 bytes. + uint32_t value; + if (!elf->memory()->ReadFully(adjusted_rel_pc - 5, &value, sizeof(value)) || + (value & 0xe000f000) != 0xe000f000) { + return 2; + } + } + return 4; + } + case ARCH_ARM64: { + if (rel_pc < 4) { + return 0; + } + return 4; + } + case ARCH_MIPS: + case ARCH_MIPS64: { + if (rel_pc < 8) { + return 0; + } + // For now, just assume no compact branches + return 8; + } + case ARCH_X86: + case ARCH_X86_64: { + if (rel_pc == 0) { + return 0; + } + return 1; + } + case ARCH_UNKNOWN: + return 0; + } +} + } // namespace unwindstack diff --git a/libunwindstack/RegsArm.cpp b/libunwindstack/RegsArm.cpp index 1b1f7ebbc..1aaa08f56 100644 --- a/libunwindstack/RegsArm.cpp +++ b/libunwindstack/RegsArm.cpp @@ -51,37 +51,6 @@ void RegsArm::set_sp(uint64_t sp) { regs_[ARM_REG_SP] = sp; } -uint64_t RegsArm::GetPcAdjustment(uint64_t rel_pc, Elf* elf) { - if (!elf->valid()) { - return 2; - } - - uint64_t load_bias = elf->GetLoadBias(); - if (rel_pc < load_bias) { - if (rel_pc < 2) { - return 0; - } - return 2; - } - uint64_t adjusted_rel_pc = rel_pc - load_bias; - if (adjusted_rel_pc < 5) { - if (adjusted_rel_pc < 2) { - return 0; - } - return 2; - } - - if (adjusted_rel_pc & 1) { - // This is a thumb instruction, it could be 2 or 4 bytes. - uint32_t value; - if (!elf->memory()->ReadFully(adjusted_rel_pc - 5, &value, sizeof(value)) || - (value & 0xe000f000) != 0xe000f000) { - return 2; - } - } - return 4; -} - bool RegsArm::SetPcFromReturnAddress(Memory*) { uint32_t lr = regs_[ARM_REG_LR]; if (regs_[ARM_REG_PC] == lr) { diff --git a/libunwindstack/RegsArm64.cpp b/libunwindstack/RegsArm64.cpp index 00b336737..5b7431a57 100644 --- a/libunwindstack/RegsArm64.cpp +++ b/libunwindstack/RegsArm64.cpp @@ -52,13 +52,6 @@ void RegsArm64::set_sp(uint64_t sp) { regs_[ARM64_REG_SP] = sp; } -uint64_t RegsArm64::GetPcAdjustment(uint64_t rel_pc, Elf*) { - if (rel_pc < 4) { - return 0; - } - return 4; -} - bool RegsArm64::SetPcFromReturnAddress(Memory*) { uint64_t lr = regs_[ARM64_REG_LR]; if (regs_[ARM64_REG_PC] == lr) { diff --git a/libunwindstack/RegsMips.cpp b/libunwindstack/RegsMips.cpp index ebefe429b..ab8469114 100644 --- a/libunwindstack/RegsMips.cpp +++ b/libunwindstack/RegsMips.cpp @@ -52,14 +52,6 @@ void RegsMips::set_sp(uint64_t sp) { regs_[MIPS_REG_SP] = static_cast<uint32_t>(sp); } -uint64_t RegsMips::GetPcAdjustment(uint64_t rel_pc, Elf*) { - if (rel_pc < 8) { - return 0; - } - // For now, just assume no compact branches - return 8; -} - bool RegsMips::SetPcFromReturnAddress(Memory*) { uint32_t ra = regs_[MIPS_REG_RA]; if (regs_[MIPS_REG_PC] == ra) { diff --git a/libunwindstack/RegsMips64.cpp b/libunwindstack/RegsMips64.cpp index be2fd22f3..7f600d362 100644 --- a/libunwindstack/RegsMips64.cpp +++ b/libunwindstack/RegsMips64.cpp @@ -52,14 +52,6 @@ void RegsMips64::set_sp(uint64_t sp) { regs_[MIPS64_REG_SP] = sp; } -uint64_t RegsMips64::GetPcAdjustment(uint64_t rel_pc, Elf*) { - if (rel_pc < 8) { - return 0; - } - // For now, just assume no compact branches - return 8; -} - bool RegsMips64::SetPcFromReturnAddress(Memory*) { uint64_t ra = regs_[MIPS64_REG_RA]; if (regs_[MIPS64_REG_PC] == ra) { diff --git a/libunwindstack/RegsX86.cpp b/libunwindstack/RegsX86.cpp index 5538fc0be..4d3c246a4 100644 --- a/libunwindstack/RegsX86.cpp +++ b/libunwindstack/RegsX86.cpp @@ -50,13 +50,6 @@ void RegsX86::set_sp(uint64_t sp) { regs_[X86_REG_SP] = static_cast<uint32_t>(sp); } -uint64_t RegsX86::GetPcAdjustment(uint64_t rel_pc, Elf*) { - if (rel_pc == 0) { - return 0; - } - return 1; -} - bool RegsX86::SetPcFromReturnAddress(Memory* process_memory) { // Attempt to get the return address from the top of the stack. uint32_t new_pc; diff --git a/libunwindstack/RegsX86_64.cpp b/libunwindstack/RegsX86_64.cpp index 5b9aa58c4..c9e245d2f 100644 --- a/libunwindstack/RegsX86_64.cpp +++ b/libunwindstack/RegsX86_64.cpp @@ -51,13 +51,6 @@ void RegsX86_64::set_sp(uint64_t sp) { regs_[X86_64_REG_SP] = sp; } -uint64_t RegsX86_64::GetPcAdjustment(uint64_t rel_pc, Elf*) { - if (rel_pc == 0) { - return 0; - } - return 1; -} - bool RegsX86_64::SetPcFromReturnAddress(Memory* process_memory) { // Attempt to get the return address from the top of the stack. uint64_t new_pc; diff --git a/libunwindstack/Unwinder.cpp b/libunwindstack/Unwinder.cpp index 1bb031947..2d867cd5e 100644 --- a/libunwindstack/Unwinder.cpp +++ b/libunwindstack/Unwinder.cpp @@ -181,7 +181,7 @@ void Unwinder::Unwind(const std::vector<std::string>* initial_map_names_to_skip, step_pc = rel_pc; } if (adjust_pc) { - pc_adjustment = regs_->GetPcAdjustment(rel_pc, elf); + pc_adjustment = GetPcAdjustment(rel_pc, elf, arch); } else { pc_adjustment = 0; } @@ -395,4 +395,54 @@ bool UnwinderFromPid::Init(ArchEnum arch) { return true; } +FrameData Unwinder::BuildFrameFromPcOnly(uint64_t pc) { + FrameData frame; + + Maps* maps = GetMaps(); + MapInfo* map_info = maps->Find(pc); + if (!map_info) { + frame.rel_pc = pc; + return frame; + } + + ArchEnum arch = Regs::CurrentArch(); + Elf* elf = map_info->GetElf(GetProcessMemory(), arch); + + uint64_t relative_pc = elf->GetRelPc(pc, map_info); + + uint64_t pc_adjustment = GetPcAdjustment(relative_pc, elf, arch); + relative_pc -= pc_adjustment; + // The debug PC may be different if the PC comes from the JIT. + uint64_t debug_pc = relative_pc; + + // If we don't have a valid ELF file, check the JIT. + if (!elf->valid()) { + JitDebug jit_debug(GetProcessMemory()); + uint64_t jit_pc = pc - pc_adjustment; + Elf* jit_elf = jit_debug.GetElf(maps, jit_pc); + if (jit_elf != nullptr) { + debug_pc = jit_pc; + elf = jit_elf; + } + } + + // Copy all the things we need into the frame for symbolization. + frame.rel_pc = relative_pc; + frame.pc = pc - pc_adjustment; + frame.map_name = map_info->name; + frame.map_elf_start_offset = map_info->elf_start_offset; + frame.map_exact_offset = map_info->offset; + frame.map_start = map_info->start; + frame.map_end = map_info->end; + frame.map_flags = map_info->flags; + frame.map_load_bias = elf->GetLoadBias(); + + if (!resolve_names_ || + !elf->GetFunctionName(relative_pc, &frame.function_name, &frame.function_offset)) { + frame.function_name = ""; + frame.function_offset = 0; + } + return frame; +} + } // namespace unwindstack diff --git a/libunwindstack/include/unwindstack/Regs.h b/libunwindstack/include/unwindstack/Regs.h index 4f761b4ef..a367e6cf1 100644 --- a/libunwindstack/include/unwindstack/Regs.h +++ b/libunwindstack/include/unwindstack/Regs.h @@ -64,8 +64,6 @@ class Regs { uint64_t dex_pc() { return dex_pc_; } void set_dex_pc(uint64_t dex_pc) { dex_pc_ = dex_pc; } - virtual uint64_t GetPcAdjustment(uint64_t rel_pc, Elf* elf) = 0; - virtual bool StepIfSignalHandler(uint64_t elf_offset, Elf* elf, Memory* process_memory) = 0; virtual bool SetPcFromReturnAddress(Memory* process_memory) = 0; @@ -110,6 +108,8 @@ class RegsImpl : public Regs { std::vector<AddressType> regs_; }; +uint64_t GetPcAdjustment(uint64_t rel_pc, Elf* elf, ArchEnum arch); + } // namespace unwindstack #endif // _LIBUNWINDSTACK_REGS_H diff --git a/libunwindstack/include/unwindstack/RegsArm.h b/libunwindstack/include/unwindstack/RegsArm.h index aa029be64..fbb34e7f8 100644 --- a/libunwindstack/include/unwindstack/RegsArm.h +++ b/libunwindstack/include/unwindstack/RegsArm.h @@ -36,8 +36,6 @@ class RegsArm : public RegsImpl<uint32_t> { ArchEnum Arch() override final; - uint64_t GetPcAdjustment(uint64_t rel_pc, Elf* elf) override; - bool SetPcFromReturnAddress(Memory* process_memory) override; bool StepIfSignalHandler(uint64_t elf_offset, Elf* elf, Memory* process_memory) override; diff --git a/libunwindstack/include/unwindstack/RegsArm64.h b/libunwindstack/include/unwindstack/RegsArm64.h index 5cd7e5ba9..2b3ddeb77 100644 --- a/libunwindstack/include/unwindstack/RegsArm64.h +++ b/libunwindstack/include/unwindstack/RegsArm64.h @@ -36,8 +36,6 @@ class RegsArm64 : public RegsImpl<uint64_t> { ArchEnum Arch() override final; - uint64_t GetPcAdjustment(uint64_t rel_pc, Elf* elf) override; - bool SetPcFromReturnAddress(Memory* process_memory) override; bool StepIfSignalHandler(uint64_t elf_offset, Elf* elf, Memory* process_memory) override; diff --git a/libunwindstack/include/unwindstack/RegsMips.h b/libunwindstack/include/unwindstack/RegsMips.h index 8164a1515..dc09b83b8 100644 --- a/libunwindstack/include/unwindstack/RegsMips.h +++ b/libunwindstack/include/unwindstack/RegsMips.h @@ -36,8 +36,6 @@ class RegsMips : public RegsImpl<uint32_t> { ArchEnum Arch() override final; - uint64_t GetPcAdjustment(uint64_t rel_pc, Elf* elf) override; - bool SetPcFromReturnAddress(Memory* process_memory) override; bool StepIfSignalHandler(uint64_t elf_offset, Elf* elf, Memory* process_memory) override; diff --git a/libunwindstack/include/unwindstack/RegsMips64.h b/libunwindstack/include/unwindstack/RegsMips64.h index c98254259..64a80dc0b 100644 --- a/libunwindstack/include/unwindstack/RegsMips64.h +++ b/libunwindstack/include/unwindstack/RegsMips64.h @@ -36,8 +36,6 @@ class RegsMips64 : public RegsImpl<uint64_t> { ArchEnum Arch() override final; - uint64_t GetPcAdjustment(uint64_t rel_pc, Elf* elf) override; - bool SetPcFromReturnAddress(Memory* process_memory) override; bool StepIfSignalHandler(uint64_t elf_offset, Elf* elf, Memory* process_memory) override; diff --git a/libunwindstack/include/unwindstack/RegsX86.h b/libunwindstack/include/unwindstack/RegsX86.h index 2323a4fc0..cfbdda661 100644 --- a/libunwindstack/include/unwindstack/RegsX86.h +++ b/libunwindstack/include/unwindstack/RegsX86.h @@ -37,8 +37,6 @@ class RegsX86 : public RegsImpl<uint32_t> { ArchEnum Arch() override final; - uint64_t GetPcAdjustment(uint64_t rel_pc, Elf* elf) override; - bool SetPcFromReturnAddress(Memory* process_memory) override; bool StepIfSignalHandler(uint64_t elf_offset, Elf* elf, Memory* process_memory) override; diff --git a/libunwindstack/include/unwindstack/RegsX86_64.h b/libunwindstack/include/unwindstack/RegsX86_64.h index 3e919a4f8..a11aef09f 100644 --- a/libunwindstack/include/unwindstack/RegsX86_64.h +++ b/libunwindstack/include/unwindstack/RegsX86_64.h @@ -37,8 +37,6 @@ class RegsX86_64 : public RegsImpl<uint64_t> { ArchEnum Arch() override final; - uint64_t GetPcAdjustment(uint64_t rel_pc, Elf* elf) override; - bool SetPcFromReturnAddress(Memory* process_memory) override; bool StepIfSignalHandler(uint64_t elf_offset, Elf* elf, Memory* process_memory) override; diff --git a/libunwindstack/include/unwindstack/Unwinder.h b/libunwindstack/include/unwindstack/Unwinder.h index 67762c0bd..4d49f236a 100644 --- a/libunwindstack/include/unwindstack/Unwinder.h +++ b/libunwindstack/include/unwindstack/Unwinder.h @@ -114,6 +114,13 @@ class Unwinder { ErrorCode LastErrorCode() { return last_error_.code; } uint64_t LastErrorAddress() { return last_error_.address; } + // Builds a frame for symbolization using the maps from this unwinder. The + // constructed frame contains just enough information to be used to symbolize + // frames collected by frame-pointer unwinding that's done outside of + // libunwindstack. This is used by tombstoned to symbolize frame pointer-based + // stack traces that are collected by tools such as GWP-ASan and MTE. + FrameData BuildFrameFromPcOnly(uint64_t pc); + protected: Unwinder(size_t max_frames) : max_frames_(max_frames) { frames_.reserve(max_frames); } diff --git a/libunwindstack/tests/DexFileTest.cpp b/libunwindstack/tests/DexFileTest.cpp index 1b54da6fb..dc935a36e 100644 --- a/libunwindstack/tests/DexFileTest.cpp +++ b/libunwindstack/tests/DexFileTest.cpp @@ -14,6 +14,7 @@ * limitations under the License. */ +#include <malloc.h> #include <stdint.h> #include <sys/types.h> #include <unistd.h> @@ -72,6 +73,37 @@ TEST(DexFileTest, from_file_open_non_zero_offset) { EXPECT_TRUE(DexFileFromFile::Create(0x100, tf.path) != nullptr); } +static constexpr size_t kNumLeakLoops = 5000; +static constexpr size_t kMaxAllowedLeakBytes = 1024; + +static void CheckForLeak(size_t loop, size_t* first_allocated_bytes, size_t* last_allocated_bytes) { + size_t allocated_bytes = mallinfo().uordblks; + if (*first_allocated_bytes == 0) { + *first_allocated_bytes = allocated_bytes; + } else if (*last_allocated_bytes > *first_allocated_bytes) { + // Check that the total memory did not increase too much over the first loop. + ASSERT_LE(*last_allocated_bytes - *first_allocated_bytes, kMaxAllowedLeakBytes) + << "Failed in loop " << loop << " first_allocated_bytes " << *first_allocated_bytes + << " last_allocated_bytes " << *last_allocated_bytes; + } + *last_allocated_bytes = allocated_bytes; +} + +TEST(DexFileTest, from_file_no_leak) { + TemporaryFile tf; + ASSERT_TRUE(tf.fd != -1); + + ASSERT_EQ(sizeof(kDexData), + static_cast<size_t>(TEMP_FAILURE_RETRY(write(tf.fd, kDexData, sizeof(kDexData))))); + + size_t first_allocated_bytes = 0; + size_t last_allocated_bytes = 0; + for (size_t i = 0; i < kNumLeakLoops; i++) { + EXPECT_TRUE(DexFileFromFile::Create(0, tf.path) != nullptr); + ASSERT_NO_FATAL_FAILURE(CheckForLeak(i, &first_allocated_bytes, &last_allocated_bytes)); + } +} + TEST(DexFileTest, from_memory_fail_too_small_for_header) { MemoryFake memory; @@ -96,6 +128,19 @@ TEST(DexFileTest, from_memory_open) { EXPECT_TRUE(DexFileFromMemory::Create(0x1000, &memory, "") != nullptr); } +TEST(DexFileTest, from_memory_no_leak) { + MemoryFake memory; + + memory.SetMemory(0x1000, kDexData, sizeof(kDexData)); + + size_t first_allocated_bytes = 0; + size_t last_allocated_bytes = 0; + for (size_t i = 0; i < kNumLeakLoops; i++) { + EXPECT_TRUE(DexFileFromMemory::Create(0x1000, &memory, "") != nullptr); + ASSERT_NO_FATAL_FAILURE(CheckForLeak(i, &first_allocated_bytes, &last_allocated_bytes)); + } +} + TEST(DexFileTest, create_using_file) { TemporaryFile tf; ASSERT_TRUE(tf.fd != -1); diff --git a/libunwindstack/tests/RegsFake.h b/libunwindstack/tests/RegsFake.h index 207d46efe..75fc9d02d 100644 --- a/libunwindstack/tests/RegsFake.h +++ b/libunwindstack/tests/RegsFake.h @@ -54,8 +54,6 @@ class RegsFake : public Regs { return fake_arch_ == ARCH_ARM || fake_arch_ == ARCH_X86 || fake_arch_ == ARCH_MIPS; } - uint64_t GetPcAdjustment(uint64_t, Elf*) override { return 2; } - bool StepIfSignalHandler(uint64_t, Elf*, Memory*) override { return false; } void FakeSetArch(ArchEnum arch) { fake_arch_ = arch; } @@ -86,7 +84,6 @@ class RegsImplFake : public RegsImpl<TypeParam> { void set_pc(uint64_t pc) override { fake_pc_ = pc; } void set_sp(uint64_t sp) override { fake_sp_ = sp; } - uint64_t GetPcAdjustment(uint64_t, Elf*) override { return 0; } bool SetPcFromReturnAddress(Memory*) override { return false; } bool StepIfSignalHandler(uint64_t, Elf*, Memory*) override { return false; } diff --git a/libunwindstack/tests/RegsTest.cpp b/libunwindstack/tests/RegsTest.cpp index 0a33e2ff4..e4fc6f070 100644 --- a/libunwindstack/tests/RegsTest.cpp +++ b/libunwindstack/tests/RegsTest.cpp @@ -93,123 +93,104 @@ TEST_F(RegsTest, regs64) { } TEST_F(RegsTest, rel_pc) { - RegsArm64 arm64; - EXPECT_EQ(4U, arm64.GetPcAdjustment(0x10, elf_.get())); - EXPECT_EQ(4U, arm64.GetPcAdjustment(0x4, elf_.get())); - EXPECT_EQ(0U, arm64.GetPcAdjustment(0x3, elf_.get())); - EXPECT_EQ(0U, arm64.GetPcAdjustment(0x2, elf_.get())); - EXPECT_EQ(0U, arm64.GetPcAdjustment(0x1, elf_.get())); - EXPECT_EQ(0U, arm64.GetPcAdjustment(0x0, elf_.get())); - - RegsX86 x86; - EXPECT_EQ(1U, x86.GetPcAdjustment(0x100, elf_.get())); - EXPECT_EQ(1U, x86.GetPcAdjustment(0x2, elf_.get())); - EXPECT_EQ(1U, x86.GetPcAdjustment(0x1, elf_.get())); - EXPECT_EQ(0U, x86.GetPcAdjustment(0x0, elf_.get())); - - RegsX86_64 x86_64; - EXPECT_EQ(1U, x86_64.GetPcAdjustment(0x100, elf_.get())); - EXPECT_EQ(1U, x86_64.GetPcAdjustment(0x2, elf_.get())); - EXPECT_EQ(1U, x86_64.GetPcAdjustment(0x1, elf_.get())); - EXPECT_EQ(0U, x86_64.GetPcAdjustment(0x0, elf_.get())); - - RegsMips mips; - EXPECT_EQ(8U, mips.GetPcAdjustment(0x10, elf_.get())); - EXPECT_EQ(8U, mips.GetPcAdjustment(0x8, elf_.get())); - EXPECT_EQ(0U, mips.GetPcAdjustment(0x7, elf_.get())); - EXPECT_EQ(0U, mips.GetPcAdjustment(0x6, elf_.get())); - EXPECT_EQ(0U, mips.GetPcAdjustment(0x5, elf_.get())); - EXPECT_EQ(0U, mips.GetPcAdjustment(0x4, elf_.get())); - EXPECT_EQ(0U, mips.GetPcAdjustment(0x3, elf_.get())); - EXPECT_EQ(0U, mips.GetPcAdjustment(0x2, elf_.get())); - EXPECT_EQ(0U, mips.GetPcAdjustment(0x1, elf_.get())); - EXPECT_EQ(0U, mips.GetPcAdjustment(0x0, elf_.get())); - - RegsMips64 mips64; - EXPECT_EQ(8U, mips64.GetPcAdjustment(0x10, elf_.get())); - EXPECT_EQ(8U, mips64.GetPcAdjustment(0x8, elf_.get())); - EXPECT_EQ(0U, mips64.GetPcAdjustment(0x7, elf_.get())); - EXPECT_EQ(0U, mips64.GetPcAdjustment(0x6, elf_.get())); - EXPECT_EQ(0U, mips64.GetPcAdjustment(0x5, elf_.get())); - EXPECT_EQ(0U, mips64.GetPcAdjustment(0x4, elf_.get())); - EXPECT_EQ(0U, mips64.GetPcAdjustment(0x3, elf_.get())); - EXPECT_EQ(0U, mips64.GetPcAdjustment(0x2, elf_.get())); - EXPECT_EQ(0U, mips64.GetPcAdjustment(0x1, elf_.get())); - EXPECT_EQ(0U, mips64.GetPcAdjustment(0x0, elf_.get())); + EXPECT_EQ(4U, GetPcAdjustment(0x10, elf_.get(), ARCH_ARM64)); + EXPECT_EQ(4U, GetPcAdjustment(0x4, elf_.get(), ARCH_ARM64)); + EXPECT_EQ(0U, GetPcAdjustment(0x3, elf_.get(), ARCH_ARM64)); + EXPECT_EQ(0U, GetPcAdjustment(0x2, elf_.get(), ARCH_ARM64)); + EXPECT_EQ(0U, GetPcAdjustment(0x1, elf_.get(), ARCH_ARM64)); + EXPECT_EQ(0U, GetPcAdjustment(0x0, elf_.get(), ARCH_ARM64)); + + EXPECT_EQ(1U, GetPcAdjustment(0x100, elf_.get(), ARCH_X86)); + EXPECT_EQ(1U, GetPcAdjustment(0x2, elf_.get(), ARCH_X86)); + EXPECT_EQ(1U, GetPcAdjustment(0x1, elf_.get(), ARCH_X86)); + EXPECT_EQ(0U, GetPcAdjustment(0x0, elf_.get(), ARCH_X86)); + + EXPECT_EQ(1U, GetPcAdjustment(0x100, elf_.get(), ARCH_X86_64)); + EXPECT_EQ(1U, GetPcAdjustment(0x2, elf_.get(), ARCH_X86_64)); + EXPECT_EQ(1U, GetPcAdjustment(0x1, elf_.get(), ARCH_X86_64)); + EXPECT_EQ(0U, GetPcAdjustment(0x0, elf_.get(), ARCH_X86_64)); + + EXPECT_EQ(8U, GetPcAdjustment(0x10, elf_.get(), ARCH_MIPS)); + EXPECT_EQ(8U, GetPcAdjustment(0x8, elf_.get(), ARCH_MIPS)); + EXPECT_EQ(0U, GetPcAdjustment(0x7, elf_.get(), ARCH_MIPS)); + EXPECT_EQ(0U, GetPcAdjustment(0x6, elf_.get(), ARCH_MIPS)); + EXPECT_EQ(0U, GetPcAdjustment(0x5, elf_.get(), ARCH_MIPS)); + EXPECT_EQ(0U, GetPcAdjustment(0x4, elf_.get(), ARCH_MIPS)); + EXPECT_EQ(0U, GetPcAdjustment(0x3, elf_.get(), ARCH_MIPS)); + EXPECT_EQ(0U, GetPcAdjustment(0x2, elf_.get(), ARCH_MIPS)); + EXPECT_EQ(0U, GetPcAdjustment(0x1, elf_.get(), ARCH_MIPS)); + EXPECT_EQ(0U, GetPcAdjustment(0x0, elf_.get(), ARCH_MIPS)); + + EXPECT_EQ(8U, GetPcAdjustment(0x10, elf_.get(), ARCH_MIPS64)); + EXPECT_EQ(8U, GetPcAdjustment(0x8, elf_.get(), ARCH_MIPS64)); + EXPECT_EQ(0U, GetPcAdjustment(0x7, elf_.get(), ARCH_MIPS64)); + EXPECT_EQ(0U, GetPcAdjustment(0x6, elf_.get(), ARCH_MIPS64)); + EXPECT_EQ(0U, GetPcAdjustment(0x5, elf_.get(), ARCH_MIPS64)); + EXPECT_EQ(0U, GetPcAdjustment(0x4, elf_.get(), ARCH_MIPS64)); + EXPECT_EQ(0U, GetPcAdjustment(0x3, elf_.get(), ARCH_MIPS64)); + EXPECT_EQ(0U, GetPcAdjustment(0x2, elf_.get(), ARCH_MIPS64)); + EXPECT_EQ(0U, GetPcAdjustment(0x1, elf_.get(), ARCH_MIPS64)); + EXPECT_EQ(0U, GetPcAdjustment(0x0, elf_.get(), ARCH_MIPS64)); } TEST_F(RegsTest, rel_pc_arm) { - RegsArm arm; - // Check fence posts. elf_->FakeSetLoadBias(0); - EXPECT_EQ(2U, arm.GetPcAdjustment(0x5, elf_.get())); - EXPECT_EQ(2U, arm.GetPcAdjustment(0x4, elf_.get())); - EXPECT_EQ(2U, arm.GetPcAdjustment(0x3, elf_.get())); - EXPECT_EQ(2U, arm.GetPcAdjustment(0x2, elf_.get())); - EXPECT_EQ(0U, arm.GetPcAdjustment(0x1, elf_.get())); - EXPECT_EQ(0U, arm.GetPcAdjustment(0x0, elf_.get())); + EXPECT_EQ(2U, GetPcAdjustment(0x5, elf_.get(), ARCH_ARM)); + EXPECT_EQ(2U, GetPcAdjustment(0x4, elf_.get(), ARCH_ARM)); + EXPECT_EQ(2U, GetPcAdjustment(0x3, elf_.get(), ARCH_ARM)); + EXPECT_EQ(2U, GetPcAdjustment(0x2, elf_.get(), ARCH_ARM)); + EXPECT_EQ(0U, GetPcAdjustment(0x1, elf_.get(), ARCH_ARM)); + EXPECT_EQ(0U, GetPcAdjustment(0x0, elf_.get(), ARCH_ARM)); elf_->FakeSetLoadBias(0x100); - EXPECT_EQ(0U, arm.GetPcAdjustment(0x1, elf_.get())); - EXPECT_EQ(2U, arm.GetPcAdjustment(0x2, elf_.get())); - EXPECT_EQ(2U, arm.GetPcAdjustment(0xff, elf_.get())); - EXPECT_EQ(2U, arm.GetPcAdjustment(0x105, elf_.get())); - EXPECT_EQ(2U, arm.GetPcAdjustment(0x104, elf_.get())); - EXPECT_EQ(2U, arm.GetPcAdjustment(0x103, elf_.get())); - EXPECT_EQ(2U, arm.GetPcAdjustment(0x102, elf_.get())); - EXPECT_EQ(0U, arm.GetPcAdjustment(0x101, elf_.get())); - EXPECT_EQ(0U, arm.GetPcAdjustment(0x100, elf_.get())); + EXPECT_EQ(0U, GetPcAdjustment(0x1, elf_.get(), ARCH_ARM)); + EXPECT_EQ(2U, GetPcAdjustment(0x2, elf_.get(), ARCH_ARM)); + EXPECT_EQ(2U, GetPcAdjustment(0xff, elf_.get(), ARCH_ARM)); + EXPECT_EQ(2U, GetPcAdjustment(0x105, elf_.get(), ARCH_ARM)); + EXPECT_EQ(2U, GetPcAdjustment(0x104, elf_.get(), ARCH_ARM)); + EXPECT_EQ(2U, GetPcAdjustment(0x103, elf_.get(), ARCH_ARM)); + EXPECT_EQ(2U, GetPcAdjustment(0x102, elf_.get(), ARCH_ARM)); + EXPECT_EQ(0U, GetPcAdjustment(0x101, elf_.get(), ARCH_ARM)); + EXPECT_EQ(0U, GetPcAdjustment(0x100, elf_.get(), ARCH_ARM)); // Check thumb instructions handling. elf_->FakeSetLoadBias(0); memory_->SetData32(0x2000, 0); - EXPECT_EQ(2U, arm.GetPcAdjustment(0x2005, elf_.get())); + EXPECT_EQ(2U, GetPcAdjustment(0x2005, elf_.get(), ARCH_ARM)); memory_->SetData32(0x2000, 0xe000f000); - EXPECT_EQ(4U, arm.GetPcAdjustment(0x2005, elf_.get())); + EXPECT_EQ(4U, GetPcAdjustment(0x2005, elf_.get(), ARCH_ARM)); elf_->FakeSetLoadBias(0x400); memory_->SetData32(0x2100, 0); - EXPECT_EQ(2U, arm.GetPcAdjustment(0x2505, elf_.get())); + EXPECT_EQ(2U, GetPcAdjustment(0x2505, elf_.get(), ARCH_ARM)); memory_->SetData32(0x2100, 0xf111f111); - EXPECT_EQ(4U, arm.GetPcAdjustment(0x2505, elf_.get())); + EXPECT_EQ(4U, GetPcAdjustment(0x2505, elf_.get(), ARCH_ARM)); } TEST_F(RegsTest, elf_invalid) { - RegsArm regs_arm; - RegsArm64 regs_arm64; - RegsX86 regs_x86; - RegsX86_64 regs_x86_64; - RegsMips regs_mips; - RegsMips64 regs_mips64; MapInfo map_info(nullptr, nullptr, 0x1000, 0x2000, 0, 0, ""); Elf* invalid_elf = new Elf(nullptr); map_info.elf.reset(invalid_elf); - regs_arm.set_pc(0x1500); - EXPECT_EQ(0x500U, invalid_elf->GetRelPc(regs_arm.pc(), &map_info)); - EXPECT_EQ(2U, regs_arm.GetPcAdjustment(0x500U, invalid_elf)); - EXPECT_EQ(2U, regs_arm.GetPcAdjustment(0x511U, invalid_elf)); + EXPECT_EQ(0x500U, invalid_elf->GetRelPc(0x1500, &map_info)); + EXPECT_EQ(2U, GetPcAdjustment(0x500U, invalid_elf, ARCH_ARM)); + EXPECT_EQ(2U, GetPcAdjustment(0x511U, invalid_elf, ARCH_ARM)); - regs_arm64.set_pc(0x1600); - EXPECT_EQ(0x600U, invalid_elf->GetRelPc(regs_arm64.pc(), &map_info)); - EXPECT_EQ(4U, regs_arm64.GetPcAdjustment(0x600U, invalid_elf)); + EXPECT_EQ(0x600U, invalid_elf->GetRelPc(0x1600, &map_info)); + EXPECT_EQ(4U, GetPcAdjustment(0x600U, invalid_elf, ARCH_ARM64)); - regs_x86.set_pc(0x1700); - EXPECT_EQ(0x700U, invalid_elf->GetRelPc(regs_x86.pc(), &map_info)); - EXPECT_EQ(1U, regs_x86.GetPcAdjustment(0x700U, invalid_elf)); + EXPECT_EQ(0x700U, invalid_elf->GetRelPc(0x1700, &map_info)); + EXPECT_EQ(1U, GetPcAdjustment(0x700U, invalid_elf, ARCH_X86)); - regs_x86_64.set_pc(0x1800); - EXPECT_EQ(0x800U, invalid_elf->GetRelPc(regs_x86_64.pc(), &map_info)); - EXPECT_EQ(1U, regs_x86_64.GetPcAdjustment(0x800U, invalid_elf)); + EXPECT_EQ(0x800U, invalid_elf->GetRelPc(0x1800, &map_info)); + EXPECT_EQ(1U, GetPcAdjustment(0x800U, invalid_elf, ARCH_X86_64)); - regs_mips.set_pc(0x1900); - EXPECT_EQ(0x900U, invalid_elf->GetRelPc(regs_mips.pc(), &map_info)); - EXPECT_EQ(8U, regs_mips.GetPcAdjustment(0x900U, invalid_elf)); + EXPECT_EQ(0x900U, invalid_elf->GetRelPc(0x1900, &map_info)); + EXPECT_EQ(8U, GetPcAdjustment(0x900U, invalid_elf, ARCH_MIPS)); - regs_mips64.set_pc(0x1a00); - EXPECT_EQ(0xa00U, invalid_elf->GetRelPc(regs_mips64.pc(), &map_info)); - EXPECT_EQ(8U, regs_mips64.GetPcAdjustment(0xa00U, invalid_elf)); + EXPECT_EQ(0xa00U, invalid_elf->GetRelPc(0x1a00, &map_info)); + EXPECT_EQ(8U, GetPcAdjustment(0xa00U, invalid_elf, ARCH_MIPS64)); } TEST_F(RegsTest, arm_verify_sp_pc) { diff --git a/libunwindstack/tests/UnwinderTest.cpp b/libunwindstack/tests/UnwinderTest.cpp index ef1950ccc..dd33aa9d7 100644 --- a/libunwindstack/tests/UnwinderTest.cpp +++ b/libunwindstack/tests/UnwinderTest.cpp @@ -161,8 +161,8 @@ TEST_F(UnwinderTest, multiple_frames) { regs_.set_pc(0x1000); regs_.set_sp(0x10000); - ElfInterfaceFake::FakePushStepData(StepData(0x1102, 0x10010, false)); - ElfInterfaceFake::FakePushStepData(StepData(0x1202, 0x10020, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x1104, 0x10010, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x1204, 0x10020, false)); ElfInterfaceFake::FakePushStepData(StepData(0, 0, true)); Unwinder unwinder(64, maps_.get(), ®s_, process_memory_); @@ -225,8 +225,8 @@ TEST_F(UnwinderTest, multiple_frames_dont_resolve_names) { regs_.set_pc(0x1000); regs_.set_sp(0x10000); - ElfInterfaceFake::FakePushStepData(StepData(0x1102, 0x10010, false)); - ElfInterfaceFake::FakePushStepData(StepData(0x1202, 0x10020, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x1104, 0x10010, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x1204, 0x10020, false)); ElfInterfaceFake::FakePushStepData(StepData(0, 0, true)); Unwinder unwinder(64, maps_.get(), ®s_, process_memory_); @@ -445,7 +445,7 @@ TEST_F(UnwinderTest, no_frames_after_finished) { TEST_F(UnwinderTest, max_frames) { for (size_t i = 0; i < 30; i++) { ElfInterfaceFake::FakePushFunctionData(FunctionData("Frame" + std::to_string(i), i)); - ElfInterfaceFake::FakePushStepData(StepData(0x1102 + i * 0x100, 0x10010 + i * 0x10, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x1104 + i * 0x100, 0x10010 + i * 0x10, false)); } regs_.set_pc(0x1000); @@ -484,12 +484,12 @@ TEST_F(UnwinderTest, verify_frames_skipped) { regs_.set_pc(0x20000); regs_.set_sp(0x10000); - ElfInterfaceFake::FakePushStepData(StepData(0x23002, 0x10010, false)); - ElfInterfaceFake::FakePushStepData(StepData(0x23102, 0x10020, false)); - ElfInterfaceFake::FakePushStepData(StepData(0x20002, 0x10030, false)); - ElfInterfaceFake::FakePushStepData(StepData(0x21002, 0x10040, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x23004, 0x10010, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x23104, 0x10020, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x20004, 0x10030, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x21004, 0x10040, false)); ElfInterfaceFake::FakePushStepData(StepData(0x1002, 0x10050, false)); - ElfInterfaceFake::FakePushStepData(StepData(0x21002, 0x10060, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x21004, 0x10060, false)); ElfInterfaceFake::FakePushStepData(StepData(0x23002, 0x10070, false)); ElfInterfaceFake::FakePushStepData(StepData(0, 0, true)); @@ -553,7 +553,7 @@ TEST_F(UnwinderTest, sp_not_in_map) { regs_.set_pc(0x1000); regs_.set_sp(0x63000); - ElfInterfaceFake::FakePushStepData(StepData(0x21002, 0x50020, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x21004, 0x50020, false)); ElfInterfaceFake::FakePushStepData(StepData(0, 0, true)); Unwinder unwinder(64, maps_.get(), ®s_, process_memory_); @@ -670,10 +670,10 @@ TEST_F(UnwinderTest, speculative_frame) { // Fake as if code called a nullptr function. regs_.set_pc(0); regs_.set_sp(0x10000); - regs_.FakeSetReturnAddress(0x1202); + regs_.FakeSetReturnAddress(0x1204); regs_.FakeSetReturnAddressValid(true); - ElfInterfaceFake::FakePushStepData(StepData(0x23102, 0x10020, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x23104, 0x10020, false)); ElfInterfaceFake::FakePushStepData(StepData(0, 0, true)); Unwinder unwinder(64, maps_.get(), ®s_, process_memory_); @@ -789,7 +789,7 @@ TEST_F(UnwinderTest, speculative_frame_not_removed_pc_bad) { // Fake as if code called a nullptr function. regs_.set_pc(0); regs_.set_sp(0x10000); - regs_.FakeSetReturnAddress(0x1202); + regs_.FakeSetReturnAddress(0x1204); regs_.FakeSetReturnAddressValid(true); Unwinder unwinder(64, maps_.get(), ®s_, process_memory_); @@ -858,8 +858,8 @@ TEST_F(UnwinderTest, map_ignore_suffixes) { // Fake as if code called a nullptr function. regs_.set_pc(0x1000); regs_.set_sp(0x10000); - ElfInterfaceFake::FakePushStepData(StepData(0x43402, 0x10010, false)); - ElfInterfaceFake::FakePushStepData(StepData(0x53502, 0x10020, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x43404, 0x10010, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x53504, 0x10020, false)); ElfInterfaceFake::FakePushStepData(StepData(0, 0, true)); Unwinder unwinder(64, maps_.get(), ®s_, process_memory_); @@ -915,11 +915,11 @@ TEST_F(UnwinderTest, sp_pc_do_not_change) { regs_.set_pc(0x1000); regs_.set_sp(0x10000); - ElfInterfaceFake::FakePushStepData(StepData(0x33402, 0x10010, false)); - ElfInterfaceFake::FakePushStepData(StepData(0x33502, 0x10020, false)); - ElfInterfaceFake::FakePushStepData(StepData(0x33502, 0x10020, false)); - ElfInterfaceFake::FakePushStepData(StepData(0x33502, 0x10020, false)); - ElfInterfaceFake::FakePushStepData(StepData(0x33502, 0x10020, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x33404, 0x10010, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x33504, 0x10020, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x33504, 0x10020, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x33504, 0x10020, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x33504, 0x10020, false)); ElfInterfaceFake::FakePushStepData(StepData(0, 0, true)); Unwinder unwinder(64, maps_.get(), ®s_, process_memory_); @@ -1113,7 +1113,7 @@ TEST_F(UnwinderTest, dex_pc_multiple_frames) { regs_.set_pc(0x1000); regs_.set_sp(0x10000); regs_.FakeSetDexPc(0xa3400); - ElfInterfaceFake::FakePushStepData(StepData(0x33402, 0x10010, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x33404, 0x10010, false)); ElfInterfaceFake::FakePushStepData(StepData(0, 0, true)); Unwinder unwinder(64, maps_.get(), ®s_, process_memory_); diff --git a/libziparchive/Android.bp b/libziparchive/Android.bp index 1bbffaf7a..786e7b3d3 100644 --- a/libziparchive/Android.bp +++ b/libziparchive/Android.bp @@ -60,6 +60,8 @@ cc_defaults { srcs: [ "zip_archive.cc", "zip_archive_stream_entry.cc", + "zip_cd_entry_map.cc", + "zip_error.cpp", "zip_writer.cc", ], @@ -210,3 +212,20 @@ sh_test { data: ["cli-tests/**/*"], target_required: ["cli-test", "ziptool"], } + +python_test_host { + name: "ziparchive_tests_large", + srcs: ["test_ziparchive_large.py"], + main: "test_ziparchive_large.py", + version: { + py2: { + enabled: true, + embedded_launcher: false, + }, + py3: { + enabled: false, + embedded_launcher: false, + }, + }, + test_suites: ["general-tests"], +} diff --git a/libziparchive/include/ziparchive/zip_archive.h b/libziparchive/include/ziparchive/zip_archive.h index 047af90f2..435bfb6b0 100644 --- a/libziparchive/include/ziparchive/zip_archive.h +++ b/libziparchive/include/ziparchive/zip_archive.h @@ -126,6 +126,9 @@ int32_t OpenArchive(const char* fileName, ZipArchiveHandle* handle); int32_t OpenArchiveFd(const int fd, const char* debugFileName, ZipArchiveHandle* handle, bool assume_ownership = true); +int32_t OpenArchiveFdRange(const int fd, const char* debugFileName, ZipArchiveHandle* handle, + off64_t length, off64_t offset, bool assume_ownership = true); + int32_t OpenArchiveFromMemory(const void* address, size_t length, const char* debugFileName, ZipArchiveHandle* handle); /* @@ -142,7 +145,7 @@ struct ZipArchiveInfo { /** The size in bytes of the archive itself. Used by zipinfo. */ off64_t archive_size; /** The number of entries in the archive. */ - size_t entry_count; + uint64_t entry_count; }; /** @@ -185,6 +188,15 @@ int32_t StartIteration(ZipArchiveHandle archive, void** cookie_ptr, const std::string_view optional_suffix = ""); /* + * Start iterating over all entries of a zip file. Use the matcher functor to + * restrict iteration to entry names that make the functor return true. + * + * Returns 0 on success and negative values on failure. + */ +int32_t StartIteration(ZipArchiveHandle archive, void** cookie_ptr, + std::function<bool(std::string_view entry_name)> matcher); + +/* * Advance to the next element in the zipfile in iteration order. * * Returns 0 on success, -1 if there are no more elements in this @@ -222,6 +234,12 @@ int32_t ExtractToMemory(ZipArchiveHandle archive, ZipEntry* entry, uint8_t* begi int GetFileDescriptor(const ZipArchiveHandle archive); +/** + * Returns the offset of the zip archive in the backing file descriptor, or 0 if the zip archive is + * not backed by a file descriptor. + */ +off64_t GetFileDescriptorOffset(const ZipArchiveHandle archive); + const char* ErrorCodeString(int32_t error_code); #if !defined(_WIN32) diff --git a/libziparchive/test_ziparchive_large.py b/libziparchive/test_ziparchive_large.py new file mode 100644 index 000000000..c29c37e9d --- /dev/null +++ b/libziparchive/test_ziparchive_large.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python +# +# Copyright (C) 2020 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Unittests for parsing files in zip64 format""" + +import os +import subprocess +import tempfile +import unittest +import zipfile +import time + +class Zip64Test(unittest.TestCase): + @staticmethod + def _AddEntriesToZip(output_zip, entries_dict=None): + for name, size in entries_dict.items(): + contents = name[0] * 1024 + file_path = tempfile.NamedTemporaryFile() + with open(file_path.name, 'w') as f: + for it in range(0, size): + f.write(contents) + output_zip.write(file_path.name, arcname = name) + + def _getEntryNames(self, zip_name): + cmd = ['ziptool', 'zipinfo', '-1', zip_name] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + output, _ = proc.communicate() + self.assertEquals(0, proc.returncode) + self.assertNotEqual(None, output) + return output.split() + + def _ExtractEntries(self, zip_name): + temp_dir = tempfile.mkdtemp() + cmd = ['ziptool', 'unzip', '-d', temp_dir, zip_name] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + proc.communicate() + self.assertEquals(0, proc.returncode) + + def test_entriesSmallerThan2G(self): + zip_path = tempfile.NamedTemporaryFile(suffix='.zip') + # Add a few entries with each of them smaller than 2GiB. But the entire zip file is larger + # than 4GiB in size. + with zipfile.ZipFile(zip_path, 'w', allowZip64=True) as output_zip: + entry_dict = {'a.txt': 1025 * 1024, 'b.txt': 1025 * 1024, 'c.txt': 1025 * 1024, + 'd.txt': 1025 * 1024, 'e.txt': 1024} + self._AddEntriesToZip(output_zip, entry_dict) + + read_names = self._getEntryNames(zip_path.name) + self.assertEquals(sorted(entry_dict.keys()), sorted(read_names)) + self._ExtractEntries(zip_path.name) + + + def test_largeNumberOfEntries(self): + zip_path = tempfile.NamedTemporaryFile(suffix='.zip') + entry_dict = {} + # Add 100k entries (more than 65535|UINT16_MAX). + for num in range(0, 100 * 1024): + entry_dict[str(num)] = 50 + + with zipfile.ZipFile(zip_path, 'w', allowZip64=True) as output_zip: + self._AddEntriesToZip(output_zip, entry_dict) + + read_names = self._getEntryNames(zip_path.name) + self.assertEquals(sorted(entry_dict.keys()), sorted(read_names)) + self._ExtractEntries(zip_path.name) + + + def test_largeCompressedEntries(self): + zip_path = tempfile.NamedTemporaryFile(suffix='.zip') + with zipfile.ZipFile(zip_path, 'w', compression=zipfile.ZIP_DEFLATED, + allowZip64=True) as output_zip: + # Add entries close to 4GiB in size. Somehow the python library will put the (un)compressed + # sizes in the extra field. Test if our ziptool should be able to parse it. + entry_dict = {'e.txt': 4095 * 1024, 'f.txt': 4095 * 1024} + self._AddEntriesToZip(output_zip, entry_dict) + + read_names = self._getEntryNames(zip_path.name) + self.assertEquals(sorted(entry_dict.keys()), sorted(read_names)) + self._ExtractEntries(zip_path.name) + + +if __name__ == '__main__': + testsuite = unittest.TestLoader().discover( + os.path.dirname(os.path.realpath(__file__))) + unittest.TextTestRunner(verbosity=2).run(testsuite) diff --git a/libziparchive/zip_archive.cc b/libziparchive/zip_archive.cc index 445150780..849b68c1e 100644 --- a/libziparchive/zip_archive.cc +++ b/libziparchive/zip_archive.cc @@ -32,6 +32,7 @@ #include <unistd.h> #include <memory> +#include <optional> #include <vector> #if defined(__APPLE__) @@ -65,6 +66,10 @@ static const bool kCrcChecksEnabled = false; // The maximum number of bytes to scan backwards for the EOCD start. static const uint32_t kMaxEOCDSearch = kMaxCommentLen + sizeof(EocdRecord); +// Set a reasonable cap (256 GiB) for the zip file size. So the data is always valid when +// we parse the fields in cd or local headers as 64 bits signed integers. +static constexpr uint64_t kMaxFileLength = 256 * static_cast<uint64_t>(1u << 30u); + /* * A Read-only Zip archive. * @@ -85,100 +90,6 @@ static const uint32_t kMaxEOCDSearch = kMaxCommentLen + sizeof(EocdRecord); * of the string length into the hash table entry. */ -/* - * Round up to the next highest power of 2. - * - * Found on http://graphics.stanford.edu/~seander/bithacks.html. - */ -static uint32_t RoundUpPower2(uint32_t val) { - val--; - val |= val >> 1; - val |= val >> 2; - val |= val >> 4; - val |= val >> 8; - val |= val >> 16; - val++; - - return val; -} - -static uint32_t ComputeHash(std::string_view name) { - return static_cast<uint32_t>(std::hash<std::string_view>{}(name)); -} - -// Convert a ZipEntry to a hash table index, verifying that it's in a valid range. -std::pair<int32_t, uint64_t> CdEntryMapZip32::GetCdEntryOffset(std::string_view name, - const uint8_t* start) const { - const uint32_t hash = ComputeHash(name); - - // NOTE: (hash_table_size - 1) is guaranteed to be non-negative. - uint32_t ent = hash & (hash_table_size_ - 1); - while (hash_table_[ent].name_offset != 0) { - if (hash_table_[ent].ToStringView(start) == name) { - return {0, hash_table_[ent].name_offset}; - } - ent = (ent + 1) & (hash_table_size_ - 1); - } - - ALOGV("Zip: Unable to find entry %.*s", static_cast<int>(name.size()), name.data()); - return {kEntryNotFound, 0}; -} - -int32_t CdEntryMapZip32::AddToMap(std::string_view name, const uint8_t* start) { - const uint64_t hash = ComputeHash(name); - uint32_t ent = hash & (hash_table_size_ - 1); - - /* - * We over-allocated the table, so we're guaranteed to find an empty slot. - * Further, we guarantee that the hashtable size is not 0. - */ - while (hash_table_[ent].name_offset != 0) { - if (hash_table_[ent].ToStringView(start) == name) { - // We've found a duplicate entry. We don't accept duplicates. - ALOGW("Zip: Found duplicate entry %.*s", static_cast<int>(name.size()), name.data()); - return kDuplicateEntry; - } - ent = (ent + 1) & (hash_table_size_ - 1); - } - - // `name` has already been validated before entry. - const char* start_char = reinterpret_cast<const char*>(start); - hash_table_[ent].name_offset = static_cast<uint32_t>(name.data() - start_char); - hash_table_[ent].name_length = static_cast<uint16_t>(name.size()); - return 0; -} - -void CdEntryMapZip32::ResetIteration() { - current_position_ = 0; -} - -std::pair<std::string_view, uint64_t> CdEntryMapZip32::Next(const uint8_t* cd_start) { - while (current_position_ < hash_table_size_) { - const auto& entry = hash_table_[current_position_]; - current_position_ += 1; - - if (entry.name_offset != 0) { - return {entry.ToStringView(cd_start), entry.name_offset}; - } - } - // We have reached the end of the hash table. - return {}; -} - -CdEntryMapZip32::CdEntryMapZip32(uint16_t num_entries) { - hash_table_size_ = RoundUpPower2(1 + (num_entries * 4) / 3); - hash_table_ = { - reinterpret_cast<ZipStringOffset*>(calloc(hash_table_size_, sizeof(ZipStringOffset))), free}; -} - -std::unique_ptr<CdEntryMapInterface> CdEntryMapZip32::Create(uint16_t num_entries) { - auto entry_map = new CdEntryMapZip32(num_entries); - CHECK(entry_map->hash_table_ != nullptr) - << "Zip: unable to allocate the " << entry_map->hash_table_size_ - << " entry hash_table, entry size: " << sizeof(ZipStringOffset); - return std::unique_ptr<CdEntryMapInterface>(entry_map); -} - #if defined(__BIONIC__) uint64_t GetOwnerTag(const ZipArchive* archive) { return android_fdsan_create_owner_tag(ANDROID_FDSAN_OWNER_TYPE_ZIPARCHIVE, @@ -186,8 +97,8 @@ uint64_t GetOwnerTag(const ZipArchive* archive) { } #endif -ZipArchive::ZipArchive(const int fd, bool assume_ownership) - : mapped_zip(fd), +ZipArchive::ZipArchive(MappedZipFile&& map, bool assume_ownership) + : mapped_zip(map), close_file(assume_ownership), directory_offset(0), central_directory(), @@ -195,7 +106,8 @@ ZipArchive::ZipArchive(const int fd, bool assume_ownership) num_entries(0) { #if defined(__BIONIC__) if (assume_ownership) { - android_fdsan_exchange_owner_tag(fd, 0, GetOwnerTag(this)); + CHECK(mapped_zip.HasFd()); + android_fdsan_exchange_owner_tag(mapped_zip.GetFileDescriptor(), 0, GetOwnerTag(this)); } #endif } @@ -218,12 +130,81 @@ ZipArchive::~ZipArchive() { } } -static int32_t MapCentralDirectory0(const char* debug_file_name, ZipArchive* archive, - off64_t file_length, uint32_t read_amount, - uint8_t* scan_buffer) { +struct CentralDirectoryInfo { + uint64_t num_records; + // The size of the central directory (in bytes). + uint64_t cd_size; + // The offset of the start of the central directory, relative + // to the start of the file. + uint64_t cd_start_offset; +}; + +static ZipError FindCentralDirectoryInfoForZip64(const char* debugFileName, ZipArchive* archive, + off64_t eocdOffset, CentralDirectoryInfo* cdInfo) { + if (eocdOffset <= sizeof(Zip64EocdLocator)) { + ALOGW("Zip: %s: Not enough space for zip64 eocd locator", debugFileName); + return kInvalidFile; + } + // We expect to find the zip64 eocd locator immediately before the zip eocd. + const int64_t locatorOffset = eocdOffset - sizeof(Zip64EocdLocator); + Zip64EocdLocator zip64EocdLocator{}; + if (!archive->mapped_zip.ReadAtOffset(reinterpret_cast<uint8_t*>((&zip64EocdLocator)), + sizeof(Zip64EocdLocator), locatorOffset)) { + ALOGW("Zip: %s: Read %zu from offset %" PRId64 " failed %s", debugFileName, + sizeof(Zip64EocdLocator), locatorOffset, debugFileName); + return kIoError; + } + + if (zip64EocdLocator.locator_signature != Zip64EocdLocator::kSignature) { + ALOGW("Zip: %s: Zip64 eocd locator signature not found at offset %" PRId64, debugFileName, + locatorOffset); + return kInvalidFile; + } + + const int64_t zip64EocdOffset = zip64EocdLocator.zip64_eocd_offset; + if (locatorOffset <= sizeof(Zip64EocdRecord) || + zip64EocdOffset > locatorOffset - sizeof(Zip64EocdRecord)) { + ALOGW("Zip: %s: Bad zip64 eocd offset %" PRId64 ", eocd locator offset %" PRId64, debugFileName, + zip64EocdOffset, locatorOffset); + return kInvalidOffset; + } + + Zip64EocdRecord zip64EocdRecord{}; + if (!archive->mapped_zip.ReadAtOffset(reinterpret_cast<uint8_t*>(&zip64EocdRecord), + sizeof(Zip64EocdRecord), zip64EocdOffset)) { + ALOGW("Zip: %s: read %zu from offset %" PRId64 " failed %s", debugFileName, + sizeof(Zip64EocdLocator), zip64EocdOffset, debugFileName); + return kIoError; + } + + if (zip64EocdRecord.record_signature != Zip64EocdRecord::kSignature) { + ALOGW("Zip: %s: Zip64 eocd record signature not found at offset %" PRId64, debugFileName, + zip64EocdOffset); + return kInvalidFile; + } + + if (zip64EocdOffset <= zip64EocdRecord.cd_size || + zip64EocdRecord.cd_start_offset > zip64EocdOffset - zip64EocdRecord.cd_size) { + ALOGW("Zip: %s: Bad offset for zip64 central directory. cd offset %" PRIu64 ", cd size %" PRIu64 + ", zip64 eocd offset %" PRIu64, + debugFileName, zip64EocdRecord.cd_start_offset, zip64EocdRecord.cd_size, zip64EocdOffset); + return kInvalidOffset; + } + + *cdInfo = {.num_records = zip64EocdRecord.num_records, + .cd_size = zip64EocdRecord.cd_size, + .cd_start_offset = zip64EocdRecord.cd_start_offset}; + + return kSuccess; +} + +static ZipError FindCentralDirectoryInfo(const char* debug_file_name, ZipArchive* archive, + off64_t file_length, uint32_t read_amount, + CentralDirectoryInfo* cdInfo) { + std::vector<uint8_t> scan_buffer(read_amount); const off64_t search_start = file_length - read_amount; - if (!archive->mapped_zip.ReadAtOffset(scan_buffer, read_amount, search_start)) { + if (!archive->mapped_zip.ReadAtOffset(scan_buffer.data(), read_amount, search_start)) { ALOGE("Zip: read %" PRId64 " from offset %" PRId64 " failed", static_cast<int64_t>(read_amount), static_cast<int64_t>(search_start)); return kIoError; @@ -252,7 +233,7 @@ static int32_t MapCentralDirectory0(const char* debug_file_name, ZipArchive* arc } const off64_t eocd_offset = search_start + i; - const EocdRecord* eocd = reinterpret_cast<const EocdRecord*>(scan_buffer + i); + auto eocd = reinterpret_cast<const EocdRecord*>(scan_buffer.data() + i); /* * Verify that there's no trailing space at the end of the central directory * and its comment. @@ -264,6 +245,13 @@ static int32_t MapCentralDirectory0(const char* debug_file_name, ZipArchive* arc return kInvalidFile; } + // One of the field is 0xFFFFFFFF, look for the zip64 EOCD instead. + if (eocd->cd_size == UINT32_MAX || eocd->cd_start_offset == UINT32_MAX) { + ALOGV("Looking for the zip64 EOCD, cd_size: %" PRIu32 "cd_start_offset: %" PRId32, + eocd->cd_size, eocd->cd_start_offset); + return FindCentralDirectoryInfoForZip64(debug_file_name, archive, eocd_offset, cdInfo); + } + /* * Grab the CD offset and size, and the number of entries in the * archive and verify that they look reasonable. @@ -273,47 +261,29 @@ static int32_t MapCentralDirectory0(const char* debug_file_name, ZipArchive* arc eocd->cd_start_offset, eocd->cd_size, static_cast<int64_t>(eocd_offset)); return kInvalidOffset; } - if (eocd->num_records == 0) { -#if defined(__ANDROID__) - ALOGW("Zip: empty archive?"); -#endif - return kEmptyArchive; - } - - ALOGV("+++ num_entries=%" PRIu32 " dir_size=%" PRIu32 " dir_offset=%" PRIu32, eocd->num_records, - eocd->cd_size, eocd->cd_start_offset); - - // It all looks good. Create a mapping for the CD, and set the fields - // in archive. - if (!archive->InitializeCentralDirectory(static_cast<off64_t>(eocd->cd_start_offset), - static_cast<size_t>(eocd->cd_size))) { - return kMmapFailed; - } - - archive->num_entries = eocd->num_records; - archive->directory_offset = eocd->cd_start_offset; - return 0; + *cdInfo = {.num_records = eocd->num_records, + .cd_size = eocd->cd_size, + .cd_start_offset = eocd->cd_start_offset}; + return kSuccess; } /* * Find the zip Central Directory and memory-map it. * - * On success, returns 0 after populating fields from the EOCD area: + * On success, returns kSuccess after populating fields from the EOCD area: * directory_offset * directory_ptr * num_entries */ -static int32_t MapCentralDirectory(const char* debug_file_name, ZipArchive* archive) { - // Test file length. We use lseek64 to make sure the file - // is small enough to be a zip file (Its size must be less than - // 0xffffffff bytes). +static ZipError MapCentralDirectory(const char* debug_file_name, ZipArchive* archive) { + // Test file length. We use lseek64 to make sure the file is small enough to be a zip file. off64_t file_length = archive->mapped_zip.GetFileLength(); if (file_length == -1) { return kInvalidFile; } - if (file_length > static_cast<off64_t>(0xffffffff)) { + if (file_length > kMaxFileLength) { ALOGV("Zip: zip file too long %" PRId64, static_cast<int64_t>(file_length)); return kInvalidFile; } @@ -340,10 +310,130 @@ static int32_t MapCentralDirectory(const char* debug_file_name, ZipArchive* arch read_amount = static_cast<uint32_t>(file_length); } - std::vector<uint8_t> scan_buffer(read_amount); - int32_t result = - MapCentralDirectory0(debug_file_name, archive, file_length, read_amount, scan_buffer.data()); - return result; + CentralDirectoryInfo cdInfo = {}; + if (auto result = + FindCentralDirectoryInfo(debug_file_name, archive, file_length, read_amount, &cdInfo); + result != kSuccess) { + return result; + } + + if (cdInfo.num_records == 0) { +#if defined(__ANDROID__) + ALOGW("Zip: empty archive?"); +#endif + return kEmptyArchive; + } + + if (cdInfo.cd_size >= SIZE_MAX) { + ALOGW("Zip: The size of central directory doesn't fit in range of size_t: %" PRIu64, + cdInfo.cd_size); + return kInvalidFile; + } + + ALOGV("+++ num_entries=%" PRIu64 " dir_size=%" PRIu64 " dir_offset=%" PRIu64, cdInfo.num_records, + cdInfo.cd_size, cdInfo.cd_start_offset); + + // It all looks good. Create a mapping for the CD, and set the fields in archive. + if (!archive->InitializeCentralDirectory(static_cast<off64_t>(cdInfo.cd_start_offset), + static_cast<size_t>(cdInfo.cd_size))) { + return kMmapFailed; + } + + archive->num_entries = cdInfo.num_records; + archive->directory_offset = cdInfo.cd_start_offset; + + return kSuccess; +} + +static ZipError ParseZip64ExtendedInfoInExtraField( + const uint8_t* extraFieldStart, uint16_t extraFieldLength, uint32_t zip32UncompressedSize, + uint32_t zip32CompressedSize, std::optional<uint32_t> zip32LocalFileHeaderOffset, + Zip64ExtendedInfo* zip64Info) { + if (extraFieldLength <= 4) { + ALOGW("Zip: Extra field isn't large enough to hold zip64 info, size %" PRIu16, + extraFieldLength); + return kInvalidFile; + } + + // Each header MUST consist of: + // Header ID - 2 bytes + // Data Size - 2 bytes + uint16_t offset = 0; + while (offset < extraFieldLength - 4) { + auto headerId = get_unaligned<uint16_t>(extraFieldStart + offset); + auto dataSize = get_unaligned<uint16_t>(extraFieldStart + offset + 2); + + offset += 4; + if (dataSize > extraFieldLength - offset) { + ALOGW("Zip: Data size exceeds the boundary of extra field, data size %" PRIu16, dataSize); + return kInvalidOffset; + } + + // Skip the other types of extensible data fields. Details in + // https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT section 4.5 + if (headerId != Zip64ExtendedInfo::kHeaderId) { + offset += dataSize; + continue; + } + + uint16_t expectedDataSize = 0; + // We expect the extended field to include both uncompressed and compressed size. + if (zip32UncompressedSize == UINT32_MAX || zip32CompressedSize == UINT32_MAX) { + expectedDataSize += 16; + } + if (zip32LocalFileHeaderOffset == UINT32_MAX) { + expectedDataSize += 8; + } + + if (expectedDataSize == 0) { + ALOGW("Zip: Data size should not be 0 in zip64 extended field"); + return kInvalidFile; + } + + if (dataSize != expectedDataSize) { + auto localOffsetString = zip32LocalFileHeaderOffset.has_value() + ? std::to_string(zip32LocalFileHeaderOffset.value()) + : "missing"; + ALOGW("Zip: Invalid data size in zip64 extended field, expect %" PRIu16 ", get %" PRIu16 + ", uncompressed size %" PRIu32 ", compressed size %" PRIu32 ", local header offset %s", + expectedDataSize, dataSize, zip32UncompressedSize, zip32CompressedSize, + localOffsetString.c_str()); + return kInvalidFile; + } + + std::optional<uint64_t> uncompressedFileSize; + std::optional<uint64_t> compressedFileSize; + std::optional<uint64_t> localHeaderOffset; + if (zip32UncompressedSize == UINT32_MAX || zip32CompressedSize == UINT32_MAX) { + uncompressedFileSize = get_unaligned<uint64_t>(extraFieldStart + offset); + compressedFileSize = get_unaligned<uint64_t>(extraFieldStart + offset + 8); + offset += 16; + + // TODO(xunchang) Support handling file large than UINT32_MAX. It's theoretically possible + // for libz to (de)compressing file larger than UINT32_MAX. But we should use our own + // bytes counter to replace stream.total_out. + if (uncompressedFileSize.value() >= UINT32_MAX || compressedFileSize.value() >= UINT32_MAX) { + ALOGW( + "Zip: File size larger than UINT32_MAX isn't supported yet. uncompressed size %" PRIu64 + ", compressed size %" PRIu64, + uncompressedFileSize.value(), compressedFileSize.value()); + return kInvalidFile; + } + } + + if (zip32LocalFileHeaderOffset == UINT32_MAX) { + localHeaderOffset = get_unaligned<uint64_t>(extraFieldStart + offset); + offset += 8; + } + + zip64Info->uncompressed_file_size = uncompressedFileSize; + zip64Info->compressed_file_size = compressedFileSize; + zip64Info->local_header_offset = localHeaderOffset; + return kSuccess; + } + + ALOGW("Zip: zip64 extended info isn't found in the extra field."); + return kInvalidFile; } /* @@ -352,17 +442,16 @@ static int32_t MapCentralDirectory(const char* debug_file_name, ZipArchive* arch * * Returns 0 on success. */ -static int32_t ParseZipArchive(ZipArchive* archive) { +static ZipError ParseZipArchive(ZipArchive* archive) { const uint8_t* const cd_ptr = archive->central_directory.GetBasePtr(); const size_t cd_length = archive->central_directory.GetMapLength(); - const uint16_t num_entries = archive->num_entries; + const uint64_t num_entries = archive->num_entries; - /* - * Create hash table. We have a minimum 75% load factor, possibly as - * low as 50% after we round off to a power of 2. There must be at - * least one unused entry to avoid an infinite loop during creation. - */ - archive->cd_entry_map = CdEntryMapZip32::Create(num_entries); + if (num_entries <= UINT16_MAX) { + archive->cd_entry_map = CdEntryMapZip32::Create(static_cast<uint16_t>(num_entries)); + } else { + archive->cd_entry_map = CdEntryMapZip64::Create(); + } if (archive->cd_entry_map == nullptr) { return kAllocationFailed; } @@ -373,9 +462,9 @@ static int32_t ParseZipArchive(ZipArchive* archive) { */ const uint8_t* const cd_end = cd_ptr + cd_length; const uint8_t* ptr = cd_ptr; - for (uint16_t i = 0; i < num_entries; i++) { + for (uint64_t i = 0; i < num_entries; i++) { if (ptr > cd_end - sizeof(CentralDirectoryRecord)) { - ALOGW("Zip: ran off the end (item #%" PRIu16 ", %zu bytes of central directory)", i, + ALOGW("Zip: ran off the end (item #%" PRIu64 ", %zu bytes of central directory)", i, cd_length); #if defined(__ANDROID__) android_errorWriteLog(0x534e4554, "36392138"); @@ -383,16 +472,9 @@ static int32_t ParseZipArchive(ZipArchive* archive) { return kInvalidFile; } - const CentralDirectoryRecord* cdr = reinterpret_cast<const CentralDirectoryRecord*>(ptr); + auto cdr = reinterpret_cast<const CentralDirectoryRecord*>(ptr); if (cdr->record_signature != CentralDirectoryRecord::kSignature) { - ALOGW("Zip: missed a central dir sig (at %" PRIu16 ")", i); - return kInvalidFile; - } - - const off64_t local_header_offset = cdr->local_file_header_offset; - if (local_header_offset >= archive->directory_offset) { - ALOGW("Zip: bad LFH offset %" PRId64 " at entry %" PRIu16, - static_cast<int64_t>(local_header_offset), i); + ALOGW("Zip: missed a central dir sig (at %" PRIu64 ")", i); return kInvalidFile; } @@ -401,15 +483,43 @@ static int32_t ParseZipArchive(ZipArchive* archive) { const uint16_t comment_length = cdr->comment_length; const uint8_t* file_name = ptr + sizeof(CentralDirectoryRecord); - if (file_name + file_name_length > cd_end) { - ALOGW("Zip: file name for entry %" PRIu16 + if (file_name_length >= cd_length || file_name > cd_end - file_name_length) { + ALOGW("Zip: file name for entry %" PRIu64 " exceeds the central directory range, file_name_length: %" PRIu16 ", cd_length: %zu", i, file_name_length, cd_length); return kInvalidEntryName; } + + const uint8_t* extra_field = file_name + file_name_length; + if (extra_length >= cd_length || extra_field > cd_end - extra_length) { + ALOGW("Zip: extra field for entry %" PRIu64 + " exceeds the central directory range, file_name_length: %" PRIu16 ", cd_length: %zu", + i, extra_length, cd_length); + return kInvalidFile; + } + + off64_t local_header_offset = cdr->local_file_header_offset; + if (local_header_offset == UINT32_MAX) { + Zip64ExtendedInfo zip64_info{}; + if (auto status = ParseZip64ExtendedInfoInExtraField( + extra_field, extra_length, cdr->uncompressed_size, cdr->compressed_size, + cdr->local_file_header_offset, &zip64_info); + status != kSuccess) { + return status; + } + CHECK(zip64_info.local_header_offset.has_value()); + local_header_offset = zip64_info.local_header_offset.value(); + } + + if (local_header_offset >= archive->directory_offset) { + ALOGW("Zip: bad LFH offset %" PRId64 " at entry %" PRIu64, + static_cast<int64_t>(local_header_offset), i); + return kInvalidFile; + } + // Check that file name is valid UTF-8 and doesn't contain NUL (U+0000) characters. if (!IsValidEntryName(file_name, file_name_length)) { - ALOGW("Zip: invalid file name at entry %" PRIu16, i); + ALOGW("Zip: invalid file name at entry %" PRIu64, i); return kInvalidEntryName; } @@ -424,7 +534,7 @@ static int32_t ParseZipArchive(ZipArchive* archive) { ptr += sizeof(CentralDirectoryRecord) + file_name_length + extra_length + comment_length; if ((ptr - cd_ptr) > static_cast<int64_t>(cd_length)) { - ALOGW("Zip: bad CD advance (%tu vs %zu) at entry %" PRIu16, ptr - cd_ptr, cd_length, i); + ALOGW("Zip: bad CD advance (%tu vs %zu) at entry %" PRIu64, ptr - cd_ptr, cd_length, i); return kInvalidFile; } } @@ -444,26 +554,44 @@ static int32_t ParseZipArchive(ZipArchive* archive) { return kInvalidFile; } - ALOGV("+++ zip good scan %" PRIu16 " entries", num_entries); + ALOGV("+++ zip good scan %" PRIu64 " entries", num_entries); - return 0; + return kSuccess; } static int32_t OpenArchiveInternal(ZipArchive* archive, const char* debug_file_name) { int32_t result = MapCentralDirectory(debug_file_name, archive); - return result != 0 ? result : ParseZipArchive(archive); + return result != kSuccess ? result : ParseZipArchive(archive); } int32_t OpenArchiveFd(int fd, const char* debug_file_name, ZipArchiveHandle* handle, bool assume_ownership) { - ZipArchive* archive = new ZipArchive(fd, assume_ownership); + ZipArchive* archive = new ZipArchive(MappedZipFile(fd), assume_ownership); *handle = archive; return OpenArchiveInternal(archive, debug_file_name); } +int32_t OpenArchiveFdRange(int fd, const char* debug_file_name, ZipArchiveHandle* handle, + off64_t length, off64_t offset, bool assume_ownership) { + ZipArchive* archive = new ZipArchive(MappedZipFile(fd, length, offset), assume_ownership); + *handle = archive; + + if (length < 0) { + ALOGW("Invalid zip length %" PRId64, length); + return kIoError; + } + + if (offset < 0) { + ALOGW("Invalid zip offset %" PRId64, offset); + return kIoError; + } + + return OpenArchiveInternal(archive, debug_file_name); +} + int32_t OpenArchive(const char* fileName, ZipArchiveHandle* handle) { const int fd = ::android::base::utf8::open(fileName, O_RDONLY | O_BINARY | O_CLOEXEC, 0); - ZipArchive* archive = new ZipArchive(fd, true); + ZipArchive* archive = new ZipArchive(MappedZipFile(fd), true); *handle = archive; if (fd < 0) { @@ -545,7 +673,7 @@ static int32_t FindEntry(const ZipArchive* archive, std::string_view entryName, return kInvalidOffset; } - const CentralDirectoryRecord* cdr = reinterpret_cast<const CentralDirectoryRecord*>(ptr); + auto cdr = reinterpret_cast<const CentralDirectoryRecord*>(ptr); // The offset of the start of the central directory in the zipfile. // We keep this lying around so that we can sanity check all our lengths @@ -564,7 +692,34 @@ static int32_t FindEntry(const ZipArchive* archive, std::string_view entryName, // Figure out the local header offset from the central directory. The // actual file data will begin after the local header and the name / // extra comments. - const off64_t local_header_offset = cdr->local_file_header_offset; + off64_t local_header_offset = cdr->local_file_header_offset; + // One of the info field is UINT32_MAX, try to parse the real value in the zip64 extended info in + // the extra field. + if (cdr->uncompressed_size == UINT32_MAX || cdr->compressed_size == UINT32_MAX || + cdr->local_file_header_offset == UINT32_MAX) { + const uint8_t* extra_field = ptr + sizeof(CentralDirectoryRecord) + cdr->file_name_length; + Zip64ExtendedInfo zip64_info{}; + if (auto status = ParseZip64ExtendedInfoInExtraField( + extra_field, cdr->extra_field_length, cdr->uncompressed_size, cdr->compressed_size, + cdr->local_file_header_offset, &zip64_info); + status != kSuccess) { + return status; + } + + if (cdr->uncompressed_size == UINT32_MAX || cdr->compressed_size == UINT32_MAX) { + CHECK(zip64_info.uncompressed_file_size.has_value()); + CHECK(zip64_info.compressed_file_size.has_value()); + // TODO(xunchang) remove the size limit and support entry length > UINT32_MAX. + data->uncompressed_length = static_cast<uint32_t>(zip64_info.uncompressed_file_size.value()); + data->compressed_length = static_cast<uint32_t>(zip64_info.compressed_file_size.value()); + } + + if (local_header_offset == UINT32_MAX) { + CHECK(zip64_info.local_header_offset.has_value()); + local_header_offset = zip64_info.local_header_offset.value(); + } + } + if (local_header_offset + static_cast<off64_t>(sizeof(LocalFileHeader)) >= cd_offset) { ALOGW("Zip: bad local hdr offset in zip"); return kInvalidOffset; @@ -577,14 +732,68 @@ static int32_t FindEntry(const ZipArchive* archive, std::string_view entryName, return kIoError; } - const LocalFileHeader* lfh = reinterpret_cast<const LocalFileHeader*>(lfh_buf); - + auto lfh = reinterpret_cast<const LocalFileHeader*>(lfh_buf); if (lfh->lfh_signature != LocalFileHeader::kSignature) { ALOGW("Zip: didn't find signature at start of lfh, offset=%" PRId64, static_cast<int64_t>(local_header_offset)); return kInvalidOffset; } + // Check that the local file header name matches the declared name in the central directory. + CHECK_LE(entryName.size(), UINT16_MAX); + auto nameLen = static_cast<uint16_t>(entryName.size()); + if (lfh->file_name_length != nameLen) { + ALOGW("Zip: lfh name length did not match central directory for %s: %" PRIu16 " %" PRIu16, + std::string(entryName).c_str(), lfh->file_name_length, nameLen); + return kInconsistentInformation; + } + const off64_t name_offset = local_header_offset + sizeof(LocalFileHeader); + if (name_offset > cd_offset - lfh->file_name_length) { + ALOGW("Zip: lfh name has invalid declared length"); + return kInvalidOffset; + } + + std::vector<uint8_t> name_buf(nameLen); + if (!archive->mapped_zip.ReadAtOffset(name_buf.data(), nameLen, name_offset)) { + ALOGW("Zip: failed reading lfh name from offset %" PRId64, static_cast<int64_t>(name_offset)); + return kIoError; + } + if (memcmp(entryName.data(), name_buf.data(), nameLen) != 0) { + ALOGW("Zip: lfh name did not match central directory"); + return kInconsistentInformation; + } + + uint64_t lfh_uncompressed_size = lfh->uncompressed_size; + uint64_t lfh_compressed_size = lfh->compressed_size; + if (lfh_uncompressed_size == UINT32_MAX || lfh_compressed_size == UINT32_MAX) { + const off64_t lfh_extra_field_offset = name_offset + lfh->file_name_length; + const uint16_t lfh_extra_field_size = lfh->extra_field_length; + if (lfh_extra_field_offset > cd_offset - lfh_extra_field_size) { + ALOGW("Zip: extra field has a bad size for entry %s", std::string(entryName).c_str()); + return kInvalidOffset; + } + + std::vector<uint8_t> local_extra_field(lfh_extra_field_size); + if (!archive->mapped_zip.ReadAtOffset(local_extra_field.data(), lfh_extra_field_size, + lfh_extra_field_offset)) { + ALOGW("Zip: failed reading lfh extra field from offset %" PRId64, lfh_extra_field_offset); + return kIoError; + } + + Zip64ExtendedInfo zip64_info{}; + if (auto status = ParseZip64ExtendedInfoInExtraField( + local_extra_field.data(), lfh_extra_field_size, lfh->uncompressed_size, + lfh->compressed_size, std::nullopt, &zip64_info); + status != kSuccess) { + return status; + } + + CHECK(zip64_info.uncompressed_file_size.has_value()); + CHECK(zip64_info.compressed_file_size.has_value()); + lfh_uncompressed_size = zip64_info.uncompressed_file_size.value(); + lfh_compressed_size = zip64_info.compressed_file_size.value(); + } + // Paranoia: Match the values specified in the local file header // to those specified in the central directory. @@ -610,12 +819,12 @@ static int32_t FindEntry(const ZipArchive* archive, std::string_view entryName, // header agree on the crc, compressed, and uncompressed sizes of the entry. if ((lfh->gpb_flags & kGPBDDFlagMask) == 0) { data->has_data_descriptor = 0; - if (data->compressed_length != lfh->compressed_size || - data->uncompressed_length != lfh->uncompressed_size || data->crc32 != lfh->crc32) { + if (data->compressed_length != lfh_compressed_size || + data->uncompressed_length != lfh_uncompressed_size || data->crc32 != lfh->crc32) { ALOGW("Zip: size/crc32 mismatch. expected {%" PRIu32 ", %" PRIu32 ", %" PRIx32 - "}, was {%" PRIu32 ", %" PRIu32 ", %" PRIx32 "}", - data->compressed_length, data->uncompressed_length, data->crc32, lfh->compressed_size, - lfh->uncompressed_size, lfh->crc32); + "}, was {%" PRIu64 ", %" PRIu64 ", %" PRIx32 "}", + data->compressed_length, data->uncompressed_length, data->crc32, lfh_compressed_size, + lfh_uncompressed_size, lfh->crc32); return kInconsistentInformation; } } else { @@ -638,30 +847,6 @@ static int32_t FindEntry(const ZipArchive* archive, std::string_view entryName, // Currently only needed to implement zipinfo. data->is_text = (cdr->internal_file_attributes & 1); - // Check that the local file header name matches the declared - // name in the central directory. - CHECK_LE(entryName.size(), UINT16_MAX); - auto nameLen = static_cast<uint16_t>(entryName.size()); - if (lfh->file_name_length != nameLen) { - ALOGW("Zip: lfh name length did not match central directory for %s: %" PRIu16 " %" PRIu16, - std::string(entryName).c_str(), lfh->file_name_length, nameLen); - return kInconsistentInformation; - } - const off64_t name_offset = local_header_offset + sizeof(LocalFileHeader); - if (name_offset + lfh->file_name_length > cd_offset) { - ALOGW("Zip: lfh name has invalid declared length"); - return kInvalidOffset; - } - std::vector<uint8_t> name_buf(nameLen); - if (!archive->mapped_zip.ReadAtOffset(name_buf.data(), nameLen, name_offset)) { - ALOGW("Zip: failed reading lfh name from offset %" PRId64, static_cast<int64_t>(name_offset)); - return kIoError; - } - if (memcmp(entryName.data(), name_buf.data(), nameLen) != 0) { - ALOGW("Zip: lfh name did not match central directory"); - return kInconsistentInformation; - } - const off64_t data_offset = local_header_offset + sizeof(LocalFileHeader) + lfh->file_name_length + lfh->extra_field_length; if (data_offset > cd_offset) { @@ -691,31 +876,40 @@ static int32_t FindEntry(const ZipArchive* archive, std::string_view entryName, struct IterationHandle { ZipArchive* archive; - std::string prefix; - std::string suffix; + std::function<bool(std::string_view)> matcher; uint32_t position = 0; - IterationHandle(ZipArchive* archive, std::string_view in_prefix, std::string_view in_suffix) - : archive(archive), prefix(in_prefix), suffix(in_suffix) {} + IterationHandle(ZipArchive* archive, std::function<bool(std::string_view)> in_matcher) + : archive(archive), matcher(std::move(in_matcher)) {} + + bool Match(std::string_view entry_name) const { return matcher(entry_name); } }; int32_t StartIteration(ZipArchiveHandle archive, void** cookie_ptr, const std::string_view optional_prefix, const std::string_view optional_suffix) { - if (archive == nullptr || archive->cd_entry_map == nullptr) { - ALOGW("Zip: Invalid ZipArchiveHandle"); - return kInvalidHandle; - } - if (optional_prefix.size() > static_cast<size_t>(UINT16_MAX) || optional_suffix.size() > static_cast<size_t>(UINT16_MAX)) { ALOGW("Zip: prefix/suffix too long"); return kInvalidEntryName; } + auto matcher = [prefix = std::string(optional_prefix), + suffix = std::string(optional_suffix)](std::string_view name) mutable { + return android::base::StartsWith(name, prefix) && android::base::EndsWith(name, suffix); + }; + return StartIteration(archive, cookie_ptr, std::move(matcher)); +} + +int32_t StartIteration(ZipArchiveHandle archive, void** cookie_ptr, + std::function<bool(std::string_view)> matcher) { + if (archive == nullptr || archive->cd_entry_map == nullptr) { + ALOGW("Zip: Invalid ZipArchiveHandle"); + return kInvalidHandle; + } archive->cd_entry_map->ResetIteration(); - *cookie_ptr = new IterationHandle(archive, optional_prefix, optional_suffix); + *cookie_ptr = new IterationHandle(archive, matcher); return 0; } @@ -765,8 +959,7 @@ int32_t Next(void* cookie, ZipEntry* data, std::string_view* name) { auto entry = archive->cd_entry_map->Next(archive->central_directory.GetBasePtr()); while (entry != std::pair<std::string_view, uint64_t>()) { const auto [entry_name, offset] = entry; - if (android::base::StartsWith(entry_name, handle->prefix) && - android::base::EndsWith(entry_name, handle->suffix)) { + if (handle->Match(entry_name)) { const int error = FindEntry(archive, entry_name, offset, data); if (!error && name) { *name = entry_name; @@ -1120,24 +1313,14 @@ int32_t ExtractEntryToFile(ZipArchiveHandle archive, ZipEntry* entry, int fd) { return ExtractToWriter(archive, entry, &writer); } -const char* ErrorCodeString(int32_t error_code) { - // Make sure that the number of entries in kErrorMessages and ErrorCodes - // match. - static_assert((-kLastErrorCode + 1) == arraysize(kErrorMessages), - "(-kLastErrorCode + 1) != arraysize(kErrorMessages)"); - - const uint32_t idx = -error_code; - if (idx < arraysize(kErrorMessages)) { - return kErrorMessages[idx]; - } - - return "Unknown return code"; -} - int GetFileDescriptor(const ZipArchiveHandle archive) { return archive->mapped_zip.GetFileDescriptor(); } +off64_t GetFileDescriptorOffset(const ZipArchiveHandle archive) { + return archive->mapped_zip.GetFileOffset(); +} + #if !defined(_WIN32) class ProcessWriter : public zip_archive::Writer { public: @@ -1177,31 +1360,65 @@ const void* MappedZipFile::GetBasePtr() const { return base_ptr_; } +off64_t MappedZipFile::GetFileOffset() const { + return fd_offset_; +} + off64_t MappedZipFile::GetFileLength() const { if (has_fd_) { - off64_t result = lseek64(fd_, 0, SEEK_END); - if (result == -1) { + if (data_length_ != -1) { + return data_length_; + } + data_length_ = lseek64(fd_, 0, SEEK_END); + if (data_length_ == -1) { ALOGE("Zip: lseek on fd %d failed: %s", fd_, strerror(errno)); } - return result; + return data_length_; } else { if (base_ptr_ == nullptr) { ALOGE("Zip: invalid file map"); return -1; } - return static_cast<off64_t>(data_length_); + return data_length_; } } // Attempts to read |len| bytes into |buf| at offset |off|. bool MappedZipFile::ReadAtOffset(uint8_t* buf, size_t len, off64_t off) const { if (has_fd_) { - if (!android::base::ReadFullyAtOffset(fd_, buf, len, off)) { + if (off < 0) { + ALOGE("Zip: invalid offset %" PRId64, off); + return false; + } + + off64_t read_offset; + if (__builtin_add_overflow(fd_offset_, off, &read_offset)) { + ALOGE("Zip: invalid read offset %" PRId64 " overflows, fd offset %" PRId64, off, fd_offset_); + return false; + } + + if (data_length_ != -1) { + off64_t read_end; + if (len > std::numeric_limits<off64_t>::max() || + __builtin_add_overflow(off, static_cast<off64_t>(len), &read_end)) { + ALOGE("Zip: invalid read length %" PRId64 " overflows, offset %" PRId64, + static_cast<off64_t>(len), off); + return false; + } + + if (read_end > data_length_) { + ALOGE("Zip: invalid read length %" PRId64 " exceeds data length %" PRId64 ", offset %" + PRId64, static_cast<off64_t>(len), data_length_, off); + return false; + } + } + + if (!android::base::ReadFullyAtOffset(fd_, buf, len, read_offset)) { ALOGE("Zip: failed to read at offset %" PRId64, off); return false; } } else { - if (off < 0 || off > static_cast<off64_t>(data_length_)) { + if (off < 0 || off > data_length_) { ALOGE("Zip: invalid offset: %" PRId64 ", data length: %" PRId64, off, data_length_); return false; } @@ -1219,7 +1436,8 @@ void CentralDirectory::Initialize(const void* map_base_ptr, off64_t cd_start_off bool ZipArchive::InitializeCentralDirectory(off64_t cd_start_offset, size_t cd_size) { if (mapped_zip.HasFd()) { directory_map = android::base::MappedFile::FromFd(mapped_zip.GetFileDescriptor(), - cd_start_offset, cd_size, PROT_READ); + mapped_zip.GetFileOffset() + cd_start_offset, + cd_size, PROT_READ); if (!directory_map) { ALOGE("Zip: failed to map central directory (offset %" PRId64 ", size %zu): %s", cd_start_offset, cd_size, strerror(errno)); diff --git a/libziparchive/zip_archive_common.h b/libziparchive/zip_archive_common.h index 8b99bde91..a92d4d27c 100644 --- a/libziparchive/zip_archive_common.h +++ b/libziparchive/zip_archive_common.h @@ -21,6 +21,8 @@ #include <inttypes.h> +#include <optional> + // The "end of central directory" (EOCD) record. Each archive // contains exactly once such record which appears at the end of // the archive. It contains archive wide information like the @@ -173,6 +175,89 @@ struct DataDescriptor { DISALLOW_COPY_AND_ASSIGN(DataDescriptor); } __attribute__((packed)); +// The zip64 end of central directory locator helps to find the zip64 EOCD. +struct Zip64EocdLocator { + static constexpr uint32_t kSignature = 0x07064b50; + + // The signature of zip64 eocd locator, must be |kSignature| + uint32_t locator_signature; + // The start disk of the zip64 eocd. This implementation assumes that each + // archive spans a single disk only. + uint32_t eocd_start_disk; + // The offset offset of the zip64 end of central directory record. + uint64_t zip64_eocd_offset; + // The total number of disks. This implementation assumes that each archive + // spans a single disk only. + uint32_t num_of_disks; + + private: + Zip64EocdLocator() = default; + DISALLOW_COPY_AND_ASSIGN(Zip64EocdLocator); +} __attribute__((packed)); + +// The optional zip64 EOCD. If one of the fields in the end of central directory +// record is too small to hold required data, the field SHOULD be set to -1 +// (0xFFFF or 0xFFFFFFFF) and the ZIP64 format record SHOULD be created. +struct Zip64EocdRecord { + static constexpr uint32_t kSignature = 0x06064b50; + + // The signature of zip64 eocd record, must be |kSignature| + uint32_t record_signature; + // Size of zip64 end of central directory record. It SHOULD be the size of the + // remaining record and SHOULD NOT include the leading 12 bytes. + uint64_t record_size; + // The version of the tool that make this archive. + uint16_t version_made_by; + // Tool version needed to extract this archive. + uint16_t version_needed; + // Number of this disk. + uint32_t disk_num; + // Number of the disk with the start of the central directory. + uint32_t cd_start_disk; + // Total number of entries in the central directory on this disk. + // This implementation assumes that each archive spans a single + // disk only. i.e, that num_records_on_disk == num_records. + uint64_t num_records_on_disk; + // The total number of central directory records. + uint64_t num_records; + // The size of the central directory in bytes. + uint64_t cd_size; + // The offset of the start of the central directory, relative to the start of + // the file. + uint64_t cd_start_offset; + + private: + Zip64EocdRecord() = default; + DISALLOW_COPY_AND_ASSIGN(Zip64EocdRecord); +} __attribute__((packed)); + +// The possible contents of the Zip64 Extended Information Extra Field. It may appear in +// the 'extra' field of a central directory record or local file header. The order of +// the fields in the zip64 extended information record is fixed, but the fields MUST +// only appear if the corresponding local or central directory record field is set to +// 0xFFFF or 0xFFFFFFFF. And this entry in the Local header MUST include BOTH original +// and compressed file size fields. +struct Zip64ExtendedInfo { + static constexpr uint16_t kHeaderId = 0x0001; + // The header tag for this 'extra' block, should be |kHeaderId|. + uint16_t header_id; + // The size in bytes of the remaining data (excluding the top 4 bytes). + uint16_t data_size; + // Size in bytes of the uncompressed file. + std::optional<uint64_t> uncompressed_file_size; + // Size in bytes of the compressed file. + std::optional<uint64_t> compressed_file_size; + // Local file header offset relative to the start of the zip file. + std::optional<uint64_t> local_header_offset; + + // This implementation assumes that each archive spans a single disk only. So + // the disk_number is not used. + // uint32_t disk_num; + private: + Zip64ExtendedInfo() = default; + DISALLOW_COPY_AND_ASSIGN(Zip64ExtendedInfo); +}; + // mask value that signifies that the entry has a DD static const uint32_t kGPBDDFlagMask = 0x0008; diff --git a/libziparchive/zip_archive_private.h b/libziparchive/zip_archive_private.h index 68977f6a1..5f5232f94 100644 --- a/libziparchive/zip_archive_private.h +++ b/libziparchive/zip_archive_private.h @@ -28,80 +28,20 @@ #include "android-base/macros.h" #include "android-base/mapped_file.h" - -static const char* kErrorMessages[] = { - "Success", - "Iteration ended", - "Zlib error", - "Invalid file", - "Invalid handle", - "Duplicate entries in archive", - "Empty archive", - "Entry not found", - "Invalid offset", - "Inconsistent information", - "Invalid entry name", - "I/O error", - "File mapping failed", - "Allocation failed", -}; - -enum ErrorCodes : int32_t { - kIterationEnd = -1, - - // We encountered a Zlib error when inflating a stream from this file. - // Usually indicates file corruption. - kZlibError = -2, - - // The input file cannot be processed as a zip archive. Usually because - // it's too small, too large or does not have a valid signature. - kInvalidFile = -3, - - // An invalid iteration / ziparchive handle was passed in as an input - // argument. - kInvalidHandle = -4, - - // The zip archive contained two (or possibly more) entries with the same - // name. - kDuplicateEntry = -5, - - // The zip archive contains no entries. - kEmptyArchive = -6, - - // The specified entry was not found in the archive. - kEntryNotFound = -7, - - // The zip archive contained an invalid local file header pointer. - kInvalidOffset = -8, - - // The zip archive contained inconsistent entry information. This could - // be because the central directory & local file header did not agree, or - // if the actual uncompressed length or crc32 do not match their declared - // values. - kInconsistentInformation = -9, - - // An invalid entry name was encountered. - kInvalidEntryName = -10, - - // An I/O related system call (read, lseek, ftruncate, map) failed. - kIoError = -11, - - // We were not able to mmap the central directory or entry contents. - kMmapFailed = -12, - - // An allocation failed. - kAllocationFailed = -13, - - kLastErrorCode = kAllocationFailed, -}; +#include "zip_cd_entry_map.h" +#include "zip_error.h" class MappedZipFile { public: explicit MappedZipFile(const int fd) - : has_fd_(true), fd_(fd), base_ptr_(nullptr), data_length_(0) {} + : has_fd_(true), fd_(fd), fd_offset_(0), base_ptr_(nullptr), data_length_(-1) {} + + explicit MappedZipFile(const int fd, off64_t length, off64_t offset) + : has_fd_(true), fd_(fd), fd_offset_(offset), base_ptr_(nullptr), data_length_(length) {} explicit MappedZipFile(const void* address, size_t length) - : has_fd_(false), fd_(-1), base_ptr_(address), data_length_(static_cast<off64_t>(length)) {} + : has_fd_(false), fd_(-1), fd_offset_(0), base_ptr_(address), + data_length_(static_cast<off64_t>(length)) {} bool HasFd() const { return has_fd_; } @@ -109,6 +49,8 @@ class MappedZipFile { const void* GetBasePtr() const; + off64_t GetFileOffset() const; + off64_t GetFileLength() const; bool ReadAtOffset(uint8_t* buf, size_t len, off64_t off) const; @@ -121,9 +63,10 @@ class MappedZipFile { const bool has_fd_; const int fd_; + const off64_t fd_offset_; const void* const base_ptr_; - const off64_t data_length_; + mutable off64_t data_length_; }; class CentralDirectory { @@ -141,75 +84,6 @@ class CentralDirectory { size_t length_; }; -// This class is the interface of the central directory entries map. The map -// helps to locate a particular cd entry based on the filename. -class CdEntryMapInterface { - public: - virtual ~CdEntryMapInterface() = default; - // Adds an entry to the map. The |name| should internally points to the - // filename field of a cd entry. And |start| points to the beginning of the - // central directory. Returns 0 on success. - virtual int32_t AddToMap(std::string_view name, const uint8_t* start) = 0; - // For the zip entry |entryName|, finds the offset of its filename field in - // the central directory. Returns a pair of [status, offset]. The value of - // the status is 0 on success. - virtual std::pair<int32_t, uint64_t> GetCdEntryOffset(std::string_view name, - const uint8_t* cd_start) const = 0; - // Resets the iterator to the beginning of the map. - virtual void ResetIteration() = 0; - // Returns the [name, cd offset] of the current element. Also increments the - // iterator to points to the next element. Returns an empty pair we have read - // past boundary. - virtual std::pair<std::string_view, uint64_t> Next(const uint8_t* cd_start) = 0; -}; - -/** - * More space efficient string representation of strings in an mmaped zipped - * file than std::string_view. Using std::string_view as an entry in the - * ZipArchive hash table wastes space. std::string_view stores a pointer to a - * string (on 64 bit, 8 bytes) and the length to read from that pointer, - * 2 bytes. Because of alignment, the structure consumes 16 bytes, wasting - * 6 bytes. - * - * ZipStringOffset stores a 4 byte offset from a fixed location in the memory - * mapped file instead of the entire address, consuming 8 bytes with alignment. - */ -struct ZipStringOffset { - uint32_t name_offset; - uint16_t name_length; - - const std::string_view ToStringView(const uint8_t* start) const { - return std::string_view{reinterpret_cast<const char*>(start + name_offset), name_length}; - } -}; - -// This implementation of CdEntryMap uses an array hash table. It uses less -// memory than std::map; and it's used as the default implementation for zip -// archives without zip64 extension. -class CdEntryMapZip32 : public CdEntryMapInterface { - public: - static std::unique_ptr<CdEntryMapInterface> Create(uint16_t num_entries); - - int32_t AddToMap(std::string_view name, const uint8_t* start) override; - std::pair<int32_t, uint64_t> GetCdEntryOffset(std::string_view name, - const uint8_t* cd_start) const override; - void ResetIteration() override; - std::pair<std::string_view, uint64_t> Next(const uint8_t* cd_start) override; - - private: - explicit CdEntryMapZip32(uint16_t num_entries); - - // We know how many entries are in the Zip archive, so we can have a - // fixed-size hash table. We define a load factor of 0.75 and over - // allocate so the maximum number entries can never be higher than - // ((4 * UINT16_MAX) / 3 + 1) which can safely fit into a uint32_t. - uint32_t hash_table_size_{0}; - std::unique_ptr<ZipStringOffset[], decltype(&free)> hash_table_{nullptr, free}; - - // The position of element for the current iteration. - uint32_t current_position_{0}; -}; - struct ZipArchive { // open Zip archive mutable MappedZipFile mapped_zip; @@ -221,10 +95,10 @@ struct ZipArchive { std::unique_ptr<android::base::MappedFile> directory_map; // number of entries in the Zip archive - uint16_t num_entries; + uint64_t num_entries; std::unique_ptr<CdEntryMapInterface> cd_entry_map; - ZipArchive(const int fd, bool assume_ownership); + ZipArchive(MappedZipFile&& map, bool assume_ownership); ZipArchive(const void* address, size_t length); ~ZipArchive(); diff --git a/libziparchive/zip_archive_test.cc b/libziparchive/zip_archive_test.cc index 091630407..69be3dfdc 100644 --- a/libziparchive/zip_archive_test.cc +++ b/libziparchive/zip_archive_test.cc @@ -14,8 +14,6 @@ * limitations under the License. */ -#include "zip_archive_private.h" - #include <errno.h> #include <fcntl.h> #include <getopt.h> @@ -23,17 +21,25 @@ #include <string.h> #include <unistd.h> +#include <map> #include <memory> +#include <set> +#include <string_view> #include <vector> #include <android-base/file.h> #include <android-base/logging.h> #include <android-base/mapped_file.h> +#include <android-base/memory.h> +#include <android-base/strings.h> #include <android-base/unique_fd.h> #include <gtest/gtest.h> #include <ziparchive/zip_archive.h> #include <ziparchive/zip_archive_stream_entry.h> +#include "zip_archive_common.h" +#include "zip_archive_private.h" + static std::string test_data_dir = android::base::GetExecutableDirectory() + "/testdata"; static const std::string kValidZip = "valid.zip"; @@ -53,6 +59,76 @@ static int32_t OpenArchiveWrapper(const std::string& name, ZipArchiveHandle* han return OpenArchive(abs_path.c_str(), handle); } +class CdEntryMapTest : public ::testing::Test { + protected: + void SetUp() override { + names_ = { + "a.txt", "b.txt", "b/", "b/c.txt", "b/d.txt", + }; + separator_ = "separator"; + header_ = "metadata"; + joined_names_ = header_ + android::base::Join(names_, separator_); + base_ptr_ = reinterpret_cast<uint8_t*>(&joined_names_[0]); + + entry_maps_.emplace_back(CdEntryMapZip32::Create(static_cast<uint16_t>(names_.size()))); + entry_maps_.emplace_back(CdEntryMapZip64::Create()); + for (auto& cd_map : entry_maps_) { + ASSERT_NE(nullptr, cd_map); + size_t offset = header_.size(); + for (const auto& name : names_) { + auto status = cd_map->AddToMap( + std::string_view{joined_names_.c_str() + offset, name.size()}, base_ptr_); + ASSERT_EQ(0, status); + offset += name.size() + separator_.size(); + } + } + } + + std::vector<std::string> names_; + // A continuous region of memory serves as a mock of the central directory. + std::string joined_names_; + // We expect some metadata at the beginning of the central directory and between filenames. + std::string header_; + std::string separator_; + + std::vector<std::unique_ptr<CdEntryMapInterface>> entry_maps_; + uint8_t* base_ptr_{nullptr}; // Points to the start of the central directory. +}; + +TEST_F(CdEntryMapTest, AddDuplicatedEntry) { + for (auto& cd_map : entry_maps_) { + std::string_view name = "b.txt"; + ASSERT_NE(0, cd_map->AddToMap(name, base_ptr_)); + } +} + +TEST_F(CdEntryMapTest, FindEntry) { + for (auto& cd_map : entry_maps_) { + uint64_t expected_offset = header_.size(); + for (const auto& name : names_) { + auto [status, offset] = cd_map->GetCdEntryOffset(name, base_ptr_); + ASSERT_EQ(status, kSuccess); + ASSERT_EQ(offset, expected_offset); + expected_offset += name.size() + separator_.size(); + } + } +} + +TEST_F(CdEntryMapTest, Iteration) { + std::set<std::string_view> expected(names_.begin(), names_.end()); + for (auto& cd_map : entry_maps_) { + cd_map->ResetIteration(); + std::set<std::string_view> entry_set; + auto ret = cd_map->Next(base_ptr_); + while (ret != std::pair<std::string_view, uint64_t>{}) { + auto [it, insert_status] = entry_set.insert(ret.first); + ASSERT_TRUE(insert_status); + ret = cd_map->Next(base_ptr_); + } + ASSERT_EQ(expected, entry_set); + } +} + TEST(ziparchive, Open) { ZipArchiveHandle handle; ASSERT_EQ(0, OpenArchiveWrapper(kValidZip, &handle)); @@ -108,6 +184,32 @@ TEST(ziparchive, OpenDoNotAssumeFdOwnership) { close(fd); } +TEST(ziparchive, OpenAssumeFdRangeOwnership) { + int fd = open((test_data_dir + "/" + kValidZip).c_str(), O_RDONLY | O_BINARY); + ASSERT_NE(-1, fd); + const off64_t length = lseek64(fd, 0, SEEK_END); + ASSERT_NE(-1, length); + ZipArchiveHandle handle; + ASSERT_EQ(0, OpenArchiveFdRange(fd, "OpenWithAssumeFdOwnership", &handle, + static_cast<size_t>(length), 0)); + CloseArchive(handle); + ASSERT_EQ(-1, lseek(fd, 0, SEEK_SET)); + ASSERT_EQ(EBADF, errno); +} + +TEST(ziparchive, OpenDoNotAssumeFdRangeOwnership) { + int fd = open((test_data_dir + "/" + kValidZip).c_str(), O_RDONLY | O_BINARY); + ASSERT_NE(-1, fd); + const off64_t length = lseek(fd, 0, SEEK_END); + ASSERT_NE(-1, length); + ZipArchiveHandle handle; + ASSERT_EQ(0, OpenArchiveFdRange(fd, "OpenWithAssumeFdOwnership", &handle, + static_cast<size_t>(length), 0, false)); + CloseArchive(handle); + ASSERT_EQ(0, lseek(fd, 0, SEEK_SET)); + close(fd); +} + TEST(ziparchive, Iteration_std_string_view) { ZipArchiveHandle handle; ASSERT_EQ(0, OpenArchiveWrapper(kValidZip, &handle)); @@ -128,32 +230,44 @@ TEST(ziparchive, Iteration_std_string_view) { CloseArchive(handle); } -static void AssertIterationOrder(const std::string_view prefix, const std::string_view suffix, +static void AssertIterationNames(void* iteration_cookie, const std::vector<std::string>& expected_names_sorted) { - ZipArchiveHandle handle; - ASSERT_EQ(0, OpenArchiveWrapper(kValidZip, &handle)); - - void* iteration_cookie; - ASSERT_EQ(0, StartIteration(handle, &iteration_cookie, prefix, suffix)); - ZipEntry data; std::vector<std::string> names; - std::string name; for (size_t i = 0; i < expected_names_sorted.size(); ++i) { ASSERT_EQ(0, Next(iteration_cookie, &data, &name)); names.push_back(name); } - // End of iteration. ASSERT_EQ(-1, Next(iteration_cookie, &data, &name)); - CloseArchive(handle); - // Assert that the names are as expected. std::sort(names.begin(), names.end()); ASSERT_EQ(expected_names_sorted, names); } +static void AssertIterationOrder(const std::string_view prefix, const std::string_view suffix, + const std::vector<std::string>& expected_names_sorted) { + ZipArchiveHandle handle; + ASSERT_EQ(0, OpenArchiveWrapper(kValidZip, &handle)); + + void* iteration_cookie; + ASSERT_EQ(0, StartIteration(handle, &iteration_cookie, prefix, suffix)); + AssertIterationNames(iteration_cookie, expected_names_sorted); + CloseArchive(handle); +} + +static void AssertIterationOrderWithMatcher(std::function<bool(std::string_view)> matcher, + const std::vector<std::string>& expected_names_sorted) { + ZipArchiveHandle handle; + ASSERT_EQ(0, OpenArchiveWrapper(kValidZip, &handle)); + + void* iteration_cookie; + ASSERT_EQ(0, StartIteration(handle, &iteration_cookie, matcher)); + AssertIterationNames(iteration_cookie, expected_names_sorted); + CloseArchive(handle); +} + TEST(ziparchive, Iteration) { static const std::vector<std::string> kExpectedMatchesSorted = {"a.txt", "b.txt", "b/", "b/c.txt", "b/d.txt"}; @@ -180,6 +294,30 @@ TEST(ziparchive, IterationWithPrefixAndSuffix) { AssertIterationOrder("b", ".txt", kExpectedMatchesSorted); } +TEST(ziparchive, IterationWithAdditionalMatchesExactly) { + static const std::vector<std::string> kExpectedMatchesSorted = {"a.txt"}; + auto matcher = [](std::string_view name) { return name == "a.txt"; }; + AssertIterationOrderWithMatcher(matcher, kExpectedMatchesSorted); +} + +TEST(ziparchive, IterationWithAdditionalMatchesWithSuffix) { + static const std::vector<std::string> kExpectedMatchesSorted = {"a.txt", "b.txt", "b/c.txt", + "b/d.txt"}; + auto matcher = [](std::string_view name) { + return name == "a.txt" || android::base::EndsWith(name, ".txt"); + }; + AssertIterationOrderWithMatcher(matcher, kExpectedMatchesSorted); +} + +TEST(ziparchive, IterationWithAdditionalMatchesWithPrefixAndSuffix) { + static const std::vector<std::string> kExpectedMatchesSorted = {"a.txt", "b/c.txt", "b/d.txt"}; + auto matcher = [](std::string_view name) { + return name == "a.txt" || + (android::base::EndsWith(name, ".txt") && android::base::StartsWith(name, "b/")); + }; + AssertIterationOrderWithMatcher(matcher, kExpectedMatchesSorted); +} + TEST(ziparchive, IterationWithBadPrefixAndSuffix) { ZipArchiveHandle handle; ASSERT_EQ(0, OpenArchiveWrapper(kValidZip, &handle)); @@ -254,6 +392,48 @@ TEST(ziparchive, TestInvalidDeclaredLength) { CloseArchive(handle); } +TEST(ziparchive, OpenArchiveFdRange) { + TemporaryFile tmp_file; + ASSERT_NE(-1, tmp_file.fd); + + const std::string leading_garbage(21, 'x'); + ASSERT_TRUE(android::base::WriteFully(tmp_file.fd, leading_garbage.c_str(), + leading_garbage.size())); + + std::string valid_content; + ASSERT_TRUE(android::base::ReadFileToString(test_data_dir + "/" + kValidZip, &valid_content)); + ASSERT_TRUE(android::base::WriteFully(tmp_file.fd, valid_content.c_str(), valid_content.size())); + + const std::string ending_garbage(42, 'x'); + ASSERT_TRUE(android::base::WriteFully(tmp_file.fd, ending_garbage.c_str(), + ending_garbage.size())); + + ZipArchiveHandle handle; + ASSERT_EQ(0, lseek(tmp_file.fd, 0, SEEK_SET)); + ASSERT_EQ(0, OpenArchiveFdRange(tmp_file.fd, "OpenArchiveFdRange", &handle, + valid_content.size(), + static_cast<off64_t>(leading_garbage.size()))); + + // An entry that's deflated. + ZipEntry data; + ASSERT_EQ(0, FindEntry(handle, "a.txt", &data)); + const uint32_t a_size = data.uncompressed_length; + ASSERT_EQ(a_size, kATxtContents.size()); + auto buffer = std::unique_ptr<uint8_t[]>(new uint8_t[a_size]); + ASSERT_EQ(0, ExtractToMemory(handle, &data, buffer.get(), a_size)); + ASSERT_EQ(0, memcmp(buffer.get(), kATxtContents.data(), a_size)); + + // An entry that's stored. + ASSERT_EQ(0, FindEntry(handle, "b.txt", &data)); + const uint32_t b_size = data.uncompressed_length; + ASSERT_EQ(b_size, kBTxtContents.size()); + buffer = std::unique_ptr<uint8_t[]>(new uint8_t[b_size]); + ASSERT_EQ(0, ExtractToMemory(handle, &data, buffer.get(), b_size)); + ASSERT_EQ(0, memcmp(buffer.get(), kBTxtContents.data(), b_size)); + + CloseArchive(handle); +} + TEST(ziparchive, ExtractToMemory) { ZipArchiveHandle handle; ASSERT_EQ(0, OpenArchiveWrapper(kValidZip, &handle)); @@ -789,3 +969,304 @@ TEST(ziparchive, Inflate) { ASSERT_EQ(0u, writer.GetOutput().size()); } } + +// The class constructs a zipfile with zip64 format, and test the parsing logic. +class Zip64ParseTest : public ::testing::Test { + protected: + struct LocalFileEntry { + std::vector<uint8_t> local_file_header; + std::string file_name; + std::vector<uint8_t> extended_field; + // Fake data to mimic the compressed bytes in the zipfile. + std::vector<uint8_t> compressed_bytes; + + size_t GetSize() const { + return local_file_header.size() + file_name.size() + extended_field.size() + + compressed_bytes.size(); + } + + void CopyToOutput(std::vector<uint8_t>* output) const { + std::copy(local_file_header.begin(), local_file_header.end(), std::back_inserter(*output)); + std::copy(file_name.begin(), file_name.end(), std::back_inserter(*output)); + std::copy(extended_field.begin(), extended_field.end(), std::back_inserter(*output)); + std::copy(compressed_bytes.begin(), compressed_bytes.end(), std::back_inserter(*output)); + } + }; + + struct CdRecordEntry { + std::vector<uint8_t> central_directory_record; + std::string file_name; + std::vector<uint8_t> extended_field; + + size_t GetSize() const { + return central_directory_record.size() + file_name.size() + extended_field.size(); + } + + void CopyToOutput(std::vector<uint8_t>* output) const { + std::copy(central_directory_record.begin(), central_directory_record.end(), + std::back_inserter(*output)); + std::copy(file_name.begin(), file_name.end(), std::back_inserter(*output)); + std::copy(extended_field.begin(), extended_field.end(), std::back_inserter(*output)); + } + }; + + static void ConstructLocalFileHeader(const std::string& name, std::vector<uint8_t>* output, + uint32_t uncompressed_size, uint32_t compressed_size) { + LocalFileHeader lfh = {}; + lfh.lfh_signature = LocalFileHeader::kSignature; + lfh.compressed_size = compressed_size; + lfh.uncompressed_size = uncompressed_size; + lfh.file_name_length = static_cast<uint16_t>(name.size()); + lfh.extra_field_length = 20; + *output = std::vector<uint8_t>(reinterpret_cast<uint8_t*>(&lfh), + reinterpret_cast<uint8_t*>(&lfh) + sizeof(LocalFileHeader)); + } + + // Put one zip64 extended info in the extended field. + static void ConstructExtendedField(const std::vector<uint64_t>& zip64_fields, + std::vector<uint8_t>* output) { + ASSERT_FALSE(zip64_fields.empty()); + uint16_t data_size = 8 * static_cast<uint16_t>(zip64_fields.size()); + std::vector<uint8_t> extended_field(data_size + 4); + android::base::put_unaligned(extended_field.data(), Zip64ExtendedInfo::kHeaderId); + android::base::put_unaligned(extended_field.data() + 2, data_size); + size_t offset = 4; + for (const auto& field : zip64_fields) { + android::base::put_unaligned(extended_field.data() + offset, field); + offset += 8; + } + + *output = std::move(extended_field); + } + + static void ConstructCentralDirectoryRecord(const std::string& name, uint32_t uncompressed_size, + uint32_t compressed_size, uint32_t local_offset, + std::vector<uint8_t>* output) { + CentralDirectoryRecord cdr = {}; + cdr.record_signature = CentralDirectoryRecord::kSignature; + cdr.compressed_size = uncompressed_size; + cdr.uncompressed_size = compressed_size; + cdr.file_name_length = static_cast<uint16_t>(name.size()); + cdr.extra_field_length = local_offset == UINT32_MAX ? 28 : 20; + cdr.local_file_header_offset = local_offset; + *output = + std::vector<uint8_t>(reinterpret_cast<uint8_t*>(&cdr), + reinterpret_cast<uint8_t*>(&cdr) + sizeof(CentralDirectoryRecord)); + } + + // Add an entry to the zipfile, construct the corresponding local header and cd entry. + void AddEntry(const std::string& name, const std::vector<uint8_t>& content, + bool uncompressed_size_in_extended, bool compressed_size_in_extended, + bool local_offset_in_extended) { + auto uncompressed_size = static_cast<uint32_t>(content.size()); + auto compressed_size = static_cast<uint32_t>(content.size()); + uint32_t local_file_header_offset = 0; + std::for_each(file_entries_.begin(), file_entries_.end(), + [&local_file_header_offset](const LocalFileEntry& file_entry) { + local_file_header_offset += file_entry.GetSize(); + }); + + std::vector<uint64_t> zip64_fields; + if (uncompressed_size_in_extended) { + zip64_fields.push_back(uncompressed_size); + uncompressed_size = UINT32_MAX; + } + if (compressed_size_in_extended) { + zip64_fields.push_back(compressed_size); + compressed_size = UINT32_MAX; + } + LocalFileEntry local_entry = { + .local_file_header = {}, + .file_name = name, + .extended_field = {}, + .compressed_bytes = content, + }; + ConstructLocalFileHeader(name, &local_entry.local_file_header, uncompressed_size, + compressed_size); + ConstructExtendedField(zip64_fields, &local_entry.extended_field); + file_entries_.push_back(std::move(local_entry)); + + if (local_offset_in_extended) { + zip64_fields.push_back(local_file_header_offset); + local_file_header_offset = UINT32_MAX; + } + CdRecordEntry cd_entry = { + .central_directory_record = {}, + .file_name = name, + .extended_field = {}, + }; + ConstructCentralDirectoryRecord(name, uncompressed_size, compressed_size, + local_file_header_offset, &cd_entry.central_directory_record); + ConstructExtendedField(zip64_fields, &cd_entry.extended_field); + cd_entries_.push_back(std::move(cd_entry)); + } + + void ConstructEocd() { + ASSERT_EQ(file_entries_.size(), cd_entries_.size()); + Zip64EocdRecord zip64_eocd = {}; + zip64_eocd.record_signature = Zip64EocdRecord::kSignature; + zip64_eocd.num_records = file_entries_.size(); + zip64_eocd.cd_size = 0; + std::for_each( + cd_entries_.begin(), cd_entries_.end(), + [&zip64_eocd](const CdRecordEntry& cd_entry) { zip64_eocd.cd_size += cd_entry.GetSize(); }); + zip64_eocd.cd_start_offset = 0; + std::for_each(file_entries_.begin(), file_entries_.end(), + [&zip64_eocd](const LocalFileEntry& file_entry) { + zip64_eocd.cd_start_offset += file_entry.GetSize(); + }); + zip64_eocd_record_ = + std::vector<uint8_t>(reinterpret_cast<uint8_t*>(&zip64_eocd), + reinterpret_cast<uint8_t*>(&zip64_eocd) + sizeof(Zip64EocdRecord)); + + Zip64EocdLocator zip64_locator = {}; + zip64_locator.locator_signature = Zip64EocdLocator::kSignature; + zip64_locator.zip64_eocd_offset = zip64_eocd.cd_start_offset + zip64_eocd.cd_size; + zip64_eocd_locator_ = + std::vector<uint8_t>(reinterpret_cast<uint8_t*>(&zip64_locator), + reinterpret_cast<uint8_t*>(&zip64_locator) + sizeof(Zip64EocdLocator)); + + EocdRecord eocd = {}; + eocd.eocd_signature = EocdRecord::kSignature, + eocd.num_records = file_entries_.size() > UINT16_MAX + ? UINT16_MAX + : static_cast<uint16_t>(file_entries_.size()); + eocd.cd_size = UINT32_MAX; + eocd.cd_start_offset = UINT32_MAX; + eocd_record_ = std::vector<uint8_t>(reinterpret_cast<uint8_t*>(&eocd), + reinterpret_cast<uint8_t*>(&eocd) + sizeof(EocdRecord)); + } + + // Concatenate all the local file entries, cd entries, and eocd metadata. + void ConstructZipFile() { + for (const auto& file_entry : file_entries_) { + file_entry.CopyToOutput(&zip_content_); + } + for (const auto& cd_entry : cd_entries_) { + cd_entry.CopyToOutput(&zip_content_); + } + std::copy(zip64_eocd_record_.begin(), zip64_eocd_record_.end(), + std::back_inserter(zip_content_)); + std::copy(zip64_eocd_locator_.begin(), zip64_eocd_locator_.end(), + std::back_inserter(zip_content_)); + std::copy(eocd_record_.begin(), eocd_record_.end(), std::back_inserter(zip_content_)); + } + + std::vector<uint8_t> zip_content_; + + std::vector<LocalFileEntry> file_entries_; + std::vector<CdRecordEntry> cd_entries_; + std::vector<uint8_t> zip64_eocd_record_; + std::vector<uint8_t> zip64_eocd_locator_; + std::vector<uint8_t> eocd_record_; +}; + +TEST_F(Zip64ParseTest, openFile) { + AddEntry("a.txt", std::vector<uint8_t>(100, 'a'), true, true, false); + ConstructEocd(); + ConstructZipFile(); + + ZipArchiveHandle handle; + ASSERT_EQ( + 0, OpenArchiveFromMemory(zip_content_.data(), zip_content_.size(), "debug_zip64", &handle)); + CloseArchive(handle); +} + +TEST_F(Zip64ParseTest, openFilelocalOffsetInExtendedField) { + AddEntry("a.txt", std::vector<uint8_t>(100, 'a'), true, true, true); + AddEntry("b.txt", std::vector<uint8_t>(200, 'b'), true, true, true); + ConstructEocd(); + ConstructZipFile(); + + ZipArchiveHandle handle; + ASSERT_EQ( + 0, OpenArchiveFromMemory(zip_content_.data(), zip_content_.size(), "debug_zip64", &handle)); + CloseArchive(handle); +} + +TEST_F(Zip64ParseTest, openFileCompressedNotInExtendedField) { + AddEntry("a.txt", std::vector<uint8_t>(100, 'a'), true, false, false); + ConstructEocd(); + ConstructZipFile(); + + ZipArchiveHandle handle; + // Zip64 extended fields must include both uncompressed and compressed size. + ASSERT_NE( + 0, OpenArchiveFromMemory(zip_content_.data(), zip_content_.size(), "debug_zip64", &handle)); + CloseArchive(handle); +} + +TEST_F(Zip64ParseTest, findEntry) { + AddEntry("a.txt", std::vector<uint8_t>(200, 'a'), true, true, true); + AddEntry("b.txt", std::vector<uint8_t>(300, 'b'), true, true, false); + ConstructEocd(); + ConstructZipFile(); + + ZipArchiveHandle handle; + ASSERT_EQ( + 0, OpenArchiveFromMemory(zip_content_.data(), zip_content_.size(), "debug_zip64", &handle)); + ZipEntry entry; + ASSERT_EQ(0, FindEntry(handle, "a.txt", &entry)); + ASSERT_EQ(200, entry.uncompressed_length); + ASSERT_EQ(200, entry.compressed_length); + + ASSERT_EQ(0, FindEntry(handle, "b.txt", &entry)); + ASSERT_EQ(300, entry.uncompressed_length); + ASSERT_EQ(300, entry.compressed_length); + CloseArchive(handle); +} + +TEST_F(Zip64ParseTest, openFileIncorrectDataSizeInLocalExtendedField) { + AddEntry("a.txt", std::vector<uint8_t>(100, 'a'), true, true, false); + ASSERT_EQ(1, file_entries_.size()); + auto& extended_field = file_entries_[0].extended_field; + // data size exceeds the extended field size in local header. + android::base::put_unaligned<uint16_t>(extended_field.data() + 2, 30); + ConstructEocd(); + ConstructZipFile(); + + ZipArchiveHandle handle; + ASSERT_EQ( + 0, OpenArchiveFromMemory(zip_content_.data(), zip_content_.size(), "debug_zip64", &handle)); + ZipEntry entry; + ASSERT_NE(0, FindEntry(handle, "a.txt", &entry)); + + CloseArchive(handle); +} + +TEST_F(Zip64ParseTest, iterates) { + std::set<std::string_view> names{"a.txt", "b.txt", "c.txt", "d.txt", "e.txt"}; + for (const auto& name : names) { + AddEntry(std::string(name), std::vector<uint8_t>(100, name[0]), true, true, true); + } + ConstructEocd(); + ConstructZipFile(); + + ZipArchiveHandle handle; + ASSERT_EQ( + 0, OpenArchiveFromMemory(zip_content_.data(), zip_content_.size(), "debug_zip64", &handle)); + + void* iteration_cookie; + ASSERT_EQ(0, StartIteration(handle, &iteration_cookie)); + std::set<std::string_view> result; + std::string_view name; + ZipEntry entry; + while (Next(iteration_cookie, &entry, &name) == 0) result.emplace(name); + ASSERT_EQ(names, result); + + CloseArchive(handle); +} + +TEST_F(Zip64ParseTest, zip64EocdWrongLocatorOffset) { + AddEntry("a.txt", std::vector<uint8_t>(1, 'a'), true, true, true); + ConstructEocd(); + zip_content_.resize(20, 'a'); + std::copy(zip64_eocd_locator_.begin(), zip64_eocd_locator_.end(), + std::back_inserter(zip_content_)); + std::copy(eocd_record_.begin(), eocd_record_.end(), std::back_inserter(zip_content_)); + + ZipArchiveHandle handle; + ASSERT_NE( + 0, OpenArchiveFromMemory(zip_content_.data(), zip_content_.size(), "debug_zip64", &handle)); + CloseArchive(handle); +} diff --git a/libziparchive/zip_cd_entry_map.cc b/libziparchive/zip_cd_entry_map.cc new file mode 100644 index 000000000..f187c0690 --- /dev/null +++ b/libziparchive/zip_cd_entry_map.cc @@ -0,0 +1,156 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "zip_cd_entry_map.h" + +#include <android-base/logging.h> +#include <log/log.h> + +/* + * Round up to the next highest power of 2. + * + * Found on http://graphics.stanford.edu/~seander/bithacks.html. + */ +static uint32_t RoundUpPower2(uint32_t val) { + val--; + val |= val >> 1; + val |= val >> 2; + val |= val >> 4; + val |= val >> 8; + val |= val >> 16; + val++; + + return val; +} + +static uint32_t ComputeHash(std::string_view name) { + return static_cast<uint32_t>(std::hash<std::string_view>{}(name)); +} + +// Convert a ZipEntry to a hash table index, verifying that it's in a valid range. +std::pair<ZipError, uint64_t> CdEntryMapZip32::GetCdEntryOffset(std::string_view name, + const uint8_t* start) const { + const uint32_t hash = ComputeHash(name); + + // NOTE: (hash_table_size - 1) is guaranteed to be non-negative. + uint32_t ent = hash & (hash_table_size_ - 1); + while (hash_table_[ent].name_offset != 0) { + if (hash_table_[ent].ToStringView(start) == name) { + return {kSuccess, hash_table_[ent].name_offset}; + } + ent = (ent + 1) & (hash_table_size_ - 1); + } + + ALOGV("Zip: Unable to find entry %.*s", static_cast<int>(name.size()), name.data()); + return {kEntryNotFound, 0}; +} + +ZipError CdEntryMapZip32::AddToMap(std::string_view name, const uint8_t* start) { + const uint64_t hash = ComputeHash(name); + uint32_t ent = hash & (hash_table_size_ - 1); + + /* + * We over-allocated the table, so we're guaranteed to find an empty slot. + * Further, we guarantee that the hashtable size is not 0. + */ + while (hash_table_[ent].name_offset != 0) { + if (hash_table_[ent].ToStringView(start) == name) { + // We've found a duplicate entry. We don't accept duplicates. + ALOGW("Zip: Found duplicate entry %.*s", static_cast<int>(name.size()), name.data()); + return kDuplicateEntry; + } + ent = (ent + 1) & (hash_table_size_ - 1); + } + + // `name` has already been validated before entry. + const char* start_char = reinterpret_cast<const char*>(start); + hash_table_[ent].name_offset = static_cast<uint32_t>(name.data() - start_char); + hash_table_[ent].name_length = static_cast<uint16_t>(name.size()); + return kSuccess; +} + +void CdEntryMapZip32::ResetIteration() { + current_position_ = 0; +} + +std::pair<std::string_view, uint64_t> CdEntryMapZip32::Next(const uint8_t* cd_start) { + while (current_position_ < hash_table_size_) { + const auto& entry = hash_table_[current_position_]; + current_position_ += 1; + + if (entry.name_offset != 0) { + return {entry.ToStringView(cd_start), entry.name_offset}; + } + } + // We have reached the end of the hash table. + return {}; +} + +CdEntryMapZip32::CdEntryMapZip32(uint16_t num_entries) { + /* + * Create hash table. We have a minimum 75% load factor, possibly as + * low as 50% after we round off to a power of 2. There must be at + * least one unused entry to avoid an infinite loop during creation. + */ + hash_table_size_ = RoundUpPower2(1 + (num_entries * 4) / 3); + hash_table_ = { + reinterpret_cast<ZipStringOffset*>(calloc(hash_table_size_, sizeof(ZipStringOffset))), free}; +} + +std::unique_ptr<CdEntryMapInterface> CdEntryMapZip32::Create(uint16_t num_entries) { + auto entry_map = new CdEntryMapZip32(num_entries); + CHECK(entry_map->hash_table_ != nullptr) + << "Zip: unable to allocate the " << entry_map->hash_table_size_ + << " entry hash_table, entry size: " << sizeof(ZipStringOffset); + return std::unique_ptr<CdEntryMapInterface>(entry_map); +} + +std::unique_ptr<CdEntryMapInterface> CdEntryMapZip64::Create() { + return std::unique_ptr<CdEntryMapInterface>(new CdEntryMapZip64()); +} + +ZipError CdEntryMapZip64::AddToMap(std::string_view name, const uint8_t* start) { + const auto [it, added] = + entry_table_.insert({name, name.data() - reinterpret_cast<const char*>(start)}); + if (!added) { + ALOGW("Zip: Found duplicate entry %.*s", static_cast<int>(name.size()), name.data()); + return kDuplicateEntry; + } + return kSuccess; +} + +std::pair<ZipError, uint64_t> CdEntryMapZip64::GetCdEntryOffset(std::string_view name, + const uint8_t* /*cd_start*/) const { + const auto it = entry_table_.find(name); + if (it == entry_table_.end()) { + ALOGV("Zip: Could not find entry %.*s", static_cast<int>(name.size()), name.data()); + return {kEntryNotFound, 0}; + } + + return {kSuccess, it->second}; +} + +void CdEntryMapZip64::ResetIteration() { + iterator_ = entry_table_.begin(); +} + +std::pair<std::string_view, uint64_t> CdEntryMapZip64::Next(const uint8_t* /*cd_start*/) { + if (iterator_ == entry_table_.end()) { + return {}; + } + + return *iterator_++; +} diff --git a/libziparchive/zip_cd_entry_map.h b/libziparchive/zip_cd_entry_map.h new file mode 100644 index 000000000..4957f754e --- /dev/null +++ b/libziparchive/zip_cd_entry_map.h @@ -0,0 +1,114 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <stdint.h> + +#include <map> +#include <memory> +#include <string_view> +#include <utility> + +#include "zip_error.h" + +// This class is the interface of the central directory entries map. The map +// helps to locate a particular cd entry based on the filename. +class CdEntryMapInterface { + public: + virtual ~CdEntryMapInterface() = default; + // Adds an entry to the map. The |name| should internally points to the + // filename field of a cd entry. And |start| points to the beginning of the + // central directory. Returns 0 on success. + virtual ZipError AddToMap(std::string_view name, const uint8_t* start) = 0; + // For the zip entry |entryName|, finds the offset of its filename field in + // the central directory. Returns a pair of [status, offset]. The value of + // the status is 0 on success. + virtual std::pair<ZipError, uint64_t> GetCdEntryOffset(std::string_view name, + const uint8_t* cd_start) const = 0; + // Resets the iterator to the beginning of the map. + virtual void ResetIteration() = 0; + // Returns the [name, cd offset] of the current element. Also increments the + // iterator to points to the next element. Returns an empty pair we have read + // past boundary. + virtual std::pair<std::string_view, uint64_t> Next(const uint8_t* cd_start) = 0; +}; + +/** + * More space efficient string representation of strings in an mmaped zipped + * file than std::string_view. Using std::string_view as an entry in the + * ZipArchive hash table wastes space. std::string_view stores a pointer to a + * string (on 64 bit, 8 bytes) and the length to read from that pointer, + * 2 bytes. Because of alignment, the structure consumes 16 bytes, wasting + * 6 bytes. + * + * ZipStringOffset stores a 4 byte offset from a fixed location in the memory + * mapped file instead of the entire address, consuming 8 bytes with alignment. + */ +struct ZipStringOffset { + uint32_t name_offset; + uint16_t name_length; + + const std::string_view ToStringView(const uint8_t* start) const { + return std::string_view{reinterpret_cast<const char*>(start + name_offset), name_length}; + } +}; + +// This implementation of CdEntryMap uses an array hash table. It uses less +// memory than std::map; and it's used as the default implementation for zip +// archives without zip64 extension. +class CdEntryMapZip32 : public CdEntryMapInterface { + public: + static std::unique_ptr<CdEntryMapInterface> Create(uint16_t num_entries); + + ZipError AddToMap(std::string_view name, const uint8_t* start) override; + std::pair<ZipError, uint64_t> GetCdEntryOffset(std::string_view name, + const uint8_t* cd_start) const override; + void ResetIteration() override; + std::pair<std::string_view, uint64_t> Next(const uint8_t* cd_start) override; + + private: + explicit CdEntryMapZip32(uint16_t num_entries); + + // We know how many entries are in the Zip archive, so we can have a + // fixed-size hash table. We define a load factor of 0.75 and over + // allocate so the maximum number entries can never be higher than + // ((4 * UINT16_MAX) / 3 + 1) which can safely fit into a uint32_t. + uint32_t hash_table_size_{0}; + std::unique_ptr<ZipStringOffset[], decltype(&free)> hash_table_{nullptr, free}; + + // The position of element for the current iteration. + uint32_t current_position_{0}; +}; + +// This implementation of CdEntryMap uses a std::map +class CdEntryMapZip64 : public CdEntryMapInterface { + public: + static std::unique_ptr<CdEntryMapInterface> Create(); + + ZipError AddToMap(std::string_view name, const uint8_t* start) override; + std::pair<ZipError, uint64_t> GetCdEntryOffset(std::string_view name, + const uint8_t* cd_start) const override; + void ResetIteration() override; + std::pair<std::string_view, uint64_t> Next(const uint8_t* cd_start) override; + + private: + CdEntryMapZip64() = default; + + std::map<std::string_view, uint64_t> entry_table_; + + std::map<std::string_view, uint64_t>::iterator iterator_; +}; diff --git a/libziparchive/zip_error.cpp b/libziparchive/zip_error.cpp new file mode 100644 index 000000000..107ec4797 --- /dev/null +++ b/libziparchive/zip_error.cpp @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2008 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "zip_error.h" + +#include <android-base/macros.h> + +static const char* kErrorMessages[] = { + "Success", + "Iteration ended", + "Zlib error", + "Invalid file", + "Invalid handle", + "Duplicate entries in archive", + "Empty archive", + "Entry not found", + "Invalid offset", + "Inconsistent information", + "Invalid entry name", + "I/O error", + "File mapping failed", + "Allocation failed", +}; + +const char* ErrorCodeString(int32_t error_code) { + // Make sure that the number of entries in kErrorMessages and the ZipError + // enum match. + static_assert((-kLastErrorCode + 1) == arraysize(kErrorMessages), + "(-kLastErrorCode + 1) != arraysize(kErrorMessages)"); + + const uint32_t idx = -error_code; + if (idx < arraysize(kErrorMessages)) { + return kErrorMessages[idx]; + } + + return "Unknown return code"; +} diff --git a/libziparchive/zip_error.h b/libziparchive/zip_error.h new file mode 100644 index 000000000..37fd55f38 --- /dev/null +++ b/libziparchive/zip_error.h @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <stdint.h> + +enum ZipError : int32_t { + kSuccess = 0, + + kIterationEnd = -1, + + // We encountered a Zlib error when inflating a stream from this file. + // Usually indicates file corruption. + kZlibError = -2, + + // The input file cannot be processed as a zip archive. Usually because + // it's too small, too large or does not have a valid signature. + kInvalidFile = -3, + + // An invalid iteration / ziparchive handle was passed in as an input + // argument. + kInvalidHandle = -4, + + // The zip archive contained two (or possibly more) entries with the same + // name. + kDuplicateEntry = -5, + + // The zip archive contains no entries. + kEmptyArchive = -6, + + // The specified entry was not found in the archive. + kEntryNotFound = -7, + + // The zip archive contained an invalid local file header pointer. + kInvalidOffset = -8, + + // The zip archive contained inconsistent entry information. This could + // be because the central directory & local file header did not agree, or + // if the actual uncompressed length or crc32 do not match their declared + // values. + kInconsistentInformation = -9, + + // An invalid entry name was encountered. + kInvalidEntryName = -10, + + // An I/O related system call (read, lseek, ftruncate, map) failed. + kIoError = -11, + + // We were not able to mmap the central directory or entry contents. + kMmapFailed = -12, + + // An allocation failed. + kAllocationFailed = -13, + + kLastErrorCode = kAllocationFailed, +}; diff --git a/libziparchive/ziptool.cpp b/libziparchive/ziptool.cpp index dd42e90b9..f345ffc4f 100644 --- a/libziparchive/ziptool.cpp +++ b/libziparchive/ziptool.cpp @@ -133,8 +133,8 @@ static void MaybeShowHeader(ZipArchiveHandle zah) { if (!flag_1 && includes.empty() && excludes.empty()) { ZipArchiveInfo info{GetArchiveInfo(zah)}; printf("Archive: %s\n", archive_name); - printf("Zip file size: %" PRId64 " bytes, number of entries: %zu\n", info.archive_size, - info.entry_count); + printf("Zip file size: %" PRId64 " bytes, number of entries: %" PRIu64 "\n", + info.archive_size, info.entry_count); } } } diff --git a/logcat/logcat.cpp b/logcat/logcat.cpp index 76a970f2a..b065855c5 100644 --- a/logcat/logcat.cpp +++ b/logcat/logcat.cpp @@ -513,7 +513,7 @@ int Logcat::Run(int argc, char** argv) { unsigned long setLogSize = 0; const char* setPruneList = nullptr; const char* setId = nullptr; - int mode = ANDROID_LOG_RDONLY; + int mode = 0; std::string forceFilters; size_t tail_lines = 0; log_time tail_time(log_time::EPOCH); @@ -591,8 +591,7 @@ int Logcat::Run(int argc, char** argv) { break; } if (long_options[option_index].name == wrap_str) { - mode |= ANDROID_LOG_WRAP | ANDROID_LOG_RDONLY | - ANDROID_LOG_NONBLOCK; + mode |= ANDROID_LOG_WRAP | ANDROID_LOG_NONBLOCK; // ToDo: implement API that supports setting a wrap timeout size_t dummy = ANDROID_LOG_WRAP_DEFAULT_TIMEOUT; if (optarg && (!ParseUint(optarg, &dummy) || dummy < 1)) { @@ -626,21 +625,19 @@ int Logcat::Run(int argc, char** argv) { case 'c': clearLog = true; - mode |= ANDROID_LOG_WRONLY; break; case 'L': - mode |= ANDROID_LOG_RDONLY | ANDROID_LOG_PSTORE | - ANDROID_LOG_NONBLOCK; + mode |= ANDROID_LOG_PSTORE | ANDROID_LOG_NONBLOCK; break; case 'd': - mode |= ANDROID_LOG_RDONLY | ANDROID_LOG_NONBLOCK; + mode |= ANDROID_LOG_NONBLOCK; break; case 't': got_t = true; - mode |= ANDROID_LOG_RDONLY | ANDROID_LOG_NONBLOCK; + mode |= ANDROID_LOG_NONBLOCK; FALLTHROUGH_INTENDED; case 'T': if (strspn(optarg, "0123456789") != strlen(optarg)) { diff --git a/logd/LogTags.cpp b/logd/LogTags.cpp index 0cc7886ea..8299e667a 100644 --- a/logd/LogTags.cpp +++ b/logd/LogTags.cpp @@ -289,9 +289,8 @@ static inline uint32_t get4LE(const char* msg) { // special pmsg event for log tags, and build up our internal // database with any found. void LogTags::ReadPersistEventLogTags() { - struct logger_list* logger_list = android_logger_list_alloc( - ANDROID_LOG_RDONLY | ANDROID_LOG_PSTORE | ANDROID_LOG_NONBLOCK, 0, - (pid_t)0); + struct logger_list* logger_list = + android_logger_list_alloc(ANDROID_LOG_PSTORE | ANDROID_LOG_NONBLOCK, 0, (pid_t)0); if (!logger_list) return; struct logger* e = android_logger_open(logger_list, LOG_ID_EVENTS); diff --git a/logd/tests/Android.bp b/logd/tests/Android.bp index d39da8a27..2519a8444 100644 --- a/logd/tests/Android.bp +++ b/logd/tests/Android.bp @@ -64,5 +64,6 @@ cc_test { test_suites: [ "cts", "vts", + "vts10", ], } diff --git a/logd/tests/logd_test.cpp b/logd/tests/logd_test.cpp index f47bee1c0..c7f348069 100644 --- a/logd/tests/logd_test.cpp +++ b/logd/tests/logd_test.cpp @@ -870,10 +870,8 @@ void __android_log_btwrite_multiple__helper(int count) { ASSERT_EQ(0, info.si_status); struct logger_list* logger_list; - ASSERT_TRUE(nullptr != - (logger_list = android_logger_list_open( - LOG_ID_EVENTS, ANDROID_LOG_RDONLY | ANDROID_LOG_NONBLOCK, - 0, pid))); + ASSERT_TRUE(nullptr != (logger_list = android_logger_list_open(LOG_ID_EVENTS, + ANDROID_LOG_NONBLOCK, 0, pid))); int expected_count = (count < 2) ? count : 2; int expected_chatty_count = (count <= 2) ? 0 : 1; diff --git a/qemu_pipe/Android.bp b/qemu_pipe/Android.bp index c6bda4a28..ad86a4e6d 100644 --- a/qemu_pipe/Android.bp +++ b/qemu_pipe/Android.bp @@ -4,6 +4,11 @@ cc_library_static { name: "libqemu_pipe", vendor_available: true, recovery_available: true, + apex_available: [ + "com.android.adbd", + // TODO(b/151398197) remove the below + "//apex_available:platform", + ], sanitize: { misc_undefined: ["integer"], }, diff --git a/storaged/main.cpp b/storaged/main.cpp index a7bda1492..bbed210de 100644 --- a/storaged/main.cpp +++ b/storaged/main.cpp @@ -71,6 +71,7 @@ int main(int argc, char** argv) { bool flag_dump_perf = false; int opt; + signal(SIGPIPE, SIG_IGN); android::base::InitLogging(argv, android::base::LogdLogger(android::base::SYSTEM)); for (;;) { |