diff options
487 files changed, 16890 insertions, 27514 deletions
diff --git a/METADATA b/METADATA new file mode 100644 index 000000000..d97975ca3 --- /dev/null +++ b/METADATA @@ -0,0 +1,3 @@ +third_party { + license_type: NOTICE +} diff --git a/PREUPLOAD.cfg b/PREUPLOAD.cfg index c8dbf77fa..dcf92be1e 100644 --- a/PREUPLOAD.cfg +++ b/PREUPLOAD.cfg @@ -3,3 +3,6 @@ clang_format = true [Builtin Hooks Options] clang_format = --commit ${PREUPLOAD_COMMIT} --style file --extensions c,h,cc,cpp + +[Hook Scripts] +aosp_hook = ${REPO_ROOT}/frameworks/base/tools/aosp/aosp_sha.sh ${PREUPLOAD_COMMIT} "." diff --git a/TEST_MAPPING b/TEST_MAPPING index 0ec505da8..79336291a 100644 --- a/TEST_MAPPING +++ b/TEST_MAPPING @@ -7,6 +7,12 @@ "name": "adb_crypto_test" }, { + "name": "adb_pairing_auth_test" + }, + { + "name": "adb_pairing_connection_test" + }, + { "name": "adb_tls_connection_test" }, { @@ -22,9 +28,6 @@ "name": "fs_mgr_vendor_overlay_test" }, { - "name": "libbase_test" - }, - { "name": "libpackagelistparser_test" }, { @@ -54,9 +57,6 @@ }, { "name": "propertyinfoserializer_tests" - }, - { - "name": "ziparchive-tests" } ] } diff --git a/adb/Android.bp b/adb/Android.bp index dee48bf80..78e0b8ecd 100644 --- a/adb/Android.bp +++ b/adb/Android.bp @@ -113,13 +113,16 @@ cc_defaults { "libadbd_core", "libadbconnection_server", "libasyncio", + "libbase", "libbrotli", "libcutils_sockets", "libdiagnose_usb", "libmdnssd", - "libbase", + "libzstd", "libadb_protos", + "libapp_processes_protos_lite", + "libprotobuf-cpp-lite", ], shared_libs: [ @@ -152,17 +155,26 @@ libadb_srcs = [ "adb_unique_fd.cpp", "adb_utils.cpp", "fdevent/fdevent.cpp", - "fdevent/fdevent_poll.cpp", "services.cpp", "sockets.cpp", "socket_spec.cpp", "sysdeps/errno.cpp", "transport.cpp", "transport_fd.cpp", - "transport_local.cpp", "types.cpp", ] +libadb_darwin_srcs = [ + "fdevent/fdevent_poll.cpp", +] + +libadb_windows_srcs = [ + "fdevent/fdevent_poll.cpp", + "sysdeps_win32.cpp", + "sysdeps/win32/errno.cpp", + "sysdeps/win32/stat.cpp", +] + libadb_posix_srcs = [ "sysdeps_unix.cpp", "sysdeps/posix/network.cpp", @@ -194,7 +206,9 @@ cc_library_host_static { "client/adb_wifi.cpp", "client/usb_libusb.cpp", "client/usb_dispatch.cpp", + "client/transport_local.cpp", "client/transport_mdns.cpp", + "client/mdns_utils.cpp", "client/transport_usb.cpp", "client/pairing/pairing_client.cpp", ], @@ -206,7 +220,7 @@ cc_library_host_static { srcs: ["client/usb_linux.cpp"] + libadb_linux_srcs, }, darwin: { - srcs: ["client/usb_osx.cpp"], + srcs: ["client/usb_osx.cpp"] + libadb_darwin_srcs, }, not_windows: { srcs: libadb_posix_srcs, @@ -215,10 +229,7 @@ cc_library_host_static { enabled: true, srcs: [ "client/usb_windows.cpp", - "sysdeps_win32.cpp", - "sysdeps/win32/errno.cpp", - "sysdeps/win32/stat.cpp", - ], + ] + libadb_windows_srcs, shared_libs: ["AdbWinApi"], }, }, @@ -244,7 +255,10 @@ cc_library_host_static { cc_test_host { name: "adb_test", defaults: ["adb_defaults"], - srcs: libadb_test_srcs, + srcs: libadb_test_srcs + [ + "client/mdns_utils_test.cpp", + ], + static_libs: [ "libadb_crypto_static", "libadb_host", @@ -304,11 +318,12 @@ cc_binary_host { static_libs: [ "libadb_crypto", "libadb_host", - "libadb_pairing_auth", - "libadb_pairing_connection", + "libadb_pairing_auth", + "libadb_pairing_connection", "libadb_protos", "libadb_tls_connection", "libandroidfw", + "libapp_processes_protos_full", "libbase", "libbrotli", "libcutils", @@ -319,13 +334,14 @@ cc_binary_host { "liblog", "liblz4", "libmdnssd", - "libprotobuf-cpp-lite", + "libprotobuf-cpp-full", "libssl", "libusb", "libutils", "liblog", "libziparchive", "libz", + "libzstd", ], // Don't add anything here, we don't want additional shared dependencies @@ -369,25 +385,27 @@ cc_library_static { compile_multilib: "both", srcs: libadb_srcs + libadb_linux_srcs + libadb_posix_srcs + [ + "daemon/adb_wifi.cpp", "daemon/auth.cpp", "daemon/jdwp_service.cpp", "daemon/logging.cpp", - "daemon/adb_wifi.cpp", + "daemon/transport_local.cpp", ], generated_headers: ["platform_tools_version"], static_libs: [ - "libadbconnection_server", "libdiagnose_usb", ], shared_libs: [ + "libadbconnection_server", "libadb_crypto", "libadb_pairing_connection", "libadb_protos", "libadb_tls_connection", "libadbd_auth", + "libapp_processes_protos_lite", "libasyncio", "libbase", "libcrypto", @@ -396,6 +414,12 @@ cc_library_static { "liblog", ], + proto: { + type: "lite", + static: true, + export_proto_headers: true, + }, + target: { android: { whole_static_libs: [ @@ -411,6 +435,7 @@ cc_library_static { exclude_shared_libs: [ "libadb_pairing_auth", "libadb_pairing_connection", + "libapp_processes_protos_lite", ], } }, @@ -448,6 +473,8 @@ cc_library { "libadbd_core", "libbrotli", "libdiagnose_usb", + "liblz4", + "libzstd", ], shared_libs: [ @@ -455,10 +482,12 @@ cc_library { "libadb_pairing_connection", "libadb_protos", "libadb_tls_connection", + "libapp_processes_protos_lite", "libasyncio", "libbase", "libcrypto_utils", "libcutils_sockets", + "libprotobuf-cpp-lite", // APEX dependencies. "libadbd_auth", @@ -514,6 +543,9 @@ cc_library { compile_multilib: "both", shared_libs: [ + "libadbconnection_server", + "libapp_processes_protos_lite", + "libprotobuf-cpp-lite", "libadb_crypto", "libadb_pairing_connection", "libadb_tls_connection", @@ -544,7 +576,9 @@ cc_library { "libbrotli", "libcutils_sockets", "libdiagnose_usb", + "liblz4", "libmdnssd", + "libzstd", ], visibility: [ @@ -577,6 +611,7 @@ cc_binary { "libadbd_services", "libasyncio", "libcap", + "liblz4", "libminijail", "libssl", ], diff --git a/adb/adb.cpp b/adb/adb.cpp index c3e9731a3..08986b77e 100644 --- a/adb/adb.cpp +++ b/adb/adb.cpp @@ -109,7 +109,9 @@ void handle_online(atransport *t) { D("adb: online"); t->online = 1; +#if ADB_HOST t->SetConnectionEstablished(true); +#endif } void handle_offline(atransport *t) @@ -1020,8 +1022,12 @@ bool handle_forward_request(const char* service, if (kill_forward) { r = remove_listener(pieces[0].c_str(), transport); } else { - r = install_listener(pieces[0], pieces[1].c_str(), transport, no_rebind, - &resolved_tcp_port, &error); + int flags = 0; + if (no_rebind) { + flags |= INSTALL_LISTENER_NO_REBIND; + } + r = install_listener(pieces[0], pieces[1].c_str(), transport, flags, &resolved_tcp_port, + &error); } if (r == INSTALL_STATUS_OK) { #if ADB_HOST @@ -1067,19 +1073,44 @@ static int SendOkay(int fd, const std::string& s) { return 0; } +static bool g_reject_kill_server = false; +void adb_set_reject_kill_server(bool value) { + g_reject_kill_server = value; +} + +static bool handle_mdns_request(std::string_view service, int reply_fd) { + if (!android::base::ConsumePrefix(&service, "mdns:")) { + return false; + } + + if (service == "check") { + std::string check = mdns_check(); + SendOkay(reply_fd, check); + return true; + } + if (service == "services") { + std::string services_list = mdns_list_discovered_services(); + SendOkay(reply_fd, services_list); + return true; + } + + return false; +} + HostRequestResult handle_host_request(std::string_view service, TransportType type, const char* serial, TransportId transport_id, int reply_fd, asocket* s) { if (service == "kill") { - fprintf(stderr, "adb server killed by remote request\n"); - fflush(stdout); - - // Send a reply even though we don't read it anymore, so that old versions - // of adb that do read it don't spew error messages. - SendOkay(reply_fd); + if (g_reject_kill_server) { + LOG(WARNING) << "adb server ignoring kill-server"; + SendFail(reply_fd, "kill-server rejected by remote server"); + } else { + fprintf(stderr, "adb server killed by remote request\n"); + SendOkay(reply_fd); - // Rely on process exit to close the socket for us. - exit(0); + // Rely on process exit to close the socket for us. + exit(0); + } } LOG(DEBUG) << "handle_host_request(" << service << ")"; @@ -1188,9 +1219,9 @@ HostRequestResult handle_host_request(std::string_view service, TransportType ty FeatureSet features = supported_features(); // Abuse features to report libusb status. if (should_use_libusb()) { - features.insert(kFeatureLibusb); + features.emplace_back(kFeatureLibusb); } - features.insert(kFeaturePushSync); + features.emplace_back(kFeaturePushSync); SendOkay(reply_fd, FeatureSetToString(features)); return HostRequestResult::Handled; } @@ -1310,6 +1341,10 @@ HostRequestResult handle_host_request(std::string_view service, TransportType ty return HostRequestResult::Handled; } + if (handle_mdns_request(service, reply_fd)) { + return HostRequestResult::Handled; + } + return HostRequestResult::Unhandled; } @@ -14,8 +14,7 @@ * limitations under the License. */ -#ifndef __ADB_H -#define __ADB_H +#pragma once #include <limits.h> #include <stdint.h> @@ -167,6 +166,7 @@ unique_fd execute_abb_command(std::string_view command); int init_jdwp(void); asocket* create_jdwp_service_socket(); asocket* create_jdwp_tracker_service_socket(); +asocket* create_app_tracker_service_socket(); unique_fd create_jdwp_connection_fd(int jdwp_pid); #endif @@ -236,6 +236,7 @@ void send_tls_request(atransport* t); void parse_banner(const std::string&, atransport* t); +#if ADB_HOST // On startup, the adb server needs to wait until all of the connected devices are ready. // To do this, we need to know when the scan has identified all of the potential new transports, and // when each transport becomes ready. @@ -249,6 +250,12 @@ void update_transport_status(); // Wait until device scan has completed and every transport is ready, or a timeout elapses. void adb_wait_for_device_initialization(); +#endif // ADB_HOST -void usb_init(); +#if ADB_HOST +// When ssh-forwarding to a remote adb server, kill-server is almost never what you actually want, +// and unfortunately, many other tools issue it. This adds a knob to reject kill-servers. +void adb_set_reject_kill_server(bool reject); #endif + +void usb_init(); diff --git a/adb/adb_listeners.cpp b/adb/adb_listeners.cpp index 29909a555..124e2d8dc 100644 --- a/adb/adb_listeners.cpp +++ b/adb/adb_listeners.cpp @@ -73,6 +73,7 @@ static auto& listener_list_mutex = *new std::mutex(); typedef std::list<std::unique_ptr<alistener>> ListenerList; static ListenerList& listener_list GUARDED_BY(listener_list_mutex) = *new ListenerList(); +#if ADB_HOST static void ss_listener_event_func(int _fd, unsigned ev, void *_l) { if (ev & FDE_READ) { unique_fd fd(adb_socket_accept(_fd, nullptr, nullptr)); @@ -88,6 +89,7 @@ static void ss_listener_event_func(int _fd, unsigned ev, void *_l) { } } } +#endif static void listener_event_func(int _fd, unsigned ev, void* _l) { @@ -164,6 +166,16 @@ void remove_all_listeners() EXCLUDES(listener_list_mutex) { } } +void enable_server_sockets() EXCLUDES(listener_list_mutex) { + std::lock_guard<std::mutex> lock(listener_list_mutex); + for (auto& l : listener_list) { + if (l->connect_to == "*smartsocket*") { + fdevent_set(l->fde, FDE_READ); + } + } +} + +#if ADB_HOST void close_smartsockets() EXCLUDES(listener_list_mutex) { std::lock_guard<std::mutex> lock(listener_list_mutex); auto pred = [](const std::unique_ptr<alistener>& listener) { @@ -171,21 +183,22 @@ void close_smartsockets() EXCLUDES(listener_list_mutex) { }; listener_list.remove_if(pred); } +#endif InstallStatus install_listener(const std::string& local_name, const char* connect_to, - atransport* transport, int no_rebind, int* resolved_tcp_port, + atransport* transport, int flags, int* resolved_tcp_port, std::string* error) EXCLUDES(listener_list_mutex) { std::lock_guard<std::mutex> lock(listener_list_mutex); for (auto& l : listener_list) { if (local_name == l->local_name) { // Can't repurpose a smartsocket. - if(l->connect_to[0] == '*') { + if (l->connect_to[0] == '*') { *error = "cannot repurpose smartsocket"; return INSTALL_STATUS_INTERNAL_ERROR; } - // Can't repurpose a listener if 'no_rebind' is true. - if (no_rebind) { + // Can't repurpose a listener if INSTALL_LISTENER_NO_REBIND is set + if (flags & INSTALL_LISTENER_NO_REBIND) { *error = "cannot rebind"; return INSTALL_STATUS_CANNOT_REBIND; } @@ -218,11 +231,17 @@ InstallStatus install_listener(const std::string& local_name, const char* connec close_on_exec(listener->fd); if (listener->connect_to == "*smartsocket*") { +#if ADB_HOST listener->fde = fdevent_create(listener->fd, ss_listener_event_func, listener.get()); +#else + LOG(FATAL) << "attempted to connect to *smartsocket* in daemon"; +#endif } else { listener->fde = fdevent_create(listener->fd, listener_event_func, listener.get()); } - fdevent_set(listener->fde, FDE_READ); + if ((flags & INSTALL_LISTENER_DISABLED) == 0) { + fdevent_set(listener->fde, FDE_READ); + } listener->transport = transport; diff --git a/adb/adb_listeners.h b/adb/adb_listeners.h index 70a2ee121..0aa774a7d 100644 --- a/adb/adb_listeners.h +++ b/adb/adb_listeners.h @@ -14,8 +14,7 @@ * limitations under the License. */ -#ifndef __ADB_LISTENERS_H -#define __ADB_LISTENERS_H +#pragma once #include "adb.h" @@ -32,8 +31,11 @@ enum InstallStatus { INSTALL_STATUS_LISTENER_NOT_FOUND = -4, }; +inline constexpr int INSTALL_LISTENER_NO_REBIND = 1 << 0; +inline constexpr int INSTALL_LISTENER_DISABLED = 1 << 1; + InstallStatus install_listener(const std::string& local_name, const char* connect_to, - atransport* transport, int no_rebind, int* resolved_tcp_port, + atransport* transport, int flags, int* resolved_tcp_port, std::string* error); std::string format_listeners(); @@ -41,6 +43,7 @@ std::string format_listeners(); InstallStatus remove_listener(const char* local_name, atransport* transport); void remove_all_listeners(void); +#if ADB_HOST +void enable_server_sockets(); void close_smartsockets(); - -#endif /* __ADB_LISTENERS_H */ +#endif diff --git a/adb/adb_trace.cpp b/adb/adb_trace.cpp index cea24fe2f..210241ce7 100644 --- a/adb/adb_trace.cpp +++ b/adb/adb_trace.cpp @@ -89,18 +89,13 @@ void start_device_log(void) { int adb_trace_mask; -std::string get_trace_setting_from_env() { +std::string get_trace_setting() { +#if ADB_HOST || !defined(__ANDROID__) const char* setting = getenv("ADB_TRACE"); if (setting == nullptr) { setting = ""; } - - return std::string(setting); -} - -std::string get_trace_setting() { -#if ADB_HOST - return get_trace_setting_from_env(); + return setting; #else return android::base::GetProperty("persist.adb.trace_mask", ""); #endif diff --git a/adb/adb_wifi.h b/adb/adb_wifi.h index 585748c91..42f414b28 100644 --- a/adb/adb_wifi.h +++ b/adb/adb_wifi.h @@ -16,6 +16,7 @@ #pragma once +#include <optional> #include <string> #include "adb.h" @@ -27,6 +28,22 @@ void adb_wifi_pair_device(const std::string& host, const std::string& password, std::string& response); bool adb_wifi_is_known_host(const std::string& host); +std::string mdns_check(); +std::string mdns_list_discovered_services(); + +struct MdnsInfo { + std::string service_name; + std::string service_type; + std::string addr; + uint16_t port = 0; + + MdnsInfo(std::string_view name, std::string_view type, std::string_view addr, uint16_t port) + : service_name(name), service_type(type), addr(addr), port(port) {} +}; + +std::optional<MdnsInfo> mdns_get_connect_service_info(std::string_view name); +std::optional<MdnsInfo> mdns_get_pairing_service_info(std::string_view name); + #else // !ADB_HOST struct AdbdAuthContext; diff --git a/adb/brotli_utils.h b/adb/brotli_utils.h deleted file mode 100644 index c5be73d0c..000000000 --- a/adb/brotli_utils.h +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright (C) 2020 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include <span> - -#include <brotli/decode.h> -#include <brotli/encode.h> - -#include "types.h" - -enum class BrotliDecodeResult { - Error, - Done, - NeedInput, - MoreOutput, -}; - -struct BrotliDecoder { - explicit BrotliDecoder(std::span<char> output_buffer) - : output_buffer_(output_buffer), - decoder_(BrotliDecoderCreateInstance(nullptr, nullptr, nullptr), - BrotliDecoderDestroyInstance) {} - - void Append(Block&& block) { input_buffer_.append(std::move(block)); } - - BrotliDecodeResult Decode(std::span<char>* output) { - size_t available_in = input_buffer_.front_size(); - const uint8_t* next_in = reinterpret_cast<const uint8_t*>(input_buffer_.front_data()); - - size_t available_out = output_buffer_.size(); - uint8_t* next_out = reinterpret_cast<uint8_t*>(output_buffer_.data()); - - BrotliDecoderResult r = BrotliDecoderDecompressStream( - decoder_.get(), &available_in, &next_in, &available_out, &next_out, nullptr); - - size_t bytes_consumed = input_buffer_.front_size() - available_in; - input_buffer_.drop_front(bytes_consumed); - - size_t bytes_emitted = output_buffer_.size() - available_out; - *output = std::span<char>(output_buffer_.data(), bytes_emitted); - - switch (r) { - case BROTLI_DECODER_RESULT_SUCCESS: - return BrotliDecodeResult::Done; - case BROTLI_DECODER_RESULT_ERROR: - return BrotliDecodeResult::Error; - case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT: - // Brotli guarantees as one of its invariants that if it returns NEEDS_MORE_INPUT, - // it will consume the entire input buffer passed in, so we don't have to worry - // about bytes left over in the front block with more input remaining. - return BrotliDecodeResult::NeedInput; - case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT: - return BrotliDecodeResult::MoreOutput; - } - } - - private: - IOVector input_buffer_; - std::span<char> output_buffer_; - std::unique_ptr<BrotliDecoderState, void (*)(BrotliDecoderState*)> decoder_; -}; - -enum class BrotliEncodeResult { - Error, - Done, - NeedInput, - MoreOutput, -}; - -template <size_t OutputBlockSize> -struct BrotliEncoder { - explicit BrotliEncoder() - : output_block_(OutputBlockSize), - output_bytes_left_(OutputBlockSize), - encoder_(BrotliEncoderCreateInstance(nullptr, nullptr, nullptr), - BrotliEncoderDestroyInstance) { - BrotliEncoderSetParameter(encoder_.get(), BROTLI_PARAM_QUALITY, 1); - } - - void Append(Block input) { input_buffer_.append(std::move(input)); } - void Finish() { finished_ = true; } - - BrotliEncodeResult Encode(Block* output) { - output->clear(); - while (true) { - size_t available_in = input_buffer_.front_size(); - const uint8_t* next_in = reinterpret_cast<const uint8_t*>(input_buffer_.front_data()); - - size_t available_out = output_bytes_left_; - uint8_t* next_out = reinterpret_cast<uint8_t*>(output_block_.data() + - (OutputBlockSize - output_bytes_left_)); - - BrotliEncoderOperation op = BROTLI_OPERATION_PROCESS; - if (finished_) { - op = BROTLI_OPERATION_FINISH; - } - - if (!BrotliEncoderCompressStream(encoder_.get(), op, &available_in, &next_in, - &available_out, &next_out, nullptr)) { - return BrotliEncodeResult::Error; - } - - size_t bytes_consumed = input_buffer_.front_size() - available_in; - input_buffer_.drop_front(bytes_consumed); - - output_bytes_left_ = available_out; - - if (BrotliEncoderIsFinished(encoder_.get())) { - output_block_.resize(OutputBlockSize - output_bytes_left_); - *output = std::move(output_block_); - return BrotliEncodeResult::Done; - } else if (output_bytes_left_ == 0) { - *output = std::move(output_block_); - output_block_.resize(OutputBlockSize); - output_bytes_left_ = OutputBlockSize; - return BrotliEncodeResult::MoreOutput; - } else if (input_buffer_.empty()) { - return BrotliEncodeResult::NeedInput; - } - } - } - - private: - bool finished_ = false; - IOVector input_buffer_; - Block output_block_; - size_t output_bytes_left_; - std::unique_ptr<BrotliEncoderState, void (*)(BrotliEncoderState*)> encoder_; -}; diff --git a/adb/client/adb_client.cpp b/adb/client/adb_client.cpp index f724cb522..a30873236 100644 --- a/adb/client/adb_client.cpp +++ b/adb/client/adb_client.cpp @@ -37,6 +37,7 @@ #include <vector> #include <android-base/file.h> +#include <android-base/no_destructor.h> #include <android-base/stringprintf.h> #include <android-base/strings.h> #include <android-base/thread_annotations.h> @@ -204,9 +205,25 @@ bool adb_kill_server() { return false; } - // The server might send OKAY, so consume that. char buf[4]; - ReadFdExactly(fd.get(), buf, 4); + if (!ReadFdExactly(fd.get(), buf, 4)) { + fprintf(stderr, "error: failed to read response from server\n"); + return false; + } + + if (memcmp(buf, "OKAY", 4) == 0) { + // Nothing to do. + } else if (memcmp(buf, "FAIL", 4) == 0) { + std::string output, error; + if (!ReadProtocolString(fd.get(), &output, &error)) { + fprintf(stderr, "error: %s\n", error.c_str()); + return false; + } + + fprintf(stderr, "error: %s\n", output.c_str()); + return false; + } + // Now that no more data is expected, wait for socket orderly shutdown or error, indicating // server death. ReadOrderlyShutdown(fd.get()); @@ -399,18 +416,20 @@ std::string format_host_command(const char* command) { return android::base::StringPrintf("%s:%s", prefix, command); } -bool adb_get_feature_set(FeatureSet* feature_set, std::string* error) { - static FeatureSet* features = nullptr; +const std::optional<FeatureSet>& adb_get_feature_set(std::string* error) { + static std::mutex feature_mutex [[clang::no_destroy]]; + static std::optional<FeatureSet> features [[clang::no_destroy]] GUARDED_BY(feature_mutex); + std::lock_guard<std::mutex> lock(feature_mutex); if (!features) { std::string result; - if (adb_query(format_host_command("features"), &result, error)) { - features = new FeatureSet(StringToFeatureSet(result)); + std::string err; + if (adb_query(format_host_command("features"), &result, &err)) { + features = StringToFeatureSet(result); + } else { + if (error) { + *error = err; + } } } - if (features) { - *feature_set = *features; - return true; - } - feature_set->clear(); - return false; + return features; } diff --git a/adb/client/adb_client.h b/adb/client/adb_client.h index 1c6cde77d..caf4e86ac 100644 --- a/adb/client/adb_client.h +++ b/adb/client/adb_client.h @@ -76,7 +76,7 @@ bool adb_status(borrowed_fd fd, std::string* _Nonnull error); std::string format_host_command(const char* _Nonnull command); // Get the feature set of the current preferred transport. -bool adb_get_feature_set(FeatureSet* _Nonnull feature_set, std::string* _Nonnull error); +const std::optional<FeatureSet>& adb_get_feature_set(std::string* _Nullable error); #if defined(__linux__) // Get the path of a file containing the path to the server executable, if the socket spec set via @@ -90,8 +90,9 @@ extern const char* _Nullable * _Nullable __adb_envp; // ADB Secure DNS service interface. Used to query what ADB Secure DNS services have been // resolved, and to run some kind of callback for each one. -using adb_secure_foreach_service_callback = std::function<void( - const char* _Nonnull service_name, const char* _Nonnull ip_address, uint16_t port)>; +using adb_secure_foreach_service_callback = + std::function<void(const char* _Nonnull service_name, const char* _Nonnull reg_type, + const char* _Nonnull ip_address, uint16_t port)>; // Queries pairing/connect services that have been discovered and resolved. // If |host_name| is not null, run |cb| only for services diff --git a/adb/client/adb_install.cpp b/adb/client/adb_install.cpp index f2de0d23d..d6f536e6e 100644 --- a/adb/client/adb_install.cpp +++ b/adb/client/adb_install.cpp @@ -57,13 +57,12 @@ enum class CmdlineOption { None, Enable, Disable }; } static bool can_use_feature(const char* feature) { - FeatureSet features; - std::string error; - if (!adb_get_feature_set(&features, &error)) { - fprintf(stderr, "error: %s\n", error.c_str()); + // We ignore errors here, if the device is missing, we'll notice when we try to push install. + auto&& features = adb_get_feature_set(nullptr); + if (!features) { return false; } - return CanUseFeature(features, feature); + return CanUseFeature(*features, feature); } static InstallMode best_install_mode() { @@ -293,7 +292,7 @@ static int install_app_legacy(int argc, const char** argv, bool use_fastdeploy) } } - if (do_sync_push(apk_file, apk_dest.c_str(), false, true)) { + if (do_sync_push(apk_file, apk_dest.c_str(), false, CompressionType::Any, false)) { result = pm_command(argc, argv); delete_device_file(apk_dest); } diff --git a/adb/client/adb_wifi.cpp b/adb/client/adb_wifi.cpp index fa7102811..61a9a480c 100644 --- a/adb/client/adb_wifi.cpp +++ b/adb/client/adb_wifi.cpp @@ -179,17 +179,21 @@ bool adb_wifi_is_known_host(const std::string& host) { void adb_wifi_pair_device(const std::string& host, const std::string& password, std::string& response) { - // Check the address for a valid address and port. - std::string parsed_host; - std::string err; - int port = -1; - if (!android::base::ParseNetAddress(host, &parsed_host, &port, nullptr, &err)) { - response = "Failed to parse address for pairing: " + err; - return; - } - if (port <= 0 || port > 65535) { - response = "Invalid port while parsing address [" + host + "]"; - return; + auto mdns_info = mdns_get_pairing_service_info(host); + + if (!mdns_info.has_value()) { + // Check the address for a valid address and port. + std::string parsed_host; + std::string err; + int port = -1; + if (!android::base::ParseNetAddress(host, &parsed_host, &port, nullptr, &err)) { + response = "Failed to parse address for pairing: " + err; + return; + } + if (port <= 0 || port > 65535) { + response = "Invalid port while parsing address [" + host + "]"; + return; + } } auto priv_key = adb_auth_get_user_privkey(); @@ -220,7 +224,11 @@ void adb_wifi_pair_device(const std::string& host, const std::string& password, PairingResultWaiter waiter; std::unique_lock<std::mutex> lock(waiter.mutex_); - if (!client->Start(host, waiter.OnResult, &waiter)) { + if (!client->Start(mdns_info.has_value() + ? android::base::StringPrintf("%s:%d", mdns_info->addr.c_str(), + mdns_info->port) + : host, + waiter.OnResult, &waiter)) { response = "Failed: Unable to start pairing client."; return; } diff --git a/adb/client/auth.cpp b/adb/client/auth.cpp index 35264c76f..b674a8161 100644 --- a/adb/client/auth.cpp +++ b/adb/client/auth.cpp @@ -145,12 +145,12 @@ static bool load_key(const std::string& file) { std::lock_guard<std::mutex> lock(g_keys_mutex); std::string fingerprint = hash_key(key.get()); - if (g_keys.find(fingerprint) != g_keys.end()) { - LOG(INFO) << "ignoring already-loaded key: " << file; - } else { - LOG(INFO) << "Loaded fingerprint=[" << SHA256BitsToHexString(fingerprint) << "]"; + bool already_loaded = (g_keys.find(fingerprint) != g_keys.end()); + if (!already_loaded) { g_keys[fingerprint] = std::move(key); } + LOG(INFO) << (already_loaded ? "ignored already-loaded" : "loaded new") << " key from '" << file + << "' with fingerprint " << SHA256BitsToHexString(fingerprint); return true; } @@ -159,23 +159,25 @@ static bool load_keys(const std::string& path, bool allow_dir = true) { struct stat st; if (stat(path.c_str(), &st) != 0) { - PLOG(ERROR) << "failed to stat '" << path << "'"; + PLOG(ERROR) << "load_keys: failed to stat '" << path << "'"; return false; } if (S_ISREG(st.st_mode)) { return load_key(path); - } else if (S_ISDIR(st.st_mode)) { + } + + if (S_ISDIR(st.st_mode)) { if (!allow_dir) { // inotify isn't recursive. It would break expectations to load keys in nested // directories but not monitor them for new keys. - LOG(WARNING) << "refusing to recurse into directory '" << path << "'"; + LOG(WARNING) << "load_keys: refusing to recurse into directory '" << path << "'"; return false; } std::unique_ptr<DIR, decltype(&closedir)> dir(opendir(path.c_str()), closedir); if (!dir) { - PLOG(ERROR) << "failed to open directory '" << path << "'"; + PLOG(ERROR) << "load_keys: failed to open directory '" << path << "'"; return false; } @@ -189,7 +191,7 @@ static bool load_keys(const std::string& path, bool allow_dir = true) { } if (!android::base::EndsWith(name, ".adb_key")) { - LOG(INFO) << "skipping non-adb_key '" << path << "/" << name << "'"; + LOG(INFO) << "skipped non-adb_key '" << path << "/" << name << "'"; continue; } @@ -198,7 +200,7 @@ static bool load_keys(const std::string& path, bool allow_dir = true) { return result; } - LOG(ERROR) << "unexpected type for '" << path << "': 0x" << std::hex << st.st_mode; + LOG(ERROR) << "load_keys: unexpected type for '" << path << "': 0x" << std::hex << st.st_mode; return false; } diff --git a/adb/client/bugreport.cpp b/adb/client/bugreport.cpp index 8ca44e8b5..e162aaad1 100644 --- a/adb/client/bugreport.cpp +++ b/adb/client/bugreport.cpp @@ -282,5 +282,5 @@ int Bugreport::SendShellCommand(const std::string& command, bool disable_shell_p bool Bugreport::DoSyncPull(const std::vector<const char*>& srcs, const char* dst, bool copy_attrs, const char* name) { - return do_sync_pull(srcs, dst, copy_attrs, name); + return do_sync_pull(srcs, dst, copy_attrs, CompressionType::None, name); } diff --git a/adb/client/commandline.cpp b/adb/client/commandline.cpp index ad4e21c4d..43772ba45 100644 --- a/adb/client/commandline.cpp +++ b/adb/client/commandline.cpp @@ -50,6 +50,8 @@ #include <unistd.h> #endif +#include <google/protobuf/text_format.h> + #include "adb.h" #include "adb_auth.h" #include "adb_client.h" @@ -57,6 +59,7 @@ #include "adb_io.h" #include "adb_unique_fd.h" #include "adb_utils.h" +#include "app_processes.pb.h" #include "bugreport.h" #include "client/file_sync_client.h" #include "commandline.h" @@ -101,7 +104,8 @@ static void help() { " connect HOST[:PORT] connect to a device via TCP/IP [default port=5555]\n" " disconnect [HOST[:PORT]]\n" " disconnect from given TCP/IP device [default port=5555], or all\n" - " pair HOST[:PORT] pair with a device for secure TCP/IP communication\n" + " pair HOST[:PORT] [PAIRING CODE]\n" + " pair with a device for secure TCP/IP communication\n" " forward --list list all forward socket connections\n" " forward [--no-rebind] LOCAL REMOTE\n" " forward socket connection using:\n" @@ -124,22 +128,26 @@ static void help() { " localfilesystem:<unix domain socket name>\n" " reverse --remove REMOTE remove specific reverse socket connection\n" " reverse --remove-all remove all reverse socket connections from device\n" + " mdns check check if mdns discovery is available\n" + " mdns services list all discovered services\n" "\n" "file transfer:\n" - " push [--sync] [-zZ] LOCAL... REMOTE\n" + " push [--sync] [-z ALGORITHM] [-Z] LOCAL... REMOTE\n" " copy local files/directories to device\n" " --sync: only push files that are newer on the host than the device\n" - " -z: enable compression\n" + " -n: dry run: push files to device without storing to the filesystem\n" + " -z: enable compression with a specified algorithm (any, none, brotli)\n" " -Z: disable compression\n" - " pull [-azZ] REMOTE... LOCAL\n" + " pull [-a] [-z ALGORITHM] [-Z] REMOTE... LOCAL\n" " copy files/dirs from device\n" " -a: preserve file timestamp and mode\n" - " -z: enable compression\n" + " -z: enable compression with a specified algorithm (any, none, brotli)\n" " -Z: disable compression\n" - " sync [-lzZ] [all|data|odm|oem|product|system|system_ext|vendor]\n" + " sync [-l] [-z ALGORITHM] [-Z] [all|data|odm|oem|product|system|system_ext|vendor]\n" " sync a local build from $ANDROID_PRODUCT_OUT to the device (default all)\n" + " -n: dry run: push files to device without storing to the filesystem\n" " -l: list files that would be copied, but don't copy them\n" - " -z: enable compression\n" + " -z: enable compression with a specified algorithm (any, none, brotli)\n" " -Z: disable compression\n" "\n" "shell:\n" @@ -197,8 +205,8 @@ static void help() { " generate adb public/private key; private key stored in FILE,\n" "\n" "scripting:\n" - " wait-for[-TRANSPORT]-STATE\n" - " wait for device to be in the given state\n" + " wait-for[-TRANSPORT]-STATE...\n" + " wait for device to be in a given state\n" " STATE: device, recovery, rescue, sideload, bootloader, or disconnect\n" " TRANSPORT: usb, local, or any [default=any]\n" " get-state print offline | bootloader | device\n" @@ -233,6 +241,7 @@ static void help() { " $ANDROID_SERIAL serial number to connect to (see -s)\n" " $ANDROID_LOG_TAGS tags to be used by logcat (see logcat --help)\n" " $ADB_LOCAL_TRANSPORT_MAX_PORT max emulator scan port (default 5585, 16 emus)\n" + " $ADB_MDNS_AUTO_CONNECT comma-separated list of mdns services to allow auto-connect (default adb-tls-connect)\n" ); // clang-format on } @@ -667,18 +676,17 @@ static int RemoteShell(bool use_shell_protocol, const std::string& type_arg, cha } static int adb_shell(int argc, const char** argv) { - FeatureSet features; - std::string error_message; - if (!adb_get_feature_set(&features, &error_message)) { - fprintf(stderr, "error: %s\n", error_message.c_str()); - return 1; + std::string error; + auto&& features = adb_get_feature_set(&error); + if (!features) { + error_exit("%s", error.c_str()); } enum PtyAllocationMode { kPtyAuto, kPtyNo, kPtyYes, kPtyDefinitely }; // Defaults. - char escape_char = '~'; // -e - bool use_shell_protocol = CanUseFeature(features, kFeatureShell2); // -x + char escape_char = '~'; // -e + bool use_shell_protocol = CanUseFeature(*features, kFeatureShell2); // -x PtyAllocationMode tty = use_shell_protocol ? kPtyAuto : kPtyDefinitely; // -t/-T // Parse shell-specific command-line options. @@ -754,7 +762,7 @@ static int adb_shell(int argc, const char** argv) { if (!use_shell_protocol) { if (shell_type_arg != kShellServiceArgPty) { fprintf(stderr, "error: %s only supports allocating a pty\n", - !CanUseFeature(features, kFeatureShell2) ? "device" : "-x"); + !CanUseFeature(*features, kFeatureShell2) ? "device" : "-x"); return 1; } else { // If we're not using the shell protocol, the type argument must be empty. @@ -774,14 +782,13 @@ static int adb_shell(int argc, const char** argv) { } static int adb_abb(int argc, const char** argv) { - FeatureSet features; - std::string error_message; - if (!adb_get_feature_set(&features, &error_message)) { - fprintf(stderr, "error: %s\n", error_message.c_str()); + std::string error; + auto&& features = adb_get_feature_set(&error); + if (!features) { + error_exit("%s", error.c_str()); return 1; } - - if (!CanUseFeature(features, kFeatureAbb)) { + if (!CanUseFeature(*features, kFeatureAbb)) { error_exit("abb is not supported by the device"); } @@ -1060,17 +1067,16 @@ static int ppp(int argc, const char** argv) { static bool wait_for_device(const char* service, std::optional<std::chrono::milliseconds> timeout = std::nullopt) { std::vector<std::string> components = android::base::Split(service, "-"); - if (components.size() < 3 || components.size() > 4) { + if (components.size() < 3) { fprintf(stderr, "adb: couldn't parse 'wait-for' command: %s\n", service); return false; } - TransportType t; - adb_get_transport(&t, nullptr, nullptr); - - // Was the caller vague about what they'd like us to wait for? - // If so, check they weren't more specific in their choice of transport type. - if (components.size() == 3) { + // If the first thing after "wait-for-" wasn't a TRANSPORT, insert whatever + // the current transport implies. + if (components[2] != "usb" && components[2] != "local" && components[2] != "any") { + TransportType t; + adb_get_transport(&t, nullptr, nullptr); auto it = components.begin() + 2; if (t == kTransportUsb) { components.insert(it, "usb"); @@ -1079,23 +1085,9 @@ static bool wait_for_device(const char* service, } else { components.insert(it, "any"); } - } else if (components[2] != "any" && components[2] != "local" && components[2] != "usb") { - fprintf(stderr, "adb: unknown type %s; expected 'any', 'local', or 'usb'\n", - components[2].c_str()); - return false; - } - - if (components[3] != "any" && components[3] != "bootloader" && components[3] != "device" && - components[3] != "recovery" && components[3] != "rescue" && components[3] != "sideload" && - components[3] != "disconnect") { - fprintf(stderr, - "adb: unknown state %s; " - "expected 'any', 'bootloader', 'device', 'recovery', 'rescue', 'sideload', or " - "'disconnect'\n", - components[3].c_str()); - return false; } + // Stitch it back together and send it over... std::string cmd = format_host_command(android::base::Join(components, "-").c_str()); if (timeout) { std::thread([timeout]() { @@ -1174,10 +1166,9 @@ int send_shell_command(const std::string& command, bool disable_shell_protocol, // Use shell protocol if it's supported and the caller doesn't explicitly // disable it. if (!disable_shell_protocol) { - FeatureSet features; - std::string error; - if (adb_get_feature_set(&features, &error)) { - use_shell_protocol = CanUseFeature(features, kFeatureShell2); + auto&& features = adb_get_feature_set(nullptr); + if (features) { + use_shell_protocol = CanUseFeature(*features, kFeatureShell2); } else { // Device was unreachable. attempt_connection = false; @@ -1326,12 +1317,38 @@ static int restore(int argc, const char** argv) { return 0; } +static CompressionType parse_compression_type(const std::string& str, bool allow_numbers) { + if (allow_numbers) { + if (str == "0") { + return CompressionType::None; + } else if (str == "1") { + return CompressionType::Any; + } + } + + if (str == "any") { + return CompressionType::Any; + } else if (str == "none") { + return CompressionType::None; + } + + if (str == "brotli") { + return CompressionType::Brotli; + } else if (str == "lz4") { + return CompressionType::LZ4; + } else if (str == "zstd") { + return CompressionType::Zstd; + } + + error_exit("unexpected compression type %s", str.c_str()); +} + static void parse_push_pull_args(const char** arg, int narg, std::vector<const char*>* srcs, - const char** dst, bool* copy_attrs, bool* sync, bool* compressed) { + const char** dst, bool* copy_attrs, bool* sync, + CompressionType* compression, bool* dry_run) { *copy_attrs = false; - const char* adb_compression = getenv("ADB_COMPRESSION"); - if (adb_compression && strcmp(adb_compression, "0") == 0) { - *compressed = false; + if (const char* adb_compression = getenv("ADB_COMPRESSION")) { + *compression = parse_compression_type(adb_compression, true); } srcs->clear(); @@ -1345,13 +1362,15 @@ static void parse_push_pull_args(const char** arg, int narg, std::vector<const c } else if (!strcmp(*arg, "-a")) { *copy_attrs = true; } else if (!strcmp(*arg, "-z")) { - if (compressed != nullptr) { - *compressed = true; + if (narg < 2) { + error_exit("-z requires an argument"); } + *compression = parse_compression_type(*++arg, false); + --narg; } else if (!strcmp(*arg, "-Z")) { - if (compressed != nullptr) { - *compressed = false; - } + *compression = CompressionType::None; + } else if (dry_run && !strcmp(*arg, "-n")) { + *dry_run = true; } else if (!strcmp(*arg, "--sync")) { if (sync != nullptr) { *sync = true; @@ -1372,17 +1391,49 @@ static void parse_push_pull_args(const char** arg, int narg, std::vector<const c } } -static int adb_connect_command(const std::string& command, TransportId* transport = nullptr) { +static int adb_connect_command(const std::string& command, TransportId* transport, + StandardStreamsCallbackInterface* callback) { std::string error; unique_fd fd(adb_connect(transport, command, &error)); if (fd < 0) { fprintf(stderr, "error: %s\n", error.c_str()); return 1; } - read_and_dump(fd); + read_and_dump(fd, false, callback); return 0; } +static int adb_connect_command(const std::string& command, TransportId* transport = nullptr) { + return adb_connect_command(command, transport, &DEFAULT_STANDARD_STREAMS_CALLBACK); +} + +// A class that prints out human readable form of the protobuf message for "track-app" service +// (received in binary format). +class TrackAppStreamsCallback : public DefaultStandardStreamsCallback { + public: + TrackAppStreamsCallback() : DefaultStandardStreamsCallback(nullptr, nullptr) {} + + // Assume the buffer contains at least 4 bytes of valid data. + void OnStdout(const char* buffer, int length) override { + if (length < 4) return; // Unexpected length received. Do nothing. + + adb::proto::AppProcesses binary_proto; + // The first 4 bytes are the length of remaining content in hexadecimal format. + binary_proto.ParseFromString(std::string(buffer + 4, length - 4)); + char summary[24]; // The following string includes digits and 16 fixed characters. + int written = snprintf(summary, sizeof(summary), "Process count: %d\n", + binary_proto.process_size()); + OnStream(nullptr, stdout, summary, written); + + std::string string_proto; + google::protobuf::TextFormat::PrintToString(binary_proto, &string_proto); + OnStream(nullptr, stdout, string_proto.data(), string_proto.length()); + } + + private: + DISALLOW_COPY_AND_ASSIGN(TrackAppStreamsCallback); +}; + static int adb_connect_command_bidirectional(const std::string& command) { std::string error; unique_fd fd(adb_connect(command, &error)); @@ -1664,14 +1715,21 @@ int adb_commandline(int argc, const char** argv) { } printf("List of devices attached\n"); return adb_query_command(query); - } - else if (!strcmp(argv[0], "connect")) { + } else if (!strcmp(argv[0], "transport-id")) { + TransportId transport_id; + std::string error; + unique_fd fd(adb_connect(&transport_id, "host:features", &error, true)); + if (fd == -1) { + error_exit("%s", error.c_str()); + } + printf("%" PRIu64 "\n", transport_id); + return 0; + } else if (!strcmp(argv[0], "connect")) { if (argc != 2) error_exit("usage: adb connect HOST[:PORT]"); std::string query = android::base::StringPrintf("host:connect:%s", argv[1]); return adb_query_command(query); - } - else if (!strcmp(argv[0], "disconnect")) { + } else if (!strcmp(argv[0], "disconnect")) { if (argc > 2) error_exit("usage: adb disconnect [HOST[:PORT]]"); std::string query = android::base::StringPrintf("host:disconnect:%s", @@ -1680,13 +1738,17 @@ int adb_commandline(int argc, const char** argv) { } else if (!strcmp(argv[0], "abb")) { return adb_abb(argc, argv); } else if (!strcmp(argv[0], "pair")) { - if (argc != 2) error_exit("usage: adb pair <host>[:<port>]"); + if (argc < 2 || argc > 3) error_exit("usage: adb pair HOST[:PORT] [PAIRING CODE]"); std::string password; - printf("Enter pairing code: "); - fflush(stdout); - if (!std::getline(std::cin, password) || password.empty()) { - error_exit("No pairing code provided"); + if (argc == 2) { + printf("Enter pairing code: "); + fflush(stdout); + if (!std::getline(std::cin, password) || password.empty()) { + error_exit("No pairing code provided"); + } + } else { + password = argv[2]; } std::string query = android::base::StringPrintf("host:pair:%s:%s", password.c_str(), argv[1]); @@ -1768,14 +1830,13 @@ int adb_commandline(int argc, const char** argv) { } return adb_connect_command(android::base::StringPrintf("tcpip:%d", port)); } else if (!strcmp(argv[0], "remount")) { - FeatureSet features; std::string error; - if (!adb_get_feature_set(&features, &error)) { - fprintf(stderr, "error: %s\n", error.c_str()); - return 1; + auto&& features = adb_get_feature_set(&error); + if (!features) { + error_exit("%s", error.c_str()); } - if (CanUseFeature(features, kFeatureRemountShell)) { + if (CanUseFeature(*features, kFeatureRemountShell)) { std::vector<const char*> args = {"shell"}; args.insert(args.cend(), argv, argv + argc); return adb_shell_noinput(args.size(), args.data()); @@ -1866,6 +1927,29 @@ int adb_commandline(int argc, const char** argv) { ReadOrderlyShutdown(fd); return 0; + } else if (!strcmp(argv[0], "mdns")) { + --argc; + if (argc < 1) error_exit("mdns requires an argument"); + ++argv; + + std::string error; + if (!adb_check_server_version(&error)) { + error_exit("failed to check server version: %s", error.c_str()); + } + + std::string query = "host:mdns:"; + if (!strcmp(argv[0], "check")) { + if (argc != 1) error_exit("mdns %s doesn't take any arguments", argv[0]); + query += "check"; + } else if (!strcmp(argv[0], "services")) { + if (argc != 1) error_exit("mdns %s doesn't take any arguments", argv[0]); + query += "services"; + printf("List of discovered mdns services\n"); + } else { + error_exit("unknown mdns command [%s]", argv[0]); + } + + return adb_query_command(query); } /* do_sync_*() commands */ else if (!strcmp(argv[0], "ls")) { @@ -1874,22 +1958,25 @@ int adb_commandline(int argc, const char** argv) { } else if (!strcmp(argv[0], "push")) { bool copy_attrs = false; bool sync = false; - bool compressed = true; + bool dry_run = false; + CompressionType compression = CompressionType::Any; std::vector<const char*> srcs; const char* dst = nullptr; - parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, ©_attrs, &sync, &compressed); + parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, ©_attrs, &sync, &compression, + &dry_run); if (srcs.empty() || !dst) error_exit("push requires an argument"); - return do_sync_push(srcs, dst, sync, compressed) ? 0 : 1; + return do_sync_push(srcs, dst, sync, compression, dry_run) ? 0 : 1; } else if (!strcmp(argv[0], "pull")) { bool copy_attrs = false; - bool compressed = true; + CompressionType compression = CompressionType::Any; std::vector<const char*> srcs; const char* dst = "."; - parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, ©_attrs, nullptr, &compressed); + parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, ©_attrs, nullptr, &compression, + nullptr); if (srcs.empty()) error_exit("pull requires an argument"); - return do_sync_pull(srcs, dst, copy_attrs, compressed) ? 0 : 1; + return do_sync_pull(srcs, dst, copy_attrs, compression) ? 0 : 1; } else if (!strcmp(argv[0], "install")) { if (argc < 2) error_exit("install requires an argument"); return install_app(argc, argv); @@ -1905,27 +1992,30 @@ int adb_commandline(int argc, const char** argv) { } else if (!strcmp(argv[0], "sync")) { std::string src; bool list_only = false; - bool compressed = true; + bool dry_run = false; + CompressionType compression = CompressionType::Any; - const char* adb_compression = getenv("ADB_COMPRESSION"); - if (adb_compression && strcmp(adb_compression, "0") == 0) { - compressed = false; + if (const char* adb_compression = getenv("ADB_COMPRESSION"); adb_compression) { + compression = parse_compression_type(adb_compression, true); } int opt; - while ((opt = getopt(argc, const_cast<char**>(argv), "lzZ")) != -1) { + while ((opt = getopt(argc, const_cast<char**>(argv), "lnz:Z")) != -1) { switch (opt) { case 'l': list_only = true; break; + case 'n': + dry_run = true; + break; case 'z': - compressed = true; + compression = parse_compression_type(optarg, false); break; case 'Z': - compressed = false; + compression = CompressionType::None; break; default: - error_exit("usage: adb sync [-lzZ] [PARTITION]"); + error_exit("usage: adb sync [-l] [-n] [-z ALGORITHM] [-Z] [PARTITION]"); } } @@ -1934,7 +2024,7 @@ int adb_commandline(int argc, const char** argv) { } else if (optind + 1 == argc) { src = argv[optind]; } else { - error_exit("usage: adb sync [-lzZ] [PARTITION]"); + error_exit("usage: adb sync [-l] [-n] [-z ALGORITHM] [-Z] [PARTITION]"); } std::vector<std::string> partitions{"data", "odm", "oem", "product", @@ -1945,7 +2035,9 @@ int adb_commandline(int argc, const char** argv) { std::string src_dir{product_file(partition)}; if (!directory_exists(src_dir)) continue; found = true; - if (!do_sync_sync(src_dir, "/" + partition, list_only, compressed)) return 1; + if (!do_sync_sync(src_dir, "/" + partition, list_only, compression, dry_run)) { + return 1; + } } } if (!found) error_exit("don't know how to sync %s partition", src.c_str()); @@ -1985,6 +2077,17 @@ int adb_commandline(int argc, const char** argv) { return adb_connect_command("jdwp"); } else if (!strcmp(argv[0], "track-jdwp")) { return adb_connect_command("track-jdwp"); + } else if (!strcmp(argv[0], "track-app")) { + std::string error; + auto&& features = adb_get_feature_set(&error); + if (!features) { + error_exit("%s", error.c_str()); + } + if (!CanUseFeature(*features, kFeatureTrackApp)) { + error_exit("track-app is not supported by the device"); + } + TrackAppStreamsCallback callback; + return adb_connect_command("track-app", nullptr, &callback); } else if (!strcmp(argv[0], "track-devices")) { if (argc > 2 || (argc == 2 && strcmp(argv[1], "-l"))) { error_exit("usage: adb track-devices [-l]"); @@ -2006,15 +2109,14 @@ int adb_commandline(int argc, const char** argv) { return 0; } else if (!strcmp(argv[0], "features")) { // Only list the features common to both the adb client and the device. - FeatureSet features; std::string error; - if (!adb_get_feature_set(&features, &error)) { - fprintf(stderr, "error: %s\n", error.c_str()); - return 1; + auto&& features = adb_get_feature_set(&error); + if (!features) { + error_exit("%s", error.c_str()); } - for (const std::string& name : features) { - if (CanUseFeature(features, name)) { + for (const std::string& name : *features) { + if (CanUseFeature(*features, name)) { printf("%s\n", name.c_str()); } } diff --git a/adb/client/fastdeploy.cpp b/adb/client/fastdeploy.cpp index de82e14e5..bc4b91bb9 100644 --- a/adb/client/fastdeploy.cpp +++ b/adb/client/fastdeploy.cpp @@ -112,7 +112,7 @@ static void push_to_device(const void* data, size_t byte_count, const char* dst, // but can't be removed until after the push. unix_close(tf.release()); - if (!do_sync_push(srcs, dst, sync, true)) { + if (!do_sync_push(srcs, dst, sync, CompressionType::Any, false)) { error_exit("Failed to push fastdeploy agent to device."); } } diff --git a/adb/client/file_sync_client.cpp b/adb/client/file_sync_client.cpp index e686973db..8bbe2a878 100644 --- a/adb/client/file_sync_client.cpp +++ b/adb/client/file_sync_client.cpp @@ -34,6 +34,7 @@ #include <memory> #include <sstream> #include <string> +#include <variant> #include <vector> #include "sysdeps.h" @@ -42,7 +43,7 @@ #include "adb_client.h" #include "adb_io.h" #include "adb_utils.h" -#include "brotli_utils.h" +#include "compression_utils.h" #include "file_sync_protocol.h" #include "line_printer.h" #include "sysdeps/errno.h" @@ -229,13 +230,19 @@ class SyncConnection { max = SYNC_DATA_MAX; // TODO: decide at runtime. std::string error; - if (!adb_get_feature_set(&features_, &error)) { + auto&& features = adb_get_feature_set(&error); + if (!features) { Error("failed to get feature set: %s", error.c_str()); } else { - have_stat_v2_ = CanUseFeature(features_, kFeatureStat2); - have_ls_v2_ = CanUseFeature(features_, kFeatureLs2); - have_sendrecv_v2_ = CanUseFeature(features_, kFeatureSendRecv2); - have_sendrecv_v2_brotli_ = CanUseFeature(features_, kFeatureSendRecv2Brotli); + features_ = &*features; + have_stat_v2_ = CanUseFeature(*features, kFeatureStat2); + have_ls_v2_ = CanUseFeature(*features, kFeatureLs2); + have_sendrecv_v2_ = CanUseFeature(*features, kFeatureSendRecv2); + have_sendrecv_v2_brotli_ = CanUseFeature(*features, kFeatureSendRecv2Brotli); + have_sendrecv_v2_lz4_ = CanUseFeature(*features, kFeatureSendRecv2LZ4); + have_sendrecv_v2_zstd_ = CanUseFeature(*features, kFeatureSendRecv2Zstd); + have_sendrecv_v2_dry_run_send_ = CanUseFeature(*features, kFeatureSendRecv2DryRunSend); + std::string error; fd.reset(adb_connect("sync:", &error)); if (fd < 0) { Error("connect failed: %s", error.c_str()); @@ -261,8 +268,27 @@ class SyncConnection { bool HaveSendRecv2() const { return have_sendrecv_v2_; } bool HaveSendRecv2Brotli() const { return have_sendrecv_v2_brotli_; } + bool HaveSendRecv2LZ4() const { return have_sendrecv_v2_lz4_; } + bool HaveSendRecv2Zstd() const { return have_sendrecv_v2_zstd_; } + bool HaveSendRecv2DryRunSend() const { return have_sendrecv_v2_dry_run_send_; } + + // Resolve a compression type which might be CompressionType::Any to a specific compression + // algorithm. + CompressionType ResolveCompressionType(CompressionType compression) const { + if (compression == CompressionType::Any) { + if (HaveSendRecv2Zstd()) { + return CompressionType::Zstd; + } else if (HaveSendRecv2LZ4()) { + return CompressionType::LZ4; + } else if (HaveSendRecv2Brotli()) { + return CompressionType::Brotli; + } + return CompressionType::None; + } + return compression; + } - const FeatureSet& Features() const { return features_; } + const FeatureSet& Features() const { return *features_; } bool IsValid() { return fd >= 0; } @@ -323,7 +349,7 @@ class SyncConnection { return WriteFdExactly(fd, buf.data(), buf.size()); } - bool SendSend2(std::string_view path, mode_t mode, bool compressed) { + bool SendSend2(std::string_view path, mode_t mode, CompressionType compression, bool dry_run) { if (path.length() > 1024) { Error("SendRequest failed: path too long: %zu", path.length()); errno = ENAMETOOLONG; @@ -339,7 +365,30 @@ class SyncConnection { syncmsg msg; msg.send_v2_setup.id = ID_SEND_V2; msg.send_v2_setup.mode = mode; - msg.send_v2_setup.flags = compressed ? kSyncFlagBrotli : kSyncFlagNone; + msg.send_v2_setup.flags = 0; + switch (compression) { + case CompressionType::None: + break; + + case CompressionType::Brotli: + msg.send_v2_setup.flags = kSyncFlagBrotli; + break; + + case CompressionType::LZ4: + msg.send_v2_setup.flags = kSyncFlagLZ4; + break; + + case CompressionType::Zstd: + msg.send_v2_setup.flags = kSyncFlagZstd; + break; + + case CompressionType::Any: + LOG(FATAL) << "unexpected CompressionType::Any"; + } + + if (dry_run) { + msg.send_v2_setup.flags |= kSyncFlagDryRun; + } buf.resize(sizeof(SyncRequest) + path.length() + sizeof(msg.send_v2_setup)); @@ -352,7 +401,7 @@ class SyncConnection { return WriteFdExactly(fd, buf.data(), buf.size()); } - bool SendRecv2(const std::string& path) { + bool SendRecv2(const std::string& path, CompressionType compression) { if (path.length() > 1024) { Error("SendRequest failed: path too long: %zu", path.length()); errno = ENAMETOOLONG; @@ -367,7 +416,26 @@ class SyncConnection { syncmsg msg; msg.recv_v2_setup.id = ID_RECV_V2; - msg.recv_v2_setup.flags = kSyncFlagBrotli; + msg.recv_v2_setup.flags = 0; + switch (compression) { + case CompressionType::None: + break; + + case CompressionType::Brotli: + msg.recv_v2_setup.flags |= kSyncFlagBrotli; + break; + + case CompressionType::LZ4: + msg.recv_v2_setup.flags |= kSyncFlagLZ4; + break; + + case CompressionType::Zstd: + msg.recv_v2_setup.flags |= kSyncFlagZstd; + break; + + case CompressionType::Any: + LOG(FATAL) << "unexpected CompressionType::Any"; + } buf.resize(sizeof(SyncRequest) + path.length() + sizeof(msg.recv_v2_setup)); @@ -494,7 +562,12 @@ class SyncConnection { // difference to "adb sync" performance. bool SendSmallFile(const std::string& path, mode_t mode, const std::string& lpath, const std::string& rpath, unsigned mtime, const char* data, - size_t data_length) { + size_t data_length, bool dry_run) { + if (dry_run) { + // We need to use send v2 for dry run. + return SendLargeFile(path, mode, lpath, rpath, mtime, CompressionType::None, dry_run); + } + std::string path_and_mode = android::base::StringPrintf("%s,%d", path.c_str(), mode); if (path_and_mode.length() > 1024) { Error("SendSmallFile failed: path too long: %zu", path_and_mode.length()); @@ -533,9 +606,21 @@ class SyncConnection { return true; } - bool SendLargeFileCompressed(const std::string& path, mode_t mode, const std::string& lpath, - const std::string& rpath, unsigned mtime) { - if (!SendSend2(path, mode, true)) { + bool SendLargeFile(const std::string& path, mode_t mode, const std::string& lpath, + const std::string& rpath, unsigned mtime, CompressionType compression, + bool dry_run) { + if (dry_run && !HaveSendRecv2DryRunSend()) { + Error("dry-run not supported by the device"); + return false; + } + + if (!HaveSendRecv2()) { + return SendLargeFileLegacy(path, mode, lpath, rpath, mtime); + } + + compression = ResolveCompressionType(compression); + + if (!SendSend2(path, mode, compression, dry_run)) { Error("failed to send ID_SEND_V2 message '%s': %s", path.c_str(), strerror(errno)); return false; } @@ -558,7 +643,30 @@ class SyncConnection { syncsendbuf sbuf; sbuf.id = ID_DATA; - BrotliEncoder<SYNC_DATA_MAX> encoder; + std::variant<std::monostate, NullEncoder, BrotliEncoder, LZ4Encoder, ZstdEncoder> + encoder_storage; + Encoder* encoder = nullptr; + switch (compression) { + case CompressionType::None: + encoder = &encoder_storage.emplace<NullEncoder>(SYNC_DATA_MAX); + break; + + case CompressionType::Brotli: + encoder = &encoder_storage.emplace<BrotliEncoder>(SYNC_DATA_MAX); + break; + + case CompressionType::LZ4: + encoder = &encoder_storage.emplace<LZ4Encoder>(SYNC_DATA_MAX); + break; + + case CompressionType::Zstd: + encoder = &encoder_storage.emplace<ZstdEncoder>(SYNC_DATA_MAX); + break; + + case CompressionType::Any: + LOG(FATAL) << "unexpected CompressionType::Any"; + } + bool sending = true; while (sending) { Block input(SYNC_DATA_MAX); @@ -569,10 +677,10 @@ class SyncConnection { } if (r == 0) { - encoder.Finish(); + encoder->Finish(); } else { input.resize(r); - encoder.Append(std::move(input)); + encoder->Append(std::move(input)); RecordBytesTransferred(r); bytes_copied += r; ReportProgress(rpath, bytes_copied, total_size); @@ -580,8 +688,8 @@ class SyncConnection { while (true) { Block output; - BrotliEncodeResult result = encoder.Encode(&output); - if (result == BrotliEncodeResult::Error) { + EncodeResult result = encoder->Encode(&output); + if (result == EncodeResult::Error) { Error("compressing '%s' locally failed", lpath.c_str()); return false; } @@ -592,12 +700,12 @@ class SyncConnection { WriteOrDie(lpath, rpath, &sbuf, sizeof(SyncRequest) + output.size()); } - if (result == BrotliEncodeResult::Done) { + if (result == EncodeResult::Done) { sending = false; break; - } else if (result == BrotliEncodeResult::NeedInput) { + } else if (result == EncodeResult::NeedInput) { break; - } else if (result == BrotliEncodeResult::MoreOutput) { + } else if (result == EncodeResult::MoreOutput) { continue; } } @@ -610,12 +718,8 @@ class SyncConnection { return WriteOrDie(lpath, rpath, &msg.data, sizeof(msg.data)); } - bool SendLargeFile(const std::string& path, mode_t mode, const std::string& lpath, - const std::string& rpath, unsigned mtime, bool compressed) { - if (compressed && HaveSendRecv2Brotli()) { - return SendLargeFileCompressed(path, mode, lpath, rpath, mtime); - } - + bool SendLargeFileLegacy(const std::string& path, mode_t mode, const std::string& lpath, + const std::string& rpath, unsigned mtime) { std::string path_and_mode = android::base::StringPrintf("%s,%d", path.c_str(), mode); if (!SendRequest(ID_SEND_V1, path_and_mode)) { Error("failed to send ID_SEND_V1 message '%s': %s", path_and_mode.c_str(), @@ -835,11 +939,14 @@ class SyncConnection { private: std::deque<std::pair<std::string, std::string>> deferred_acknowledgements_; Block acknowledgement_buffer_; - FeatureSet features_; + const FeatureSet* features_ = nullptr; bool have_stat_v2_; bool have_ls_v2_; bool have_sendrecv_v2_; bool have_sendrecv_v2_brotli_; + bool have_sendrecv_v2_lz4_; + bool have_sendrecv_v2_zstd_; + bool have_sendrecv_v2_dry_run_send_; TransferLedger global_ledger_; TransferLedger current_ledger_; @@ -921,7 +1028,8 @@ static bool sync_stat_fallback(SyncConnection& sc, const std::string& path, stru } static bool sync_send(SyncConnection& sc, const std::string& lpath, const std::string& rpath, - unsigned mtime, mode_t mode, bool sync, bool compressed) { + unsigned mtime, mode_t mode, bool sync, CompressionType compression, + bool dry_run) { if (sync) { struct stat st; if (sync_lstat(sc, rpath, &st)) { @@ -942,7 +1050,7 @@ static bool sync_send(SyncConnection& sc, const std::string& lpath, const std::s } buf[data_length++] = '\0'; - if (!sc.SendSmallFile(rpath, mode, lpath, rpath, mtime, buf, data_length)) { + if (!sc.SendSmallFile(rpath, mode, lpath, rpath, mtime, buf, data_length, dry_run)) { return false; } return sc.ReadAcknowledgements(); @@ -960,11 +1068,12 @@ static bool sync_send(SyncConnection& sc, const std::string& lpath, const std::s sc.Error("failed to read all of '%s': %s", lpath.c_str(), strerror(errno)); return false; } - if (!sc.SendSmallFile(rpath, mode, lpath, rpath, mtime, data.data(), data.size())) { + if (!sc.SendSmallFile(rpath, mode, lpath, rpath, mtime, data.data(), data.size(), + dry_run)) { return false; } } else { - if (!sc.SendLargeFile(rpath, mode, lpath, rpath, mtime, compressed)) { + if (!sc.SendLargeFile(rpath, mode, lpath, rpath, mtime, compression, dry_run)) { return false; } } @@ -1027,8 +1136,10 @@ static bool sync_recv_v1(SyncConnection& sc, const char* rpath, const char* lpat } static bool sync_recv_v2(SyncConnection& sc, const char* rpath, const char* lpath, const char* name, - uint64_t expected_size) { - if (!sc.SendRecv2(rpath)) return false; + uint64_t expected_size, CompressionType compression) { + compression = sc.ResolveCompressionType(compression); + + if (!sc.SendRecv2(rpath, compression)) return false; adb_unlink(lpath); unique_fd lfd(adb_creat(lpath, 0644)); @@ -1040,9 +1151,33 @@ static bool sync_recv_v2(SyncConnection& sc, const char* rpath, const char* lpat uint64_t bytes_copied = 0; Block buffer(SYNC_DATA_MAX); - BrotliDecoder decoder(std::span(buffer.data(), buffer.size())); - bool reading = true; - while (reading) { + std::variant<std::monostate, NullDecoder, BrotliDecoder, LZ4Decoder, ZstdDecoder> + decoder_storage; + Decoder* decoder = nullptr; + + std::span buffer_span(buffer.data(), buffer.size()); + switch (compression) { + case CompressionType::None: + decoder = &decoder_storage.emplace<NullDecoder>(buffer_span); + break; + + case CompressionType::Brotli: + decoder = &decoder_storage.emplace<BrotliDecoder>(buffer_span); + break; + + case CompressionType::LZ4: + decoder = &decoder_storage.emplace<LZ4Decoder>(buffer_span); + break; + + case CompressionType::Zstd: + decoder = &decoder_storage.emplace<ZstdDecoder>(buffer_span); + break; + + case CompressionType::Any: + LOG(FATAL) << "unexpected CompressionType::Any"; + } + + while (true) { syncmsg msg; if (!ReadFdExactly(sc.fd, &msg.data, sizeof(msg.data))) { adb_unlink(lpath); @@ -1050,35 +1185,34 @@ static bool sync_recv_v2(SyncConnection& sc, const char* rpath, const char* lpat } if (msg.data.id == ID_DONE) { - adb_unlink(lpath); - sc.Error("unexpected ID_DONE"); - return false; - } - - if (msg.data.id != ID_DATA) { + if (!decoder->Finish()) { + sc.Error("unexpected ID_DONE"); + return false; + } + } else if (msg.data.id != ID_DATA) { adb_unlink(lpath); sc.ReportCopyFailure(rpath, lpath, msg); return false; - } - - if (msg.data.size > sc.max) { - sc.Error("msg.data.size too large: %u (max %zu)", msg.data.size, sc.max); - adb_unlink(lpath); - return false; - } + } else { + if (msg.data.size > sc.max) { + sc.Error("msg.data.size too large: %u (max %zu)", msg.data.size, sc.max); + adb_unlink(lpath); + return false; + } - Block block(msg.data.size); - if (!ReadFdExactly(sc.fd, block.data(), msg.data.size)) { - adb_unlink(lpath); - return false; + Block block(msg.data.size); + if (!ReadFdExactly(sc.fd, block.data(), msg.data.size)) { + adb_unlink(lpath); + return false; + } + decoder->Append(std::move(block)); } - decoder.Append(std::move(block)); while (true) { std::span<char> output; - BrotliDecodeResult result = decoder.Decode(&output); + DecodeResult result = decoder->Decode(&output); - if (result == BrotliDecodeResult::Error) { + if (result == DecodeResult::Error) { sc.Error("decompress failed"); adb_unlink(lpath); return false; @@ -1093,42 +1227,27 @@ static bool sync_recv_v2(SyncConnection& sc, const char* rpath, const char* lpat } bytes_copied += output.size(); - - sc.RecordBytesTransferred(msg.data.size); + sc.RecordBytesTransferred(output.size()); sc.ReportProgress(name != nullptr ? name : rpath, bytes_copied, expected_size); - if (result == BrotliDecodeResult::NeedInput) { + if (result == DecodeResult::NeedInput) { break; - } else if (result == BrotliDecodeResult::MoreOutput) { + } else if (result == DecodeResult::MoreOutput) { continue; - } else if (result == BrotliDecodeResult::Done) { - reading = false; - break; + } else if (result == DecodeResult::Done) { + sc.RecordFilesTransferred(1); + return true; } else { - LOG(FATAL) << "invalid BrotliDecodeResult: " << static_cast<int>(result); + LOG(FATAL) << "invalid DecodeResult: " << static_cast<int>(result); } } } - - syncmsg msg; - if (!ReadFdExactly(sc.fd, &msg.data, sizeof(msg.data))) { - sc.Error("failed to read ID_DONE"); - return false; - } - - if (msg.data.id != ID_DONE) { - sc.Error("unexpected message after transfer: id = %d (expected ID_DONE)", msg.data.id); - return false; - } - - sc.RecordFilesTransferred(1); - return true; } static bool sync_recv(SyncConnection& sc, const char* rpath, const char* lpath, const char* name, - uint64_t expected_size, bool compressed) { - if (sc.HaveSendRecv2() && compressed) { - return sync_recv_v2(sc, rpath, lpath, name, expected_size); + uint64_t expected_size, CompressionType compression) { + if (sc.HaveSendRecv2()) { + return sync_recv_v2(sc, rpath, lpath, name, expected_size, compression); } else { return sync_recv_v1(sc, rpath, lpath, name, expected_size); } @@ -1210,7 +1329,8 @@ static bool is_root_dir(std::string_view path) { } static bool copy_local_dir_remote(SyncConnection& sc, std::string lpath, std::string rpath, - bool check_timestamps, bool list_only, bool compressed) { + bool check_timestamps, bool list_only, + CompressionType compression, bool dry_run) { sc.NewTransfer(); // Make sure that both directory paths end in a slash. @@ -1292,7 +1412,8 @@ static bool copy_local_dir_remote(SyncConnection& sc, std::string lpath, std::st if (list_only) { sc.Println("would push: %s -> %s", ci.lpath.c_str(), ci.rpath.c_str()); } else { - if (!sync_send(sc, ci.lpath, ci.rpath, ci.time, ci.mode, false, compressed)) { + if (!sync_send(sc, ci.lpath, ci.rpath, ci.time, ci.mode, false, compression, + dry_run)) { return false; } } @@ -1308,7 +1429,7 @@ static bool copy_local_dir_remote(SyncConnection& sc, std::string lpath, std::st } bool do_sync_push(const std::vector<const char*>& srcs, const char* dst, bool sync, - bool compressed) { + CompressionType compression, bool dry_run) { SyncConnection sc; if (!sc.IsValid()) return false; @@ -1373,7 +1494,8 @@ bool do_sync_push(const std::vector<const char*>& srcs, const char* dst, bool sy dst_dir.append(android::base::Basename(src_path)); } - success &= copy_local_dir_remote(sc, src_path, dst_dir, sync, false, compressed); + success &= + copy_local_dir_remote(sc, src_path, dst_dir, sync, false, compression, dry_run); continue; } else if (!should_push_file(st.st_mode)) { sc.Warning("skipping special file '%s' (mode = 0o%o)", src_path, st.st_mode); @@ -1394,7 +1516,8 @@ bool do_sync_push(const std::vector<const char*>& srcs, const char* dst, bool sy sc.NewTransfer(); sc.SetExpectedTotalBytes(st.st_size); - success &= sync_send(sc, src_path, dst_path, st.st_mtime, st.st_mode, sync, compressed); + success &= sync_send(sc, src_path, dst_path, st.st_mtime, st.st_mode, sync, compression, + dry_run); sc.ReportTransferRate(src_path, TransferDirection::push); } @@ -1435,14 +1558,14 @@ static bool remote_build_list(SyncConnection& sc, std::vector<copyinfo>* file_li } }; - if (!sync_ls(sc, rpath.c_str(), callback)) { + if (!sync_ls(sc, rpath, callback)) { return false; } // Check each symlink we found to see whether it's a file or directory. for (copyinfo& link_ci : linklist) { struct stat st; - if (!sync_stat_fallback(sc, link_ci.rpath.c_str(), &st)) { + if (!sync_stat_fallback(sc, link_ci.rpath, &st)) { sc.Warning("stat failed for path %s: %s", link_ci.rpath.c_str(), strerror(errno)); continue; } @@ -1480,7 +1603,7 @@ static int set_time_and_mode(const std::string& lpath, time_t time, } static bool copy_remote_dir_local(SyncConnection& sc, std::string rpath, std::string lpath, - bool copy_attrs, bool compressed) { + bool copy_attrs, CompressionType compression) { sc.NewTransfer(); // Make sure that both directory paths end in a slash. @@ -1510,7 +1633,7 @@ static bool copy_remote_dir_local(SyncConnection& sc, std::string rpath, std::st continue; } - if (!sync_recv(sc, ci.rpath.c_str(), ci.lpath.c_str(), nullptr, ci.size, compressed)) { + if (!sync_recv(sc, ci.rpath.c_str(), ci.lpath.c_str(), nullptr, ci.size, compression)) { return false; } @@ -1528,7 +1651,7 @@ static bool copy_remote_dir_local(SyncConnection& sc, std::string rpath, std::st } bool do_sync_pull(const std::vector<const char*>& srcs, const char* dst, bool copy_attrs, - bool compressed, const char* name) { + CompressionType compression, const char* name) { SyncConnection sc; if (!sc.IsValid()) return false; @@ -1602,7 +1725,7 @@ bool do_sync_pull(const std::vector<const char*>& srcs, const char* dst, bool co dst_dir.append(android::base::Basename(src_path)); } - success &= copy_remote_dir_local(sc, src_path, dst_dir, copy_attrs, compressed); + success &= copy_remote_dir_local(sc, src_path, dst_dir, copy_attrs, compression); continue; } else if (!should_pull_file(src_st.st_mode)) { sc.Warning("skipping special file '%s' (mode = 0o%o)", src_path, src_st.st_mode); @@ -1621,7 +1744,7 @@ bool do_sync_pull(const std::vector<const char*>& srcs, const char* dst, bool co sc.NewTransfer(); sc.SetExpectedTotalBytes(src_st.st_size); - if (!sync_recv(sc, src_path, dst_path, name, src_st.st_size, compressed)) { + if (!sync_recv(sc, src_path, dst_path, name, src_st.st_size, compression)) { success = false; continue; } @@ -1638,11 +1761,11 @@ bool do_sync_pull(const std::vector<const char*>& srcs, const char* dst, bool co } bool do_sync_sync(const std::string& lpath, const std::string& rpath, bool list_only, - bool compressed) { + CompressionType compression, bool dry_run) { SyncConnection sc; if (!sc.IsValid()) return false; - bool success = copy_local_dir_remote(sc, lpath, rpath, true, list_only, compressed); + bool success = copy_local_dir_remote(sc, lpath, rpath, true, list_only, compression, dry_run); if (!list_only) { sc.ReportOverallTransferRate(TransferDirection::push); } diff --git a/adb/client/file_sync_client.h b/adb/client/file_sync_client.h index de3f19245..cb8ca9323 100644 --- a/adb/client/file_sync_client.h +++ b/adb/client/file_sync_client.h @@ -19,11 +19,13 @@ #include <string> #include <vector> +#include "file_sync_protocol.h" + bool do_sync_ls(const char* path); bool do_sync_push(const std::vector<const char*>& srcs, const char* dst, bool sync, - bool compressed); + CompressionType compression, bool dry_run); bool do_sync_pull(const std::vector<const char*>& srcs, const char* dst, bool copy_attrs, - bool compressed, const char* name = nullptr); + CompressionType compression, const char* name = nullptr); bool do_sync_sync(const std::string& lpath, const std::string& rpath, bool list_only, - bool compressed); + CompressionType compression, bool dry_run); diff --git a/adb/client/incremental.cpp b/adb/client/incremental.cpp index c8f69c381..3033059d0 100644 --- a/adb/client/incremental.cpp +++ b/adb/client/incremental.cpp @@ -42,7 +42,7 @@ static std::pair<unique_fd, std::vector<char>> read_signature(Size file_size, struct stat st; if (stat(signature_file.c_str(), &st)) { if (!silent) { - fprintf(stderr, "Failed to stat signature file %s. Abort.\n", signature_file.c_str()); + fprintf(stderr, "Failed to stat signature file %s.\n", signature_file.c_str()); } return {}; } @@ -50,11 +50,21 @@ static std::pair<unique_fd, std::vector<char>> read_signature(Size file_size, unique_fd fd(adb_open(signature_file.c_str(), O_RDONLY)); if (fd < 0) { if (!silent) { - fprintf(stderr, "Failed to open signature file: %s. Abort.\n", signature_file.c_str()); + fprintf(stderr, "Failed to open signature file: %s.\n", signature_file.c_str()); } return {}; } + std::vector<char> invalid_signature; + + if (st.st_size > kMaxSignatureSize) { + if (!silent) { + fprintf(stderr, "Signature is too long. Max allowed is %d. Abort.\n", + kMaxSignatureSize); + } + return {std::move(fd), std::move(invalid_signature)}; + } + auto [signature, tree_size] = read_id_sig_headers(fd); if (auto expected = verity_tree_size_for_file(file_size); tree_size != expected) { if (!silent) { @@ -62,7 +72,7 @@ static std::pair<unique_fd, std::vector<char>> read_signature(Size file_size, "Verity tree size mismatch in signature file: %s [was %lld, expected %lld].\n", signature_file.c_str(), (long long)tree_size, (long long)expected); } - return {}; + return {std::move(fd), std::move(invalid_signature)}; } return {std::move(fd), std::move(signature)}; @@ -72,9 +82,11 @@ static std::pair<unique_fd, std::vector<char>> read_signature(Size file_size, static std::pair<unique_fd, std::string> read_and_encode_signature(Size file_size, std::string signature_file, bool silent) { + std::string encoded_signature; + auto [fd, signature] = read_signature(file_size, std::move(signature_file), silent); - if (!fd.ok()) { - return {}; + if (!fd.ok() || signature.empty()) { + return {std::move(fd), std::move(encoded_signature)}; } size_t base64_len = 0; @@ -82,9 +94,10 @@ static std::pair<unique_fd, std::string> read_and_encode_signature(Size file_siz if (!silent) { fprintf(stderr, "Fail to estimate base64 encoded length. Abort.\n"); } - return {}; + return {std::move(fd), std::move(encoded_signature)}; } - std::string encoded_signature(base64_len, '\0'); + + encoded_signature.resize(base64_len, '\0'); encoded_signature.resize(EVP_EncodeBlock((uint8_t*)encoded_signature.data(), (const uint8_t*)signature.data(), signature.size())); @@ -109,7 +122,7 @@ static unique_fd start_install(const Files& files, const Args& passthrough_args, } auto [signature_fd, signature] = read_and_encode_signature(st.st_size, file, silent); - if (!signature_fd.ok()) { + if (signature_fd.ok() && signature.empty()) { return {}; } @@ -138,9 +151,12 @@ bool can_install(const Files& files) { return false; } - auto [fd, _] = read_signature(st.st_size, file, true); - if (!fd.ok()) { - return false; + if (android::base::EndsWithIgnoreCase(file, ".apk")) { + // Signature has to be present for APKs. + auto [fd, _] = read_signature(st.st_size, file, /*silent=*/true); + if (!fd.ok()) { + return false; + } } } return true; @@ -157,23 +173,19 @@ std::optional<Process> install(const Files& files, const Args& passthrough_args, std::string adb_path = android::base::GetExecutablePath(); - auto osh = adb_get_os_handle(connection_fd.get()); -#ifdef _WIN32 - auto fd_param = std::to_string(reinterpret_cast<intptr_t>(osh)); -#else /* !_WIN32 a.k.a. Unix */ + auto osh = cast_handle_to_int(adb_get_os_handle(connection_fd.get())); auto fd_param = std::to_string(osh); -#endif // pipe for child process to write output int print_fds[2]; if (adb_socketpair(print_fds) != 0) { if (!silent) { - fprintf(stderr, "Failed to create socket pair for child to print to parent\n"); + fprintf(stderr, "adb: failed to create socket pair for child to print to parent\n"); } return {}; } auto [pipe_read_fd, pipe_write_fd] = print_fds; - auto pipe_write_fd_param = std::to_string(intptr_t(adb_get_os_handle(pipe_write_fd))); + auto pipe_write_fd_param = std::to_string(cast_handle_to_int(adb_get_os_handle(pipe_write_fd))); close_on_exec(pipe_read_fd); std::vector<std::string> args(std::move(files)); @@ -195,10 +207,15 @@ std::optional<Process> install(const Files& files, const Args& passthrough_args, Result result = wait_for_installation(pipe_read_fd); adb_close(pipe_read_fd); - if (result == Result::Success) { - // adb client exits now but inc-server can continue - serverKiller.release(); + if (result != Result::Success) { + if (!silent) { + fprintf(stderr, "adb: install command failed"); + } + return {}; } + + // adb client exits now but inc-server can continue + serverKiller.release(); return child; } diff --git a/adb/client/incremental_server.cpp b/adb/client/incremental_server.cpp index bfe18c0b2..0654a11b4 100644 --- a/adb/client/incremental_server.cpp +++ b/adb/client/incremental_server.cpp @@ -171,6 +171,8 @@ class File { const std::vector<BlockIdx>& PriorityBlocks() const { return priority_blocks_; } + bool hasTree() const { return tree_fd_.ok(); } + std::vector<bool> sentBlocks; NumBlocks sentBlocksCount = 0; @@ -349,6 +351,9 @@ std::optional<RequestCommand> IncrementalServer::ReadRequest(bool blocking) { bool IncrementalServer::SendTreeBlocksForDataBlock(const FileId fileId, const BlockIdx blockIdx) { auto& file = files_[fileId]; + if (!file.hasTree()) { + return true; + } const int32_t data_block_count = numBytesToNumBlocks(file.size); const int32_t total_nodes_count(file.sentTreeBlocks.size()); @@ -670,7 +675,8 @@ static std::pair<unique_fd, int64_t> open_signature(int64_t file_size, const cha unique_fd fd(adb_open(signature_file.c_str(), O_RDONLY)); if (fd < 0) { - error_exit("inc-server: failed to open file '%s'.", signature_file.c_str()); + D("No signature file found for '%s'('%s')", filepath, signature_file.c_str()); + return {}; } auto [tree_offset, tree_size] = skip_id_sig_headers(fd); diff --git a/adb/client/incremental_utils.cpp b/adb/client/incremental_utils.cpp index 076b7665e..dd117d228 100644 --- a/adb/client/incremental_utils.cpp +++ b/adb/client/incremental_utils.cpp @@ -24,6 +24,7 @@ #include <ziparchive/zip_archive.h> #include <ziparchive/zip_writer.h> +#include <array> #include <cinttypes> #include <numeric> #include <unordered_set> @@ -32,6 +33,8 @@ #include "adb_trace.h" #include "sysdeps.h" +using namespace std::literals; + namespace incremental { static constexpr inline int32_t offsetToBlockIndex(int64_t offset) { @@ -299,16 +302,24 @@ static std::pair<ZipArchiveHandle, std::unique_ptr<android::base::MappedFile>> o return {zip, std::move(mapping)}; } -// TODO(b/151676293): avoid using libziparchive as it reads local file headers -// which causes additional performance cost. Instead, only read from central directory. static std::vector<int32_t> InstallationPriorityBlocks(borrowed_fd fd, Size fileSize) { + static constexpr std::array<std::string_view, 3> additional_matches = { + "resources.arsc"sv, "AndroidManifest.xml"sv, "classes.dex"sv}; auto [zip, _] = openZipArchive(fd, fileSize); if (!zip) { return {}; } + auto matcher = [](std::string_view entry_name) { + if (entry_name.starts_with("lib/"sv) && entry_name.ends_with(".so"sv)) { + return true; + } + return std::any_of(additional_matches.begin(), additional_matches.end(), + [entry_name](std::string_view i) { return i == entry_name; }); + }; + void* cookie = nullptr; - if (StartIteration(zip, &cookie) != 0) { + if (StartIteration(zip, &cookie, std::move(matcher)) != 0) { D("%s failed at StartIteration: %d", __func__, errno); return {}; } @@ -317,8 +328,13 @@ static std::vector<int32_t> InstallationPriorityBlocks(borrowed_fd fd, Size file ZipEntry entry; std::string_view entryName; while (Next(cookie, &entry, &entryName) == 0) { - if (entryName == "resources.arsc" || entryName == "AndroidManifest.xml" || - entryName.starts_with("lib/")) { + if (entryName == "classes.dex"sv) { + // Only the head is needed for installation + int32_t startBlockIndex = offsetToBlockIndex(entry.offset); + appendBlocks(startBlockIndex, 2, &installationPriorityBlocks); + D("\tadding to priority blocks: '%.*s' (%d)", (int)entryName.size(), entryName.data(), + 2); + } else { // Full entries are needed for installation off64_t entryStartOffset = entry.offset; off64_t entryEndOffset = @@ -330,12 +346,8 @@ static std::vector<int32_t> InstallationPriorityBlocks(borrowed_fd fd, Size file int32_t endBlockIndex = offsetToBlockIndex(entryEndOffset); int32_t numNewBlocks = endBlockIndex - startBlockIndex + 1; appendBlocks(startBlockIndex, numNewBlocks, &installationPriorityBlocks); - D("\tadding to priority blocks: '%.*s'", (int)entryName.size(), entryName.data()); - } else if (entryName == "classes.dex") { - // Only the head is needed for installation - int32_t startBlockIndex = offsetToBlockIndex(entry.offset); - appendBlocks(startBlockIndex, 2, &installationPriorityBlocks); - D("\tadding to priority blocks: '%.*s'", (int)entryName.size(), entryName.data()); + D("\tadding to priority blocks: '%.*s' (%d)", (int)entryName.size(), entryName.data(), + numNewBlocks); } } @@ -346,7 +358,7 @@ static std::vector<int32_t> InstallationPriorityBlocks(borrowed_fd fd, Size file std::vector<int32_t> PriorityBlocksForFile(const std::string& filepath, borrowed_fd fd, Size fileSize) { - if (!android::base::EndsWithIgnoreCase(filepath, ".apk")) { + if (!android::base::EndsWithIgnoreCase(filepath, ".apk"sv)) { return {}; } off64_t signerOffset = SignerBlockOffset(fd, fileSize); diff --git a/adb/client/incremental_utils.h b/adb/client/incremental_utils.h index fe2914dde..ebf18f8cb 100644 --- a/adb/client/incremental_utils.h +++ b/adb/client/incremental_utils.h @@ -25,14 +25,13 @@ #include <stdint.h> -#include <android-base/off64_t.h> - namespace incremental { using Size = int64_t; constexpr int kBlockSize = 4096; constexpr int kSha256DigestSize = 32; constexpr int kDigestSize = kSha256DigestSize; +constexpr int kMaxSignatureSize = 8096; // incrementalfs.h constexpr std::string_view IDSIG = ".idsig"; diff --git a/adb/client/main.cpp b/adb/client/main.cpp index 78f7b8f20..a19bd6d31 100644 --- a/adb/client/main.cpp +++ b/adb/client/main.cpp @@ -105,7 +105,12 @@ int adb_server_main(int is_daemon, const std::string& socket_spec, int ack_reply fdevent_run_on_main_thread([]() { exit(0); }); }); - char* leak = getenv("ADB_LEAK"); + const char* reject_kill_server = getenv("ADB_REJECT_KILL_SERVER"); + if (reject_kill_server && strcmp(reject_kill_server, "1") == 0) { + adb_set_reject_kill_server(true); + } + + const char* leak = getenv("ADB_LEAK"); if (leak && strcmp(leak, "1") == 0) { intentionally_leak(); } @@ -140,9 +145,10 @@ int adb_server_main(int is_daemon, const std::string& socket_spec, int ack_reply auto start = std::chrono::steady_clock::now(); // If we told a previous adb server to quit because of version mismatch, we can get to this - // point before it's finished exiting. Retry for a while to give it some time. - while (install_listener(socket_spec, "*smartsocket*", nullptr, 0, nullptr, &error) != - INSTALL_STATUS_OK) { + // point before it's finished exiting. Retry for a while to give it some time. Don't actually + // accept any connections until adb_wait_for_device_initialization finishes below. + while (install_listener(socket_spec, "*smartsocket*", nullptr, INSTALL_LISTENER_DISABLED, + nullptr, &error) != INSTALL_STATUS_OK) { if (std::chrono::steady_clock::now() - start > 0.5s) { LOG(FATAL) << "could not install *smartsocket* listener: " << error; } @@ -163,12 +169,14 @@ int adb_server_main(int is_daemon, const std::string& socket_spec, int ack_reply PLOG(FATAL) << "setsid() failed"; } #endif + } - // Wait for the USB scan to complete before notifying the parent that we're up. - // We need to perform this in a thread, because we would otherwise block the event loop. - std::thread notify_thread([ack_reply_fd]() { - adb_wait_for_device_initialization(); + // Wait for the USB scan to complete before notifying the parent that we're up. + // We need to perform this in a thread, because we would otherwise block the event loop. + std::thread notify_thread([ack_reply_fd]() { + adb_wait_for_device_initialization(); + if (ack_reply_fd >= 0) { // Any error output written to stderr now goes to adb.log. We could // keep around a copy of the stderr fd and use that to write any errors // encountered by the following code, but that is probably overkill. @@ -194,9 +202,13 @@ int adb_server_main(int is_daemon, const std::string& socket_spec, int ack_reply } unix_close(ack_reply_fd); #endif - }); - notify_thread.detach(); - } + } + // We don't accept() client connections until this point: this way, clients + // can't see wonky state early in startup even if they're connecting directly + // to the server instead of going through the adb program. + fdevent_run_on_main_thread([] { enable_server_sockets(); }); + }); + notify_thread.detach(); #if defined(__linux__) // Write our location to .android/adb.$PORT, so that older clients can exec us. diff --git a/adb/client/mdns_utils.cpp b/adb/client/mdns_utils.cpp new file mode 100644 index 000000000..8666b18e9 --- /dev/null +++ b/adb/client/mdns_utils.cpp @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "client/mdns_utils.h" + +#include <android-base/strings.h> + +namespace mdns { + +// <Instance>.<Service>.<Domain> +std::optional<MdnsInstance> mdns_parse_instance_name(std::string_view name) { + CHECK(!name.empty()); + + // Return the whole name if it doesn't fall under <Instance>.<Service>.<Domain> or + // <Instance>.<Service> + bool has_local_suffix = false; + // Strip the local suffix, if any + { + std::string local_suffix = ".local"; + local_suffix += android::base::EndsWith(name, ".") ? "." : ""; + + if (android::base::ConsumeSuffix(&name, local_suffix)) { + if (name.empty()) { + return std::nullopt; + } + has_local_suffix = true; + } + } + + std::string transport; + // Strip the transport suffix, if any + { + std::string add_dot = (!has_local_suffix && android::base::EndsWith(name, ".")) ? "." : ""; + std::array<std::string, 2> transport_suffixes{"._tcp", "._udp"}; + + for (const auto& t : transport_suffixes) { + if (android::base::ConsumeSuffix(&name, t + add_dot)) { + if (name.empty()) { + return std::nullopt; + } + transport = t.substr(1); + break; + } + } + + if (has_local_suffix && transport.empty()) { + return std::nullopt; + } + } + + if (!has_local_suffix && transport.empty()) { + return std::make_optional<MdnsInstance>(name, "", ""); + } + + // Split the service name from the instance name + auto pos = name.rfind("."); + if (pos == 0 || pos == std::string::npos || pos == name.size() - 1) { + return std::nullopt; + } + + return std::make_optional<MdnsInstance>(name.substr(0, pos), name.substr(pos + 1), transport); +} + +} // namespace mdns diff --git a/adb/client/mdns_utils.h b/adb/client/mdns_utils.h new file mode 100644 index 000000000..40d095ddc --- /dev/null +++ b/adb/client/mdns_utils.h @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <optional> +#include <string_view> + +#include "adb_wifi.h" + +namespace mdns { + +struct MdnsInstance { + std::string instance_name; // "my name" + std::string service_name; // "_adb-tls-connect" + std::string transport_type; // either "_tcp" or "_udp" + + MdnsInstance(std::string_view inst, std::string_view serv, std::string_view trans) + : instance_name(inst), service_name(serv), transport_type(trans) {} +}; + +// This parser is based on https://tools.ietf.org/html/rfc6763#section-4.1 for +// structured service instance names, where the whole name is in the format +// <Instance>.<Service>.<Domain>. +// +// In our case, we ignore <Domain> portion of the name, which +// we always assume to be ".local", or link-local mDNS. +// +// The string can be in one of the following forms: +// - <Instance>.<Service>.<Domain>.? +// - e.g. "instance._service._tcp.local" (or "...local.") +// - <Instance>.<Service>.? (must contain either "_tcp" or "_udp" at the end) +// - e.g. "instance._service._tcp" (or "..._tcp.) +// - <Instance> (can contain dots '.') +// - e.g. "myname", "name.", "my.name." +// +// Returns an MdnsInstance with the appropriate fields filled in (instance name is never empty), +// otherwise returns std::nullopt. +std::optional<MdnsInstance> mdns_parse_instance_name(std::string_view name); + +} // namespace mdns diff --git a/adb/client/mdns_utils_test.cpp b/adb/client/mdns_utils_test.cpp new file mode 100644 index 000000000..ec7152957 --- /dev/null +++ b/adb/client/mdns_utils_test.cpp @@ -0,0 +1,173 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "client/mdns_utils.h" + +#include <gtest/gtest.h> + +namespace mdns { + +TEST(mdns_utils, mdns_parse_instance_name) { + // Just the instance name + { + std::string str = "."; + auto res = mdns_parse_instance_name(str); + ASSERT_TRUE(res.has_value()); + EXPECT_EQ(str, res->instance_name); + EXPECT_TRUE(res->service_name.empty()); + EXPECT_TRUE(res->transport_type.empty()); + } + { + std::string str = "my.name"; + auto res = mdns_parse_instance_name(str); + ASSERT_TRUE(res.has_value()); + EXPECT_EQ(str, res->instance_name); + EXPECT_TRUE(res->service_name.empty()); + EXPECT_TRUE(res->transport_type.empty()); + } + { + std::string str = "my.name."; + auto res = mdns_parse_instance_name(str); + ASSERT_TRUE(res.has_value()); + EXPECT_EQ(str, res->instance_name); + EXPECT_TRUE(res->service_name.empty()); + EXPECT_TRUE(res->transport_type.empty()); + } + + // With "_tcp", "_udp" transport type + for (const std::string_view transport : {"._tcp", "._udp"}) { + { + std::string str = android::base::StringPrintf("%s", transport.data()); + auto res = mdns_parse_instance_name(str); + EXPECT_FALSE(res.has_value()); + } + { + std::string str = android::base::StringPrintf("%s.", transport.data()); + auto res = mdns_parse_instance_name(str); + EXPECT_FALSE(res.has_value()); + } + { + std::string str = android::base::StringPrintf("service%s", transport.data()); + auto res = mdns_parse_instance_name(str); + EXPECT_FALSE(res.has_value()); + } + { + std::string str = android::base::StringPrintf(".service%s", transport.data()); + auto res = mdns_parse_instance_name(str); + EXPECT_FALSE(res.has_value()); + } + { + std::string str = android::base::StringPrintf("service.%s", transport.data()); + auto res = mdns_parse_instance_name(str); + EXPECT_FALSE(res.has_value()); + } + { + std::string str = android::base::StringPrintf("my.service%s", transport.data()); + auto res = mdns_parse_instance_name(str); + ASSERT_TRUE(res.has_value()); + EXPECT_EQ(res->instance_name, "my"); + EXPECT_EQ(res->service_name, "service"); + EXPECT_EQ(res->transport_type, transport.substr(1)); + } + { + std::string str = android::base::StringPrintf("my.service%s.", transport.data()); + auto res = mdns_parse_instance_name(str); + ASSERT_TRUE(res.has_value()); + EXPECT_EQ(res->instance_name, "my"); + EXPECT_EQ(res->service_name, "service"); + EXPECT_EQ(res->transport_type, transport.substr(1)); + } + { + std::string str = android::base::StringPrintf("my..service%s", transport.data()); + auto res = mdns_parse_instance_name(str); + ASSERT_TRUE(res.has_value()); + EXPECT_EQ(res->instance_name, "my."); + EXPECT_EQ(res->service_name, "service"); + EXPECT_EQ(res->transport_type, transport.substr(1)); + } + { + std::string str = android::base::StringPrintf("my.name.service%s.", transport.data()); + auto res = mdns_parse_instance_name(str); + ASSERT_TRUE(res.has_value()); + EXPECT_EQ(res->instance_name, "my.name"); + EXPECT_EQ(res->service_name, "service"); + EXPECT_EQ(res->transport_type, transport.substr(1)); + } + { + std::string str = android::base::StringPrintf("name.service.%s.", transport.data()); + auto res = mdns_parse_instance_name(str); + EXPECT_FALSE(res.has_value()); + } + + // With ".local" domain + { + std::string str = ".local"; + auto res = mdns_parse_instance_name(str); + EXPECT_FALSE(res.has_value()); + } + { + std::string str = ".local."; + auto res = mdns_parse_instance_name(str); + EXPECT_FALSE(res.has_value()); + } + { + std::string str = "name.local"; + auto res = mdns_parse_instance_name(str); + EXPECT_FALSE(res.has_value()); + } + { + std::string str = android::base::StringPrintf("%s.local", transport.data()); + auto res = mdns_parse_instance_name(str); + EXPECT_FALSE(res.has_value()); + } + { + std::string str = android::base::StringPrintf("service%s.local", transport.data()); + auto res = mdns_parse_instance_name(str); + EXPECT_FALSE(res.has_value()); + } + { + std::string str = android::base::StringPrintf("name.service%s.local", transport.data()); + auto res = mdns_parse_instance_name(str); + ASSERT_TRUE(res.has_value()); + EXPECT_EQ(res->instance_name, "name"); + EXPECT_EQ(res->service_name, "service"); + EXPECT_EQ(res->transport_type, transport.substr(1)); + } + { + std::string str = + android::base::StringPrintf("name.service%s.local.", transport.data()); + auto res = mdns_parse_instance_name(str); + ASSERT_TRUE(res.has_value()); + EXPECT_EQ(res->instance_name, "name"); + EXPECT_EQ(res->service_name, "service"); + EXPECT_EQ(res->transport_type, transport.substr(1)); + } + { + std::string str = + android::base::StringPrintf("name.service%s..local.", transport.data()); + auto res = mdns_parse_instance_name(str); + EXPECT_FALSE(res.has_value()); + } + { + std::string str = + android::base::StringPrintf("name.service.%s.local.", transport.data()); + auto res = mdns_parse_instance_name(str); + EXPECT_FALSE(res.has_value()); + } + } +} + +} // namespace mdns diff --git a/adb/client/pairing/pairing_client.cpp b/adb/client/pairing/pairing_client.cpp index 04bbcebd4..937a5bd4c 100644 --- a/adb/client/pairing/pairing_client.cpp +++ b/adb/client/pairing/pairing_client.cpp @@ -141,11 +141,7 @@ bool PairingClientImpl::StartConnection() { cert_.size(), priv_key_.data(), priv_key_.size())); CHECK(connection_); -#ifdef _WIN32 int osh = cast_handle_to_int(adb_get_os_handle(fd.release())); -#else - int osh = adb_get_os_handle(fd.release()); -#endif if (!pairing_connection_start(connection_.get(), osh, OnPairingResult, this)) { LOG(ERROR) << "PairingClient failed to start the PairingConnection"; state_ = State::Stopped; diff --git a/adb/transport_local.cpp b/adb/client/transport_local.cpp index 5ec8e1665..15a072466 100644 --- a/adb/transport_local.cpp +++ b/adb/client/transport_local.cpp @@ -38,10 +38,6 @@ #include <android-base/thread_annotations.h> #include <cutils/sockets.h> -#if !ADB_HOST -#include <android-base/properties.h> -#endif - #include "adb.h" #include "adb_io.h" #include "adb_unique_fd.h" @@ -49,8 +45,6 @@ #include "socket_spec.h" #include "sysdeps/chrono.h" -#if ADB_HOST - // Android Wear has been using port 5601 in all of its documentation/tooling, // but we search for emulators on ports [5554, 5555 + ADB_LOCAL_TRANSPORT_MAX]. // Avoid stomping on their port by restricting the active scanning range. @@ -76,9 +70,8 @@ static void adb_local_transport_max_port_env_override() { // We keep a map from emulator port to transport. // TODO: weak_ptr? -static auto& local_transports GUARDED_BY(local_transports_lock) = - *new std::unordered_map<int, atransport*>(); -#endif /* ADB_HOST */ +static std::unordered_map<int, atransport*> local_transports + [[clang::no_destroy]] GUARDED_BY(local_transports_lock); bool local_connect(int port) { std::string dummy; @@ -140,21 +133,19 @@ void connect_device(const std::string& address, std::string* response) { } } - int local_connect_arbitrary_ports(int console_port, int adb_port, std::string* error) { unique_fd fd; -#if ADB_HOST if (find_emulator_transport_by_adb_port(adb_port) != nullptr || find_emulator_transport_by_console_port(console_port) != nullptr) { return -1; } - const char *host = getenv("ADBHOST"); + const char* host = getenv("ADBHOST"); if (host) { fd.reset(network_connect(host, adb_port, SOCK_STREAM, 0, error)); } -#endif + if (fd < 0) { fd.reset(network_loopback_client(adb_port, SOCK_STREAM, error)); } @@ -173,8 +164,6 @@ int local_connect_arbitrary_ports(int console_port, int adb_port, std::string* e return -1; } -#if ADB_HOST - static void PollAllLocalPortsForEmulator() { // Try to connect to any number of running emulator instances. for (int port = DEFAULT_ADB_LOCAL_TRANSPORT_PORT; port <= adb_local_transport_max_port; @@ -194,8 +183,8 @@ struct RetryPort { // Retry emulators just kicked. static std::vector<RetryPort>& retry_ports = *new std::vector<RetryPort>; -std::mutex &retry_ports_lock = *new std::mutex; -std::condition_variable &retry_ports_cond = *new std::condition_variable; +std::mutex& retry_ports_lock = *new std::mutex; +std::condition_variable& retry_ports_cond = *new std::condition_variable; static void client_socket_thread(std::string_view) { adb_thread_setname("client_socket_thread"); @@ -220,7 +209,7 @@ static void client_socket_thread(std::string_view) { std::vector<RetryPort> next_ports; for (auto& port : ports) { VLOG(TRANSPORT) << "retry port " << port.port << ", last retry_count " - << port.retry_count; + << port.retry_count; if (local_connect(port.port)) { VLOG(TRANSPORT) << "retry port " << port.port << " successfully"; continue; @@ -240,77 +229,12 @@ static void client_socket_thread(std::string_view) { } } -#else // !ADB_HOST - -void server_socket_thread(std::function<unique_fd(std::string_view, std::string*)> listen_func, - std::string_view addr) { - adb_thread_setname("server socket"); - - unique_fd serverfd; - std::string error; - - while (serverfd == -1) { - errno = 0; - serverfd = listen_func(addr, &error); - if (errno == EAFNOSUPPORT || errno == EINVAL || errno == EPROTONOSUPPORT) { - D("unrecoverable error: '%s'", error.c_str()); - return; - } else if (serverfd < 0) { - D("server: cannot bind socket yet: %s", error.c_str()); - std::this_thread::sleep_for(1s); - continue; - } - close_on_exec(serverfd.get()); - } - - while (true) { - D("server: trying to get new connection from fd %d", serverfd.get()); - unique_fd fd(adb_socket_accept(serverfd, nullptr, nullptr)); - if (fd >= 0) { - D("server: new connection on fd %d", fd.get()); - close_on_exec(fd.get()); - disable_tcp_nagle(fd.get()); - std::string serial = android::base::StringPrintf("host-%d", fd.get()); - // We don't care about port value in "register_socket_transport" as it is used - // only from ADB_HOST. "server_socket_thread" is never called from ADB_HOST. - register_socket_transport( - std::move(fd), std::move(serial), 0, 1, - [](atransport*) { return ReconnectResult::Abort; }, false); - } - } - D("transport: server_socket_thread() exiting"); -} - -#endif - -#if !ADB_HOST -unique_fd adb_listen(std::string_view addr, std::string* error) { - return unique_fd{socket_spec_listen(addr, error, nullptr)}; -} -#endif - void local_init(const std::string& addr) { -#if ADB_HOST D("transport: local client init"); std::thread(client_socket_thread, addr).detach(); adb_local_transport_max_port_env_override(); -#elif !defined(__ANDROID__) - // Host adbd. - D("transport: local server init"); - std::thread(server_socket_thread, adb_listen, addr).detach(); -#else - D("transport: local server init"); - // For the adbd daemon in the system image we need to distinguish - // between the device, and the emulator. - if (addr.starts_with("tcp:") && use_qemu_goldfish()) { - std::thread(qemu_socket_thread, addr).detach(); - } else { - std::thread(server_socket_thread, adb_listen, addr).detach(); - } -#endif // !ADB_HOST } -#if ADB_HOST struct EmulatorConnection : public FdConnection { EmulatorConnection(unique_fd fd, int local_port) : FdConnection(std::move(fd)), local_port_(local_port) {} @@ -336,7 +260,7 @@ struct EmulatorConnection : public FdConnection { /* Only call this function if you already hold local_transports_lock. */ static atransport* find_emulator_transport_by_adb_port_locked(int adb_port) - REQUIRES(local_transports_lock) { + REQUIRES(local_transports_lock) { auto it = local_transports.find(adb_port); if (it == local_transports.end()) { return nullptr; @@ -352,7 +276,6 @@ atransport* find_emulator_transport_by_adb_port(int adb_port) { atransport* find_emulator_transport_by_console_port(int console_port) { return find_transport(getEmulatorSerialString(console_port).c_str()); } -#endif std::string getEmulatorSerialString(int console_port) { return android::base::StringPrintf("emulator-%d", console_port); @@ -363,7 +286,6 @@ int init_socket_transport(atransport* t, unique_fd fd, int adb_port, int local) t->type = kTransportLocal; -#if ADB_HOST // Emulator connection. if (local) { auto emulator_connection = std::make_unique<EmulatorConnection>(std::move(fd), adb_port); @@ -380,7 +302,6 @@ int init_socket_transport(atransport* t, unique_fd fd, int adb_port, int local) return fail; } -#endif // Regular tcp connection. auto fd_connection = std::make_unique<FdConnection>(std::move(fd)); diff --git a/adb/client/transport_mdns.cpp b/adb/client/transport_mdns.cpp index 22b9b1808..961121202 100644 --- a/adb/client/transport_mdns.cpp +++ b/adb/client/transport_mdns.cpp @@ -26,6 +26,7 @@ #include <memory> #include <thread> +#include <unordered_set> #include <vector> #include <android-base/stringprintf.h> @@ -37,32 +38,81 @@ #include "adb_trace.h" #include "adb_utils.h" #include "adb_wifi.h" +#include "client/mdns_utils.h" #include "fdevent/fdevent.h" #include "sysdeps.h" static DNSServiceRef service_refs[kNumADBDNSServices]; static fdevent* service_ref_fdes[kNumADBDNSServices]; +static auto& g_autoconn_whitelist = *new std::unordered_set<int>(); -static int adb_DNSServiceIndexByName(const char* regType) { +static int adb_DNSServiceIndexByName(std::string_view regType) { for (int i = 0; i < kNumADBDNSServices; ++i) { - if (!strncmp(regType, kADBDNSServices[i], strlen(kADBDNSServices[i]))) { + if (!strncmp(regType.data(), kADBDNSServices[i], strlen(kADBDNSServices[i]))) { return i; } } return -1; } -static bool adb_DNSServiceShouldConnect(const char* regType, const char* serviceName) { - int index = adb_DNSServiceIndexByName(regType); - if (index == kADBTransportServiceRefIndex) { - // Ignore adb-EMULATOR* service names, as it interferes with the - // emulator ports that are already connected. - if (android::base::StartsWith(serviceName, "adb-EMULATOR")) { - LOG(INFO) << "Ignoring emulator transport service [" << serviceName << "]"; - return false; +static void config_auto_connect_services() { + // ADB_MDNS_AUTO_CONNECT is a comma-delimited list of mdns services + // that are allowed to auto-connect. By default, only allow "adb-tls-connect" + // to auto-connect, since this is filtered down to auto-connect only to paired + // devices. + g_autoconn_whitelist.insert(kADBSecureConnectServiceRefIndex); + const char* srvs = getenv("ADB_MDNS_AUTO_CONNECT"); + if (!srvs) { + return; + } + + if (strcmp(srvs, "0") == 0) { + D("Disabling all auto-connecting"); + g_autoconn_whitelist.clear(); + return; + } + + if (strcmp(srvs, "1") == 0) { + D("Allow all auto-connecting"); + g_autoconn_whitelist.insert(kADBTransportServiceRefIndex); + return; + } + + // Selectively choose which services to allow auto-connect. + // E.g. ADB_MDNS_AUTO_CONNECT=adb,adb-tls-connect would allow + // _adb._tcp and _adb-tls-connnect._tcp services to auto-connect. + auto srvs_list = android::base::Split(srvs, ","); + std::unordered_set<int> new_whitelist; + for (const auto& item : srvs_list) { + auto full_srv = android::base::StringPrintf("_%s._tcp", item.data()); + int idx = adb_DNSServiceIndexByName(full_srv); + if (idx >= 0) { + new_whitelist.insert(idx); } } - return (index == kADBTransportServiceRefIndex || index == kADBSecureConnectServiceRefIndex); + + if (!new_whitelist.empty()) { + g_autoconn_whitelist = std::move(new_whitelist); + } +} + +static bool adb_DNSServiceShouldAutoConnect(const char* regType, const char* serviceName) { + // Try to auto-connect to any "_adb" or "_adb-tls-connect" services excluding emulator services. + int index = adb_DNSServiceIndexByName(regType); + if (index != kADBTransportServiceRefIndex && index != kADBSecureConnectServiceRefIndex) { + return false; + } + if (g_autoconn_whitelist.find(index) == g_autoconn_whitelist.end()) { + D("Auto-connect for regType '%s' disabled", regType); + return false; + } + // Ignore adb-EMULATOR* service names, as it interferes with the + // emulator ports that are already connected. + if (android::base::StartsWith(serviceName, "adb-EMULATOR")) { + LOG(INFO) << "Ignoring emulator transport service [" << serviceName << "]"; + return false; + } + return true; } // Use adb_DNSServiceRefSockFD() instead of calling DNSServiceRefSockFD() @@ -95,7 +145,7 @@ class AsyncServiceRef { return initialized_; } - virtual ~AsyncServiceRef() { + void DestroyServiceRef() { if (!initialized_) { return; } @@ -103,9 +153,13 @@ class AsyncServiceRef { // Order matters here! Must destroy the fdevent first since it has a // reference to |sdRef_|. fdevent_destroy(fde_); + D("DNSServiceRefDeallocate(sdRef=%p)", sdRef_); DNSServiceRefDeallocate(sdRef_); + initialized_ = false; } + virtual ~AsyncServiceRef() { DestroyServiceRef(); } + protected: DNSServiceRef sdRef_; @@ -154,6 +208,7 @@ class ResolvedService : public AsyncServiceRef { if (ret != kDNSServiceErr_NoError) { D("Got %d from DNSServiceGetAddrInfo.", ret); } else { + D("DNSServiceGetAddrInfo(sdRef=%p, hosttarget=%s)", sdRef_, hosttarget); Initialize(); } @@ -167,14 +222,14 @@ class ResolvedService : public AsyncServiceRef { } std::string response; - connect_device(android::base::StringPrintf(addr_format_.c_str(), ip_addr_, port_), + connect_device(android::base::StringPrintf("%s.%s", serviceName_.c_str(), regType_.c_str()), &response); D("Secure connect to %s regtype %s (%s:%hu) : %s", serviceName_.c_str(), regType_.c_str(), ip_addr_, port_, response.c_str()); return true; } - void Connect(const sockaddr* address) { + bool AddToServiceRegistry(const sockaddr* address) { sa_family_ = address->sa_family; if (sa_family_ == AF_INET) { @@ -185,26 +240,53 @@ class ResolvedService : public AsyncServiceRef { addr_format_ = "[%s]:%hu"; } else { // Should be impossible D("mDNS resolved non-IP address."); - return; + return false; } // Winsock version requires the const cast Because Microsoft. if (!inet_ntop(sa_family_, const_cast<void*>(ip_addr_data_), ip_addr_, sizeof(ip_addr_))) { D("Could not convert IP address to string."); - return; + return false; } - // adb secure service needs to do something different from just - // connecting here. - if (adb_DNSServiceShouldConnect(regType_.c_str(), serviceName_.c_str())) { + // Add to the service registry before trying to auto-connect, since socket_spec_connect will + // check these registries for the ip address when connecting via mdns instance name. + int adbSecureServiceType = serviceIndex(); + ServiceRegistry* services = nullptr; + switch (adbSecureServiceType) { + case kADBTransportServiceRefIndex: + services = sAdbTransportServices; + break; + case kADBSecurePairingServiceRefIndex: + services = sAdbSecurePairingServices; + break; + case kADBSecureConnectServiceRefIndex: + services = sAdbSecureConnectServices; + break; + default: + LOG(WARNING) << "No registry available for reg_type=[" << regType_ << "]"; + return false; + } + + if (!services->empty()) { + // Remove the previous resolved service, if any. + services->erase(std::remove_if(services->begin(), services->end(), + [&](std::unique_ptr<ResolvedService>& service) { + return (serviceName_ == service->serviceName()); + })); + } + services->push_back(std::unique_ptr<ResolvedService>(this)); + + if (adb_DNSServiceShouldAutoConnect(regType_.c_str(), serviceName_.c_str())) { std::string response; - D("Attempting to serviceName=[%s], regtype=[%s] ipaddr=(%s:%hu)", serviceName_.c_str(), - regType_.c_str(), ip_addr_, port_); + D("Attempting to connect serviceName=[%s], regtype=[%s] ipaddr=(%s:%hu)", + serviceName_.c_str(), regType_.c_str(), ip_addr_, port_); int index = adb_DNSServiceIndexByName(regType_.c_str()); if (index == kADBSecureConnectServiceRefIndex) { ConnectSecureWifiDevice(); } else { - connect_device(android::base::StringPrintf(addr_format_.c_str(), ip_addr_, port_), + connect_device(android::base::StringPrintf("%s.%s", serviceName_.c_str(), + regType_.c_str()), &response); D("Connect to %s regtype %s (%s:%hu) : %s", serviceName_.c_str(), regType_.c_str(), ip_addr_, port_, response.c_str()); @@ -214,17 +296,7 @@ class ResolvedService : public AsyncServiceRef { serviceName_.c_str(), regType_.c_str(), ip_addr_, port_); } - int adbSecureServiceType = serviceIndex(); - switch (adbSecureServiceType) { - case kADBSecurePairingServiceRefIndex: - sAdbSecurePairingServices->push_back(this); - break; - case kADBSecureConnectServiceRefIndex: - sAdbSecureConnectServices->push_back(this); - break; - default: - break; - } + return true; } int serviceIndex() const { return adb_DNSServiceIndexByName(regType_.c_str()); } @@ -233,18 +305,23 @@ class ResolvedService : public AsyncServiceRef { std::string serviceName() const { return serviceName_; } + std::string regType() const { return regType_; } + std::string ipAddress() const { return ip_addr_; } uint16_t port() const { return port_; } - using ServiceRegistry = std::vector<ResolvedService*>; + using ServiceRegistry = std::vector<std::unique_ptr<ResolvedService>>; + + // unencrypted tcp connections + static ServiceRegistry* sAdbTransportServices; static ServiceRegistry* sAdbSecurePairingServices; static ServiceRegistry* sAdbSecureConnectServices; - static void initAdbSecure(); + static void initAdbServiceRegistries(); - static void forEachService(const ServiceRegistry& services, const std::string& hostname, + static void forEachService(const ServiceRegistry& services, std::string_view hostname, adb_secure_foreach_service_callback cb); static bool connectByServiceName(const ServiceRegistry& services, @@ -264,13 +341,19 @@ class ResolvedService : public AsyncServiceRef { }; // static -std::vector<ResolvedService*>* ResolvedService::sAdbSecurePairingServices = NULL; +ResolvedService::ServiceRegistry* ResolvedService::sAdbTransportServices = NULL; + +// static +ResolvedService::ServiceRegistry* ResolvedService::sAdbSecurePairingServices = NULL; // static -std::vector<ResolvedService*>* ResolvedService::sAdbSecureConnectServices = NULL; +ResolvedService::ServiceRegistry* ResolvedService::sAdbSecureConnectServices = NULL; // static -void ResolvedService::initAdbSecure() { +void ResolvedService::initAdbServiceRegistries() { + if (!sAdbTransportServices) { + sAdbTransportServices = new ServiceRegistry; + } if (!sAdbSecurePairingServices) { sAdbSecurePairingServices = new ServiceRegistry; } @@ -281,19 +364,20 @@ void ResolvedService::initAdbSecure() { // static void ResolvedService::forEachService(const ServiceRegistry& services, - const std::string& wanted_service_name, + std::string_view wanted_service_name, adb_secure_foreach_service_callback cb) { - initAdbSecure(); + initAdbServiceRegistries(); - for (auto service : services) { + for (const auto& service : services) { auto service_name = service->serviceName(); + auto reg_type = service->regType(); auto ip = service->ipAddress(); auto port = service->port(); - if (wanted_service_name == "") { - cb(service_name.c_str(), ip.c_str(), port); + if (wanted_service_name.empty()) { + cb(service_name.c_str(), reg_type.c_str(), ip.c_str(), port); } else if (service_name == wanted_service_name) { - cb(service_name.c_str(), ip.c_str(), port); + cb(service_name.c_str(), reg_type.c_str(), ip.c_str(), port); } } } @@ -301,8 +385,8 @@ void ResolvedService::forEachService(const ServiceRegistry& services, // static bool ResolvedService::connectByServiceName(const ServiceRegistry& services, const std::string& service_name) { - initAdbSecure(); - for (auto service : services) { + initAdbServiceRegistries(); + for (const auto& service : services) { if (service_name == service->serviceName()) { D("Got service_name match [%s]", service->serviceName().c_str()); return service->ConnectSecureWifiDevice(); @@ -314,14 +398,12 @@ bool ResolvedService::connectByServiceName(const ServiceRegistry& services, void adb_secure_foreach_pairing_service(const char* service_name, adb_secure_foreach_service_callback cb) { - ResolvedService::forEachService(*ResolvedService::sAdbSecurePairingServices, - service_name ? service_name : "", cb); + ResolvedService::forEachService(*ResolvedService::sAdbSecurePairingServices, service_name, cb); } void adb_secure_foreach_connect_service(const char* service_name, adb_secure_foreach_service_callback cb) { - ResolvedService::forEachService(*ResolvedService::sAdbSecureConnectServices, - service_name ? service_name : "", cb); + ResolvedService::forEachService(*ResolvedService::sAdbSecureConnectServices, service_name, cb); } bool adb_secure_connect_by_service_name(const char* service_name) { @@ -329,23 +411,28 @@ bool adb_secure_connect_by_service_name(const char* service_name) { service_name); } -static void DNSSD_API register_service_ip(DNSServiceRef /*sdRef*/, - DNSServiceFlags /*flags*/, +static void DNSSD_API register_service_ip(DNSServiceRef sdRef, DNSServiceFlags flags, uint32_t /*interfaceIndex*/, - DNSServiceErrorType /*errorCode*/, - const char* /*hostname*/, - const sockaddr* address, - uint32_t /*ttl*/, - void* context) { - D("Got IP for service."); + DNSServiceErrorType errorCode, const char* hostname, + const sockaddr* address, uint32_t ttl, void* context) { + D("%s: sdRef=%p flags=0x%08x errorCode=%u ttl=%u", __func__, sdRef, flags, errorCode, ttl); std::unique_ptr<ResolvedService> data( reinterpret_cast<ResolvedService*>(context)); - data->Connect(address); + // Only resolve the address once. If the address or port changes, we'll just get another + // registration. + data->DestroyServiceRef(); - // For ADB Secure services, keep those ResolvedService's around - // for later processing with secure connection establishment. - if (data->serviceIndex() != kADBTransportServiceRefIndex) { - data.release(); + if (errorCode != kDNSServiceErr_NoError) { + D("Got error while looking up ipaddr [%u]", errorCode); + return; + } + + if (flags & kDNSServiceFlagsAdd) { + D("Resolved IP address for [%s]. Adding to service registry.", hostname); + auto* ptr = data.release(); + if (!ptr->AddToServiceRegistry(address)) { + data.reset(ptr); + } } } @@ -395,9 +482,13 @@ class DiscoveredService : public AsyncServiceRef { }; static void adb_RemoveDNSService(const char* regType, const char* serviceName) { + D("%s: regType=[%s] serviceName=[%s]", __func__, regType, serviceName); int index = adb_DNSServiceIndexByName(regType); ResolvedService::ServiceRegistry* services; switch (index) { + case kADBTransportServiceRefIndex: + services = ResolvedService::sAdbTransportServices; + break; case kADBSecurePairingServiceRefIndex: services = ResolvedService::sAdbSecurePairingServices; break; @@ -408,10 +499,15 @@ static void adb_RemoveDNSService(const char* regType, const char* serviceName) { return; } + if (services->empty()) { + return; + } + std::string sName(serviceName); - services->erase(std::remove_if( - services->begin(), services->end(), - [&sName](ResolvedService* service) { return (sName == service->serviceName()); })); + services->erase(std::remove_if(services->begin(), services->end(), + [&sName](std::unique_ptr<ResolvedService>& service) { + return (sName == service->serviceName()); + })); } // Returns the version the device wanted to advertise, @@ -521,8 +617,15 @@ static void DNSSD_API on_service_browsed(DNSServiceRef sdRef, DNSServiceFlags fl } void init_mdns_transport_discovery_thread(void) { - int errorCodes[kNumADBDNSServices]; + config_auto_connect_services(); + std::string res; + std::for_each(g_autoconn_whitelist.begin(), g_autoconn_whitelist.end(), [&](const int& i) { + res += kADBDNSServices[i]; + res += ","; + }); + D("mdns auto-connect whitelist: [%s]", res.data()); + int errorCodes[kNumADBDNSServices]; for (int i = 0; i < kNumADBDNSServices; ++i) { errorCodes[i] = DNSServiceBrowse(&service_refs[i], 0, 0, kADBDNSServices[i], nullptr, on_service_browsed, nullptr); @@ -542,6 +645,117 @@ void init_mdns_transport_discovery_thread(void) { } void init_mdns_transport_discovery(void) { - ResolvedService::initAdbSecure(); + ResolvedService::initAdbServiceRegistries(); std::thread(init_mdns_transport_discovery_thread).detach(); } + +std::string mdns_check() { + uint32_t daemon_version; + uint32_t sz = sizeof(daemon_version); + + auto dnserr = DNSServiceGetProperty(kDNSServiceProperty_DaemonVersion, &daemon_version, &sz); + std::string result = "ERROR: mdns daemon unavailable"; + if (dnserr != kDNSServiceErr_NoError) { + return result; + } + + result = android::base::StringPrintf("mdns daemon version [%u]", daemon_version); + return result; +} + +std::string mdns_list_discovered_services() { + std::string result; + auto cb = [&](const char* service_name, const char* reg_type, const char* ip_addr, + uint16_t port) { + result += android::base::StringPrintf("%s\t%s\t%s:%u\n", service_name, reg_type, ip_addr, + port); + }; + + ResolvedService::forEachService(*ResolvedService::sAdbTransportServices, "", cb); + ResolvedService::forEachService(*ResolvedService::sAdbSecureConnectServices, "", cb); + ResolvedService::forEachService(*ResolvedService::sAdbSecurePairingServices, "", cb); + return result; +} + +std::optional<MdnsInfo> mdns_get_connect_service_info(std::string_view name) { + CHECK(!name.empty()); + + // only adb server creates these registries + if (!ResolvedService::sAdbTransportServices && !ResolvedService::sAdbSecureConnectServices) { + return std::nullopt; + } + CHECK(ResolvedService::sAdbTransportServices); + CHECK(ResolvedService::sAdbSecureConnectServices); + + auto mdns_instance = mdns::mdns_parse_instance_name(name); + if (!mdns_instance.has_value()) { + D("Failed to parse mDNS name [%s]", name.data()); + return std::nullopt; + } + + std::optional<MdnsInfo> info; + auto cb = [&](const char* service_name, const char* reg_type, const char* ip_addr, + uint16_t port) { info.emplace(service_name, reg_type, ip_addr, port); }; + + std::string reg_type; + if (!mdns_instance->service_name.empty()) { + reg_type = android::base::StringPrintf("%s.%s", mdns_instance->service_name.data(), + mdns_instance->transport_type.data()); + int index = adb_DNSServiceIndexByName(reg_type); + switch (index) { + case kADBTransportServiceRefIndex: + ResolvedService::forEachService(*ResolvedService::sAdbTransportServices, + mdns_instance->instance_name, cb); + break; + case kADBSecureConnectServiceRefIndex: + ResolvedService::forEachService(*ResolvedService::sAdbSecureConnectServices, + mdns_instance->instance_name, cb); + break; + default: + D("Unknown reg_type [%s]", reg_type.data()); + return std::nullopt; + } + return info; + } + + for (const auto& service : + {ResolvedService::sAdbTransportServices, ResolvedService::sAdbSecureConnectServices}) { + ResolvedService::forEachService(*service, name, cb); + if (info.has_value()) { + return info; + } + } + + return std::nullopt; +} + +std::optional<MdnsInfo> mdns_get_pairing_service_info(std::string_view name) { + CHECK(!name.empty()); + + auto mdns_instance = mdns::mdns_parse_instance_name(name); + if (!mdns_instance.has_value()) { + D("Failed to parse mDNS pairing name [%s]", name.data()); + return std::nullopt; + } + + std::optional<MdnsInfo> info; + auto cb = [&](const char* service_name, const char* reg_type, const char* ip_addr, + uint16_t port) { info.emplace(service_name, reg_type, ip_addr, port); }; + + // Verify it's a pairing service if user explicitly inputs it. + if (!mdns_instance->service_name.empty()) { + auto reg_type = android::base::StringPrintf("%s.%s", mdns_instance->service_name.data(), + mdns_instance->transport_type.data()); + int index = adb_DNSServiceIndexByName(reg_type); + switch (index) { + case kADBSecurePairingServiceRefIndex: + break; + default: + D("Not an adb pairing reg_type [%s]", reg_type.data()); + return std::nullopt; + } + } + + ResolvedService::forEachService(*ResolvedService::sAdbSecurePairingServices, name, cb); + return info; +} diff --git a/adb/compression_utils.h b/adb/compression_utils.h new file mode 100644 index 000000000..a74710823 --- /dev/null +++ b/adb/compression_utils.h @@ -0,0 +1,486 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <algorithm> +#include <memory> +#include <span> + +#include <android-base/logging.h> + +#include <brotli/decode.h> +#include <brotli/encode.h> +#include <lz4frame.h> +#include <zstd.h> + +#include "types.h" + +enum class DecodeResult { + Error, + Done, + NeedInput, + MoreOutput, +}; + +enum class EncodeResult { + Error, + Done, + NeedInput, + MoreOutput, +}; + +struct Decoder { + void Append(Block&& block) { input_buffer_.append(std::move(block)); } + bool Finish() { + bool old = std::exchange(finished_, true); + if (old) { + LOG(FATAL) << "Decoder::Finish called while already finished?"; + return false; + } + return true; + } + + virtual DecodeResult Decode(std::span<char>* output) = 0; + + protected: + Decoder(std::span<char> output_buffer) : output_buffer_(output_buffer) {} + ~Decoder() = default; + + bool finished_ = false; + IOVector input_buffer_; + std::span<char> output_buffer_; +}; + +struct Encoder { + void Append(Block input) { input_buffer_.append(std::move(input)); } + bool Finish() { + bool old = std::exchange(finished_, true); + if (old) { + LOG(FATAL) << "Decoder::Finish called while already finished?"; + return false; + } + return true; + } + + virtual EncodeResult Encode(Block* output) = 0; + + protected: + explicit Encoder(size_t output_block_size) : output_block_size_(output_block_size) {} + ~Encoder() = default; + + const size_t output_block_size_; + bool finished_ = false; + IOVector input_buffer_; +}; + +struct NullDecoder final : public Decoder { + explicit NullDecoder(std::span<char> output_buffer) : Decoder(output_buffer) {} + + DecodeResult Decode(std::span<char>* output) final { + size_t available_out = output_buffer_.size(); + void* p = output_buffer_.data(); + while (available_out > 0 && !input_buffer_.empty()) { + size_t len = std::min(available_out, input_buffer_.front_size()); + p = mempcpy(p, input_buffer_.front_data(), len); + available_out -= len; + input_buffer_.drop_front(len); + } + *output = std::span(output_buffer_.data(), static_cast<char*>(p)); + if (input_buffer_.empty()) { + return finished_ ? DecodeResult::Done : DecodeResult::NeedInput; + } + return DecodeResult::MoreOutput; + } +}; + +struct NullEncoder final : public Encoder { + explicit NullEncoder(size_t output_block_size) : Encoder(output_block_size) {} + + EncodeResult Encode(Block* output) final { + output->clear(); + output->resize(output_block_size_); + + size_t available_out = output->size(); + void* p = output->data(); + + while (available_out > 0 && !input_buffer_.empty()) { + size_t len = std::min(available_out, input_buffer_.front_size()); + p = mempcpy(p, input_buffer_.front_data(), len); + available_out -= len; + input_buffer_.drop_front(len); + } + + output->resize(output->size() - available_out); + + if (input_buffer_.empty()) { + return finished_ ? EncodeResult::Done : EncodeResult::NeedInput; + } + return EncodeResult::MoreOutput; + } +}; + +struct BrotliDecoder final : public Decoder { + explicit BrotliDecoder(std::span<char> output_buffer) + : Decoder(output_buffer), + decoder_(BrotliDecoderCreateInstance(nullptr, nullptr, nullptr), + BrotliDecoderDestroyInstance) {} + + DecodeResult Decode(std::span<char>* output) final { + size_t available_in = input_buffer_.front_size(); + const uint8_t* next_in = reinterpret_cast<const uint8_t*>(input_buffer_.front_data()); + + size_t available_out = output_buffer_.size(); + uint8_t* next_out = reinterpret_cast<uint8_t*>(output_buffer_.data()); + + BrotliDecoderResult r = BrotliDecoderDecompressStream( + decoder_.get(), &available_in, &next_in, &available_out, &next_out, nullptr); + + size_t bytes_consumed = input_buffer_.front_size() - available_in; + input_buffer_.drop_front(bytes_consumed); + + size_t bytes_emitted = output_buffer_.size() - available_out; + *output = std::span<char>(output_buffer_.data(), bytes_emitted); + + switch (r) { + case BROTLI_DECODER_RESULT_SUCCESS: + // We need to wait for ID_DONE from the other end. + return finished_ ? DecodeResult::Done : DecodeResult::NeedInput; + case BROTLI_DECODER_RESULT_ERROR: + return DecodeResult::Error; + case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT: + // Brotli guarantees as one of its invariants that if it returns NEEDS_MORE_INPUT, + // it will consume the entire input buffer passed in, so we don't have to worry + // about bytes left over in the front block with more input remaining. + return DecodeResult::NeedInput; + case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT: + return DecodeResult::MoreOutput; + } + } + + private: + std::unique_ptr<BrotliDecoderState, void (*)(BrotliDecoderState*)> decoder_; +}; + +struct BrotliEncoder final : public Encoder { + explicit BrotliEncoder(size_t output_block_size) + : Encoder(output_block_size), + output_block_(output_block_size_), + output_bytes_left_(output_block_size_), + encoder_(BrotliEncoderCreateInstance(nullptr, nullptr, nullptr), + BrotliEncoderDestroyInstance) { + BrotliEncoderSetParameter(encoder_.get(), BROTLI_PARAM_QUALITY, 1); + } + + EncodeResult Encode(Block* output) final { + output->clear(); + + while (true) { + size_t available_in = input_buffer_.front_size(); + const uint8_t* next_in = reinterpret_cast<const uint8_t*>(input_buffer_.front_data()); + + size_t available_out = output_bytes_left_; + uint8_t* next_out = reinterpret_cast<uint8_t*>( + output_block_.data() + (output_block_size_ - output_bytes_left_)); + + BrotliEncoderOperation op = BROTLI_OPERATION_PROCESS; + if (finished_) { + op = BROTLI_OPERATION_FINISH; + } + + if (!BrotliEncoderCompressStream(encoder_.get(), op, &available_in, &next_in, + &available_out, &next_out, nullptr)) { + return EncodeResult::Error; + } + + size_t bytes_consumed = input_buffer_.front_size() - available_in; + input_buffer_.drop_front(bytes_consumed); + + output_bytes_left_ = available_out; + + if (BrotliEncoderIsFinished(encoder_.get())) { + output_block_.resize(output_block_size_ - output_bytes_left_); + *output = std::move(output_block_); + return EncodeResult::Done; + } else if (output_bytes_left_ == 0) { + *output = std::move(output_block_); + output_block_.resize(output_block_size_); + output_bytes_left_ = output_block_size_; + return EncodeResult::MoreOutput; + } else if (input_buffer_.empty()) { + return EncodeResult::NeedInput; + } + } + } + + private: + Block output_block_; + size_t output_bytes_left_; + std::unique_ptr<BrotliEncoderState, void (*)(BrotliEncoderState*)> encoder_; +}; + +struct LZ4Decoder final : public Decoder { + explicit LZ4Decoder(std::span<char> output_buffer) + : Decoder(output_buffer), decoder_(nullptr, nullptr) { + LZ4F_dctx* dctx; + if (LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) != 0) { + LOG(FATAL) << "failed to initialize LZ4 decompression context"; + } + decoder_ = std::unique_ptr<LZ4F_dctx, decltype(&LZ4F_freeDecompressionContext)>( + dctx, LZ4F_freeDecompressionContext); + } + + DecodeResult Decode(std::span<char>* output) final { + size_t available_in = input_buffer_.front_size(); + const char* next_in = input_buffer_.front_data(); + + size_t available_out = output_buffer_.size(); + char* next_out = output_buffer_.data(); + + size_t rc = LZ4F_decompress(decoder_.get(), next_out, &available_out, next_in, + &available_in, nullptr); + if (LZ4F_isError(rc)) { + LOG(ERROR) << "LZ4F_decompress failed: " << LZ4F_getErrorName(rc); + return DecodeResult::Error; + } + + input_buffer_.drop_front(available_in); + + if (rc == 0) { + if (!input_buffer_.empty()) { + LOG(ERROR) << "LZ4 stream hit end before reading all data"; + return DecodeResult::Error; + } + lz4_done_ = true; + } + + *output = std::span<char>(output_buffer_.data(), available_out); + + if (finished_) { + return input_buffer_.empty() && lz4_done_ ? DecodeResult::Done + : DecodeResult::MoreOutput; + } + + return DecodeResult::NeedInput; + } + + private: + bool lz4_done_ = false; + std::unique_ptr<LZ4F_dctx, LZ4F_errorCode_t (*)(LZ4F_dctx*)> decoder_; +}; + +struct LZ4Encoder final : public Encoder { + explicit LZ4Encoder(size_t output_block_size) + : Encoder(output_block_size), encoder_(nullptr, nullptr) { + LZ4F_cctx* cctx; + if (LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) != 0) { + LOG(FATAL) << "failed to initialize LZ4 compression context"; + } + encoder_ = std::unique_ptr<LZ4F_cctx, decltype(&LZ4F_freeCompressionContext)>( + cctx, LZ4F_freeCompressionContext); + Block header(LZ4F_HEADER_SIZE_MAX); + size_t rc = LZ4F_compressBegin(encoder_.get(), header.data(), header.size(), nullptr); + if (LZ4F_isError(rc)) { + LOG(FATAL) << "LZ4F_compressBegin failed: %s", LZ4F_getErrorName(rc); + } + header.resize(rc); + output_buffer_.append(std::move(header)); + } + + // As an optimization, only emit a block if we have an entire output block ready, or we're done. + bool OutputReady() const { + return output_buffer_.size() >= output_block_size_ || lz4_finalized_; + } + + // TODO: Switch the output type to IOVector to remove a copy? + EncodeResult Encode(Block* output) final { + size_t available_in = input_buffer_.front_size(); + const char* next_in = input_buffer_.front_data(); + + // LZ4 makes no guarantees about being able to recover from trying to compress with an + // insufficiently large output buffer. LZ4F_compressBound tells us how much buffer we + // need to compress a given number of bytes, but the smallest value seems to be bigger + // than SYNC_DATA_MAX, so we need to buffer ourselves. + + // Input size chosen to be a local maximum for LZ4F_compressBound (i.e. the block size). + constexpr size_t max_input_size = 65536; + const size_t encode_block_size = LZ4F_compressBound(max_input_size, nullptr); + + if (available_in != 0) { + if (lz4_finalized_) { + LOG(ERROR) << "LZ4Encoder received data after Finish?"; + return EncodeResult::Error; + } + + available_in = std::min(available_in, max_input_size); + + Block encode_block(encode_block_size); + size_t available_out = encode_block.capacity(); + char* next_out = encode_block.data(); + + size_t rc = LZ4F_compressUpdate(encoder_.get(), next_out, available_out, next_in, + available_in, nullptr); + if (LZ4F_isError(rc)) { + LOG(ERROR) << "LZ4F_compressUpdate failed: " << LZ4F_getErrorName(rc); + return EncodeResult::Error; + } + + input_buffer_.drop_front(available_in); + + available_out -= rc; + next_out += rc; + + encode_block.resize(encode_block_size - available_out); + output_buffer_.append(std::move(encode_block)); + } + + if (finished_ && !lz4_finalized_) { + lz4_finalized_ = true; + + Block final_block(encode_block_size + 4); + size_t rc = LZ4F_compressEnd(encoder_.get(), final_block.data(), final_block.size(), + nullptr); + if (LZ4F_isError(rc)) { + LOG(ERROR) << "LZ4F_compressEnd failed: " << LZ4F_getErrorName(rc); + return EncodeResult::Error; + } + + final_block.resize(rc); + output_buffer_.append(std::move(final_block)); + } + + if (OutputReady()) { + size_t len = std::min(output_block_size_, output_buffer_.size()); + *output = output_buffer_.take_front(len).coalesce(); + } else { + output->clear(); + } + + if (lz4_finalized_ && output_buffer_.empty()) { + return EncodeResult::Done; + } else if (OutputReady()) { + return EncodeResult::MoreOutput; + } + return EncodeResult::NeedInput; + } + + private: + bool lz4_finalized_ = false; + std::unique_ptr<LZ4F_cctx, LZ4F_errorCode_t (*)(LZ4F_cctx*)> encoder_; + IOVector output_buffer_; +}; + +struct ZstdDecoder final : public Decoder { + explicit ZstdDecoder(std::span<char> output_buffer) + : Decoder(output_buffer), decoder_(ZSTD_createDStream(), ZSTD_freeDStream) { + if (!decoder_) { + LOG(FATAL) << "failed to initialize Zstd decompression context"; + } + } + + DecodeResult Decode(std::span<char>* output) final { + ZSTD_inBuffer in; + in.src = input_buffer_.front_data(); + in.size = input_buffer_.front_size(); + in.pos = 0; + + ZSTD_outBuffer out; + out.dst = output_buffer_.data(); + // The standard specifies size() as returning size_t, but our current version of + // libc++ returns a signed value instead. + out.size = static_cast<size_t>(output_buffer_.size()); + out.pos = 0; + + size_t rc = ZSTD_decompressStream(decoder_.get(), &out, &in); + if (ZSTD_isError(rc)) { + LOG(ERROR) << "ZSTD_decompressStream failed: " << ZSTD_getErrorName(rc); + return DecodeResult::Error; + } + + input_buffer_.drop_front(in.pos); + if (rc == 0) { + if (!input_buffer_.empty()) { + LOG(ERROR) << "Zstd stream hit end before reading all data"; + return DecodeResult::Error; + } + zstd_done_ = true; + } + + *output = std::span<char>(output_buffer_.data(), out.pos); + + if (finished_) { + return input_buffer_.empty() && zstd_done_ ? DecodeResult::Done + : DecodeResult::MoreOutput; + } + return DecodeResult::NeedInput; + } + + private: + bool zstd_done_ = false; + std::unique_ptr<ZSTD_DStream, size_t (*)(ZSTD_DStream*)> decoder_; +}; + +struct ZstdEncoder final : public Encoder { + explicit ZstdEncoder(size_t output_block_size) + : Encoder(output_block_size), encoder_(ZSTD_createCStream(), ZSTD_freeCStream) { + if (!encoder_) { + LOG(FATAL) << "failed to initialize Zstd compression context"; + } + ZSTD_CCtx_setParameter(encoder_.get(), ZSTD_c_compressionLevel, 1); + } + + EncodeResult Encode(Block* output) final { + ZSTD_inBuffer in; + in.src = input_buffer_.front_data(); + in.size = input_buffer_.front_size(); + in.pos = 0; + + output->resize(output_block_size_); + + ZSTD_outBuffer out; + out.dst = output->data(); + out.size = static_cast<size_t>(output->size()); + out.pos = 0; + + ZSTD_EndDirective end_directive = finished_ ? ZSTD_e_end : ZSTD_e_continue; + size_t rc = ZSTD_compressStream2(encoder_.get(), &out, &in, end_directive); + if (ZSTD_isError(rc)) { + LOG(ERROR) << "ZSTD_compressStream2 failed: " << ZSTD_getErrorName(rc); + return EncodeResult::Error; + } + + input_buffer_.drop_front(in.pos); + output->resize(out.pos); + + if (rc == 0) { + // Zstd finished flushing its data. + if (finished_) { + if (!input_buffer_.empty()) { + LOG(ERROR) << "ZSTD_compressStream2 finished early"; + return EncodeResult::Error; + } + return EncodeResult::Done; + } else { + return input_buffer_.empty() ? EncodeResult::NeedInput : EncodeResult::MoreOutput; + } + } else { + return EncodeResult::MoreOutput; + } + } + + private: + std::unique_ptr<ZSTD_CStream, size_t (*)(ZSTD_CStream*)> encoder_; +}; diff --git a/adb/coverage/.gitignore b/adb/coverage/.gitignore new file mode 100644 index 000000000..b6a258204 --- /dev/null +++ b/adb/coverage/.gitignore @@ -0,0 +1,2 @@ +/adbd.profdata +/report diff --git a/adb/coverage/gen_coverage.sh b/adb/coverage/gen_coverage.sh new file mode 100755 index 000000000..43d45f0d8 --- /dev/null +++ b/adb/coverage/gen_coverage.sh @@ -0,0 +1,119 @@ +#!/bin/bash + +set -euxo pipefail + +OUTPUT_DIR=$(dirname "$0") +. "$OUTPUT_DIR"/include.sh + +TRACEDIR=`mktemp -d` + +### Make sure we can connect to the device. + +# Get the device's wlan0 address. +IP_ADDR=$(adb shell ip route get 0.0.0.0 oif wlan0 | sed -En -e 's/.*src (\S+)\s.*/\1/p') +REMOTE_PORT=5555 +REMOTE=$IP_ADDR:$REMOTE_PORT +LOCAL_SERIAL=$(adb shell getprop ro.serialno) + +# Check that we can connect to it. +adb disconnect + +TRANSPORT_ID=$(adb transport-id) +adb tcpip $REMOTE_PORT +adb -t $TRANSPORT_ID wait-for-disconnect + +adb connect $REMOTE + +REMOTE_FETCHED_SERIAL=$(adb -s $REMOTE shell getprop ro.serialno) + +if [[ "$LOCAL_SERIAL" != "$REMOTE_FETCHED_SERIAL" ]]; then + echo "Mismatch: local serial = $LOCAL_SERIAL, remote serial = $REMOTE_FETCHED_SERIAL" + exit 1 +fi + +# Back to USB, and make sure adbd is root. +adb -s $REMOTE usb +adb disconnect $REMOTE + +adb wait-for-device root +adb root +adb wait-for-device + +TRANSPORT_ID=$(adb transport-id) +adb usb +adb -t $TRANSPORT_ID wait-for-disconnect + +adb wait-for-device + +### Run the adb unit tests and fetch traces from them. +mkdir "$TRACEDIR"/test_traces +adb shell rm -rf /data/local/tmp/adb_coverage +adb shell mkdir /data/local/tmp/adb_coverage + +for TEST in $ADB_TESTS; do + adb shell LLVM_PROFILE_FILE=/data/local/tmp/adb_coverage/$TEST.profraw /data/nativetest64/$TEST/$TEST + adb pull /data/local/tmp/adb_coverage/$TEST.profraw "$TRACEDIR"/test_traces/ +done + +adb pull /data/local/tmp/adb_coverage "$TRACEDIR"/test_traces + +# Clear logcat and increase the buffer to something ridiculous so we can fetch the pids of adbd later. +adb shell logcat -c -G128M + +# Turn on extremely verbose logging so as to not count debug logging against us. +adb shell setprop persist.adb.trace_mask 1 + +### Run test_device.py over USB. +TRANSPORT_ID=$(adb transport-id) +adb shell killall adbd +adb -t $TRANSPORT_ID wait-for-disconnect + +adb wait-for-device shell rm -rf "/data/misc/trace/*" /data/local/tmp/adb_coverage/ +"$OUTPUT_DIR"/../test_device.py + +# Do a usb reset to exercise the disconnect code. +adb_usbreset +adb wait-for-device + +# Dump traces from the currently running adbd. +adb shell killall -37 adbd + +echo Waiting for adbd to finish dumping traces +sleep 5 + +# Restart adbd in tcp mode. +TRANSPORT_ID=$(adb transport-id) +adb tcpip $REMOTE_PORT +adb -t $TRANSPORT_ID wait-for-disconnect + +adb connect $REMOTE +adb -s $REMOTE wait-for-device + +# Instead of running test_device.py again, which takes forever, do some I/O back and forth instead. +dd if=/dev/zero bs=1024 count=10240 | adb -s $REMOTE raw sink:10485760 +adb -s $REMOTE raw source:10485760 | dd of=/dev/null bs=1024 count=10240 + +# Dump traces again. +adb disconnect $REMOTE +adb shell killall -37 adbd + +echo Waiting for adbd to finish dumping traces +sleep 5 + +adb pull /data/misc/trace "$TRACEDIR"/ +echo Pulled traces to $TRACEDIR + +# Identify which of the trace files are actually adbd, in case something else exited simultaneously. +ADBD_PIDS=$(adb shell "logcat -d -s adbd --format=process | grep 'adbd started' | cut -c 3-7 | tr -d ' ' | sort | uniq") +mkdir "$TRACEDIR"/adbd_traces + +adb shell 'setprop persist.adb.trace_mask 0; killall adbd' + +IFS=$'\n' +for PID in $ADBD_PIDS; do + cp "$TRACEDIR"/trace/clang-$PID-*.profraw "$TRACEDIR"/adbd_traces 2>/dev/null || true +done +unset IFS + +### Merge the traces. +llvm-profdata merge --output="$OUTPUT_DIR"/adbd.profdata "$TRACEDIR"/adbd_traces/* "$TRACEDIR"/test_traces/* diff --git a/adb/coverage/include.sh b/adb/coverage/include.sh new file mode 100644 index 000000000..45ebc3475 --- /dev/null +++ b/adb/coverage/include.sh @@ -0,0 +1,5 @@ +ADB_TESTS="adbd_test adb_crypto_test adb_pairing_auth_test adb_pairing_connection_test adb_tls_connection_test" +ADB_TEST_BINARIES="" +for TEST in $ADB_TESTS; do + ADB_TEST_BINARIES="--object=$ANDROID_PRODUCT_OUT/data/nativetest64/$TEST/$TEST $ADB_TEST_BINARIES" +done diff --git a/adb/coverage/report.sh b/adb/coverage/report.sh new file mode 100755 index 000000000..257310c6a --- /dev/null +++ b/adb/coverage/report.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -euxo pipefail + +OUTPUT_DIR=$(realpath $(dirname "$0")) +. "$OUTPUT_DIR"/include.sh + +rm -rf "$OUTPUT_DIR"/report + +cd $ANDROID_BUILD_TOP +llvm-cov show --instr-profile="$OUTPUT_DIR"/adbd.profdata \ + $ANDROID_PRODUCT_OUT/apex/com.android.adbd/bin/adbd \ + /proc/self/cwd/system/core/adb \ + $ADB_TEST_BINARIES \ + --show-region-summary=false \ + --format=html -o "$OUTPUT_DIR"/report + +llvm-cov report --instr-profile="$OUTPUT_DIR"/adbd.profdata \ + $ANDROID_PRODUCT_OUT/apex/com.android.adbd/bin/adbd \ + /proc/self/cwd/system/core/adb \ + $ADB_TEST_BINARIES \ + --show-region-summary=false diff --git a/adb/coverage/show.sh b/adb/coverage/show.sh new file mode 100755 index 000000000..3b2faa304 --- /dev/null +++ b/adb/coverage/show.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -euxo pipefail + +OUTPUT_DIR=$(realpath $(dirname "$0")) +. "$OUTPUT_DIR"/include.sh + +BASE_PATH=/proc/self/cwd/system/core/adb +PATHS="" +if [[ $# == 0 ]]; then + PATHS=$BASE_PATH +else + for arg in "$@"; do + PATHS="$PATHS $BASE_PATH/$arg" + done +fi + +cd $ANDROID_BUILD_TOP +llvm-cov show --instr-profile="$OUTPUT_DIR"/adbd.profdata \ + $ANDROID_PRODUCT_OUT/apex/com.android.adbd/bin/adbd \ + $PATHS \ + $ADB_TEST_BINARIES diff --git a/adb/daemon/file_sync_service.cpp b/adb/daemon/file_sync_service.cpp index 07f6e65e8..513b8dd8f 100644 --- a/adb/daemon/file_sync_service.cpp +++ b/adb/daemon/file_sync_service.cpp @@ -35,6 +35,7 @@ #include <optional> #include <span> #include <string> +#include <variant> #include <vector> #include <android-base/file.h> @@ -57,7 +58,7 @@ #include "adb_io.h" #include "adb_trace.h" #include "adb_utils.h" -#include "brotli_utils.h" +#include "compression_utils.h" #include "file_sync_protocol.h" #include "security_log_tags.h" #include "sysdeps/errno.h" @@ -266,47 +267,75 @@ static bool SendSyncFailErrno(borrowed_fd fd, const std::string& reason) { return SendSyncFail(fd, StringPrintf("%s: %s", reason.c_str(), strerror(errno))); } -static bool handle_send_file_compressed(borrowed_fd s, unique_fd fd, uint32_t* timestamp) { +static bool handle_send_file_data(borrowed_fd s, unique_fd fd, uint32_t* timestamp, + CompressionType compression) { syncmsg msg; - Block decode_buffer(SYNC_DATA_MAX); - BrotliDecoder decoder(std::span(decode_buffer.data(), decode_buffer.size())); + Block buffer(SYNC_DATA_MAX); + std::span<char> buffer_span(buffer.data(), buffer.size()); + std::variant<std::monostate, NullDecoder, BrotliDecoder, LZ4Decoder, ZstdDecoder> + decoder_storage; + Decoder* decoder = nullptr; + + switch (compression) { + case CompressionType::None: + decoder = &decoder_storage.emplace<NullDecoder>(buffer_span); + break; + + case CompressionType::Brotli: + decoder = &decoder_storage.emplace<BrotliDecoder>(buffer_span); + break; + + case CompressionType::LZ4: + decoder = &decoder_storage.emplace<LZ4Decoder>(buffer_span); + break; + + case CompressionType::Zstd: + decoder = &decoder_storage.emplace<ZstdDecoder>(buffer_span); + break; + + case CompressionType::Any: + LOG(FATAL) << "unexpected CompressionType::Any"; + } + while (true) { if (!ReadFdExactly(s, &msg.data, sizeof(msg.data))) return false; - if (msg.data.id != ID_DATA) { - if (msg.data.id == ID_DONE) { - *timestamp = msg.data.size; - return true; - } + if (msg.data.id == ID_DONE) { + *timestamp = msg.data.size; + decoder->Finish(); + } else if (msg.data.id == ID_DATA) { + Block block(msg.data.size); + if (!ReadFdExactly(s, block.data(), msg.data.size)) return false; + decoder->Append(std::move(block)); + } else { SendSyncFail(s, "invalid data message"); return false; } - Block block(msg.data.size); - if (!ReadFdExactly(s, block.data(), msg.data.size)) return false; - decoder.Append(std::move(block)); - while (true) { std::span<char> output; - BrotliDecodeResult result = decoder.Decode(&output); - if (result == BrotliDecodeResult::Error) { + DecodeResult result = decoder->Decode(&output); + if (result == DecodeResult::Error) { SendSyncFailErrno(s, "decompress failed"); return false; } - if (!WriteFdExactly(fd, output.data(), output.size())) { - SendSyncFailErrno(s, "write failed"); - return false; + // fd is -1 if the client is pushing with --dry-run. + if (fd != -1) { + if (!WriteFdExactly(fd, output.data(), output.size())) { + SendSyncFailErrno(s, "write failed"); + return false; + } } - if (result == BrotliDecodeResult::NeedInput) { + if (result == DecodeResult::NeedInput) { break; - } else if (result == BrotliDecodeResult::MoreOutput) { + } else if (result == DecodeResult::MoreOutput) { continue; - } else if (result == BrotliDecodeResult::Done) { - break; + } else if (result == DecodeResult::Done) { + return true; } else { - LOG(FATAL) << "invalid BrotliDecodeResult: " << static_cast<int>(result); + LOG(FATAL) << "invalid DecodeResult: " << static_cast<int>(result); } } } @@ -314,102 +343,67 @@ static bool handle_send_file_compressed(borrowed_fd s, unique_fd fd, uint32_t* t __builtin_unreachable(); } -static bool handle_send_file_uncompressed(borrowed_fd s, unique_fd fd, uint32_t* timestamp, - std::vector<char>& buffer) { +static bool handle_send_file(borrowed_fd s, const char* path, uint32_t* timestamp, uid_t uid, + gid_t gid, uint64_t capabilities, mode_t mode, + CompressionType compression, bool dry_run, std::vector<char>& buffer, + bool do_unlink) { syncmsg msg; + unique_fd fd; - while (true) { - if (!ReadFdExactly(s, &msg.data, sizeof(msg.data))) return false; + if (!dry_run) { + __android_log_security_bswrite(SEC_TAG_ADB_SEND_FILE, path); + fd.reset(adb_open_mode(path, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, mode)); - if (msg.data.id != ID_DATA) { - if (msg.data.id == ID_DONE) { - *timestamp = msg.data.size; - return true; + if (fd < 0 && errno == ENOENT) { + if (!secure_mkdirs(Dirname(path))) { + SendSyncFailErrno(s, "secure_mkdirs failed"); + goto fail; } - SendSyncFail(s, "invalid data message"); - return false; + fd.reset(adb_open_mode(path, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, mode)); } - - if (msg.data.size > buffer.size()) { // TODO: resize buffer? - SendSyncFail(s, "oversize data message"); - return false; - } - if (!ReadFdExactly(s, &buffer[0], msg.data.size)) return false; - if (!WriteFdExactly(fd, &buffer[0], msg.data.size)) { - SendSyncFailErrno(s, "write failed"); - return false; - } - } -} - -static bool handle_send_file(borrowed_fd s, const char* path, uint32_t* timestamp, uid_t uid, - gid_t gid, uint64_t capabilities, mode_t mode, bool compressed, - std::vector<char>& buffer, bool do_unlink) { - int rc; - syncmsg msg; - - __android_log_security_bswrite(SEC_TAG_ADB_SEND_FILE, path); - - unique_fd fd(adb_open_mode(path, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, mode)); - - if (fd < 0 && errno == ENOENT) { - if (!secure_mkdirs(Dirname(path))) { - SendSyncFailErrno(s, "secure_mkdirs failed"); - goto fail; + if (fd < 0 && errno == EEXIST) { + fd.reset(adb_open_mode(path, O_WRONLY | O_CLOEXEC, mode)); } - fd.reset(adb_open_mode(path, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, mode)); - } - if (fd < 0 && errno == EEXIST) { - fd.reset(adb_open_mode(path, O_WRONLY | O_CLOEXEC, mode)); - } - if (fd < 0) { - SendSyncFailErrno(s, "couldn't create file"); - goto fail; - } else { - if (fchown(fd.get(), uid, gid) == -1) { - SendSyncFailErrno(s, "fchown failed"); + if (fd < 0) { + SendSyncFailErrno(s, "couldn't create file"); goto fail; - } + } else { + if (fchown(fd.get(), uid, gid) == -1) { + SendSyncFailErrno(s, "fchown failed"); + goto fail; + } #if defined(__ANDROID__) - // Not all filesystems support setting SELinux labels. http://b/23530370. - selinux_android_restorecon(path, 0); + // Not all filesystems support setting SELinux labels. http://b/23530370. + selinux_android_restorecon(path, 0); #endif - // fchown clears the setuid bit - restore it if present. - // Ignore the result of calling fchmod. It's not supported - // by all filesystems, so we don't check for success. b/12441485 - fchmod(fd.get(), mode); - } + // fchown clears the setuid bit - restore it if present. + // Ignore the result of calling fchmod. It's not supported + // by all filesystems, so we don't check for success. b/12441485 + fchmod(fd.get(), mode); + } - { - rc = posix_fadvise(fd.get(), 0, 0, - POSIX_FADV_SEQUENTIAL | POSIX_FADV_NOREUSE | POSIX_FADV_WILLNEED); + int rc = posix_fadvise(fd.get(), 0, 0, + POSIX_FADV_SEQUENTIAL | POSIX_FADV_NOREUSE | POSIX_FADV_WILLNEED); if (rc != 0) { D("[ Failed to fadvise: %s ]", strerror(rc)); } + } - bool result; - if (compressed) { - result = handle_send_file_compressed(s, std::move(fd), timestamp); - } else { - result = handle_send_file_uncompressed(s, std::move(fd), timestamp, buffer); - } - - if (!result) { - goto fail; - } - - if (!update_capabilities(path, capabilities)) { - SendSyncFailErrno(s, "update_capabilities failed"); - goto fail; - } + if (!handle_send_file_data(s, std::move(fd), timestamp, compression)) { + goto fail; + } - msg.status.id = ID_OKAY; - msg.status.msglen = 0; - return WriteFdExactly(s, &msg.status, sizeof(msg.status)); + if (!update_capabilities(path, capabilities)) { + SendSyncFailErrno(s, "update_capabilities failed"); + goto fail; } + msg.status.id = ID_OKAY; + msg.status.msglen = 0; + return WriteFdExactly(s, &msg.status, sizeof(msg.status)); + fail: // If there's a problem on the device, we'll send an ID_FAIL message and // close the socket. Unfortunately the kernel will sometimes throw that @@ -448,7 +442,7 @@ extern bool handle_send_link(int s, const std::string& path, uint32_t* timestamp, std::vector<char>& buffer) __attribute__((error("no symlinks on Windows"))); #else -static bool handle_send_link(int s, const std::string& path, uint32_t* timestamp, +static bool handle_send_link(int s, const std::string& path, uint32_t* timestamp, bool dry_run, std::vector<char>& buffer) { syncmsg msg; @@ -467,19 +461,21 @@ static bool handle_send_link(int s, const std::string& path, uint32_t* timestamp if (!ReadFdExactly(s, &buffer[0], len)) return false; std::string buf_link; - if (!android::base::Readlink(path, &buf_link) || (buf_link != &buffer[0])) { - adb_unlink(path.c_str()); - auto ret = symlink(&buffer[0], path.c_str()); - if (ret && errno == ENOENT) { - if (!secure_mkdirs(Dirname(path))) { - SendSyncFailErrno(s, "secure_mkdirs failed"); + if (!dry_run) { + if (!android::base::Readlink(path, &buf_link) || (buf_link != &buffer[0])) { + adb_unlink(path.c_str()); + auto ret = symlink(&buffer[0], path.c_str()); + if (ret && errno == ENOENT) { + if (!secure_mkdirs(Dirname(path))) { + SendSyncFailErrno(s, "secure_mkdirs failed"); + return false; + } + ret = symlink(&buffer[0], path.c_str()); + } + if (ret) { + SendSyncFailErrno(s, "symlink failed"); return false; } - ret = symlink(&buffer[0], path.c_str()); - } - if (ret) { - SendSyncFailErrno(s, "symlink failed"); - return false; } } @@ -499,12 +495,15 @@ static bool handle_send_link(int s, const std::string& path, uint32_t* timestamp } #endif -static bool send_impl(int s, const std::string& path, mode_t mode, bool compressed, - std::vector<char>& buffer) { +static bool send_impl(int s, const std::string& path, mode_t mode, CompressionType compression, + bool dry_run, std::vector<char>& buffer) { // Don't delete files before copying if they are not "regular" or symlinks. struct stat st; - bool do_unlink = (lstat(path.c_str(), &st) == -1) || S_ISREG(st.st_mode) || - (S_ISLNK(st.st_mode) && !S_ISLNK(mode)); + bool do_unlink = false; + if (!dry_run) { + do_unlink = (lstat(path.c_str(), &st) == -1) || S_ISREG(st.st_mode) || + (S_ISLNK(st.st_mode) && !S_ISLNK(mode)); + } if (do_unlink) { adb_unlink(path.c_str()); } @@ -512,7 +511,7 @@ static bool send_impl(int s, const std::string& path, mode_t mode, bool compress bool result; uint32_t timestamp; if (S_ISLNK(mode)) { - result = handle_send_link(s, path, ×tamp, buffer); + result = handle_send_link(s, path, ×tamp, dry_run, buffer); } else { // Copy user permission bits to "group" and "other" permissions. mode &= 0777; @@ -522,12 +521,12 @@ static bool send_impl(int s, const std::string& path, mode_t mode, bool compress uid_t uid = -1; gid_t gid = -1; uint64_t capabilities = 0; - if (should_use_fs_config(path)) { + if (should_use_fs_config(path) && !dry_run) { adbd_fs_config(path.c_str(), 0, nullptr, &uid, &gid, &mode, &capabilities); } result = handle_send_file(s, path.c_str(), ×tamp, uid, gid, capabilities, mode, - compressed, buffer, do_unlink); + compression, dry_run, buffer, do_unlink); } if (!result) { @@ -560,7 +559,7 @@ static bool do_send_v1(int s, const std::string& spec, std::vector<char>& buffer return false; } - return send_impl(s, path, mode, false, buffer); + return send_impl(s, path, mode, CompressionType::None, false, buffer); } static bool do_send_v2(int s, const std::string& path, std::vector<char>& buffer) { @@ -574,46 +573,94 @@ static bool do_send_v2(int s, const std::string& path, std::vector<char>& buffer PLOG(ERROR) << "failed to read send_v2 setup packet"; } - bool compressed = false; + bool dry_run = false; + std::optional<CompressionType> compression; + + uint32_t orig_flags = msg.send_v2_setup.flags; if (msg.send_v2_setup.flags & kSyncFlagBrotli) { msg.send_v2_setup.flags &= ~kSyncFlagBrotli; - compressed = true; + if (compression) { + SendSyncFail(s, android::base::StringPrintf("multiple compression flags received: %d", + orig_flags)); + return false; + } + compression = CompressionType::Brotli; + } + if (msg.send_v2_setup.flags & kSyncFlagLZ4) { + msg.send_v2_setup.flags &= ~kSyncFlagLZ4; + if (compression) { + SendSyncFail(s, android::base::StringPrintf("multiple compression flags received: %d", + orig_flags)); + return false; + } + compression = CompressionType::LZ4; + } + if (msg.send_v2_setup.flags & kSyncFlagZstd) { + msg.send_v2_setup.flags &= ~kSyncFlagZstd; + if (compression) { + SendSyncFail(s, android::base::StringPrintf("multiple compression flags received: %d", + orig_flags)); + return false; + } + compression = CompressionType::Zstd; + } + if (msg.send_v2_setup.flags & kSyncFlagDryRun) { + msg.send_v2_setup.flags &= ~kSyncFlagDryRun; + dry_run = true; } + if (msg.send_v2_setup.flags) { SendSyncFail(s, android::base::StringPrintf("unknown flags: %d", msg.send_v2_setup.flags)); return false; } errno = 0; - return send_impl(s, path, msg.send_v2_setup.mode, compressed, buffer); + return send_impl(s, path, msg.send_v2_setup.mode, compression.value_or(CompressionType::None), + dry_run, buffer); } -static bool recv_uncompressed(borrowed_fd s, unique_fd fd, std::vector<char>& buffer) { - syncmsg msg; - msg.data.id = ID_DATA; - std::optional<BrotliEncoder<SYNC_DATA_MAX>> encoder; - while (true) { - int r = adb_read(fd.get(), &buffer[0], buffer.size() - sizeof(msg.data)); - if (r <= 0) { - if (r == 0) break; - SendSyncFailErrno(s, "read failed"); - return false; - } - msg.data.size = r; +static bool recv_impl(borrowed_fd s, const char* path, CompressionType compression, + std::vector<char>& buffer) { + __android_log_security_bswrite(SEC_TAG_ADB_RECV_FILE, path); - if (!WriteFdExactly(s, &msg.data, sizeof(msg.data)) || !WriteFdExactly(s, &buffer[0], r)) { - return false; - } + unique_fd fd(adb_open(path, O_RDONLY | O_CLOEXEC)); + if (fd < 0) { + SendSyncFailErrno(s, "open failed"); + return false; } - return true; -} + int rc = posix_fadvise(fd.get(), 0, 0, POSIX_FADV_SEQUENTIAL | POSIX_FADV_NOREUSE); + if (rc != 0) { + D("[ Failed to fadvise: %s ]", strerror(rc)); + } -static bool recv_compressed(borrowed_fd s, unique_fd fd) { syncmsg msg; msg.data.id = ID_DATA; - BrotliEncoder<SYNC_DATA_MAX> encoder; + std::variant<std::monostate, NullEncoder, BrotliEncoder, LZ4Encoder, ZstdEncoder> + encoder_storage; + Encoder* encoder; + + switch (compression) { + case CompressionType::None: + encoder = &encoder_storage.emplace<NullEncoder>(SYNC_DATA_MAX); + break; + + case CompressionType::Brotli: + encoder = &encoder_storage.emplace<BrotliEncoder>(SYNC_DATA_MAX); + break; + + case CompressionType::LZ4: + encoder = &encoder_storage.emplace<LZ4Encoder>(SYNC_DATA_MAX); + break; + + case CompressionType::Zstd: + encoder = &encoder_storage.emplace<ZstdEncoder>(SYNC_DATA_MAX); + break; + + case CompressionType::Any: + LOG(FATAL) << "unexpected CompressionType::Any"; + } bool sending = true; while (sending) { @@ -625,16 +672,16 @@ static bool recv_compressed(borrowed_fd s, unique_fd fd) { } if (r == 0) { - encoder.Finish(); + encoder->Finish(); } else { input.resize(r); - encoder.Append(std::move(input)); + encoder->Append(std::move(input)); } while (true) { Block output; - BrotliEncodeResult result = encoder.Encode(&output); - if (result == BrotliEncodeResult::Error) { + EncodeResult result = encoder->Encode(&output); + if (result == EncodeResult::Error) { SendSyncFailErrno(s, "compress failed"); return false; } @@ -647,53 +694,24 @@ static bool recv_compressed(borrowed_fd s, unique_fd fd) { } } - if (result == BrotliEncodeResult::Done) { + if (result == EncodeResult::Done) { sending = false; break; - } else if (result == BrotliEncodeResult::NeedInput) { + } else if (result == EncodeResult::NeedInput) { break; - } else if (result == BrotliEncodeResult::MoreOutput) { + } else if (result == EncodeResult::MoreOutput) { continue; } } } - return true; -} - -static bool recv_impl(borrowed_fd s, const char* path, bool compressed, std::vector<char>& buffer) { - __android_log_security_bswrite(SEC_TAG_ADB_RECV_FILE, path); - - unique_fd fd(adb_open(path, O_RDONLY | O_CLOEXEC)); - if (fd < 0) { - SendSyncFailErrno(s, "open failed"); - return false; - } - - int rc = posix_fadvise(fd.get(), 0, 0, POSIX_FADV_SEQUENTIAL | POSIX_FADV_NOREUSE); - if (rc != 0) { - D("[ Failed to fadvise: %s ]", strerror(rc)); - } - - bool result; - if (compressed) { - result = recv_compressed(s, std::move(fd)); - } else { - result = recv_uncompressed(s, std::move(fd), buffer); - } - - if (!result) { - return false; - } - - syncmsg msg; msg.data.id = ID_DONE; msg.data.size = 0; return WriteFdExactly(s, &msg.data, sizeof(msg.data)); } static bool do_recv_v1(borrowed_fd s, const char* path, std::vector<char>& buffer) { - return recv_impl(s, path, false, buffer); + return recv_impl(s, path, CompressionType::None, buffer); } static bool do_recv_v2(borrowed_fd s, const char* path, std::vector<char>& buffer) { @@ -707,17 +725,42 @@ static bool do_recv_v2(borrowed_fd s, const char* path, std::vector<char>& buffe PLOG(ERROR) << "failed to read recv_v2 setup packet"; } - bool compressed = false; + std::optional<CompressionType> compression; + uint32_t orig_flags = msg.recv_v2_setup.flags; if (msg.recv_v2_setup.flags & kSyncFlagBrotli) { msg.recv_v2_setup.flags &= ~kSyncFlagBrotli; - compressed = true; + if (compression) { + SendSyncFail(s, android::base::StringPrintf("multiple compression flags received: %d", + orig_flags)); + return false; + } + compression = CompressionType::Brotli; + } + if (msg.recv_v2_setup.flags & kSyncFlagLZ4) { + msg.recv_v2_setup.flags &= ~kSyncFlagLZ4; + if (compression) { + SendSyncFail(s, android::base::StringPrintf("multiple compression flags received: %d", + orig_flags)); + return false; + } + compression = CompressionType::LZ4; } + if (msg.recv_v2_setup.flags & kSyncFlagZstd) { + msg.recv_v2_setup.flags &= ~kSyncFlagZstd; + if (compression) { + SendSyncFail(s, android::base::StringPrintf("multiple compression flags received: %d", + orig_flags)); + return false; + } + compression = CompressionType::Zstd; + } + if (msg.recv_v2_setup.flags) { SendSyncFail(s, android::base::StringPrintf("unknown flags: %d", msg.recv_v2_setup.flags)); return false; } - return recv_impl(s, path, compressed, buffer); + return recv_impl(s, path, compression.value_or(CompressionType::None), buffer); } static const char* sync_id_to_name(uint32_t id) { diff --git a/adb/daemon/jdwp_service.cpp b/adb/daemon/jdwp_service.cpp index b92a7dec0..adae9f7f4 100644 --- a/adb/daemon/jdwp_service.cpp +++ b/adb/daemon/jdwp_service.cpp @@ -16,11 +16,13 @@ #if !ADB_HOST +#if !defined(__ANDROID_RECOVERY__) #define TRACE_TAG JDWP #include "sysdeps.h" #include <errno.h> +#include <inttypes.h> #include <stdio.h> #include <stdlib.h> #include <string.h> @@ -33,6 +35,7 @@ #include <thread> #include <vector> +#include <adbconnection/process_info.h> #include <adbconnection/server.h> #include <android-base/cmsg.h> #include <android-base/unique_fd.h> @@ -41,6 +44,7 @@ #include "adb_io.h" #include "adb_unique_fd.h" #include "adb_utils.h" +#include "app_processes.pb.h" using android::base::borrowed_fd; using android::base::unique_fd; @@ -132,18 +136,24 @@ using android::base::unique_fd; ** for each JDWP process, we record its pid and its connected socket **/ +enum class TrackerKind { + kJdwp, + kApp, +}; + static void jdwp_process_event(int socket, unsigned events, void* _proc); static void jdwp_process_list_updated(void); +static void app_process_list_updated(void); struct JdwpProcess; static auto& _jdwp_list = *new std::list<std::unique_ptr<JdwpProcess>>(); struct JdwpProcess { - JdwpProcess(unique_fd socket, pid_t pid) { - CHECK(pid != 0); + JdwpProcess(unique_fd socket, ProcessInfo process) { + CHECK(process.pid != 0); this->socket = socket; - this->pid = pid; + this->process = process; this->fde = fdevent_create(socket.release(), jdwp_process_event, this); if (!this->fde) { @@ -171,17 +181,19 @@ struct JdwpProcess { } borrowed_fd socket = -1; - int32_t pid = -1; + ProcessInfo process; fdevent* fde = nullptr; std::vector<unique_fd> out_fds; }; +// Populate the list of processes for "track-jdwp" service. static size_t jdwp_process_list(char* buffer, size_t bufferlen) { std::string temp; for (auto& proc : _jdwp_list) { - std::string next = std::to_string(proc->pid) + "\n"; + if (!proc->process.debuggable) continue; + std::string next = std::to_string(proc->process.pid) + "\n"; if (temp.length() + next.length() > bufferlen) { D("truncating JDWP process list (max len = %zu)", bufferlen); break; @@ -193,7 +205,44 @@ static size_t jdwp_process_list(char* buffer, size_t bufferlen) { return temp.length(); } -static size_t jdwp_process_list_msg(char* buffer, size_t bufferlen) { +// Populate the list of processes for "track-app" service. +// The list is a protobuf message in the binary format for efficiency. +static size_t app_process_list(char* buffer, size_t bufferlen) { + adb::proto::AppProcesses output; // result that's guaranteed to fit in the given buffer + adb::proto::AppProcesses temp; // temporary result that may be longer than the given buffer + std::string serialized_message; + + for (auto& proc : _jdwp_list) { + if (!proc->process.debuggable && !proc->process.profileable) continue; + auto* entry = temp.add_process(); + entry->set_pid(proc->process.pid); + entry->set_debuggable(proc->process.debuggable); + entry->set_profileable(proc->process.profileable); + entry->set_architecture(proc->process.arch_name, proc->process.arch_name_length); + temp.SerializeToString(&serialized_message); + if (serialized_message.size() > bufferlen) { + D("truncating app process list (max len = %zu)", bufferlen); + break; + } + output = temp; + } + output.SerializeToString(&serialized_message); + memcpy(buffer, serialized_message.data(), serialized_message.length()); + return serialized_message.length(); +} + +// Populate the list of processes for either "track-jdwp" or "track-app" services, +// depending on the given kind. +static size_t process_list(TrackerKind kind, char* buffer, size_t bufferlen) { + switch (kind) { + case TrackerKind::kJdwp: + return jdwp_process_list(buffer, bufferlen); + case TrackerKind::kApp: + return app_process_list(buffer, bufferlen); + } +} + +static size_t process_list_msg(TrackerKind kind, char* buffer, size_t bufferlen) { // Message is length-prefixed with 4 hex digits in ASCII. static constexpr size_t header_len = 4; if (bufferlen < header_len) { @@ -201,7 +250,7 @@ static size_t jdwp_process_list_msg(char* buffer, size_t bufferlen) { } char head[header_len + 1]; - size_t len = jdwp_process_list(buffer + header_len, bufferlen - header_len); + size_t len = process_list(kind, buffer + header_len, bufferlen - header_len); snprintf(head, sizeof head, "%04zx", len); memcpy(buffer, head, header_len); return len + header_len; @@ -213,7 +262,7 @@ static void jdwp_process_event(int socket, unsigned events, void* _proc) { if (events & FDE_READ) { // We already have the PID, if we can read from the socket, we've probably hit EOF. - D("terminating JDWP connection %d", proc->pid); + D("terminating JDWP connection %" PRId64, proc->process.pid); goto CloseProcess; } @@ -223,11 +272,12 @@ static void jdwp_process_event(int socket, unsigned events, void* _proc) { int fd = proc->out_fds.back().get(); if (android::base::SendFileDescriptors(socket, "", 1, fd) != 1) { - D("sending new file descriptor to JDWP %d failed: %s", proc->pid, strerror(errno)); + D("sending new file descriptor to JDWP %" PRId64 " failed: %s", proc->process.pid, + strerror(errno)); goto CloseProcess; } - D("sent file descriptor %d to JDWP process %d", fd, proc->pid); + D("sent file descriptor %d to JDWP process %" PRId64, fd, proc->process.pid); proc->out_fds.pop_back(); if (proc->out_fds.empty()) { @@ -238,15 +288,20 @@ static void jdwp_process_event(int socket, unsigned events, void* _proc) { return; CloseProcess: + bool debuggable = proc->process.debuggable; + bool profileable = proc->process.profileable; proc->RemoveFromList(); - jdwp_process_list_updated(); + if (debuggable) jdwp_process_list_updated(); + if (debuggable || profileable) app_process_list_updated(); } unique_fd create_jdwp_connection_fd(int pid) { D("looking for pid %d in JDWP process list", pid); for (auto& proc : _jdwp_list) { - if (proc->pid == pid) { + // Don't allow JDWP connection to a non-debuggable process. + if (!proc->process.debuggable) continue; + if (proc->process.pid == static_cast<uint64_t>(pid)) { int fds[2]; if (adb_socketpair(fds) < 0) { @@ -338,18 +393,22 @@ asocket* create_jdwp_service_socket(void) { **/ struct JdwpTracker : public asocket { + TrackerKind kind; bool need_initial; + + explicit JdwpTracker(TrackerKind k, bool initial) : kind(k), need_initial(initial) {} }; static auto& _jdwp_trackers = *new std::vector<std::unique_ptr<JdwpTracker>>(); -static void jdwp_process_list_updated(void) { +static void process_list_updated(TrackerKind kind) { std::string data; - data.resize(1024); - data.resize(jdwp_process_list_msg(&data[0], data.size())); + const int kMaxLength = kind == TrackerKind::kJdwp ? 1024 : 2048; + data.resize(kMaxLength); + data.resize(process_list_msg(kind, &data[0], data.size())); for (auto& t : _jdwp_trackers) { - if (t->peer) { + if (t->kind == kind && t->peer) { // The tracker might not have been connected yet. apacket::payload_type payload(data.begin(), data.end()); t->peer->enqueue(t->peer, std::move(payload)); @@ -357,6 +416,14 @@ static void jdwp_process_list_updated(void) { } } +static void jdwp_process_list_updated(void) { + process_list_updated(TrackerKind::kJdwp); +} + +static void app_process_list_updated(void) { + process_list_updated(TrackerKind::kApp); +} + static void jdwp_tracker_close(asocket* s) { D("LS(%d): destroying jdwp tracker service", s->id); @@ -380,7 +447,7 @@ static void jdwp_tracker_ready(asocket* s) { if (t->need_initial) { apacket::payload_type data; data.resize(s->get_max_payload()); - data.resize(jdwp_process_list_msg(&data[0], data.size())); + data.resize(process_list_msg(t->kind, &data[0], data.size())); t->need_initial = false; s->peer->enqueue(s->peer, std::move(data)); } @@ -393,8 +460,8 @@ static int jdwp_tracker_enqueue(asocket* s, apacket::payload_type) { return -1; } -asocket* create_jdwp_tracker_service_socket(void) { - auto t = std::make_unique<JdwpTracker>(); +static asocket* create_process_tracker_service_socket(TrackerKind kind) { + auto t = std::make_unique<JdwpTracker>(kind, true); if (!t) { LOG(FATAL) << "failed to allocate JdwpTracker"; } @@ -407,7 +474,6 @@ asocket* create_jdwp_tracker_service_socket(void) { t->ready = jdwp_tracker_ready; t->enqueue = jdwp_tracker_enqueue; t->close = jdwp_tracker_close; - t->need_initial = true; asocket* result = t.get(); @@ -416,23 +482,56 @@ asocket* create_jdwp_tracker_service_socket(void) { return result; } +asocket* create_jdwp_tracker_service_socket() { + return create_process_tracker_service_socket(TrackerKind::kJdwp); +} + +asocket* create_app_tracker_service_socket() { + return create_process_tracker_service_socket(TrackerKind::kApp); +} + int init_jdwp(void) { std::thread([]() { adb_thread_setname("jdwp control"); - adbconnection_listen([](int fd, pid_t pid) { - LOG(INFO) << "jdwp connection from " << pid; - fdevent_run_on_main_thread([fd, pid] { + adbconnection_listen([](int fd, ProcessInfo process) { + LOG(INFO) << "jdwp connection from " << process.pid; + fdevent_run_on_main_thread([fd, process] { unique_fd ufd(fd); - auto proc = std::make_unique<JdwpProcess>(std::move(ufd), pid); + auto proc = std::make_unique<JdwpProcess>(std::move(ufd), process); if (!proc) { LOG(FATAL) << "failed to allocate JdwpProcess"; } _jdwp_list.emplace_back(std::move(proc)); - jdwp_process_list_updated(); + if (process.debuggable) jdwp_process_list_updated(); + if (process.debuggable || process.profileable) app_process_list_updated(); }); }); }).detach(); return 0; } +#else // !defined(__ANDROID_RECOVERY) +#include "adb.h" + +asocket* create_jdwp_service_socket(void) { + return nullptr; +} + +unique_fd create_jdwp_connection_fd(int pid) { + return {}; +} + +asocket* create_app_tracker_service_socket() { + return nullptr; +} + +asocket* create_jdwp_tracker_service_socket() { + return nullptr; +} + +int init_jdwp() { + return 0; +} + +#endif /* defined(__ANDROID_RECOVERY__) */ #endif /* !ADB_HOST */ diff --git a/adb/daemon/main.cpp b/adb/daemon/main.cpp index 658e24456..eb28668b9 100644 --- a/adb/daemon/main.cpp +++ b/adb/daemon/main.cpp @@ -157,12 +157,6 @@ static void drop_privileges(int server_port) { LOG(FATAL) << "Could not set SELinux context"; } } - std::string error; - std::string local_name = - android::base::StringPrintf("tcp:%d", server_port); - if (install_listener(local_name, "*smartsocket*", nullptr, 0, nullptr, &error)) { - LOG(FATAL) << "Could not install *smartsocket* listener: " << error; - } } } #endif @@ -280,6 +274,8 @@ int adbd_main(int server_port) { setup_adb(addrs); } + LOG(INFO) << "adbd started"; + D("adbd_main(): pre init_jdwp()"); init_jdwp(); D("adbd_main(): post init_jdwp()"); diff --git a/adb/daemon/mdns.cpp b/adb/daemon/mdns.cpp index fa692c039..c1e766eba 100644 --- a/adb/daemon/mdns.cpp +++ b/adb/daemon/mdns.cpp @@ -150,11 +150,11 @@ static std::string RandomAlphaNumString(size_t len) { for (size_t i = 0; i < len; ++i) { uint8_t val = dist(mt); if (val < 10) { - ret += '0' + val; + ret += static_cast<char>('0' + val); } else if (val < 36) { - ret += 'A' + (val - 10); + ret += static_cast<char>('A' + (val - 10)); } else { - ret += 'a' + (val - 36); + ret += static_cast<char>('a' + (val - 36)); } } return ret; diff --git a/adb/daemon/services.cpp b/adb/daemon/services.cpp index 6bbf66e8e..a9d1fe844 100644 --- a/adb/daemon/services.cpp +++ b/adb/daemon/services.cpp @@ -241,6 +241,8 @@ asocket* daemon_service_to_socket(std::string_view name) { return create_jdwp_service_socket(); } else if (name == "track-jdwp") { return create_jdwp_tracker_service_socket(); + } else if (name == "track-app") { + return create_app_tracker_service_socket(); } else if (android::base::ConsumePrefix(&name, "sink:")) { uint64_t byte_count = 0; if (!ParseUint(&byte_count, name)) { diff --git a/adb/daemon/shell_service.cpp b/adb/daemon/shell_service.cpp index fbfae1e44..dbca4adb6 100644 --- a/adb/daemon/shell_service.cpp +++ b/adb/daemon/shell_service.cpp @@ -646,15 +646,21 @@ unique_fd* Subprocess::PollLoop(SubprocessPollfds* pfds) { } // After handling all of the events we've received, check to see if any fds have died. - if (stdinout_pfd.revents & (POLLHUP | POLLRDHUP | POLLERR | POLLNVAL)) { + auto poll_finished = [](int events) { + // Don't return failure until we've read out all of the fd's incoming data. + return (events & POLLIN) == 0 && + (events & (POLLHUP | POLLRDHUP | POLLERR | POLLNVAL)) != 0; + }; + + if (poll_finished(stdinout_pfd.revents)) { return &stdinout_sfd_; } - if (stderr_pfd.revents & (POLLHUP | POLLRDHUP | POLLERR | POLLNVAL)) { + if (poll_finished(stderr_pfd.revents)) { return &stderr_sfd_; } - if (protocol_pfd.revents & (POLLHUP | POLLRDHUP | POLLERR | POLLNVAL)) { + if (poll_finished(protocol_pfd.revents)) { return &protocol_sfd_; } } // while (!dead_sfd) diff --git a/adb/daemon/transport_local.cpp b/adb/daemon/transport_local.cpp new file mode 100644 index 000000000..9e0b88737 --- /dev/null +++ b/adb/daemon/transport_local.cpp @@ -0,0 +1,117 @@ +/* + * Copyright (C) 2007 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define TRACE_TAG TRANSPORT + +#include "sysdeps.h" +#include "transport.h" + +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/types.h> + +#include <condition_variable> +#include <functional> +#include <memory> +#include <mutex> +#include <thread> +#include <unordered_map> +#include <vector> + +#include <android-base/parsenetaddress.h> +#include <android-base/stringprintf.h> +#include <android-base/thread_annotations.h> +#include <cutils/sockets.h> + +#if !ADB_HOST +#include <android-base/properties.h> +#endif + +#include "adb.h" +#include "adb_io.h" +#include "adb_unique_fd.h" +#include "adb_utils.h" +#include "socket_spec.h" +#include "sysdeps/chrono.h" + +void server_socket_thread(std::function<unique_fd(std::string_view, std::string*)> listen_func, + std::string_view addr) { + adb_thread_setname("server socket"); + + unique_fd serverfd; + std::string error; + + while (serverfd == -1) { + errno = 0; + serverfd = listen_func(addr, &error); + if (errno == EAFNOSUPPORT || errno == EINVAL || errno == EPROTONOSUPPORT) { + D("unrecoverable error: '%s'", error.c_str()); + return; + } else if (serverfd < 0) { + D("server: cannot bind socket yet: %s", error.c_str()); + std::this_thread::sleep_for(1s); + continue; + } + close_on_exec(serverfd.get()); + } + + while (true) { + D("server: trying to get new connection from fd %d", serverfd.get()); + unique_fd fd(adb_socket_accept(serverfd, nullptr, nullptr)); + if (fd >= 0) { + D("server: new connection on fd %d", fd.get()); + close_on_exec(fd.get()); + disable_tcp_nagle(fd.get()); + std::string serial = android::base::StringPrintf("host-%d", fd.get()); + // We don't care about port value in "register_socket_transport" as it is used + // only from ADB_HOST. "server_socket_thread" is never called from ADB_HOST. + register_socket_transport( + std::move(fd), std::move(serial), 0, 1, + [](atransport*) { return ReconnectResult::Abort; }, false); + } + } + D("transport: server_socket_thread() exiting"); +} + +unique_fd adb_listen(std::string_view addr, std::string* error) { + return unique_fd{socket_spec_listen(addr, error, nullptr)}; +} + +void local_init(const std::string& addr) { +#if !defined(__ANDROID__) + // Host adbd. + D("transport: local server init"); + std::thread(server_socket_thread, adb_listen, addr).detach(); +#else + D("transport: local server init"); + // For the adbd daemon in the system image we need to distinguish + // between the device, and the emulator. + if (addr.starts_with("tcp:") && use_qemu_goldfish()) { + std::thread(qemu_socket_thread, addr).detach(); + } else { + std::thread(server_socket_thread, adb_listen, addr).detach(); + } +#endif // !ADB_HOST +} + +int init_socket_transport(atransport* t, unique_fd fd, int adb_port, int local) { + t->type = kTransportLocal; + auto fd_connection = std::make_unique<FdConnection>(std::move(fd)); + t->SetConnection(std::make_unique<BlockingConnectionAdapter>(std::move(fd_connection))); + return 0; +} diff --git a/adb/fastdeploy/proto/ApkEntry.proto b/adb/fastdeploy/proto/ApkEntry.proto index d84c5a54e..ed5056e68 100644 --- a/adb/fastdeploy/proto/ApkEntry.proto +++ b/adb/fastdeploy/proto/ApkEntry.proto @@ -5,7 +5,6 @@ package com.android.fastdeploy; option java_package = "com.android.fastdeploy"; option java_outer_classname = "ApkEntryProto"; option java_multiple_files = true; -option optimize_for = LITE_RUNTIME; message APKDump { string name = 1; diff --git a/adb/fdevent/fdevent.cpp b/adb/fdevent/fdevent.cpp index fd550200f..70cb9b3a6 100644 --- a/adb/fdevent/fdevent.cpp +++ b/adb/fdevent/fdevent.cpp @@ -27,7 +27,10 @@ #include "adb_utils.h" #include "fdevent.h" #include "fdevent_epoll.h" + +#if !defined(__linux__) #include "fdevent_poll.h" +#endif using namespace std::chrono_literals; using std::chrono::duration_cast; diff --git a/adb/fdevent/fdevent.h b/adb/fdevent/fdevent.h index 9fc3b2c0c..bb3af746f 100644 --- a/adb/fdevent/fdevent.h +++ b/adb/fdevent/fdevent.h @@ -79,8 +79,8 @@ struct fdevent_context { unique_fd Destroy(fdevent* fde); protected: - virtual void Register(fdevent*) {} - virtual void Unregister(fdevent*) {} + virtual void Register(fdevent*) = 0; + virtual void Unregister(fdevent*) = 0; public: // Change which events should cause notifications. diff --git a/adb/fdevent/fdevent_poll.cpp b/adb/fdevent/fdevent_poll.cpp index ac86c08e1..21c1ba093 100644 --- a/adb/fdevent/fdevent_poll.cpp +++ b/adb/fdevent/fdevent_poll.cpp @@ -211,3 +211,7 @@ void fdevent_context_poll::Interrupt() { PLOG(FATAL) << "failed to write to fdevent interrupt fd"; } } + +void fdevent_context_poll::Register(fdevent*) {} + +void fdevent_context_poll::Unregister(fdevent*) {} diff --git a/adb/fdevent/fdevent_poll.h b/adb/fdevent/fdevent_poll.h index 98abab283..8803e3ec9 100644 --- a/adb/fdevent/fdevent_poll.h +++ b/adb/fdevent/fdevent_poll.h @@ -48,6 +48,9 @@ struct fdevent_context_poll final : public fdevent_context { fdevent_context_poll(); virtual ~fdevent_context_poll(); + virtual void Register(fdevent* fde) final; + virtual void Unregister(fdevent* fde) final; + virtual void Set(fdevent* fde, unsigned events) final; virtual void Loop() final; diff --git a/adb/file_sync_protocol.h b/adb/file_sync_protocol.h index fd9a5169e..5234c20ae 100644 --- a/adb/file_sync_protocol.h +++ b/adb/file_sync_protocol.h @@ -92,6 +92,17 @@ struct __attribute__((packed)) sync_dent_v2 { enum SyncFlag : uint32_t { kSyncFlagNone = 0, kSyncFlagBrotli = 1, + kSyncFlagLZ4 = 2, + kSyncFlagZstd = 4, + kSyncFlagDryRun = 0x8000'0000U, +}; + +enum class CompressionType { + None, + Any, + Brotli, + LZ4, + Zstd, }; // send_v1 sent the path in a buffer, followed by a comma and the mode as a string. diff --git a/adb/libs/adbconnection/Android.bp b/adb/libs/adbconnection/Android.bp index ce2ab51ef..f7d2dc171 100644 --- a/adb/libs/adbconnection/Android.bp +++ b/adb/libs/adbconnection/Android.bp @@ -53,6 +53,7 @@ cc_library { linux: { version_script: "libadbconnection_client.map.txt", }, + darwin: { enabled: false }, }, stubs: { symbol_file: "libadbconnection_client.map.txt", diff --git a/adb/libs/adbconnection/adbconnection_client.cpp b/adb/libs/adbconnection/adbconnection_client.cpp index ee48abb03..7e1614835 100644 --- a/adb/libs/adbconnection/adbconnection_client.cpp +++ b/adb/libs/adbconnection/adbconnection_client.cpp @@ -29,6 +29,8 @@ #include <android-base/logging.h> #include <android-base/unique_fd.h> +#include "adbconnection/process_info.h" + using android::base::unique_fd; static constexpr char kJdwpControlName[] = "\0jdwp-control"; @@ -60,6 +62,8 @@ AdbConnectionClientContext* adbconnection_client_new( std::optional<uint64_t> pid; std::optional<bool> debuggable; + std::optional<bool> profileable; + std::optional<std::string> architecture; for (size_t i = 0; i < info_count; ++i) { auto info = info_elems[i]; @@ -77,7 +81,23 @@ AdbConnectionClientContext* adbconnection_client_new( LOG(ERROR) << "multiple debuggable entries in AdbConnectionClientInfo, ignoring"; continue; } - debuggable = info->data.pid; + debuggable = info->data.debuggable; + break; + + case AdbConnectionClientInfoType::profileable: + if (profileable) { + LOG(ERROR) << "multiple profileable entries in AdbConnectionClientInfo, ignoring"; + continue; + } + profileable = info->data.profileable; + break; + + case AdbConnectionClientInfoType::architecture: + if (architecture) { + LOG(ERROR) << "multiple architecture entries in AdbConnectionClientInfo, ignoring"; + continue; + } + architecture = std::string(info->data.architecture.name, info->data.architecture.size); break; } } @@ -92,6 +112,16 @@ AdbConnectionClientContext* adbconnection_client_new( return nullptr; } + if (!profileable) { + LOG(ERROR) << "AdbConnectionClientInfo missing required field profileable"; + return nullptr; + } + + if (!architecture) { + LOG(ERROR) << "AdbConnectionClientInfo missing required field architecture"; + return nullptr; + } + ctx->control_socket_.reset(socket(AF_UNIX, SOCK_SEQPACKET | SOCK_CLOEXEC, 0)); if (ctx->control_socket_ < 0) { PLOG(ERROR) << "failed to create Unix domain socket"; @@ -110,7 +140,13 @@ AdbConnectionClientContext* adbconnection_client_new( int rc = connect(ctx->control_socket_.get(), reinterpret_cast<sockaddr*>(&addr), addr_len); if (rc != 0) { - PLOG(ERROR) << "failed to connect to jdwp control socket"; + if (errno == ECONNREFUSED) { + // On userdebug devices, every Java process is debuggable, so if adbd is explicitly turned + // off, this would spew enormous amounts of red-herring errors. + LOG(DEBUG) << "failed to connect to jdwp control socket, adbd not running?"; + } else { + PLOG(ERROR) << "failed to connect to jdwp control socket"; + } return nullptr; } @@ -120,10 +156,10 @@ AdbConnectionClientContext* adbconnection_client_new( return nullptr; } - uint32_t pid_u32 = static_cast<uint32_t>(*pid); - rc = TEMP_FAILURE_RETRY(write(ctx->control_socket_.get(), &pid_u32, sizeof(pid_u32))); - if (rc != sizeof(pid_u32)) { - PLOG(ERROR) << "failed to send JDWP process pid to adbd"; + ProcessInfo process(*pid, *debuggable, *profileable, *architecture); + rc = TEMP_FAILURE_RETRY(write(ctx->control_socket_.get(), &process, sizeof(process))); + if (rc != sizeof(process)) { + PLOG(ERROR) << "failed to send JDWP process info to adbd"; } return ctx.release(); diff --git a/adb/libs/adbconnection/adbconnection_server.cpp b/adb/libs/adbconnection/adbconnection_server.cpp index 939da2f64..aac9615ed 100644 --- a/adb/libs/adbconnection/adbconnection_server.cpp +++ b/adb/libs/adbconnection/adbconnection_server.cpp @@ -28,6 +28,8 @@ #include <android-base/logging.h> #include <android-base/unique_fd.h> +#include "adbconnection/process_info.h" + using android::base::unique_fd; #define JDWP_CONTROL_NAME "\0jdwp-control" @@ -36,7 +38,7 @@ using android::base::unique_fd; static_assert(JDWP_CONTROL_NAME_LEN <= sizeof(reinterpret_cast<sockaddr_un*>(0)->sun_path)); // Listen for incoming jdwp clients forever. -void adbconnection_listen(void (*callback)(int fd, pid_t pid)) { +void adbconnection_listen(void (*callback)(int fd, ProcessInfo process)) { sockaddr_un addr = {}; socklen_t addrlen = JDWP_CONTROL_NAME_LEN + sizeof(addr.sun_family); @@ -106,16 +108,13 @@ void adbconnection_listen(void (*callback)(int fd, pid_t pid)) { << ") in pending connections"; } - // Massively oversized buffer: we're expecting an int32_t from the other end. - char buf[32]; - int rc = TEMP_FAILURE_RETRY(recv(it->get(), buf, sizeof(buf), MSG_DONTWAIT)); - if (rc != 4) { + ProcessInfo process; + int rc = TEMP_FAILURE_RETRY(recv(it->get(), &process, sizeof(process), MSG_DONTWAIT)); + if (rc != sizeof(process)) { LOG(ERROR) << "received data of incorrect size from JDWP client: read " << rc - << ", expected 4"; + << ", expected " << sizeof(process); } else { - int32_t pid; - memcpy(&pid, buf, sizeof(pid)); - callback(it->release(), static_cast<pid_t>(pid)); + callback(it->release(), process); } if (epoll_ctl(epfd.get(), EPOLL_CTL_DEL, event.data.fd, nullptr) != 0) { diff --git a/adb/libs/adbconnection/include/adbconnection/client.h b/adb/libs/adbconnection/include/adbconnection/client.h index 692fea00d..a74cd36e0 100644 --- a/adb/libs/adbconnection/include/adbconnection/client.h +++ b/adb/libs/adbconnection/include/adbconnection/client.h @@ -28,6 +28,8 @@ struct AdbConnectionClientContext; enum AdbConnectionClientInfoType { pid, debuggable, + profileable, + architecture, }; struct AdbConnectionClientInfo { @@ -35,11 +37,17 @@ struct AdbConnectionClientInfo { union { uint64_t pid; bool debuggable; + bool profileable; + struct { + const char* name; + size_t size; + } architecture; } data; }; // Construct a context and connect to adbd. // Returns null if we fail to connect to adbd. +// Note this is an apex interface as it's loaded by ART. AdbConnectionClientContext* adbconnection_client_new( const AdbConnectionClientInfo* const* info_elems, size_t info_count); diff --git a/adb/libs/adbconnection/include/adbconnection/process_info.h b/adb/libs/adbconnection/include/adbconnection/process_info.h new file mode 100644 index 000000000..d22669978 --- /dev/null +++ b/adb/libs/adbconnection/include/adbconnection/process_info.h @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <stdint.h> +#include <string.h> +#include <string> + +struct ProcessInfo { + static constexpr size_t kMaxArchNameLength = 16; + + uint64_t pid; + bool debuggable; + bool profileable; + int32_t arch_name_length; // length of architecture name in bytes + char arch_name[kMaxArchNameLength]; // ISA name, e.g., "arm64" + + ProcessInfo() : pid(0), debuggable(false), profileable(false), arch_name_length(0) {} + + ProcessInfo(uint64_t pid, bool dbg, bool prof, const std::string& arch) + : pid(pid), debuggable(dbg), profileable(prof) { + arch_name_length = std::min(arch.size(), kMaxArchNameLength); + memcpy(arch_name, arch.data(), arch_name_length); + } +}; diff --git a/adb/libs/adbconnection/include/adbconnection/server.h b/adb/libs/adbconnection/include/adbconnection/server.h index 57ca6cdee..b1059bad4 100644 --- a/adb/libs/adbconnection/include/adbconnection/server.h +++ b/adb/libs/adbconnection/include/adbconnection/server.h @@ -20,7 +20,7 @@ #include <android-base/unique_fd.h> -extern "C" { +#include "adbconnection/process_info.h" -void adbconnection_listen(void (*callback)(int fd, pid_t pid)); -} +// Note this is NOT an apex interface as it's linked only into adbd. +void adbconnection_listen(void (*callback)(int fd, ProcessInfo process)); diff --git a/adb/pairing_connection/Android.bp b/adb/pairing_connection/Android.bp index 707161bcb..9595511b7 100644 --- a/adb/pairing_connection/Android.bp +++ b/adb/pairing_connection/Android.bp @@ -84,7 +84,7 @@ cc_library { static_libs: [ "libadb_protos", // Statically link libadb_tls_connection because it is not - // ABI-stable. + // ABI-stable. "libadb_tls_connection", "libprotobuf-cpp-lite", ], @@ -133,7 +133,6 @@ cc_defaults { "//frameworks/base/services:__subpackages__", ], - host_supported: true, recovery_available: false, stl: "libc++_static", diff --git a/adb/pairing_connection/tests/pairing_connection_test.cpp b/adb/pairing_connection/tests/pairing_connection_test.cpp index b6e09f190..86b66aaee 100644 --- a/adb/pairing_connection/tests/pairing_connection_test.cpp +++ b/adb/pairing_connection/tests/pairing_connection_test.cpp @@ -440,7 +440,7 @@ TEST_F(AdbPairingConnectionTest, MultipleClientsAllFail) { EXPECT_FALSE(*(server_waiter.is_valid_)); } -TEST_F(AdbPairingConnectionTest, MultipleClientsOnePass) { +TEST_F(AdbPairingConnectionTest, DISABLED_MultipleClientsOnePass) { // Send multiple clients with bad passwords, but send the last one with the // correct password. std::vector<uint8_t> pswd{0x01, 0x03, 0x05, 0x07}; diff --git a/adb/proto/Android.bp b/adb/proto/Android.bp index f7cba9522..086d10e16 100644 --- a/adb/proto/Android.bp +++ b/adb/proto/Android.bp @@ -71,3 +71,64 @@ cc_library_static { "//apex_available:platform", ], } + +cc_defaults { + name: "libapp_processes_protos_defaults", + cflags: [ + "-Wall", + "-Wextra", + "-Wthread-safety", + "-Werror", + ], + + compile_multilib: "both", + + srcs: [ + "app_processes.proto", + ], + target: { + windows: { + compile_multilib: "first", + enabled: true, + }, + }, + + visibility: [ + "//system/core/adb:__subpackages__", + + // This needs to be visible to minadbd, even though it's removed via exclude_shared_libs. + "//bootable/recovery/minadbd:__subpackages__", + ], + + stl: "libc++_static", + + apex_available: [ + "com.android.adbd", + "test_com.android.adbd", + ], +} + +cc_library { + name: "libapp_processes_protos_lite", + defaults: ["libapp_processes_protos_defaults"], + + apex_available: ["//apex_available:platform"], + + proto: { + export_proto_headers: true, + type: "lite", + }, + + host_supported: true, + recovery_available: true, +} + +cc_library_host_static { + name: "libapp_processes_protos_full", + defaults: ["libapp_processes_protos_defaults"], + + proto: { + export_proto_headers: true, + type: "full", + }, +} diff --git a/base/process_test.cpp b/adb/proto/app_processes.proto index 056f6679f..118364591 100644 --- a/base/process_test.cpp +++ b/adb/proto/app_processes.proto @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019 The Android Open Source Project + * Copyright (C) 2020 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,22 +14,20 @@ * limitations under the License. */ -#include "android-base/process.h" +syntax = "proto3"; -#include <unistd.h> +option java_package = "com.android.server.adb.protos"; +option java_outer_classname = "AppProcessesProto"; -#include <gtest/gtest.h> +package adb.proto; -TEST(process, find_ourselves) { -#if defined(__linux__) - bool found_our_pid = false; - for (const auto& pid : android::base::AllPids{}) { - if (pid == getpid()) { - found_our_pid = true; - } - } - - EXPECT_TRUE(found_our_pid); +message ProcessEntry { + int64 pid = 1; + bool debuggable = 2; + bool profileable = 3; + string architecture = 4; +} -#endif +message AppProcesses { + repeated ProcessEntry process = 1; } diff --git a/adb/services.cpp b/adb/services.cpp index 853d65897..19a9030f2 100644 --- a/adb/services.cpp +++ b/adb/services.cpp @@ -95,56 +95,6 @@ unique_fd service_to_fd(std::string_view name, atransport* transport) { } #if ADB_HOST -struct state_info { - TransportType transport_type; - std::string serial; - TransportId transport_id; - ConnectionState state; -}; - -static void wait_for_state(unique_fd fd, state_info* sinfo) { - D("wait_for_state %d", sinfo->state); - - while (true) { - bool is_ambiguous = false; - std::string error = "unknown error"; - const char* serial = sinfo->serial.length() ? sinfo->serial.c_str() : nullptr; - atransport* t = acquire_one_transport(sinfo->transport_type, serial, sinfo->transport_id, - &is_ambiguous, &error); - if (sinfo->state == kCsOffline) { - // wait-for-disconnect uses kCsOffline, we don't actually want to wait for 'offline'. - if (t == nullptr) { - SendOkay(fd); - break; - } - } else if (t != nullptr && - (sinfo->state == kCsAny || sinfo->state == t->GetConnectionState())) { - SendOkay(fd); - break; - } - - if (!is_ambiguous) { - adb_pollfd pfd = {.fd = fd.get(), .events = POLLIN}; - int rc = adb_poll(&pfd, 1, 100); - if (rc < 0) { - SendFail(fd, error); - break; - } else if (rc > 0 && (pfd.revents & POLLHUP) != 0) { - // The other end of the socket is closed, probably because the other side was - // terminated, bail out. - break; - } - - // Try again... - } else { - SendFail(fd, error); - break; - } - } - - D("wait_for_state is done"); -} - void connect_emulator(const std::string& port_spec, std::string* response) { std::vector<std::string> pieces = android::base::Split(port_spec, ","); if (pieces.size() != 2) { @@ -201,6 +151,91 @@ static void pair_service(unique_fd fd, std::string host, std::string password) { adb_wifi_pair_device(host, password, response); SendProtocolString(fd.get(), response); } + +static void wait_service(unique_fd fd, std::string serial, TransportId transport_id, + std::string spec) { + std::vector<std::string> components = android::base::Split(spec, "-"); + if (components.size() < 2) { + SendFail(fd, "short wait-for-: " + spec); + return; + } + + TransportType transport_type; + if (components[0] == "local") { + transport_type = kTransportLocal; + } else if (components[0] == "usb") { + transport_type = kTransportUsb; + } else if (components[0] == "any") { + transport_type = kTransportAny; + } else { + SendFail(fd, "bad wait-for- transport: " + spec); + return; + } + + std::vector<ConnectionState> states; + for (size_t i = 1; i < components.size(); ++i) { + if (components[i] == "device") { + states.push_back(kCsDevice); + } else if (components[i] == "recovery") { + states.push_back(kCsRecovery); + } else if (components[i] == "rescue") { + states.push_back(kCsRescue); + } else if (components[i] == "sideload") { + states.push_back(kCsSideload); + } else if (components[i] == "bootloader") { + states.push_back(kCsBootloader); + } else if (components[i] == "any") { + states.push_back(kCsAny); + } else if (components[i] == "disconnect") { + states.push_back(kCsOffline); + } else { + SendFail(fd, "bad wait-for- state: " + spec); + return; + } + } + + while (true) { + bool is_ambiguous = false; + std::string error = "unknown error"; + atransport* t = + acquire_one_transport(transport_type, !serial.empty() ? serial.c_str() : nullptr, + transport_id, &is_ambiguous, &error); + + for (const auto& state : states) { + if (state == kCsOffline) { + // Special case for wait-for-disconnect: + // We want to wait for USB devices to completely disappear, but TCP devices can + // go into the offline state, since we automatically reconnect. + if (!t) { + SendOkay(fd); + return; + } else if (!t->GetUsbHandle()) { + SendOkay(fd); + return; + } + } else { + if (t && (state == kCsAny || state == t->GetConnectionState())) { + SendOkay(fd); + return; + } + } + } + + if (is_ambiguous) { + SendFail(fd, error); + return; + } + + // Sleep before retrying. + adb_pollfd pfd = {.fd = fd.get(), .events = POLLIN}; + if (adb_poll(&pfd, 1, 100) != 0) { + // The other end of the socket is closed, probably because the + // client terminated. Bail out. + SendFail(fd, error); + return; + } + } +} #endif #if ADB_HOST @@ -211,45 +246,10 @@ asocket* host_service_to_socket(std::string_view name, std::string_view serial, } else if (name == "track-devices-l") { return create_device_tracker(true); } else if (android::base::ConsumePrefix(&name, "wait-for-")) { - std::shared_ptr<state_info> sinfo = std::make_shared<state_info>(); - if (sinfo == nullptr) { - fprintf(stderr, "couldn't allocate state_info: %s", strerror(errno)); - return nullptr; - } - - sinfo->serial = serial; - sinfo->transport_id = transport_id; - - if (android::base::ConsumePrefix(&name, "local")) { - sinfo->transport_type = kTransportLocal; - } else if (android::base::ConsumePrefix(&name, "usb")) { - sinfo->transport_type = kTransportUsb; - } else if (android::base::ConsumePrefix(&name, "any")) { - sinfo->transport_type = kTransportAny; - } else { - return nullptr; - } - - if (name == "-device") { - sinfo->state = kCsDevice; - } else if (name == "-recovery") { - sinfo->state = kCsRecovery; - } else if (name == "-rescue") { - sinfo->state = kCsRescue; - } else if (name == "-sideload") { - sinfo->state = kCsSideload; - } else if (name == "-bootloader") { - sinfo->state = kCsBootloader; - } else if (name == "-any") { - sinfo->state = kCsAny; - } else if (name == "-disconnect") { - sinfo->state = kCsOffline; - } else { - return nullptr; - } - - unique_fd fd = create_service_thread( - "wait", [sinfo](unique_fd fd) { wait_for_state(std::move(fd), sinfo.get()); }); + std::string spec(name); + unique_fd fd = + create_service_thread("wait", std::bind(wait_service, std::placeholders::_1, + std::string(serial), transport_id, spec)); return create_local_socket(std::move(fd)); } else if (android::base::ConsumePrefix(&name, "connect:")) { std::string host(name); diff --git a/adb/socket.h b/adb/socket.h index 4276851d2..062320489 100644 --- a/adb/socket.h +++ b/adb/socket.h @@ -108,7 +108,10 @@ asocket* create_local_service_socket(std::string_view destination, atransport* t asocket *create_remote_socket(unsigned id, atransport *t); void connect_to_remote(asocket* s, std::string_view destination); + +#if ADB_HOST void connect_to_smartsocket(asocket *s); +#endif // Internal functions that are only made available here for testing purposes. namespace internal { diff --git a/adb/socket_spec.cpp b/adb/socket_spec.cpp index d17036cec..5cad70d09 100644 --- a/adb/socket_spec.cpp +++ b/adb/socket_spec.cpp @@ -30,6 +30,7 @@ #include "adb.h" #include "adb_utils.h" +#include "adb_wifi.h" #include "sysdeps.h" using namespace std::string_literals; @@ -103,12 +104,6 @@ bool parse_tcp_socket_spec(std::string_view spec, std::string* hostname, int* po if (!android::base::ParseNetAddress(addr, &hostname_value, &port_value, serial, error)) { return false; } - - if (port_value == -1) { - *error = "missing port in specification: "; - *error += spec; - return false; - } } if (hostname) { @@ -201,7 +196,24 @@ bool socket_spec_connect(unique_fd* fd, std::string_view address, int* port, std fd->reset(network_loopback_client(port_value, SOCK_STREAM, error)); } else { #if ADB_HOST - fd->reset(network_connect(hostname, port_value, SOCK_STREAM, 0, error)); + // Check if the address is an mdns service we can connect to. + if (auto mdns_info = mdns_get_connect_service_info(address.substr(4)); + mdns_info != std::nullopt) { + fd->reset(network_connect(mdns_info->addr, mdns_info->port, SOCK_STREAM, 0, error)); + if (fd->get() != -1) { + // TODO(joshuaduong): We still show the ip address for the serial. Change it to + // use the mdns instance name, so we can adjust to address changes on + // reconnects. + port_value = mdns_info->port; + if (serial) { + *serial = android::base::StringPrintf("%s.%s", + mdns_info->service_name.c_str(), + mdns_info->service_type.c_str()); + } + } + } else { + fd->reset(network_connect(hostname, port_value, SOCK_STREAM, 0, error)); + } #else // Disallow arbitrary connections in adbd. *error = "adbd does not support arbitrary tcp connections"; diff --git a/adb/socket_spec_test.cpp b/adb/socket_spec_test.cpp index e9d5270ca..e83c34c45 100644 --- a/adb/socket_spec_test.cpp +++ b/adb/socket_spec_test.cpp @@ -24,6 +24,13 @@ #include <android-base/stringprintf.h> #include <gtest/gtest.h> +TEST(socket_spec, parse_tcp_socket_spec_failure) { + std::string hostname, error, serial; + int port; + EXPECT_FALSE(parse_tcp_socket_spec("sneakernet:5037", &hostname, &port, &serial, &error)); + EXPECT_TRUE(error.find("sneakernet") != std::string::npos); +} + TEST(socket_spec, parse_tcp_socket_spec_just_port) { std::string hostname, error, serial; int port; @@ -134,6 +141,19 @@ TEST(socket_spec, socket_spec_listen_connect_tcp) { EXPECT_NE(client_fd.get(), -1); } +TEST(socket_spec, socket_spec_connect_failure) { + std::string error, serial; + int port; + unique_fd client_fd; + EXPECT_FALSE(socket_spec_connect(&client_fd, "tcp:", &port, &serial, &error)); + EXPECT_FALSE(socket_spec_connect(&client_fd, "acceptfd:", &port, &serial, &error)); + EXPECT_FALSE(socket_spec_connect(&client_fd, "vsock:", &port, &serial, &error)); + EXPECT_FALSE(socket_spec_connect(&client_fd, "vsock:x", &port, &serial, &error)); + EXPECT_FALSE(socket_spec_connect(&client_fd, "vsock:5", &port, &serial, &error)); + EXPECT_FALSE(socket_spec_connect(&client_fd, "vsock:5:x", &port, &serial, &error)); + EXPECT_FALSE(socket_spec_connect(&client_fd, "sneakernet:", &port, &serial, &error)); +} + TEST(socket_spec, socket_spec_listen_connect_localfilesystem) { std::string error, serial; int port; @@ -152,3 +172,16 @@ TEST(socket_spec, socket_spec_listen_connect_localfilesystem) { EXPECT_NE(client_fd.get(), -1); } } + +TEST(socket_spec, is_socket_spec) { + EXPECT_TRUE(is_socket_spec("tcp:blah")); + EXPECT_TRUE(is_socket_spec("acceptfd:blah")); + EXPECT_TRUE(is_socket_spec("local:blah")); + EXPECT_TRUE(is_socket_spec("localreserved:blah")); +} + +TEST(socket_spec, is_local_socket_spec) { + EXPECT_TRUE(is_local_socket_spec("local:blah")); + EXPECT_TRUE(is_local_socket_spec("tcp:localhost")); + EXPECT_FALSE(is_local_socket_spec("tcp:www.google.com")); +} diff --git a/adb/sockets.cpp b/adb/sockets.cpp index 423af67f1..13a473776 100644 --- a/adb/sockets.cpp +++ b/adb/sockets.cpp @@ -520,6 +520,7 @@ void connect_to_remote(asocket* s, std::string_view destination) { send_packet(p, s->transport); } +#if ADB_HOST /* this is used by magic sockets to rig local sockets to send the go-ahead message when they connect */ static void local_socket_ready_notify(asocket* s) { @@ -584,8 +585,6 @@ static unsigned unhex(const char* s, int len) { return n; } -#if ADB_HOST - namespace internal { // Parses a host service string of the following format: @@ -714,15 +713,11 @@ bool parse_host_service(std::string_view* out_serial, std::string_view* out_comm } // namespace internal -#endif // ADB_HOST - static int smart_socket_enqueue(asocket* s, apacket::payload_type data) { -#if ADB_HOST std::string_view service; std::string_view serial; TransportId transport_id = 0; TransportType type = kTransportAny; -#endif D("SS(%d): enqueue %zu", s->id, data.size()); @@ -755,7 +750,6 @@ static int smart_socket_enqueue(asocket* s, apacket::payload_type data) { D("SS(%d): '%s'", s->id, (char*)(s->smart_socket_data.data() + 4)); -#if ADB_HOST service = std::string_view(s->smart_socket_data).substr(4); // TODO: These should be handled in handle_host_request. @@ -841,16 +835,6 @@ static int smart_socket_enqueue(asocket* s, apacket::payload_type data) { s2->ready(s2); return 0; } -#else /* !ADB_HOST */ - if (s->transport == nullptr) { - std::string error_msg = "unknown failure"; - s->transport = acquire_one_transport(kTransportAny, nullptr, 0, nullptr, &error_msg); - if (s->transport == nullptr) { - SendFail(s->peer->fd, error_msg); - goto fail; - } - } -#endif if (!s->transport) { SendFail(s->peer->fd, "device offline (no transport)"); @@ -922,6 +906,7 @@ void connect_to_smartsocket(asocket* s) { ss->peer = s; s->ready(s); } +#endif size_t asocket::get_max_payload() const { size_t max_payload = MAX_PAYLOAD; diff --git a/adb/sysdeps.h b/adb/sysdeps.h index 1eed0d20a..7326ab137 100644 --- a/adb/sysdeps.h +++ b/adb/sysdeps.h @@ -671,6 +671,10 @@ static inline int adb_get_os_handle(borrowed_fd fd) { return fd.get(); } +static inline int cast_handle_to_int(int fd) { + return fd; +} + // A very simple wrapper over a launched child process class Process { public: diff --git a/adb/sysdeps/errno.cpp b/adb/sysdeps/errno.cpp index 9a37ea2fd..e6af68b48 100644 --- a/adb/sysdeps/errno.cpp +++ b/adb/sysdeps/errno.cpp @@ -24,7 +24,7 @@ #include "adb.h" -// Use the linux asm-generic values for errno (which are used on all android archs but mips). +// Use the linux asm-generic values for errno (which are used on all android architectures). #define ERRNO_VALUES() \ ERRNO_VALUE(EACCES, 13); \ ERRNO_VALUE(EEXIST, 17); \ @@ -48,7 +48,7 @@ ERRNO_VALUE(ETXTBSY, 26) // Make sure these values are actually correct. -#if defined(__linux__) && !defined(__mips__) +#if defined(__linux__) #define ERRNO_VALUE(error_name, wire_value) static_assert((error_name) == (wire_value), "") ERRNO_VALUES(); #undef ERRNO_VALUE diff --git a/adb/test_adb.py b/adb/test_adb.py index c872fb0f7..b9f0d5487 100755 --- a/adb/test_adb.py +++ b/adb/test_adb.py @@ -25,6 +25,7 @@ import os import random import select import socket +import string import struct import subprocess import sys @@ -32,6 +33,7 @@ import threading import time import unittest import warnings +from importlib import util def find_open_port(): # Find an open port. @@ -576,6 +578,169 @@ class PowerTest(unittest.TestCase): # If the power event was detected, the adb shell command should be broken very quickly. self.assertLess(end - start, 2) +"""Use 'adb mdns check' to see if mdns discovery is available.""" +def is_adb_mdns_available(): + with adb_server() as server_port: + output = subprocess.check_output(["adb", "-P", str(server_port), + "mdns", "check"]).strip() + return output.startswith(b"mdns daemon version") + +"""Check if we have zeroconf python library installed""" +def is_zeroconf_installed(): + zeroconf_spec = util.find_spec("zeroconf") + return zeroconf_spec is not None + +@contextlib.contextmanager +def zeroconf_context(ipversion): + from zeroconf import Zeroconf + """Context manager for a zeroconf instance + + This creates a zeroconf instance and returns it. + """ + + try: + zeroconf = Zeroconf(ip_version=ipversion) + yield zeroconf + finally: + zeroconf.close() + +@contextlib.contextmanager +def zeroconf_register_service(zeroconf_ctx, info): + """Context manager for a zeroconf service + + Registers a service and unregisters it on cleanup. Returns the ServiceInfo + supplied. + """ + + try: + zeroconf_ctx.register_service(info) + yield info + finally: + zeroconf_ctx.unregister_service(info) + +"""Should match the service names listed in adb_mdns.h""" +class MdnsTest: + """Tests for adb mdns.""" + + class Base(unittest.TestCase): + @staticmethod + def _mdns_services(port): + output = subprocess.check_output(["adb", "-P", str(port), "mdns", "services"]) + return [x.split("\t") for x in output.decode("utf8").strip().splitlines()[1:]] + + @staticmethod + def _devices(port): + output = subprocess.check_output(["adb", "-P", str(port), "devices"]) + return [x.split("\t") for x in output.decode("utf8").strip().splitlines()[1:]] + + @contextlib.contextmanager + def _adb_mdns_connect(self, server_port, mdns_instance, serial, should_connect): + """Context manager for an ADB connection. + + This automatically disconnects when done with the connection. + """ + + output = subprocess.check_output(["adb", "-P", str(server_port), "connect", mdns_instance]) + if should_connect: + self.assertEqual(output.strip(), "connected to {}".format(serial).encode("utf8")) + else: + self.assertTrue(output.startswith("failed to resolve host: '{}'" + .format(mdns_instance).encode("utf8"))) + + try: + yield + finally: + # Perform best-effort disconnection. Discard the output. + subprocess.Popen(["adb", "disconnect", serial], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + + @unittest.skipIf(not is_zeroconf_installed(), "zeroconf library not installed") + def test_mdns_services_register_unregister(self): + """Ensure that `adb mdns services` correctly adds and removes a service + """ + from zeroconf import IPVersion, ServiceInfo + + with adb_server() as server_port: + output = subprocess.check_output(["adb", "-P", str(server_port), + "mdns", "services"]).strip() + self.assertTrue(output.startswith(b"List of discovered mdns services")) + + """TODO(joshuaduong): Add ipv6 tests once we have it working in adb""" + """Register/Unregister a service""" + with zeroconf_context(IPVersion.V4Only) as zc: + serv_instance = "my_fake_test_service" + serv_type = "_" + self.service_name + "._tcp." + serv_ipaddr = socket.inet_aton("1.2.3.4") + serv_port = 12345 + service_info = ServiceInfo( + serv_type + "local.", + name=serv_instance + "." + serv_type + "local.", + addresses=[serv_ipaddr], + port=serv_port) + with zeroconf_register_service(zc, service_info) as info: + """Give adb some time to register the service""" + time.sleep(1) + self.assertTrue(any((serv_instance in line and serv_type in line) + for line in MdnsTest._mdns_services(server_port))) + + """Give adb some time to unregister the service""" + time.sleep(1) + self.assertFalse(any((serv_instance in line and serv_type in line) + for line in MdnsTest._mdns_services(server_port))) + + @unittest.skipIf(not is_zeroconf_installed(), "zeroconf library not installed") + def test_mdns_connect(self): + """Ensure that `adb connect` by mdns instance name works (for non-pairing services) + """ + from zeroconf import IPVersion, ServiceInfo + + with adb_server() as server_port: + with zeroconf_context(IPVersion.V4Only) as zc: + serv_instance = "fakeadbd-" + ''.join( + random.choice(string.ascii_letters) for i in range(4)) + serv_type = "_" + self.service_name + "._tcp." + serv_ipaddr = socket.inet_aton("127.0.0.1") + should_connect = self.service_name != "adb-tls-pairing" + with fake_adbd() as (port, _): + service_info = ServiceInfo( + serv_type + "local.", + name=serv_instance + "." + serv_type + "local.", + addresses=[serv_ipaddr], + port=port) + with zeroconf_register_service(zc, service_info) as info: + """Give adb some time to register the service""" + time.sleep(1) + self.assertTrue(any((serv_instance in line and serv_type in line) + for line in MdnsTest._mdns_services(server_port))) + full_name = '.'.join([serv_instance, serv_type]) + with self._adb_mdns_connect(server_port, serv_instance, full_name, + should_connect): + if should_connect: + self.assertEqual(MdnsTest._devices(server_port), + [[full_name, "device"]]) + + """Give adb some time to unregister the service""" + time.sleep(1) + self.assertFalse(any((serv_instance in line and serv_type in line) + for line in MdnsTest._mdns_services(server_port))) + + +@unittest.skipIf(not is_adb_mdns_available(), "mdns feature not available") +class MdnsTestAdb(MdnsTest.Base): + service_name = "adb" + + +@unittest.skipIf(not is_adb_mdns_available(), "mdns feature not available") +class MdnsTestAdbTlsConnect(MdnsTest.Base): + service_name = "adb-tls-connect" + + +@unittest.skipIf(not is_adb_mdns_available(), "mdns feature not available") +class MdnsTestAdbTlsPairing(MdnsTest.Base): + service_name = "adb-tls-pairing" + def main(): """Main entrypoint.""" diff --git a/adb/test_device.py b/adb/test_device.py index 6a9ff89ef..c1caafcf6 100755 --- a/adb/test_device.py +++ b/adb/test_device.py @@ -77,16 +77,18 @@ def requires_non_root(func): class DeviceTest(unittest.TestCase): - def setUp(self): - self.device = adb.get_device() + device = adb.get_device() class AbbTest(DeviceTest): def test_smoke(self): - result = subprocess.run(['adb', 'abb'], capture_output=True) - self.assertEqual(1, result.returncode) - expected_output = b"cmd: No service specified; use -l to list all services\n" - self.assertEqual(expected_output, result.stderr) + abb = subprocess.run(['adb', 'abb'], capture_output=True) + cmd = subprocess.run(['adb', 'shell', 'cmd'], capture_output=True) + + # abb squashes all failures to 1. + self.assertEqual(abb.returncode == 0, cmd.returncode == 0) + self.assertEqual(abb.stdout, cmd.stdout) + self.assertEqual(abb.stderr, cmd.stderr) class ForwardReverseTest(DeviceTest): def _test_no_rebind(self, description, direction_list, direction, @@ -753,535 +755,615 @@ def make_random_device_files(device, in_dir, num_files, prefix='device_tmpfile') return files -class FileOperationsTest(DeviceTest): - SCRATCH_DIR = '/data/local/tmp' - DEVICE_TEMP_FILE = SCRATCH_DIR + '/adb_test_file' - DEVICE_TEMP_DIR = SCRATCH_DIR + '/adb_test_dir' +class FileOperationsTest: + class Base(DeviceTest): + SCRATCH_DIR = '/data/local/tmp' + DEVICE_TEMP_FILE = SCRATCH_DIR + '/adb_test_file' + DEVICE_TEMP_DIR = SCRATCH_DIR + '/adb_test_dir' - def _verify_remote(self, checksum, remote_path): - dev_md5, _ = self.device.shell([get_md5_prog(self.device), - remote_path])[0].split() - self.assertEqual(checksum, dev_md5) + def setUp(self): + self.previous_env = os.environ.get("ADB_COMPRESSION") + os.environ["ADB_COMPRESSION"] = self.compression - def _verify_local(self, checksum, local_path): - with open(local_path, 'rb') as host_file: - host_md5 = compute_md5(host_file.read()) - self.assertEqual(host_md5, checksum) + def tearDown(self): + if self.previous_env is None: + del os.environ["ADB_COMPRESSION"] + else: + os.environ["ADB_COMPRESSION"] = self.previous_env - def test_push(self): - """Push a randomly generated file to specified device.""" - kbytes = 512 - tmp = tempfile.NamedTemporaryFile(mode='wb', delete=False) - rand_str = os.urandom(1024 * kbytes) - tmp.write(rand_str) - tmp.close() + def _verify_remote(self, checksum, remote_path): + dev_md5, _ = self.device.shell([get_md5_prog(self.device), + remote_path])[0].split() + self.assertEqual(checksum, dev_md5) - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_FILE]) - self.device.push(local=tmp.name, remote=self.DEVICE_TEMP_FILE) + def _verify_local(self, checksum, local_path): + with open(local_path, 'rb') as host_file: + host_md5 = compute_md5(host_file.read()) + self.assertEqual(host_md5, checksum) - self._verify_remote(compute_md5(rand_str), self.DEVICE_TEMP_FILE) - self.device.shell(['rm', '-f', self.DEVICE_TEMP_FILE]) + def test_push(self): + """Push a randomly generated file to specified device.""" + kbytes = 512 + tmp = tempfile.NamedTemporaryFile(mode='wb', delete=False) + rand_str = os.urandom(1024 * kbytes) + tmp.write(rand_str) + tmp.close() - os.remove(tmp.name) + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_FILE]) + self.device.push(local=tmp.name, remote=self.DEVICE_TEMP_FILE) - def test_push_dir(self): - """Push a randomly generated directory of files to the device.""" - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - self.device.shell(['mkdir', self.DEVICE_TEMP_DIR]) + self._verify_remote(compute_md5(rand_str), self.DEVICE_TEMP_FILE) + self.device.shell(['rm', '-f', self.DEVICE_TEMP_FILE]) - try: - host_dir = tempfile.mkdtemp() + os.remove(tmp.name) - # Make sure the temp directory isn't setuid, or else adb will complain. - os.chmod(host_dir, 0o700) + def test_push_dir(self): + """Push a randomly generated directory of files to the device.""" + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + self.device.shell(['mkdir', self.DEVICE_TEMP_DIR]) - # Create 32 random files. - temp_files = make_random_host_files(in_dir=host_dir, num_files=32) - self.device.push(host_dir, self.DEVICE_TEMP_DIR) + try: + host_dir = tempfile.mkdtemp() - for temp_file in temp_files: - remote_path = posixpath.join(self.DEVICE_TEMP_DIR, - os.path.basename(host_dir), - temp_file.base_name) - self._verify_remote(temp_file.checksum, remote_path) + # Make sure the temp directory isn't setuid, or else adb will complain. + os.chmod(host_dir, 0o700) + + # Create 32 random files. + temp_files = make_random_host_files(in_dir=host_dir, num_files=32) + self.device.push(host_dir, self.DEVICE_TEMP_DIR) + + for temp_file in temp_files: + remote_path = posixpath.join(self.DEVICE_TEMP_DIR, + os.path.basename(host_dir), + temp_file.base_name) + self._verify_remote(temp_file.checksum, remote_path) + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if host_dir is not None: + shutil.rmtree(host_dir) + + def disabled_test_push_empty(self): + """Push an empty directory to the device.""" self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - finally: - if host_dir is not None: - shutil.rmtree(host_dir) + self.device.shell(['mkdir', self.DEVICE_TEMP_DIR]) - def disabled_test_push_empty(self): - """Push an empty directory to the device.""" - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - self.device.shell(['mkdir', self.DEVICE_TEMP_DIR]) + try: + host_dir = tempfile.mkdtemp() - try: - host_dir = tempfile.mkdtemp() + # Make sure the temp directory isn't setuid, or else adb will complain. + os.chmod(host_dir, 0o700) - # Make sure the temp directory isn't setuid, or else adb will complain. - os.chmod(host_dir, 0o700) + # Create an empty directory. + empty_dir_path = os.path.join(host_dir, 'empty') + os.mkdir(empty_dir_path); - # Create an empty directory. - empty_dir_path = os.path.join(host_dir, 'empty') - os.mkdir(empty_dir_path); + self.device.push(empty_dir_path, self.DEVICE_TEMP_DIR) - self.device.push(empty_dir_path, self.DEVICE_TEMP_DIR) + remote_path = os.path.join(self.DEVICE_TEMP_DIR, "empty") + test_empty_cmd = ["[", "-d", remote_path, "]"] + rc, _, _ = self.device.shell_nocheck(test_empty_cmd) - remote_path = os.path.join(self.DEVICE_TEMP_DIR, "empty") - test_empty_cmd = ["[", "-d", remote_path, "]"] - rc, _, _ = self.device.shell_nocheck(test_empty_cmd) + self.assertEqual(rc, 0) + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if host_dir is not None: + shutil.rmtree(host_dir) - self.assertEqual(rc, 0) - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - finally: - if host_dir is not None: - shutil.rmtree(host_dir) + @unittest.skipIf(sys.platform == "win32", "symlinks require elevated privileges on windows") + def test_push_symlink(self): + """Push a symlink. + + Bug: http://b/31491920 + """ + try: + host_dir = tempfile.mkdtemp() - @unittest.skipIf(sys.platform == "win32", "symlinks require elevated privileges on windows") - def test_push_symlink(self): - """Push a symlink. + # Make sure the temp directory isn't setuid, or else adb will + # complain. + os.chmod(host_dir, 0o700) - Bug: http://b/31491920 - """ - try: - host_dir = tempfile.mkdtemp() + with open(os.path.join(host_dir, 'foo'), 'w') as f: + f.write('foo') + + symlink_path = os.path.join(host_dir, 'symlink') + os.symlink('foo', symlink_path) - # Make sure the temp directory isn't setuid, or else adb will - # complain. - os.chmod(host_dir, 0o700) + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + self.device.shell(['mkdir', self.DEVICE_TEMP_DIR]) + self.device.push(symlink_path, self.DEVICE_TEMP_DIR) + rc, out, _ = self.device.shell_nocheck( + ['cat', posixpath.join(self.DEVICE_TEMP_DIR, 'symlink')]) + self.assertEqual(0, rc) + self.assertEqual(out.strip(), 'foo') + finally: + if host_dir is not None: + shutil.rmtree(host_dir) - with open(os.path.join(host_dir, 'foo'), 'w') as f: - f.write('foo') + def test_multiple_push(self): + """Push multiple files to the device in one adb push command. - symlink_path = os.path.join(host_dir, 'symlink') - os.symlink('foo', symlink_path) + Bug: http://b/25324823 + """ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) self.device.shell(['mkdir', self.DEVICE_TEMP_DIR]) - self.device.push(symlink_path, self.DEVICE_TEMP_DIR) - rc, out, _ = self.device.shell_nocheck( - ['cat', posixpath.join(self.DEVICE_TEMP_DIR, 'symlink')]) - self.assertEqual(0, rc) - self.assertEqual(out.strip(), 'foo') - finally: - if host_dir is not None: - shutil.rmtree(host_dir) - def test_multiple_push(self): - """Push multiple files to the device in one adb push command. + try: + host_dir = tempfile.mkdtemp() - Bug: http://b/25324823 - """ + # Create some random files and a subdirectory containing more files. + temp_files = make_random_host_files(in_dir=host_dir, num_files=4) - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - self.device.shell(['mkdir', self.DEVICE_TEMP_DIR]) + subdir = os.path.join(host_dir, 'subdir') + os.mkdir(subdir) + subdir_temp_files = make_random_host_files(in_dir=subdir, + num_files=4) - try: - host_dir = tempfile.mkdtemp() + paths = [x.full_path for x in temp_files] + paths.append(subdir) + self.device._simple_call(['push'] + paths + [self.DEVICE_TEMP_DIR]) - # Create some random files and a subdirectory containing more files. - temp_files = make_random_host_files(in_dir=host_dir, num_files=4) + for temp_file in temp_files: + remote_path = posixpath.join(self.DEVICE_TEMP_DIR, + temp_file.base_name) + self._verify_remote(temp_file.checksum, remote_path) - subdir = os.path.join(host_dir, 'subdir') - os.mkdir(subdir) - subdir_temp_files = make_random_host_files(in_dir=subdir, - num_files=4) + for subdir_temp_file in subdir_temp_files: + remote_path = posixpath.join(self.DEVICE_TEMP_DIR, + # BROKEN: http://b/25394682 + # 'subdir'; + temp_file.base_name) + self._verify_remote(temp_file.checksum, remote_path) - paths = [x.full_path for x in temp_files] - paths.append(subdir) - self.device._simple_call(['push'] + paths + [self.DEVICE_TEMP_DIR]) - for temp_file in temp_files: - remote_path = posixpath.join(self.DEVICE_TEMP_DIR, - temp_file.base_name) - self._verify_remote(temp_file.checksum, remote_path) + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if host_dir is not None: + shutil.rmtree(host_dir) - for subdir_temp_file in subdir_temp_files: - remote_path = posixpath.join(self.DEVICE_TEMP_DIR, - # BROKEN: http://b/25394682 - # 'subdir'; - temp_file.base_name) - self._verify_remote(temp_file.checksum, remote_path) + @requires_non_root + def test_push_error_reporting(self): + """Make sure that errors that occur while pushing a file get reported + Bug: http://b/26816782 + """ + with tempfile.NamedTemporaryFile() as tmp_file: + tmp_file.write(b'\0' * 1024 * 1024) + tmp_file.flush() + try: + self.device.push(local=tmp_file.name, remote='/system/') + self.fail('push should not have succeeded') + except subprocess.CalledProcessError as e: + output = e.output + + self.assertTrue(b'Permission denied' in output or + b'Read-only file system' in output) + + @requires_non_root + def test_push_directory_creation(self): + """Regression test for directory creation. + + Bug: http://b/110953234 + """ + with tempfile.NamedTemporaryFile() as tmp_file: + tmp_file.write(b'\0' * 1024 * 1024) + tmp_file.flush() + remote_path = self.DEVICE_TEMP_DIR + '/test_push_directory_creation' + self.device.shell(['rm', '-rf', remote_path]) - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - finally: - if host_dir is not None: - shutil.rmtree(host_dir) + remote_path += '/filename' + self.device.push(local=tmp_file.name, remote=remote_path) - @requires_non_root - def test_push_error_reporting(self): - """Make sure that errors that occur while pushing a file get reported + def disabled_test_push_multiple_slash_root(self): + """Regression test for pushing to //data/local/tmp. + + Bug: http://b/141311284 + + Disabled because this broken on the adbd side as well: b/141943968 + """ + with tempfile.NamedTemporaryFile() as tmp_file: + tmp_file.write('\0' * 1024 * 1024) + tmp_file.flush() + remote_path = '/' + self.DEVICE_TEMP_DIR + '/test_push_multiple_slash_root' + self.device.shell(['rm', '-rf', remote_path]) + self.device.push(local=tmp_file.name, remote=remote_path) + + def _test_pull(self, remote_file, checksum): + tmp_write = tempfile.NamedTemporaryFile(mode='wb', delete=False) + tmp_write.close() + self.device.pull(remote=remote_file, local=tmp_write.name) + with open(tmp_write.name, 'rb') as tmp_read: + host_contents = tmp_read.read() + host_md5 = compute_md5(host_contents) + self.assertEqual(checksum, host_md5) + os.remove(tmp_write.name) + + @requires_non_root + def test_pull_error_reporting(self): + self.device.shell(['touch', self.DEVICE_TEMP_FILE]) + self.device.shell(['chmod', 'a-rwx', self.DEVICE_TEMP_FILE]) - Bug: http://b/26816782 - """ - with tempfile.NamedTemporaryFile() as tmp_file: - tmp_file.write(b'\0' * 1024 * 1024) - tmp_file.flush() try: - self.device.push(local=tmp_file.name, remote='/system/') - self.fail('push should not have succeeded') + output = self.device.pull(remote=self.DEVICE_TEMP_FILE, local='x') except subprocess.CalledProcessError as e: output = e.output - self.assertTrue(b'Permission denied' in output or - b'Read-only file system' in output) + self.assertIn(b'Permission denied', output) + + self.device.shell(['rm', '-f', self.DEVICE_TEMP_FILE]) + + def test_pull(self): + """Pull a randomly generated file from specified device.""" + kbytes = 512 + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_FILE]) + cmd = ['dd', 'if=/dev/urandom', + 'of={}'.format(self.DEVICE_TEMP_FILE), 'bs=1024', + 'count={}'.format(kbytes)] + self.device.shell(cmd) + dev_md5, _ = self.device.shell( + [get_md5_prog(self.device), self.DEVICE_TEMP_FILE])[0].split() + self._test_pull(self.DEVICE_TEMP_FILE, dev_md5) + self.device.shell_nocheck(['rm', self.DEVICE_TEMP_FILE]) + + def test_pull_dir(self): + """Pull a randomly generated directory of files from the device.""" + try: + host_dir = tempfile.mkdtemp() - @requires_non_root - def test_push_directory_creation(self): - """Regression test for directory creation. + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR]) - Bug: http://b/110953234 - """ - with tempfile.NamedTemporaryFile() as tmp_file: - tmp_file.write(b'\0' * 1024 * 1024) - tmp_file.flush() - remote_path = self.DEVICE_TEMP_DIR + '/test_push_directory_creation' - self.device.shell(['rm', '-rf', remote_path]) + # Populate device directory with random files. + temp_files = make_random_device_files( + self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32) - remote_path += '/filename' - self.device.push(local=tmp_file.name, remote=remote_path) + self.device.pull(remote=self.DEVICE_TEMP_DIR, local=host_dir) - def disabled_test_push_multiple_slash_root(self): - """Regression test for pushing to //data/local/tmp. + for temp_file in temp_files: + host_path = os.path.join( + host_dir, posixpath.basename(self.DEVICE_TEMP_DIR), + temp_file.base_name) + self._verify_local(temp_file.checksum, host_path) - Bug: http://b/141311284 + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if host_dir is not None: + shutil.rmtree(host_dir) - Disabled because this broken on the adbd side as well: b/141943968 - """ - with tempfile.NamedTemporaryFile() as tmp_file: - tmp_file.write('\0' * 1024 * 1024) - tmp_file.flush() - remote_path = '/' + self.DEVICE_TEMP_DIR + '/test_push_multiple_slash_root' - self.device.shell(['rm', '-rf', remote_path]) - self.device.push(local=tmp_file.name, remote=remote_path) - - def _test_pull(self, remote_file, checksum): - tmp_write = tempfile.NamedTemporaryFile(mode='wb', delete=False) - tmp_write.close() - self.device.pull(remote=remote_file, local=tmp_write.name) - with open(tmp_write.name, 'rb') as tmp_read: - host_contents = tmp_read.read() - host_md5 = compute_md5(host_contents) - self.assertEqual(checksum, host_md5) - os.remove(tmp_write.name) - - @requires_non_root - def test_pull_error_reporting(self): - self.device.shell(['touch', self.DEVICE_TEMP_FILE]) - self.device.shell(['chmod', 'a-rwx', self.DEVICE_TEMP_FILE]) + def test_pull_dir_symlink(self): + """Pull a directory into a symlink to a directory. - try: - output = self.device.pull(remote=self.DEVICE_TEMP_FILE, local='x') - except subprocess.CalledProcessError as e: - output = e.output - - self.assertIn(b'Permission denied', output) - - self.device.shell(['rm', '-f', self.DEVICE_TEMP_FILE]) - - def test_pull(self): - """Pull a randomly generated file from specified device.""" - kbytes = 512 - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_FILE]) - cmd = ['dd', 'if=/dev/urandom', - 'of={}'.format(self.DEVICE_TEMP_FILE), 'bs=1024', - 'count={}'.format(kbytes)] - self.device.shell(cmd) - dev_md5, _ = self.device.shell( - [get_md5_prog(self.device), self.DEVICE_TEMP_FILE])[0].split() - self._test_pull(self.DEVICE_TEMP_FILE, dev_md5) - self.device.shell_nocheck(['rm', self.DEVICE_TEMP_FILE]) - - def test_pull_dir(self): - """Pull a randomly generated directory of files from the device.""" - try: - host_dir = tempfile.mkdtemp() + Bug: http://b/27362811 + """ + if os.name != 'posix': + raise unittest.SkipTest('requires POSIX') - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR]) + try: + host_dir = tempfile.mkdtemp() + real_dir = os.path.join(host_dir, 'dir') + symlink = os.path.join(host_dir, 'symlink') + os.mkdir(real_dir) + os.symlink(real_dir, symlink) - # Populate device directory with random files. - temp_files = make_random_device_files( - self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32) + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR]) - self.device.pull(remote=self.DEVICE_TEMP_DIR, local=host_dir) + # Populate device directory with random files. + temp_files = make_random_device_files( + self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32) - for temp_file in temp_files: - host_path = os.path.join( - host_dir, posixpath.basename(self.DEVICE_TEMP_DIR), - temp_file.base_name) - self._verify_local(temp_file.checksum, host_path) + self.device.pull(remote=self.DEVICE_TEMP_DIR, local=symlink) - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - finally: - if host_dir is not None: - shutil.rmtree(host_dir) + for temp_file in temp_files: + host_path = os.path.join( + real_dir, posixpath.basename(self.DEVICE_TEMP_DIR), + temp_file.base_name) + self._verify_local(temp_file.checksum, host_path) - def test_pull_dir_symlink(self): - """Pull a directory into a symlink to a directory. + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if host_dir is not None: + shutil.rmtree(host_dir) - Bug: http://b/27362811 - """ - if os.name != 'posix': - raise unittest.SkipTest('requires POSIX') + def test_pull_dir_symlink_collision(self): + """Pull a directory into a colliding symlink to directory.""" + if os.name != 'posix': + raise unittest.SkipTest('requires POSIX') - try: - host_dir = tempfile.mkdtemp() - real_dir = os.path.join(host_dir, 'dir') - symlink = os.path.join(host_dir, 'symlink') - os.mkdir(real_dir) - os.symlink(real_dir, symlink) + try: + host_dir = tempfile.mkdtemp() + real_dir = os.path.join(host_dir, 'real') + tmp_dirname = os.path.basename(self.DEVICE_TEMP_DIR) + symlink = os.path.join(host_dir, tmp_dirname) + os.mkdir(real_dir) + os.symlink(real_dir, symlink) - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR]) + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR]) - # Populate device directory with random files. - temp_files = make_random_device_files( - self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32) + # Populate device directory with random files. + temp_files = make_random_device_files( + self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32) - self.device.pull(remote=self.DEVICE_TEMP_DIR, local=symlink) + self.device.pull(remote=self.DEVICE_TEMP_DIR, local=host_dir) - for temp_file in temp_files: - host_path = os.path.join( - real_dir, posixpath.basename(self.DEVICE_TEMP_DIR), - temp_file.base_name) - self._verify_local(temp_file.checksum, host_path) + for temp_file in temp_files: + host_path = os.path.join(real_dir, temp_file.base_name) + self._verify_local(temp_file.checksum, host_path) - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - finally: - if host_dir is not None: - shutil.rmtree(host_dir) + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if host_dir is not None: + shutil.rmtree(host_dir) - def test_pull_dir_symlink_collision(self): - """Pull a directory into a colliding symlink to directory.""" - if os.name != 'posix': - raise unittest.SkipTest('requires POSIX') + def test_pull_dir_nonexistent(self): + """Pull a directory of files from the device to a nonexistent path.""" + try: + host_dir = tempfile.mkdtemp() + dest_dir = os.path.join(host_dir, 'dest') - try: - host_dir = tempfile.mkdtemp() - real_dir = os.path.join(host_dir, 'real') - tmp_dirname = os.path.basename(self.DEVICE_TEMP_DIR) - symlink = os.path.join(host_dir, tmp_dirname) - os.mkdir(real_dir) - os.symlink(real_dir, symlink) + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR]) - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR]) + # Populate device directory with random files. + temp_files = make_random_device_files( + self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32) - # Populate device directory with random files. - temp_files = make_random_device_files( - self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32) + self.device.pull(remote=self.DEVICE_TEMP_DIR, local=dest_dir) - self.device.pull(remote=self.DEVICE_TEMP_DIR, local=host_dir) + for temp_file in temp_files: + host_path = os.path.join(dest_dir, temp_file.base_name) + self._verify_local(temp_file.checksum, host_path) - for temp_file in temp_files: - host_path = os.path.join(real_dir, temp_file.base_name) - self._verify_local(temp_file.checksum, host_path) + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if host_dir is not None: + shutil.rmtree(host_dir) - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - finally: - if host_dir is not None: - shutil.rmtree(host_dir) + # selinux prevents adbd from accessing symlinks on /data/local/tmp. + def disabled_test_pull_symlink_dir(self): + """Pull a symlink to a directory of symlinks to files.""" + try: + host_dir = tempfile.mkdtemp() - def test_pull_dir_nonexistent(self): - """Pull a directory of files from the device to a nonexistent path.""" - try: - host_dir = tempfile.mkdtemp() - dest_dir = os.path.join(host_dir, 'dest') + remote_dir = posixpath.join(self.DEVICE_TEMP_DIR, 'contents') + remote_links = posixpath.join(self.DEVICE_TEMP_DIR, 'links') + remote_symlink = posixpath.join(self.DEVICE_TEMP_DIR, 'symlink') - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR]) + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + self.device.shell(['mkdir', '-p', remote_dir, remote_links]) + self.device.shell(['ln', '-s', remote_links, remote_symlink]) - # Populate device directory with random files. - temp_files = make_random_device_files( - self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32) + # Populate device directory with random files. + temp_files = make_random_device_files( + self.device, in_dir=remote_dir, num_files=32) - self.device.pull(remote=self.DEVICE_TEMP_DIR, local=dest_dir) + for temp_file in temp_files: + self.device.shell( + ['ln', '-s', '../contents/{}'.format(temp_file.base_name), + posixpath.join(remote_links, temp_file.base_name)]) - for temp_file in temp_files: - host_path = os.path.join(dest_dir, temp_file.base_name) - self._verify_local(temp_file.checksum, host_path) + self.device.pull(remote=remote_symlink, local=host_dir) - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - finally: - if host_dir is not None: - shutil.rmtree(host_dir) + for temp_file in temp_files: + host_path = os.path.join( + host_dir, 'symlink', temp_file.base_name) + self._verify_local(temp_file.checksum, host_path) - # selinux prevents adbd from accessing symlinks on /data/local/tmp. - def disabled_test_pull_symlink_dir(self): - """Pull a symlink to a directory of symlinks to files.""" - try: - host_dir = tempfile.mkdtemp() + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if host_dir is not None: + shutil.rmtree(host_dir) + + def test_pull_empty(self): + """Pull a directory containing an empty directory from the device.""" + try: + host_dir = tempfile.mkdtemp() - remote_dir = posixpath.join(self.DEVICE_TEMP_DIR, 'contents') - remote_links = posixpath.join(self.DEVICE_TEMP_DIR, 'links') - remote_symlink = posixpath.join(self.DEVICE_TEMP_DIR, 'symlink') + remote_empty_path = posixpath.join(self.DEVICE_TEMP_DIR, 'empty') + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + self.device.shell(['mkdir', '-p', remote_empty_path]) - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - self.device.shell(['mkdir', '-p', remote_dir, remote_links]) - self.device.shell(['ln', '-s', remote_links, remote_symlink]) + self.device.pull(remote=remote_empty_path, local=host_dir) + self.assertTrue(os.path.isdir(os.path.join(host_dir, 'empty'))) + finally: + if host_dir is not None: + shutil.rmtree(host_dir) - # Populate device directory with random files. - temp_files = make_random_device_files( - self.device, in_dir=remote_dir, num_files=32) + def test_multiple_pull(self): + """Pull a randomly generated directory of files from the device.""" - for temp_file in temp_files: - self.device.shell( - ['ln', '-s', '../contents/{}'.format(temp_file.base_name), - posixpath.join(remote_links, temp_file.base_name)]) + try: + host_dir = tempfile.mkdtemp() + + subdir = posixpath.join(self.DEVICE_TEMP_DIR, 'subdir') + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + self.device.shell(['mkdir', '-p', subdir]) + + # Create some random files and a subdirectory containing more files. + temp_files = make_random_device_files( + self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=4) + + subdir_temp_files = make_random_device_files( + self.device, in_dir=subdir, num_files=4, prefix='subdir_') + + paths = [x.full_path for x in temp_files] + paths.append(subdir) + self.device._simple_call(['pull'] + paths + [host_dir]) + + for temp_file in temp_files: + local_path = os.path.join(host_dir, temp_file.base_name) + self._verify_local(temp_file.checksum, local_path) - self.device.pull(remote=remote_symlink, local=host_dir) + for subdir_temp_file in subdir_temp_files: + local_path = os.path.join(host_dir, + 'subdir', + subdir_temp_file.base_name) + self._verify_local(subdir_temp_file.checksum, local_path) + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if host_dir is not None: + shutil.rmtree(host_dir) + + def verify_sync(self, device, temp_files, device_dir): + """Verifies that a list of temp files was synced to the device.""" + # Confirm that every file on the device mirrors that on the host. for temp_file in temp_files: - host_path = os.path.join( - host_dir, 'symlink', temp_file.base_name) - self._verify_local(temp_file.checksum, host_path) + device_full_path = posixpath.join( + device_dir, temp_file.base_name) + dev_md5, _ = device.shell( + [get_md5_prog(self.device), device_full_path])[0].split() + self.assertEqual(temp_file.checksum, dev_md5) - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - finally: - if host_dir is not None: - shutil.rmtree(host_dir) + def test_sync(self): + """Sync a host directory to the data partition.""" - def test_pull_empty(self): - """Pull a directory containing an empty directory from the device.""" - try: - host_dir = tempfile.mkdtemp() + try: + base_dir = tempfile.mkdtemp() - remote_empty_path = posixpath.join(self.DEVICE_TEMP_DIR, 'empty') - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - self.device.shell(['mkdir', '-p', remote_empty_path]) + # Create mirror device directory hierarchy within base_dir. + full_dir_path = base_dir + self.DEVICE_TEMP_DIR + os.makedirs(full_dir_path) - self.device.pull(remote=remote_empty_path, local=host_dir) - self.assertTrue(os.path.isdir(os.path.join(host_dir, 'empty'))) - finally: - if host_dir is not None: - shutil.rmtree(host_dir) + # Create 32 random files within the host mirror. + temp_files = make_random_host_files( + in_dir=full_dir_path, num_files=32) - def test_multiple_pull(self): - """Pull a randomly generated directory of files from the device.""" + # Clean up any stale files on the device. + device = adb.get_device() # pylint: disable=no-member + device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - try: - host_dir = tempfile.mkdtemp() + old_product_out = os.environ.get('ANDROID_PRODUCT_OUT') + os.environ['ANDROID_PRODUCT_OUT'] = base_dir + device.sync('data') + if old_product_out is None: + del os.environ['ANDROID_PRODUCT_OUT'] + else: + os.environ['ANDROID_PRODUCT_OUT'] = old_product_out - subdir = posixpath.join(self.DEVICE_TEMP_DIR, 'subdir') - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - self.device.shell(['mkdir', '-p', subdir]) + self.verify_sync(device, temp_files, self.DEVICE_TEMP_DIR) - # Create some random files and a subdirectory containing more files. - temp_files = make_random_device_files( - self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=4) + #self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if base_dir is not None: + shutil.rmtree(base_dir) - subdir_temp_files = make_random_device_files( - self.device, in_dir=subdir, num_files=4, prefix='subdir_') + def test_push_sync(self): + """Sync a host directory to a specific path.""" - paths = [x.full_path for x in temp_files] - paths.append(subdir) - self.device._simple_call(['pull'] + paths + [host_dir]) + try: + temp_dir = tempfile.mkdtemp() + temp_files = make_random_host_files(in_dir=temp_dir, num_files=32) - for temp_file in temp_files: - local_path = os.path.join(host_dir, temp_file.base_name) - self._verify_local(temp_file.checksum, local_path) + device_dir = posixpath.join(self.DEVICE_TEMP_DIR, 'sync_src_dst') - for subdir_temp_file in subdir_temp_files: - local_path = os.path.join(host_dir, - 'subdir', - subdir_temp_file.base_name) - self._verify_local(subdir_temp_file.checksum, local_path) + # Clean up any stale files on the device. + device = adb.get_device() # pylint: disable=no-member + device.shell(['rm', '-rf', device_dir]) - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - finally: - if host_dir is not None: - shutil.rmtree(host_dir) - - def verify_sync(self, device, temp_files, device_dir): - """Verifies that a list of temp files was synced to the device.""" - # Confirm that every file on the device mirrors that on the host. - for temp_file in temp_files: - device_full_path = posixpath.join( - device_dir, temp_file.base_name) - dev_md5, _ = device.shell( - [get_md5_prog(self.device), device_full_path])[0].split() - self.assertEqual(temp_file.checksum, dev_md5) - - def test_sync(self): - """Sync a host directory to the data partition.""" + device.push(temp_dir, device_dir, sync=True) - try: - base_dir = tempfile.mkdtemp() + self.verify_sync(device, temp_files, device_dir) - # Create mirror device directory hierarchy within base_dir. - full_dir_path = base_dir + self.DEVICE_TEMP_DIR - os.makedirs(full_dir_path) + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if temp_dir is not None: + shutil.rmtree(temp_dir) - # Create 32 random files within the host mirror. - temp_files = make_random_host_files( - in_dir=full_dir_path, num_files=32) + def test_push_dry_run_nonexistent_file(self): + """Push with dry run.""" - # Clean up any stale files on the device. - device = adb.get_device() # pylint: disable=no-member - device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + for file_size in [8, 1024 * 1024]: + try: + device_dir = posixpath.join(self.DEVICE_TEMP_DIR, 'push_dry_run') + device_file = posixpath.join(device_dir, 'file') - old_product_out = os.environ.get('ANDROID_PRODUCT_OUT') - os.environ['ANDROID_PRODUCT_OUT'] = base_dir - device.sync('data') - if old_product_out is None: - del os.environ['ANDROID_PRODUCT_OUT'] - else: - os.environ['ANDROID_PRODUCT_OUT'] = old_product_out + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + self.device.shell(['mkdir', '-p', device_dir]) - self.verify_sync(device, temp_files, self.DEVICE_TEMP_DIR) + host_dir = tempfile.mkdtemp() + host_file = posixpath.join(host_dir, 'file') - #self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - finally: - if base_dir is not None: - shutil.rmtree(base_dir) + with open(host_file, "w") as f: + f.write('x' * file_size) - def test_push_sync(self): - """Sync a host directory to a specific path.""" + self.device._simple_call(['push', '-n', host_file, device_file]) + rc, _, _ = self.device.shell_nocheck(['[', '-e', device_file, ']']) + self.assertNotEqual(0, rc) - try: - temp_dir = tempfile.mkdtemp() - temp_files = make_random_host_files(in_dir=temp_dir, num_files=32) + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if host_dir is not None: + shutil.rmtree(host_dir) - device_dir = posixpath.join(self.DEVICE_TEMP_DIR, 'sync_src_dst') + def test_push_dry_run_existent_file(self): + """Push with dry run.""" - # Clean up any stale files on the device. - device = adb.get_device() # pylint: disable=no-member - device.shell(['rm', '-rf', device_dir]) + for file_size in [8, 1024 * 1024]: + try: + device_dir = posixpath.join(self.DEVICE_TEMP_DIR, 'push_dry_run') + device_file = posixpath.join(device_dir, 'file') - device.push(temp_dir, device_dir, sync=True) + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + self.device.shell(['mkdir', '-p', device_dir]) + self.device.shell(['echo', 'foo', '>', device_file]) - self.verify_sync(device, temp_files, device_dir) + host_dir = tempfile.mkdtemp() + host_file = posixpath.join(host_dir, 'file') - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - finally: - if temp_dir is not None: - shutil.rmtree(temp_dir) - - def test_unicode_paths(self): - """Ensure that we can support non-ASCII paths, even on Windows.""" - name = u'로보카 폴리' - - self.device.shell(['rm', '-f', '/data/local/tmp/adb-test-*']) - remote_path = u'/data/local/tmp/adb-test-{}'.format(name) - - ## push. - tf = tempfile.NamedTemporaryFile('wb', suffix=name, delete=False) - tf.close() - self.device.push(tf.name, remote_path) - os.remove(tf.name) - self.assertFalse(os.path.exists(tf.name)) - - # Verify that the device ended up with the expected UTF-8 path - output = self.device.shell( - ['ls', '/data/local/tmp/adb-test-*'])[0].strip() - self.assertEqual(remote_path, output) - - # pull. - self.device.pull(remote_path, tf.name) - self.assertTrue(os.path.exists(tf.name)) - os.remove(tf.name) - self.device.shell(['rm', '-f', '/data/local/tmp/adb-test-*']) + with open(host_file, "w") as f: + f.write('x' * file_size) + + self.device._simple_call(['push', '-n', host_file, device_file]) + stdout, stderr = self.device.shell(['cat', device_file]) + self.assertEqual(stdout.strip(), "foo") + + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if host_dir is not None: + shutil.rmtree(host_dir) + + def test_unicode_paths(self): + """Ensure that we can support non-ASCII paths, even on Windows.""" + name = u'로보카 폴리' + + self.device.shell(['rm', '-f', '/data/local/tmp/adb-test-*']) + remote_path = u'/data/local/tmp/adb-test-{}'.format(name) + + ## push. + tf = tempfile.NamedTemporaryFile('wb', suffix=name, delete=False) + tf.close() + self.device.push(tf.name, remote_path) + os.remove(tf.name) + self.assertFalse(os.path.exists(tf.name)) + + # Verify that the device ended up with the expected UTF-8 path + output = self.device.shell( + ['ls', '/data/local/tmp/adb-test-*'])[0].strip() + self.assertEqual(remote_path, output) + + # pull. + self.device.pull(remote_path, tf.name) + self.assertTrue(os.path.exists(tf.name)) + os.remove(tf.name) + self.device.shell(['rm', '-f', '/data/local/tmp/adb-test-*']) + + +class FileOperationsTestUncompressed(FileOperationsTest.Base): + compression = "none" + + +class FileOperationsTestBrotli(FileOperationsTest.Base): + compression = "brotli" + + +class FileOperationsTestLZ4(FileOperationsTest.Base): + compression = "lz4" + + +class FileOperationsTestZstd(FileOperationsTest.Base): + compression = "zstd" class DeviceOfflineTest(DeviceTest): diff --git a/adb/tools/Android.bp b/adb/tools/Android.bp index 71e32b78f..a7af53cee 100644 --- a/adb/tools/Android.bp +++ b/adb/tools/Android.bp @@ -34,3 +34,26 @@ cc_binary_host { ], }, } + +cc_binary_host { + name: "adb_usbreset", + + defaults: ["adb_defaults"], + + srcs: [ + "adb_usbreset.cpp", + ], + + static_libs: [ + "libbase", + "libusb", + ], + + stl: "libc++_static", + + dist: { + targets: [ + "sdk", + ], + }, +} diff --git a/adb/tools/adb_usbreset.cpp b/adb/tools/adb_usbreset.cpp new file mode 100644 index 000000000..6f141bd92 --- /dev/null +++ b/adb/tools/adb_usbreset.cpp @@ -0,0 +1,188 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <err.h> +#include <getopt.h> +#include <stdarg.h> +#include <stdlib.h> +#include <unistd.h> + +#include <string> +#include <string_view> +#include <variant> +#include <vector> + +#include <libusb/libusb.h> + +struct AllDevices {}; +struct SingleDevice {}; +struct Serial { + std::string_view serial; +}; + +using DeviceSelection = std::variant<std::monostate, AllDevices, SingleDevice, Serial>; + +[[noreturn]] static void Usage(int rc) { + fprintf(stderr, "usage: [ANDROID_SERIAL=SERIAL] usbreset [-d] [-s SERIAL]\n"); + fprintf(stderr, "\t-a --all\t\tReset all connected devices\n"); + fprintf(stderr, "\t-d --device\t\tReset the single connected device\n"); + fprintf(stderr, "\t-s --serial\t\tReset device with specified serial\n"); + exit(rc); +} + +static void SetOption(DeviceSelection* out, DeviceSelection in) { + if (!std::get_if<std::monostate>(out)) { + printf("error: multiple device selection options provided\n"); + Usage(1); + } + + *out = in; +} + +static __attribute__((format(printf, 2, 3))) void PrintLibusbError(int err, const char* fmt, ...) { + va_list args; + va_start(args, fmt); + vprintf(fmt, args); + vprintf(fmt, args); + va_end(args); + + printf(": %s", libusb_strerror(static_cast<libusb_error>(err))); +} + +static bool IsAdbInterface(const libusb_interface_descriptor* desc) { + return desc->bInterfaceClass == 0xFF && desc->bInterfaceSubClass == 0x42 && + desc->bInterfaceProtocol == 0x1; +} + +int main(int argc, char** argv) { + std::variant<std::monostate, AllDevices, SingleDevice, Serial> selection; + + static constexpr struct option long_opts[] = { + {"all", 0, 0, 'a'}, {"help", 0, 0, 'h'}, {"serial", required_argument, 0, 's'}, + {"device", 0, 0, 'd'}, {0, 0, 0, 0}, + }; + + int opt; + while ((opt = getopt_long(argc, argv, "adhs:", long_opts, nullptr)) != -1) { + if (opt == 'h') { + Usage(0); + } else if (opt == 'a') { + SetOption(&selection, AllDevices{}); + } else if (opt == 's') { + SetOption(&selection, Serial{optarg}); + } else if (opt == 'd') { + SetOption(&selection, Serial{optarg}); + } else { + errx(1, "unknown option: '%c'", opt); + } + } + + if (std::get_if<std::monostate>(&selection)) { + const char* env = getenv("ANDROID_SERIAL"); + if (env) { + SetOption(&selection, Serial{env}); + } else { + fprintf(stderr, "adb_usbreset: no device specified\n"); + Usage(1); + } + } + + libusb_context* ctx; + int rc = libusb_init(&ctx); + if (rc != LIBUSB_SUCCESS) { + PrintLibusbError(rc, "error: failed to initialize libusb"); + exit(1); + } + + libusb_device** device_list; + ssize_t device_count = libusb_get_device_list(ctx, &device_list); + if (device_count < 0) { + PrintLibusbError(device_count, "error: failed to list devices"); + exit(1); + } + + std::vector<std::pair<std::string, libusb_device_handle*>> selected_devices; + for (int i = 0; i < device_count; ++i) { + libusb_device* device = device_list[i]; + libusb_device_descriptor device_desc; + + // Always succeeds for LIBUSB_API_VERSION >= 0x01000102. + libusb_get_device_descriptor(device, &device_desc); + static_assert(LIBUSB_API_VERSION >= 0x01000102); + + libusb_config_descriptor* config_desc; + rc = libusb_get_active_config_descriptor(device, &config_desc); + if (rc != 0) { + PrintLibusbError(rc, "warning: failed to get config descriptor"); + continue; + } + + bool found_adb_interface = false; + for (int i = 0; i < config_desc->bNumInterfaces; ++i) { + if (IsAdbInterface(&config_desc->interface[i].altsetting[0])) { + found_adb_interface = true; + break; + } + } + + if (found_adb_interface) { + libusb_device_handle* device_handle; + rc = libusb_open(device, &device_handle); + if (rc != 0) { + PrintLibusbError(rc, "warning: failed to open device"); + continue; + } + + char buf[128]; + rc = libusb_get_string_descriptor_ascii(device_handle, device_desc.iSerialNumber, + reinterpret_cast<unsigned char*>(buf), + sizeof(buf)); + + if (rc < 0) { + PrintLibusbError(rc, "warning: failed to get device serial"); + continue; + } + + std::string serial(buf, buf + rc); + if (auto s = std::get_if<Serial>(&selection)) { + if (s->serial == serial) { + selected_devices.push_back(std::make_pair(std::move(serial), device_handle)); + } + } else { + selected_devices.push_back(std::make_pair(std::move(serial), device_handle)); + } + } + } + + if (selected_devices.empty()) { + errx(1, "no devices match criteria"); + } else if (std::get_if<SingleDevice>(&selection) && selected_devices.size() != 1) { + errx(1, "more than 1 device connected"); + } + + bool success = true; + for (auto& [serial, device_handle] : selected_devices) { + rc = libusb_reset_device(device_handle); + // libusb_reset_device will try to restore the previous state, and will return + // LIBUSB_ERROR_NOT_FOUND if it can't. + if (rc == 0 || rc == LIBUSB_ERROR_NOT_FOUND) { + printf("%s: successfully reset\n", serial.c_str()); + } else { + PrintLibusbError(rc, "%s: failed to reset", serial.c_str()); + success = false; + } + } + + return !success; +} diff --git a/adb/transport.cpp b/adb/transport.cpp index fe286dee4..c33d5afcb 100644 --- a/adb/transport.cpp +++ b/adb/transport.cpp @@ -29,7 +29,6 @@ #include <unistd.h> #include <algorithm> -#include <deque> #include <list> #include <memory> #include <mutex> @@ -40,6 +39,7 @@ #include <adb/crypto/x509_generator.h> #include <adb/tls/tls_connection.h> #include <android-base/logging.h> +#include <android-base/no_destructor.h> #include <android-base/parsenetaddress.h> #include <android-base/stringprintf.h> #include <android-base/strings.h> @@ -81,8 +81,12 @@ const char* const kFeatureAbb = "abb"; const char* const kFeatureFixedPushSymlinkTimestamp = "fixed_push_symlink_timestamp"; const char* const kFeatureAbbExec = "abb_exec"; const char* const kFeatureRemountShell = "remount_shell"; +const char* const kFeatureTrackApp = "track_app"; const char* const kFeatureSendRecv2 = "sendrecv_v2"; const char* const kFeatureSendRecv2Brotli = "sendrecv_v2_brotli"; +const char* const kFeatureSendRecv2LZ4 = "sendrecv_v2_lz4"; +const char* const kFeatureSendRecv2Zstd = "sendrecv_v2_zstd"; +const char* const kFeatureSendRecv2DryRunSend = "sendrecv_v2_dry_run_send"; namespace { @@ -499,12 +503,8 @@ bool FdConnection::DoTlsHandshake(RSA* key, std::string* auth_key) { auto x509 = GenerateX509Certificate(evp_pkey.get()); auto x509_str = X509ToPEMString(x509.get()); auto evp_str = Key::ToPEMString(evp_pkey.get()); -#ifdef _WIN32 - int osh = cast_handle_to_int(adb_get_os_handle(fd_)); -#else - int osh = adb_get_os_handle(fd_); -#endif + int osh = cast_handle_to_int(adb_get_os_handle(fd_)); #if ADB_HOST tls_ = TlsConnection::Create(TlsConnection::Role::Client, x509_str, evp_str, osh); #else @@ -929,6 +929,7 @@ static void transport_destroy(atransport* t) { remove_transport(t); } +#if ADB_HOST static int qual_match(const std::string& to_test, const char* prefix, const std::string& qual, bool sanitize_qual) { if (to_test.empty()) /* Return true if both the qual and to_test are empty strings. */ @@ -1084,10 +1085,13 @@ void ConnectionWaitable::SetConnectionEstablished(bool success) { } cv_.notify_one(); } +#endif atransport::~atransport() { +#if ADB_HOST // If the connection callback had not been run before, run it now. SetConnectionEstablished(false); +#endif } int atransport::Write(apacket* p) { @@ -1170,25 +1174,30 @@ size_t atransport::get_max_payload() const { } const FeatureSet& supported_features() { - // Local static allocation to avoid global non-POD variables. - static const FeatureSet* features = new FeatureSet{ - kFeatureShell2, - kFeatureCmd, - kFeatureStat2, - kFeatureLs2, - kFeatureFixedPushMkdir, - kFeatureApex, - kFeatureAbb, - kFeatureFixedPushSymlinkTimestamp, - kFeatureAbbExec, - kFeatureRemountShell, - kFeatureSendRecv2, - kFeatureSendRecv2Brotli, - // Increment ADB_SERVER_VERSION when adding a feature that adbd needs - // to know about. Otherwise, the client can be stuck running an old - // version of the server even after upgrading their copy of adb. - // (http://b/24370690) - }; + static const android::base::NoDestructor<FeatureSet> features([] { + return FeatureSet{ + kFeatureShell2, + kFeatureCmd, + kFeatureStat2, + kFeatureLs2, + kFeatureFixedPushMkdir, + kFeatureApex, + kFeatureAbb, + kFeatureFixedPushSymlinkTimestamp, + kFeatureAbbExec, + kFeatureRemountShell, + kFeatureTrackApp, + kFeatureSendRecv2, + kFeatureSendRecv2Brotli, + kFeatureSendRecv2LZ4, + kFeatureSendRecv2Zstd, + kFeatureSendRecv2DryRunSend, + // Increment ADB_SERVER_VERSION when adding a feature that adbd needs + // to know about. Otherwise, the client can be stuck running an old + // version of the server even after upgrading their copy of adb. + // (http://b/24370690) + }; + }()); return *features; } @@ -1202,16 +1211,20 @@ FeatureSet StringToFeatureSet(const std::string& features_string) { return FeatureSet(); } - auto names = android::base::Split(features_string, ","); - return FeatureSet(names.begin(), names.end()); + return android::base::Split(features_string, ","); +} + +template <class Range, class Value> +static bool contains(const Range& r, const Value& v) { + return std::find(std::begin(r), std::end(r), v) != std::end(r); } bool CanUseFeature(const FeatureSet& feature_set, const std::string& feature) { - return feature_set.count(feature) > 0 && supported_features().count(feature) > 0; + return contains(feature_set, feature) && contains(supported_features(), feature); } bool atransport::has_feature(const std::string& feature) const { - return features_.count(feature) > 0; + return contains(features_, feature); } void atransport::SetFeatures(const std::string& features_string) { @@ -1233,6 +1246,7 @@ void atransport::RunDisconnects() { disconnects_.clear(); } +#if ADB_HOST bool atransport::MatchesTarget(const std::string& target) const { if (!serial.empty()) { if (target == serial) { @@ -1276,8 +1290,6 @@ ReconnectResult atransport::Reconnect() { return reconnect_(this); } -#if ADB_HOST - // We use newline as our delimiter, make sure to never output it. static std::string sanitize(std::string str, bool alphanumeric) { auto pred = alphanumeric ? [](const char c) { return !isalnum(c); } @@ -1359,7 +1371,7 @@ void close_usb_devices(std::function<bool(const atransport*)> predicate, bool re void close_usb_devices(bool reset) { close_usb_devices([](const atransport*) { return true; }, reset); } -#endif // ADB_HOST +#endif bool register_socket_transport(unique_fd s, std::string serial, int port, int local, atransport::ReconnectCallback reconnect, bool use_tls, int* error) { @@ -1399,7 +1411,9 @@ bool register_socket_transport(unique_fd s, std::string serial, int port, int lo lock.unlock(); +#if ADB_HOST auto waitable = t->connection_waitable(); +#endif register_transport(t); if (local == 1) { @@ -1407,6 +1421,7 @@ bool register_socket_transport(unique_fd s, std::string serial, int port, int lo return true; } +#if ADB_HOST if (!waitable->WaitForConnection(std::chrono::seconds(10))) { if (error) *error = ETIMEDOUT; return false; @@ -1416,6 +1431,7 @@ bool register_socket_transport(unique_fd s, std::string serial, int port, int lo if (error) *error = EPERM; return false; } +#endif return true; } @@ -1446,14 +1462,9 @@ void kick_all_tcp_devices() { t->Kick(); } } -#if ADB_HOST reconnect_handler.CheckForKicked(); -#endif } -#endif - -#if ADB_HOST void register_usb_transport(usb_handle* usb, const char* serial, const char* devpath, unsigned writeable) { atransport* t = new atransport(writeable ? kCsOffline : kCsNoPerm); @@ -1475,9 +1486,7 @@ void register_usb_transport(usb_handle* usb, const char* serial, const char* dev register_transport(t); } -#endif -#if ADB_HOST // This should only be used for transports with connection_state == kCsNoPerm. void unregister_usb_transport(usb_handle* usb) { std::lock_guard<std::recursive_mutex> lock(transport_lock); @@ -1524,8 +1533,7 @@ std::shared_ptr<RSA> atransport::NextKey() { keys_.pop_front(); } - std::shared_ptr<RSA> result = keys_[0]; - return result; + return Key(); } void atransport::ResetKeys() { diff --git a/adb/transport.h b/adb/transport.h index 5bc1b5c49..b1f27445c 100644 --- a/adb/transport.h +++ b/adb/transport.h @@ -30,7 +30,7 @@ #include <string> #include <string_view> #include <thread> -#include <unordered_set> +#include <vector> #include <android-base/macros.h> #include <android-base/thread_annotations.h> @@ -40,7 +40,10 @@ #include "adb_unique_fd.h" #include "types.h" -typedef std::unordered_set<std::string> FeatureSet; +// Even though the feature set is used as a set, we only have a dozen or two +// of available features at any moment. Vector works much better in terms of +// both memory usage and performance for these sizes. +using FeatureSet = std::vector<std::string>; namespace adb { namespace tls { @@ -82,10 +85,18 @@ extern const char* const kFeatureAbbExec; extern const char* const kFeatureFixedPushSymlinkTimestamp; // Implement `adb remount` via shelling out to /system/bin/remount. extern const char* const kFeatureRemountShell; +// adbd supports `track-app` service reporting debuggable/profileable apps. +extern const char* const kFeatureTrackApp; // adbd supports version 2 of send/recv. extern const char* const kFeatureSendRecv2; // adbd supports brotli for send/recv v2. extern const char* const kFeatureSendRecv2Brotli; +// adbd supports LZ4 for send/recv v2. +extern const char* const kFeatureSendRecv2LZ4; +// adbd supports Zstd for send/recv v2. +extern const char* const kFeatureSendRecv2Zstd; +// adbd supports dry-run send for send/recv v2. +extern const char* const kFeatureSendRecv2DryRunSend; TransportId NextTransportId(); @@ -253,9 +264,12 @@ class atransport : public enable_weak_from_this<atransport> { : id(NextTransportId()), kicked_(false), connection_state_(state), - connection_waitable_(std::make_shared<ConnectionWaitable>()), connection_(nullptr), reconnect_(std::move(reconnect)) { +#if ADB_HOST + connection_waitable_ = std::make_shared<ConnectionWaitable>(); +#endif + // Initialize protocol to min version for compatibility with older versions. // Version will be updated post-connect. protocol_version = A_VERSION_MIN; @@ -341,6 +355,7 @@ class atransport : public enable_weak_from_this<atransport> { void RemoveDisconnect(adisconnect* disconnect); void RunDisconnects(); +#if ADB_HOST // Returns true if |target| matches this transport. A matching |target| can be any of: // * <serial> // * <devpath> @@ -365,6 +380,7 @@ class atransport : public enable_weak_from_this<atransport> { // Attempts to reconnect with the underlying Connection. ReconnectResult Reconnect(); +#endif private: std::atomic<bool> kicked_; @@ -383,9 +399,11 @@ class atransport : public enable_weak_from_this<atransport> { std::deque<std::shared_ptr<RSA>> keys_; #endif +#if ADB_HOST // A sharable object that can be used to wait for the atransport's // connection to be established. std::shared_ptr<ConnectionWaitable> connection_waitable_; +#endif // The underlying connection object. std::shared_ptr<Connection> connection_ GUARDED_BY(mutex_); @@ -425,10 +443,17 @@ void init_reconnect_handler(void); void init_transport_registration(void); void init_mdns_transport_discovery(void); std::string list_transports(bool long_listing); + +#if ADB_HOST atransport* find_transport(const char* serial); + void kick_all_tcp_devices(); +#endif + void kick_all_transports(); + void kick_all_tcp_tls_transports(); + #if !ADB_HOST void kick_all_transports_by_auth_key(std::string_view auth_key); #endif diff --git a/adb/transport_test.cpp b/adb/transport_test.cpp index 00beb3a2b..8579ff4c2 100644 --- a/adb/transport_test.cpp +++ b/adb/transport_test.cpp @@ -66,7 +66,7 @@ TEST_F(TransportTest, SetFeatures) { ASSERT_TRUE(t.has_feature("bar")); t.SetFeatures(FeatureSetToString(FeatureSet{"foo", "bar", "foo"})); - ASSERT_EQ(2U, t.features().size()); + ASSERT_LE(2U, t.features().size()); ASSERT_TRUE(t.has_feature("foo")); ASSERT_TRUE(t.has_feature("bar")); @@ -127,6 +127,7 @@ TEST_F(TransportTest, parse_banner_features) { ASSERT_EQ(std::string("baz"), t.device); } +#if ADB_HOST TEST_F(TransportTest, test_matches_target) { std::string serial = "foo"; std::string devpath = "/path/to/bar"; @@ -183,3 +184,4 @@ TEST_F(TransportTest, test_matches_target_local) { EXPECT_FALSE(t.MatchesTarget("abc:100.100.100.100")); } } +#endif diff --git a/adb/types.cpp b/adb/types.cpp index 26b77ab62..9cdf32b93 100644 --- a/adb/types.cpp +++ b/adb/types.cpp @@ -51,7 +51,7 @@ void IOVector::drop_front(IOVector::size_type len) { auto dropped = 0u; while (dropped < len) { const auto next = chain_[start_index_].size() - begin_offset_; - if (dropped + next < len) { + if (dropped + next <= len) { pop_front_block(); dropped += next; } else { diff --git a/adb/types.h b/adb/types.h index deca7eafb..620aa8eb7 100644 --- a/adb/types.h +++ b/adb/types.h @@ -155,7 +155,7 @@ struct IOVector { return nullptr; } - return chain_.front().data() + begin_offset_; + return chain_[start_index_].data() + begin_offset_; } size_type front_size() const { @@ -163,7 +163,7 @@ struct IOVector { return 0; } - return chain_.front().size() - begin_offset_; + return chain_[start_index_].size() - begin_offset_; } size_type size() const { return chain_length_ - begin_offset_; } diff --git a/adb/types_test.cpp b/adb/types_test.cpp index 2c99f9512..41fa1db25 100644 --- a/adb/types_test.cpp +++ b/adb/types_test.cpp @@ -117,3 +117,20 @@ TEST(IOVector, misaligned_split) { ASSERT_EQ(1ULL, bc.size()); ASSERT_EQ(create_block("x"), bc.coalesce()); } + +TEST(IOVector, drop_front) { + IOVector vec; + + vec.append(create_block('x', 2)); + vec.append(create_block('y', 1000)); + ASSERT_EQ(2U, vec.front_size()); + ASSERT_EQ(1002U, vec.size()); + + vec.drop_front(1); + ASSERT_EQ(1U, vec.front_size()); + ASSERT_EQ(1001U, vec.size()); + + vec.drop_front(1); + ASSERT_EQ(1000U, vec.front_size()); + ASSERT_EQ(1000U, vec.size()); +} @@ -0,0 +1 @@ +../libbase
\ No newline at end of file diff --git a/base/.clang-format b/base/.clang-format deleted file mode 120000 index fd0645fdf..000000000 --- a/base/.clang-format +++ /dev/null @@ -1 +0,0 @@ -../.clang-format-2
\ No newline at end of file diff --git a/base/Android.bp b/base/Android.bp deleted file mode 100644 index 4556f9bf1..000000000 --- a/base/Android.bp +++ /dev/null @@ -1,225 +0,0 @@ -// -// Copyright (C) 2015 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -cc_defaults { - name: "libbase_cflags_defaults", - cflags: [ - "-Wall", - "-Werror", - "-Wextra", - ], - target: { - android: { - cflags: [ - "-D_FILE_OFFSET_BITS=64", - ], - }, - }, -} - -cc_library_headers { - name: "libbase_headers", - vendor_available: true, - ramdisk_available: true, - recovery_available: true, - host_supported: true, - native_bridge_supported: true, - export_include_dirs: ["include"], - - target: { - linux_bionic: { - enabled: true, - }, - windows: { - enabled: true, - }, - }, - apex_available: [ - "//apex_available:anyapex", - "//apex_available:platform", - ], - min_sdk_version: "29", -} - -cc_defaults { - name: "libbase_defaults", - defaults: ["libbase_cflags_defaults"], - srcs: [ - "abi_compatibility.cpp", - "chrono_utils.cpp", - "cmsg.cpp", - "file.cpp", - "liblog_symbols.cpp", - "logging.cpp", - "mapped_file.cpp", - "parsebool.cpp", - "parsenetaddress.cpp", - "process.cpp", - "properties.cpp", - "stringprintf.cpp", - "strings.cpp", - "threads.cpp", - "test_utils.cpp", - ], - - cppflags: ["-Wexit-time-destructors"], - shared_libs: ["liblog"], - target: { - android: { - sanitize: { - misc_undefined: ["integer"], - }, - - }, - linux: { - srcs: [ - "errors_unix.cpp", - ], - }, - darwin: { - srcs: [ - "errors_unix.cpp", - ], - }, - linux_bionic: { - enabled: true, - }, - windows: { - srcs: [ - "errors_windows.cpp", - "utf8.cpp", - ], - exclude_srcs: [ - "cmsg.cpp", - ], - enabled: true, - }, - }, -} - -cc_library { - name: "libbase", - defaults: ["libbase_defaults"], - vendor_available: true, - ramdisk_available: true, - recovery_available: true, - host_supported: true, - native_bridge_supported: true, - vndk: { - enabled: true, - support_system_process: true, - }, - header_libs: [ - "libbase_headers", - ], - export_header_lib_headers: ["libbase_headers"], - static_libs: ["fmtlib"], - whole_static_libs: ["fmtlib"], - export_static_lib_headers: ["fmtlib"], - apex_available: [ - "//apex_available:anyapex", - "//apex_available:platform", - ], - min_sdk_version: "29", -} - -cc_library_static { - name: "libbase_ndk", - defaults: ["libbase_defaults"], - sdk_version: "current", - stl: "c++_static", - export_include_dirs: ["include"], - static_libs: ["fmtlib_ndk"], - whole_static_libs: ["fmtlib_ndk"], - export_static_lib_headers: ["fmtlib_ndk"], -} - -// Tests -// ------------------------------------------------------------------------------ -cc_test { - name: "libbase_test", - defaults: ["libbase_cflags_defaults"], - host_supported: true, - srcs: [ - "cmsg_test.cpp", - "endian_test.cpp", - "errors_test.cpp", - "expected_test.cpp", - "file_test.cpp", - "logging_splitters_test.cpp", - "logging_test.cpp", - "macros_test.cpp", - "mapped_file_test.cpp", - "no_destructor_test.cpp", - "parsedouble_test.cpp", - "parsebool_test.cpp", - "parseint_test.cpp", - "parsenetaddress_test.cpp", - "process_test.cpp", - "properties_test.cpp", - "result_test.cpp", - "scopeguard_test.cpp", - "stringprintf_test.cpp", - "strings_test.cpp", - "test_main.cpp", - "test_utils_test.cpp", - ], - target: { - android: { - sanitize: { - misc_undefined: ["integer"], - }, - }, - linux: { - srcs: ["chrono_utils_test.cpp"], - }, - windows: { - srcs: ["utf8_test.cpp"], - cflags: ["-Wno-unused-parameter"], - enabled: true, - }, - }, - local_include_dirs: ["."], - shared_libs: ["libbase"], - compile_multilib: "both", - multilib: { - lib32: { - suffix: "32", - }, - lib64: { - suffix: "64", - }, - }, - test_suites: ["device-tests"], -} - -cc_benchmark { - name: "libbase_benchmark", - defaults: ["libbase_cflags_defaults"], - - srcs: ["format_benchmark.cpp"], - shared_libs: ["libbase"], - - compile_multilib: "both", - multilib: { - lib32: { - suffix: "32", - }, - lib64: { - suffix: "64", - }, - }, -} diff --git a/base/CPPLINT.cfg b/base/CPPLINT.cfg deleted file mode 100644 index d94a89c34..000000000 --- a/base/CPPLINT.cfg +++ /dev/null @@ -1,2 +0,0 @@ -set noparent -filter=-build/header_guard,-build/include,-build/c++11,-whitespace/operators diff --git a/base/OWNERS b/base/OWNERS deleted file mode 100644 index 97777f703..000000000 --- a/base/OWNERS +++ /dev/null @@ -1,3 +0,0 @@ -enh@google.com -jmgao@google.com -tomcherry@google.com diff --git a/base/README.md b/base/README.md deleted file mode 100644 index 2ef5c10b5..000000000 --- a/base/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# libbase - -## Who is this library for? - -This library is a collection of convenience functions to make common tasks -easier and less error-prone. - -In this context, "error-prone" covers both "hard to do correctly" and -"hard to do with good performance", but as a general purpose library, -libbase's primary focus is on making it easier to do things easily and -correctly when a compromise has to be made between "simplest API" on the -one hand and "fastest implementation" on the other. Though obviously -the ideal is to have both. - -## Should my routine be added? - -The intention is to cover the 80% use cases, not be all things to all users. - -If you have a routine that's really useful in your project, -congratulations. But that doesn't mean it should be here rather than -just in your project. - -The question for libbase is "should everyone be doing this?"/"does this -make everyone's code cleaner/safer?". Historically we've considered the -bar for inclusion to be "are there at least three *unrelated* projects -that would be cleaned up by doing so". - -If your routine is actually something from a future C++ standard (that -isn't yet in libc++), or it's widely used in another library, that helps -show that there's precedent. Being able to say "so-and-so has used this -API for n years" is a good way to reduce concerns about API choices. - -## Any other restrictions? - -Unlike most Android code, code in libbase has to build for Mac and -Windows too. - -Code here is also expected to have good test coverage. - -By its nature, it's difficult to change libbase API. It's often best -to start using your routine just in your project, and let it "graduate" -after you're certain that the API is solid. diff --git a/base/abi_compatibility.cpp b/base/abi_compatibility.cpp deleted file mode 100644 index 06a780171..000000000 --- a/base/abi_compatibility.cpp +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <memory> - -#include "android-base/cmsg.h" -#include "android-base/file.h" -#include "android-base/mapped_file.h" -#include "android-base/unique_fd.h" - -namespace android { -namespace base { - -// These ABI-compatibility shims are in a separate file for two reasons: -// 1. If they were in the file with the actual functions, it prevents calls to -// those functions by other functions in the file, due to ambiguity. -// 2. We will hopefully be able to delete these quickly. - -#if !defined(_WIN32) -ssize_t SendFileDescriptorVector(int sockfd, const void* data, size_t len, - const std::vector<int>& fds) { - return SendFileDescriptorVector(borrowed_fd(sockfd), data, len, fds); -} - -ssize_t ReceiveFileDescriptorVector(int sockfd, void* data, size_t len, size_t max_fds, - std::vector<unique_fd>* fds) { - return ReceiveFileDescriptorVector(borrowed_fd(sockfd), data, len, max_fds, fds); -} -#endif - -bool ReadFdToString(int fd, std::string* content) { - return ReadFdToString(borrowed_fd(fd), content); -} - -bool WriteStringToFd(const std::string& content, int fd) { - return WriteStringToFd(content, borrowed_fd(fd)); -} - -bool ReadFully(int fd, void* data, size_t byte_count) { - return ReadFully(borrowed_fd(fd), data, byte_count); -} - -bool ReadFullyAtOffset(int fd, void* data, size_t byte_count, off64_t offset) { - return ReadFullyAtOffset(borrowed_fd(fd), data, byte_count, offset); -} - -bool WriteFully(int fd, const void* data, size_t byte_count) { - return WriteFully(borrowed_fd(fd), data, byte_count); -} - -#if defined(__LP64__) -#define MAPPEDFILE_FROMFD _ZN10MappedFile6FromFdEilmi -#else -#define MAPPEDFILE_FROMFD _ZN10MappedFile6FromFdEixmi -#endif - -#pragma clang diagnostic ignored "-Wreturn-type-c-linkage" -extern "C" std::unique_ptr<MappedFile> MAPPEDFILE_FROMFD(int fd, off64_t offset, size_t length, - int prot) { - return MappedFile::FromFd(fd, offset, length, prot); -} - -} // namespace base -} // namespace android diff --git a/base/chrono_utils.cpp b/base/chrono_utils.cpp deleted file mode 100644 index 19080a5a7..000000000 --- a/base/chrono_utils.cpp +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (C) 2017 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "android-base/chrono_utils.h" - -#include <time.h> - -namespace android { -namespace base { - -boot_clock::time_point boot_clock::now() { -#ifdef __linux__ - timespec ts; - clock_gettime(CLOCK_BOOTTIME, &ts); - return boot_clock::time_point(std::chrono::seconds(ts.tv_sec) + - std::chrono::nanoseconds(ts.tv_nsec)); -#else - // Darwin and Windows do not support clock_gettime. - return boot_clock::time_point(); -#endif // __linux__ -} - -std::ostream& operator<<(std::ostream& os, const Timer& t) { - os << t.duration().count() << "ms"; - return os; -} - -} // namespace base -} // namespace android diff --git a/base/chrono_utils_test.cpp b/base/chrono_utils_test.cpp deleted file mode 100644 index da442f455..000000000 --- a/base/chrono_utils_test.cpp +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (C) 2017 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "android-base/chrono_utils.h" - -#include <time.h> - -#include <chrono> -#include <sstream> -#include <string> -#include <thread> - -#include <gtest/gtest.h> - -namespace android { -namespace base { - -std::chrono::seconds GetBootTimeSeconds() { - struct timespec now; - clock_gettime(CLOCK_BOOTTIME, &now); - - auto now_tp = boot_clock::time_point(std::chrono::seconds(now.tv_sec) + - std::chrono::nanoseconds(now.tv_nsec)); - return std::chrono::duration_cast<std::chrono::seconds>(now_tp.time_since_epoch()); -} - -// Tests (at least) the seconds accuracy of the boot_clock::now() method. -TEST(ChronoUtilsTest, BootClockNowSeconds) { - auto now = GetBootTimeSeconds(); - auto boot_seconds = - std::chrono::duration_cast<std::chrono::seconds>(boot_clock::now().time_since_epoch()); - EXPECT_EQ(now, boot_seconds); -} - -template <typename T> -void ExpectAboutEqual(T expected, T actual) { - auto expected_upper_bound = expected * 1.05f; - auto expected_lower_bound = expected * .95; - EXPECT_GT(expected_upper_bound, actual); - EXPECT_LT(expected_lower_bound, actual); -} - -TEST(ChronoUtilsTest, TimerDurationIsSane) { - auto start = boot_clock::now(); - Timer t; - std::this_thread::sleep_for(50ms); - auto stop = boot_clock::now(); - auto stop_timer = t.duration(); - - auto expected = std::chrono::duration_cast<std::chrono::milliseconds>(stop - start); - ExpectAboutEqual(expected, stop_timer); -} - -TEST(ChronoUtilsTest, TimerOstream) { - Timer t; - std::this_thread::sleep_for(50ms); - auto stop_timer = t.duration().count(); - std::stringstream os; - os << t; - decltype(stop_timer) stop_timer_from_stream; - os >> stop_timer_from_stream; - EXPECT_NE(0, stop_timer); - ExpectAboutEqual(stop_timer, stop_timer_from_stream); -} - -} // namespace base -} // namespace android diff --git a/base/cmsg.cpp b/base/cmsg.cpp deleted file mode 100644 index 1fa873c82..000000000 --- a/base/cmsg.cpp +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <android-base/cmsg.h> - -#include <errno.h> -#include <fcntl.h> -#include <stdlib.h> -#include <sys/socket.h> -#include <sys/user.h> - -#include <memory> - -#include <android-base/logging.h> - -namespace android { -namespace base { - -ssize_t SendFileDescriptorVector(borrowed_fd sockfd, const void* data, size_t len, - const std::vector<int>& fds) { - size_t cmsg_space = CMSG_SPACE(sizeof(int) * fds.size()); - size_t cmsg_len = CMSG_LEN(sizeof(int) * fds.size()); - if (cmsg_space >= PAGE_SIZE) { - errno = ENOMEM; - return -1; - } - - alignas(struct cmsghdr) char cmsg_buf[cmsg_space]; - iovec iov = {.iov_base = const_cast<void*>(data), .iov_len = len}; - msghdr msg = { - .msg_name = nullptr, - .msg_namelen = 0, - .msg_iov = &iov, - .msg_iovlen = 1, - .msg_control = cmsg_buf, - // We can't cast to the actual type of the field, because it's different across platforms. - .msg_controllen = static_cast<unsigned int>(cmsg_space), - .msg_flags = 0, - }; - - struct cmsghdr* cmsg = CMSG_FIRSTHDR(&msg); - cmsg->cmsg_level = SOL_SOCKET; - cmsg->cmsg_type = SCM_RIGHTS; - cmsg->cmsg_len = cmsg_len; - - int* cmsg_fds = reinterpret_cast<int*>(CMSG_DATA(cmsg)); - for (size_t i = 0; i < fds.size(); ++i) { - cmsg_fds[i] = fds[i]; - } - -#if defined(__linux__) - int flags = MSG_NOSIGNAL; -#else - int flags = 0; -#endif - - return TEMP_FAILURE_RETRY(sendmsg(sockfd.get(), &msg, flags)); -} - -ssize_t ReceiveFileDescriptorVector(borrowed_fd sockfd, void* data, size_t len, size_t max_fds, - std::vector<unique_fd>* fds) { - fds->clear(); - - size_t cmsg_space = CMSG_SPACE(sizeof(int) * max_fds); - if (cmsg_space >= PAGE_SIZE) { - errno = ENOMEM; - return -1; - } - - alignas(struct cmsghdr) char cmsg_buf[cmsg_space]; - iovec iov = {.iov_base = const_cast<void*>(data), .iov_len = len}; - msghdr msg = { - .msg_name = nullptr, - .msg_namelen = 0, - .msg_iov = &iov, - .msg_iovlen = 1, - .msg_control = cmsg_buf, - // We can't cast to the actual type of the field, because it's different across platforms. - .msg_controllen = static_cast<unsigned int>(cmsg_space), - .msg_flags = 0, - }; - - int flags = MSG_TRUNC | MSG_CTRUNC; -#if defined(__linux__) - flags |= MSG_CMSG_CLOEXEC | MSG_NOSIGNAL; -#endif - - ssize_t rc = TEMP_FAILURE_RETRY(recvmsg(sockfd.get(), &msg, flags)); - - if (rc == -1) { - return -1; - } - - int error = 0; - if ((msg.msg_flags & MSG_TRUNC)) { - LOG(ERROR) << "message was truncated when receiving file descriptors"; - error = EMSGSIZE; - } else if ((msg.msg_flags & MSG_CTRUNC)) { - LOG(ERROR) << "control message was truncated when receiving file descriptors"; - error = EMSGSIZE; - } - - std::vector<unique_fd> received_fds; - struct cmsghdr* cmsg; - for (cmsg = CMSG_FIRSTHDR(&msg); cmsg != nullptr; cmsg = CMSG_NXTHDR(&msg, cmsg)) { - if (cmsg->cmsg_level != SOL_SOCKET || cmsg->cmsg_type != SCM_RIGHTS) { - LOG(ERROR) << "received unexpected cmsg: [" << cmsg->cmsg_level << ", " << cmsg->cmsg_type - << "]"; - error = EBADMSG; - continue; - } - - // There isn't a macro that does the inverse of CMSG_LEN, so hack around it ourselves, with - // some asserts to ensure that CMSG_LEN behaves as we expect. -#if defined(__linux__) -#define CMSG_ASSERT static_assert -#else -// CMSG_LEN is somehow not constexpr on darwin. -#define CMSG_ASSERT CHECK -#endif - CMSG_ASSERT(CMSG_LEN(0) + 1 * sizeof(int) == CMSG_LEN(1 * sizeof(int))); - CMSG_ASSERT(CMSG_LEN(0) + 2 * sizeof(int) == CMSG_LEN(2 * sizeof(int))); - CMSG_ASSERT(CMSG_LEN(0) + 3 * sizeof(int) == CMSG_LEN(3 * sizeof(int))); - CMSG_ASSERT(CMSG_LEN(0) + 4 * sizeof(int) == CMSG_LEN(4 * sizeof(int))); - - if (cmsg->cmsg_len % sizeof(int) != 0) { - LOG(FATAL) << "cmsg_len(" << cmsg->cmsg_len << ") not aligned to sizeof(int)"; - } else if (cmsg->cmsg_len <= CMSG_LEN(0)) { - LOG(FATAL) << "cmsg_len(" << cmsg->cmsg_len << ") not long enough to hold any data"; - } - - int* cmsg_fds = reinterpret_cast<int*>(CMSG_DATA(cmsg)); - size_t cmsg_fdcount = static_cast<size_t>(cmsg->cmsg_len - CMSG_LEN(0)) / sizeof(int); - for (size_t i = 0; i < cmsg_fdcount; ++i) { -#if !defined(__linux__) - // Linux uses MSG_CMSG_CLOEXEC instead of doing this manually. - fcntl(cmsg_fds[i], F_SETFD, FD_CLOEXEC); -#endif - received_fds.emplace_back(cmsg_fds[i]); - } - } - - if (error != 0) { - errno = error; - return -1; - } - - if (received_fds.size() > max_fds) { - LOG(ERROR) << "received too many file descriptors, expected " << fds->size() << ", received " - << received_fds.size(); - errno = EMSGSIZE; - return -1; - } - - *fds = std::move(received_fds); - return rc; -} - -} // namespace base -} // namespace android diff --git a/base/cmsg_test.cpp b/base/cmsg_test.cpp deleted file mode 100644 index 9ee5c8253..000000000 --- a/base/cmsg_test.cpp +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <android-base/cmsg.h> - -#include <android-base/file.h> -#include <android-base/logging.h> -#include <android-base/unique_fd.h> -#include <gtest/gtest.h> - -#if !defined(_WIN32) - -using android::base::ReceiveFileDescriptors; -using android::base::SendFileDescriptors; -using android::base::unique_fd; - -static ino_t GetInode(int fd) { - struct stat st; - if (fstat(fd, &st) != 0) { - PLOG(FATAL) << "fstat failed"; - } - - return st.st_ino; -} - -struct CmsgTest : ::testing::TestWithParam<bool> { - bool Seqpacket() { return GetParam(); } - - void SetUp() override { - ASSERT_TRUE( - android::base::Socketpair(Seqpacket() ? SOCK_SEQPACKET : SOCK_STREAM, &send, &recv)); - int dup1 = dup(tmp1.fd); - ASSERT_NE(-1, dup1); - int dup2 = dup(tmp2.fd); - ASSERT_NE(-1, dup2); - - fd1.reset(dup1); - fd2.reset(dup2); - - ino1 = GetInode(dup1); - ino2 = GetInode(dup2); - } - - unique_fd send; - unique_fd recv; - - TemporaryFile tmp1; - TemporaryFile tmp2; - - unique_fd fd1; - unique_fd fd2; - - ino_t ino1; - ino_t ino2; -}; - -TEST_P(CmsgTest, smoke) { - ASSERT_EQ(1, SendFileDescriptors(send.get(), "x", 1, fd1.get())); - - char buf[2]; - unique_fd received; - ASSERT_EQ(1, ReceiveFileDescriptors(recv.get(), buf, 2, &received)); - ASSERT_EQ('x', buf[0]); - ASSERT_NE(-1, received.get()); - - ASSERT_EQ(ino1, GetInode(received.get())); -} - -TEST_P(CmsgTest, msg_trunc) { - ASSERT_EQ(2, SendFileDescriptors(send.get(), "ab", 2, fd1.get(), fd2.get())); - - char buf[2]; - unique_fd received1, received2; - - ssize_t rc = ReceiveFileDescriptors(recv.get(), buf, 1, &received1, &received2); - if (Seqpacket()) { - ASSERT_EQ(-1, rc); - ASSERT_EQ(EMSGSIZE, errno); - ASSERT_EQ(-1, received1.get()); - ASSERT_EQ(-1, received2.get()); - } else { - ASSERT_EQ(1, rc); - ASSERT_NE(-1, received1.get()); - ASSERT_NE(-1, received2.get()); - ASSERT_EQ(ino1, GetInode(received1.get())); - ASSERT_EQ(ino2, GetInode(received2.get())); - ASSERT_EQ(1, read(recv.get(), buf, 2)); - } -} - -TEST_P(CmsgTest, msg_ctrunc) { - ASSERT_EQ(1, SendFileDescriptors(send.get(), "a", 1, fd1.get(), fd2.get())); - - char buf[2]; - unique_fd received; - ASSERT_EQ(-1, ReceiveFileDescriptors(recv.get(), buf, 1, &received)); - ASSERT_EQ(EMSGSIZE, errno); - ASSERT_EQ(-1, received.get()); -} - -TEST_P(CmsgTest, peek) { - ASSERT_EQ(1, SendFileDescriptors(send.get(), "a", 1, fd1.get())); - - char buf[2]; - ASSERT_EQ(1, ::recv(recv.get(), buf, sizeof(buf), MSG_PEEK)); - ASSERT_EQ('a', buf[0]); - - unique_fd received; - ASSERT_EQ(1, ReceiveFileDescriptors(recv.get(), buf, 1, &received)); - ASSERT_EQ(ino1, GetInode(received.get())); -} - -TEST_P(CmsgTest, stream_fd_association) { - if (Seqpacket()) { - return; - } - - // fds are associated with the first byte of the write. - ASSERT_EQ(1, TEMP_FAILURE_RETRY(write(send.get(), "a", 1))); - ASSERT_EQ(2, SendFileDescriptors(send.get(), "bc", 2, fd1.get())); - ASSERT_EQ(1, SendFileDescriptors(send.get(), "d", 1, fd2.get())); - char buf[2]; - ASSERT_EQ(2, TEMP_FAILURE_RETRY(read(recv.get(), buf, 2))); - ASSERT_EQ(0, memcmp(buf, "ab", 2)); - - std::vector<unique_fd> received1; - ssize_t rc = ReceiveFileDescriptorVector(recv.get(), buf, 1, 1, &received1); - ASSERT_EQ(1, rc); - ASSERT_EQ('c', buf[0]); - ASSERT_TRUE(received1.empty()); - - unique_fd received2; - rc = ReceiveFileDescriptors(recv.get(), buf, 1, &received2); - ASSERT_EQ(1, rc); - ASSERT_EQ('d', buf[0]); - ASSERT_EQ(ino2, GetInode(received2.get())); -} - -TEST_P(CmsgTest, multiple_fd_ordering) { - ASSERT_EQ(1, SendFileDescriptors(send.get(), "a", 1, fd1.get(), fd2.get())); - - char buf[2]; - unique_fd received1, received2; - ASSERT_EQ(1, ReceiveFileDescriptors(recv.get(), buf, 1, &received1, &received2)); - - ASSERT_NE(-1, received1.get()); - ASSERT_NE(-1, received2.get()); - - ASSERT_EQ(ino1, GetInode(received1.get())); - ASSERT_EQ(ino2, GetInode(received2.get())); -} - -TEST_P(CmsgTest, separate_fd_ordering) { - ASSERT_EQ(1, SendFileDescriptors(send.get(), "a", 1, fd1.get())); - ASSERT_EQ(1, SendFileDescriptors(send.get(), "b", 1, fd2.get())); - - char buf[2]; - unique_fd received1, received2; - ASSERT_EQ(1, ReceiveFileDescriptors(recv.get(), buf, 1, &received1)); - ASSERT_EQ(1, ReceiveFileDescriptors(recv.get(), buf, 1, &received2)); - - ASSERT_NE(-1, received1.get()); - ASSERT_NE(-1, received2.get()); - - ASSERT_EQ(ino1, GetInode(received1.get())); - ASSERT_EQ(ino2, GetInode(received2.get())); -} - -TEST_P(CmsgTest, separate_fds_no_coalescing) { - unique_fd sent1(dup(tmp1.fd)); - unique_fd sent2(dup(tmp2.fd)); - - ASSERT_EQ(1, SendFileDescriptors(send.get(), "", 1, fd1.get())); - ASSERT_EQ(1, SendFileDescriptors(send.get(), "", 1, fd2.get())); - - char buf[2]; - std::vector<unique_fd> received; - ASSERT_EQ(1, ReceiveFileDescriptorVector(recv.get(), buf, 2, 2, &received)); - ASSERT_EQ(1U, received.size()); - ASSERT_EQ(ino1, GetInode(received[0].get())); - - ASSERT_EQ(1, ReceiveFileDescriptorVector(recv.get(), buf, 2, 2, &received)); - ASSERT_EQ(1U, received.size()); - ASSERT_EQ(ino2, GetInode(received[0].get())); -} - -INSTANTIATE_TEST_CASE_P(CmsgTest, CmsgTest, testing::Bool()); - -#endif diff --git a/base/endian_test.cpp b/base/endian_test.cpp deleted file mode 100644 index 963ab1348..000000000 --- a/base/endian_test.cpp +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright (C) 2017 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "android-base/endian.h" - -#include <gtest/gtest.h> - -TEST(endian, constants) { - ASSERT_TRUE(__LITTLE_ENDIAN == LITTLE_ENDIAN); - ASSERT_TRUE(__BIG_ENDIAN == BIG_ENDIAN); - ASSERT_TRUE(__BYTE_ORDER == BYTE_ORDER); - - ASSERT_EQ(__LITTLE_ENDIAN, __BYTE_ORDER); -} - -TEST(endian, smoke) { - static constexpr uint16_t le16 = 0x1234; - static constexpr uint32_t le32 = 0x12345678; - static constexpr uint64_t le64 = 0x123456789abcdef0; - - static constexpr uint16_t be16 = 0x3412; - static constexpr uint32_t be32 = 0x78563412; - static constexpr uint64_t be64 = 0xf0debc9a78563412; - - ASSERT_EQ(be16, htons(le16)); - ASSERT_EQ(be32, htonl(le32)); - ASSERT_EQ(be64, htonq(le64)); - - ASSERT_EQ(le16, ntohs(be16)); - ASSERT_EQ(le32, ntohl(be32)); - ASSERT_EQ(le64, ntohq(be64)); - - ASSERT_EQ(be16, htobe16(le16)); - ASSERT_EQ(be32, htobe32(le32)); - ASSERT_EQ(be64, htobe64(le64)); - - ASSERT_EQ(le16, betoh16(be16)); - ASSERT_EQ(le32, betoh32(be32)); - ASSERT_EQ(le64, betoh64(be64)); - - ASSERT_EQ(le16, htole16(le16)); - ASSERT_EQ(le32, htole32(le32)); - ASSERT_EQ(le64, htole64(le64)); - - ASSERT_EQ(le16, letoh16(le16)); - ASSERT_EQ(le32, letoh32(le32)); - ASSERT_EQ(le64, letoh64(le64)); - - ASSERT_EQ(le16, be16toh(be16)); - ASSERT_EQ(le32, be32toh(be32)); - ASSERT_EQ(le64, be64toh(be64)); - - ASSERT_EQ(le16, le16toh(le16)); - ASSERT_EQ(le32, le32toh(le32)); - ASSERT_EQ(le64, le64toh(le64)); -} diff --git a/base/errors_test.cpp b/base/errors_test.cpp deleted file mode 100644 index 8e7cdd1da..000000000 --- a/base/errors_test.cpp +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (C) 2016 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "android-base/errors.h" - -#include <gtest/gtest.h> - -namespace android { -namespace base { - -// Error strings aren't consistent enough across systems to test the output, -// just make sure we can compile correctly and nothing crashes even if we send -// it possibly bogus error codes. -TEST(ErrorsTest, TestSystemErrorString) { - SystemErrorCodeToString(-1); - SystemErrorCodeToString(0); - SystemErrorCodeToString(1); -} - -} // namespace base -} // namespace android diff --git a/base/errors_unix.cpp b/base/errors_unix.cpp deleted file mode 100644 index 48269b675..000000000 --- a/base/errors_unix.cpp +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) 2016 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "android-base/errors.h" - -#include <errno.h> -#include <string.h> - -namespace android { -namespace base { - -std::string SystemErrorCodeToString(int error_code) { - return strerror(error_code); -} - -} // namespace base -} // namespace android diff --git a/base/errors_windows.cpp b/base/errors_windows.cpp deleted file mode 100644 index a5ff51188..000000000 --- a/base/errors_windows.cpp +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (C) 2016 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "android-base/errors.h" - -#include <windows.h> - -#include "android-base/stringprintf.h" -#include "android-base/strings.h" -#include "android-base/utf8.h" - -// A Windows error code is a DWORD. It's simpler to use an int error code for -// both Unix and Windows if possible, but if this fails we'll need a different -// function signature for each. -static_assert(sizeof(int) >= sizeof(DWORD), - "Windows system error codes are too large to fit in an int."); - -namespace android { -namespace base { - -static constexpr DWORD kErrorMessageBufferSize = 256; - -std::string SystemErrorCodeToString(int int_error_code) { - WCHAR msgbuf[kErrorMessageBufferSize]; - DWORD error_code = int_error_code; - DWORD flags = FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS; - DWORD len = FormatMessageW(flags, nullptr, error_code, 0, msgbuf, - kErrorMessageBufferSize, nullptr); - if (len == 0) { - return android::base::StringPrintf( - "Error %lu while retrieving message for error %lu", GetLastError(), - error_code); - } - - // Convert UTF-16 to UTF-8. - std::string msg; - if (!android::base::WideToUTF8(msgbuf, &msg)) { - return android::base::StringPrintf( - "Error %lu while converting message for error %lu from UTF-16 to UTF-8", - GetLastError(), error_code); - } - - // Messages returned by the system end with line breaks. - msg = android::base::Trim(msg); - - // There are many Windows error messages compared to POSIX, so include the - // numeric error code for easier, quicker, accurate identification. Use - // decimal instead of hex because there are decimal ranges like 10000-11999 - // for Winsock. - android::base::StringAppendF(&msg, " (%lu)", error_code); - return msg; -} - -} // namespace base -} // namespace android diff --git a/base/expected_test.cpp b/base/expected_test.cpp deleted file mode 100644 index 47e396a22..000000000 --- a/base/expected_test.cpp +++ /dev/null @@ -1,876 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "android-base/expected.h" - -#include <cstdio> -#include <memory> -#include <string> - -#include <gtest/gtest.h> - -using android::base::expected; -using android::base::unexpected; - -typedef expected<int, int> exp_int; -typedef expected<double, double> exp_double; -typedef expected<std::string, std::string> exp_string; -typedef expected<std::pair<std::string, int>, int> exp_pair; -typedef expected<void, int> exp_void; - -struct T { - int a; - int b; - T() = default; - T(int a, int b) noexcept : a(a), b(b) {} -}; -bool operator==(const T& x, const T& y) { - return x.a == y.a && x.b == y.b; -} -bool operator!=(const T& x, const T& y) { - return x.a != y.a || x.b != y.b; -} - -struct E { - std::string message; - int cause; - E(const std::string& message, int cause) : message(message), cause(cause) {} -}; - -typedef expected<T,E> exp_complex; - -TEST(Expected, testDefaultConstructible) { - exp_int e; - EXPECT_TRUE(e.has_value()); - EXPECT_EQ(0, e.value()); - - exp_complex e2; - EXPECT_TRUE(e2.has_value()); - EXPECT_EQ(T(0,0), e2.value()); - - exp_void e3; - EXPECT_TRUE(e3.has_value()); -} - -TEST(Expected, testCopyConstructible) { - exp_int e; - exp_int e2 = e; - - EXPECT_TRUE(e.has_value()); - EXPECT_TRUE(e2.has_value()); - EXPECT_EQ(0, e.value()); - EXPECT_EQ(0, e2.value()); - - exp_void e3; - exp_void e4 = e3; - EXPECT_TRUE(e3.has_value()); - EXPECT_TRUE(e4.has_value()); -} - -TEST(Expected, testMoveConstructible) { - exp_int e; - exp_int e2 = std::move(e); - - EXPECT_TRUE(e.has_value()); - EXPECT_TRUE(e2.has_value()); - EXPECT_EQ(0, e.value()); - EXPECT_EQ(0, e2.value()); - - exp_string e3(std::string("hello")); - exp_string e4 = std::move(e3); - - EXPECT_TRUE(e3.has_value()); - EXPECT_TRUE(e4.has_value()); - EXPECT_EQ("", e3.value()); // e3 is moved - EXPECT_EQ("hello", e4.value()); - - exp_void e5; - exp_void e6 = std::move(e5); - EXPECT_TRUE(e5.has_value()); - EXPECT_TRUE(e6.has_value()); -} - -TEST(Expected, testCopyConstructibleFromConvertibleType) { - exp_double e = 3.3f; - exp_int e2 = e; - - EXPECT_TRUE(e.has_value()); - EXPECT_TRUE(e2.has_value()); - EXPECT_EQ(3.3f, e.value()); - EXPECT_EQ(3, e2.value()); -} - -TEST(Expected, testMoveConstructibleFromConvertibleType) { - exp_double e = 3.3f; - exp_int e2 = std::move(e); - - EXPECT_TRUE(e.has_value()); - EXPECT_TRUE(e2.has_value()); - EXPECT_EQ(3.3f, e.value()); - EXPECT_EQ(3, e2.value()); -} - -TEST(Expected, testConstructibleFromValue) { - exp_int e = 3; - exp_double e2 = 5.5f; - exp_string e3 = std::string("hello"); - exp_complex e4 = T(10, 20); - exp_void e5 = {}; - - EXPECT_TRUE(e.has_value()); - EXPECT_TRUE(e2.has_value()); - EXPECT_TRUE(e3.has_value()); - EXPECT_TRUE(e4.has_value()); - EXPECT_TRUE(e5.has_value()); - EXPECT_EQ(3, e.value()); - EXPECT_EQ(5.5f, e2.value()); - EXPECT_EQ("hello", e3.value()); - EXPECT_EQ(T(10,20), e4.value()); -} - -TEST(Expected, testConstructibleFromMovedValue) { - std::string hello = "hello"; - exp_string e = std::move(hello); - - EXPECT_TRUE(e.has_value()); - EXPECT_EQ("hello", e.value()); - EXPECT_EQ("", hello); -} - -TEST(Expected, testConstructibleFromConvertibleValue) { - exp_int e = 3.3f; // double to int - exp_string e2 = "hello"; // char* to std::string - EXPECT_TRUE(e.has_value()); - EXPECT_EQ(3, e.value()); - - EXPECT_TRUE(e2.has_value()); - EXPECT_EQ("hello", e2.value()); -} - -TEST(Expected, testConstructibleFromUnexpected) { - exp_int::unexpected_type unexp = unexpected(10); - exp_int e = unexp; - - exp_double::unexpected_type unexp2 = unexpected(10.5f); - exp_double e2 = unexp2; - - exp_string::unexpected_type unexp3 = unexpected(std::string("error")); - exp_string e3 = unexp3; - - exp_void::unexpected_type unexp4 = unexpected(10); - exp_void e4 = unexp4; - - EXPECT_FALSE(e.has_value()); - EXPECT_FALSE(e2.has_value()); - EXPECT_FALSE(e3.has_value()); - EXPECT_FALSE(e4.has_value()); - EXPECT_EQ(10, e.error()); - EXPECT_EQ(10.5f, e2.error()); - EXPECT_EQ("error", e3.error()); - EXPECT_EQ(10, e4.error()); -} - -TEST(Expected, testMoveConstructibleFromUnexpected) { - exp_int e = unexpected(10); - exp_double e2 = unexpected(10.5f); - exp_string e3 = unexpected(std::string("error")); - exp_void e4 = unexpected(10); - - EXPECT_FALSE(e.has_value()); - EXPECT_FALSE(e2.has_value()); - EXPECT_FALSE(e3.has_value()); - EXPECT_FALSE(e4.has_value()); - EXPECT_EQ(10, e.error()); - EXPECT_EQ(10.5f, e2.error()); - EXPECT_EQ("error", e3.error()); - EXPECT_EQ(10, e4.error()); -} - -TEST(Expected, testConstructibleByForwarding) { - exp_string e(std::in_place, 5, 'a'); - EXPECT_TRUE(e.has_value()); - EXPECT_EQ("aaaaa", e.value()); - - exp_string e2({'a', 'b', 'c'}); - EXPECT_TRUE(e2.has_value()); - EXPECT_EQ("abc", e2.value()); - - exp_pair e3({"hello", 30}); - EXPECT_TRUE(e3.has_value()); - EXPECT_EQ("hello",e3->first); - EXPECT_EQ(30,e3->second); - - exp_void e4({}); - EXPECT_TRUE(e4.has_value()); -} - -TEST(Expected, testDestructible) { - bool destroyed = false; - struct T { - bool* flag_; - T(bool* flag) : flag_(flag) {} - ~T() { *flag_ = true; } - }; - { - expected<T, int> exp = T(&destroyed); - } - EXPECT_TRUE(destroyed); -} - -TEST(Expected, testAssignable) { - exp_int e = 10; - exp_int e2 = 20; - e = e2; - - EXPECT_EQ(20, e.value()); - EXPECT_EQ(20, e2.value()); - - exp_int e3 = 10; - exp_int e4 = 20; - e3 = std::move(e4); - - EXPECT_EQ(20, e3.value()); - EXPECT_EQ(20, e4.value()); - - exp_void e5 = unexpected(10); - ASSERT_FALSE(e5.has_value()); - exp_void e6; - e5 = e6; - - EXPECT_TRUE(e5.has_value()); - EXPECT_TRUE(e6.has_value()); -} - -TEST(Expected, testAssignableFromValue) { - exp_int e = 10; - e = 20; - EXPECT_EQ(20, e.value()); - - exp_double e2 = 3.5f; - e2 = 10.5f; - EXPECT_EQ(10.5f, e2.value()); - - exp_string e3 = "hello"; - e3 = "world"; - EXPECT_EQ("world", e3.value()); - - exp_void e4 = unexpected(10); - ASSERT_FALSE(e4.has_value()); - e4 = {}; - EXPECT_TRUE(e4.has_value()); -} - -TEST(Expected, testAssignableFromUnexpected) { - exp_int e = 10; - e = unexpected(30); - EXPECT_FALSE(e.has_value()); - EXPECT_EQ(30, e.error()); - - exp_double e2 = 3.5f; - e2 = unexpected(10.5f); - EXPECT_FALSE(e2.has_value()); - EXPECT_EQ(10.5f, e2.error()); - - exp_string e3 = "hello"; - e3 = unexpected("world"); - EXPECT_FALSE(e3.has_value()); - EXPECT_EQ("world", e3.error()); - - exp_void e4 = {}; - e4 = unexpected(10); - EXPECT_FALSE(e4.has_value()); - EXPECT_EQ(10, e4.error()); -} - -TEST(Expected, testAssignableFromMovedValue) { - std::string world = "world"; - exp_string e = "hello"; - e = std::move(world); - - EXPECT_TRUE(e.has_value()); - EXPECT_EQ("world", e.value()); - EXPECT_EQ("", world); -} - -TEST(Expected, testAssignableFromMovedUnexpected) { - std::string world = "world"; - exp_string e = "hello"; - e = unexpected(std::move(world)); - - EXPECT_FALSE(e.has_value()); - EXPECT_EQ("world", e.error()); - EXPECT_EQ("", world); -} - -TEST(Expected, testEmplace) { - struct T { - int a; - double b; - T() {} - T(int a, double b) noexcept : a(a), b(b) {} - }; - expected<T, int> exp; - T& t = exp.emplace(3, 10.5f); - - EXPECT_TRUE(exp.has_value()); - EXPECT_EQ(3, t.a); - EXPECT_EQ(10.5f, t.b); - EXPECT_EQ(3, exp.value().a); - EXPECT_EQ(10.5, exp.value().b); - - exp_void e = unexpected(10); - ASSERT_FALSE(e.has_value()); - e.emplace(); - EXPECT_TRUE(e.has_value()); -} - -TEST(Expected, testSwapExpectedExpected) { - exp_int e = 10; - exp_int e2 = 20; - e.swap(e2); - - EXPECT_TRUE(e.has_value()); - EXPECT_TRUE(e2.has_value()); - EXPECT_EQ(20, e.value()); - EXPECT_EQ(10, e2.value()); - - exp_void e3; - exp_void e4; - e3.swap(e4); - - EXPECT_TRUE(e3.has_value()); - EXPECT_TRUE(e4.has_value()); -} - -TEST(Expected, testSwapUnexpectedUnexpected) { - exp_int e = unexpected(10); - exp_int e2 = unexpected(20); - e.swap(e2); - EXPECT_FALSE(e.has_value()); - EXPECT_FALSE(e2.has_value()); - EXPECT_EQ(20, e.error()); - EXPECT_EQ(10, e2.error()); - - exp_void e3 = unexpected(10); - exp_void e4 = unexpected(20); - e3.swap(e4); - EXPECT_FALSE(e3.has_value()); - EXPECT_FALSE(e4.has_value()); - EXPECT_EQ(20, e3.error()); - EXPECT_EQ(10, e4.error()); -} - -TEST(Expected, testSwapExpectedUnepected) { - exp_int e = 10; - exp_int e2 = unexpected(30); - e.swap(e2); - EXPECT_FALSE(e.has_value()); - EXPECT_TRUE(e2.has_value()); - EXPECT_EQ(30, e.error()); - EXPECT_EQ(10, e2.value()); - - exp_void e3; - exp_void e4 = unexpected(10); - e3.swap(e4); - EXPECT_FALSE(e3.has_value()); - EXPECT_TRUE(e4.has_value()); - EXPECT_EQ(10, e3.error()); -} - -TEST(Expected, testDereference) { - struct T { - int a; - double b; - T() {} - T(int a, double b) : a(a), b(b) {} - }; - expected<T, int> exp = T(3, 10.5f); - - EXPECT_EQ(3, exp->a); - EXPECT_EQ(10.5f, exp->b); - - EXPECT_EQ(3, (*exp).a); - EXPECT_EQ(10.5f, (*exp).b); -} - -TEST(Expected, testTest) { - exp_int e = 10; - EXPECT_TRUE(e.ok()); - EXPECT_TRUE(e.has_value()); - - exp_int e2 = unexpected(10); - EXPECT_FALSE(e2.ok()); - EXPECT_FALSE(e2.has_value()); -} - -TEST(Expected, testGetValue) { - exp_int e = 10; - EXPECT_EQ(10, e.value()); - EXPECT_EQ(10, e.value_or(20)); - - exp_int e2 = unexpected(10); - EXPECT_EQ(10, e2.error()); - EXPECT_EQ(20, e2.value_or(20)); -} - -TEST(Expected, testSameValues) { - exp_int e = 10; - exp_int e2 = 10; - EXPECT_TRUE(e == e2); - EXPECT_TRUE(e2 == e); - EXPECT_FALSE(e != e2); - EXPECT_FALSE(e2 != e); - - exp_void e3; - exp_void e4; - EXPECT_TRUE(e3 == e4); - EXPECT_TRUE(e4 == e3); - EXPECT_FALSE(e3 != e4); - EXPECT_FALSE(e4 != e3); -} - -TEST(Expected, testDifferentValues) { - exp_int e = 10; - exp_int e2 = 20; - EXPECT_FALSE(e == e2); - EXPECT_FALSE(e2 == e); - EXPECT_TRUE(e != e2); - EXPECT_TRUE(e2 != e); -} - -TEST(Expected, testValueWithError) { - exp_int e = 10; - exp_int e2 = unexpected(10); - EXPECT_FALSE(e == e2); - EXPECT_FALSE(e2 == e); - EXPECT_TRUE(e != e2); - EXPECT_TRUE(e2 != e); - - exp_void e3; - exp_void e4 = unexpected(10); - EXPECT_FALSE(e3 == e4); - EXPECT_FALSE(e4 == e3); - EXPECT_TRUE(e3 != e4); - EXPECT_TRUE(e4 != e3); -} - -TEST(Expected, testSameErrors) { - exp_int e = unexpected(10); - exp_int e2 = unexpected(10); - EXPECT_TRUE(e == e2); - EXPECT_TRUE(e2 == e); - EXPECT_FALSE(e != e2); - EXPECT_FALSE(e2 != e); - - exp_void e3 = unexpected(10); - exp_void e4 = unexpected(10); - EXPECT_TRUE(e3 == e4); - EXPECT_TRUE(e4 == e3); - EXPECT_FALSE(e3 != e4); - EXPECT_FALSE(e4 != e3); -} - -TEST(Expected, testDifferentErrors) { - exp_int e = unexpected(10); - exp_int e2 = unexpected(20); - EXPECT_FALSE(e == e2); - EXPECT_FALSE(e2 == e); - EXPECT_TRUE(e != e2); - EXPECT_TRUE(e2 != e); - - exp_void e3 = unexpected(10); - exp_void e4 = unexpected(20); - EXPECT_FALSE(e3 == e4); - EXPECT_FALSE(e4 == e3); - EXPECT_TRUE(e3 != e4); - EXPECT_TRUE(e4 != e3); -} - -TEST(Expected, testCompareWithSameError) { - exp_int e = unexpected(10); - exp_int::unexpected_type error = 10; - EXPECT_TRUE(e == error); - EXPECT_TRUE(error == e); - EXPECT_FALSE(e != error); - EXPECT_FALSE(error != e); - - exp_void e2 = unexpected(10); - exp_void::unexpected_type error2 = 10; - EXPECT_TRUE(e2 == error2); - EXPECT_TRUE(error2 == e2); - EXPECT_FALSE(e2 != error2); - EXPECT_FALSE(error2 != e2); -} - -TEST(Expected, testCompareWithDifferentError) { - exp_int e = unexpected(10); - exp_int::unexpected_type error = 20; - EXPECT_FALSE(e == error); - EXPECT_FALSE(error == e); - EXPECT_TRUE(e != error); - EXPECT_TRUE(error != e); - - exp_void e2 = unexpected(10); - exp_void::unexpected_type error2 = 20; - EXPECT_FALSE(e2 == error2); - EXPECT_FALSE(error2 == e2); - EXPECT_TRUE(e2 != error2); - EXPECT_TRUE(error2 != e2); -} - -TEST(Expected, testCompareDifferentType) { - expected<int,int> e = 10; - expected<int32_t, int> e2 = 10; - EXPECT_TRUE(e == e2); - e2 = 20; - EXPECT_FALSE(e == e2); - - expected<std::string_view,int> e3 = "hello"; - expected<std::string,int> e4 = "hello"; - EXPECT_TRUE(e3 == e4); - e4 = "world"; - EXPECT_FALSE(e3 == e4); - - expected<void,int> e5; - expected<int,int> e6 = 10; - EXPECT_FALSE(e5 == e6); - EXPECT_FALSE(e6 == e5); -} - -TEST(Expected, testDivideExample) { - struct QR { - int quotient; - int remainder; - QR(int q, int r) noexcept : quotient(q), remainder(r) {} - bool operator==(const QR& rhs) const { - return quotient == rhs.quotient && remainder == rhs.remainder; - } - bool operator!=(const QR& rhs) const { - return quotient != rhs.quotient || remainder == rhs.remainder; - } - }; - - auto divide = [](int x, int y) -> expected<QR,E> { - if (y == 0) { - return unexpected(E("divide by zero", -1)); - } else { - return QR(x / y, x % y); - } - }; - - EXPECT_FALSE(divide(10, 0).ok()); - EXPECT_EQ("divide by zero", divide(10, 0).error().message); - EXPECT_EQ(-1, divide(10, 0).error().cause); - - EXPECT_TRUE(divide(10, 3).ok()); - EXPECT_EQ(QR(3, 1), *divide(10, 3)); -} - -TEST(Expected, testPair) { - auto test = [](bool yes) -> exp_pair { - if (yes) { - return exp_pair({"yes", 42}); - } else { - return unexpected(42); - } - }; - - auto r = test(true); - EXPECT_TRUE(r.ok()); - EXPECT_EQ("yes", r->first); -} - -TEST(Expected, testVoid) { - auto test = [](bool ok) -> exp_void { - if (ok) { - return {}; - } else { - return unexpected(10); - } - }; - - auto r = test(true); - EXPECT_TRUE(r.ok()); - r = test(false); - EXPECT_FALSE(r.ok()); - EXPECT_EQ(10, r.error()); -} - -// copied from result_test.cpp -struct ConstructorTracker { - static size_t constructor_called; - static size_t copy_constructor_called; - static size_t move_constructor_called; - static size_t copy_assignment_called; - static size_t move_assignment_called; - - template <typename T, - typename std::enable_if_t<std::is_convertible_v<T, std::string>>* = nullptr> - ConstructorTracker(T&& string) : string(string) { - ++constructor_called; - } - ConstructorTracker(const ConstructorTracker& ct) { - ++copy_constructor_called; - string = ct.string; - } - ConstructorTracker(ConstructorTracker&& ct) noexcept { - ++move_constructor_called; - string = std::move(ct.string); - } - ConstructorTracker& operator=(const ConstructorTracker& ct) { - ++copy_assignment_called; - string = ct.string; - return *this; - } - ConstructorTracker& operator=(ConstructorTracker&& ct) noexcept { - ++move_assignment_called; - string = std::move(ct.string); - return *this; - } - static void Reset() { - constructor_called = 0; - copy_constructor_called = 0; - move_constructor_called = 0; - copy_assignment_called = 0; - move_assignment_called = 0; - } - std::string string; -}; - -size_t ConstructorTracker::constructor_called = 0; -size_t ConstructorTracker::copy_constructor_called = 0; -size_t ConstructorTracker::move_constructor_called = 0; -size_t ConstructorTracker::copy_assignment_called = 0; -size_t ConstructorTracker::move_assignment_called = 0; - -typedef expected<ConstructorTracker, int> exp_track; - -TEST(Expected, testNumberOfCopies) { - // default constructor - ConstructorTracker::Reset(); - exp_track e("hello"); - EXPECT_EQ(1U, ConstructorTracker::constructor_called); - EXPECT_EQ(0U, ConstructorTracker::copy_constructor_called); - EXPECT_EQ(0U, ConstructorTracker::move_constructor_called); - EXPECT_EQ(0U, ConstructorTracker::copy_assignment_called); - EXPECT_EQ(0U, ConstructorTracker::move_assignment_called); - - // copy constructor - ConstructorTracker::Reset(); - exp_track e2 = e; - EXPECT_EQ(0U, ConstructorTracker::constructor_called); - EXPECT_EQ(1U, ConstructorTracker::copy_constructor_called); - EXPECT_EQ(0U, ConstructorTracker::move_constructor_called); - EXPECT_EQ(0U, ConstructorTracker::copy_assignment_called); - EXPECT_EQ(0U, ConstructorTracker::move_assignment_called); - - // move constructor - ConstructorTracker::Reset(); - exp_track e3 = std::move(e); - EXPECT_EQ(0U, ConstructorTracker::constructor_called); - EXPECT_EQ(0U, ConstructorTracker::copy_constructor_called); - EXPECT_EQ(1U, ConstructorTracker::move_constructor_called); - EXPECT_EQ(0U, ConstructorTracker::copy_assignment_called); - EXPECT_EQ(0U, ConstructorTracker::move_assignment_called); - - // construct from lvalue - ConstructorTracker::Reset(); - ConstructorTracker ct = "hello"; - exp_track e4(ct); - EXPECT_EQ(1U, ConstructorTracker::constructor_called); - EXPECT_EQ(1U, ConstructorTracker::copy_constructor_called); - EXPECT_EQ(0U, ConstructorTracker::move_constructor_called); - EXPECT_EQ(0U, ConstructorTracker::copy_assignment_called); - EXPECT_EQ(0U, ConstructorTracker::move_assignment_called); - - // construct from rvalue - ConstructorTracker::Reset(); - ConstructorTracker ct2 = "hello"; - exp_track e5(std::move(ct2)); - EXPECT_EQ(1U, ConstructorTracker::constructor_called); - EXPECT_EQ(0U, ConstructorTracker::copy_constructor_called); - EXPECT_EQ(1U, ConstructorTracker::move_constructor_called); - EXPECT_EQ(0U, ConstructorTracker::copy_assignment_called); - EXPECT_EQ(0U, ConstructorTracker::move_assignment_called); - - // copy assignment - ConstructorTracker::Reset(); - exp_track e6 = "hello"; - exp_track e7 = "world"; - e7 = e6; - EXPECT_EQ(2U, ConstructorTracker::constructor_called); - EXPECT_EQ(0U, ConstructorTracker::copy_constructor_called); - EXPECT_EQ(0U, ConstructorTracker::move_constructor_called); - EXPECT_EQ(1U, ConstructorTracker::copy_assignment_called); - EXPECT_EQ(0U, ConstructorTracker::move_assignment_called); - - // move assignment - ConstructorTracker::Reset(); - exp_track e8 = "hello"; - exp_track e9 = "world"; - e9 = std::move(e8); - EXPECT_EQ(2U, ConstructorTracker::constructor_called); - EXPECT_EQ(0U, ConstructorTracker::copy_constructor_called); - EXPECT_EQ(0U, ConstructorTracker::move_constructor_called); - EXPECT_EQ(0U, ConstructorTracker::copy_assignment_called); - EXPECT_EQ(1U, ConstructorTracker::move_assignment_called); - - // swap - ConstructorTracker::Reset(); - exp_track e10 = "hello"; - exp_track e11 = "world"; - std::swap(e10, e11); - EXPECT_EQ(2U, ConstructorTracker::constructor_called); - EXPECT_EQ(0U, ConstructorTracker::copy_constructor_called); - EXPECT_EQ(1U, ConstructorTracker::move_constructor_called); - EXPECT_EQ(0U, ConstructorTracker::copy_assignment_called); - EXPECT_EQ(2U, ConstructorTracker::move_assignment_called); -} - -TEST(Expected, testNoCopyOnReturn) { - auto test = [](const std::string& in) -> exp_track { - if (in.empty()) { - return "literal string"; - } - if (in == "test2") { - return ConstructorTracker(in + in + "2"); - } - ConstructorTracker result(in + " " + in); - return result; - }; - - ConstructorTracker::Reset(); - auto result1 = test(""); - ASSERT_TRUE(result1.ok()); - EXPECT_EQ("literal string", result1->string); - EXPECT_EQ(1U, ConstructorTracker::constructor_called); - EXPECT_EQ(0U, ConstructorTracker::copy_constructor_called); - EXPECT_EQ(0U, ConstructorTracker::move_constructor_called); - EXPECT_EQ(0U, ConstructorTracker::copy_assignment_called); - EXPECT_EQ(0U, ConstructorTracker::move_assignment_called); - - ConstructorTracker::Reset(); - auto result2 = test("test2"); - ASSERT_TRUE(result2.ok()); - EXPECT_EQ("test2test22", result2->string); - EXPECT_EQ(1U, ConstructorTracker::constructor_called); - EXPECT_EQ(0U, ConstructorTracker::copy_constructor_called); - EXPECT_EQ(1U, ConstructorTracker::move_constructor_called); - EXPECT_EQ(0U, ConstructorTracker::copy_assignment_called); - EXPECT_EQ(0U, ConstructorTracker::move_assignment_called); - - ConstructorTracker::Reset(); - auto result3 = test("test3"); - ASSERT_TRUE(result3.ok()); - EXPECT_EQ("test3 test3", result3->string); - EXPECT_EQ(1U, ConstructorTracker::constructor_called); - EXPECT_EQ(0U, ConstructorTracker::copy_constructor_called); - EXPECT_EQ(1U, ConstructorTracker::move_constructor_called); - EXPECT_EQ(0U, ConstructorTracker::copy_assignment_called); - EXPECT_EQ(0U, ConstructorTracker::move_assignment_called); -} - -TEST(Expected, testNested) { - expected<exp_string, std::string> e = "hello"; - - EXPECT_TRUE(e.ok()); - EXPECT_TRUE(e.has_value()); - EXPECT_TRUE(e.value().has_value()); - EXPECT_TRUE(e->ok()); - EXPECT_EQ("hello", e.value().value()); - - expected<exp_string, std::string> e2 = unexpected("world"); - EXPECT_FALSE(e2.has_value()); - EXPECT_FALSE(e2.ok()); - EXPECT_EQ("world", e2.error()); - - expected<exp_string, std::string> e3 = exp_string(unexpected("world")); - EXPECT_TRUE(e3.has_value()); - EXPECT_FALSE(e3.value().has_value()); - EXPECT_TRUE(e3.ok()); - EXPECT_FALSE(e3->ok()); - EXPECT_EQ("world", e3.value().error()); -} - -constexpr bool equals(const char* a, const char* b) { - return (a == nullptr && b == nullptr) || - (a != nullptr && b != nullptr && *a == *b && - (*a == '\0' || equals(a + 1, b + 1))); -} - -TEST(Expected, testConstexpr) { - // Compliation error will occur if these expressions can't be - // evaluated at compile time - constexpr exp_int e(3); - constexpr exp_int::unexpected_type err(3); - constexpr int i = 4; - - // default constructor - static_assert(exp_int().value() == 0); - // copy constructor - static_assert(exp_int(e).value() == 3); - // move constructor - static_assert(exp_int(exp_int(4)).value() == 4); - // copy construct from value - static_assert(exp_int(i).value() == 4); - // copy construct from unexpected - static_assert(exp_int(err).error() == 3); - // move costruct from unexpected - static_assert(exp_int(unexpected(3)).error() == 3); - // observers - static_assert(*exp_int(3) == 3); - static_assert(exp_int(3).has_value() == true); - static_assert(exp_int(3).value_or(4) == 3); - - typedef expected<const char*, int> exp_s; - constexpr exp_s s("hello"); - constexpr const char* c = "hello"; - static_assert(equals(exp_s().value(), nullptr)); - static_assert(equals(exp_s(s).value(), "hello")); - static_assert(equals(exp_s(exp_s("hello")).value(), "hello")); - static_assert(equals(exp_s("hello").value(), "hello")); - static_assert(equals(exp_s(c).value(), "hello")); -} - -TEST(Expected, testWithNonConstructible) { - struct AssertNotConstructed { - AssertNotConstructed() = delete; - }; - - expected<int, AssertNotConstructed> v(42); - EXPECT_TRUE(v.has_value()); - EXPECT_EQ(42, v.value()); - - expected<AssertNotConstructed, int> e(unexpected(42)); - EXPECT_FALSE(e.has_value()); - EXPECT_EQ(42, e.error()); -} - -TEST(Expected, testWithMoveOnlyType) { - typedef expected<std::unique_ptr<int>,std::unique_ptr<int>> exp_ptr; - exp_ptr e(std::make_unique<int>(3)); - exp_ptr e2(unexpected(std::make_unique<int>(4))); - - EXPECT_TRUE(e.has_value()); - EXPECT_FALSE(e2.has_value()); - EXPECT_EQ(3, *(e.value())); - EXPECT_EQ(4, *(e2.error())); - - e2 = std::move(e); - EXPECT_TRUE(e.has_value()); - EXPECT_TRUE(e2.has_value()); - EXPECT_EQ(3, *(e2.value())); -} diff --git a/base/file.cpp b/base/file.cpp deleted file mode 100644 index 97cc2b27f..000000000 --- a/base/file.cpp +++ /dev/null @@ -1,522 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "android-base/file.h" - -#include <errno.h> -#include <fcntl.h> -#include <ftw.h> -#include <libgen.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sys/stat.h> -#include <sys/types.h> -#include <unistd.h> - -#include <memory> -#include <mutex> -#include <string> -#include <vector> - -#if defined(__APPLE__) -#include <mach-o/dyld.h> -#endif -#if defined(_WIN32) -#include <direct.h> -#include <windows.h> -#define O_NOFOLLOW 0 -#define OS_PATH_SEPARATOR '\\' -#else -#define OS_PATH_SEPARATOR '/' -#endif - -#include "android-base/logging.h" // and must be after windows.h for ERROR -#include "android-base/macros.h" // For TEMP_FAILURE_RETRY on Darwin. -#include "android-base/unique_fd.h" -#include "android-base/utf8.h" - -namespace { - -#ifdef _WIN32 -static int mkstemp(char* name_template, size_t size_in_chars) { - std::wstring path; - CHECK(android::base::UTF8ToWide(name_template, &path)) - << "path can't be converted to wchar: " << name_template; - if (_wmktemp_s(path.data(), path.size() + 1) != 0) { - return -1; - } - - // Use open() to match the close() that TemporaryFile's destructor does. - // Use O_BINARY to match base file APIs. - int fd = _wopen(path.c_str(), O_CREAT | O_EXCL | O_RDWR | O_BINARY, S_IRUSR | S_IWUSR); - if (fd < 0) { - return -1; - } - - std::string path_utf8; - CHECK(android::base::WideToUTF8(path, &path_utf8)) << "path can't be converted to utf8"; - CHECK(strcpy_s(name_template, size_in_chars, path_utf8.c_str()) == 0) - << "utf8 path can't be assigned back to name_template"; - - return fd; -} - -static char* mkdtemp(char* name_template, size_t size_in_chars) { - std::wstring path; - CHECK(android::base::UTF8ToWide(name_template, &path)) - << "path can't be converted to wchar: " << name_template; - - if (_wmktemp_s(path.data(), path.size() + 1) != 0) { - return nullptr; - } - - if (_wmkdir(path.c_str()) != 0) { - return nullptr; - } - - std::string path_utf8; - CHECK(android::base::WideToUTF8(path, &path_utf8)) << "path can't be converted to utf8"; - CHECK(strcpy_s(name_template, size_in_chars, path_utf8.c_str()) == 0) - << "utf8 path can't be assigned back to name_template"; - - return name_template; -} -#endif - -std::string GetSystemTempDir() { -#if defined(__ANDROID__) - const auto* tmpdir = getenv("TMPDIR"); - if (tmpdir == nullptr) tmpdir = "/data/local/tmp"; - if (access(tmpdir, R_OK | W_OK | X_OK) == 0) { - return tmpdir; - } - // Tests running in app context can't access /data/local/tmp, - // so try current directory if /data/local/tmp is not accessible. - return "."; -#elif defined(_WIN32) - wchar_t tmp_dir_w[MAX_PATH]; - DWORD result = GetTempPathW(std::size(tmp_dir_w), tmp_dir_w); // checks TMP env - CHECK_NE(result, 0ul) << "GetTempPathW failed, error: " << GetLastError(); - CHECK_LT(result, std::size(tmp_dir_w)) << "path truncated to: " << result; - - // GetTempPath() returns a path with a trailing slash, but init() - // does not expect that, so remove it. - if (tmp_dir_w[result - 1] == L'\\') { - tmp_dir_w[result - 1] = L'\0'; - } - - std::string tmp_dir; - CHECK(android::base::WideToUTF8(tmp_dir_w, &tmp_dir)) << "path can't be converted to utf8"; - - return tmp_dir; -#else - const auto* tmpdir = getenv("TMPDIR"); - if (tmpdir == nullptr) tmpdir = "/tmp"; - return tmpdir; -#endif -} - -} // namespace - -TemporaryFile::TemporaryFile() { - init(GetSystemTempDir()); -} - -TemporaryFile::TemporaryFile(const std::string& tmp_dir) { - init(tmp_dir); -} - -TemporaryFile::~TemporaryFile() { - if (fd != -1) { - close(fd); - } - if (remove_file_) { - unlink(path); - } -} - -int TemporaryFile::release() { - int result = fd; - fd = -1; - return result; -} - -void TemporaryFile::init(const std::string& tmp_dir) { - snprintf(path, sizeof(path), "%s%cTemporaryFile-XXXXXX", tmp_dir.c_str(), OS_PATH_SEPARATOR); -#if defined(_WIN32) - fd = mkstemp(path, sizeof(path)); -#else - fd = mkstemp(path); -#endif -} - -TemporaryDir::TemporaryDir() { - init(GetSystemTempDir()); -} - -TemporaryDir::~TemporaryDir() { - if (!remove_dir_and_contents_) return; - - auto callback = [](const char* child, const struct stat*, int file_type, struct FTW*) -> int { - switch (file_type) { - case FTW_D: - case FTW_DP: - case FTW_DNR: - if (rmdir(child) == -1) { - PLOG(ERROR) << "rmdir " << child; - } - break; - case FTW_NS: - default: - if (rmdir(child) != -1) break; - // FALLTHRU (for gcc, lint, pcc, etc; and following for clang) - FALLTHROUGH_INTENDED; - case FTW_F: - case FTW_SL: - case FTW_SLN: - if (unlink(child) == -1) { - PLOG(ERROR) << "unlink " << child; - } - break; - } - return 0; - }; - - nftw(path, callback, 128, FTW_DEPTH | FTW_MOUNT | FTW_PHYS); -} - -bool TemporaryDir::init(const std::string& tmp_dir) { - snprintf(path, sizeof(path), "%s%cTemporaryDir-XXXXXX", tmp_dir.c_str(), OS_PATH_SEPARATOR); -#if defined(_WIN32) - return (mkdtemp(path, sizeof(path)) != nullptr); -#else - return (mkdtemp(path) != nullptr); -#endif -} - -namespace android { -namespace base { - -// Versions of standard library APIs that support UTF-8 strings. -using namespace android::base::utf8; - -bool ReadFdToString(borrowed_fd fd, std::string* content) { - content->clear(); - - // Although original we had small files in mind, this code gets used for - // very large files too, where the std::string growth heuristics might not - // be suitable. https://code.google.com/p/android/issues/detail?id=258500. - struct stat sb; - if (fstat(fd.get(), &sb) != -1 && sb.st_size > 0) { - content->reserve(sb.st_size); - } - - char buf[BUFSIZ] __attribute__((__uninitialized__)); - ssize_t n; - while ((n = TEMP_FAILURE_RETRY(read(fd.get(), &buf[0], sizeof(buf)))) > 0) { - content->append(buf, n); - } - return (n == 0) ? true : false; -} - -bool ReadFileToString(const std::string& path, std::string* content, bool follow_symlinks) { - content->clear(); - - int flags = O_RDONLY | O_CLOEXEC | O_BINARY | (follow_symlinks ? 0 : O_NOFOLLOW); - android::base::unique_fd fd(TEMP_FAILURE_RETRY(open(path.c_str(), flags))); - if (fd == -1) { - return false; - } - return ReadFdToString(fd, content); -} - -bool WriteStringToFd(const std::string& content, borrowed_fd fd) { - const char* p = content.data(); - size_t left = content.size(); - while (left > 0) { - ssize_t n = TEMP_FAILURE_RETRY(write(fd.get(), p, left)); - if (n == -1) { - return false; - } - p += n; - left -= n; - } - return true; -} - -static bool CleanUpAfterFailedWrite(const std::string& path) { - // Something went wrong. Let's not leave a corrupt file lying around. - int saved_errno = errno; - unlink(path.c_str()); - errno = saved_errno; - return false; -} - -#if !defined(_WIN32) -bool WriteStringToFile(const std::string& content, const std::string& path, - mode_t mode, uid_t owner, gid_t group, - bool follow_symlinks) { - int flags = O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC | O_BINARY | - (follow_symlinks ? 0 : O_NOFOLLOW); - android::base::unique_fd fd(TEMP_FAILURE_RETRY(open(path.c_str(), flags, mode))); - if (fd == -1) { - PLOG(ERROR) << "android::WriteStringToFile open failed"; - return false; - } - - // We do an explicit fchmod here because we assume that the caller really - // meant what they said and doesn't want the umask-influenced mode. - if (fchmod(fd, mode) == -1) { - PLOG(ERROR) << "android::WriteStringToFile fchmod failed"; - return CleanUpAfterFailedWrite(path); - } - if (fchown(fd, owner, group) == -1) { - PLOG(ERROR) << "android::WriteStringToFile fchown failed"; - return CleanUpAfterFailedWrite(path); - } - if (!WriteStringToFd(content, fd)) { - PLOG(ERROR) << "android::WriteStringToFile write failed"; - return CleanUpAfterFailedWrite(path); - } - return true; -} -#endif - -bool WriteStringToFile(const std::string& content, const std::string& path, - bool follow_symlinks) { - int flags = O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC | O_BINARY | - (follow_symlinks ? 0 : O_NOFOLLOW); - android::base::unique_fd fd(TEMP_FAILURE_RETRY(open(path.c_str(), flags, 0666))); - if (fd == -1) { - return false; - } - return WriteStringToFd(content, fd) || CleanUpAfterFailedWrite(path); -} - -bool ReadFully(borrowed_fd fd, void* data, size_t byte_count) { - uint8_t* p = reinterpret_cast<uint8_t*>(data); - size_t remaining = byte_count; - while (remaining > 0) { - ssize_t n = TEMP_FAILURE_RETRY(read(fd.get(), p, remaining)); - if (n <= 0) return false; - p += n; - remaining -= n; - } - return true; -} - -#if defined(_WIN32) -// Windows implementation of pread. Note that this DOES move the file descriptors read position, -// but it does so atomically. -static ssize_t pread(borrowed_fd fd, void* data, size_t byte_count, off64_t offset) { - DWORD bytes_read; - OVERLAPPED overlapped; - memset(&overlapped, 0, sizeof(OVERLAPPED)); - overlapped.Offset = static_cast<DWORD>(offset); - overlapped.OffsetHigh = static_cast<DWORD>(offset >> 32); - if (!ReadFile(reinterpret_cast<HANDLE>(_get_osfhandle(fd.get())), data, - static_cast<DWORD>(byte_count), &bytes_read, &overlapped)) { - // In case someone tries to read errno (since this is masquerading as a POSIX call) - errno = EIO; - return -1; - } - return static_cast<ssize_t>(bytes_read); -} -#endif - -bool ReadFullyAtOffset(borrowed_fd fd, void* data, size_t byte_count, off64_t offset) { - uint8_t* p = reinterpret_cast<uint8_t*>(data); - while (byte_count > 0) { - ssize_t n = TEMP_FAILURE_RETRY(pread(fd.get(), p, byte_count, offset)); - if (n <= 0) return false; - p += n; - byte_count -= n; - offset += n; - } - return true; -} - -bool WriteFully(borrowed_fd fd, const void* data, size_t byte_count) { - const uint8_t* p = reinterpret_cast<const uint8_t*>(data); - size_t remaining = byte_count; - while (remaining > 0) { - ssize_t n = TEMP_FAILURE_RETRY(write(fd.get(), p, remaining)); - if (n == -1) return false; - p += n; - remaining -= n; - } - return true; -} - -bool RemoveFileIfExists(const std::string& path, std::string* err) { - struct stat st; -#if defined(_WIN32) - // TODO: Windows version can't handle symbolic links correctly. - int result = stat(path.c_str(), &st); - bool file_type_removable = (result == 0 && S_ISREG(st.st_mode)); -#else - int result = lstat(path.c_str(), &st); - bool file_type_removable = (result == 0 && (S_ISREG(st.st_mode) || S_ISLNK(st.st_mode))); -#endif - if (result == -1) { - if (errno == ENOENT || errno == ENOTDIR) return true; - if (err != nullptr) *err = strerror(errno); - return false; - } - - if (result == 0) { - if (!file_type_removable) { - if (err != nullptr) { - *err = "is not a regular file or symbolic link"; - } - return false; - } - if (unlink(path.c_str()) == -1) { - if (err != nullptr) { - *err = strerror(errno); - } - return false; - } - } - return true; -} - -#if !defined(_WIN32) -bool Readlink(const std::string& path, std::string* result) { - result->clear(); - - // Most Linux file systems (ext2 and ext4, say) limit symbolic links to - // 4095 bytes. Since we'll copy out into the string anyway, it doesn't - // waste memory to just start there. We add 1 so that we can recognize - // whether it actually fit (rather than being truncated to 4095). - std::vector<char> buf(4095 + 1); - while (true) { - ssize_t size = readlink(path.c_str(), &buf[0], buf.size()); - // Unrecoverable error? - if (size == -1) return false; - // It fit! (If size == buf.size(), it may have been truncated.) - if (static_cast<size_t>(size) < buf.size()) { - result->assign(&buf[0], size); - return true; - } - // Double our buffer and try again. - buf.resize(buf.size() * 2); - } -} -#endif - -#if !defined(_WIN32) -bool Realpath(const std::string& path, std::string* result) { - result->clear(); - - // realpath may exit with EINTR. Retry if so. - char* realpath_buf = nullptr; - do { - realpath_buf = realpath(path.c_str(), nullptr); - } while (realpath_buf == nullptr && errno == EINTR); - - if (realpath_buf == nullptr) { - return false; - } - result->assign(realpath_buf); - free(realpath_buf); - return true; -} -#endif - -std::string GetExecutablePath() { -#if defined(__linux__) - std::string path; - android::base::Readlink("/proc/self/exe", &path); - return path; -#elif defined(__APPLE__) - char path[PATH_MAX + 1]; - uint32_t path_len = sizeof(path); - int rc = _NSGetExecutablePath(path, &path_len); - if (rc < 0) { - std::unique_ptr<char> path_buf(new char[path_len]); - _NSGetExecutablePath(path_buf.get(), &path_len); - return path_buf.get(); - } - return path; -#elif defined(_WIN32) - char path[PATH_MAX + 1]; - DWORD result = GetModuleFileName(NULL, path, sizeof(path) - 1); - if (result == 0 || result == sizeof(path) - 1) return ""; - path[PATH_MAX - 1] = 0; - return path; -#else -#error unknown OS -#endif -} - -std::string GetExecutableDirectory() { - return Dirname(GetExecutablePath()); -} - -std::string Basename(const std::string& path) { - // Copy path because basename may modify the string passed in. - std::string result(path); - -#if !defined(__BIONIC__) - // Use lock because basename() may write to a process global and return a - // pointer to that. Note that this locking strategy only works if all other - // callers to basename in the process also grab this same lock, but its - // better than nothing. Bionic's basename returns a thread-local buffer. - static std::mutex& basename_lock = *new std::mutex(); - std::lock_guard<std::mutex> lock(basename_lock); -#endif - - // Note that if std::string uses copy-on-write strings, &str[0] will cause - // the copy to be made, so there is no chance of us accidentally writing to - // the storage for 'path'. - char* name = basename(&result[0]); - - // In case basename returned a pointer to a process global, copy that string - // before leaving the lock. - result.assign(name); - - return result; -} - -std::string Dirname(const std::string& path) { - // Copy path because dirname may modify the string passed in. - std::string result(path); - -#if !defined(__BIONIC__) - // Use lock because dirname() may write to a process global and return a - // pointer to that. Note that this locking strategy only works if all other - // callers to dirname in the process also grab this same lock, but its - // better than nothing. Bionic's dirname returns a thread-local buffer. - static std::mutex& dirname_lock = *new std::mutex(); - std::lock_guard<std::mutex> lock(dirname_lock); -#endif - - // Note that if std::string uses copy-on-write strings, &str[0] will cause - // the copy to be made, so there is no chance of us accidentally writing to - // the storage for 'path'. - char* parent = dirname(&result[0]); - - // In case dirname returned a pointer to a process global, copy that string - // before leaving the lock. - result.assign(parent); - - return result; -} - -} // namespace base -} // namespace android diff --git a/base/file_test.cpp b/base/file_test.cpp deleted file mode 100644 index 120228d94..000000000 --- a/base/file_test.cpp +++ /dev/null @@ -1,385 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "android-base/file.h" - -#include "android-base/utf8.h" - -#include <gtest/gtest.h> - -#include <errno.h> -#include <fcntl.h> -#include <unistd.h> -#include <wchar.h> - -#include <string> - -#if !defined(_WIN32) -#include <pwd.h> -#else -#include <windows.h> -#endif - -#include "android-base/logging.h" // and must be after windows.h for ERROR - -TEST(file, ReadFileToString_ENOENT) { - std::string s("hello"); - errno = 0; - ASSERT_FALSE(android::base::ReadFileToString("/proc/does-not-exist", &s)); - EXPECT_EQ(ENOENT, errno); - EXPECT_EQ("", s); // s was cleared. -} - -TEST(file, ReadFileToString_WriteStringToFile) { - TemporaryFile tf; - ASSERT_NE(tf.fd, -1) << tf.path; - ASSERT_TRUE(android::base::WriteStringToFile("abc", tf.path)) - << strerror(errno); - std::string s; - ASSERT_TRUE(android::base::ReadFileToString(tf.path, &s)) - << strerror(errno); - EXPECT_EQ("abc", s); -} - -// symlinks require elevated privileges on Windows. -#if !defined(_WIN32) -TEST(file, ReadFileToString_WriteStringToFile_symlink) { - TemporaryFile target, link; - ASSERT_EQ(0, unlink(link.path)); - ASSERT_EQ(0, symlink(target.path, link.path)); - ASSERT_FALSE(android::base::WriteStringToFile("foo", link.path, false)); - ASSERT_EQ(ELOOP, errno); - ASSERT_TRUE(android::base::WriteStringToFile("foo", link.path, true)); - - std::string s; - ASSERT_FALSE(android::base::ReadFileToString(link.path, &s)); - ASSERT_EQ(ELOOP, errno); - ASSERT_TRUE(android::base::ReadFileToString(link.path, &s, true)); - ASSERT_EQ("foo", s); -} -#endif - -// WriteStringToFile2 is explicitly for setting Unix permissions, which make no -// sense on Windows. -#if !defined(_WIN32) -TEST(file, WriteStringToFile2) { - TemporaryFile tf; - ASSERT_NE(tf.fd, -1) << tf.path; - ASSERT_TRUE(android::base::WriteStringToFile("abc", tf.path, 0660, - getuid(), getgid())) - << strerror(errno); - struct stat sb; - ASSERT_EQ(0, stat(tf.path, &sb)); - ASSERT_EQ(0660U, static_cast<unsigned int>(sb.st_mode & ~S_IFMT)); - ASSERT_EQ(getuid(), sb.st_uid); - ASSERT_EQ(getgid(), sb.st_gid); - std::string s; - ASSERT_TRUE(android::base::ReadFileToString(tf.path, &s)) - << strerror(errno); - EXPECT_EQ("abc", s); -} -#endif - -#if defined(_WIN32) -TEST(file, NonUnicodeCharsWindows) { - constexpr auto kMaxEnvVariableValueSize = 32767; - std::wstring old_tmp; - old_tmp.resize(kMaxEnvVariableValueSize); - old_tmp.resize(GetEnvironmentVariableW(L"TMP", old_tmp.data(), old_tmp.size())); - if (old_tmp.empty()) { - // Can't continue with empty TMP folder. - return; - } - - std::wstring new_tmp = old_tmp; - if (new_tmp.back() != L'\\') { - new_tmp.push_back(L'\\'); - } - - { - auto path(new_tmp + L"锦绣成都\\"); - _wmkdir(path.c_str()); - ASSERT_TRUE(SetEnvironmentVariableW(L"TMP", path.c_str())); - - TemporaryFile tf; - ASSERT_NE(tf.fd, -1) << tf.path; - ASSERT_TRUE(android::base::WriteStringToFd("abc", tf.fd)); - - ASSERT_EQ(0, lseek(tf.fd, 0, SEEK_SET)) << strerror(errno); - - std::string s; - ASSERT_TRUE(android::base::ReadFdToString(tf.fd, &s)) << strerror(errno); - EXPECT_EQ("abc", s); - } - { - auto path(new_tmp + L"директория с длинным именем\\"); - _wmkdir(path.c_str()); - ASSERT_TRUE(SetEnvironmentVariableW(L"TMP", path.c_str())); - - TemporaryFile tf; - ASSERT_NE(tf.fd, -1) << tf.path; - ASSERT_TRUE(android::base::WriteStringToFd("abc", tf.fd)); - - ASSERT_EQ(0, lseek(tf.fd, 0, SEEK_SET)) << strerror(errno); - - std::string s; - ASSERT_TRUE(android::base::ReadFdToString(tf.fd, &s)) << strerror(errno); - EXPECT_EQ("abc", s); - } - { - auto path(new_tmp + L"äüöß weiß\\"); - _wmkdir(path.c_str()); - ASSERT_TRUE(SetEnvironmentVariableW(L"TMP", path.c_str())); - - TemporaryFile tf; - ASSERT_NE(tf.fd, -1) << tf.path; - ASSERT_TRUE(android::base::WriteStringToFd("abc", tf.fd)); - - ASSERT_EQ(0, lseek(tf.fd, 0, SEEK_SET)) << strerror(errno); - - std::string s; - ASSERT_TRUE(android::base::ReadFdToString(tf.fd, &s)) << strerror(errno); - EXPECT_EQ("abc", s); - } - - SetEnvironmentVariableW(L"TMP", old_tmp.c_str()); -} - -TEST(file, RootDirectoryWindows) { - constexpr auto kMaxEnvVariableValueSize = 32767; - std::wstring old_tmp; - bool tmp_is_empty = false; - old_tmp.resize(kMaxEnvVariableValueSize); - old_tmp.resize(GetEnvironmentVariableW(L"TMP", old_tmp.data(), old_tmp.size())); - if (old_tmp.empty()) { - tmp_is_empty = (GetLastError() == ERROR_ENVVAR_NOT_FOUND); - } - SetEnvironmentVariableW(L"TMP", L"C:"); - - TemporaryFile tf; - ASSERT_NE(tf.fd, -1) << tf.path; - - SetEnvironmentVariableW(L"TMP", tmp_is_empty ? nullptr : old_tmp.c_str()); -} -#endif - -TEST(file, WriteStringToFd) { - TemporaryFile tf; - ASSERT_NE(tf.fd, -1) << tf.path; - ASSERT_TRUE(android::base::WriteStringToFd("abc", tf.fd)); - - ASSERT_EQ(0, lseek(tf.fd, 0, SEEK_SET)) << strerror(errno); - - std::string s; - ASSERT_TRUE(android::base::ReadFdToString(tf.fd, &s)) << strerror(errno); - EXPECT_EQ("abc", s); -} - -TEST(file, WriteFully) { - TemporaryFile tf; - ASSERT_NE(tf.fd, -1) << tf.path; - ASSERT_TRUE(android::base::WriteFully(tf.fd, "abc", 3)); - - ASSERT_EQ(0, lseek(tf.fd, 0, SEEK_SET)) << strerror(errno); - - std::string s; - s.resize(3); - ASSERT_TRUE(android::base::ReadFully(tf.fd, &s[0], s.size())) - << strerror(errno); - EXPECT_EQ("abc", s); - - ASSERT_EQ(0, lseek(tf.fd, 0, SEEK_SET)) << strerror(errno); - - s.resize(1024); - ASSERT_FALSE(android::base::ReadFully(tf.fd, &s[0], s.size())); -} - -TEST(file, RemoveFileIfExists) { - TemporaryFile tf; - ASSERT_NE(tf.fd, -1) << tf.path; - close(tf.fd); - tf.fd = -1; - std::string err; - ASSERT_TRUE(android::base::RemoveFileIfExists(tf.path, &err)) << err; - ASSERT_TRUE(android::base::RemoveFileIfExists(tf.path)); - TemporaryDir td; - ASSERT_FALSE(android::base::RemoveFileIfExists(td.path)); - ASSERT_FALSE(android::base::RemoveFileIfExists(td.path, &err)); - ASSERT_EQ("is not a regular file or symbolic link", err); -} - -TEST(file, RemoveFileIfExists_ENOTDIR) { - TemporaryFile tf; - close(tf.fd); - tf.fd = -1; - std::string err{"xxx"}; - ASSERT_TRUE(android::base::RemoveFileIfExists(std::string{tf.path} + "/abc", &err)); - ASSERT_EQ("xxx", err); -} - -#if !defined(_WIN32) -TEST(file, RemoveFileIfExists_EACCES) { - // EACCES -- one of the directories in the path has no search permission - // root can bypass permission restrictions, so drop root. - if (getuid() == 0) { - passwd* shell = getpwnam("shell"); - setgid(shell->pw_gid); - setuid(shell->pw_uid); - } - - TemporaryDir td; - TemporaryFile tf(td.path); - close(tf.fd); - tf.fd = -1; - std::string err{"xxx"}; - // Remove dir's search permission. - ASSERT_TRUE(chmod(td.path, S_IRUSR | S_IWUSR) == 0); - ASSERT_FALSE(android::base::RemoveFileIfExists(tf.path, &err)); - ASSERT_EQ("Permission denied", err); - // Set dir's search permission again. - ASSERT_TRUE(chmod(td.path, S_IRWXU) == 0); - ASSERT_TRUE(android::base::RemoveFileIfExists(tf.path, &err)); -} -#endif - -TEST(file, Readlink) { -#if !defined(_WIN32) - // Linux doesn't allow empty symbolic links. - std::string min("x"); - // ext2 and ext4 both have PAGE_SIZE limits. - // If file encryption is enabled, there's extra overhead to store the - // size of the encrypted symlink target. There's also an off-by-one - // in current kernels (and marlin/sailfish where we're seeing this - // failure are still on 3.18, far from current). http://b/33306057. - std::string max(static_cast<size_t>(4096 - 2 - 1 - 1), 'x'); - - TemporaryDir td; - std::string min_path{std::string(td.path) + "/" + "min"}; - std::string max_path{std::string(td.path) + "/" + "max"}; - - ASSERT_EQ(0, symlink(min.c_str(), min_path.c_str())); - ASSERT_EQ(0, symlink(max.c_str(), max_path.c_str())); - - std::string result; - - result = "wrong"; - ASSERT_TRUE(android::base::Readlink(min_path, &result)); - ASSERT_EQ(min, result); - - result = "wrong"; - ASSERT_TRUE(android::base::Readlink(max_path, &result)); - ASSERT_EQ(max, result); -#endif -} - -TEST(file, Realpath) { -#if !defined(_WIN32) - TemporaryDir td; - std::string basename = android::base::Basename(td.path); - std::string dir_name = android::base::Dirname(td.path); - std::string base_dir_name = android::base::Basename(dir_name); - - { - std::string path = dir_name + "/../" + base_dir_name + "/" + basename; - std::string result; - ASSERT_TRUE(android::base::Realpath(path, &result)); - ASSERT_EQ(td.path, result); - } - - { - std::string path = std::string(td.path) + "/.."; - std::string result; - ASSERT_TRUE(android::base::Realpath(path, &result)); - ASSERT_EQ(dir_name, result); - } - - { - errno = 0; - std::string path = std::string(td.path) + "/foo.noent"; - std::string result = "wrong"; - ASSERT_TRUE(!android::base::Realpath(path, &result)); - ASSERT_TRUE(result.empty()); - ASSERT_EQ(ENOENT, errno); - } -#endif -} - -TEST(file, GetExecutableDirectory) { - std::string path = android::base::GetExecutableDirectory(); - ASSERT_NE("", path); - ASSERT_NE(android::base::GetExecutablePath(), path); - ASSERT_EQ('/', path[0]); - ASSERT_NE('/', path[path.size() - 1]); -} - -TEST(file, GetExecutablePath) { - ASSERT_NE("", android::base::GetExecutablePath()); -} - -TEST(file, Basename) { - EXPECT_EQ("sh", android::base::Basename("/system/bin/sh")); - EXPECT_EQ("sh", android::base::Basename("sh")); - EXPECT_EQ("sh", android::base::Basename("/system/bin/sh/")); -} - -TEST(file, Dirname) { - EXPECT_EQ("/system/bin", android::base::Dirname("/system/bin/sh")); - EXPECT_EQ(".", android::base::Dirname("sh")); - EXPECT_EQ("/system/bin", android::base::Dirname("/system/bin/sh/")); -} - -TEST(file, ReadFileToString_capacity) { - TemporaryFile tf; - ASSERT_NE(tf.fd, -1) << tf.path; - - // For a huge file, the overhead should still be small. - std::string s; - size_t size = 16 * 1024 * 1024; - ASSERT_TRUE(android::base::WriteStringToFile(std::string(size, 'x'), tf.path)); - ASSERT_TRUE(android::base::ReadFileToString(tf.path, &s)); - EXPECT_EQ(size, s.size()); - EXPECT_LT(s.capacity(), size + 16); - - // Even for weird badly-aligned sizes. - size += 12345; - ASSERT_TRUE(android::base::WriteStringToFile(std::string(size, 'x'), tf.path)); - ASSERT_TRUE(android::base::ReadFileToString(tf.path, &s)); - EXPECT_EQ(size, s.size()); - EXPECT_LT(s.capacity(), size + 16); - - // We'll shrink an enormous string if you read a small file into it. - size = 64; - ASSERT_TRUE(android::base::WriteStringToFile(std::string(size, 'x'), tf.path)); - ASSERT_TRUE(android::base::ReadFileToString(tf.path, &s)); - EXPECT_EQ(size, s.size()); - EXPECT_LT(s.capacity(), size + 16); -} - -TEST(file, ReadFileToString_capacity_0) { - TemporaryFile tf; - ASSERT_NE(tf.fd, -1) << tf.path; - - // Because /proc reports its files as zero-length, we don't actually trust - // any file that claims to be zero-length. Rather than add increasingly - // complex heuristics for shrinking the passed-in string in that case, we - // currently leave it alone. - std::string s; - size_t initial_capacity = s.capacity(); - ASSERT_TRUE(android::base::WriteStringToFile("", tf.path)); - ASSERT_TRUE(android::base::ReadFileToString(tf.path, &s)); - EXPECT_EQ(0U, s.size()); - EXPECT_EQ(initial_capacity, s.capacity()); -} diff --git a/base/format_benchmark.cpp b/base/format_benchmark.cpp deleted file mode 100644 index 9590b2365..000000000 --- a/base/format_benchmark.cpp +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "android-base/format.h" - -#include <limits> - -#include <benchmark/benchmark.h> - -#include "android-base/stringprintf.h" - -using android::base::StringPrintf; - -static void BenchmarkFormatInt(benchmark::State& state) { - for (auto _ : state) { - benchmark::DoNotOptimize(fmt::format("{} {} {}", 42, std::numeric_limits<int>::min(), - std::numeric_limits<int>::max())); - } -} - -BENCHMARK(BenchmarkFormatInt); - -static void BenchmarkStringPrintfInt(benchmark::State& state) { - for (auto _ : state) { - benchmark::DoNotOptimize(StringPrintf("%d %d %d", 42, std::numeric_limits<int>::min(), - std::numeric_limits<int>::max())); - } -} - -BENCHMARK(BenchmarkStringPrintfInt); - -static void BenchmarkFormatFloat(benchmark::State& state) { - for (auto _ : state) { - benchmark::DoNotOptimize(fmt::format("{} {} {}", 42.42, std::numeric_limits<float>::min(), - std::numeric_limits<float>::max())); - } -} - -BENCHMARK(BenchmarkFormatFloat); - -static void BenchmarkStringPrintfFloat(benchmark::State& state) { - for (auto _ : state) { - benchmark::DoNotOptimize(StringPrintf("%f %f %f", 42.42, std::numeric_limits<float>::min(), - std::numeric_limits<float>::max())); - } -} - -BENCHMARK(BenchmarkStringPrintfFloat); - -static void BenchmarkFormatStrings(benchmark::State& state) { - for (auto _ : state) { - benchmark::DoNotOptimize(fmt::format("{} hello there {}", "hi,", "!!")); - } -} - -BENCHMARK(BenchmarkFormatStrings); - -static void BenchmarkStringPrintfStrings(benchmark::State& state) { - for (auto _ : state) { - benchmark::DoNotOptimize(StringPrintf("%s hello there %s", "hi,", "!!")); - } -} - -BENCHMARK(BenchmarkStringPrintfStrings); - -// Run the benchmark -BENCHMARK_MAIN(); diff --git a/base/include/android-base/chrono_utils.h b/base/include/android-base/chrono_utils.h deleted file mode 100644 index 11fcf710e..000000000 --- a/base/include/android-base/chrono_utils.h +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (C) 2017 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include <chrono> -#include <sstream> - -#if __cplusplus > 201103L && !defined(__WIN32) // C++14 -using namespace std::chrono_literals; -#endif - -namespace android { -namespace base { - -// A std::chrono clock based on CLOCK_BOOTTIME. -class boot_clock { - public: - typedef std::chrono::nanoseconds duration; - typedef std::chrono::time_point<boot_clock, duration> time_point; - - static time_point now(); -}; - -class Timer { - public: - Timer() : start_(boot_clock::now()) {} - - std::chrono::milliseconds duration() const { - return std::chrono::duration_cast<std::chrono::milliseconds>(boot_clock::now() - start_); - } - - private: - boot_clock::time_point start_; -}; - -std::ostream& operator<<(std::ostream& os, const Timer& t); - -} // namespace base -} // namespace android diff --git a/base/include/android-base/cmsg.h b/base/include/android-base/cmsg.h deleted file mode 100644 index e4197b109..000000000 --- a/base/include/android-base/cmsg.h +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include <sys/stat.h> -#include <sys/types.h> - -#include <type_traits> -#include <vector> - -#include <android-base/collections.h> -#include <android-base/macros.h> -#include <android-base/unique_fd.h> - -namespace android { -namespace base { - -#if !defined(_WIN32) - -// Helpers for sending and receiving file descriptors across Unix domain sockets. -// -// The cmsg(3) API is very hard to get right, with multiple landmines that can -// lead to death. Almost all of the uses of cmsg in Android make at least one of -// the following mistakes: -// -// - not aligning the cmsg buffer -// - leaking fds if more fds are received than expected -// - blindly dereferencing CMSG_DATA without checking the header -// - using CMSG_SPACE instead of CMSG_LEN for .cmsg_len -// - using CMSG_LEN instead of CMSG_SPACE for .msg_controllen -// - using a length specified in number of fds instead of bytes -// -// These functions wrap the hard-to-use cmsg API with an easier to use abstraction. - -// Send file descriptors across a Unix domain socket. -// -// Note that the write can return short if the socket type is SOCK_STREAM. When -// this happens, file descriptors are still sent to the other end, but with -// truncated data. For this reason, using SOCK_SEQPACKET or SOCK_DGRAM is recommended. -ssize_t SendFileDescriptorVector(borrowed_fd sock, const void* data, size_t len, - const std::vector<int>& fds); - -// Receive file descriptors from a Unix domain socket. -// -// If more FDs (or bytes, for datagram sockets) are received than expected, -// -1 is returned with errno set to EMSGSIZE, and all received FDs are thrown away. -ssize_t ReceiveFileDescriptorVector(borrowed_fd sock, void* data, size_t len, size_t max_fds, - std::vector<android::base::unique_fd>* fds); - -// Helper for SendFileDescriptorVector that constructs a std::vector for you, e.g.: -// SendFileDescriptors(sock, "foo", 3, std::move(fd1), std::move(fd2)) -template <typename... Args> -ssize_t SendFileDescriptors(borrowed_fd sock, const void* data, size_t len, Args&&... sent_fds) { - // Do not allow implicit conversion to int: people might try to do something along the lines of: - // SendFileDescriptors(..., std::move(a_unique_fd)) - // and be surprised when the unique_fd isn't closed afterwards. - AssertType<int>(std::forward<Args>(sent_fds)...); - std::vector<int> fds; - Append(fds, std::forward<Args>(sent_fds)...); - return SendFileDescriptorVector(sock, data, len, fds); -} - -// Helper for ReceiveFileDescriptorVector that receives an exact number of file descriptors. -// If more file descriptors are received than requested, -1 is returned with errno set to EMSGSIZE. -// If fewer file descriptors are received than requested, -1 is returned with errno set to ENOMSG. -// In both cases, all arguments are cleared and any received FDs are thrown away. -template <typename... Args> -ssize_t ReceiveFileDescriptors(borrowed_fd sock, void* data, size_t len, Args&&... received_fds) { - std::vector<unique_fd*> fds; - Append(fds, std::forward<Args>(received_fds)...); - - std::vector<unique_fd> result; - ssize_t rc = ReceiveFileDescriptorVector(sock, data, len, fds.size(), &result); - if (rc == -1 || result.size() != fds.size()) { - int err = rc == -1 ? errno : ENOMSG; - for (unique_fd* fd : fds) { - fd->reset(); - } - errno = err; - return -1; - } - - for (size_t i = 0; i < fds.size(); ++i) { - *fds[i] = std::move(result[i]); - } - return rc; -} - -#endif - -} // namespace base -} // namespace android diff --git a/base/include/android-base/collections.h b/base/include/android-base/collections.h deleted file mode 100644 index be0683ab9..000000000 --- a/base/include/android-base/collections.h +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include <utility> - -namespace android { -namespace base { - -// Helpers for converting a variadic template parameter pack to a homogeneous collection. -// Parameters must be implictly convertible to the contained type (including via move/copy ctors). -// -// Use as follows: -// -// template <typename... Args> -// std::vector<int> CreateVector(Args&&... args) { -// std::vector<int> result; -// Append(result, std::forward<Args>(args)...); -// return result; -// } -template <typename CollectionType, typename T> -void Append(CollectionType& collection, T&& arg) { - collection.push_back(std::forward<T>(arg)); -} - -template <typename CollectionType, typename T, typename... Args> -void Append(CollectionType& collection, T&& arg, Args&&... args) { - collection.push_back(std::forward<T>(arg)); - return Append(collection, std::forward<Args>(args)...); -} - -// Assert that all of the arguments in a variadic template parameter pack are of a given type -// after std::decay. -template <typename T, typename Arg, typename... Args> -void AssertType(Arg&&) { - static_assert(std::is_same<T, typename std::decay<Arg>::type>::value); -} - -template <typename T, typename Arg, typename... Args> -void AssertType(Arg&&, Args&&... args) { - static_assert(std::is_same<T, typename std::decay<Arg>::type>::value); - AssertType<T>(std::forward<Args>(args)...); -} - -} // namespace base -} // namespace android diff --git a/base/include/android-base/endian.h b/base/include/android-base/endian.h deleted file mode 100644 index 8fa6365ee..000000000 --- a/base/include/android-base/endian.h +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (C) 2017 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -/* A cross-platform equivalent of bionic's <sys/endian.h>. */ - -/* For __BIONIC__ and __GLIBC__ */ -#include <sys/cdefs.h> - -#if defined(__BIONIC__) - -#include <sys/endian.h> - -#elif defined(__GLIBC__) - -/* glibc's <endian.h> is like bionic's <sys/endian.h>. */ -#include <endian.h> - -/* glibc keeps htons and htonl in <netinet/in.h>. */ -#include <netinet/in.h> - -/* glibc doesn't have the 64-bit variants. */ -#define htonq(x) htobe64(x) -#define ntohq(x) be64toh(x) - -/* glibc has different names to BSD for these. */ -#define betoh16(x) be16toh(x) -#define betoh32(x) be32toh(x) -#define betoh64(x) be64toh(x) -#define letoh16(x) le16toh(x) -#define letoh32(x) le32toh(x) -#define letoh64(x) le64toh(x) - -#else - -#if defined(__APPLE__) -/* macOS has some of the basics. */ -#include <sys/_endian.h> -#else -/* Windows has some of the basics as well. */ -#include <sys/param.h> -#include <winsock2.h> -/* winsock2.h *must* be included before the following four macros. */ -#define htons(x) __builtin_bswap16(x) -#define htonl(x) __builtin_bswap32(x) -#define ntohs(x) __builtin_bswap16(x) -#define ntohl(x) __builtin_bswap32(x) -#endif - -/* Neither macOS nor Windows have the rest. */ - -#define __LITTLE_ENDIAN 1234 -#define __BIG_ENDIAN 4321 -#define __BYTE_ORDER __LITTLE_ENDIAN - -#define htonq(x) __builtin_bswap64(x) - -#define ntohq(x) __builtin_bswap64(x) - -#define htobe16(x) __builtin_bswap16(x) -#define htobe32(x) __builtin_bswap32(x) -#define htobe64(x) __builtin_bswap64(x) - -#define betoh16(x) __builtin_bswap16(x) -#define betoh32(x) __builtin_bswap32(x) -#define betoh64(x) __builtin_bswap64(x) - -#define htole16(x) (x) -#define htole32(x) (x) -#define htole64(x) (x) - -#define letoh16(x) (x) -#define letoh32(x) (x) -#define letoh64(x) (x) - -#define be16toh(x) __builtin_bswap16(x) -#define be32toh(x) __builtin_bswap32(x) -#define be64toh(x) __builtin_bswap64(x) - -#define le16toh(x) (x) -#define le32toh(x) (x) -#define le64toh(x) (x) - -#endif diff --git a/base/include/android-base/errors.h b/base/include/android-base/errors.h deleted file mode 100644 index 06f29fcac..000000000 --- a/base/include/android-base/errors.h +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (C) 2016 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Portable error handling functions. This is only necessary for host-side -// code that needs to be cross-platform; code that is only run on Unix should -// just use errno and strerror() for simplicity. -// -// There is some complexity since Windows has (at least) three different error -// numbers, not all of which share the same type: -// * errno: for C runtime errors. -// * GetLastError(): Windows non-socket errors. -// * WSAGetLastError(): Windows socket errors. -// errno can be passed to strerror() on all platforms, but the other two require -// special handling to get the error string. Refer to Microsoft documentation -// to determine which error code to check for each function. - -#pragma once - -#include <string> - -namespace android { -namespace base { - -// Returns a string describing the given system error code. |error_code| must -// be errno on Unix or GetLastError()/WSAGetLastError() on Windows. Passing -// errno on Windows has undefined behavior. -std::string SystemErrorCodeToString(int error_code); - -} // namespace base -} // namespace android diff --git a/base/include/android-base/expected.h b/base/include/android-base/expected.h deleted file mode 100644 index 9470344e8..000000000 --- a/base/include/android-base/expected.h +++ /dev/null @@ -1,744 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include <algorithm> -#include <initializer_list> -#include <type_traits> -#include <utility> -#include <variant> - -// android::base::expected is an Android implementation of the std::expected -// proposal. -// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2018/p0323r7.html -// -// Usage: -// using android::base::expected; -// using android::base::unexpected; -// -// expected<double,std::string> safe_divide(double i, double j) { -// if (j == 0) return unexpected("divide by zero"); -// else return i / j; -// } -// -// void test() { -// auto q = safe_divide(10, 0); -// if (q) { printf("%f\n", q.value()); } -// else { printf("%s\n", q.error().c_str()); } -// } -// -// When the proposal becomes part of the standard and is implemented by -// libcxx, this will be removed and android::base::expected will be -// type alias to std::expected. -// - -namespace android { -namespace base { - -// Synopsis -template<class T, class E> - class expected; - -template<class E> - class unexpected; -template<class E> - unexpected(E) -> unexpected<E>; - -template<class E> - class bad_expected_access; - -template<> - class bad_expected_access<void>; - -struct unexpect_t { - explicit unexpect_t() = default; -}; -inline constexpr unexpect_t unexpect{}; - -// macros for SFINAE -#define _ENABLE_IF(...) \ - , std::enable_if_t<(__VA_ARGS__)>* = nullptr - -// Define NODISCARD_EXPECTED to prevent expected<T,E> from being -// ignored when used as a return value. This is off by default. -#ifdef NODISCARD_EXPECTED -#define _NODISCARD_ [[nodiscard]] -#else -#define _NODISCARD_ -#endif - -// Class expected -template<class T, class E> -class _NODISCARD_ expected { - public: - using value_type = T; - using error_type = E; - using unexpected_type = unexpected<E>; - - template<class U> - using rebind = expected<U, error_type>; - - // constructors - constexpr expected() = default; - constexpr expected(const expected& rhs) = default; - constexpr expected(expected&& rhs) noexcept = default; - - template<class U, class G _ENABLE_IF( - std::is_constructible_v<T, const U&> && - std::is_constructible_v<E, const G&> && - !std::is_constructible_v<T, expected<U, G>&> && - !std::is_constructible_v<T, expected<U, G>&&> && - !std::is_constructible_v<T, const expected<U, G>&> && - !std::is_constructible_v<T, const expected<U, G>&&> && - !std::is_convertible_v<expected<U, G>&, T> && - !std::is_convertible_v<expected<U, G>&&, T> && - !std::is_convertible_v<const expected<U, G>&, T> && - !std::is_convertible_v<const expected<U, G>&&, T> && - !(!std::is_convertible_v<const U&, T> || - !std::is_convertible_v<const G&, E>) /* non-explicit */ - )> - // NOLINTNEXTLINE(google-explicit-constructor) - constexpr expected(const expected<U, G>& rhs) { - if (rhs.has_value()) var_ = rhs.value(); - else var_ = unexpected(rhs.error()); - } - - template<class U, class G _ENABLE_IF( - std::is_constructible_v<T, const U&> && - std::is_constructible_v<E, const G&> && - !std::is_constructible_v<T, expected<U, G>&> && - !std::is_constructible_v<T, expected<U, G>&&> && - !std::is_constructible_v<T, const expected<U, G>&> && - !std::is_constructible_v<T, const expected<U, G>&&> && - !std::is_convertible_v<expected<U, G>&, T> && - !std::is_convertible_v<expected<U, G>&&, T> && - !std::is_convertible_v<const expected<U, G>&, T> && - !std::is_convertible_v<const expected<U, G>&&, T> && - (!std::is_convertible_v<const U&, T> || - !std::is_convertible_v<const G&, E>) /* explicit */ - )> - constexpr explicit expected(const expected<U, G>& rhs) { - if (rhs.has_value()) var_ = rhs.value(); - else var_ = unexpected(rhs.error()); - } - - template<class U, class G _ENABLE_IF( - std::is_constructible_v<T, const U&> && - std::is_constructible_v<E, const G&> && - !std::is_constructible_v<T, expected<U, G>&> && - !std::is_constructible_v<T, expected<U, G>&&> && - !std::is_constructible_v<T, const expected<U, G>&> && - !std::is_constructible_v<T, const expected<U, G>&&> && - !std::is_convertible_v<expected<U, G>&, T> && - !std::is_convertible_v<expected<U, G>&&, T> && - !std::is_convertible_v<const expected<U, G>&, T> && - !std::is_convertible_v<const expected<U, G>&&, T> && - !(!std::is_convertible_v<const U&, T> || - !std::is_convertible_v<const G&, E>) /* non-explicit */ - )> - // NOLINTNEXTLINE(google-explicit-constructor) - constexpr expected(expected<U, G>&& rhs) { - if (rhs.has_value()) var_ = std::move(rhs.value()); - else var_ = unexpected(std::move(rhs.error())); - } - - template<class U, class G _ENABLE_IF( - std::is_constructible_v<T, const U&> && - std::is_constructible_v<E, const G&> && - !std::is_constructible_v<T, expected<U, G>&> && - !std::is_constructible_v<T, expected<U, G>&&> && - !std::is_constructible_v<T, const expected<U, G>&> && - !std::is_constructible_v<T, const expected<U, G>&&> && - !std::is_convertible_v<expected<U, G>&, T> && - !std::is_convertible_v<expected<U, G>&&, T> && - !std::is_convertible_v<const expected<U, G>&, T> && - !std::is_convertible_v<const expected<U, G>&&, T> && - (!std::is_convertible_v<const U&, T> || - !std::is_convertible_v<const G&, E>) /* explicit */ - )> - constexpr explicit expected(expected<U, G>&& rhs) { - if (rhs.has_value()) var_ = std::move(rhs.value()); - else var_ = unexpected(std::move(rhs.error())); - } - - template <class U = T _ENABLE_IF( - std::is_constructible_v<T, U&&> && - !std::is_same_v<std::remove_cv_t<std::remove_reference_t<U>>, std::in_place_t> && - !std::is_same_v<expected<T, E>, std::remove_cv_t<std::remove_reference_t<U>>> && - !std::is_same_v<unexpected<E>, std::remove_cv_t<std::remove_reference_t<U>>> && - std::is_convertible_v<U&&, T> /* non-explicit */ - )> - // NOLINTNEXTLINE(google-explicit-constructor,bugprone-forwarding-reference-overload) - constexpr expected(U&& v) : var_(std::in_place_index<0>, std::forward<U>(v)) {} - - template <class U = T _ENABLE_IF( - std::is_constructible_v<T, U&&> && - !std::is_same_v<std::remove_cv_t<std::remove_reference_t<U>>, std::in_place_t> && - !std::is_same_v<expected<T, E>, std::remove_cv_t<std::remove_reference_t<U>>> && - !std::is_same_v<unexpected<E>, std::remove_cv_t<std::remove_reference_t<U>>> && - !std::is_convertible_v<U&&, T> /* explicit */ - )> - // NOLINTNEXTLINE(bugprone-forwarding-reference-overload) - constexpr explicit expected(U&& v) : var_(std::in_place_index<0>, T(std::forward<U>(v))) {} - - template<class G = E _ENABLE_IF( - std::is_constructible_v<E, const G&> && - std::is_convertible_v<const G&, E> /* non-explicit */ - )> - // NOLINTNEXTLINE(google-explicit-constructor) - constexpr expected(const unexpected<G>& e) - : var_(std::in_place_index<1>, e.value()) {} - - template<class G = E _ENABLE_IF( - std::is_constructible_v<E, const G&> && - !std::is_convertible_v<const G&, E> /* explicit */ - )> - constexpr explicit expected(const unexpected<G>& e) - : var_(std::in_place_index<1>, E(e.value())) {} - - template<class G = E _ENABLE_IF( - std::is_constructible_v<E, G&&> && - std::is_convertible_v<G&&, E> /* non-explicit */ - )> - // NOLINTNEXTLINE(google-explicit-constructor) - constexpr expected(unexpected<G>&& e) - : var_(std::in_place_index<1>, std::move(e.value())) {} - - template<class G = E _ENABLE_IF( - std::is_constructible_v<E, G&&> && - !std::is_convertible_v<G&&, E> /* explicit */ - )> - constexpr explicit expected(unexpected<G>&& e) - : var_(std::in_place_index<1>, E(std::move(e.value()))) {} - - template<class... Args _ENABLE_IF( - std::is_constructible_v<T, Args&&...> - )> - constexpr explicit expected(std::in_place_t, Args&&... args) - : var_(std::in_place_index<0>, std::forward<Args>(args)...) {} - - template<class U, class... Args _ENABLE_IF( - std::is_constructible_v<T, std::initializer_list<U>&, Args...> - )> - constexpr explicit expected(std::in_place_t, std::initializer_list<U> il, Args&&... args) - : var_(std::in_place_index<0>, il, std::forward<Args>(args)...) {} - - template<class... Args _ENABLE_IF( - std::is_constructible_v<E, Args...> - )> - constexpr explicit expected(unexpect_t, Args&&... args) - : var_(unexpected_type(std::forward<Args>(args)...)) {} - - template<class U, class... Args _ENABLE_IF( - std::is_constructible_v<E, std::initializer_list<U>&, Args...> - )> - constexpr explicit expected(unexpect_t, std::initializer_list<U> il, Args&&... args) - : var_(unexpected_type(il, std::forward<Args>(args)...)) {} - - // destructor - ~expected() = default; - - // assignment - // Note: SFNAIE doesn't work here because assignment operator should be - // non-template. We could workaround this by defining a templated parent class - // having the assignment operator. This incomplete implementation however - // doesn't allow us to copy assign expected<T,E> even when T is non-copy - // assignable. The copy assignment will fail by the underlying std::variant - // anyway though the error message won't be clear. - expected& operator=(const expected& rhs) = default; - - // Note for SFNAIE above applies to here as well - expected& operator=(expected&& rhs) noexcept( - std::is_nothrow_move_assignable_v<T>&& std::is_nothrow_move_assignable_v<E>) = default; - - template <class U = T _ENABLE_IF( - !std::is_void_v<T> && - !std::is_same_v<expected<T, E>, std::remove_cv_t<std::remove_reference_t<U>>> && - !std::conjunction_v<std::is_scalar<T>, std::is_same<T, std::decay_t<U>>> && - std::is_constructible_v<T, U> && std::is_assignable_v<T&, U> && - std::is_nothrow_move_constructible_v<E>)> - expected& operator=(U&& rhs) { - var_ = T(std::forward<U>(rhs)); - return *this; - } - - template<class G = E> - expected& operator=(const unexpected<G>& rhs) { - var_ = rhs; - return *this; - } - - template<class G = E _ENABLE_IF( - std::is_nothrow_move_constructible_v<G> && - std::is_move_assignable_v<G> - )> - expected& operator=(unexpected<G>&& rhs) { - var_ = std::move(rhs); - return *this; - } - - // modifiers - template<class... Args _ENABLE_IF( - std::is_nothrow_constructible_v<T, Args...> - )> - T& emplace(Args&&... args) { - expected(std::in_place, std::forward<Args>(args)...).swap(*this); - return value(); - } - - template<class U, class... Args _ENABLE_IF( - std::is_nothrow_constructible_v<T, std::initializer_list<U>&, Args...> - )> - T& emplace(std::initializer_list<U> il, Args&&... args) { - expected(std::in_place, il, std::forward<Args>(args)...).swap(*this); - return value(); - } - - // swap - template<typename U = T, typename = std::enable_if_t<( - std::is_swappable_v<U> && - std::is_swappable_v<E> && - (std::is_move_constructible_v<U> || - std::is_move_constructible_v<E>))>> - void swap(expected& rhs) noexcept( - std::is_nothrow_move_constructible_v<T> && - std::is_nothrow_swappable_v<T> && - std::is_nothrow_move_constructible_v<E> && - std::is_nothrow_swappable_v<E>) { - var_.swap(rhs.var_); - } - - // observers - constexpr const T* operator->() const { return std::addressof(value()); } - constexpr T* operator->() { return std::addressof(value()); } - constexpr const T& operator*() const& { return value(); } - constexpr T& operator*() & { return value(); } - constexpr const T&& operator*() const&& { return std::move(std::get<T>(var_)); } - constexpr T&& operator*() && { return std::move(std::get<T>(var_)); } - - constexpr explicit operator bool() const noexcept { return has_value(); } - constexpr bool has_value() const noexcept { return var_.index() == 0; } - constexpr bool ok() const noexcept { return has_value(); } - - constexpr const T& value() const& { return std::get<T>(var_); } - constexpr T& value() & { return std::get<T>(var_); } - constexpr const T&& value() const&& { return std::move(std::get<T>(var_)); } - constexpr T&& value() && { return std::move(std::get<T>(var_)); } - - constexpr const E& error() const& { return std::get<unexpected_type>(var_).value(); } - constexpr E& error() & { return std::get<unexpected_type>(var_).value(); } - constexpr const E&& error() const&& { return std::move(std::get<unexpected_type>(var_)).value(); } - constexpr E&& error() && { return std::move(std::get<unexpected_type>(var_)).value(); } - - template<class U _ENABLE_IF( - std::is_copy_constructible_v<T> && - std::is_convertible_v<U, T> - )> - constexpr T value_or(U&& v) const& { - if (has_value()) return value(); - else return static_cast<T>(std::forward<U>(v)); - } - - template<class U _ENABLE_IF( - std::is_move_constructible_v<T> && - std::is_convertible_v<U, T> - )> - constexpr T value_or(U&& v) && { - if (has_value()) return std::move(value()); - else return static_cast<T>(std::forward<U>(v)); - } - - // expected equality operators - template<class T1, class E1, class T2, class E2> - friend constexpr bool operator==(const expected<T1, E1>& x, const expected<T2, E2>& y); - template<class T1, class E1, class T2, class E2> - friend constexpr bool operator!=(const expected<T1, E1>& x, const expected<T2, E2>& y); - - // Comparison with unexpected<E> - template<class T1, class E1, class E2> - friend constexpr bool operator==(const expected<T1, E1>&, const unexpected<E2>&); - template<class T1, class E1, class E2> - friend constexpr bool operator==(const unexpected<E2>&, const expected<T1, E1>&); - template<class T1, class E1, class E2> - friend constexpr bool operator!=(const expected<T1, E1>&, const unexpected<E2>&); - template<class T1, class E1, class E2> - friend constexpr bool operator!=(const unexpected<E2>&, const expected<T1, E1>&); - - // Specialized algorithms - template<class T1, class E1> - friend void swap(expected<T1, E1>&, expected<T1, E1>&) noexcept; - - private: - std::variant<value_type, unexpected_type> var_; -}; - -template<class T1, class E1, class T2, class E2> -constexpr bool operator==(const expected<T1, E1>& x, const expected<T2, E2>& y) { - if (x.has_value() != y.has_value()) return false; - if (!x.has_value()) return x.error() == y.error(); - return *x == *y; -} - -template<class T1, class E1, class T2, class E2> -constexpr bool operator!=(const expected<T1, E1>& x, const expected<T2, E2>& y) { - return !(x == y); -} - -// Comparison with unexpected<E> -template<class T1, class E1, class E2> -constexpr bool operator==(const expected<T1, E1>& x, const unexpected<E2>& y) { - return !x.has_value() && (x.error() == y.value()); -} -template<class T1, class E1, class E2> -constexpr bool operator==(const unexpected<E2>& x, const expected<T1, E1>& y) { - return !y.has_value() && (x.value() == y.error()); -} -template<class T1, class E1, class E2> -constexpr bool operator!=(const expected<T1, E1>& x, const unexpected<E2>& y) { - return x.has_value() || (x.error() != y.value()); -} -template<class T1, class E1, class E2> -constexpr bool operator!=(const unexpected<E2>& x, const expected<T1, E1>& y) { - return y.has_value() || (x.value() != y.error()); -} - -template<class E> -class _NODISCARD_ expected<void, E> { - public: - using value_type = void; - using error_type = E; - using unexpected_type = unexpected<E>; - - // constructors - constexpr expected() = default; - constexpr expected(const expected& rhs) = default; - constexpr expected(expected&& rhs) noexcept = default; - - template<class U, class G _ENABLE_IF( - std::is_void_v<U> && - std::is_convertible_v<const G&, E> /* non-explicit */ - )> - // NOLINTNEXTLINE(google-explicit-constructor) - constexpr expected(const expected<U, G>& rhs) { - if (!rhs.has_value()) var_ = unexpected(rhs.error()); - } - - template<class U, class G _ENABLE_IF( - std::is_void_v<U> && - !std::is_convertible_v<const G&, E> /* explicit */ - )> - constexpr explicit expected(const expected<U, G>& rhs) { - if (!rhs.has_value()) var_ = unexpected(rhs.error()); - } - - template<class U, class G _ENABLE_IF( - std::is_void_v<U> && - std::is_convertible_v<const G&&, E> /* non-explicit */ - )> - // NOLINTNEXTLINE(google-explicit-constructor) - constexpr expected(expected<U, G>&& rhs) { - if (!rhs.has_value()) var_ = unexpected(std::move(rhs.error())); - } - - template<class U, class G _ENABLE_IF( - std::is_void_v<U> && - !std::is_convertible_v<const G&&, E> /* explicit */ - )> - constexpr explicit expected(expected<U, G>&& rhs) { - if (!rhs.has_value()) var_ = unexpected(std::move(rhs.error())); - } - - template<class G = E _ENABLE_IF( - std::is_constructible_v<E, const G&> && - std::is_convertible_v<const G&, E> /* non-explicit */ - )> - // NOLINTNEXTLINE(google-explicit-constructor) - constexpr expected(const unexpected<G>& e) - : var_(std::in_place_index<1>, e.value()) {} - - template<class G = E _ENABLE_IF( - std::is_constructible_v<E, const G&> && - !std::is_convertible_v<const G&, E> /* explicit */ - )> - constexpr explicit expected(const unexpected<G>& e) - : var_(std::in_place_index<1>, E(e.value())) {} - - template<class G = E _ENABLE_IF( - std::is_constructible_v<E, G&&> && - std::is_convertible_v<G&&, E> /* non-explicit */ - )> - // NOLINTNEXTLINE(google-explicit-constructor) - constexpr expected(unexpected<G>&& e) - : var_(std::in_place_index<1>, std::move(e.value())) {} - - template<class G = E _ENABLE_IF( - std::is_constructible_v<E, G&&> && - !std::is_convertible_v<G&&, E> /* explicit */ - )> - constexpr explicit expected(unexpected<G>&& e) - : var_(std::in_place_index<1>, E(std::move(e.value()))) {} - - template<class... Args _ENABLE_IF( - sizeof...(Args) == 0 - )> - constexpr explicit expected(std::in_place_t, Args&&...) {} - - template<class... Args _ENABLE_IF( - std::is_constructible_v<E, Args...> - )> - constexpr explicit expected(unexpect_t, Args&&... args) - : var_(unexpected_type(std::forward<Args>(args)...)) {} - - template<class U, class... Args _ENABLE_IF( - std::is_constructible_v<E, std::initializer_list<U>&, Args...> - )> - constexpr explicit expected(unexpect_t, std::initializer_list<U> il, Args&&... args) - : var_(unexpected_type(il, std::forward<Args>(args)...)) {} - - // destructor - ~expected() = default; - - // assignment - // Note: SFNAIE doesn't work here because assignment operator should be - // non-template. We could workaround this by defining a templated parent class - // having the assignment operator. This incomplete implementation however - // doesn't allow us to copy assign expected<T,E> even when T is non-copy - // assignable. The copy assignment will fail by the underlying std::variant - // anyway though the error message won't be clear. - expected& operator=(const expected& rhs) = default; - - // Note for SFNAIE above applies to here as well - expected& operator=(expected&& rhs) noexcept(std::is_nothrow_move_assignable_v<E>) = default; - - template<class G = E> - expected& operator=(const unexpected<G>& rhs) { - var_ = rhs; - return *this; - } - - template<class G = E _ENABLE_IF( - std::is_nothrow_move_constructible_v<G> && - std::is_move_assignable_v<G> - )> - expected& operator=(unexpected<G>&& rhs) { - var_ = std::move(rhs); - return *this; - } - - // modifiers - void emplace() { - var_ = std::monostate(); - } - - // swap - template<typename = std::enable_if_t< - std::is_swappable_v<E>> - > - void swap(expected& rhs) noexcept(std::is_nothrow_move_constructible_v<E>) { - var_.swap(rhs.var_); - } - - // observers - constexpr explicit operator bool() const noexcept { return has_value(); } - constexpr bool has_value() const noexcept { return var_.index() == 0; } - constexpr bool ok() const noexcept { return has_value(); } - - constexpr void value() const& { if (!has_value()) std::get<0>(var_); } - - constexpr const E& error() const& { return std::get<unexpected_type>(var_).value(); } - constexpr E& error() & { return std::get<unexpected_type>(var_).value(); } - constexpr const E&& error() const&& { return std::move(std::get<unexpected_type>(var_)).value(); } - constexpr E&& error() && { return std::move(std::get<unexpected_type>(var_)).value(); } - - // expected equality operators - template<class E1, class E2> - friend constexpr bool operator==(const expected<void, E1>& x, const expected<void, E2>& y); - - // Specialized algorithms - template<class T1, class E1> - friend void swap(expected<T1, E1>&, expected<T1, E1>&) noexcept; - - private: - std::variant<std::monostate, unexpected_type> var_; -}; - -template<class E1, class E2> -constexpr bool operator==(const expected<void, E1>& x, const expected<void, E2>& y) { - if (x.has_value() != y.has_value()) return false; - if (!x.has_value()) return x.error() == y.error(); - return true; -} - -template<class T1, class E1, class E2> -constexpr bool operator==(const expected<T1, E1>& x, const expected<void, E2>& y) { - if (x.has_value() != y.has_value()) return false; - if (!x.has_value()) return x.error() == y.error(); - return false; -} - -template<class E1, class T2, class E2> -constexpr bool operator==(const expected<void, E1>& x, const expected<T2, E2>& y) { - if (x.has_value() != y.has_value()) return false; - if (!x.has_value()) return x.error() == y.error(); - return false; -} - -template<class E> -class unexpected { - public: - // constructors - constexpr unexpected(const unexpected&) = default; - constexpr unexpected(unexpected&&) noexcept(std::is_nothrow_move_constructible_v<E>) = default; - - template <class Err = E _ENABLE_IF( - std::is_constructible_v<E, Err> && - !std::is_same_v<std::remove_cv_t<std::remove_reference_t<E>>, std::in_place_t> && - !std::is_same_v<std::remove_cv_t<std::remove_reference_t<E>>, unexpected>)> - // NOLINTNEXTLINE(google-explicit-constructor,bugprone-forwarding-reference-overload) - constexpr unexpected(Err&& e) : val_(std::forward<Err>(e)) {} - - template<class U, class... Args _ENABLE_IF( - std::is_constructible_v<E, std::initializer_list<U>&, Args...> - )> - constexpr explicit unexpected(std::in_place_t, std::initializer_list<U> il, Args&&... args) - : val_(il, std::forward<Args>(args)...) {} - - template<class Err _ENABLE_IF( - std::is_constructible_v<E, Err> && - !std::is_constructible_v<E, unexpected<Err>&> && - !std::is_constructible_v<E, unexpected<Err>> && - !std::is_constructible_v<E, const unexpected<Err>&> && - !std::is_constructible_v<E, const unexpected<Err>> && - !std::is_convertible_v<unexpected<Err>&, E> && - !std::is_convertible_v<unexpected<Err>, E> && - !std::is_convertible_v<const unexpected<Err>&, E> && - !std::is_convertible_v<const unexpected<Err>, E> && - std::is_convertible_v<Err, E> /* non-explicit */ - )> - // NOLINTNEXTLINE(google-explicit-constructor) - constexpr unexpected(const unexpected<Err>& rhs) - : val_(rhs.value()) {} - - template<class Err _ENABLE_IF( - std::is_constructible_v<E, Err> && - !std::is_constructible_v<E, unexpected<Err>&> && - !std::is_constructible_v<E, unexpected<Err>> && - !std::is_constructible_v<E, const unexpected<Err>&> && - !std::is_constructible_v<E, const unexpected<Err>> && - !std::is_convertible_v<unexpected<Err>&, E> && - !std::is_convertible_v<unexpected<Err>, E> && - !std::is_convertible_v<const unexpected<Err>&, E> && - !std::is_convertible_v<const unexpected<Err>, E> && - !std::is_convertible_v<Err, E> /* explicit */ - )> - constexpr explicit unexpected(const unexpected<Err>& rhs) - : val_(E(rhs.value())) {} - - template<class Err _ENABLE_IF( - std::is_constructible_v<E, Err> && - !std::is_constructible_v<E, unexpected<Err>&> && - !std::is_constructible_v<E, unexpected<Err>> && - !std::is_constructible_v<E, const unexpected<Err>&> && - !std::is_constructible_v<E, const unexpected<Err>> && - !std::is_convertible_v<unexpected<Err>&, E> && - !std::is_convertible_v<unexpected<Err>, E> && - !std::is_convertible_v<const unexpected<Err>&, E> && - !std::is_convertible_v<const unexpected<Err>, E> && - std::is_convertible_v<Err, E> /* non-explicit */ - )> - // NOLINTNEXTLINE(google-explicit-constructor) - constexpr unexpected(unexpected<Err>&& rhs) - : val_(std::move(rhs.value())) {} - - template<class Err _ENABLE_IF( - std::is_constructible_v<E, Err> && - !std::is_constructible_v<E, unexpected<Err>&> && - !std::is_constructible_v<E, unexpected<Err>> && - !std::is_constructible_v<E, const unexpected<Err>&> && - !std::is_constructible_v<E, const unexpected<Err>> && - !std::is_convertible_v<unexpected<Err>&, E> && - !std::is_convertible_v<unexpected<Err>, E> && - !std::is_convertible_v<const unexpected<Err>&, E> && - !std::is_convertible_v<const unexpected<Err>, E> && - !std::is_convertible_v<Err, E> /* explicit */ - )> - constexpr explicit unexpected(unexpected<Err>&& rhs) - : val_(E(std::move(rhs.value()))) {} - - // assignment - constexpr unexpected& operator=(const unexpected&) = default; - constexpr unexpected& operator=(unexpected&&) noexcept(std::is_nothrow_move_assignable_v<E>) = - default; - template<class Err = E> - constexpr unexpected& operator=(const unexpected<Err>& rhs) { - val_ = rhs.value(); - return *this; - } - template<class Err = E> - constexpr unexpected& operator=(unexpected<Err>&& rhs) { - val_ = std::forward<E>(rhs.value()); - return *this; - } - - // observer - constexpr const E& value() const& noexcept { return val_; } - constexpr E& value() & noexcept { return val_; } - constexpr const E&& value() const&& noexcept { return std::move(val_); } - constexpr E&& value() && noexcept { return std::move(val_); } - - void swap(unexpected& other) noexcept(std::is_nothrow_swappable_v<E>) { - std::swap(val_, other.val_); - } - - template<class E1, class E2> - friend constexpr bool - operator==(const unexpected<E1>& e1, const unexpected<E2>& e2); - template<class E1, class E2> - friend constexpr bool - operator!=(const unexpected<E1>& e1, const unexpected<E2>& e2); - - template<class E1> - friend void swap(unexpected<E1>& x, unexpected<E1>& y) noexcept(noexcept(x.swap(y))); - - private: - E val_; -}; - -template<class E1, class E2> -constexpr bool -operator==(const unexpected<E1>& e1, const unexpected<E2>& e2) { - return e1.value() == e2.value(); -} - -template<class E1, class E2> -constexpr bool -operator!=(const unexpected<E1>& e1, const unexpected<E2>& e2) { - return e1.value() != e2.value(); -} - -template<class E1> -void swap(unexpected<E1>& x, unexpected<E1>& y) noexcept(noexcept(x.swap(y))) { - x.swap(y); -} - -// TODO: bad_expected_access class - -#undef _ENABLE_IF -#undef _NODISCARD_ - -} // namespace base -} // namespace android diff --git a/base/include/android-base/file.h b/base/include/android-base/file.h deleted file mode 100644 index c62256277..000000000 --- a/base/include/android-base/file.h +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include <sys/stat.h> -#include <sys/types.h> - -#include <string> - -#include "android-base/macros.h" -#include "android-base/off64_t.h" -#include "android-base/unique_fd.h" - -#if !defined(_WIN32) && !defined(O_BINARY) -/** Windows needs O_BINARY, but Unix never mangles line endings. */ -#define O_BINARY 0 -#endif - -#if defined(_WIN32) && !defined(O_CLOEXEC) -/** Windows has O_CLOEXEC but calls it O_NOINHERIT for some reason. */ -#define O_CLOEXEC O_NOINHERIT -#endif - -class TemporaryFile { - public: - TemporaryFile(); - explicit TemporaryFile(const std::string& tmp_dir); - ~TemporaryFile(); - - // Release the ownership of fd, caller is reponsible for closing the - // fd or stream properly. - int release(); - // Don't remove the temporary file in the destructor. - void DoNotRemove() { remove_file_ = false; } - - int fd; - char path[1024]; - - private: - void init(const std::string& tmp_dir); - - bool remove_file_ = true; - - DISALLOW_COPY_AND_ASSIGN(TemporaryFile); -}; - -class TemporaryDir { - public: - TemporaryDir(); - ~TemporaryDir(); - // Don't remove the temporary dir in the destructor. - void DoNotRemove() { remove_dir_and_contents_ = false; } - - char path[1024]; - - private: - bool init(const std::string& tmp_dir); - - bool remove_dir_and_contents_ = true; - - DISALLOW_COPY_AND_ASSIGN(TemporaryDir); -}; - -namespace android { -namespace base { - -bool ReadFdToString(borrowed_fd fd, std::string* content); -bool ReadFileToString(const std::string& path, std::string* content, - bool follow_symlinks = false); - -bool WriteStringToFile(const std::string& content, const std::string& path, - bool follow_symlinks = false); -bool WriteStringToFd(const std::string& content, borrowed_fd fd); - -#if !defined(_WIN32) -bool WriteStringToFile(const std::string& content, const std::string& path, - mode_t mode, uid_t owner, gid_t group, - bool follow_symlinks = false); -#endif - -bool ReadFully(borrowed_fd fd, void* data, size_t byte_count); - -// Reads `byte_count` bytes from the file descriptor at the specified offset. -// Returns false if there was an IO error or EOF was reached before reading `byte_count` bytes. -// -// NOTE: On Linux/Mac, this function wraps pread, which provides atomic read support without -// modifying the read pointer of the file descriptor. On Windows, however, the read pointer does -// get modified. This means that ReadFullyAtOffset can be used concurrently with other calls to the -// same function, but concurrently seeking or reading incrementally can lead to unexpected -// behavior. -bool ReadFullyAtOffset(borrowed_fd fd, void* data, size_t byte_count, off64_t offset); - -bool WriteFully(borrowed_fd fd, const void* data, size_t byte_count); - -bool RemoveFileIfExists(const std::string& path, std::string* err = nullptr); - -#if !defined(_WIN32) -bool Realpath(const std::string& path, std::string* result); -bool Readlink(const std::string& path, std::string* result); -#endif - -std::string GetExecutablePath(); -std::string GetExecutableDirectory(); - -// Like the regular basename and dirname, but thread-safe on all -// platforms and capable of correctly handling exotic Windows paths. -std::string Basename(const std::string& path); -std::string Dirname(const std::string& path); - -} // namespace base -} // namespace android diff --git a/base/include/android-base/format.h b/base/include/android-base/format.h deleted file mode 100644 index 330040d8a..000000000 --- a/base/include/android-base/format.h +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -// We include fmtlib here as an alias, since libbase will have fmtlib statically linked already. -// It is accessed through its normal fmt:: namespace. -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wshadow" -#include <fmt/chrono.h> -#pragma clang diagnostic pop -#include <fmt/core.h> -#include <fmt/format.h> -#include <fmt/ostream.h> -#include <fmt/printf.h> diff --git a/base/include/android-base/logging.h b/base/include/android-base/logging.h deleted file mode 100644 index 26827fb83..000000000 --- a/base/include/android-base/logging.h +++ /dev/null @@ -1,464 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -// -// Google-style C++ logging. -// - -// This header provides a C++ stream interface to logging. -// -// To log: -// -// LOG(INFO) << "Some text; " << some_value; -// -// Replace `INFO` with any severity from `enum LogSeverity`. -// -// To log the result of a failed function and include the string -// representation of `errno` at the end: -// -// PLOG(ERROR) << "Write failed"; -// -// The output will be something like `Write failed: I/O error`. -// Remember this as 'P' as in perror(3). -// -// To output your own types, simply implement operator<< as normal. -// -// By default, output goes to logcat on Android and stderr on the host. -// A process can use `SetLogger` to decide where all logging goes. -// Implementations are provided for logcat, stderr, and dmesg. -// -// By default, the process' name is used as the log tag. -// Code can choose a specific log tag by defining LOG_TAG -// before including this header. - -// This header also provides assertions: -// -// CHECK(must_be_true); -// CHECK_EQ(a, b) << z_is_interesting_too; - -// NOTE: For Windows, you must include logging.h after windows.h to allow the -// following code to suppress the evil ERROR macro: -#ifdef _WIN32 -// windows.h includes wingdi.h which defines an evil macro ERROR. -#ifdef ERROR -#undef ERROR -#endif -#endif - -#include <functional> -#include <memory> -#include <ostream> - -#include "android-base/errno_restorer.h" -#include "android-base/macros.h" - -// Note: DO NOT USE DIRECTLY. Use LOG_TAG instead. -#ifdef _LOG_TAG_INTERNAL -#error "_LOG_TAG_INTERNAL must not be defined" -#endif -#ifdef LOG_TAG -#define _LOG_TAG_INTERNAL LOG_TAG -#else -#define _LOG_TAG_INTERNAL nullptr -#endif - -namespace android { -namespace base { - -enum LogSeverity { - VERBOSE, - DEBUG, - INFO, - WARNING, - ERROR, - FATAL_WITHOUT_ABORT, // For loggability tests, this is considered identical to FATAL. - FATAL, -}; - -enum LogId { - DEFAULT, - MAIN, - SYSTEM, - RADIO, - CRASH, -}; - -using LogFunction = std::function<void(LogId, LogSeverity, const char*, const char*, - unsigned int, const char*)>; -using AbortFunction = std::function<void(const char*)>; - -// Loggers for use with InitLogging/SetLogger. - -// Log to the kernel log (dmesg). -void KernelLogger(LogId, LogSeverity, const char*, const char*, unsigned int, const char*); -// Log to stderr in the full logcat format (with pid/tid/time/tag details). -void StderrLogger(LogId, LogSeverity, const char*, const char*, unsigned int, const char*); -// Log just the message to stdout/stderr (without pid/tid/time/tag details). -// The choice of stdout versus stderr is based on the severity. -// Errors are also prefixed by the program name (as with err(3)/error(3)). -// Useful for replacing printf(3)/perror(3)/err(3)/error(3) in command-line tools. -void StdioLogger(LogId, LogSeverity, const char*, const char*, unsigned int, const char*); - -void DefaultAborter(const char* abort_message); - -void SetDefaultTag(const std::string& tag); - -// The LogdLogger sends chunks of up to ~4000 bytes at a time to logd. It does not prevent other -// threads from writing to logd between sending each chunk, so other threads may interleave their -// messages. If preventing interleaving is required, then a custom logger that takes a lock before -// calling this logger should be provided. -class LogdLogger { - public: - explicit LogdLogger(LogId default_log_id = android::base::MAIN); - - void operator()(LogId, LogSeverity, const char* tag, const char* file, - unsigned int line, const char* message); - - private: - LogId default_log_id_; -}; - -// Configure logging based on ANDROID_LOG_TAGS environment variable. -// We need to parse a string that looks like -// -// *:v jdwp:d dalvikvm:d dalvikvm-gc:i dalvikvmi:i -// -// The tag (or '*' for the global level) comes first, followed by a colon and a -// letter indicating the minimum priority level we're expected to log. This can -// be used to reveal or conceal logs with specific tags. -#ifdef __ANDROID__ -#define INIT_LOGGING_DEFAULT_LOGGER LogdLogger() -#else -#define INIT_LOGGING_DEFAULT_LOGGER StderrLogger -#endif -void InitLogging(char* argv[], - LogFunction&& logger = INIT_LOGGING_DEFAULT_LOGGER, - AbortFunction&& aborter = DefaultAborter); -#undef INIT_LOGGING_DEFAULT_LOGGER - -// Replace the current logger. -void SetLogger(LogFunction&& logger); - -// Replace the current aborter. -void SetAborter(AbortFunction&& aborter); - -// A helper macro that produces an expression that accepts both a qualified name and an -// unqualified name for a LogSeverity, and returns a LogSeverity value. -// Note: DO NOT USE DIRECTLY. This is an implementation detail. -#define SEVERITY_LAMBDA(severity) ([&]() { \ - using ::android::base::VERBOSE; \ - using ::android::base::DEBUG; \ - using ::android::base::INFO; \ - using ::android::base::WARNING; \ - using ::android::base::ERROR; \ - using ::android::base::FATAL_WITHOUT_ABORT; \ - using ::android::base::FATAL; \ - return (severity); }()) - -#ifdef __clang_analyzer__ -// Clang's static analyzer does not see the conditional statement inside -// LogMessage's destructor that will abort on FATAL severity. -#define ABORT_AFTER_LOG_FATAL for (;; abort()) - -struct LogAbortAfterFullExpr { - ~LogAbortAfterFullExpr() __attribute__((noreturn)) { abort(); } - explicit operator bool() const { return false; } -}; -// Provides an expression that evaluates to the truthiness of `x`, automatically -// aborting if `c` is true. -#define ABORT_AFTER_LOG_EXPR_IF(c, x) (((c) && ::android::base::LogAbortAfterFullExpr()) || (x)) -// Note to the static analyzer that we always execute FATAL logs in practice. -#define MUST_LOG_MESSAGE(severity) (SEVERITY_LAMBDA(severity) == ::android::base::FATAL) -#else -#define ABORT_AFTER_LOG_FATAL -#define ABORT_AFTER_LOG_EXPR_IF(c, x) (x) -#define MUST_LOG_MESSAGE(severity) false -#endif -#define ABORT_AFTER_LOG_FATAL_EXPR(x) ABORT_AFTER_LOG_EXPR_IF(true, x) - -// Defines whether the given severity will be logged or silently swallowed. -#define WOULD_LOG(severity) \ - (UNLIKELY(::android::base::ShouldLog(SEVERITY_LAMBDA(severity), _LOG_TAG_INTERNAL)) || \ - MUST_LOG_MESSAGE(severity)) - -// Get an ostream that can be used for logging at the given severity and to the default -// destination. -// -// Notes: -// 1) This will not check whether the severity is high enough. One should use WOULD_LOG to filter -// usage manually. -// 2) This does not save and restore errno. -#define LOG_STREAM(severity) \ - ::android::base::LogMessage(__FILE__, __LINE__, SEVERITY_LAMBDA(severity), _LOG_TAG_INTERNAL, \ - -1) \ - .stream() - -// Logs a message to logcat on Android otherwise to stderr. If the severity is -// FATAL it also causes an abort. For example: -// -// LOG(FATAL) << "We didn't expect to reach here"; -#define LOG(severity) LOGGING_PREAMBLE(severity) && LOG_STREAM(severity) - -// Checks if we want to log something, and sets up appropriate RAII objects if -// so. -// Note: DO NOT USE DIRECTLY. This is an implementation detail. -#define LOGGING_PREAMBLE(severity) \ - (WOULD_LOG(severity) && \ - ABORT_AFTER_LOG_EXPR_IF((SEVERITY_LAMBDA(severity)) == ::android::base::FATAL, true) && \ - ::android::base::ErrnoRestorer()) - -// A variant of LOG that also logs the current errno value. To be used when -// library calls fail. -#define PLOG(severity) \ - LOGGING_PREAMBLE(severity) && \ - ::android::base::LogMessage(__FILE__, __LINE__, SEVERITY_LAMBDA(severity), \ - _LOG_TAG_INTERNAL, errno) \ - .stream() - -// Marker that code is yet to be implemented. -#define UNIMPLEMENTED(level) \ - LOG(level) << __PRETTY_FUNCTION__ << " unimplemented " - -// Check whether condition x holds and LOG(FATAL) if not. The value of the -// expression x is only evaluated once. Extra logging can be appended using << -// after. For example: -// -// CHECK(false == true) results in a log message of -// "Check failed: false == true". -#define CHECK(x) \ - LIKELY((x)) || ABORT_AFTER_LOG_FATAL_EXPR(false) || \ - ::android::base::LogMessage(__FILE__, __LINE__, ::android::base::FATAL, _LOG_TAG_INTERNAL, \ - -1) \ - .stream() \ - << "Check failed: " #x << " " - -// clang-format off -// Helper for CHECK_xx(x,y) macros. -#define CHECK_OP(LHS, RHS, OP) \ - for (auto _values = ::android::base::MakeEagerEvaluator(LHS, RHS); \ - UNLIKELY(!(_values.lhs OP _values.rhs)); \ - /* empty */) \ - ABORT_AFTER_LOG_FATAL \ - ::android::base::LogMessage(__FILE__, __LINE__, ::android::base::FATAL, _LOG_TAG_INTERNAL, -1) \ - .stream() \ - << "Check failed: " << #LHS << " " << #OP << " " << #RHS << " (" #LHS "=" << _values.lhs \ - << ", " #RHS "=" << _values.rhs << ") " -// clang-format on - -// Check whether a condition holds between x and y, LOG(FATAL) if not. The value -// of the expressions x and y is evaluated once. Extra logging can be appended -// using << after. For example: -// -// CHECK_NE(0 == 1, false) results in -// "Check failed: false != false (0==1=false, false=false) ". -#define CHECK_EQ(x, y) CHECK_OP(x, y, == ) -#define CHECK_NE(x, y) CHECK_OP(x, y, != ) -#define CHECK_LE(x, y) CHECK_OP(x, y, <= ) -#define CHECK_LT(x, y) CHECK_OP(x, y, < ) -#define CHECK_GE(x, y) CHECK_OP(x, y, >= ) -#define CHECK_GT(x, y) CHECK_OP(x, y, > ) - -// clang-format off -// Helper for CHECK_STRxx(s1,s2) macros. -#define CHECK_STROP(s1, s2, sense) \ - while (UNLIKELY((strcmp(s1, s2) == 0) != (sense))) \ - ABORT_AFTER_LOG_FATAL \ - ::android::base::LogMessage(__FILE__, __LINE__, ::android::base::FATAL, \ - _LOG_TAG_INTERNAL, -1) \ - .stream() \ - << "Check failed: " << "\"" << (s1) << "\"" \ - << ((sense) ? " == " : " != ") << "\"" << (s2) << "\"" -// clang-format on - -// Check for string (const char*) equality between s1 and s2, LOG(FATAL) if not. -#define CHECK_STREQ(s1, s2) CHECK_STROP(s1, s2, true) -#define CHECK_STRNE(s1, s2) CHECK_STROP(s1, s2, false) - -// Perform the pthread function call(args), LOG(FATAL) on error. -#define CHECK_PTHREAD_CALL(call, args, what) \ - do { \ - int rc = call args; \ - if (rc != 0) { \ - errno = rc; \ - ABORT_AFTER_LOG_FATAL \ - PLOG(FATAL) << #call << " failed for " << (what); \ - } \ - } while (false) - -// CHECK that can be used in a constexpr function. For example: -// -// constexpr int half(int n) { -// return -// DCHECK_CONSTEXPR(n >= 0, , 0) -// CHECK_CONSTEXPR((n & 1) == 0), -// << "Extra debugging output: n = " << n, 0) -// n / 2; -// } -#define CHECK_CONSTEXPR(x, out, dummy) \ - (UNLIKELY(!(x))) \ - ? (LOG(FATAL) << "Check failed: " << #x out, dummy) \ - : - -// DCHECKs are debug variants of CHECKs only enabled in debug builds. Generally -// CHECK should be used unless profiling identifies a CHECK as being in -// performance critical code. -#if defined(NDEBUG) && !defined(__clang_analyzer__) -static constexpr bool kEnableDChecks = false; -#else -static constexpr bool kEnableDChecks = true; -#endif - -#define DCHECK(x) \ - if (::android::base::kEnableDChecks) CHECK(x) -#define DCHECK_EQ(x, y) \ - if (::android::base::kEnableDChecks) CHECK_EQ(x, y) -#define DCHECK_NE(x, y) \ - if (::android::base::kEnableDChecks) CHECK_NE(x, y) -#define DCHECK_LE(x, y) \ - if (::android::base::kEnableDChecks) CHECK_LE(x, y) -#define DCHECK_LT(x, y) \ - if (::android::base::kEnableDChecks) CHECK_LT(x, y) -#define DCHECK_GE(x, y) \ - if (::android::base::kEnableDChecks) CHECK_GE(x, y) -#define DCHECK_GT(x, y) \ - if (::android::base::kEnableDChecks) CHECK_GT(x, y) -#define DCHECK_STREQ(s1, s2) \ - if (::android::base::kEnableDChecks) CHECK_STREQ(s1, s2) -#define DCHECK_STRNE(s1, s2) \ - if (::android::base::kEnableDChecks) CHECK_STRNE(s1, s2) -#if defined(NDEBUG) && !defined(__clang_analyzer__) -#define DCHECK_CONSTEXPR(x, out, dummy) -#else -#define DCHECK_CONSTEXPR(x, out, dummy) CHECK_CONSTEXPR(x, out, dummy) -#endif - -// Temporary class created to evaluate the LHS and RHS, used with -// MakeEagerEvaluator to infer the types of LHS and RHS. -template <typename LHS, typename RHS> -struct EagerEvaluator { - constexpr EagerEvaluator(LHS l, RHS r) : lhs(l), rhs(r) { - } - LHS lhs; - RHS rhs; -}; - -// Helper function for CHECK_xx. -template <typename LHS, typename RHS> -constexpr EagerEvaluator<LHS, RHS> MakeEagerEvaluator(LHS lhs, RHS rhs) { - return EagerEvaluator<LHS, RHS>(lhs, rhs); -} - -// Explicitly instantiate EagerEvalue for pointers so that char*s aren't treated -// as strings. To compare strings use CHECK_STREQ and CHECK_STRNE. We rely on -// signed/unsigned warnings to protect you against combinations not explicitly -// listed below. -#define EAGER_PTR_EVALUATOR(T1, T2) \ - template <> \ - struct EagerEvaluator<T1, T2> { \ - EagerEvaluator(T1 l, T2 r) \ - : lhs(reinterpret_cast<const void*>(l)), \ - rhs(reinterpret_cast<const void*>(r)) { \ - } \ - const void* lhs; \ - const void* rhs; \ - } -EAGER_PTR_EVALUATOR(const char*, const char*); -EAGER_PTR_EVALUATOR(const char*, char*); -EAGER_PTR_EVALUATOR(char*, const char*); -EAGER_PTR_EVALUATOR(char*, char*); -EAGER_PTR_EVALUATOR(const unsigned char*, const unsigned char*); -EAGER_PTR_EVALUATOR(const unsigned char*, unsigned char*); -EAGER_PTR_EVALUATOR(unsigned char*, const unsigned char*); -EAGER_PTR_EVALUATOR(unsigned char*, unsigned char*); -EAGER_PTR_EVALUATOR(const signed char*, const signed char*); -EAGER_PTR_EVALUATOR(const signed char*, signed char*); -EAGER_PTR_EVALUATOR(signed char*, const signed char*); -EAGER_PTR_EVALUATOR(signed char*, signed char*); - -// Data for the log message, not stored in LogMessage to avoid increasing the -// stack size. -class LogMessageData; - -// A LogMessage is a temporarily scoped object used by LOG and the unlikely part -// of a CHECK. The destructor will abort if the severity is FATAL. -class LogMessage { - public: - // LogId has been deprecated, but this constructor must exist for prebuilts. - LogMessage(const char* file, unsigned int line, LogId, LogSeverity severity, const char* tag, - int error); - LogMessage(const char* file, unsigned int line, LogSeverity severity, const char* tag, int error); - - ~LogMessage(); - - // Returns the stream associated with the message, the LogMessage performs - // output when it goes out of scope. - std::ostream& stream(); - - // The routine that performs the actual logging. - static void LogLine(const char* file, unsigned int line, LogSeverity severity, const char* tag, - const char* msg); - - private: - const std::unique_ptr<LogMessageData> data_; - - DISALLOW_COPY_AND_ASSIGN(LogMessage); -}; - -// Get the minimum severity level for logging. -LogSeverity GetMinimumLogSeverity(); - -// Set the minimum severity level for logging, returning the old severity. -LogSeverity SetMinimumLogSeverity(LogSeverity new_severity); - -// Return whether or not a log message with the associated tag should be logged. -bool ShouldLog(LogSeverity severity, const char* tag); - -// Allows to temporarily change the minimum severity level for logging. -class ScopedLogSeverity { - public: - explicit ScopedLogSeverity(LogSeverity level); - ~ScopedLogSeverity(); - - private: - LogSeverity old_; -}; - -} // namespace base -} // namespace android - -namespace std { // NOLINT(cert-dcl58-cpp) - -// Emit a warning of ostream<< with std::string*. The intention was most likely to print *string. -// -// Note: for this to work, we need to have this in a namespace. -// Note: using a pragma because "-Wgcc-compat" (included in "-Weverything") complains about -// diagnose_if. -// Note: to print the pointer, use "<< static_cast<const void*>(string_pointer)" instead. -// Note: a not-recommended alternative is to let Clang ignore the warning by adding -// -Wno-user-defined-warnings to CPPFLAGS. -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wgcc-compat" -#define OSTREAM_STRING_POINTER_USAGE_WARNING \ - __attribute__((diagnose_if(true, "Unexpected logging of string pointer", "warning"))) -inline OSTREAM_STRING_POINTER_USAGE_WARNING -std::ostream& operator<<(std::ostream& stream, const std::string* string_pointer) { - return stream << static_cast<const void*>(string_pointer); -} -#pragma clang diagnostic pop - -} // namespace std diff --git a/base/include/android-base/macros.h b/base/include/android-base/macros.h deleted file mode 100644 index 5abf5141f..000000000 --- a/base/include/android-base/macros.h +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include <stddef.h> // for size_t -#include <unistd.h> // for TEMP_FAILURE_RETRY - -#include <utility> - -// bionic and glibc both have TEMP_FAILURE_RETRY, but eg Mac OS' libc doesn't. -#ifndef TEMP_FAILURE_RETRY -#define TEMP_FAILURE_RETRY(exp) \ - ({ \ - decltype(exp) _rc; \ - do { \ - _rc = (exp); \ - } while (_rc == -1 && errno == EINTR); \ - _rc; \ - }) -#endif - -// A macro to disallow the copy constructor and operator= functions -// This must be placed in the private: declarations for a class. -// -// For disallowing only assign or copy, delete the relevant operator or -// constructor, for example: -// void operator=(const TypeName&) = delete; -// Note, that most uses of DISALLOW_ASSIGN and DISALLOW_COPY are broken -// semantically, one should either use disallow both or neither. Try to -// avoid these in new code. -#define DISALLOW_COPY_AND_ASSIGN(TypeName) \ - TypeName(const TypeName&) = delete; \ - void operator=(const TypeName&) = delete - -// A macro to disallow all the implicit constructors, namely the -// default constructor, copy constructor and operator= functions. -// -// This should be used in the private: declarations for a class -// that wants to prevent anyone from instantiating it. This is -// especially useful for classes containing only static methods. -#define DISALLOW_IMPLICIT_CONSTRUCTORS(TypeName) \ - TypeName() = delete; \ - DISALLOW_COPY_AND_ASSIGN(TypeName) - -// The arraysize(arr) macro returns the # of elements in an array arr. -// The expression is a compile-time constant, and therefore can be -// used in defining new arrays, for example. If you use arraysize on -// a pointer by mistake, you will get a compile-time error. -// -// One caveat is that arraysize() doesn't accept any array of an -// anonymous type or a type defined inside a function. In these rare -// cases, you have to use the unsafe ARRAYSIZE_UNSAFE() macro below. This is -// due to a limitation in C++'s template system. The limitation might -// eventually be removed, but it hasn't happened yet. - -// This template function declaration is used in defining arraysize. -// Note that the function doesn't need an implementation, as we only -// use its type. -template <typename T, size_t N> -char(&ArraySizeHelper(T(&array)[N]))[N]; // NOLINT(readability/casting) - -#define arraysize(array) (sizeof(ArraySizeHelper(array))) - -#define SIZEOF_MEMBER(t, f) sizeof(std::declval<t>().f) - -// Changing this definition will cause you a lot of pain. A majority of -// vendor code defines LIKELY and UNLIKELY this way, and includes -// this header through an indirect path. -#define LIKELY( exp ) (__builtin_expect( (exp) != 0, true )) -#define UNLIKELY( exp ) (__builtin_expect( (exp) != 0, false )) - -#define WARN_UNUSED __attribute__((warn_unused_result)) - -// A deprecated function to call to create a false use of the parameter, for -// example: -// int foo(int x) { UNUSED(x); return 10; } -// to avoid compiler warnings. Going forward we prefer ATTRIBUTE_UNUSED. -template <typename... T> -void UNUSED(const T&...) { -} - -// An attribute to place on a parameter to a function, for example: -// int foo(int x ATTRIBUTE_UNUSED) { return 10; } -// to avoid compiler warnings. -#define ATTRIBUTE_UNUSED __attribute__((__unused__)) - -// The FALLTHROUGH_INTENDED macro can be used to annotate implicit fall-through -// between switch labels: -// switch (x) { -// case 40: -// case 41: -// if (truth_is_out_there) { -// ++x; -// FALLTHROUGH_INTENDED; // Use instead of/along with annotations in -// // comments. -// } else { -// return x; -// } -// case 42: -// ... -// -// As shown in the example above, the FALLTHROUGH_INTENDED macro should be -// followed by a semicolon. It is designed to mimic control-flow statements -// like 'break;', so it can be placed in most places where 'break;' can, but -// only if there are no statements on the execution path between it and the -// next switch label. -// -// When compiled with clang, the FALLTHROUGH_INTENDED macro is expanded to -// [[clang::fallthrough]] attribute, which is analysed when performing switch -// labels fall-through diagnostic ('-Wimplicit-fallthrough'). See clang -// documentation on language extensions for details: -// http://clang.llvm.org/docs/LanguageExtensions.html#clang__fallthrough -// -// When used with unsupported compilers, the FALLTHROUGH_INTENDED macro has no -// effect on diagnostics. -// -// In either case this macro has no effect on runtime behavior and performance -// of code. -#ifndef FALLTHROUGH_INTENDED -#define FALLTHROUGH_INTENDED [[clang::fallthrough]] // NOLINT -#endif - -// Current ABI string -#if defined(__arm__) -#define ABI_STRING "arm" -#elif defined(__aarch64__) -#define ABI_STRING "arm64" -#elif defined(__i386__) -#define ABI_STRING "x86" -#elif defined(__x86_64__) -#define ABI_STRING "x86_64" -#elif defined(__mips__) && !defined(__LP64__) -#define ABI_STRING "mips" -#elif defined(__mips__) && defined(__LP64__) -#define ABI_STRING "mips64" -#endif diff --git a/base/include/android-base/mapped_file.h b/base/include/android-base/mapped_file.h deleted file mode 100644 index 8c37f4305..000000000 --- a/base/include/android-base/mapped_file.h +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include <sys/types.h> - -#include <memory> - -#include "android-base/macros.h" -#include "android-base/off64_t.h" -#include "android-base/unique_fd.h" - -#if defined(_WIN32) -#include <windows.h> -#define PROT_READ 1 -#define PROT_WRITE 2 -using os_handle = HANDLE; -#else -#include <sys/mman.h> -using os_handle = int; -#endif - -namespace android { -namespace base { - -/** - * A region of a file mapped into memory (for grepping: also known as MmapFile or file mapping). - */ -class MappedFile { - public: - /** - * Creates a new mapping of the file pointed to by `fd`. Unlike the underlying OS primitives, - * `offset` does not need to be page-aligned. If `PROT_WRITE` is set in `prot`, the mapping - * will be writable, otherwise it will be read-only. Mappings are always `MAP_SHARED`. - */ - static std::unique_ptr<MappedFile> FromFd(borrowed_fd fd, off64_t offset, size_t length, - int prot); - - /** - * Same thing, but using the raw OS file handle instead of a CRT wrapper. - */ - static std::unique_ptr<MappedFile> FromOsHandle(os_handle h, off64_t offset, size_t length, - int prot); - - /** - * Removes the mapping. - */ - ~MappedFile(); - - /** - * Not copyable but movable. - */ - MappedFile(MappedFile&& other); - MappedFile& operator=(MappedFile&& other); - - char* data() const { return base_ + offset_; } - size_t size() const { return size_; } - - private: - DISALLOW_IMPLICIT_CONSTRUCTORS(MappedFile); - - void Close(); - - char* base_; - size_t size_; - - size_t offset_; - -#if defined(_WIN32) - MappedFile(char* base, size_t size, size_t offset, HANDLE handle) - : base_(base), size_(size), offset_(offset), handle_(handle) {} - HANDLE handle_; -#else - MappedFile(char* base, size_t size, size_t offset) : base_(base), size_(size), offset_(offset) {} -#endif -}; - -} // namespace base -} // namespace android diff --git a/base/include/android-base/memory.h b/base/include/android-base/memory.h deleted file mode 100644 index 0277a034e..000000000 --- a/base/include/android-base/memory.h +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -namespace android { -namespace base { - -// Use memcpy for access to unaligned data on targets with alignment -// restrictions. The compiler will generate appropriate code to access these -// structures without generating alignment exceptions. -template <typename T> -static inline T get_unaligned(const void* address) { - T result; - memcpy(&result, address, sizeof(T)); - return result; -} - -template <typename T> -static inline void put_unaligned(void* address, T v) { - memcpy(address, &v, sizeof(T)); -} - -} // namespace base -} // namespace android diff --git a/base/include/android-base/no_destructor.h b/base/include/android-base/no_destructor.h deleted file mode 100644 index ce0dc9f2a..000000000 --- a/base/include/android-base/no_destructor.h +++ /dev/null @@ -1,94 +0,0 @@ -#pragma once - -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <utility> - -#include "android-base/macros.h" - -namespace android { -namespace base { - -// A wrapper that makes it easy to create an object of type T with static -// storage duration that: -// - is only constructed on first access -// - never invokes the destructor -// in order to satisfy the styleguide ban on global constructors and -// destructors. -// -// Runtime constant example: -// const std::string& GetLineSeparator() { -// // Forwards to std::string(size_t, char, const Allocator&) constructor. -// static const base::NoDestructor<std::string> s(5, '-'); -// return *s; -// } -// -// More complex initialization with a lambda: -// const std::string& GetSessionNonce() { -// static const base::NoDestructor<std::string> nonce([] { -// std::string s(16); -// crypto::RandString(s.data(), s.size()); -// return s; -// }()); -// return *nonce; -// } -// -// NoDestructor<T> stores the object inline, so it also avoids a pointer -// indirection and a malloc. Also note that since C++11 static local variable -// initialization is thread-safe and so is this pattern. Code should prefer to -// use NoDestructor<T> over: -// - A function scoped static T* or T& that is dynamically initialized. -// - A global base::LazyInstance<T>. -// -// Note that since the destructor is never run, this *will* leak memory if used -// as a stack or member variable. Furthermore, a NoDestructor<T> should never -// have global scope as that may require a static initializer. -template <typename T> -class NoDestructor { - public: - // Not constexpr; just write static constexpr T x = ...; if the value should - // be a constexpr. - template <typename... Args> - explicit NoDestructor(Args&&... args) { - new (storage_) T(std::forward<Args>(args)...); - } - - // Allows copy and move construction of the contained type, to allow - // construction from an initializer list, e.g. for std::vector. - explicit NoDestructor(const T& x) { new (storage_) T(x); } - explicit NoDestructor(T&& x) { new (storage_) T(std::move(x)); } - - NoDestructor(const NoDestructor&) = delete; - NoDestructor& operator=(const NoDestructor&) = delete; - - ~NoDestructor() = default; - - const T& operator*() const { return *get(); } - T& operator*() { return *get(); } - - const T* operator->() const { return get(); } - T* operator->() { return get(); } - - const T* get() const { return reinterpret_cast<const T*>(storage_); } - T* get() { return reinterpret_cast<T*>(storage_); } - - private: - alignas(T) char storage_[sizeof(T)]; -}; - -} // namespace base -} // namespace android diff --git a/base/include/android-base/parsebool.h b/base/include/android-base/parsebool.h deleted file mode 100644 index b2bd0217e..000000000 --- a/base/include/android-base/parsebool.h +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include <string_view> - -namespace android { -namespace base { - -// Parse the given string as yes or no inactivation of some sort. Return one of the -// ParseBoolResult enumeration values. -// -// The following values parse as true: -// -// 1 -// on -// true -// y -// yes -// -// -// The following values parse as false: -// -// 0 -// false -// n -// no -// off -// -// Anything else is a parse error. -// -// The purpose of this function is to have a single canonical parser for yes-or-no indications -// throughout the system. - -enum class ParseBoolResult { - kError, - kFalse, - kTrue, -}; - -ParseBoolResult ParseBool(std::string_view s); - -} // namespace base -} // namespace android diff --git a/base/include/android-base/parsedouble.h b/base/include/android-base/parsedouble.h deleted file mode 100644 index ccffba232..000000000 --- a/base/include/android-base/parsedouble.h +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (C) 2016 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include <errno.h> -#include <stdlib.h> - -#include <limits> -#include <string> - -namespace android { -namespace base { - -// Parse floating value in the string 's' and sets 'out' to that value if it exists. -// Optionally allows the caller to define a 'min' and 'max' beyond which -// otherwise valid values will be rejected. Returns boolean success. -template <typename T, T (*strtox)(const char* str, char** endptr)> -static inline bool ParseFloatingPoint(const char* s, T* out, T min, T max) { - errno = 0; - char* end; - T result = strtox(s, &end); - if (errno != 0 || s == end || *end != '\0') { - return false; - } - if (result < min || max < result) { - return false; - } - if (out != nullptr) { - *out = result; - } - return true; -} - -// Parse double value in the string 's' and sets 'out' to that value if it exists. -// Optionally allows the caller to define a 'min' and 'max' beyond which -// otherwise valid values will be rejected. Returns boolean success. -static inline bool ParseDouble(const char* s, double* out, - double min = std::numeric_limits<double>::lowest(), - double max = std::numeric_limits<double>::max()) { - return ParseFloatingPoint<double, strtod>(s, out, min, max); -} -static inline bool ParseDouble(const std::string& s, double* out, - double min = std::numeric_limits<double>::lowest(), - double max = std::numeric_limits<double>::max()) { - return ParseFloatingPoint<double, strtod>(s.c_str(), out, min, max); -} - -// Parse float value in the string 's' and sets 'out' to that value if it exists. -// Optionally allows the caller to define a 'min' and 'max' beyond which -// otherwise valid values will be rejected. Returns boolean success. -static inline bool ParseFloat(const char* s, float* out, - float min = std::numeric_limits<float>::lowest(), - float max = std::numeric_limits<float>::max()) { - return ParseFloatingPoint<float, strtof>(s, out, min, max); -} -static inline bool ParseFloat(const std::string& s, float* out, - float min = std::numeric_limits<float>::lowest(), - float max = std::numeric_limits<float>::max()) { - return ParseFloatingPoint<float, strtof>(s.c_str(), out, min, max); -} - -} // namespace base -} // namespace android diff --git a/base/include/android-base/parseint.h b/base/include/android-base/parseint.h deleted file mode 100644 index be8b97b78..000000000 --- a/base/include/android-base/parseint.h +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include <errno.h> -#include <stdlib.h> -#include <string.h> - -#include <limits> -#include <string> -#include <type_traits> - -namespace android { -namespace base { - -// Parses the unsigned decimal or hexadecimal integer in the string 's' and sets -// 'out' to that value if it is specified. Optionally allows the caller to define -// a 'max' beyond which otherwise valid values will be rejected. Returns boolean -// success; 'out' is untouched if parsing fails. -template <typename T> -bool ParseUint(const char* s, T* out, T max = std::numeric_limits<T>::max(), - bool allow_suffixes = false) { - static_assert(std::is_unsigned<T>::value, "ParseUint can only be used with unsigned types"); - while (isspace(*s)) { - s++; - } - - if (s[0] == '-') { - errno = EINVAL; - return false; - } - - int base = (s[0] == '0' && (s[1] == 'x' || s[1] == 'X')) ? 16 : 10; - errno = 0; - char* end; - unsigned long long int result = strtoull(s, &end, base); - if (errno != 0) return false; - if (end == s) { - errno = EINVAL; - return false; - } - if (*end != '\0') { - const char* suffixes = "bkmgtpe"; - const char* suffix; - if ((!allow_suffixes || (suffix = strchr(suffixes, tolower(*end))) == nullptr) || - __builtin_mul_overflow(result, 1ULL << (10 * (suffix - suffixes)), &result)) { - errno = EINVAL; - return false; - } - } - if (max < result) { - errno = ERANGE; - return false; - } - if (out != nullptr) { - *out = static_cast<T>(result); - } - return true; -} - -// TODO: string_view -template <typename T> -bool ParseUint(const std::string& s, T* out, T max = std::numeric_limits<T>::max(), - bool allow_suffixes = false) { - return ParseUint(s.c_str(), out, max, allow_suffixes); -} - -template <typename T> -bool ParseByteCount(const char* s, T* out, T max = std::numeric_limits<T>::max()) { - return ParseUint(s, out, max, true); -} - -// TODO: string_view -template <typename T> -bool ParseByteCount(const std::string& s, T* out, T max = std::numeric_limits<T>::max()) { - return ParseByteCount(s.c_str(), out, max); -} - -// Parses the signed decimal or hexadecimal integer in the string 's' and sets -// 'out' to that value if it is specified. Optionally allows the caller to define -// a 'min' and 'max' beyond which otherwise valid values will be rejected. Returns -// boolean success; 'out' is untouched if parsing fails. -template <typename T> -bool ParseInt(const char* s, T* out, - T min = std::numeric_limits<T>::min(), - T max = std::numeric_limits<T>::max()) { - static_assert(std::is_signed<T>::value, "ParseInt can only be used with signed types"); - while (isspace(*s)) { - s++; - } - - int base = (s[0] == '0' && (s[1] == 'x' || s[1] == 'X')) ? 16 : 10; - errno = 0; - char* end; - long long int result = strtoll(s, &end, base); - if (errno != 0) { - return false; - } - if (s == end || *end != '\0') { - errno = EINVAL; - return false; - } - if (result < min || max < result) { - errno = ERANGE; - return false; - } - if (out != nullptr) { - *out = static_cast<T>(result); - } - return true; -} - -// TODO: string_view -template <typename T> -bool ParseInt(const std::string& s, T* out, - T min = std::numeric_limits<T>::min(), - T max = std::numeric_limits<T>::max()) { - return ParseInt(s.c_str(), out, min, max); -} - -} // namespace base -} // namespace android diff --git a/base/include/android-base/parsenetaddress.h b/base/include/android-base/parsenetaddress.h deleted file mode 100644 index 47f8b5f6e..000000000 --- a/base/include/android-base/parsenetaddress.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (C) 2016 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include <string> - -namespace android { -namespace base { - -// Parses |address| into |host| and |port|. -// -// If |address| doesn't contain a port number, the default value is taken from -// |port|. If |canonical_address| is non-null it will be set to "host:port" or -// "[host]:port" as appropriate. -// -// On failure, returns false and fills |error|. -bool ParseNetAddress(const std::string& address, std::string* host, int* port, - std::string* canonical_address, std::string* error); - -} // namespace base -} // namespace android diff --git a/base/include/android-base/process.h b/base/include/android-base/process.h deleted file mode 100644 index 69ed3fba5..000000000 --- a/base/include/android-base/process.h +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include <dirent.h> -#include <sys/types.h> - -#include <iterator> -#include <memory> -#include <vector> - -namespace android { -namespace base { - -class AllPids { - class PidIterator { - public: - PidIterator(DIR* dir) : dir_(dir, closedir) { Increment(); } - PidIterator& operator++() { - Increment(); - return *this; - } - bool operator==(const PidIterator& other) const { return pid_ == other.pid_; } - bool operator!=(const PidIterator& other) const { return !(*this == other); } - long operator*() const { return pid_; } - // iterator traits - using difference_type = pid_t; - using value_type = pid_t; - using pointer = const pid_t*; - using reference = const pid_t&; - using iterator_category = std::input_iterator_tag; - - private: - void Increment(); - - pid_t pid_ = -1; - std::unique_ptr<DIR, decltype(&closedir)> dir_; - }; - - public: - PidIterator begin() { return opendir("/proc"); } - PidIterator end() { return nullptr; } -}; - -} // namespace base -} // namespace android diff --git a/base/include/android-base/properties.h b/base/include/android-base/properties.h deleted file mode 100644 index 49f1f3184..000000000 --- a/base/include/android-base/properties.h +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright (C) 2016 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include <sys/cdefs.h> - -#include <chrono> -#include <limits> -#include <optional> -#include <string> - -struct prop_info; - -namespace android { -namespace base { - -// Returns the current value of the system property `key`, -// or `default_value` if the property is empty or doesn't exist. -std::string GetProperty(const std::string& key, const std::string& default_value); - -// Returns true if the system property `key` has the value "1", "y", "yes", "on", or "true", -// false for "0", "n", "no", "off", or "false", or `default_value` otherwise. -bool GetBoolProperty(const std::string& key, bool default_value); - -// Returns the signed integer corresponding to the system property `key`. -// If the property is empty, doesn't exist, doesn't have an integer value, or is outside -// the optional bounds, returns `default_value`. -template <typename T> T GetIntProperty(const std::string& key, - T default_value, - T min = std::numeric_limits<T>::min(), - T max = std::numeric_limits<T>::max()); - -// Returns the unsigned integer corresponding to the system property `key`. -// If the property is empty, doesn't exist, doesn't have an integer value, or is outside -// the optional bound, returns `default_value`. -template <typename T> T GetUintProperty(const std::string& key, - T default_value, - T max = std::numeric_limits<T>::max()); - -// Sets the system property `key` to `value`. -bool SetProperty(const std::string& key, const std::string& value); - -// Waits for the system property `key` to have the value `expected_value`. -// Times out after `relative_timeout`. -// Returns true on success, false on timeout. -#if defined(__BIONIC__) -bool WaitForProperty(const std::string& key, const std::string& expected_value, - std::chrono::milliseconds relative_timeout = std::chrono::milliseconds::max()); -#endif - -// Waits for the system property `key` to be created. -// Times out after `relative_timeout`. -// Returns true on success, false on timeout. -#if defined(__BIONIC__) -bool WaitForPropertyCreation(const std::string& key, std::chrono::milliseconds relative_timeout = - std::chrono::milliseconds::max()); -#endif - -#if defined(__BIONIC__) && __cplusplus >= 201703L -// Cached system property lookup. For code that needs to read the same property multiple times, -// this class helps optimize those lookups. -class CachedProperty { - public: - explicit CachedProperty(const char* property_name); - - // Returns the current value of the underlying system property as cheaply as possible. - // The returned pointer is valid until the next call to Get. Because most callers are going - // to want to parse the string returned here and cached that as well, this function performs - // no locking, and is completely thread unsafe. It is the caller's responsibility to provide a - // lock for thread-safety. - // - // Note: *changed can be set to true even if the contents of the property remain the same. - const char* Get(bool* changed = nullptr); - - private: - std::string property_name_; - const prop_info* prop_info_; - std::optional<uint32_t> cached_area_serial_; - std::optional<uint32_t> cached_property_serial_; - char cached_value_[92]; - bool is_read_only_; - const char* read_only_property_; -}; -#endif - -} // namespace base -} // namespace android diff --git a/base/include/android-base/result.h b/base/include/android-base/result.h deleted file mode 100644 index 56a4f3e80..000000000 --- a/base/include/android-base/result.h +++ /dev/null @@ -1,235 +0,0 @@ -/* - * Copyright (C) 2017 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// This file contains classes for returning a successful result along with an optional -// arbitrarily typed return value or for returning a failure result along with an optional string -// indicating why the function failed. - -// There are 3 classes that implement this functionality and one additional helper type. -// -// Result<T> either contains a member of type T that can be accessed using similar semantics as -// std::optional<T> or it contains a ResultError describing an error, which can be accessed via -// Result<T>::error(). -// -// ResultError is a type that contains both a std::string describing the error and a copy of errno -// from when the error occurred. ResultError can be used in an ostream directly to print its -// string value. -// -// Result<void> is the correct return type for a function that either returns successfully or -// returns an error value. Returning {} from a function that returns Result<void> is the -// correct way to indicate that a function without a return type has completed successfully. -// -// A successful Result<T> is constructed implicitly from any type that can be implicitly converted -// to T or from the constructor arguments for T. This allows you to return a type T directly from -// a function that returns Result<T>. -// -// Error and ErrnoError are used to construct a Result<T> that has failed. The Error class takes -// an ostream as an input and are implicitly cast to a Result<T> containing that failure. -// ErrnoError() is a helper function to create an Error class that appends ": " + strerror(errno) -// to the end of the failure string to aid in interacting with C APIs. Alternatively, an errno -// value can be directly specified via the Error() constructor. -// -// Errorf and ErrnoErrorf accept the format string syntax of the fmblib (https://fmt.dev). -// Errorf("{} errors", num) is equivalent to Error() << num << " errors". -// -// ResultError can be used in the ostream and when using Error/Errorf to construct a Result<T>. -// In this case, the string that the ResultError takes is passed through the stream normally, but -// the errno is passed to the Result<T>. This can be used to pass errno from a failing C function up -// multiple callers. Note that when the outer Result<T> is created with ErrnoError/ErrnoErrorf then -// the errno from the inner ResultError is not passed. Also when multiple ResultError objects are -// used, the errno of the last one is respected. -// -// ResultError can also directly construct a Result<T>. This is particularly useful if you have a -// function that return Result<T> but you have a Result<U> and want to return its error. In this -// case, you can return the .error() from the Result<U> to construct the Result<T>. - -// An example of how to use these is below: -// Result<U> CalculateResult(const T& input) { -// U output; -// if (!SomeOtherCppFunction(input, &output)) { -// return Errorf("SomeOtherCppFunction {} failed", input); -// } -// if (!c_api_function(output)) { -// return ErrnoErrorf("c_api_function {} failed", output); -// } -// return output; -// } -// -// auto output = CalculateResult(input); -// if (!output) return Error() << "CalculateResult failed: " << output.error(); -// UseOutput(*output); - -#pragma once - -#include <errno.h> - -#include <sstream> -#include <string> - -#include "android-base/expected.h" -#include "android-base/format.h" - -namespace android { -namespace base { - -struct ResultError { - template <typename T> - ResultError(T&& message, int code) : message_(std::forward<T>(message)), code_(code) {} - - template <typename T> - // NOLINTNEXTLINE(google-explicit-constructor) - operator android::base::expected<T, ResultError>() { - return android::base::unexpected(ResultError(message_, code_)); - } - - std::string message() const { return message_; } - int code() const { return code_; } - - private: - std::string message_; - int code_; -}; - -inline bool operator==(const ResultError& lhs, const ResultError& rhs) { - return lhs.message() == rhs.message() && lhs.code() == rhs.code(); -} - -inline bool operator!=(const ResultError& lhs, const ResultError& rhs) { - return !(lhs == rhs); -} - -inline std::ostream& operator<<(std::ostream& os, const ResultError& t) { - os << t.message(); - return os; -} - -class Error { - public: - Error() : errno_(0), append_errno_(false) {} - // NOLINTNEXTLINE(google-explicit-constructor) - Error(int errno_to_append) : errno_(errno_to_append), append_errno_(true) {} - - template <typename T> - // NOLINTNEXTLINE(google-explicit-constructor) - operator android::base::expected<T, ResultError>() { - return android::base::unexpected(ResultError(str(), errno_)); - } - - template <typename T> - Error& operator<<(T&& t) { - // NOLINTNEXTLINE(bugprone-suspicious-semicolon) - if constexpr (std::is_same_v<std::remove_cv_t<std::remove_reference_t<T>>, ResultError>) { - errno_ = t.code(); - return (*this) << t.message(); - } - int saved = errno; - ss_ << t; - errno = saved; - return *this; - } - - const std::string str() const { - std::string str = ss_.str(); - if (append_errno_) { - if (str.empty()) { - return strerror(errno_); - } - return std::move(str) + ": " + strerror(errno_); - } - return str; - } - - Error(const Error&) = delete; - Error(Error&&) = delete; - Error& operator=(const Error&) = delete; - Error& operator=(Error&&) = delete; - - template <typename T, typename... Args> - friend Error ErrorfImpl(const T&& fmt, const Args&... args); - - template <typename T, typename... Args> - friend Error ErrnoErrorfImpl(const T&& fmt, const Args&... args); - - private: - Error(bool append_errno, int errno_to_append, const std::string& message) - : errno_(errno_to_append), append_errno_(append_errno) { - (*this) << message; - } - - std::stringstream ss_; - int errno_; - const bool append_errno_; -}; - -inline Error ErrnoError() { - return Error(errno); -} - -inline int ErrorCode(int code) { - return code; -} - -// Return the error code of the last ResultError object, if any. -// Otherwise, return `code` as it is. -template <typename T, typename... Args> -inline int ErrorCode(int code, T&& t, const Args&... args) { - if constexpr (std::is_same_v<std::remove_cv_t<std::remove_reference_t<T>>, ResultError>) { - return ErrorCode(t.code(), args...); - } - return ErrorCode(code, args...); -} - -template <typename T, typename... Args> -inline Error ErrorfImpl(const T&& fmt, const Args&... args) { - return Error(false, ErrorCode(0, args...), fmt::format(fmt, args...)); -} - -template <typename T, typename... Args> -inline Error ErrnoErrorfImpl(const T&& fmt, const Args&... args) { - return Error(true, errno, fmt::format(fmt, args...)); -} - -#define Errorf(fmt, ...) android::base::ErrorfImpl(FMT_STRING(fmt), ##__VA_ARGS__) -#define ErrnoErrorf(fmt, ...) android::base::ErrnoErrorfImpl(FMT_STRING(fmt), ##__VA_ARGS__) - -template <typename T> -using Result = android::base::expected<T, ResultError>; - -// Macros for testing the results of functions that return android::base::Result. -// These also work with base::android::expected. - -#define CHECK_RESULT_OK(stmt) \ - do { \ - const auto& tmp = (stmt); \ - CHECK(tmp.ok()) << tmp.error(); \ - } while (0) - -#define ASSERT_RESULT_OK(stmt) \ - do { \ - const auto& tmp = (stmt); \ - ASSERT_TRUE(tmp.ok()) << tmp.error(); \ - } while (0) - -#define EXPECT_RESULT_OK(stmt) \ - do { \ - auto tmp = (stmt); \ - EXPECT_TRUE(tmp.ok()) << tmp.error(); \ - } while (0) - -// TODO: Maybe add RETURN_IF_ERROR() and ASSIGN_OR_RETURN() - -} // namespace base -} // namespace android diff --git a/base/include/android-base/scopeguard.h b/base/include/android-base/scopeguard.h deleted file mode 100644 index 5a224d634..000000000 --- a/base/include/android-base/scopeguard.h +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include <utility> // for std::move, std::forward - -namespace android { -namespace base { - -// ScopeGuard ensures that the specified functor is executed no matter how the -// current scope exits. -template <typename F> -class ScopeGuard { - public: - ScopeGuard(F&& f) : f_(std::forward<F>(f)), active_(true) {} - - ScopeGuard(ScopeGuard&& that) noexcept : f_(std::move(that.f_)), active_(that.active_) { - that.active_ = false; - } - - template <typename Functor> - ScopeGuard(ScopeGuard<Functor>&& that) : f_(std::move(that.f_)), active_(that.active_) { - that.active_ = false; - } - - ~ScopeGuard() { - if (active_) f_(); - } - - ScopeGuard() = delete; - ScopeGuard(const ScopeGuard&) = delete; - void operator=(const ScopeGuard&) = delete; - void operator=(ScopeGuard&& that) = delete; - - void Disable() { active_ = false; } - - bool active() const { return active_; } - - private: - template <typename Functor> - friend class ScopeGuard; - - F f_; - bool active_; -}; - -template <typename F> -ScopeGuard<F> make_scope_guard(F&& f) { - return ScopeGuard<F>(std::forward<F>(f)); -} - -} // namespace base -} // namespace android diff --git a/base/include/android-base/stringprintf.h b/base/include/android-base/stringprintf.h deleted file mode 100644 index 93c56afd7..000000000 --- a/base/include/android-base/stringprintf.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (C) 2011 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include <stdarg.h> -#include <string> - -namespace android { -namespace base { - -// These printf-like functions are implemented in terms of vsnprintf, so they -// use the same attribute for compile-time format string checking. - -// Returns a string corresponding to printf-like formatting of the arguments. -std::string StringPrintf(const char* fmt, ...) __attribute__((__format__(__printf__, 1, 2))); - -// Appends a printf-like formatting of the arguments to 'dst'. -void StringAppendF(std::string* dst, const char* fmt, ...) - __attribute__((__format__(__printf__, 2, 3))); - -// Appends a printf-like formatting of the arguments to 'dst'. -void StringAppendV(std::string* dst, const char* format, va_list ap) - __attribute__((__format__(__printf__, 2, 0))); - -} // namespace base -} // namespace android diff --git a/base/include/android-base/strings.h b/base/include/android-base/strings.h deleted file mode 100644 index 14d534a3b..000000000 --- a/base/include/android-base/strings.h +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include <sstream> -#include <string> -#include <string_view> -#include <vector> - -namespace android { -namespace base { - -// Splits a string into a vector of strings. -// -// The string is split at each occurrence of a character in delimiters. -// -// The empty string is not a valid delimiter list. -std::vector<std::string> Split(const std::string& s, - const std::string& delimiters); - -// Trims whitespace off both ends of the given string. -std::string Trim(const std::string& s); - -// Joins a container of things into a single string, using the given separator. -template <typename ContainerT, typename SeparatorT> -std::string Join(const ContainerT& things, SeparatorT separator) { - if (things.empty()) { - return ""; - } - - std::ostringstream result; - result << *things.begin(); - for (auto it = std::next(things.begin()); it != things.end(); ++it) { - result << separator << *it; - } - return result.str(); -} - -// We instantiate the common cases in strings.cpp. -extern template std::string Join(const std::vector<std::string>&, char); -extern template std::string Join(const std::vector<const char*>&, char); -extern template std::string Join(const std::vector<std::string>&, const std::string&); -extern template std::string Join(const std::vector<const char*>&, const std::string&); - -// Tests whether 's' starts with 'prefix'. -bool StartsWith(std::string_view s, std::string_view prefix); -bool StartsWith(std::string_view s, char prefix); -bool StartsWithIgnoreCase(std::string_view s, std::string_view prefix); - -// Tests whether 's' ends with 'suffix'. -bool EndsWith(std::string_view s, std::string_view suffix); -bool EndsWith(std::string_view s, char suffix); -bool EndsWithIgnoreCase(std::string_view s, std::string_view suffix); - -// Tests whether 'lhs' equals 'rhs', ignoring case. -bool EqualsIgnoreCase(std::string_view lhs, std::string_view rhs); - -// Removes `prefix` from the start of the given string and returns true (if -// it was present), false otherwise. -inline bool ConsumePrefix(std::string_view* s, std::string_view prefix) { - if (!StartsWith(*s, prefix)) return false; - s->remove_prefix(prefix.size()); - return true; -} - -// Removes `suffix` from the end of the given string and returns true (if -// it was present), false otherwise. -inline bool ConsumeSuffix(std::string_view* s, std::string_view suffix) { - if (!EndsWith(*s, suffix)) return false; - s->remove_suffix(suffix.size()); - return true; -} - -// Replaces `from` with `to` in `s`, once if `all == false`, or as many times as -// there are matches if `all == true`. -[[nodiscard]] std::string StringReplace(std::string_view s, std::string_view from, - std::string_view to, bool all); - -} // namespace base -} // namespace android diff --git a/base/include/android-base/test_utils.h b/base/include/android-base/test_utils.h deleted file mode 100644 index f3d7cb031..000000000 --- a/base/include/android-base/test_utils.h +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include <regex> -#include <string> - -#include <android-base/file.h> -#include <android-base/macros.h> - -class CapturedStdFd { - public: - CapturedStdFd(int std_fd); - ~CapturedStdFd(); - - std::string str(); - - void Start(); - void Stop(); - void Reset(); - - private: - int fd() const; - - TemporaryFile temp_file_; - int std_fd_; - int old_fd_ = -1; - - DISALLOW_COPY_AND_ASSIGN(CapturedStdFd); -}; - -class CapturedStderr : public CapturedStdFd { - public: - CapturedStderr() : CapturedStdFd(STDERR_FILENO) {} -}; - -class CapturedStdout : public CapturedStdFd { - public: - CapturedStdout() : CapturedStdFd(STDOUT_FILENO) {} -}; - -#define ASSERT_MATCH(str, pattern) \ - do { \ - auto __s = (str); \ - if (!std::regex_search(__s, std::regex((pattern)))) { \ - FAIL() << "regex mismatch: expected " << (pattern) << " in:\n" << __s; \ - } \ - } while (0) - -#define ASSERT_NOT_MATCH(str, pattern) \ - do { \ - auto __s = (str); \ - if (std::regex_search(__s, std::regex((pattern)))) { \ - FAIL() << "regex mismatch: expected to not find " << (pattern) << " in:\n" << __s; \ - } \ - } while (0) - -#define EXPECT_MATCH(str, pattern) \ - do { \ - auto __s = (str); \ - if (!std::regex_search(__s, std::regex((pattern)))) { \ - ADD_FAILURE() << "regex mismatch: expected " << (pattern) << " in:\n" << __s; \ - } \ - } while (0) - -#define EXPECT_NOT_MATCH(str, pattern) \ - do { \ - auto __s = (str); \ - if (std::regex_search(__s, std::regex((pattern)))) { \ - ADD_FAILURE() << "regex mismatch: expected to not find " << (pattern) << " in:\n" << __s; \ - } \ - } while (0) diff --git a/base/include/android-base/thread_annotations.h b/base/include/android-base/thread_annotations.h deleted file mode 100644 index 53fe6dae9..000000000 --- a/base/include/android-base/thread_annotations.h +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright (C) 2016 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include <mutex> - -#define THREAD_ANNOTATION_ATTRIBUTE__(x) __attribute__((x)) - -#define CAPABILITY(x) \ - THREAD_ANNOTATION_ATTRIBUTE__(capability(x)) - -#define SCOPED_CAPABILITY \ - THREAD_ANNOTATION_ATTRIBUTE__(scoped_lockable) - -#define SHARED_CAPABILITY(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(shared_capability(__VA_ARGS__)) - -#define GUARDED_BY(x) \ - THREAD_ANNOTATION_ATTRIBUTE__(guarded_by(x)) - -#define PT_GUARDED_BY(x) \ - THREAD_ANNOTATION_ATTRIBUTE__(pt_guarded_by(x)) - -#define EXCLUSIVE_LOCKS_REQUIRED(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(exclusive_locks_required(__VA_ARGS__)) - -#define SHARED_LOCKS_REQUIRED(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(shared_locks_required(__VA_ARGS__)) - -#define ACQUIRED_BEFORE(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(acquired_before(__VA_ARGS__)) - -#define ACQUIRED_AFTER(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(acquired_after(__VA_ARGS__)) - -#define REQUIRES(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(requires_capability(__VA_ARGS__)) - -#define REQUIRES_SHARED(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(requires_shared_capability(__VA_ARGS__)) - -#define ACQUIRE(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(acquire_capability(__VA_ARGS__)) - -#define ACQUIRE_SHARED(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(acquire_shared_capability(__VA_ARGS__)) - -#define RELEASE(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(release_capability(__VA_ARGS__)) - -#define RELEASE_SHARED(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(release_shared_capability(__VA_ARGS__)) - -#define TRY_ACQUIRE(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(try_acquire_capability(__VA_ARGS__)) - -#define TRY_ACQUIRE_SHARED(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(try_acquire_shared_capability(__VA_ARGS__)) - -#define EXCLUDES(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(locks_excluded(__VA_ARGS__)) - -#define ASSERT_CAPABILITY(x) \ - THREAD_ANNOTATION_ATTRIBUTE__(assert_capability(x)) - -#define ASSERT_SHARED_CAPABILITY(x) \ - THREAD_ANNOTATION_ATTRIBUTE__(assert_shared_capability(x)) - -#define RETURN_CAPABILITY(x) \ - THREAD_ANNOTATION_ATTRIBUTE__(lock_returned(x)) - -#define EXCLUSIVE_LOCK_FUNCTION(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(exclusive_lock_function(__VA_ARGS__)) - -#define EXCLUSIVE_TRYLOCK_FUNCTION(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(exclusive_trylock_function(__VA_ARGS__)) - -#define SHARED_LOCK_FUNCTION(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(shared_lock_function(__VA_ARGS__)) - -#define SHARED_TRYLOCK_FUNCTION(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(shared_trylock_function(__VA_ARGS__)) - -#define UNLOCK_FUNCTION(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(unlock_function(__VA_ARGS__)) - -#define SCOPED_LOCKABLE \ - THREAD_ANNOTATION_ATTRIBUTE__(scoped_lockable) - -#define LOCK_RETURNED(x) \ - THREAD_ANNOTATION_ATTRIBUTE__(lock_returned(x)) - -#define NO_THREAD_SAFETY_ANALYSIS \ - THREAD_ANNOTATION_ATTRIBUTE__(no_thread_safety_analysis) - -namespace android { -namespace base { - -// A class to help thread safety analysis deal with std::unique_lock and condition_variable. -// -// Clang's thread safety analysis currently doesn't perform alias analysis, so movable types -// like std::unique_lock can't be marked with thread safety annotations. This helper allows -// for manual assertion of lock state in a scope. -// -// For example: -// -// std::mutex mutex; -// std::condition_variable cv; -// std::vector<int> vec GUARDED_BY(mutex); -// -// int pop() { -// std::unique_lock lock(mutex); -// ScopedLockAssertion lock_assertion(mutex); -// cv.wait(lock, []() { -// ScopedLockAssertion lock_assertion(mutex); -// return !vec.empty(); -// }); -// -// int result = vec.back(); -// vec.pop_back(); -// return result; -// } -class SCOPED_CAPABILITY ScopedLockAssertion { - public: - ScopedLockAssertion(std::mutex& mutex) ACQUIRE(mutex) {} - ~ScopedLockAssertion() RELEASE() {} -}; - -} // namespace base -} // namespace android diff --git a/base/include/android-base/unique_fd.h b/base/include/android-base/unique_fd.h deleted file mode 100644 index c4a0aad0f..000000000 --- a/base/include/android-base/unique_fd.h +++ /dev/null @@ -1,293 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include <dirent.h> -#include <errno.h> -#include <fcntl.h> - -#if !defined(_WIN32) -#include <sys/socket.h> -#endif - -#include <stdio.h> -#include <sys/types.h> -#include <unistd.h> - -// DO NOT INCLUDE OTHER LIBBASE HEADERS! -// This file gets used in libbinder, and libbinder is used everywhere. -// Including other headers from libbase frequently results in inclusion of -// android-base/macros.h, which causes macro collisions. - -// Container for a file descriptor that automatically closes the descriptor as -// it goes out of scope. -// -// unique_fd ufd(open("/some/path", "r")); -// if (ufd.get() == -1) return error; -// -// // Do something useful, possibly including 'return'. -// -// return 0; // Descriptor is closed for you. -// -// unique_fd is also known as ScopedFd/ScopedFD/scoped_fd; mentioned here to help -// you find this class if you're searching for one of those names. - -#if defined(__BIONIC__) -#include <android/fdsan.h> -#endif - -namespace android { -namespace base { - -struct DefaultCloser { -#if defined(__BIONIC__) - static void Tag(int fd, void* old_addr, void* new_addr) { - if (android_fdsan_exchange_owner_tag) { - uint64_t old_tag = android_fdsan_create_owner_tag(ANDROID_FDSAN_OWNER_TYPE_UNIQUE_FD, - reinterpret_cast<uint64_t>(old_addr)); - uint64_t new_tag = android_fdsan_create_owner_tag(ANDROID_FDSAN_OWNER_TYPE_UNIQUE_FD, - reinterpret_cast<uint64_t>(new_addr)); - android_fdsan_exchange_owner_tag(fd, old_tag, new_tag); - } - } - static void Close(int fd, void* addr) { - if (android_fdsan_close_with_tag) { - uint64_t tag = android_fdsan_create_owner_tag(ANDROID_FDSAN_OWNER_TYPE_UNIQUE_FD, - reinterpret_cast<uint64_t>(addr)); - android_fdsan_close_with_tag(fd, tag); - } else { - close(fd); - } - } -#else - static void Close(int fd) { - // Even if close(2) fails with EINTR, the fd will have been closed. - // Using TEMP_FAILURE_RETRY will either lead to EBADF or closing someone - // else's fd. - // http://lkml.indiana.edu/hypermail/linux/kernel/0509.1/0877.html - ::close(fd); - } -#endif -}; - -template <typename Closer> -class unique_fd_impl final { - public: - unique_fd_impl() {} - - explicit unique_fd_impl(int fd) { reset(fd); } - ~unique_fd_impl() { reset(); } - - unique_fd_impl(const unique_fd_impl&) = delete; - void operator=(const unique_fd_impl&) = delete; - unique_fd_impl(unique_fd_impl&& other) noexcept { reset(other.release()); } - unique_fd_impl& operator=(unique_fd_impl&& s) noexcept { - int fd = s.fd_; - s.fd_ = -1; - reset(fd, &s); - return *this; - } - - void reset(int new_value = -1) { reset(new_value, nullptr); } - - int get() const { return fd_; } - -#if !defined(ANDROID_BASE_UNIQUE_FD_DISABLE_IMPLICIT_CONVERSION) - // unique_fd's operator int is dangerous, but we have way too much code that - // depends on it, so make this opt-in at first. - operator int() const { return get(); } // NOLINT -#endif - - bool operator>=(int rhs) const { return get() >= rhs; } - bool operator<(int rhs) const { return get() < rhs; } - bool operator==(int rhs) const { return get() == rhs; } - bool operator!=(int rhs) const { return get() != rhs; } - bool operator==(const unique_fd_impl& rhs) const { return get() == rhs.get(); } - bool operator!=(const unique_fd_impl& rhs) const { return get() != rhs.get(); } - - // Catch bogus error checks (i.e.: "!fd" instead of "fd != -1"). - bool operator!() const = delete; - - bool ok() const { return get() >= 0; } - - int release() __attribute__((warn_unused_result)) { - tag(fd_, this, nullptr); - int ret = fd_; - fd_ = -1; - return ret; - } - - private: - void reset(int new_value, void* previous_tag) { - int previous_errno = errno; - - if (fd_ != -1) { - close(fd_, this); - } - - fd_ = new_value; - if (new_value != -1) { - tag(new_value, previous_tag, this); - } - - errno = previous_errno; - } - - int fd_ = -1; - - // Template magic to use Closer::Tag if available, and do nothing if not. - // If Closer::Tag exists, this implementation is preferred, because int is a better match. - // If not, this implementation is SFINAEd away, and the no-op below is the only one that exists. - template <typename T = Closer> - static auto tag(int fd, void* old_tag, void* new_tag) - -> decltype(T::Tag(fd, old_tag, new_tag), void()) { - T::Tag(fd, old_tag, new_tag); - } - - template <typename T = Closer> - static void tag(long, void*, void*) { - // No-op. - } - - // Same as above, to select between Closer::Close(int) and Closer::Close(int, void*). - template <typename T = Closer> - static auto close(int fd, void* tag_value) -> decltype(T::Close(fd, tag_value), void()) { - T::Close(fd, tag_value); - } - - template <typename T = Closer> - static auto close(int fd, void*) -> decltype(T::Close(fd), void()) { - T::Close(fd); - } -}; - -using unique_fd = unique_fd_impl<DefaultCloser>; - -#if !defined(_WIN32) - -// Inline functions, so that they can be used header-only. -template <typename Closer> -inline bool Pipe(unique_fd_impl<Closer>* read, unique_fd_impl<Closer>* write, - int flags = O_CLOEXEC) { - int pipefd[2]; - -#if defined(__linux__) - if (pipe2(pipefd, flags) != 0) { - return false; - } -#else // defined(__APPLE__) - if (flags & ~(O_CLOEXEC | O_NONBLOCK)) { - return false; - } - if (pipe(pipefd) != 0) { - return false; - } - - if (flags & O_CLOEXEC) { - if (fcntl(pipefd[0], F_SETFD, FD_CLOEXEC) != 0 || fcntl(pipefd[1], F_SETFD, FD_CLOEXEC) != 0) { - close(pipefd[0]); - close(pipefd[1]); - return false; - } - } - if (flags & O_NONBLOCK) { - if (fcntl(pipefd[0], F_SETFL, O_NONBLOCK) != 0 || fcntl(pipefd[1], F_SETFL, O_NONBLOCK) != 0) { - close(pipefd[0]); - close(pipefd[1]); - return false; - } - } -#endif - - read->reset(pipefd[0]); - write->reset(pipefd[1]); - return true; -} - -template <typename Closer> -inline bool Socketpair(int domain, int type, int protocol, unique_fd_impl<Closer>* left, - unique_fd_impl<Closer>* right) { - int sockfd[2]; - if (socketpair(domain, type, protocol, sockfd) != 0) { - return false; - } - left->reset(sockfd[0]); - right->reset(sockfd[1]); - return true; -} - -template <typename Closer> -inline bool Socketpair(int type, unique_fd_impl<Closer>* left, unique_fd_impl<Closer>* right) { - return Socketpair(AF_UNIX, type, 0, left, right); -} - -// Using fdopen with unique_fd correctly is more annoying than it should be, -// because fdopen doesn't close the file descriptor received upon failure. -inline FILE* Fdopen(unique_fd&& ufd, const char* mode) { - int fd = ufd.release(); - FILE* file = fdopen(fd, mode); - if (!file) { - close(fd); - } - return file; -} - -// Using fdopendir with unique_fd correctly is more annoying than it should be, -// because fdopen doesn't close the file descriptor received upon failure. -inline DIR* Fdopendir(unique_fd&& ufd) { - int fd = ufd.release(); - DIR* dir = fdopendir(fd); - if (dir == nullptr) { - close(fd); - } - return dir; -} - -#endif // !defined(_WIN32) - -// A wrapper type that can be implicitly constructed from either int or unique_fd. -struct borrowed_fd { - /* implicit */ borrowed_fd(int fd) : fd_(fd) {} // NOLINT - template <typename T> - /* implicit */ borrowed_fd(const unique_fd_impl<T>& ufd) : fd_(ufd.get()) {} // NOLINT - - int get() const { return fd_; } - - bool operator>=(int rhs) const { return get() >= rhs; } - bool operator<(int rhs) const { return get() < rhs; } - bool operator==(int rhs) const { return get() == rhs; } - bool operator!=(int rhs) const { return get() != rhs; } - - private: - int fd_ = -1; -}; -} // namespace base -} // namespace android - -template <typename T> -int close(const android::base::unique_fd_impl<T>&) - __attribute__((__unavailable__("close called on unique_fd"))); - -template <typename T> -FILE* fdopen(const android::base::unique_fd_impl<T>&, const char* mode) - __attribute__((__unavailable__("fdopen takes ownership of the fd passed in; either dup the " - "unique_fd, or use android::base::Fdopen to pass ownership"))); - -template <typename T> -DIR* fdopendir(const android::base::unique_fd_impl<T>&) __attribute__(( - __unavailable__("fdopendir takes ownership of the fd passed in; either dup the " - "unique_fd, or use android::base::Fdopendir to pass ownership"))); diff --git a/base/include/android-base/utf8.h b/base/include/android-base/utf8.h deleted file mode 100644 index 1a414ec79..000000000 --- a/base/include/android-base/utf8.h +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#ifdef _WIN32 -#include <sys/types.h> -#include <string> -#else -// Bring in prototypes for standard APIs so that we can import them into the utf8 namespace. -#include <fcntl.h> // open -#include <stdio.h> // fopen -#include <sys/stat.h> // mkdir -#include <unistd.h> // unlink -#endif - -namespace android { -namespace base { - -// Only available on Windows because this is only needed on Windows. -#ifdef _WIN32 -// Convert size number of UTF-16 wchar_t's to UTF-8. Returns whether the -// conversion was done successfully. -bool WideToUTF8(const wchar_t* utf16, const size_t size, std::string* utf8); - -// Convert a NULL-terminated string of UTF-16 characters to UTF-8. Returns -// whether the conversion was done successfully. -bool WideToUTF8(const wchar_t* utf16, std::string* utf8); - -// Convert a UTF-16 std::wstring (including any embedded NULL characters) to -// UTF-8. Returns whether the conversion was done successfully. -bool WideToUTF8(const std::wstring& utf16, std::string* utf8); - -// Convert size number of UTF-8 char's to UTF-16. Returns whether the conversion -// was done successfully. -bool UTF8ToWide(const char* utf8, const size_t size, std::wstring* utf16); - -// Convert a NULL-terminated string of UTF-8 characters to UTF-16. Returns -// whether the conversion was done successfully. -bool UTF8ToWide(const char* utf8, std::wstring* utf16); - -// Convert a UTF-8 std::string (including any embedded NULL characters) to -// UTF-16. Returns whether the conversion was done successfully. -bool UTF8ToWide(const std::string& utf8, std::wstring* utf16); - -// Convert a file system path, represented as a NULL-terminated string of -// UTF-8 characters, to a UTF-16 string representing the same file system -// path using the Windows extended-lengh path representation. -// -// See https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#MAXPATH: -// ```The Windows API has many functions that also have Unicode versions to -// permit an extended-length path for a maximum total path length of 32,767 -// characters. To specify an extended-length path, use the "\\?\" prefix. -// For example, "\\?\D:\very long path".``` -// -// Returns whether the conversion was done successfully. -bool UTF8PathToWindowsLongPath(const char* utf8, std::wstring* utf16); -#endif - -// The functions in the utf8 namespace take UTF-8 strings. For Windows, these -// are wrappers, for non-Windows these just expose existing APIs. To call these -// functions, use: -// -// // anonymous namespace to avoid conflict with existing open(), unlink(), etc. -// namespace { -// // Import functions into anonymous namespace. -// using namespace android::base::utf8; -// -// void SomeFunction(const char* name) { -// int fd = open(name, ...); // Calls android::base::utf8::open(). -// ... -// unlink(name); // Calls android::base::utf8::unlink(). -// } -// } -namespace utf8 { - -#ifdef _WIN32 -FILE* fopen(const char* name, const char* mode); -int mkdir(const char* name, mode_t mode); -int open(const char* name, int flags, ...); -int unlink(const char* name); -#else -using ::fopen; -using ::mkdir; -using ::open; -using ::unlink; -#endif - -} // namespace utf8 -} // namespace base -} // namespace android diff --git a/base/liblog_symbols.cpp b/base/liblog_symbols.cpp deleted file mode 100644 index 1f4b69b53..000000000 --- a/base/liblog_symbols.cpp +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (C) 2020 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "liblog_symbols.h" - -#if defined(__ANDROID_SDK_VERSION__) && (__ANDROID_SDK_VERSION__ <= 29) -#define USE_DLSYM -#endif - -#ifdef USE_DLSYM -#include <dlfcn.h> -#endif - -namespace android { -namespace base { - -#ifdef USE_DLSYM - -const std::optional<LibLogFunctions>& GetLibLogFunctions() { - static std::optional<LibLogFunctions> liblog_functions = []() -> std::optional<LibLogFunctions> { - void* liblog_handle = dlopen("liblog.so", RTLD_NOW); - if (liblog_handle == nullptr) { - return {}; - } - - LibLogFunctions real_liblog_functions = {}; - -#define DLSYM(name) \ - real_liblog_functions.name = \ - reinterpret_cast<decltype(LibLogFunctions::name)>(dlsym(liblog_handle, #name)); \ - if (real_liblog_functions.name == nullptr) { \ - return {}; \ - } - - DLSYM(__android_log_set_logger) - DLSYM(__android_log_write_log_message) - DLSYM(__android_log_logd_logger) - DLSYM(__android_log_stderr_logger) - DLSYM(__android_log_set_aborter) - DLSYM(__android_log_call_aborter) - DLSYM(__android_log_default_aborter) - DLSYM(__android_log_set_minimum_priority); - DLSYM(__android_log_get_minimum_priority); - DLSYM(__android_log_set_default_tag); -#undef DLSYM - - return real_liblog_functions; - }(); - - return liblog_functions; -} - -#else - -const std::optional<LibLogFunctions>& GetLibLogFunctions() { - static std::optional<LibLogFunctions> liblog_functions = []() -> std::optional<LibLogFunctions> { - return LibLogFunctions{ - .__android_log_set_logger = __android_log_set_logger, - .__android_log_write_log_message = __android_log_write_log_message, - .__android_log_logd_logger = __android_log_logd_logger, - .__android_log_stderr_logger = __android_log_stderr_logger, - .__android_log_set_aborter = __android_log_set_aborter, - .__android_log_call_aborter = __android_log_call_aborter, - .__android_log_default_aborter = __android_log_default_aborter, - .__android_log_set_minimum_priority = __android_log_set_minimum_priority, - .__android_log_get_minimum_priority = __android_log_get_minimum_priority, - .__android_log_set_default_tag = __android_log_set_default_tag, - }; - }(); - return liblog_functions; -} - -#endif - -} // namespace base -} // namespace android diff --git a/base/liblog_symbols.h b/base/liblog_symbols.h deleted file mode 100644 index 2e6b47f7d..000000000 --- a/base/liblog_symbols.h +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (C) 2020 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include <optional> - -#include <android/log.h> - -namespace android { -namespace base { - -struct LibLogFunctions { - void (*__android_log_set_logger)(__android_logger_function logger); - void (*__android_log_write_log_message)(struct __android_log_message* log_message); - - void (*__android_log_logd_logger)(const struct __android_log_message* log_message); - void (*__android_log_stderr_logger)(const struct __android_log_message* log_message); - - void (*__android_log_set_aborter)(__android_aborter_function aborter); - void (*__android_log_call_aborter)(const char* abort_message); - void (*__android_log_default_aborter)(const char* abort_message); - int32_t (*__android_log_set_minimum_priority)(int32_t priority); - int32_t (*__android_log_get_minimum_priority)(); - void (*__android_log_set_default_tag)(const char* tag); -}; - -const std::optional<LibLogFunctions>& GetLibLogFunctions(); - -} // namespace base -} // namespace android diff --git a/base/logging.cpp b/base/logging.cpp deleted file mode 100644 index 5bd21da66..000000000 --- a/base/logging.cpp +++ /dev/null @@ -1,585 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#if defined(_WIN32) -#include <windows.h> -#endif - -#include "android-base/logging.h" - -#include <fcntl.h> -#include <inttypes.h> -#include <libgen.h> -#include <time.h> - -// For getprogname(3) or program_invocation_short_name. -#if defined(__ANDROID__) || defined(__APPLE__) -#include <stdlib.h> -#elif defined(__GLIBC__) -#include <errno.h> -#endif - -#if defined(__linux__) -#include <sys/uio.h> -#endif - -#include <atomic> -#include <iostream> -#include <limits> -#include <mutex> -#include <optional> -#include <sstream> -#include <string> -#include <utility> -#include <vector> - -#include <android/log.h> -#ifdef __ANDROID__ -#include <android/set_abort_message.h> -#else -#include <sys/types.h> -#include <unistd.h> -#endif - -#include <android-base/file.h> -#include <android-base/macros.h> -#include <android-base/parseint.h> -#include <android-base/strings.h> -#include <android-base/threads.h> - -#include "liblog_symbols.h" -#include "logging_splitters.h" - -namespace android { -namespace base { - -// BSD-based systems like Android/macOS have getprogname(). Others need us to provide one. -#if defined(__GLIBC__) || defined(_WIN32) -static const char* getprogname() { -#if defined(__GLIBC__) - return program_invocation_short_name; -#elif defined(_WIN32) - static bool first = true; - static char progname[MAX_PATH] = {}; - - if (first) { - snprintf(progname, sizeof(progname), "%s", - android::base::Basename(android::base::GetExecutablePath()).c_str()); - first = false; - } - - return progname; -#endif -} -#endif - -static const char* GetFileBasename(const char* file) { - // We can't use basename(3) even on Unix because the Mac doesn't - // have a non-modifying basename. - const char* last_slash = strrchr(file, '/'); - if (last_slash != nullptr) { - return last_slash + 1; - } -#if defined(_WIN32) - const char* last_backslash = strrchr(file, '\\'); - if (last_backslash != nullptr) { - return last_backslash + 1; - } -#endif - return file; -} - -#if defined(__linux__) -static int OpenKmsg() { -#if defined(__ANDROID__) - // pick up 'file w /dev/kmsg' environment from daemon's init rc file - const auto val = getenv("ANDROID_FILE__dev_kmsg"); - if (val != nullptr) { - int fd; - if (android::base::ParseInt(val, &fd, 0)) { - auto flags = fcntl(fd, F_GETFL); - if ((flags != -1) && ((flags & O_ACCMODE) == O_WRONLY)) return fd; - } - } -#endif - return TEMP_FAILURE_RETRY(open("/dev/kmsg", O_WRONLY | O_CLOEXEC)); -} -#endif - -static LogId log_id_tToLogId(int32_t buffer_id) { - switch (buffer_id) { - case LOG_ID_MAIN: - return MAIN; - case LOG_ID_SYSTEM: - return SYSTEM; - case LOG_ID_RADIO: - return RADIO; - case LOG_ID_CRASH: - return CRASH; - case LOG_ID_DEFAULT: - default: - return DEFAULT; - } -} - -static int32_t LogIdTolog_id_t(LogId log_id) { - switch (log_id) { - case MAIN: - return LOG_ID_MAIN; - case SYSTEM: - return LOG_ID_SYSTEM; - case RADIO: - return LOG_ID_RADIO; - case CRASH: - return LOG_ID_CRASH; - case DEFAULT: - default: - return LOG_ID_DEFAULT; - } -} - -static LogSeverity PriorityToLogSeverity(int priority) { - switch (priority) { - case ANDROID_LOG_DEFAULT: - return INFO; - case ANDROID_LOG_VERBOSE: - return VERBOSE; - case ANDROID_LOG_DEBUG: - return DEBUG; - case ANDROID_LOG_INFO: - return INFO; - case ANDROID_LOG_WARN: - return WARNING; - case ANDROID_LOG_ERROR: - return ERROR; - case ANDROID_LOG_FATAL: - return FATAL; - default: - return FATAL; - } -} - -static int32_t LogSeverityToPriority(LogSeverity severity) { - switch (severity) { - case VERBOSE: - return ANDROID_LOG_VERBOSE; - case DEBUG: - return ANDROID_LOG_DEBUG; - case INFO: - return ANDROID_LOG_INFO; - case WARNING: - return ANDROID_LOG_WARN; - case ERROR: - return ANDROID_LOG_ERROR; - case FATAL_WITHOUT_ABORT: - case FATAL: - default: - return ANDROID_LOG_FATAL; - } -} - -static LogFunction& Logger() { -#ifdef __ANDROID__ - static auto& logger = *new LogFunction(LogdLogger()); -#else - static auto& logger = *new LogFunction(StderrLogger); -#endif - return logger; -} - -static AbortFunction& Aborter() { - static auto& aborter = *new AbortFunction(DefaultAborter); - return aborter; -} - -// Only used for Q fallback. -static std::recursive_mutex& TagLock() { - static auto& tag_lock = *new std::recursive_mutex(); - return tag_lock; -} -// Only used for Q fallback. -static std::string* gDefaultTag; - -void SetDefaultTag(const std::string& tag) { - static auto& liblog_functions = GetLibLogFunctions(); - if (liblog_functions) { - liblog_functions->__android_log_set_default_tag(tag.c_str()); - } else { - std::lock_guard<std::recursive_mutex> lock(TagLock()); - if (gDefaultTag != nullptr) { - delete gDefaultTag; - gDefaultTag = nullptr; - } - if (!tag.empty()) { - gDefaultTag = new std::string(tag); - } - } -} - -static bool gInitialized = false; - -// Only used for Q fallback. -static LogSeverity gMinimumLogSeverity = INFO; - -#if defined(__linux__) -static void KernelLogLine(const char* msg, int length, android::base::LogSeverity severity, - const char* tag) { - // clang-format off - static constexpr int kLogSeverityToKernelLogLevel[] = { - [android::base::VERBOSE] = 7, // KERN_DEBUG (there is no verbose kernel log - // level) - [android::base::DEBUG] = 7, // KERN_DEBUG - [android::base::INFO] = 6, // KERN_INFO - [android::base::WARNING] = 4, // KERN_WARNING - [android::base::ERROR] = 3, // KERN_ERROR - [android::base::FATAL_WITHOUT_ABORT] = 2, // KERN_CRIT - [android::base::FATAL] = 2, // KERN_CRIT - }; - // clang-format on - static_assert(arraysize(kLogSeverityToKernelLogLevel) == android::base::FATAL + 1, - "Mismatch in size of kLogSeverityToKernelLogLevel and values in LogSeverity"); - - static int klog_fd = OpenKmsg(); - if (klog_fd == -1) return; - - int level = kLogSeverityToKernelLogLevel[severity]; - - // The kernel's printk buffer is only 1024 bytes. - // TODO: should we automatically break up long lines into multiple lines? - // Or we could log but with something like "..." at the end? - char buf[1024] __attribute__((__uninitialized__)); - size_t size = snprintf(buf, sizeof(buf), "<%d>%s: %.*s\n", level, tag, length, msg); - if (size > sizeof(buf)) { - size = snprintf(buf, sizeof(buf), "<%d>%s: %zu-byte message too long for printk\n", - level, tag, size); - } - - iovec iov[1]; - iov[0].iov_base = buf; - iov[0].iov_len = size; - TEMP_FAILURE_RETRY(writev(klog_fd, iov, 1)); -} - -void KernelLogger(android::base::LogId, android::base::LogSeverity severity, const char* tag, - const char*, unsigned int, const char* full_message) { - SplitByLines(full_message, KernelLogLine, severity, tag); -} -#endif - -void StderrLogger(LogId, LogSeverity severity, const char* tag, const char* file, unsigned int line, - const char* message) { - struct tm now; - time_t t = time(nullptr); - -#if defined(_WIN32) - localtime_s(&now, &t); -#else - localtime_r(&t, &now); -#endif - auto output_string = - StderrOutputGenerator(now, getpid(), GetThreadId(), severity, tag, file, line, message); - - fputs(output_string.c_str(), stderr); -} - -void StdioLogger(LogId, LogSeverity severity, const char* /*tag*/, const char* /*file*/, - unsigned int /*line*/, const char* message) { - if (severity >= WARNING) { - fflush(stdout); - fprintf(stderr, "%s: %s\n", GetFileBasename(getprogname()), message); - } else { - fprintf(stdout, "%s\n", message); - } -} - -void DefaultAborter(const char* abort_message) { -#ifdef __ANDROID__ - android_set_abort_message(abort_message); -#else - UNUSED(abort_message); -#endif - abort(); -} - -static void LogdLogChunk(LogId id, LogSeverity severity, const char* tag, const char* message) { - int32_t lg_id = LogIdTolog_id_t(id); - int32_t priority = LogSeverityToPriority(severity); - - static auto& liblog_functions = GetLibLogFunctions(); - if (liblog_functions) { - __android_log_message log_message = {sizeof(__android_log_message), lg_id, priority, tag, - static_cast<const char*>(nullptr), 0, message}; - liblog_functions->__android_log_logd_logger(&log_message); - } else { - __android_log_buf_print(lg_id, priority, tag, "%s", message); - } -} - -LogdLogger::LogdLogger(LogId default_log_id) : default_log_id_(default_log_id) {} - -void LogdLogger::operator()(LogId id, LogSeverity severity, const char* tag, const char* file, - unsigned int line, const char* message) { - if (id == DEFAULT) { - id = default_log_id_; - } - - SplitByLogdChunks(id, severity, tag, file, line, message, LogdLogChunk); -} - -void InitLogging(char* argv[], LogFunction&& logger, AbortFunction&& aborter) { - SetLogger(std::forward<LogFunction>(logger)); - SetAborter(std::forward<AbortFunction>(aborter)); - - if (gInitialized) { - return; - } - - gInitialized = true; - - // Stash the command line for later use. We can use /proc/self/cmdline on - // Linux to recover this, but we don't have that luxury on the Mac/Windows, - // and there are a couple of argv[0] variants that are commonly used. - if (argv != nullptr) { - SetDefaultTag(basename(argv[0])); - } - - const char* tags = getenv("ANDROID_LOG_TAGS"); - if (tags == nullptr) { - return; - } - - std::vector<std::string> specs = Split(tags, " "); - for (size_t i = 0; i < specs.size(); ++i) { - // "tag-pattern:[vdiwefs]" - std::string spec(specs[i]); - if (spec.size() == 3 && StartsWith(spec, "*:")) { - switch (spec[2]) { - case 'v': - SetMinimumLogSeverity(VERBOSE); - continue; - case 'd': - SetMinimumLogSeverity(DEBUG); - continue; - case 'i': - SetMinimumLogSeverity(INFO); - continue; - case 'w': - SetMinimumLogSeverity(WARNING); - continue; - case 'e': - SetMinimumLogSeverity(ERROR); - continue; - case 'f': - SetMinimumLogSeverity(FATAL_WITHOUT_ABORT); - continue; - // liblog will even suppress FATAL if you say 's' for silent, but that's - // crazy! - case 's': - SetMinimumLogSeverity(FATAL_WITHOUT_ABORT); - continue; - } - } - LOG(FATAL) << "unsupported '" << spec << "' in ANDROID_LOG_TAGS (" << tags - << ")"; - } -} - -void SetLogger(LogFunction&& logger) { - Logger() = std::move(logger); - - static auto& liblog_functions = GetLibLogFunctions(); - if (liblog_functions) { - liblog_functions->__android_log_set_logger([](const struct __android_log_message* log_message) { - auto log_id = log_id_tToLogId(log_message->buffer_id); - auto severity = PriorityToLogSeverity(log_message->priority); - - Logger()(log_id, severity, log_message->tag, log_message->file, log_message->line, - log_message->message); - }); - } -} - -void SetAborter(AbortFunction&& aborter) { - Aborter() = std::move(aborter); - - static auto& liblog_functions = GetLibLogFunctions(); - if (liblog_functions) { - liblog_functions->__android_log_set_aborter( - [](const char* abort_message) { Aborter()(abort_message); }); - } -} - -// This indirection greatly reduces the stack impact of having lots of -// checks/logging in a function. -class LogMessageData { - public: - LogMessageData(const char* file, unsigned int line, LogSeverity severity, const char* tag, - int error) - : file_(GetFileBasename(file)), - line_number_(line), - severity_(severity), - tag_(tag), - error_(error) {} - - const char* GetFile() const { - return file_; - } - - unsigned int GetLineNumber() const { - return line_number_; - } - - LogSeverity GetSeverity() const { - return severity_; - } - - const char* GetTag() const { return tag_; } - - int GetError() const { - return error_; - } - - std::ostream& GetBuffer() { - return buffer_; - } - - std::string ToString() const { - return buffer_.str(); - } - - private: - std::ostringstream buffer_; - const char* const file_; - const unsigned int line_number_; - const LogSeverity severity_; - const char* const tag_; - const int error_; - - DISALLOW_COPY_AND_ASSIGN(LogMessageData); -}; - -LogMessage::LogMessage(const char* file, unsigned int line, LogId, LogSeverity severity, - const char* tag, int error) - : LogMessage(file, line, severity, tag, error) {} - -LogMessage::LogMessage(const char* file, unsigned int line, LogSeverity severity, const char* tag, - int error) - : data_(new LogMessageData(file, line, severity, tag, error)) {} - -LogMessage::~LogMessage() { - // Check severity again. This is duplicate work wrt/ LOG macros, but not LOG_STREAM. - if (!WOULD_LOG(data_->GetSeverity())) { - return; - } - - // Finish constructing the message. - if (data_->GetError() != -1) { - data_->GetBuffer() << ": " << strerror(data_->GetError()); - } - std::string msg(data_->ToString()); - - if (data_->GetSeverity() == FATAL) { -#ifdef __ANDROID__ - // Set the bionic abort message early to avoid liblog doing it - // with the individual lines, so that we get the whole message. - android_set_abort_message(msg.c_str()); -#endif - } - - LogLine(data_->GetFile(), data_->GetLineNumber(), data_->GetSeverity(), data_->GetTag(), - msg.c_str()); - - // Abort if necessary. - if (data_->GetSeverity() == FATAL) { - static auto& liblog_functions = GetLibLogFunctions(); - if (liblog_functions) { - liblog_functions->__android_log_call_aborter(msg.c_str()); - } else { - Aborter()(msg.c_str()); - } - } -} - -std::ostream& LogMessage::stream() { - return data_->GetBuffer(); -} - -void LogMessage::LogLine(const char* file, unsigned int line, LogSeverity severity, const char* tag, - const char* message) { - static auto& liblog_functions = GetLibLogFunctions(); - int32_t priority = LogSeverityToPriority(severity); - if (liblog_functions) { - __android_log_message log_message = { - sizeof(__android_log_message), LOG_ID_DEFAULT, priority, tag, file, line, message}; - liblog_functions->__android_log_write_log_message(&log_message); - } else { - if (tag == nullptr) { - std::lock_guard<std::recursive_mutex> lock(TagLock()); - if (gDefaultTag == nullptr) { - gDefaultTag = new std::string(getprogname()); - } - - Logger()(DEFAULT, severity, gDefaultTag->c_str(), file, line, message); - } else { - Logger()(DEFAULT, severity, tag, file, line, message); - } - } -} - -LogSeverity GetMinimumLogSeverity() { - static auto& liblog_functions = GetLibLogFunctions(); - if (liblog_functions) { - return PriorityToLogSeverity(liblog_functions->__android_log_get_minimum_priority()); - } else { - return gMinimumLogSeverity; - } -} - -bool ShouldLog(LogSeverity severity, const char* tag) { - static auto& liblog_functions = GetLibLogFunctions(); - // Even though we're not using the R liblog functions in this function, if we're running on Q, - // we need to fall back to using gMinimumLogSeverity, since __android_log_is_loggable() will not - // take into consideration the value from SetMinimumLogSeverity(). - if (liblog_functions) { - int32_t priority = LogSeverityToPriority(severity); - return __android_log_is_loggable(priority, tag, ANDROID_LOG_INFO); - } else { - return severity >= gMinimumLogSeverity; - } -} - -LogSeverity SetMinimumLogSeverity(LogSeverity new_severity) { - static auto& liblog_functions = GetLibLogFunctions(); - if (liblog_functions) { - int32_t priority = LogSeverityToPriority(new_severity); - return PriorityToLogSeverity(liblog_functions->__android_log_set_minimum_priority(priority)); - } else { - LogSeverity old_severity = gMinimumLogSeverity; - gMinimumLogSeverity = new_severity; - return old_severity; - } -} - -ScopedLogSeverity::ScopedLogSeverity(LogSeverity new_severity) { - old_ = SetMinimumLogSeverity(new_severity); -} - -ScopedLogSeverity::~ScopedLogSeverity() { - SetMinimumLogSeverity(old_); -} - -} // namespace base -} // namespace android diff --git a/base/logging_splitters.h b/base/logging_splitters.h deleted file mode 100644 index 2ec2b205d..000000000 --- a/base/logging_splitters.h +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Copyright (C) 2020 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include <inttypes.h> - -#include <android-base/logging.h> -#include <android-base/stringprintf.h> - -#define LOGGER_ENTRY_MAX_PAYLOAD 4068 // This constant is not in the NDK. - -namespace android { -namespace base { - -// This splits the message up line by line, by calling log_function with a pointer to the start of -// each line and the size up to the newline character. It sends size = -1 for the final line. -template <typename F, typename... Args> -static void SplitByLines(const char* msg, const F& log_function, Args&&... args) { - const char* newline = strchr(msg, '\n'); - while (newline != nullptr) { - log_function(msg, newline - msg, args...); - msg = newline + 1; - newline = strchr(msg, '\n'); - } - - log_function(msg, -1, args...); -} - -// This splits the message up into chunks that logs can process delimited by new lines. It calls -// log_function with the exact null terminated message that should be sent to logd. -// Note, despite the loops and snprintf's, if severity is not fatal and there are no new lines, -// this function simply calls log_function with msg without any extra overhead. -template <typename F> -static void SplitByLogdChunks(LogId log_id, LogSeverity severity, const char* tag, const char* file, - unsigned int line, const char* msg, const F& log_function) { - // The maximum size of a payload, after the log header that logd will accept is - // LOGGER_ENTRY_MAX_PAYLOAD, so subtract the other elements in the payload to find the size of - // the string that we can log in each pass. - // The protocol is documented in liblog/README.protocol.md. - // Specifically we subtract a byte for the priority, the length of the tag + its null terminator, - // and an additional byte for the null terminator on the payload. We subtract an additional 32 - // bytes for slack, similar to java/android/util/Log.java. - ptrdiff_t max_size = LOGGER_ENTRY_MAX_PAYLOAD - strlen(tag) - 35; - if (max_size <= 0) { - abort(); - } - // If we're logging a fatal message, we'll append the file and line numbers. - bool add_file = file != nullptr && (severity == FATAL || severity == FATAL_WITHOUT_ABORT); - - std::string file_header; - if (add_file) { - file_header = StringPrintf("%s:%u] ", file, line); - } - int file_header_size = file_header.size(); - - __attribute__((uninitialized)) char logd_chunk[max_size + 1]; - ptrdiff_t chunk_position = 0; - - auto call_log_function = [&]() { - log_function(log_id, severity, tag, logd_chunk); - chunk_position = 0; - }; - - auto write_to_logd_chunk = [&](const char* message, int length) { - int size_written = 0; - const char* new_line = chunk_position > 0 ? "\n" : ""; - if (add_file) { - size_written = snprintf(logd_chunk + chunk_position, sizeof(logd_chunk) - chunk_position, - "%s%s%.*s", new_line, file_header.c_str(), length, message); - } else { - size_written = snprintf(logd_chunk + chunk_position, sizeof(logd_chunk) - chunk_position, - "%s%.*s", new_line, length, message); - } - - // This should never fail, if it does and we set size_written to 0, which will skip this line - // and move to the next one. - if (size_written < 0) { - size_written = 0; - } - chunk_position += size_written; - }; - - const char* newline = strchr(msg, '\n'); - while (newline != nullptr) { - // If we have data in the buffer and this next line doesn't fit, write the buffer. - if (chunk_position != 0 && chunk_position + (newline - msg) + 1 + file_header_size > max_size) { - call_log_function(); - } - - // Otherwise, either the next line fits or we have any empty buffer and too large of a line to - // ever fit, in both cases, we add it to the buffer and continue. - write_to_logd_chunk(msg, newline - msg); - - msg = newline + 1; - newline = strchr(msg, '\n'); - } - - // If we have left over data in the buffer and we can fit the rest of msg, add it to the buffer - // then write the buffer. - if (chunk_position != 0 && - chunk_position + static_cast<int>(strlen(msg)) + 1 + file_header_size <= max_size) { - write_to_logd_chunk(msg, -1); - call_log_function(); - } else { - // If the buffer is not empty and we can't fit the rest of msg into it, write its contents. - if (chunk_position != 0) { - call_log_function(); - } - // Then write the rest of the msg. - if (add_file) { - snprintf(logd_chunk, sizeof(logd_chunk), "%s%s", file_header.c_str(), msg); - log_function(log_id, severity, tag, logd_chunk); - } else { - log_function(log_id, severity, tag, msg); - } - } -} - -static std::pair<int, int> CountSizeAndNewLines(const char* message) { - int size = 0; - int new_lines = 0; - while (*message != '\0') { - size++; - if (*message == '\n') { - ++new_lines; - } - ++message; - } - return {size, new_lines}; -} - -// This adds the log header to each line of message and returns it as a string intended to be -// written to stderr. -static std::string StderrOutputGenerator(const struct tm& now, int pid, uint64_t tid, - LogSeverity severity, const char* tag, const char* file, - unsigned int line, const char* message) { - char timestamp[32]; - strftime(timestamp, sizeof(timestamp), "%m-%d %H:%M:%S", &now); - - static const char log_characters[] = "VDIWEFF"; - static_assert(arraysize(log_characters) - 1 == FATAL + 1, - "Mismatch in size of log_characters and values in LogSeverity"); - char severity_char = log_characters[severity]; - std::string line_prefix; - if (file != nullptr) { - line_prefix = StringPrintf("%s %c %s %5d %5" PRIu64 " %s:%u] ", tag ? tag : "nullptr", - severity_char, timestamp, pid, tid, file, line); - } else { - line_prefix = StringPrintf("%s %c %s %5d %5" PRIu64 " ", tag ? tag : "nullptr", severity_char, - timestamp, pid, tid); - } - - auto [size, new_lines] = CountSizeAndNewLines(message); - std::string output_string; - output_string.reserve(size + new_lines * line_prefix.size() + 1); - - auto concat_lines = [&](const char* message, int size) { - output_string.append(line_prefix); - if (size == -1) { - output_string.append(message); - } else { - output_string.append(message, size); - } - output_string.append("\n"); - }; - SplitByLines(message, concat_lines); - return output_string; -} - -} // namespace base -} // namespace android diff --git a/base/logging_splitters_test.cpp b/base/logging_splitters_test.cpp deleted file mode 100644 index 679d19ed1..000000000 --- a/base/logging_splitters_test.cpp +++ /dev/null @@ -1,325 +0,0 @@ -/* - * Copyright (C) 2020 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "logging_splitters.h" - -#include <string> -#include <vector> - -#include <android-base/strings.h> -#include <gtest/gtest.h> - -namespace android { -namespace base { - -void TestNewlineSplitter(const std::string& input, - const std::vector<std::string>& expected_output) { - std::vector<std::string> output; - auto logger_function = [&](const char* msg, int length) { - if (length == -1) { - output.push_back(msg); - } else { - output.push_back(std::string(msg, length)); - } - }; - SplitByLines(input.c_str(), logger_function); - - EXPECT_EQ(expected_output, output); -} - -TEST(logging_splitters, NewlineSplitter_EmptyString) { - TestNewlineSplitter("", std::vector<std::string>{""}); -} - -TEST(logging_splitters, NewlineSplitter_BasicString) { - TestNewlineSplitter("normal string", std::vector<std::string>{"normal string"}); -} - -TEST(logging_splitters, NewlineSplitter_ormalBasicStringTrailingNewline) { - TestNewlineSplitter("normal string\n", std::vector<std::string>{"normal string", ""}); -} - -TEST(logging_splitters, NewlineSplitter_MultilineTrailing) { - TestNewlineSplitter("normal string\nsecond string\nthirdstring", - std::vector<std::string>{"normal string", "second string", "thirdstring"}); -} - -TEST(logging_splitters, NewlineSplitter_MultilineTrailingNewline) { - TestNewlineSplitter( - "normal string\nsecond string\nthirdstring\n", - std::vector<std::string>{"normal string", "second string", "thirdstring", ""}); -} - -TEST(logging_splitters, NewlineSplitter_MultilineEmbeddedNewlines) { - TestNewlineSplitter( - "normal string\n\n\nsecond string\n\nthirdstring\n", - std::vector<std::string>{"normal string", "", "", "second string", "", "thirdstring", ""}); -} - -void TestLogdChunkSplitter(const std::string& tag, const std::string& file, - const std::string& input, - const std::vector<std::string>& expected_output) { - std::vector<std::string> output; - auto logger_function = [&](LogId, LogSeverity, const char*, const char* msg) { - output.push_back(msg); - }; - - SplitByLogdChunks(MAIN, FATAL, tag.c_str(), file.empty() ? nullptr : file.c_str(), 1000, - input.c_str(), logger_function); - - auto return_lengths = [&] { - std::string sizes; - sizes += "expected_output sizes:"; - for (const auto& string : expected_output) { - sizes += " " + std::to_string(string.size()); - } - sizes += "\noutput sizes:"; - for (const auto& string : output) { - sizes += " " + std::to_string(string.size()); - } - return sizes; - }; - - EXPECT_EQ(expected_output, output) << return_lengths(); -} - -TEST(logging_splitters, LogdChunkSplitter_EmptyString) { - TestLogdChunkSplitter("tag", "", "", std::vector<std::string>{""}); -} - -TEST(logging_splitters, LogdChunkSplitter_BasicString) { - TestLogdChunkSplitter("tag", "", "normal string", std::vector<std::string>{"normal string"}); -} - -TEST(logging_splitters, LogdChunkSplitter_NormalBasicStringTrailingNewline) { - TestLogdChunkSplitter("tag", "", "normal string\n", std::vector<std::string>{"normal string\n"}); -} - -TEST(logging_splitters, LogdChunkSplitter_MultilineTrailing) { - TestLogdChunkSplitter("tag", "", "normal string\nsecond string\nthirdstring", - std::vector<std::string>{"normal string\nsecond string\nthirdstring"}); -} - -TEST(logging_splitters, LogdChunkSplitter_MultilineTrailingNewline) { - TestLogdChunkSplitter("tag", "", "normal string\nsecond string\nthirdstring\n", - std::vector<std::string>{"normal string\nsecond string\nthirdstring\n"}); -} - -TEST(logging_splitters, LogdChunkSplitter_MultilineEmbeddedNewlines) { - TestLogdChunkSplitter( - "tag", "", "normal string\n\n\nsecond string\n\nthirdstring\n", - std::vector<std::string>{"normal string\n\n\nsecond string\n\nthirdstring\n"}); -} - -// This test should return the same string, the logd logger itself will truncate down to size. -// This has historically been the behavior both in libbase and liblog. -TEST(logging_splitters, LogdChunkSplitter_HugeLineNoNewline) { - auto long_string = std::string(LOGGER_ENTRY_MAX_PAYLOAD, 'x'); - ASSERT_EQ(LOGGER_ENTRY_MAX_PAYLOAD, static_cast<int>(long_string.size())); - - TestLogdChunkSplitter("tag", "", long_string, std::vector{long_string}); -} - -std::string ReduceToMaxSize(const std::string& tag, const std::string& string) { - return string.substr(0, LOGGER_ENTRY_MAX_PAYLOAD - tag.size() - 35); -} - -TEST(logging_splitters, LogdChunkSplitter_MultipleHugeLineNoNewline) { - auto long_string_x = std::string(LOGGER_ENTRY_MAX_PAYLOAD, 'x'); - auto long_string_y = std::string(LOGGER_ENTRY_MAX_PAYLOAD, 'y'); - auto long_string_z = std::string(LOGGER_ENTRY_MAX_PAYLOAD, 'z'); - - auto long_strings = long_string_x + '\n' + long_string_y + '\n' + long_string_z; - - std::string tag = "tag"; - std::vector expected = {ReduceToMaxSize(tag, long_string_x), ReduceToMaxSize(tag, long_string_y), - long_string_z}; - - TestLogdChunkSplitter(tag, "", long_strings, expected); -} - -// With a ~4k buffer, we should print 2 long strings per logger call. -TEST(logging_splitters, LogdChunkSplitter_Multiple2kLines) { - std::vector expected = { - std::string(2000, 'a') + '\n' + std::string(2000, 'b'), - std::string(2000, 'c') + '\n' + std::string(2000, 'd'), - std::string(2000, 'e') + '\n' + std::string(2000, 'f'), - }; - - auto long_strings = Join(expected, '\n'); - - TestLogdChunkSplitter("tag", "", long_strings, expected); -} - -TEST(logging_splitters, LogdChunkSplitter_ExactSizedLines) { - const char* tag = "tag"; - ptrdiff_t max_size = LOGGER_ENTRY_MAX_PAYLOAD - strlen(tag) - 35; - auto long_string_a = std::string(max_size, 'a'); - auto long_string_b = std::string(max_size, 'b'); - auto long_string_c = std::string(max_size, 'c'); - - auto long_strings = long_string_a + '\n' + long_string_b + '\n' + long_string_c; - - TestLogdChunkSplitter(tag, "", long_strings, - std::vector{long_string_a, long_string_b, long_string_c}); -} - -TEST(logging_splitters, LogdChunkSplitter_UnderEqualOver) { - std::string tag = "tag"; - ptrdiff_t max_size = LOGGER_ENTRY_MAX_PAYLOAD - tag.size() - 35; - - auto first_string_size = 1000; - auto first_string = std::string(first_string_size, 'a'); - auto second_string_size = max_size - first_string_size - 1; - auto second_string = std::string(second_string_size, 'b'); - - auto exact_string = std::string(max_size, 'c'); - - auto large_string = std::string(max_size + 50, 'd'); - - auto final_string = std::string("final string!\n\nfinal \n \n final \n"); - - std::vector expected = {first_string + '\n' + second_string, exact_string, - ReduceToMaxSize(tag, large_string), final_string}; - - std::vector input_strings = {first_string + '\n' + second_string, exact_string, large_string, - final_string}; - auto long_strings = Join(input_strings, '\n'); - - TestLogdChunkSplitter(tag, "", long_strings, expected); -} - -TEST(logging_splitters, LogdChunkSplitter_WithFile) { - std::string tag = "tag"; - std::string file = "/path/to/myfile.cpp"; - int line = 1000; - auto file_header = StringPrintf("%s:%d] ", file.c_str(), line); - ptrdiff_t max_size = LOGGER_ENTRY_MAX_PAYLOAD - tag.size() - 35; - - auto first_string_size = 1000; - auto first_string = std::string(first_string_size, 'a'); - auto second_string_size = max_size - first_string_size - 1 - 2 * file_header.size(); - auto second_string = std::string(second_string_size, 'b'); - - auto exact_string = std::string(max_size - file_header.size(), 'c'); - - auto large_string = std::string(max_size + 50, 'd'); - - auto final_string = std::string("final string!"); - - std::vector expected = { - file_header + first_string + '\n' + file_header + second_string, file_header + exact_string, - file_header + ReduceToMaxSize(file_header + tag, large_string), file_header + final_string}; - - std::vector input_strings = {first_string + '\n' + second_string, exact_string, large_string, - final_string}; - auto long_strings = Join(input_strings, '\n'); - - TestLogdChunkSplitter(tag, file, long_strings, expected); -} - -// We set max_size based off of tag, so if it's too large, the buffer will be sized wrong. -// We could recover from this, but it's certainly an error for someone to attempt to use a tag this -// large, so we abort instead. -TEST(logging_splitters, LogdChunkSplitter_TooLongTag) { - auto long_tag = std::string(5000, 'x'); - auto logger_function = [](LogId, LogSeverity, const char*, const char*) {}; - ASSERT_DEATH( - SplitByLogdChunks(MAIN, ERROR, long_tag.c_str(), nullptr, 0, "message", logger_function), ""); -} - -// We do handle excessively large file names correctly however. -TEST(logging_splitters, LogdChunkSplitter_TooLongFile) { - auto long_file = std::string(5000, 'x'); - std::string tag = "tag"; - - std::vector expected = {ReduceToMaxSize(tag, long_file), ReduceToMaxSize(tag, long_file)}; - - TestLogdChunkSplitter(tag, long_file, "can't see me\nor me", expected); -} - -void TestStderrOutputGenerator(const char* tag, const char* file, int line, const char* message, - const std::string& expected) { - // All log messages will show "01-01 00:00:00" - struct tm now = { - .tm_sec = 0, - .tm_min = 0, - .tm_hour = 0, - .tm_mday = 1, - .tm_mon = 0, - .tm_year = 1970, - }; - - int pid = 1234; // All log messages will have 1234 for their PID. - uint64_t tid = 4321; // All log messages will have 4321 for their TID. - - auto result = StderrOutputGenerator(now, pid, tid, ERROR, tag, file, line, message); - EXPECT_EQ(expected, result); -} - -TEST(logging_splitters, StderrOutputGenerator_Basic) { - TestStderrOutputGenerator(nullptr, nullptr, 0, "simple message", - "nullptr E 01-01 00:00:00 1234 4321 simple message\n"); - TestStderrOutputGenerator("tag", nullptr, 0, "simple message", - "tag E 01-01 00:00:00 1234 4321 simple message\n"); - TestStderrOutputGenerator( - "tag", "/path/to/some/file", 0, "simple message", - "tag E 01-01 00:00:00 1234 4321 /path/to/some/file:0] simple message\n"); -} - -TEST(logging_splitters, StderrOutputGenerator_NewlineTagAndFile) { - TestStderrOutputGenerator("tag\n\n", nullptr, 0, "simple message", - "tag\n\n E 01-01 00:00:00 1234 4321 simple message\n"); - TestStderrOutputGenerator( - "tag", "/path/to/some/file\n\n", 0, "simple message", - "tag E 01-01 00:00:00 1234 4321 /path/to/some/file\n\n:0] simple message\n"); -} - -TEST(logging_splitters, StderrOutputGenerator_TrailingNewLine) { - TestStderrOutputGenerator( - "tag", nullptr, 0, "simple message\n", - "tag E 01-01 00:00:00 1234 4321 simple message\ntag E 01-01 00:00:00 1234 4321 \n"); -} - -TEST(logging_splitters, StderrOutputGenerator_MultiLine) { - const char* expected_result = - "tag E 01-01 00:00:00 1234 4321 simple message\n" - "tag E 01-01 00:00:00 1234 4321 \n" - "tag E 01-01 00:00:00 1234 4321 \n" - "tag E 01-01 00:00:00 1234 4321 another message \n" - "tag E 01-01 00:00:00 1234 4321 \n" - "tag E 01-01 00:00:00 1234 4321 final message \n" - "tag E 01-01 00:00:00 1234 4321 \n" - "tag E 01-01 00:00:00 1234 4321 \n" - "tag E 01-01 00:00:00 1234 4321 \n"; - - TestStderrOutputGenerator("tag", nullptr, 0, - "simple message\n\n\nanother message \n\n final message \n\n\n", - expected_result); -} - -TEST(logging_splitters, StderrOutputGenerator_MultiLineLong) { - auto long_string_a = std::string(4000, 'a'); - auto long_string_b = std::string(4000, 'b'); - - auto message = long_string_a + '\n' + long_string_b; - auto expected_result = "tag E 01-01 00:00:00 1234 4321 " + long_string_a + '\n' + - "tag E 01-01 00:00:00 1234 4321 " + long_string_b + '\n'; - TestStderrOutputGenerator("tag", nullptr, 0, message.c_str(), expected_result); -} - -} // namespace base -} // namespace android diff --git a/base/logging_test.cpp b/base/logging_test.cpp deleted file mode 100644 index 593e2c100..000000000 --- a/base/logging_test.cpp +++ /dev/null @@ -1,673 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "android-base/logging.h" - -#include <libgen.h> - -#if defined(_WIN32) -#include <signal.h> -#endif - -#include <regex> -#include <string> -#include <thread> - -#include "android-base/file.h" -#include "android-base/scopeguard.h" -#include "android-base/stringprintf.h" -#include "android-base/test_utils.h" - -#include <gtest/gtest.h> - -#ifdef __ANDROID__ -#define HOST_TEST(suite, name) TEST(suite, DISABLED_ ## name) -#else -#define HOST_TEST(suite, name) TEST(suite, name) -#endif - -#if defined(_WIN32) -static void ExitSignalAbortHandler(int) { - _exit(3); -} -#endif - -static void SuppressAbortUI() { -#if defined(_WIN32) - // We really just want to call _set_abort_behavior(0, _CALL_REPORTFAULT) to - // suppress the Windows Error Reporting dialog box, but that API is not - // available in the OS-supplied C Runtime, msvcrt.dll, that we currently - // use (it is available in the Visual Studio C runtime). - // - // Instead, we setup a SIGABRT handler, which is called in abort() right - // before calling Windows Error Reporting. In the handler, we exit the - // process just like abort() does. - ASSERT_NE(SIG_ERR, signal(SIGABRT, ExitSignalAbortHandler)); -#endif -} - -TEST(logging, CHECK) { - ASSERT_DEATH({SuppressAbortUI(); CHECK(false);}, "Check failed: false "); - CHECK(true); - - ASSERT_DEATH({SuppressAbortUI(); CHECK_EQ(0, 1);}, "Check failed: 0 == 1 "); - CHECK_EQ(0, 0); - - ASSERT_DEATH({SuppressAbortUI(); CHECK_STREQ("foo", "bar");}, - R"(Check failed: "foo" == "bar")"); - CHECK_STREQ("foo", "foo"); - - // Test whether CHECK() and CHECK_STREQ() have a dangling if with no else. - bool flag = false; - if (true) - CHECK(true); - else - flag = true; - EXPECT_FALSE(flag) << "CHECK macro probably has a dangling if with no else"; - - flag = false; - if (true) - CHECK_STREQ("foo", "foo"); - else - flag = true; - EXPECT_FALSE(flag) << "CHECK_STREQ probably has a dangling if with no else"; -} - -TEST(logging, DCHECK) { - if (android::base::kEnableDChecks) { - ASSERT_DEATH({SuppressAbortUI(); DCHECK(false);}, "DCheck failed: false "); - } - DCHECK(true); - - if (android::base::kEnableDChecks) { - ASSERT_DEATH({SuppressAbortUI(); DCHECK_EQ(0, 1);}, "DCheck failed: 0 == 1 "); - } - DCHECK_EQ(0, 0); - - if (android::base::kEnableDChecks) { - ASSERT_DEATH({SuppressAbortUI(); DCHECK_STREQ("foo", "bar");}, - R"(DCheck failed: "foo" == "bar")"); - } - DCHECK_STREQ("foo", "foo"); - - // No testing whether we have a dangling else, possibly. That's inherent to the if (constexpr) - // setup we intentionally chose to force type-checks of debug code even in release builds (so - // we don't get more bit-rot). -} - - -#define CHECK_WOULD_LOG_DISABLED(severity) \ - static_assert(android::base::severity < android::base::FATAL, "Bad input"); \ - for (size_t i = static_cast<size_t>(android::base::severity) + 1; \ - i <= static_cast<size_t>(android::base::FATAL); \ - ++i) { \ - { \ - android::base::ScopedLogSeverity sls2(static_cast<android::base::LogSeverity>(i)); \ - EXPECT_FALSE(WOULD_LOG(severity)) << i; \ - } \ - { \ - android::base::ScopedLogSeverity sls2(static_cast<android::base::LogSeverity>(i)); \ - EXPECT_FALSE(WOULD_LOG(::android::base::severity)) << i; \ - } \ - } \ - -#define CHECK_WOULD_LOG_ENABLED(severity) \ - for (size_t i = static_cast<size_t>(android::base::VERBOSE); \ - i <= static_cast<size_t>(android::base::severity); \ - ++i) { \ - { \ - android::base::ScopedLogSeverity sls2(static_cast<android::base::LogSeverity>(i)); \ - EXPECT_TRUE(WOULD_LOG(severity)) << i; \ - } \ - { \ - android::base::ScopedLogSeverity sls2(static_cast<android::base::LogSeverity>(i)); \ - EXPECT_TRUE(WOULD_LOG(::android::base::severity)) << i; \ - } \ - } \ - -TEST(logging, WOULD_LOG_FATAL) { - CHECK_WOULD_LOG_ENABLED(FATAL); -} - -TEST(logging, WOULD_LOG_FATAL_WITHOUT_ABORT_enabled) { - CHECK_WOULD_LOG_ENABLED(FATAL_WITHOUT_ABORT); -} - -TEST(logging, WOULD_LOG_ERROR_disabled) { - CHECK_WOULD_LOG_DISABLED(ERROR); -} - -TEST(logging, WOULD_LOG_ERROR_enabled) { - CHECK_WOULD_LOG_ENABLED(ERROR); -} - -TEST(logging, WOULD_LOG_WARNING_disabled) { - CHECK_WOULD_LOG_DISABLED(WARNING); -} - -TEST(logging, WOULD_LOG_WARNING_enabled) { - CHECK_WOULD_LOG_ENABLED(WARNING); -} - -TEST(logging, WOULD_LOG_INFO_disabled) { - CHECK_WOULD_LOG_DISABLED(INFO); -} - -TEST(logging, WOULD_LOG_INFO_enabled) { - CHECK_WOULD_LOG_ENABLED(INFO); -} - -TEST(logging, WOULD_LOG_DEBUG_disabled) { - CHECK_WOULD_LOG_DISABLED(DEBUG); -} - -TEST(logging, WOULD_LOG_DEBUG_enabled) { - CHECK_WOULD_LOG_ENABLED(DEBUG); -} - -TEST(logging, WOULD_LOG_VERBOSE_disabled) { - CHECK_WOULD_LOG_DISABLED(VERBOSE); -} - -TEST(logging, WOULD_LOG_VERBOSE_enabled) { - CHECK_WOULD_LOG_ENABLED(VERBOSE); -} - -#undef CHECK_WOULD_LOG_DISABLED -#undef CHECK_WOULD_LOG_ENABLED - - -#if !defined(_WIN32) -static std::string make_log_pattern(android::base::LogSeverity severity, - const char* message) { - static const char log_characters[] = "VDIWEFF"; - static_assert(arraysize(log_characters) - 1 == android::base::FATAL + 1, - "Mismatch in size of log_characters and values in LogSeverity"); - char log_char = log_characters[severity]; - std::string holder(__FILE__); - return android::base::StringPrintf( - "%c \\d+-\\d+ \\d+:\\d+:\\d+ \\s*\\d+ \\s*\\d+ %s:\\d+] %s", - log_char, basename(&holder[0]), message); -} -#endif - -static void CheckMessage(const std::string& output, android::base::LogSeverity severity, - const char* expected, const char* expected_tag = nullptr) { - // We can't usefully check the output of any of these on Windows because we - // don't have std::regex, but we can at least make sure we printed at least as - // many characters are in the log message. - ASSERT_GT(output.length(), strlen(expected)); - ASSERT_NE(nullptr, strstr(output.c_str(), expected)) << output; - if (expected_tag != nullptr) { - ASSERT_NE(nullptr, strstr(output.c_str(), expected_tag)) << output; - } - -#if !defined(_WIN32) - std::string regex_str; - if (expected_tag != nullptr) { - regex_str.append(expected_tag); - regex_str.append(" "); - } - regex_str.append(make_log_pattern(severity, expected)); - std::regex message_regex(regex_str); - ASSERT_TRUE(std::regex_search(output, message_regex)) << output; -#endif -} - -static void CheckMessage(CapturedStderr& cap, android::base::LogSeverity severity, - const char* expected, const char* expected_tag = nullptr) { - cap.Stop(); - std::string output = cap.str(); - return CheckMessage(output, severity, expected, expected_tag); -} - -#define CHECK_LOG_STREAM_DISABLED(severity) \ - { \ - android::base::ScopedLogSeverity sls1(android::base::FATAL); \ - CapturedStderr cap1; \ - LOG_STREAM(severity) << "foo bar"; \ - cap1.Stop(); \ - ASSERT_EQ("", cap1.str()); \ - } \ - { \ - android::base::ScopedLogSeverity sls1(android::base::FATAL); \ - CapturedStderr cap1; \ - LOG_STREAM(::android::base::severity) << "foo bar"; \ - cap1.Stop(); \ - ASSERT_EQ("", cap1.str()); \ - } - -#define CHECK_LOG_STREAM_ENABLED(severity) \ - { \ - android::base::ScopedLogSeverity sls2(android::base::severity); \ - CapturedStderr cap2; \ - LOG_STREAM(severity) << "foobar"; \ - CheckMessage(cap2, android::base::severity, "foobar"); \ - } \ - { \ - android::base::ScopedLogSeverity sls2(android::base::severity); \ - CapturedStderr cap2; \ - LOG_STREAM(::android::base::severity) << "foobar"; \ - CheckMessage(cap2, android::base::severity, "foobar"); \ - } \ - -TEST(logging, LOG_STREAM_FATAL_WITHOUT_ABORT_enabled) { - ASSERT_NO_FATAL_FAILURE(CHECK_LOG_STREAM_ENABLED(FATAL_WITHOUT_ABORT)); -} - -TEST(logging, LOG_STREAM_ERROR_disabled) { - CHECK_LOG_STREAM_DISABLED(ERROR); -} - -TEST(logging, LOG_STREAM_ERROR_enabled) { - ASSERT_NO_FATAL_FAILURE(CHECK_LOG_STREAM_ENABLED(ERROR)); -} - -TEST(logging, LOG_STREAM_WARNING_disabled) { - CHECK_LOG_STREAM_DISABLED(WARNING); -} - -TEST(logging, LOG_STREAM_WARNING_enabled) { - ASSERT_NO_FATAL_FAILURE(CHECK_LOG_STREAM_ENABLED(WARNING)); -} - -TEST(logging, LOG_STREAM_INFO_disabled) { - CHECK_LOG_STREAM_DISABLED(INFO); -} - -TEST(logging, LOG_STREAM_INFO_enabled) { - ASSERT_NO_FATAL_FAILURE(CHECK_LOG_STREAM_ENABLED(INFO)); -} - -TEST(logging, LOG_STREAM_DEBUG_disabled) { - CHECK_LOG_STREAM_DISABLED(DEBUG); -} - -TEST(logging, LOG_STREAM_DEBUG_enabled) { - ASSERT_NO_FATAL_FAILURE(CHECK_LOG_STREAM_ENABLED(DEBUG)); -} - -TEST(logging, LOG_STREAM_VERBOSE_disabled) { - CHECK_LOG_STREAM_DISABLED(VERBOSE); -} - -TEST(logging, LOG_STREAM_VERBOSE_enabled) { - ASSERT_NO_FATAL_FAILURE(CHECK_LOG_STREAM_ENABLED(VERBOSE)); -} - -#undef CHECK_LOG_STREAM_DISABLED -#undef CHECK_LOG_STREAM_ENABLED - -#define CHECK_LOG_DISABLED(severity) \ - { \ - android::base::ScopedLogSeverity sls1(android::base::FATAL); \ - CapturedStderr cap1; \ - LOG(severity) << "foo bar"; \ - cap1.Stop(); \ - ASSERT_EQ("", cap1.str()); \ - } \ - { \ - android::base::ScopedLogSeverity sls1(android::base::FATAL); \ - CapturedStderr cap1; \ - LOG(::android::base::severity) << "foo bar"; \ - cap1.Stop(); \ - ASSERT_EQ("", cap1.str()); \ - } - -#define CHECK_LOG_ENABLED(severity) \ - { \ - android::base::ScopedLogSeverity sls2(android::base::severity); \ - CapturedStderr cap2; \ - LOG(severity) << "foobar"; \ - CheckMessage(cap2, android::base::severity, "foobar"); \ - } \ - { \ - android::base::ScopedLogSeverity sls2(android::base::severity); \ - CapturedStderr cap2; \ - LOG(::android::base::severity) << "foobar"; \ - CheckMessage(cap2, android::base::severity, "foobar"); \ - } \ - -TEST(logging, LOG_FATAL) { - ASSERT_DEATH({SuppressAbortUI(); LOG(FATAL) << "foobar";}, "foobar"); - ASSERT_DEATH({SuppressAbortUI(); LOG(::android::base::FATAL) << "foobar";}, "foobar"); -} - -TEST(logging, LOG_FATAL_WITHOUT_ABORT_enabled) { - ASSERT_NO_FATAL_FAILURE(CHECK_LOG_ENABLED(FATAL_WITHOUT_ABORT)); -} - -TEST(logging, LOG_ERROR_disabled) { - CHECK_LOG_DISABLED(ERROR); -} - -TEST(logging, LOG_ERROR_enabled) { - ASSERT_NO_FATAL_FAILURE(CHECK_LOG_ENABLED(ERROR)); -} - -TEST(logging, LOG_WARNING_disabled) { - CHECK_LOG_DISABLED(WARNING); -} - -TEST(logging, LOG_WARNING_enabled) { - ASSERT_NO_FATAL_FAILURE(CHECK_LOG_ENABLED(WARNING)); -} - -TEST(logging, LOG_INFO_disabled) { - CHECK_LOG_DISABLED(INFO); -} - -TEST(logging, LOG_INFO_enabled) { - ASSERT_NO_FATAL_FAILURE(CHECK_LOG_ENABLED(INFO)); -} - -TEST(logging, LOG_DEBUG_disabled) { - CHECK_LOG_DISABLED(DEBUG); -} - -TEST(logging, LOG_DEBUG_enabled) { - ASSERT_NO_FATAL_FAILURE(CHECK_LOG_ENABLED(DEBUG)); -} - -TEST(logging, LOG_VERBOSE_disabled) { - CHECK_LOG_DISABLED(VERBOSE); -} - -TEST(logging, LOG_VERBOSE_enabled) { - ASSERT_NO_FATAL_FAILURE(CHECK_LOG_ENABLED(VERBOSE)); -} - -#undef CHECK_LOG_DISABLED -#undef CHECK_LOG_ENABLED - -TEST(logging, LOG_complex_param) { -#define CHECK_LOG_COMBINATION(use_scoped_log_severity_info, use_logging_severity_info) \ - { \ - android::base::ScopedLogSeverity sls( \ - (use_scoped_log_severity_info) ? ::android::base::INFO : ::android::base::WARNING); \ - CapturedStderr cap; \ - LOG((use_logging_severity_info) ? ::android::base::INFO : ::android::base::WARNING) \ - << "foobar"; \ - if ((use_scoped_log_severity_info) || !(use_logging_severity_info)) { \ - ASSERT_NO_FATAL_FAILURE(CheckMessage( \ - cap, (use_logging_severity_info) ? ::android::base::INFO : ::android::base::WARNING, \ - "foobar")); \ - } else { \ - cap.Stop(); \ - ASSERT_EQ("", cap.str()); \ - } \ - } - - CHECK_LOG_COMBINATION(false,false); - CHECK_LOG_COMBINATION(false,true); - CHECK_LOG_COMBINATION(true,false); - CHECK_LOG_COMBINATION(true,true); - -#undef CHECK_LOG_COMBINATION -} - - -TEST(logging, LOG_does_not_clobber_errno) { - CapturedStderr cap; - errno = 12345; - LOG(INFO) << (errno = 67890); - EXPECT_EQ(12345, errno) << "errno was not restored"; - - ASSERT_NO_FATAL_FAILURE(CheckMessage(cap, android::base::INFO, "67890")); -} - -TEST(logging, PLOG_does_not_clobber_errno) { - CapturedStderr cap; - errno = 12345; - PLOG(INFO) << (errno = 67890); - EXPECT_EQ(12345, errno) << "errno was not restored"; - - ASSERT_NO_FATAL_FAILURE(CheckMessage(cap, android::base::INFO, "67890")); -} - -TEST(logging, LOG_does_not_have_dangling_if) { - CapturedStderr cap; // So the logging below has no side-effects. - - // Do the test two ways: once where we hypothesize that LOG()'s if - // will evaluate to true (when severity is high enough) and once when we - // expect it to evaluate to false (when severity is not high enough). - bool flag = false; - if (true) - LOG(INFO) << "foobar"; - else - flag = true; - - EXPECT_FALSE(flag) << "LOG macro probably has a dangling if with no else"; - - flag = false; - if (true) - LOG(VERBOSE) << "foobar"; - else - flag = true; - - EXPECT_FALSE(flag) << "LOG macro probably has a dangling if with no else"; -} - -#define CHECK_PLOG_DISABLED(severity) \ - { \ - android::base::ScopedLogSeverity sls1(android::base::FATAL); \ - CapturedStderr cap1; \ - PLOG(severity) << "foo bar"; \ - cap1.Stop(); \ - ASSERT_EQ("", cap1.str()); \ - } \ - { \ - android::base::ScopedLogSeverity sls1(android::base::FATAL); \ - CapturedStderr cap1; \ - PLOG(severity) << "foo bar"; \ - cap1.Stop(); \ - ASSERT_EQ("", cap1.str()); \ - } - -#define CHECK_PLOG_ENABLED(severity) \ - { \ - android::base::ScopedLogSeverity sls2(android::base::severity); \ - CapturedStderr cap2; \ - errno = ENOENT; \ - PLOG(severity) << "foobar"; \ - CheckMessage(cap2, android::base::severity, "foobar: No such file or directory"); \ - } \ - { \ - android::base::ScopedLogSeverity sls2(android::base::severity); \ - CapturedStderr cap2; \ - errno = ENOENT; \ - PLOG(severity) << "foobar"; \ - CheckMessage(cap2, android::base::severity, "foobar: No such file or directory"); \ - } \ - -TEST(logging, PLOG_FATAL) { - ASSERT_DEATH({SuppressAbortUI(); PLOG(FATAL) << "foobar";}, "foobar"); - ASSERT_DEATH({SuppressAbortUI(); PLOG(::android::base::FATAL) << "foobar";}, "foobar"); -} - -TEST(logging, PLOG_FATAL_WITHOUT_ABORT_enabled) { - ASSERT_NO_FATAL_FAILURE(CHECK_PLOG_ENABLED(FATAL_WITHOUT_ABORT)); -} - -TEST(logging, PLOG_ERROR_disabled) { - CHECK_PLOG_DISABLED(ERROR); -} - -TEST(logging, PLOG_ERROR_enabled) { - ASSERT_NO_FATAL_FAILURE(CHECK_PLOG_ENABLED(ERROR)); -} - -TEST(logging, PLOG_WARNING_disabled) { - CHECK_PLOG_DISABLED(WARNING); -} - -TEST(logging, PLOG_WARNING_enabled) { - ASSERT_NO_FATAL_FAILURE(CHECK_PLOG_ENABLED(WARNING)); -} - -TEST(logging, PLOG_INFO_disabled) { - CHECK_PLOG_DISABLED(INFO); -} - -TEST(logging, PLOG_INFO_enabled) { - ASSERT_NO_FATAL_FAILURE(CHECK_PLOG_ENABLED(INFO)); -} - -TEST(logging, PLOG_DEBUG_disabled) { - CHECK_PLOG_DISABLED(DEBUG); -} - -TEST(logging, PLOG_DEBUG_enabled) { - ASSERT_NO_FATAL_FAILURE(CHECK_PLOG_ENABLED(DEBUG)); -} - -TEST(logging, PLOG_VERBOSE_disabled) { - CHECK_PLOG_DISABLED(VERBOSE); -} - -TEST(logging, PLOG_VERBOSE_enabled) { - ASSERT_NO_FATAL_FAILURE(CHECK_PLOG_ENABLED(VERBOSE)); -} - -#undef CHECK_PLOG_DISABLED -#undef CHECK_PLOG_ENABLED - - -TEST(logging, UNIMPLEMENTED) { - std::string expected = android::base::StringPrintf("%s unimplemented ", __PRETTY_FUNCTION__); - - CapturedStderr cap; - errno = ENOENT; - UNIMPLEMENTED(ERROR); - ASSERT_NO_FATAL_FAILURE(CheckMessage(cap, android::base::ERROR, expected.c_str())); -} - -static void NoopAborter(const char* msg ATTRIBUTE_UNUSED) { - LOG(ERROR) << "called noop"; -} - -TEST(logging, LOG_FATAL_NOOP_ABORTER) { - CapturedStderr cap; - { - android::base::SetAborter(NoopAborter); - - android::base::ScopedLogSeverity sls(android::base::ERROR); - LOG(FATAL) << "foobar"; - cap.Stop(); - - android::base::SetAborter(android::base::DefaultAborter); - } - std::string output = cap.str(); - ASSERT_NO_FATAL_FAILURE(CheckMessage(output, android::base::FATAL, "foobar")); - ASSERT_NO_FATAL_FAILURE(CheckMessage(output, android::base::ERROR, "called noop")); - - ASSERT_DEATH({SuppressAbortUI(); LOG(FATAL) << "foobar";}, "foobar"); -} - -struct CountLineAborter { - static void CountLineAborterFunction(const char* msg) { - while (*msg != 0) { - if (*msg == '\n') { - newline_count++; - } - msg++; - } - } - static size_t newline_count; -}; -size_t CountLineAborter::newline_count = 0; - -TEST(logging, LOG_FATAL_ABORTER_MESSAGE) { - CountLineAborter::newline_count = 0; - android::base::SetAborter(CountLineAborter::CountLineAborterFunction); - - android::base::ScopedLogSeverity sls(android::base::ERROR); - CapturedStderr cap; - LOG(FATAL) << "foo\nbar"; - - EXPECT_EQ(CountLineAborter::newline_count, 1U); -} - -__attribute__((constructor)) void TestLoggingInConstructor() { - LOG(ERROR) << "foobar"; -} - -TEST(logging, StdioLogger) { - CapturedStderr cap_err; - CapturedStdout cap_out; - android::base::SetLogger(android::base::StdioLogger); - LOG(INFO) << "out"; - LOG(ERROR) << "err"; - cap_err.Stop(); - cap_out.Stop(); - - // For INFO we expect just the literal "out\n". - ASSERT_EQ("out\n", cap_out.str()); - // Whereas ERROR logging includes the program name. - ASSERT_EQ(android::base::Basename(android::base::GetExecutablePath()) + ": err\n", cap_err.str()); -} - -TEST(logging, ForkSafe) { -#if !defined(_WIN32) - using namespace android::base; - SetLogger( - [&](LogId, LogSeverity, const char*, const char*, unsigned int, const char*) { sleep(3); }); - - auto guard = make_scope_guard([&] { -#ifdef __ANDROID__ - SetLogger(LogdLogger()); -#else - SetLogger(StderrLogger); -#endif - }); - - auto thread = std::thread([] { - LOG(ERROR) << "This should sleep for 3 seconds, long enough to fork another process, if there " - "is no intervention"; - }); - thread.detach(); - - auto pid = fork(); - ASSERT_NE(-1, pid); - - if (pid == 0) { - // Reset the logger, so the next message doesn't sleep(). - SetLogger([](LogId, LogSeverity, const char*, const char*, unsigned int, const char*) {}); - LOG(ERROR) << "This should succeed in the child, only if libbase is forksafe."; - _exit(EXIT_SUCCESS); - } - - // Wait for up to 3 seconds for the child to exit. - int tries = 3; - bool found_child = false; - while (tries-- > 0) { - auto result = waitpid(pid, nullptr, WNOHANG); - EXPECT_NE(-1, result); - if (result == pid) { - found_child = true; - break; - } - sleep(1); - } - - EXPECT_TRUE(found_child); - - // Kill the child if it did not exit. - if (!found_child) { - kill(pid, SIGKILL); - } -#endif -} diff --git a/base/macros_test.cpp b/base/macros_test.cpp deleted file mode 100644 index 2b522db5c..000000000 --- a/base/macros_test.cpp +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "android-base/macros.h" - -#include <stdint.h> - -#include <gtest/gtest.h> - -TEST(macros, SIZEOF_MEMBER_macro) { - struct S { - int32_t i32; - double d; - }; - ASSERT_EQ(4U, SIZEOF_MEMBER(S, i32)); - ASSERT_EQ(8U, SIZEOF_MEMBER(S, d)); -} diff --git a/base/mapped_file.cpp b/base/mapped_file.cpp deleted file mode 100644 index fff345384..000000000 --- a/base/mapped_file.cpp +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "android-base/mapped_file.h" - -#include <utility> - -#include <errno.h> - -namespace android { -namespace base { - -static constexpr char kEmptyBuffer[] = {'0'}; - -static off64_t InitPageSize() { -#if defined(_WIN32) - SYSTEM_INFO si; - GetSystemInfo(&si); - return si.dwAllocationGranularity; -#else - return sysconf(_SC_PAGE_SIZE); -#endif -} - -std::unique_ptr<MappedFile> MappedFile::FromFd(borrowed_fd fd, off64_t offset, size_t length, - int prot) { -#if defined(_WIN32) - return FromOsHandle(reinterpret_cast<HANDLE>(_get_osfhandle(fd.get())), offset, length, prot); -#else - return FromOsHandle(fd.get(), offset, length, prot); -#endif -} - -std::unique_ptr<MappedFile> MappedFile::FromOsHandle(os_handle h, off64_t offset, size_t length, - int prot) { - static const off64_t page_size = InitPageSize(); - size_t slop = offset % page_size; - off64_t file_offset = offset - slop; - off64_t file_length = length + slop; - -#if defined(_WIN32) - HANDLE handle = CreateFileMappingW( - h, nullptr, (prot & PROT_WRITE) ? PAGE_READWRITE : PAGE_READONLY, 0, 0, nullptr); - if (handle == nullptr) { - // http://b/119818070 "app crashes when reading asset of zero length". - // Return a MappedFile that's only valid for reading the size. - if (length == 0 && ::GetLastError() == ERROR_FILE_INVALID) { - return std::unique_ptr<MappedFile>( - new MappedFile(const_cast<char*>(kEmptyBuffer), 0, 0, nullptr)); - } - return nullptr; - } - void* base = MapViewOfFile(handle, (prot & PROT_WRITE) ? FILE_MAP_ALL_ACCESS : FILE_MAP_READ, 0, - file_offset, file_length); - if (base == nullptr) { - CloseHandle(handle); - return nullptr; - } - return std::unique_ptr<MappedFile>( - new MappedFile(static_cast<char*>(base), length, slop, handle)); -#else - void* base = mmap(nullptr, file_length, prot, MAP_SHARED, h, file_offset); - if (base == MAP_FAILED) { - // http://b/119818070 "app crashes when reading asset of zero length". - // mmap fails with EINVAL for a zero length region. - if (errno == EINVAL && length == 0) { - return std::unique_ptr<MappedFile>(new MappedFile(const_cast<char*>(kEmptyBuffer), 0, 0)); - } - return nullptr; - } - return std::unique_ptr<MappedFile>(new MappedFile(static_cast<char*>(base), length, slop)); -#endif -} - -MappedFile::MappedFile(MappedFile&& other) - : base_(std::exchange(other.base_, nullptr)), - size_(std::exchange(other.size_, 0)), - offset_(std::exchange(other.offset_, 0)) -#ifdef _WIN32 - , - handle_(std::exchange(other.handle_, nullptr)) -#endif -{ -} - -MappedFile& MappedFile::operator=(MappedFile&& other) { - Close(); - base_ = std::exchange(other.base_, nullptr); - size_ = std::exchange(other.size_, 0); - offset_ = std::exchange(other.offset_, 0); -#ifdef _WIN32 - handle_ = std::exchange(other.handle_, nullptr); -#endif - return *this; -} - -MappedFile::~MappedFile() { - Close(); -} - -void MappedFile::Close() { -#if defined(_WIN32) - if (base_ != nullptr && size_ != 0) UnmapViewOfFile(base_); - if (handle_ != nullptr) CloseHandle(handle_); - handle_ = nullptr; -#else - if (base_ != nullptr && size_ != 0) munmap(base_, size_ + offset_); -#endif - - base_ = nullptr; - offset_ = size_ = 0; -} - -} // namespace base -} // namespace android diff --git a/base/mapped_file_test.cpp b/base/mapped_file_test.cpp deleted file mode 100644 index d21703c78..000000000 --- a/base/mapped_file_test.cpp +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "android-base/mapped_file.h" - -#include <gtest/gtest.h> - -#include <errno.h> -#include <fcntl.h> -#include <unistd.h> - -#include <string> - -#include "android-base/file.h" - -TEST(mapped_file, smoke) { - TemporaryFile tf; - ASSERT_TRUE(tf.fd != -1); - ASSERT_TRUE(android::base::WriteStringToFd("hello world", tf.fd)); - - auto m = android::base::MappedFile::FromFd(tf.fd, 3, 2, PROT_READ); - ASSERT_EQ(2u, m->size()); - ASSERT_EQ('l', m->data()[0]); - ASSERT_EQ('o', m->data()[1]); -} - -TEST(mapped_file, zero_length_mapping) { - // http://b/119818070 "app crashes when reading asset of zero length". - // mmap fails with EINVAL for a zero length region. - TemporaryFile tf; - ASSERT_TRUE(tf.fd != -1); - - auto m = android::base::MappedFile::FromFd(tf.fd, 4096, 0, PROT_READ); - EXPECT_EQ(0u, m->size()); - EXPECT_NE(nullptr, m->data()); -} diff --git a/base/no_destructor_test.cpp b/base/no_destructor_test.cpp deleted file mode 100644 index f19468abf..000000000 --- a/base/no_destructor_test.cpp +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "android-base/no_destructor.h" - -#include <gtest/gtest.h> - -struct __attribute__((packed)) Bomb { - Bomb() : magic_(123) {} - - ~Bomb() { exit(42); } - - int get() const { return magic_; } - - private: - [[maybe_unused]] char padding_; - int magic_; -}; - -TEST(no_destructor, bomb) { - ASSERT_EXIT(({ - { - Bomb b; - if (b.get() != 123) exit(1); - } - - exit(0); - }), - ::testing::ExitedWithCode(42), ""); -} - -TEST(no_destructor, defused) { - ASSERT_EXIT(({ - { - android::base::NoDestructor<Bomb> b; - if (b->get() != 123) exit(1); - } - - exit(0); - }), - ::testing::ExitedWithCode(0), ""); -} - -TEST(no_destructor, operators) { - android::base::NoDestructor<Bomb> b; - const android::base::NoDestructor<Bomb>& c = b; - ASSERT_EQ(123, b.get()->get()); - ASSERT_EQ(123, b->get()); - ASSERT_EQ(123, (*b).get()); - ASSERT_EQ(123, c.get()->get()); - ASSERT_EQ(123, c->get()); - ASSERT_EQ(123, (*c).get()); -} diff --git a/base/parsebool.cpp b/base/parsebool.cpp deleted file mode 100644 index ff96fe9a9..000000000 --- a/base/parsebool.cpp +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "android-base/parsebool.h" -#include <errno.h> - -namespace android { -namespace base { - -ParseBoolResult ParseBool(std::string_view s) { - if (s == "1" || s == "y" || s == "yes" || s == "on" || s == "true") { - return ParseBoolResult::kTrue; - } - if (s == "0" || s == "n" || s == "no" || s == "off" || s == "false") { - return ParseBoolResult::kFalse; - } - return ParseBoolResult::kError; -} - -} // namespace base -} // namespace android diff --git a/base/parsebool_test.cpp b/base/parsebool_test.cpp deleted file mode 100644 index a0819940b..000000000 --- a/base/parsebool_test.cpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "android-base/parsebool.h" - -#include <errno.h> - -#include <gtest/gtest.h> -#include <string_view> - -using android::base::ParseBool; -using android::base::ParseBoolResult; - -TEST(parsebool, true_) { - static const char* yes[] = { - "1", "on", "true", "y", "yes", - }; - for (const char* s : yes) { - ASSERT_EQ(ParseBoolResult::kTrue, ParseBool(s)); - } -} - -TEST(parsebool, false_) { - static const char* no[] = { - "0", "false", "n", "no", "off", - }; - for (const char* s : no) { - ASSERT_EQ(ParseBoolResult::kFalse, ParseBool(s)); - } -} - -TEST(parsebool, invalid) { - ASSERT_EQ(ParseBoolResult::kError, ParseBool("blarg")); - ASSERT_EQ(ParseBoolResult::kError, ParseBool("")); -} diff --git a/base/parsedouble_test.cpp b/base/parsedouble_test.cpp deleted file mode 100644 index ec3c10c74..000000000 --- a/base/parsedouble_test.cpp +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (C) 2016 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "android-base/parsedouble.h" - -#include <gtest/gtest.h> - -TEST(parsedouble, double_smoke) { - double d; - ASSERT_FALSE(android::base::ParseDouble("", &d)); - ASSERT_FALSE(android::base::ParseDouble("x", &d)); - ASSERT_FALSE(android::base::ParseDouble("123.4x", &d)); - - ASSERT_TRUE(android::base::ParseDouble("123.4", &d)); - ASSERT_DOUBLE_EQ(123.4, d); - ASSERT_TRUE(android::base::ParseDouble("-123.4", &d)); - ASSERT_DOUBLE_EQ(-123.4, d); - - ASSERT_TRUE(android::base::ParseDouble("0", &d, 0.0)); - ASSERT_DOUBLE_EQ(0.0, d); - ASSERT_FALSE(android::base::ParseDouble("0", &d, 1e-9)); - ASSERT_FALSE(android::base::ParseDouble("3.0", &d, -1.0, 2.0)); - ASSERT_TRUE(android::base::ParseDouble("1.0", &d, 0.0, 2.0)); - ASSERT_DOUBLE_EQ(1.0, d); - - ASSERT_FALSE(android::base::ParseDouble("123.4x", nullptr)); - ASSERT_TRUE(android::base::ParseDouble("-123.4", nullptr)); - ASSERT_FALSE(android::base::ParseDouble("3.0", nullptr, -1.0, 2.0)); - ASSERT_TRUE(android::base::ParseDouble("1.0", nullptr, 0.0, 2.0)); -} - -TEST(parsedouble, float_smoke) { - float f; - ASSERT_FALSE(android::base::ParseFloat("", &f)); - ASSERT_FALSE(android::base::ParseFloat("x", &f)); - ASSERT_FALSE(android::base::ParseFloat("123.4x", &f)); - - ASSERT_TRUE(android::base::ParseFloat("123.4", &f)); - ASSERT_FLOAT_EQ(123.4, f); - ASSERT_TRUE(android::base::ParseFloat("-123.4", &f)); - ASSERT_FLOAT_EQ(-123.4, f); - - ASSERT_TRUE(android::base::ParseFloat("0", &f, 0.0)); - ASSERT_FLOAT_EQ(0.0, f); - ASSERT_FALSE(android::base::ParseFloat("0", &f, 1e-9)); - ASSERT_FALSE(android::base::ParseFloat("3.0", &f, -1.0, 2.0)); - ASSERT_TRUE(android::base::ParseFloat("1.0", &f, 0.0, 2.0)); - ASSERT_FLOAT_EQ(1.0, f); - - ASSERT_FALSE(android::base::ParseFloat("123.4x", nullptr)); - ASSERT_TRUE(android::base::ParseFloat("-123.4", nullptr)); - ASSERT_FALSE(android::base::ParseFloat("3.0", nullptr, -1.0, 2.0)); - ASSERT_TRUE(android::base::ParseFloat("1.0", nullptr, 0.0, 2.0)); -} diff --git a/base/parseint_test.cpp b/base/parseint_test.cpp deleted file mode 100644 index e449c3345..000000000 --- a/base/parseint_test.cpp +++ /dev/null @@ -1,203 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "android-base/parseint.h" - -#include <errno.h> - -#include <gtest/gtest.h> - -TEST(parseint, signed_smoke) { - errno = 0; - int i = 0; - ASSERT_FALSE(android::base::ParseInt("x", &i)); - ASSERT_EQ(EINVAL, errno); - errno = 0; - ASSERT_FALSE(android::base::ParseInt("123x", &i)); - ASSERT_EQ(EINVAL, errno); - - ASSERT_TRUE(android::base::ParseInt("123", &i)); - ASSERT_EQ(123, i); - ASSERT_EQ(0, errno); - i = 0; - EXPECT_TRUE(android::base::ParseInt(" 123", &i)); - EXPECT_EQ(123, i); - ASSERT_TRUE(android::base::ParseInt("-123", &i)); - ASSERT_EQ(-123, i); - i = 0; - EXPECT_TRUE(android::base::ParseInt(" -123", &i)); - EXPECT_EQ(-123, i); - - short s = 0; - ASSERT_TRUE(android::base::ParseInt("1234", &s)); - ASSERT_EQ(1234, s); - - ASSERT_TRUE(android::base::ParseInt("12", &i, 0, 15)); - ASSERT_EQ(12, i); - errno = 0; - ASSERT_FALSE(android::base::ParseInt("-12", &i, 0, 15)); - ASSERT_EQ(ERANGE, errno); - errno = 0; - ASSERT_FALSE(android::base::ParseInt("16", &i, 0, 15)); - ASSERT_EQ(ERANGE, errno); - - errno = 0; - ASSERT_FALSE(android::base::ParseInt<int>("x", nullptr)); - ASSERT_EQ(EINVAL, errno); - errno = 0; - ASSERT_FALSE(android::base::ParseInt<int>("123x", nullptr)); - ASSERT_EQ(EINVAL, errno); - ASSERT_TRUE(android::base::ParseInt<int>("1234", nullptr)); -} - -TEST(parseint, unsigned_smoke) { - errno = 0; - unsigned int i = 0u; - ASSERT_FALSE(android::base::ParseUint("x", &i)); - ASSERT_EQ(EINVAL, errno); - errno = 0; - ASSERT_FALSE(android::base::ParseUint("123x", &i)); - ASSERT_EQ(EINVAL, errno); - - ASSERT_TRUE(android::base::ParseUint("123", &i)); - ASSERT_EQ(123u, i); - ASSERT_EQ(0, errno); - i = 0u; - EXPECT_TRUE(android::base::ParseUint(" 123", &i)); - EXPECT_EQ(123u, i); - errno = 0; - ASSERT_FALSE(android::base::ParseUint("-123", &i)); - EXPECT_EQ(EINVAL, errno); - errno = 0; - EXPECT_FALSE(android::base::ParseUint(" -123", &i)); - EXPECT_EQ(EINVAL, errno); - - unsigned short s = 0u; - ASSERT_TRUE(android::base::ParseUint("1234", &s)); - ASSERT_EQ(1234u, s); - - ASSERT_TRUE(android::base::ParseUint("12", &i, 15u)); - ASSERT_EQ(12u, i); - errno = 0; - ASSERT_FALSE(android::base::ParseUint("-12", &i, 15u)); - ASSERT_EQ(EINVAL, errno); - errno = 0; - ASSERT_FALSE(android::base::ParseUint("16", &i, 15u)); - ASSERT_EQ(ERANGE, errno); - - errno = 0; - ASSERT_FALSE(android::base::ParseUint<unsigned short>("x", nullptr)); - ASSERT_EQ(EINVAL, errno); - errno = 0; - ASSERT_FALSE(android::base::ParseUint<unsigned short>("123x", nullptr)); - ASSERT_EQ(EINVAL, errno); - ASSERT_TRUE(android::base::ParseUint<unsigned short>("1234", nullptr)); - - errno = 0; - unsigned long long int lli; - EXPECT_FALSE(android::base::ParseUint("-123", &lli)); - EXPECT_EQ(EINVAL, errno); - errno = 0; - EXPECT_FALSE(android::base::ParseUint(" -123", &lli)); - EXPECT_EQ(EINVAL, errno); -} - -TEST(parseint, no_implicit_octal) { - int i = 0; - ASSERT_TRUE(android::base::ParseInt("0123", &i)); - ASSERT_EQ(123, i); - - unsigned int u = 0u; - ASSERT_TRUE(android::base::ParseUint("0123", &u)); - ASSERT_EQ(123u, u); -} - -TEST(parseint, explicit_hex) { - int i = 0; - ASSERT_TRUE(android::base::ParseInt("0x123", &i)); - ASSERT_EQ(0x123, i); - i = 0; - EXPECT_TRUE(android::base::ParseInt(" 0x123", &i)); - EXPECT_EQ(0x123, i); - - unsigned int u = 0u; - ASSERT_TRUE(android::base::ParseUint("0x123", &u)); - ASSERT_EQ(0x123u, u); - u = 0u; - EXPECT_TRUE(android::base::ParseUint(" 0x123", &u)); - EXPECT_EQ(0x123u, u); -} - -TEST(parseint, string) { - int i = 0; - ASSERT_TRUE(android::base::ParseInt(std::string("123"), &i)); - ASSERT_EQ(123, i); - - unsigned int u = 0u; - ASSERT_TRUE(android::base::ParseUint(std::string("123"), &u)); - ASSERT_EQ(123u, u); -} - -TEST(parseint, untouched_on_failure) { - int i = 123; - ASSERT_FALSE(android::base::ParseInt("456x", &i)); - ASSERT_EQ(123, i); - - unsigned int u = 123u; - ASSERT_FALSE(android::base::ParseUint("456x", &u)); - ASSERT_EQ(123u, u); -} - -TEST(parseint, ParseByteCount) { - uint64_t i = 0; - ASSERT_TRUE(android::base::ParseByteCount("123b", &i)); - ASSERT_EQ(123ULL, i); - - ASSERT_TRUE(android::base::ParseByteCount("8k", &i)); - ASSERT_EQ(8ULL * 1024, i); - - ASSERT_TRUE(android::base::ParseByteCount("8M", &i)); - ASSERT_EQ(8ULL * 1024 * 1024, i); - - ASSERT_TRUE(android::base::ParseByteCount("6g", &i)); - ASSERT_EQ(6ULL * 1024 * 1024 * 1024, i); - - ASSERT_TRUE(android::base::ParseByteCount("1T", &i)); - ASSERT_EQ(1ULL * 1024 * 1024 * 1024 * 1024, i); - - ASSERT_TRUE(android::base::ParseByteCount("2p", &i)); - ASSERT_EQ(2ULL * 1024 * 1024 * 1024 * 1024 * 1024, i); - - ASSERT_TRUE(android::base::ParseByteCount("4e", &i)); - ASSERT_EQ(4ULL * 1024 * 1024 * 1024 * 1024 * 1024 * 1024, i); -} - -TEST(parseint, ParseByteCount_invalid_suffix) { - unsigned u; - ASSERT_FALSE(android::base::ParseByteCount("1x", &u)); -} - -TEST(parseint, ParseByteCount_overflow) { - uint64_t u64; - ASSERT_FALSE(android::base::ParseByteCount("4294967295E", &u64)); - - uint16_t u16; - ASSERT_TRUE(android::base::ParseByteCount("63k", &u16)); - ASSERT_EQ(63U * 1024, u16); - ASSERT_TRUE(android::base::ParseByteCount("65535b", &u16)); - ASSERT_EQ(65535U, u16); - ASSERT_FALSE(android::base::ParseByteCount("65k", &u16)); -} diff --git a/base/parsenetaddress.cpp b/base/parsenetaddress.cpp deleted file mode 100644 index dd80f6da2..000000000 --- a/base/parsenetaddress.cpp +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (C) 2016 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "android-base/parsenetaddress.h" - -#include <algorithm> - -#include "android-base/stringprintf.h" -#include "android-base/strings.h" - -namespace android { -namespace base { - -bool ParseNetAddress(const std::string& address, std::string* host, int* port, - std::string* canonical_address, std::string* error) { - host->clear(); - - bool ipv6 = true; - bool saw_port = false; - size_t colons = std::count(address.begin(), address.end(), ':'); - size_t dots = std::count(address.begin(), address.end(), '.'); - std::string port_str; - if (address[0] == '[') { - // [::1]:123 - if (address.rfind("]:") == std::string::npos) { - *error = StringPrintf("bad IPv6 address '%s'", address.c_str()); - return false; - } - *host = address.substr(1, (address.find("]:") - 1)); - port_str = address.substr(address.rfind("]:") + 2); - saw_port = true; - } else if (dots == 0 && colons >= 2 && colons <= 7) { - // ::1 - *host = address; - } else if (colons <= 1) { - // 1.2.3.4 or some.accidental.domain.com - ipv6 = false; - std::vector<std::string> pieces = Split(address, ":"); - *host = pieces[0]; - if (pieces.size() > 1) { - port_str = pieces[1]; - saw_port = true; - } - } - - if (host->empty()) { - *error = StringPrintf("no host in '%s'", address.c_str()); - return false; - } - - if (saw_port) { - if (sscanf(port_str.c_str(), "%d", port) != 1 || *port <= 0 || - *port > 65535) { - *error = StringPrintf("bad port number '%s' in '%s'", port_str.c_str(), - address.c_str()); - return false; - } - } - - if (canonical_address != nullptr) { - *canonical_address = - StringPrintf(ipv6 ? "[%s]:%d" : "%s:%d", host->c_str(), *port); - } - - return true; -} - -} // namespace base -} // namespace android diff --git a/base/parsenetaddress_test.cpp b/base/parsenetaddress_test.cpp deleted file mode 100644 index a3bfac860..000000000 --- a/base/parsenetaddress_test.cpp +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright (C) 2016 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "android-base/parsenetaddress.h" - -#include <gtest/gtest.h> - -using android::base::ParseNetAddress; - -TEST(ParseNetAddressTest, TestUrl) { - std::string canonical, host, error; - int port = 123; - - EXPECT_TRUE( - ParseNetAddress("www.google.com", &host, &port, &canonical, &error)); - EXPECT_EQ("www.google.com:123", canonical); - EXPECT_EQ("www.google.com", host); - EXPECT_EQ(123, port); - - EXPECT_TRUE( - ParseNetAddress("www.google.com:666", &host, &port, &canonical, &error)); - EXPECT_EQ("www.google.com:666", canonical); - EXPECT_EQ("www.google.com", host); - EXPECT_EQ(666, port); -} - -TEST(ParseNetAddressTest, TestIpv4) { - std::string canonical, host, error; - int port = 123; - - EXPECT_TRUE(ParseNetAddress("1.2.3.4", &host, &port, &canonical, &error)); - EXPECT_EQ("1.2.3.4:123", canonical); - EXPECT_EQ("1.2.3.4", host); - EXPECT_EQ(123, port); - - EXPECT_TRUE(ParseNetAddress("1.2.3.4:666", &host, &port, &canonical, &error)); - EXPECT_EQ("1.2.3.4:666", canonical); - EXPECT_EQ("1.2.3.4", host); - EXPECT_EQ(666, port); -} - -TEST(ParseNetAddressTest, TestIpv6) { - std::string canonical, host, error; - int port = 123; - - EXPECT_TRUE(ParseNetAddress("::1", &host, &port, &canonical, &error)); - EXPECT_EQ("[::1]:123", canonical); - EXPECT_EQ("::1", host); - EXPECT_EQ(123, port); - - EXPECT_TRUE(ParseNetAddress("fe80::200:5aee:feaa:20a2", &host, &port, - &canonical, &error)); - EXPECT_EQ("[fe80::200:5aee:feaa:20a2]:123", canonical); - EXPECT_EQ("fe80::200:5aee:feaa:20a2", host); - EXPECT_EQ(123, port); - - EXPECT_TRUE(ParseNetAddress("[::1]:666", &host, &port, &canonical, &error)); - EXPECT_EQ("[::1]:666", canonical); - EXPECT_EQ("::1", host); - EXPECT_EQ(666, port); - - EXPECT_TRUE(ParseNetAddress("[fe80::200:5aee:feaa:20a2]:666", &host, &port, - &canonical, &error)); - EXPECT_EQ("[fe80::200:5aee:feaa:20a2]:666", canonical); - EXPECT_EQ("fe80::200:5aee:feaa:20a2", host); - EXPECT_EQ(666, port); -} - -TEST(ParseNetAddressTest, TestInvalidAddress) { - std::string canonical, host; - int port; - - std::string failure_cases[] = { - // Invalid IPv4. - "1.2.3.4:", - "1.2.3.4::", - ":123", - - // Invalid IPv6. - ":1", - "::::::::1", - "[::1", - "[::1]", - "[::1]:", - "[::1]::", - - // Invalid port. - "1.2.3.4:-1", - "1.2.3.4:0", - "1.2.3.4:65536" - "1.2.3.4:hello", - "[::1]:-1", - "[::1]:0", - "[::1]:65536", - "[::1]:hello", - }; - - for (const auto& address : failure_cases) { - // Failure should give some non-empty error string. - std::string error; - EXPECT_FALSE(ParseNetAddress(address, &host, &port, &canonical, &error)); - EXPECT_NE("", error); - } -} - -// Null canonical address argument. -TEST(ParseNetAddressTest, TestNullCanonicalAddress) { - std::string host, error; - int port = 42; - - EXPECT_TRUE(ParseNetAddress("www.google.com", &host, &port, nullptr, &error)); - EXPECT_TRUE(ParseNetAddress("1.2.3.4", &host, &port, nullptr, &error)); - EXPECT_TRUE(ParseNetAddress("::1", &host, &port, nullptr, &error)); -} diff --git a/base/process.cpp b/base/process.cpp deleted file mode 100644 index b8cabf657..000000000 --- a/base/process.cpp +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "android-base/process.h" - -namespace android { -namespace base { - -void AllPids::PidIterator::Increment() { - if (!dir_) { - return; - } - - dirent* de; - while ((de = readdir(dir_.get())) != nullptr) { - pid_t pid = atoi(de->d_name); - if (pid != 0) { - pid_ = pid; - return; - } - } - pid_ = -1; -} - -} // namespace base -} // namespace android diff --git a/base/properties.cpp b/base/properties.cpp deleted file mode 100644 index 35e41a86e..000000000 --- a/base/properties.cpp +++ /dev/null @@ -1,258 +0,0 @@ -/* - * Copyright (C) 2016 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "android-base/properties.h" - -#if defined(__BIONIC__) -#define _REALLY_INCLUDE_SYS__SYSTEM_PROPERTIES_H_ -#include <sys/system_properties.h> -#include <sys/_system_properties.h> -#endif - -#include <algorithm> -#include <chrono> -#include <limits> -#include <map> -#include <string> - -#include <android-base/parsebool.h> -#include <android-base/parseint.h> -#include <android-base/strings.h> - -namespace android { -namespace base { - -bool GetBoolProperty(const std::string& key, bool default_value) { - switch (ParseBool(GetProperty(key, ""))) { - case ParseBoolResult::kError: - return default_value; - case ParseBoolResult::kFalse: - return false; - case ParseBoolResult::kTrue: - return true; - } - __builtin_unreachable(); -} - -template <typename T> -T GetIntProperty(const std::string& key, T default_value, T min, T max) { - T result; - std::string value = GetProperty(key, ""); - if (!value.empty() && android::base::ParseInt(value, &result, min, max)) return result; - return default_value; -} - -template <typename T> -T GetUintProperty(const std::string& key, T default_value, T max) { - T result; - std::string value = GetProperty(key, ""); - if (!value.empty() && android::base::ParseUint(value, &result, max)) return result; - return default_value; -} - -template int8_t GetIntProperty(const std::string&, int8_t, int8_t, int8_t); -template int16_t GetIntProperty(const std::string&, int16_t, int16_t, int16_t); -template int32_t GetIntProperty(const std::string&, int32_t, int32_t, int32_t); -template int64_t GetIntProperty(const std::string&, int64_t, int64_t, int64_t); - -template uint8_t GetUintProperty(const std::string&, uint8_t, uint8_t); -template uint16_t GetUintProperty(const std::string&, uint16_t, uint16_t); -template uint32_t GetUintProperty(const std::string&, uint32_t, uint32_t); -template uint64_t GetUintProperty(const std::string&, uint64_t, uint64_t); - -#if !defined(__BIONIC__) -static std::map<std::string, std::string>& g_properties = *new std::map<std::string, std::string>; -static int __system_property_set(const char* key, const char* value) { - g_properties[key] = value; - return 0; -} -#endif - -std::string GetProperty(const std::string& key, const std::string& default_value) { - std::string property_value; -#if defined(__BIONIC__) - const prop_info* pi = __system_property_find(key.c_str()); - if (pi == nullptr) return default_value; - - __system_property_read_callback(pi, - [](void* cookie, const char*, const char* value, unsigned) { - auto property_value = reinterpret_cast<std::string*>(cookie); - *property_value = value; - }, - &property_value); -#else - auto it = g_properties.find(key); - if (it == g_properties.end()) return default_value; - property_value = it->second; -#endif - // If the property exists but is empty, also return the default value. - // Since we can't remove system properties, "empty" is traditionally - // the same as "missing" (this was true for cutils' property_get). - return property_value.empty() ? default_value : property_value; -} - -bool SetProperty(const std::string& key, const std::string& value) { - return (__system_property_set(key.c_str(), value.c_str()) == 0); -} - -#if defined(__BIONIC__) - -struct WaitForPropertyData { - bool done; - const std::string* expected_value; - unsigned last_read_serial; -}; - -static void WaitForPropertyCallback(void* data_ptr, const char*, const char* value, unsigned serial) { - WaitForPropertyData* data = reinterpret_cast<WaitForPropertyData*>(data_ptr); - if (*data->expected_value == value) { - data->done = true; - } else { - data->last_read_serial = serial; - } -} - -// TODO: chrono_utils? -static void DurationToTimeSpec(timespec& ts, const std::chrono::milliseconds d) { - auto s = std::chrono::duration_cast<std::chrono::seconds>(d); - auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(d - s); - ts.tv_sec = std::min<std::chrono::seconds::rep>(s.count(), std::numeric_limits<time_t>::max()); - ts.tv_nsec = ns.count(); -} - -// TODO: boot_clock? -using AbsTime = std::chrono::time_point<std::chrono::steady_clock>; - -static void UpdateTimeSpec(timespec& ts, std::chrono::milliseconds relative_timeout, - const AbsTime& start_time) { - auto now = std::chrono::steady_clock::now(); - auto time_elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(now - start_time); - if (time_elapsed >= relative_timeout) { - ts = { 0, 0 }; - } else { - auto remaining_timeout = relative_timeout - time_elapsed; - DurationToTimeSpec(ts, remaining_timeout); - } -} - -// Waits for the system property `key` to be created. -// Times out after `relative_timeout`. -// Sets absolute_timeout which represents absolute time for the timeout. -// Returns nullptr on timeout. -static const prop_info* WaitForPropertyCreation(const std::string& key, - const std::chrono::milliseconds& relative_timeout, - const AbsTime& start_time) { - // Find the property's prop_info*. - const prop_info* pi; - unsigned global_serial = 0; - while ((pi = __system_property_find(key.c_str())) == nullptr) { - // The property doesn't even exist yet. - // Wait for a global change and then look again. - timespec ts; - UpdateTimeSpec(ts, relative_timeout, start_time); - if (!__system_property_wait(nullptr, global_serial, &global_serial, &ts)) return nullptr; - } - return pi; -} - -bool WaitForProperty(const std::string& key, const std::string& expected_value, - std::chrono::milliseconds relative_timeout) { - auto start_time = std::chrono::steady_clock::now(); - const prop_info* pi = WaitForPropertyCreation(key, relative_timeout, start_time); - if (pi == nullptr) return false; - - WaitForPropertyData data; - data.expected_value = &expected_value; - data.done = false; - while (true) { - timespec ts; - // Check whether the property has the value we're looking for? - __system_property_read_callback(pi, WaitForPropertyCallback, &data); - if (data.done) return true; - - // It didn't, so wait for the property to change before checking again. - UpdateTimeSpec(ts, relative_timeout, start_time); - uint32_t unused; - if (!__system_property_wait(pi, data.last_read_serial, &unused, &ts)) return false; - } -} - -bool WaitForPropertyCreation(const std::string& key, - std::chrono::milliseconds relative_timeout) { - auto start_time = std::chrono::steady_clock::now(); - return (WaitForPropertyCreation(key, relative_timeout, start_time) != nullptr); -} - -CachedProperty::CachedProperty(const char* property_name) - : property_name_(property_name), - prop_info_(nullptr), - cached_area_serial_(0), - cached_property_serial_(0), - is_read_only_(android::base::StartsWith(property_name, "ro.")), - read_only_property_(nullptr) { - static_assert(sizeof(cached_value_) == PROP_VALUE_MAX); -} - -const char* CachedProperty::Get(bool* changed) { - std::optional<uint32_t> initial_property_serial_ = cached_property_serial_; - - // Do we have a `struct prop_info` yet? - if (prop_info_ == nullptr) { - // `__system_property_find` is expensive, so only retry if a property - // has been created since last time we checked. - uint32_t property_area_serial = __system_property_area_serial(); - if (property_area_serial != cached_area_serial_) { - prop_info_ = __system_property_find(property_name_.c_str()); - cached_area_serial_ = property_area_serial; - } - } - - if (prop_info_ != nullptr) { - // Only bother re-reading the property if it's actually changed since last time. - uint32_t property_serial = __system_property_serial(prop_info_); - if (property_serial != cached_property_serial_) { - __system_property_read_callback( - prop_info_, - [](void* data, const char*, const char* value, uint32_t serial) { - CachedProperty* instance = reinterpret_cast<CachedProperty*>(data); - instance->cached_property_serial_ = serial; - // Read only properties can be larger than PROP_VALUE_MAX, but also never change value - // or location, thus we return the pointer from the shared memory directly. - if (instance->is_read_only_) { - instance->read_only_property_ = value; - } else { - strlcpy(instance->cached_value_, value, PROP_VALUE_MAX); - } - }, - this); - } - } - - if (changed) { - *changed = cached_property_serial_ != initial_property_serial_; - } - - if (is_read_only_) { - return read_only_property_; - } else { - return cached_value_; - } -} - -#endif - -} // namespace base -} // namespace android diff --git a/base/properties_test.cpp b/base/properties_test.cpp deleted file mode 100644 index c30c41e8a..000000000 --- a/base/properties_test.cpp +++ /dev/null @@ -1,257 +0,0 @@ -/* - * Copyright (C) 2016 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "android-base/properties.h" - -#include <gtest/gtest.h> - -#include <atomic> -#include <chrono> -#include <string> -#include <thread> - -#if !defined(_WIN32) -using namespace std::literals; -#endif - -TEST(properties, smoke) { - android::base::SetProperty("debug.libbase.property_test", "hello"); - - std::string s = android::base::GetProperty("debug.libbase.property_test", ""); - ASSERT_EQ("hello", s); - - android::base::SetProperty("debug.libbase.property_test", "world"); - s = android::base::GetProperty("debug.libbase.property_test", ""); - ASSERT_EQ("world", s); - - s = android::base::GetProperty("this.property.does.not.exist", ""); - ASSERT_EQ("", s); - - s = android::base::GetProperty("this.property.does.not.exist", "default"); - ASSERT_EQ("default", s); -} - -TEST(properties, empty) { - // Because you can't delete a property, people "delete" them by - // setting them to the empty string. In that case we'd want to - // keep the default value (like cutils' property_get did). - android::base::SetProperty("debug.libbase.property_test", ""); - std::string s = android::base::GetProperty("debug.libbase.property_test", "default"); - ASSERT_EQ("default", s); -} - -static void CheckGetBoolProperty(bool expected, const std::string& value, bool default_value) { - android::base::SetProperty("debug.libbase.property_test", value.c_str()); - ASSERT_EQ(expected, android::base::GetBoolProperty("debug.libbase.property_test", default_value)); -} - -TEST(properties, GetBoolProperty_true) { - CheckGetBoolProperty(true, "1", false); - CheckGetBoolProperty(true, "y", false); - CheckGetBoolProperty(true, "yes", false); - CheckGetBoolProperty(true, "on", false); - CheckGetBoolProperty(true, "true", false); -} - -TEST(properties, GetBoolProperty_false) { - CheckGetBoolProperty(false, "0", true); - CheckGetBoolProperty(false, "n", true); - CheckGetBoolProperty(false, "no", true); - CheckGetBoolProperty(false, "off", true); - CheckGetBoolProperty(false, "false", true); -} - -TEST(properties, GetBoolProperty_default) { - CheckGetBoolProperty(true, "burp", true); - CheckGetBoolProperty(false, "burp", false); -} - -template <typename T> void CheckGetIntProperty() { - // Positive and negative. - android::base::SetProperty("debug.libbase.property_test", "-12"); - EXPECT_EQ(T(-12), android::base::GetIntProperty<T>("debug.libbase.property_test", 45)); - android::base::SetProperty("debug.libbase.property_test", "12"); - EXPECT_EQ(T(12), android::base::GetIntProperty<T>("debug.libbase.property_test", 45)); - - // Default value. - android::base::SetProperty("debug.libbase.property_test", ""); - EXPECT_EQ(T(45), android::base::GetIntProperty<T>("debug.libbase.property_test", 45)); - - // Bounds checks. - android::base::SetProperty("debug.libbase.property_test", "0"); - EXPECT_EQ(T(45), android::base::GetIntProperty<T>("debug.libbase.property_test", 45, 1, 2)); - android::base::SetProperty("debug.libbase.property_test", "1"); - EXPECT_EQ(T(1), android::base::GetIntProperty<T>("debug.libbase.property_test", 45, 1, 2)); - android::base::SetProperty("debug.libbase.property_test", "2"); - EXPECT_EQ(T(2), android::base::GetIntProperty<T>("debug.libbase.property_test", 45, 1, 2)); - android::base::SetProperty("debug.libbase.property_test", "3"); - EXPECT_EQ(T(45), android::base::GetIntProperty<T>("debug.libbase.property_test", 45, 1, 2)); -} - -template <typename T> void CheckGetUintProperty() { - // Positive. - android::base::SetProperty("debug.libbase.property_test", "12"); - EXPECT_EQ(T(12), android::base::GetUintProperty<T>("debug.libbase.property_test", 45)); - - // Default value. - android::base::SetProperty("debug.libbase.property_test", ""); - EXPECT_EQ(T(45), android::base::GetUintProperty<T>("debug.libbase.property_test", 45)); - - // Bounds checks. - android::base::SetProperty("debug.libbase.property_test", "12"); - EXPECT_EQ(T(12), android::base::GetUintProperty<T>("debug.libbase.property_test", 33, 22)); - android::base::SetProperty("debug.libbase.property_test", "12"); - EXPECT_EQ(T(5), android::base::GetUintProperty<T>("debug.libbase.property_test", 5, 10)); -} - -TEST(properties, GetIntProperty_int8_t) { CheckGetIntProperty<int8_t>(); } -TEST(properties, GetIntProperty_int16_t) { CheckGetIntProperty<int16_t>(); } -TEST(properties, GetIntProperty_int32_t) { CheckGetIntProperty<int32_t>(); } -TEST(properties, GetIntProperty_int64_t) { CheckGetIntProperty<int64_t>(); } - -TEST(properties, GetUintProperty_uint8_t) { CheckGetUintProperty<uint8_t>(); } -TEST(properties, GetUintProperty_uint16_t) { CheckGetUintProperty<uint16_t>(); } -TEST(properties, GetUintProperty_uint32_t) { CheckGetUintProperty<uint32_t>(); } -TEST(properties, GetUintProperty_uint64_t) { CheckGetUintProperty<uint64_t>(); } - -TEST(properties, WaitForProperty) { -#if defined(__BIONIC__) - std::atomic<bool> flag{false}; - std::thread thread([&]() { - std::this_thread::sleep_for(100ms); - android::base::SetProperty("debug.libbase.WaitForProperty_test", "a"); - while (!flag) std::this_thread::yield(); - android::base::SetProperty("debug.libbase.WaitForProperty_test", "b"); - }); - - ASSERT_TRUE(android::base::WaitForProperty("debug.libbase.WaitForProperty_test", "a", 1s)); - flag = true; - ASSERT_TRUE(android::base::WaitForProperty("debug.libbase.WaitForProperty_test", "b", 1s)); - thread.join(); -#else - GTEST_LOG_(INFO) << "This test does nothing on the host.\n"; -#endif -} - -TEST(properties, WaitForProperty_timeout) { -#if defined(__BIONIC__) - auto t0 = std::chrono::steady_clock::now(); - ASSERT_FALSE(android::base::WaitForProperty("debug.libbase.WaitForProperty_timeout_test", "a", - 200ms)); - auto t1 = std::chrono::steady_clock::now(); - - ASSERT_GE(std::chrono::duration_cast<std::chrono::milliseconds>(t1 - t0), 200ms); - // Upper bounds on timing are inherently flaky, but let's try... - ASSERT_LT(std::chrono::duration_cast<std::chrono::milliseconds>(t1 - t0), 600ms); -#else - GTEST_LOG_(INFO) << "This test does nothing on the host.\n"; -#endif -} - -TEST(properties, WaitForProperty_MaxTimeout) { -#if defined(__BIONIC__) - std::atomic<bool> flag{false}; - std::thread thread([&]() { - android::base::SetProperty("debug.libbase.WaitForProperty_test", "a"); - while (!flag) std::this_thread::yield(); - std::this_thread::sleep_for(500ms); - android::base::SetProperty("debug.libbase.WaitForProperty_test", "b"); - }); - - ASSERT_TRUE(android::base::WaitForProperty("debug.libbase.WaitForProperty_test", "a", 1s)); - flag = true; - // Test that this does not immediately return false due to overflow issues with the timeout. - ASSERT_TRUE(android::base::WaitForProperty("debug.libbase.WaitForProperty_test", "b")); - thread.join(); -#else - GTEST_LOG_(INFO) << "This test does nothing on the host.\n"; -#endif -} - -TEST(properties, WaitForProperty_NegativeTimeout) { -#if defined(__BIONIC__) - std::atomic<bool> flag{false}; - std::thread thread([&]() { - android::base::SetProperty("debug.libbase.WaitForProperty_test", "a"); - while (!flag) std::this_thread::yield(); - std::this_thread::sleep_for(500ms); - android::base::SetProperty("debug.libbase.WaitForProperty_test", "b"); - }); - - ASSERT_TRUE(android::base::WaitForProperty("debug.libbase.WaitForProperty_test", "a", 1s)); - flag = true; - // Assert that this immediately returns with a negative timeout - ASSERT_FALSE(android::base::WaitForProperty("debug.libbase.WaitForProperty_test", "b", -100ms)); - thread.join(); -#else - GTEST_LOG_(INFO) << "This test does nothing on the host.\n"; -#endif -} - -TEST(properties, WaitForPropertyCreation) { -#if defined(__BIONIC__) - std::thread thread([&]() { - std::this_thread::sleep_for(100ms); - android::base::SetProperty("debug.libbase.WaitForPropertyCreation_test", "a"); - }); - - ASSERT_TRUE(android::base::WaitForPropertyCreation( - "debug.libbase.WaitForPropertyCreation_test", 1s)); - thread.join(); -#else - GTEST_LOG_(INFO) << "This test does nothing on the host.\n"; -#endif -} - -TEST(properties, WaitForPropertyCreation_timeout) { -#if defined(__BIONIC__) - auto t0 = std::chrono::steady_clock::now(); - ASSERT_FALSE(android::base::WaitForPropertyCreation( - "debug.libbase.WaitForPropertyCreation_timeout_test", 200ms)); - auto t1 = std::chrono::steady_clock::now(); - - ASSERT_GE(std::chrono::duration_cast<std::chrono::milliseconds>(t1 - t0), 200ms); - // Upper bounds on timing are inherently flaky, but let's try... - ASSERT_LT(std::chrono::duration_cast<std::chrono::milliseconds>(t1 - t0), 600ms); -#else - GTEST_LOG_(INFO) << "This test does nothing on the host.\n"; -#endif -} - -TEST(properties, CachedProperty) { -#if defined(__BIONIC__) - android::base::CachedProperty cached_property("debug.libbase.CachedProperty_test"); - bool changed; - cached_property.Get(&changed); - - android::base::SetProperty("debug.libbase.CachedProperty_test", "foo"); - ASSERT_STREQ("foo", cached_property.Get(&changed)); - ASSERT_TRUE(changed); - - ASSERT_STREQ("foo", cached_property.Get(&changed)); - ASSERT_FALSE(changed); - - android::base::SetProperty("debug.libbase.CachedProperty_test", "bar"); - ASSERT_STREQ("bar", cached_property.Get(&changed)); - ASSERT_TRUE(changed); - - ASSERT_STREQ("bar", cached_property.Get(&changed)); - ASSERT_FALSE(changed); - -#else - GTEST_LOG_(INFO) << "This test does nothing on the host.\n"; -#endif -} diff --git a/base/result_test.cpp b/base/result_test.cpp deleted file mode 100644 index c0ac0fdaf..000000000 --- a/base/result_test.cpp +++ /dev/null @@ -1,422 +0,0 @@ -/* - * Copyright (C) 2017 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "android-base/result.h" - -#include "errno.h" - -#include <istream> -#include <string> - -#include <gtest/gtest.h> - -using namespace std::string_literals; - -namespace android { -namespace base { - -TEST(result, result_accessors) { - Result<std::string> result = "success"; - ASSERT_RESULT_OK(result); - ASSERT_TRUE(result.has_value()); - - EXPECT_EQ("success", *result); - EXPECT_EQ("success", result.value()); - - EXPECT_EQ('s', result->data()[0]); -} - -TEST(result, result_accessors_rvalue) { - ASSERT_TRUE(Result<std::string>("success").ok()); - ASSERT_TRUE(Result<std::string>("success").has_value()); - - EXPECT_EQ("success", *Result<std::string>("success")); - EXPECT_EQ("success", Result<std::string>("success").value()); - - EXPECT_EQ('s', Result<std::string>("success")->data()[0]); -} - -TEST(result, result_void) { - Result<void> ok = {}; - EXPECT_RESULT_OK(ok); - ok.value(); // should not crash - ASSERT_DEATH(ok.error(), ""); - - Result<void> fail = Error() << "failure" << 1; - EXPECT_FALSE(fail.ok()); - EXPECT_EQ("failure1", fail.error().message()); - EXPECT_EQ(0, fail.error().code()); - EXPECT_TRUE(ok != fail); - ASSERT_DEATH(fail.value(), ""); - - auto test = [](bool ok) -> Result<void> { - if (ok) return {}; - else return Error() << "failure" << 1; - }; - EXPECT_TRUE(test(true).ok()); - EXPECT_FALSE(test(false).ok()); - test(true).value(); // should not crash - ASSERT_DEATH(test(true).error(), ""); - ASSERT_DEATH(test(false).value(), ""); - EXPECT_EQ("failure1", test(false).error().message()); -} - -TEST(result, result_error) { - Result<void> result = Error() << "failure" << 1; - ASSERT_FALSE(result.ok()); - ASSERT_FALSE(result.has_value()); - - EXPECT_EQ(0, result.error().code()); - EXPECT_EQ("failure1", result.error().message()); -} - -TEST(result, result_error_empty) { - Result<void> result = Error(); - ASSERT_FALSE(result.ok()); - ASSERT_FALSE(result.has_value()); - - EXPECT_EQ(0, result.error().code()); - EXPECT_EQ("", result.error().message()); -} - -TEST(result, result_error_rvalue) { - // Error() and ErrnoError() aren't actually used to create a Result<T> object. - // Under the hood, they are an intermediate class that can be implicitly constructed into a - // Result<T>. This is needed both to create the ostream and because Error() itself, by - // definition will not know what the type, T, of the underlying Result<T> object that it would - // create is. - - auto MakeRvalueErrorResult = []() -> Result<void> { return Error() << "failure" << 1; }; - ASSERT_FALSE(MakeRvalueErrorResult().ok()); - ASSERT_FALSE(MakeRvalueErrorResult().has_value()); - - EXPECT_EQ(0, MakeRvalueErrorResult().error().code()); - EXPECT_EQ("failure1", MakeRvalueErrorResult().error().message()); -} - -TEST(result, result_errno_error) { - constexpr int test_errno = 6; - errno = test_errno; - Result<void> result = ErrnoError() << "failure" << 1; - - ASSERT_FALSE(result.ok()); - ASSERT_FALSE(result.has_value()); - - EXPECT_EQ(test_errno, result.error().code()); - EXPECT_EQ("failure1: "s + strerror(test_errno), result.error().message()); -} - -TEST(result, result_errno_error_no_text) { - constexpr int test_errno = 6; - errno = test_errno; - Result<void> result = ErrnoError(); - - ASSERT_FALSE(result.ok()); - ASSERT_FALSE(result.has_value()); - - EXPECT_EQ(test_errno, result.error().code()); - EXPECT_EQ(strerror(test_errno), result.error().message()); -} - -TEST(result, result_error_from_other_result) { - auto error_text = "test error"s; - Result<void> result = Error() << error_text; - - ASSERT_FALSE(result.ok()); - ASSERT_FALSE(result.has_value()); - - Result<std::string> result2 = result.error(); - - ASSERT_FALSE(result2.ok()); - ASSERT_FALSE(result2.has_value()); - - EXPECT_EQ(0, result2.error().code()); - EXPECT_EQ(error_text, result2.error().message()); -} - -TEST(result, result_error_through_ostream) { - auto error_text = "test error"s; - Result<void> result = Error() << error_text; - - ASSERT_FALSE(result.ok()); - ASSERT_FALSE(result.has_value()); - - Result<std::string> result2 = Error() << result.error(); - - ASSERT_FALSE(result2.ok()); - ASSERT_FALSE(result2.has_value()); - - EXPECT_EQ(0, result2.error().code()); - EXPECT_EQ(error_text, result2.error().message()); -} - -TEST(result, result_errno_error_through_ostream) { - auto error_text = "test error"s; - constexpr int test_errno = 6; - errno = 6; - Result<void> result = ErrnoError() << error_text; - - errno = 0; - - ASSERT_FALSE(result.ok()); - ASSERT_FALSE(result.has_value()); - - Result<std::string> result2 = Error() << result.error(); - - ASSERT_FALSE(result2.ok()); - ASSERT_FALSE(result2.has_value()); - - EXPECT_EQ(test_errno, result2.error().code()); - EXPECT_EQ(error_text + ": " + strerror(test_errno), result2.error().message()); -} - -TEST(result, constructor_forwarding) { - auto result = Result<std::string>(std::in_place, 5, 'a'); - - ASSERT_RESULT_OK(result); - ASSERT_TRUE(result.has_value()); - - EXPECT_EQ("aaaaa", *result); -} - -struct ConstructorTracker { - static size_t constructor_called; - static size_t copy_constructor_called; - static size_t move_constructor_called; - static size_t copy_assignment_called; - static size_t move_assignment_called; - - template <typename T> - ConstructorTracker(T&& string) : string(string) { - ++constructor_called; - } - - ConstructorTracker(const ConstructorTracker& ct) { - ++copy_constructor_called; - string = ct.string; - } - ConstructorTracker(ConstructorTracker&& ct) noexcept { - ++move_constructor_called; - string = std::move(ct.string); - } - ConstructorTracker& operator=(const ConstructorTracker& ct) { - ++copy_assignment_called; - string = ct.string; - return *this; - } - ConstructorTracker& operator=(ConstructorTracker&& ct) noexcept { - ++move_assignment_called; - string = std::move(ct.string); - return *this; - } - - std::string string; -}; - -size_t ConstructorTracker::constructor_called = 0; -size_t ConstructorTracker::copy_constructor_called = 0; -size_t ConstructorTracker::move_constructor_called = 0; -size_t ConstructorTracker::copy_assignment_called = 0; -size_t ConstructorTracker::move_assignment_called = 0; - -Result<ConstructorTracker> ReturnConstructorTracker(const std::string& in) { - if (in.empty()) { - return "literal string"; - } - if (in == "test2") { - return ConstructorTracker(in + in + "2"); - } - ConstructorTracker result(in + " " + in); - return result; -}; - -TEST(result, no_copy_on_return) { - // If returning parameters that may be used to implicitly construct the type T of Result<T>, - // then those parameters are forwarded to the construction of Result<T>. - - // If returning an prvalue or xvalue, it will be move constructed during the construction of - // Result<T>. - - // This check ensures that that is the case, and particularly that no copy constructors - // are called. - - auto result1 = ReturnConstructorTracker(""); - ASSERT_RESULT_OK(result1); - EXPECT_EQ("literal string", result1->string); - EXPECT_EQ(1U, ConstructorTracker::constructor_called); - EXPECT_EQ(0U, ConstructorTracker::copy_constructor_called); - EXPECT_EQ(0U, ConstructorTracker::move_constructor_called); - EXPECT_EQ(0U, ConstructorTracker::copy_assignment_called); - EXPECT_EQ(0U, ConstructorTracker::move_assignment_called); - - auto result2 = ReturnConstructorTracker("test2"); - ASSERT_RESULT_OK(result2); - EXPECT_EQ("test2test22", result2->string); - EXPECT_EQ(2U, ConstructorTracker::constructor_called); - EXPECT_EQ(0U, ConstructorTracker::copy_constructor_called); - EXPECT_EQ(1U, ConstructorTracker::move_constructor_called); - EXPECT_EQ(0U, ConstructorTracker::copy_assignment_called); - EXPECT_EQ(0U, ConstructorTracker::move_assignment_called); - - auto result3 = ReturnConstructorTracker("test3"); - ASSERT_RESULT_OK(result3); - EXPECT_EQ("test3 test3", result3->string); - EXPECT_EQ(3U, ConstructorTracker::constructor_called); - EXPECT_EQ(0U, ConstructorTracker::copy_constructor_called); - EXPECT_EQ(2U, ConstructorTracker::move_constructor_called); - EXPECT_EQ(0U, ConstructorTracker::copy_assignment_called); - EXPECT_EQ(0U, ConstructorTracker::move_assignment_called); -} - -// Below two tests require that we do not hide the move constructor with our forwarding reference -// constructor. This is done with by disabling the forwarding reference constructor if its first -// and only type is Result<T>. -TEST(result, result_result_with_success) { - auto return_result_result_with_success = []() -> Result<Result<void>> { return Result<void>(); }; - auto result = return_result_result_with_success(); - ASSERT_RESULT_OK(result); - ASSERT_RESULT_OK(*result); - - auto inner_result = result.value(); - ASSERT_RESULT_OK(inner_result); -} - -TEST(result, result_result_with_failure) { - auto return_result_result_with_error = []() -> Result<Result<void>> { - return Result<void>(ResultError("failure string", 6)); - }; - auto result = return_result_result_with_error(); - ASSERT_RESULT_OK(result); - ASSERT_FALSE(result->ok()); - EXPECT_EQ("failure string", (*result).error().message()); - EXPECT_EQ(6, (*result).error().code()); -} - -// This test requires that we disable the forwarding reference constructor if Result<T> is the -// *only* type that we are forwarding. In otherwords, if we are forwarding Result<T>, int to -// construct a Result<T>, then we still need the constructor. -TEST(result, result_two_parameter_constructor_same_type) { - struct TestStruct { - TestStruct(int value) : value_(value) {} - TestStruct(Result<TestStruct> result, int value) : value_(result->value_ * value) {} - int value_; - }; - - auto return_test_struct = []() -> Result<TestStruct> { - return Result<TestStruct>(std::in_place, Result<TestStruct>(std::in_place, 6), 6); - }; - - auto result = return_test_struct(); - ASSERT_RESULT_OK(result); - EXPECT_EQ(36, result->value_); -} - -TEST(result, die_on_access_failed_result) { - Result<std::string> result = Error(); - ASSERT_DEATH(*result, ""); -} - -TEST(result, die_on_get_error_succesful_result) { - Result<std::string> result = "success"; - ASSERT_DEATH(result.error(), ""); -} - -template <class CharT> -std::basic_ostream<CharT>& SetErrnoToTwo(std::basic_ostream<CharT>& ss) { - errno = 2; - return ss; -} - -TEST(result, preserve_errno) { - errno = 1; - int old_errno = errno; - Result<int> result = Error() << "Failed" << SetErrnoToTwo<char>; - ASSERT_FALSE(result.ok()); - EXPECT_EQ(old_errno, errno); - - errno = 1; - old_errno = errno; - Result<int> result2 = ErrnoError() << "Failed" << SetErrnoToTwo<char>; - ASSERT_FALSE(result2.ok()); - EXPECT_EQ(old_errno, errno); - EXPECT_EQ(old_errno, result2.error().code()); -} - -TEST(result, error_with_fmt) { - Result<int> result = Errorf("{} {}!", "hello", "world"); - EXPECT_EQ("hello world!", result.error().message()); - - result = Errorf("{} {}!", std::string("hello"), std::string("world")); - EXPECT_EQ("hello world!", result.error().message()); - - result = Errorf("{1} {0}!", "world", "hello"); - EXPECT_EQ("hello world!", result.error().message()); - - result = Errorf("hello world!"); - EXPECT_EQ("hello world!", result.error().message()); - - Result<int> result2 = Errorf("error occurred with {}", result.error()); - EXPECT_EQ("error occurred with hello world!", result2.error().message()); - - constexpr int test_errno = 6; - errno = test_errno; - result = ErrnoErrorf("{} {}!", "hello", "world"); - EXPECT_EQ(test_errno, result.error().code()); - EXPECT_EQ("hello world!: "s + strerror(test_errno), result.error().message()); -} - -TEST(result, error_with_fmt_carries_errno) { - constexpr int inner_errno = 6; - errno = inner_errno; - Result<int> inner_result = ErrnoErrorf("inner failure"); - errno = 0; - EXPECT_EQ(inner_errno, inner_result.error().code()); - - // outer_result is created with Errorf, but its error code is got from inner_result. - Result<int> outer_result = Errorf("outer failure caused by {}", inner_result.error()); - EXPECT_EQ(inner_errno, outer_result.error().code()); - EXPECT_EQ("outer failure caused by inner failure: "s + strerror(inner_errno), - outer_result.error().message()); - - // now both result objects are created with ErrnoErrorf. errno from the inner_result - // is not passed to outer_result. - constexpr int outer_errno = 10; - errno = outer_errno; - outer_result = ErrnoErrorf("outer failure caused by {}", inner_result.error()); - EXPECT_EQ(outer_errno, outer_result.error().code()); - EXPECT_EQ("outer failure caused by inner failure: "s + strerror(inner_errno) + ": "s + - strerror(outer_errno), - outer_result.error().message()); -} - -TEST(result, errno_chaining_multiple) { - constexpr int errno1 = 6; - errno = errno1; - Result<int> inner1 = ErrnoErrorf("error1"); - - constexpr int errno2 = 10; - errno = errno2; - Result<int> inner2 = ErrnoErrorf("error2"); - - // takes the error code of inner2 since its the last one. - Result<int> outer = Errorf("two errors: {}, {}", inner1.error(), inner2.error()); - EXPECT_EQ(errno2, outer.error().code()); - EXPECT_EQ("two errors: error1: "s + strerror(errno1) + ", error2: "s + strerror(errno2), - outer.error().message()); -} - -} // namespace base -} // namespace android diff --git a/base/scopeguard_test.cpp b/base/scopeguard_test.cpp deleted file mode 100644 index 9236d7b78..000000000 --- a/base/scopeguard_test.cpp +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (C) 2017 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "android-base/scopeguard.h" - -#include <utility> -#include <vector> - -#include <gtest/gtest.h> - -TEST(scopeguard, normal) { - bool guarded_var = true; - { - auto scopeguard = android::base::make_scope_guard([&guarded_var] { guarded_var = false; }); - } - ASSERT_FALSE(guarded_var); -} - -TEST(scopeguard, disabled) { - bool guarded_var = true; - { - auto scopeguard = android::base::make_scope_guard([&guarded_var] { guarded_var = false; }); - scopeguard.Disable(); - } - ASSERT_TRUE(guarded_var); -} - -TEST(scopeguard, moved) { - int guarded_var = true; - auto scopeguard = android::base::make_scope_guard([&guarded_var] { guarded_var = false; }); - { decltype(scopeguard) new_guard(std::move(scopeguard)); } - EXPECT_FALSE(scopeguard.active()); - ASSERT_FALSE(guarded_var); -} - -TEST(scopeguard, vector) { - int guarded_var = 0; - { - std::vector<android::base::ScopeGuard<std::function<void()>>> scopeguards; - scopeguards.emplace_back(android::base::make_scope_guard( - std::bind([](int& guarded_var) { guarded_var++; }, std::ref(guarded_var)))); - scopeguards.emplace_back(android::base::make_scope_guard( - std::bind([](int& guarded_var) { guarded_var++; }, std::ref(guarded_var)))); - } - ASSERT_EQ(guarded_var, 2); -} diff --git a/base/stringprintf.cpp b/base/stringprintf.cpp deleted file mode 100644 index e83ab1316..000000000 --- a/base/stringprintf.cpp +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (C) 2011 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "android-base/stringprintf.h" - -#include <stdio.h> - -#include <string> - -namespace android { -namespace base { - -void StringAppendV(std::string* dst, const char* format, va_list ap) { - // First try with a small fixed size buffer - char space[1024] __attribute__((__uninitialized__)); - - // It's possible for methods that use a va_list to invalidate - // the data in it upon use. The fix is to make a copy - // of the structure before using it and use that copy instead. - va_list backup_ap; - va_copy(backup_ap, ap); - int result = vsnprintf(space, sizeof(space), format, backup_ap); - va_end(backup_ap); - - if (result < static_cast<int>(sizeof(space))) { - if (result >= 0) { - // Normal case -- everything fit. - dst->append(space, result); - return; - } - - if (result < 0) { - // Just an error. - return; - } - } - - // Increase the buffer size to the size requested by vsnprintf, - // plus one for the closing \0. - int length = result + 1; - char* buf = new char[length]; - - // Restore the va_list before we use it again - va_copy(backup_ap, ap); - result = vsnprintf(buf, length, format, backup_ap); - va_end(backup_ap); - - if (result >= 0 && result < length) { - // It fit - dst->append(buf, result); - } - delete[] buf; -} - -std::string StringPrintf(const char* fmt, ...) { - va_list ap; - va_start(ap, fmt); - std::string result; - StringAppendV(&result, fmt, ap); - va_end(ap); - return result; -} - -void StringAppendF(std::string* dst, const char* format, ...) { - va_list ap; - va_start(ap, format); - StringAppendV(dst, format, ap); - va_end(ap); -} - -} // namespace base -} // namespace android diff --git a/base/stringprintf_test.cpp b/base/stringprintf_test.cpp deleted file mode 100644 index fc009b1d7..000000000 --- a/base/stringprintf_test.cpp +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (C) 2011 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "android-base/stringprintf.h" - -#include <gtest/gtest.h> - -#include <string> - -TEST(StringPrintfTest, HexSizeT) { - size_t size = 0x00107e59; - EXPECT_EQ("00107e59", android::base::StringPrintf("%08zx", size)); - EXPECT_EQ("0x00107e59", android::base::StringPrintf("0x%08zx", size)); -} - -TEST(StringPrintfTest, StringAppendF) { - std::string s("a"); - android::base::StringAppendF(&s, "b"); - EXPECT_EQ("ab", s); -} - -TEST(StringPrintfTest, Errno) { - errno = 123; - android::base::StringPrintf("hello %s", "world"); - EXPECT_EQ(123, errno); -} - -void TestN(size_t n) { - char* buf = new char[n + 1]; - memset(buf, 'x', n); - buf[n] = '\0'; - std::string s(android::base::StringPrintf("%s", buf)); - EXPECT_EQ(buf, s); - delete[] buf; -} - -TEST(StringPrintfTest, At1023) { - TestN(1023); -} - -TEST(StringPrintfTest, At1024) { - TestN(1024); -} - -TEST(StringPrintfTest, At1025) { - TestN(1025); -} diff --git a/base/strings.cpp b/base/strings.cpp deleted file mode 100644 index 40b2bf270..000000000 --- a/base/strings.cpp +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "android-base/strings.h" - -#include <stdlib.h> -#include <string.h> - -#include <string> -#include <vector> - -namespace android { -namespace base { - -#define CHECK_NE(a, b) \ - if ((a) == (b)) abort(); - -std::vector<std::string> Split(const std::string& s, - const std::string& delimiters) { - CHECK_NE(delimiters.size(), 0U); - - std::vector<std::string> result; - - size_t base = 0; - size_t found; - while (true) { - found = s.find_first_of(delimiters, base); - result.push_back(s.substr(base, found - base)); - if (found == s.npos) break; - base = found + 1; - } - - return result; -} - -std::string Trim(const std::string& s) { - std::string result; - - if (s.size() == 0) { - return result; - } - - size_t start_index = 0; - size_t end_index = s.size() - 1; - - // Skip initial whitespace. - while (start_index < s.size()) { - if (!isspace(s[start_index])) { - break; - } - start_index++; - } - - // Skip terminating whitespace. - while (end_index >= start_index) { - if (!isspace(s[end_index])) { - break; - } - end_index--; - } - - // All spaces, no beef. - if (end_index < start_index) { - return ""; - } - // Start_index is the first non-space, end_index is the last one. - return s.substr(start_index, end_index - start_index + 1); -} - -// These cases are probably the norm, so we mark them extern in the header to -// aid compile time and binary size. -template std::string Join(const std::vector<std::string>&, char); -template std::string Join(const std::vector<const char*>&, char); -template std::string Join(const std::vector<std::string>&, const std::string&); -template std::string Join(const std::vector<const char*>&, const std::string&); - -bool StartsWith(std::string_view s, std::string_view prefix) { - return s.substr(0, prefix.size()) == prefix; -} - -bool StartsWith(std::string_view s, char prefix) { - return !s.empty() && s.front() == prefix; -} - -bool StartsWithIgnoreCase(std::string_view s, std::string_view prefix) { - return s.size() >= prefix.size() && strncasecmp(s.data(), prefix.data(), prefix.size()) == 0; -} - -bool EndsWith(std::string_view s, std::string_view suffix) { - return s.size() >= suffix.size() && s.substr(s.size() - suffix.size(), suffix.size()) == suffix; -} - -bool EndsWith(std::string_view s, char suffix) { - return !s.empty() && s.back() == suffix; -} - -bool EndsWithIgnoreCase(std::string_view s, std::string_view suffix) { - return s.size() >= suffix.size() && - strncasecmp(s.data() + (s.size() - suffix.size()), suffix.data(), suffix.size()) == 0; -} - -bool EqualsIgnoreCase(std::string_view lhs, std::string_view rhs) { - return lhs.size() == rhs.size() && strncasecmp(lhs.data(), rhs.data(), lhs.size()) == 0; -} - -std::string StringReplace(std::string_view s, std::string_view from, std::string_view to, - bool all) { - if (from.empty()) return std::string(s); - - std::string result; - std::string_view::size_type start_pos = 0; - do { - std::string_view::size_type pos = s.find(from, start_pos); - if (pos == std::string_view::npos) break; - - result.append(s.data() + start_pos, pos - start_pos); - result.append(to.data(), to.size()); - - start_pos = pos + from.size(); - } while (all); - result.append(s.data() + start_pos, s.size() - start_pos); - return result; -} - -} // namespace base -} // namespace android diff --git a/base/strings_test.cpp b/base/strings_test.cpp deleted file mode 100644 index 5ae309446..000000000 --- a/base/strings_test.cpp +++ /dev/null @@ -1,356 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "android-base/strings.h" - -#include <gtest/gtest.h> - -#include <string> -#include <vector> -#include <set> -#include <unordered_set> - -TEST(strings, split_empty) { - std::vector<std::string> parts = android::base::Split("", ","); - ASSERT_EQ(1U, parts.size()); - ASSERT_EQ("", parts[0]); -} - -TEST(strings, split_single) { - std::vector<std::string> parts = android::base::Split("foo", ","); - ASSERT_EQ(1U, parts.size()); - ASSERT_EQ("foo", parts[0]); -} - -TEST(strings, split_simple) { - std::vector<std::string> parts = android::base::Split("foo,bar,baz", ","); - ASSERT_EQ(3U, parts.size()); - ASSERT_EQ("foo", parts[0]); - ASSERT_EQ("bar", parts[1]); - ASSERT_EQ("baz", parts[2]); -} - -TEST(strings, split_with_empty_part) { - std::vector<std::string> parts = android::base::Split("foo,,bar", ","); - ASSERT_EQ(3U, parts.size()); - ASSERT_EQ("foo", parts[0]); - ASSERT_EQ("", parts[1]); - ASSERT_EQ("bar", parts[2]); -} - -TEST(strings, split_with_trailing_empty_part) { - std::vector<std::string> parts = android::base::Split("foo,bar,", ","); - ASSERT_EQ(3U, parts.size()); - ASSERT_EQ("foo", parts[0]); - ASSERT_EQ("bar", parts[1]); - ASSERT_EQ("", parts[2]); -} - -TEST(strings, split_null_char) { - std::vector<std::string> parts = - android::base::Split(std::string("foo\0bar", 7), std::string("\0", 1)); - ASSERT_EQ(2U, parts.size()); - ASSERT_EQ("foo", parts[0]); - ASSERT_EQ("bar", parts[1]); -} - -TEST(strings, split_any) { - std::vector<std::string> parts = android::base::Split("foo:bar,baz", ",:"); - ASSERT_EQ(3U, parts.size()); - ASSERT_EQ("foo", parts[0]); - ASSERT_EQ("bar", parts[1]); - ASSERT_EQ("baz", parts[2]); -} - -TEST(strings, split_any_with_empty_part) { - std::vector<std::string> parts = android::base::Split("foo:,bar", ",:"); - ASSERT_EQ(3U, parts.size()); - ASSERT_EQ("foo", parts[0]); - ASSERT_EQ("", parts[1]); - ASSERT_EQ("bar", parts[2]); -} - -TEST(strings, trim_empty) { - ASSERT_EQ("", android::base::Trim("")); -} - -TEST(strings, trim_already_trimmed) { - ASSERT_EQ("foo", android::base::Trim("foo")); -} - -TEST(strings, trim_left) { - ASSERT_EQ("foo", android::base::Trim(" foo")); -} - -TEST(strings, trim_right) { - ASSERT_EQ("foo", android::base::Trim("foo ")); -} - -TEST(strings, trim_both) { - ASSERT_EQ("foo", android::base::Trim(" foo ")); -} - -TEST(strings, trim_no_trim_middle) { - ASSERT_EQ("foo bar", android::base::Trim("foo bar")); -} - -TEST(strings, trim_other_whitespace) { - ASSERT_EQ("foo", android::base::Trim("\v\tfoo\n\f")); -} - -TEST(strings, join_nothing) { - std::vector<std::string> list = {}; - ASSERT_EQ("", android::base::Join(list, ',')); -} - -TEST(strings, join_single) { - std::vector<std::string> list = {"foo"}; - ASSERT_EQ("foo", android::base::Join(list, ',')); -} - -TEST(strings, join_simple) { - std::vector<std::string> list = {"foo", "bar", "baz"}; - ASSERT_EQ("foo,bar,baz", android::base::Join(list, ',')); -} - -TEST(strings, join_separator_in_vector) { - std::vector<std::string> list = {",", ","}; - ASSERT_EQ(",,,", android::base::Join(list, ',')); -} - -TEST(strings, join_simple_ints) { - std::set<int> list = {1, 2, 3}; - ASSERT_EQ("1,2,3", android::base::Join(list, ',')); -} - -TEST(strings, join_unordered_set) { - std::unordered_set<int> list = {1, 2}; - ASSERT_TRUE("1,2" == android::base::Join(list, ',') || - "2,1" == android::base::Join(list, ',')); -} - -TEST(strings, StartsWith_empty) { - ASSERT_FALSE(android::base::StartsWith("", "foo")); - ASSERT_TRUE(android::base::StartsWith("", "")); -} - -TEST(strings, StartsWithIgnoreCase_empty) { - ASSERT_FALSE(android::base::StartsWithIgnoreCase("", "foo")); - ASSERT_TRUE(android::base::StartsWithIgnoreCase("", "")); -} - -TEST(strings, StartsWith_simple) { - ASSERT_TRUE(android::base::StartsWith("foo", "")); - ASSERT_TRUE(android::base::StartsWith("foo", "f")); - ASSERT_TRUE(android::base::StartsWith("foo", "fo")); - ASSERT_TRUE(android::base::StartsWith("foo", "foo")); -} - -TEST(strings, StartsWithIgnoreCase_simple) { - ASSERT_TRUE(android::base::StartsWithIgnoreCase("foo", "")); - ASSERT_TRUE(android::base::StartsWithIgnoreCase("foo", "f")); - ASSERT_TRUE(android::base::StartsWithIgnoreCase("foo", "F")); - ASSERT_TRUE(android::base::StartsWithIgnoreCase("foo", "fo")); - ASSERT_TRUE(android::base::StartsWithIgnoreCase("foo", "fO")); - ASSERT_TRUE(android::base::StartsWithIgnoreCase("foo", "Fo")); - ASSERT_TRUE(android::base::StartsWithIgnoreCase("foo", "FO")); - ASSERT_TRUE(android::base::StartsWithIgnoreCase("foo", "foo")); - ASSERT_TRUE(android::base::StartsWithIgnoreCase("foo", "foO")); - ASSERT_TRUE(android::base::StartsWithIgnoreCase("foo", "fOo")); - ASSERT_TRUE(android::base::StartsWithIgnoreCase("foo", "fOO")); - ASSERT_TRUE(android::base::StartsWithIgnoreCase("foo", "Foo")); - ASSERT_TRUE(android::base::StartsWithIgnoreCase("foo", "FoO")); - ASSERT_TRUE(android::base::StartsWithIgnoreCase("foo", "FOo")); - ASSERT_TRUE(android::base::StartsWithIgnoreCase("foo", "FOO")); -} - -TEST(strings, StartsWith_prefix_too_long) { - ASSERT_FALSE(android::base::StartsWith("foo", "foobar")); -} - -TEST(strings, StartsWithIgnoreCase_prefix_too_long) { - ASSERT_FALSE(android::base::StartsWithIgnoreCase("foo", "foobar")); - ASSERT_FALSE(android::base::StartsWithIgnoreCase("foo", "FOOBAR")); -} - -TEST(strings, StartsWith_contains_prefix) { - ASSERT_FALSE(android::base::StartsWith("foobar", "oba")); - ASSERT_FALSE(android::base::StartsWith("foobar", "bar")); -} - -TEST(strings, StartsWithIgnoreCase_contains_prefix) { - ASSERT_FALSE(android::base::StartsWithIgnoreCase("foobar", "oba")); - ASSERT_FALSE(android::base::StartsWithIgnoreCase("foobar", "OBA")); - ASSERT_FALSE(android::base::StartsWithIgnoreCase("foobar", "bar")); - ASSERT_FALSE(android::base::StartsWithIgnoreCase("foobar", "BAR")); -} - -TEST(strings, StartsWith_char) { - ASSERT_FALSE(android::base::StartsWith("", 'f')); - ASSERT_TRUE(android::base::StartsWith("foo", 'f')); - ASSERT_FALSE(android::base::StartsWith("foo", 'o')); -} - -TEST(strings, EndsWith_empty) { - ASSERT_FALSE(android::base::EndsWith("", "foo")); - ASSERT_TRUE(android::base::EndsWith("", "")); -} - -TEST(strings, EndsWithIgnoreCase_empty) { - ASSERT_FALSE(android::base::EndsWithIgnoreCase("", "foo")); - ASSERT_FALSE(android::base::EndsWithIgnoreCase("", "FOO")); - ASSERT_TRUE(android::base::EndsWithIgnoreCase("", "")); -} - -TEST(strings, EndsWith_simple) { - ASSERT_TRUE(android::base::EndsWith("foo", "")); - ASSERT_TRUE(android::base::EndsWith("foo", "o")); - ASSERT_TRUE(android::base::EndsWith("foo", "oo")); - ASSERT_TRUE(android::base::EndsWith("foo", "foo")); -} - -TEST(strings, EndsWithIgnoreCase_simple) { - ASSERT_TRUE(android::base::EndsWithIgnoreCase("foo", "")); - ASSERT_TRUE(android::base::EndsWithIgnoreCase("foo", "o")); - ASSERT_TRUE(android::base::EndsWithIgnoreCase("foo", "O")); - ASSERT_TRUE(android::base::EndsWithIgnoreCase("foo", "oo")); - ASSERT_TRUE(android::base::EndsWithIgnoreCase("foo", "oO")); - ASSERT_TRUE(android::base::EndsWithIgnoreCase("foo", "Oo")); - ASSERT_TRUE(android::base::EndsWithIgnoreCase("foo", "OO")); - ASSERT_TRUE(android::base::EndsWithIgnoreCase("foo", "foo")); - ASSERT_TRUE(android::base::EndsWithIgnoreCase("foo", "foO")); - ASSERT_TRUE(android::base::EndsWithIgnoreCase("foo", "fOo")); - ASSERT_TRUE(android::base::EndsWithIgnoreCase("foo", "fOO")); - ASSERT_TRUE(android::base::EndsWithIgnoreCase("foo", "Foo")); - ASSERT_TRUE(android::base::EndsWithIgnoreCase("foo", "FoO")); - ASSERT_TRUE(android::base::EndsWithIgnoreCase("foo", "FOo")); - ASSERT_TRUE(android::base::EndsWithIgnoreCase("foo", "FOO")); -} - -TEST(strings, EndsWith_prefix_too_long) { - ASSERT_FALSE(android::base::EndsWith("foo", "foobar")); -} - -TEST(strings, EndsWithIgnoreCase_prefix_too_long) { - ASSERT_FALSE(android::base::EndsWithIgnoreCase("foo", "foobar")); - ASSERT_FALSE(android::base::EndsWithIgnoreCase("foo", "FOOBAR")); -} - -TEST(strings, EndsWith_contains_prefix) { - ASSERT_FALSE(android::base::EndsWith("foobar", "oba")); - ASSERT_FALSE(android::base::EndsWith("foobar", "foo")); -} - -TEST(strings, EndsWithIgnoreCase_contains_prefix) { - ASSERT_FALSE(android::base::EndsWithIgnoreCase("foobar", "OBA")); - ASSERT_FALSE(android::base::EndsWithIgnoreCase("foobar", "FOO")); -} - -TEST(strings, StartsWith_std_string) { - ASSERT_TRUE(android::base::StartsWith("hello", std::string{"hell"})); - ASSERT_FALSE(android::base::StartsWith("goodbye", std::string{"hell"})); -} - -TEST(strings, StartsWithIgnoreCase_std_string) { - ASSERT_TRUE(android::base::StartsWithIgnoreCase("HeLlO", std::string{"hell"})); - ASSERT_FALSE(android::base::StartsWithIgnoreCase("GoOdByE", std::string{"hell"})); -} - -TEST(strings, EndsWith_std_string) { - ASSERT_TRUE(android::base::EndsWith("hello", std::string{"lo"})); - ASSERT_FALSE(android::base::EndsWith("goodbye", std::string{"lo"})); -} - -TEST(strings, EndsWithIgnoreCase_std_string) { - ASSERT_TRUE(android::base::EndsWithIgnoreCase("HeLlO", std::string{"lo"})); - ASSERT_FALSE(android::base::EndsWithIgnoreCase("GoOdByE", std::string{"lo"})); -} - -TEST(strings, EndsWith_char) { - ASSERT_FALSE(android::base::EndsWith("", 'o')); - ASSERT_TRUE(android::base::EndsWith("foo", 'o')); - ASSERT_FALSE(android::base::EndsWith("foo", "f")); -} - -TEST(strings, EqualsIgnoreCase) { - ASSERT_TRUE(android::base::EqualsIgnoreCase("foo", "FOO")); - ASSERT_TRUE(android::base::EqualsIgnoreCase("FOO", "foo")); - ASSERT_FALSE(android::base::EqualsIgnoreCase("foo", "bar")); - ASSERT_FALSE(android::base::EqualsIgnoreCase("foo", "fool")); -} - -TEST(strings, ubsan_28729303) { - android::base::Split("/dev/null", ":"); -} - -TEST(strings, ConsumePrefix) { - std::string_view s{"foo.bar"}; - ASSERT_FALSE(android::base::ConsumePrefix(&s, "bar.")); - ASSERT_EQ("foo.bar", s); - ASSERT_TRUE(android::base::ConsumePrefix(&s, "foo.")); - ASSERT_EQ("bar", s); -} - -TEST(strings, ConsumeSuffix) { - std::string_view s{"foo.bar"}; - ASSERT_FALSE(android::base::ConsumeSuffix(&s, ".foo")); - ASSERT_EQ("foo.bar", s); - ASSERT_TRUE(android::base::ConsumeSuffix(&s, ".bar")); - ASSERT_EQ("foo", s); -} - -TEST(strings, StringReplace_false) { - // No change. - ASSERT_EQ("abcabc", android::base::StringReplace("abcabc", "z", "Z", false)); - ASSERT_EQ("", android::base::StringReplace("", "z", "Z", false)); - ASSERT_EQ("abcabc", android::base::StringReplace("abcabc", "", "Z", false)); - - // Equal lengths. - ASSERT_EQ("Abcabc", android::base::StringReplace("abcabc", "a", "A", false)); - ASSERT_EQ("aBcabc", android::base::StringReplace("abcabc", "b", "B", false)); - ASSERT_EQ("abCabc", android::base::StringReplace("abcabc", "c", "C", false)); - - // Longer replacement. - ASSERT_EQ("foobcabc", android::base::StringReplace("abcabc", "a", "foo", false)); - ASSERT_EQ("afoocabc", android::base::StringReplace("abcabc", "b", "foo", false)); - ASSERT_EQ("abfooabc", android::base::StringReplace("abcabc", "c", "foo", false)); - - // Shorter replacement. - ASSERT_EQ("xxyz", android::base::StringReplace("abcxyz", "abc", "x", false)); - ASSERT_EQ("axyz", android::base::StringReplace("abcxyz", "bcx", "x", false)); - ASSERT_EQ("abcx", android::base::StringReplace("abcxyz", "xyz", "x", false)); -} - -TEST(strings, StringReplace_true) { - // No change. - ASSERT_EQ("abcabc", android::base::StringReplace("abcabc", "z", "Z", true)); - ASSERT_EQ("", android::base::StringReplace("", "z", "Z", true)); - ASSERT_EQ("abcabc", android::base::StringReplace("abcabc", "", "Z", true)); - - // Equal lengths. - ASSERT_EQ("AbcAbc", android::base::StringReplace("abcabc", "a", "A", true)); - ASSERT_EQ("aBcaBc", android::base::StringReplace("abcabc", "b", "B", true)); - ASSERT_EQ("abCabC", android::base::StringReplace("abcabc", "c", "C", true)); - - // Longer replacement. - ASSERT_EQ("foobcfoobc", android::base::StringReplace("abcabc", "a", "foo", true)); - ASSERT_EQ("afoocafooc", android::base::StringReplace("abcabc", "b", "foo", true)); - ASSERT_EQ("abfooabfoo", android::base::StringReplace("abcabc", "c", "foo", true)); - - // Shorter replacement. - ASSERT_EQ("xxyzx", android::base::StringReplace("abcxyzabc", "abc", "x", true)); - ASSERT_EQ("<xx>", android::base::StringReplace("<abcabc>", "abc", "x", true)); -} diff --git a/base/test_main.cpp b/base/test_main.cpp deleted file mode 100644 index 7fa6a8425..000000000 --- a/base/test_main.cpp +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <gtest/gtest.h> - -#include "android-base/logging.h" - -int main(int argc, char** argv) { - ::testing::InitGoogleTest(&argc, argv); - android::base::InitLogging(argv, android::base::StderrLogger); - return RUN_ALL_TESTS(); -} diff --git a/base/test_utils.cpp b/base/test_utils.cpp deleted file mode 100644 index 36b4cdfa4..000000000 --- a/base/test_utils.cpp +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "android-base/test_utils.h" - -#include <fcntl.h> -#include <stdio.h> -#include <stdlib.h> -#include <sys/stat.h> -#include <unistd.h> - -#include <string> - -#include <android-base/file.h> -#include <android-base/logging.h> - -CapturedStdFd::CapturedStdFd(int std_fd) : std_fd_(std_fd), old_fd_(-1) { - Start(); -} - -CapturedStdFd::~CapturedStdFd() { - if (old_fd_ != -1) { - Stop(); - } -} - -int CapturedStdFd::fd() const { - return temp_file_.fd; -} - -std::string CapturedStdFd::str() { - std::string result; - CHECK_EQ(0, TEMP_FAILURE_RETRY(lseek(fd(), 0, SEEK_SET))); - android::base::ReadFdToString(fd(), &result); - return result; -} - -void CapturedStdFd::Reset() { - // Do not reset while capturing. - CHECK_EQ(-1, old_fd_); - CHECK_EQ(0, TEMP_FAILURE_RETRY(lseek(fd(), 0, SEEK_SET))); - CHECK_EQ(0, ftruncate(fd(), 0)); -} - -void CapturedStdFd::Start() { -#if defined(_WIN32) - // On Windows, stderr is often buffered, so make sure it is unbuffered so - // that we can immediately read back what was written to stderr. - if (std_fd_ == STDERR_FILENO) CHECK_EQ(0, setvbuf(stderr, nullptr, _IONBF, 0)); -#endif - old_fd_ = dup(std_fd_); - CHECK_NE(-1, old_fd_); - CHECK_NE(-1, dup2(fd(), std_fd_)); -} - -void CapturedStdFd::Stop() { - CHECK_NE(-1, old_fd_); - CHECK_NE(-1, dup2(old_fd_, std_fd_)); - close(old_fd_); - old_fd_ = -1; - // Note: cannot restore prior setvbuf() setting. -} diff --git a/base/test_utils_test.cpp b/base/test_utils_test.cpp deleted file mode 100644 index 15a79dd9d..000000000 --- a/base/test_utils_test.cpp +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright (C) 2017 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <stdio.h> - -#include "android-base/test_utils.h" - -#include <gtest/gtest-spi.h> -#include <gtest/gtest.h> - -namespace android { -namespace base { - -TEST(TestUtilsTest, AssertMatch) { - ASSERT_MATCH("foobar", R"(fo+baz?r)"); - EXPECT_FATAL_FAILURE(ASSERT_MATCH("foobar", R"(foobaz)"), "regex mismatch"); -} - -TEST(TestUtilsTest, AssertNotMatch) { - ASSERT_NOT_MATCH("foobar", R"(foobaz)"); - EXPECT_FATAL_FAILURE(ASSERT_NOT_MATCH("foobar", R"(foobar)"), "regex mismatch"); -} - -TEST(TestUtilsTest, ExpectMatch) { - EXPECT_MATCH("foobar", R"(fo+baz?r)"); - EXPECT_NONFATAL_FAILURE(EXPECT_MATCH("foobar", R"(foobaz)"), "regex mismatch"); -} - -TEST(TestUtilsTest, ExpectNotMatch) { - EXPECT_NOT_MATCH("foobar", R"(foobaz)"); - EXPECT_NONFATAL_FAILURE(EXPECT_NOT_MATCH("foobar", R"(foobar)"), "regex mismatch"); -} - -TEST(TestUtilsTest, CaptureStdout_smoke) { - CapturedStdout cap; - printf("This should be captured.\n"); - cap.Stop(); - printf("This will not be captured.\n"); - ASSERT_EQ("This should be captured.\n", cap.str()); - - cap.Start(); - printf("And this text should be captured too.\n"); - cap.Stop(); - ASSERT_EQ("This should be captured.\nAnd this text should be captured too.\n", cap.str()); - - printf("Still not going to be captured.\n"); - cap.Reset(); - cap.Start(); - printf("Only this will be captured.\n"); - ASSERT_EQ("Only this will be captured.\n", cap.str()); -} - -TEST(TestUtilsTest, CaptureStderr_smoke) { - CapturedStderr cap; - fprintf(stderr, "This should be captured.\n"); - cap.Stop(); - fprintf(stderr, "This will not be captured.\n"); - ASSERT_EQ("This should be captured.\n", cap.str()); - - cap.Start(); - fprintf(stderr, "And this text should be captured too.\n"); - cap.Stop(); - ASSERT_EQ("This should be captured.\nAnd this text should be captured too.\n", cap.str()); - - fprintf(stderr, "Still not going to be captured.\n"); - cap.Reset(); - cap.Start(); - fprintf(stderr, "Only this will be captured.\n"); - ASSERT_EQ("Only this will be captured.\n", cap.str()); -} - -} // namespace base -} // namespace android diff --git a/base/threads.cpp b/base/threads.cpp deleted file mode 100644 index 48f6197ef..000000000 --- a/base/threads.cpp +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <android-base/threads.h> - -#include <stdint.h> -#include <unistd.h> - -#if defined(__APPLE__) -#include <pthread.h> -#elif defined(__linux__) && !defined(__ANDROID__) -#include <syscall.h> -#elif defined(_WIN32) -#include <windows.h> -#endif - -namespace android { -namespace base { - -uint64_t GetThreadId() { -#if defined(__BIONIC__) - return gettid(); -#elif defined(__APPLE__) - uint64_t tid; - pthread_threadid_np(NULL, &tid); - return tid; -#elif defined(__linux__) - return syscall(__NR_gettid); -#elif defined(_WIN32) - return GetCurrentThreadId(); -#endif -} - -} // namespace base -} // namespace android - -#if defined(__GLIBC__) -int tgkill(int tgid, int tid, int sig) { - return syscall(__NR_tgkill, tgid, tid, sig); -} -#endif diff --git a/base/utf8.cpp b/base/utf8.cpp deleted file mode 100644 index adb46d09c..000000000 --- a/base/utf8.cpp +++ /dev/null @@ -1,235 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <windows.h> - -#include "android-base/utf8.h" - -#include <fcntl.h> -#include <stdio.h> - -#include <algorithm> -#include <string> - -#include "android-base/logging.h" - -namespace android { -namespace base { - -// Helper to set errno based on GetLastError() after WideCharToMultiByte()/MultiByteToWideChar(). -static void SetErrnoFromLastError() { - switch (GetLastError()) { - case ERROR_NO_UNICODE_TRANSLATION: - errno = EILSEQ; - break; - default: - errno = EINVAL; - break; - } -} - -bool WideToUTF8(const wchar_t* utf16, const size_t size, std::string* utf8) { - utf8->clear(); - - if (size == 0) { - return true; - } - - // TODO: Consider using std::wstring_convert once libcxx is supported on - // Windows. - - // Only Vista or later has this flag that causes WideCharToMultiByte() to - // return an error on invalid characters. - const DWORD flags = -#if (WINVER >= 0x0600) - WC_ERR_INVALID_CHARS; -#else - 0; -#endif - - const int chars_required = WideCharToMultiByte(CP_UTF8, flags, utf16, size, - NULL, 0, NULL, NULL); - if (chars_required <= 0) { - SetErrnoFromLastError(); - return false; - } - - // This could potentially throw a std::bad_alloc exception. - utf8->resize(chars_required); - - const int result = WideCharToMultiByte(CP_UTF8, flags, utf16, size, - &(*utf8)[0], chars_required, NULL, - NULL); - if (result != chars_required) { - SetErrnoFromLastError(); - CHECK_LE(result, chars_required) << "WideCharToMultiByte wrote " << result - << " chars to buffer of " << chars_required << " chars"; - utf8->clear(); - return false; - } - - return true; -} - -bool WideToUTF8(const wchar_t* utf16, std::string* utf8) { - // Compute string length of NULL-terminated string with wcslen(). - return WideToUTF8(utf16, wcslen(utf16), utf8); -} - -bool WideToUTF8(const std::wstring& utf16, std::string* utf8) { - // Use the stored length of the string which allows embedded NULL characters - // to be converted. - return WideToUTF8(utf16.c_str(), utf16.length(), utf8); -} - -// Internal helper function that takes MultiByteToWideChar() flags. -static bool UTF8ToWideWithFlags(const char* utf8, const size_t size, std::wstring* utf16, - const DWORD flags) { - utf16->clear(); - - if (size == 0) { - return true; - } - - // TODO: Consider using std::wstring_convert once libcxx is supported on - // Windows. - const int chars_required = MultiByteToWideChar(CP_UTF8, flags, utf8, size, - NULL, 0); - if (chars_required <= 0) { - SetErrnoFromLastError(); - return false; - } - - // This could potentially throw a std::bad_alloc exception. - utf16->resize(chars_required); - - const int result = MultiByteToWideChar(CP_UTF8, flags, utf8, size, - &(*utf16)[0], chars_required); - if (result != chars_required) { - SetErrnoFromLastError(); - CHECK_LE(result, chars_required) << "MultiByteToWideChar wrote " << result - << " chars to buffer of " << chars_required << " chars"; - utf16->clear(); - return false; - } - - return true; -} - -bool UTF8ToWide(const char* utf8, const size_t size, std::wstring* utf16) { - // If strictly interpreting as UTF-8 succeeds, return success. - if (UTF8ToWideWithFlags(utf8, size, utf16, MB_ERR_INVALID_CHARS)) { - return true; - } - - const int saved_errno = errno; - - // Fallback to non-strict interpretation, allowing invalid characters and - // converting as best as possible, and return false to signify a problem. - (void)UTF8ToWideWithFlags(utf8, size, utf16, 0); - errno = saved_errno; - return false; -} - -bool UTF8ToWide(const char* utf8, std::wstring* utf16) { - // Compute string length of NULL-terminated string with strlen(). - return UTF8ToWide(utf8, strlen(utf8), utf16); -} - -bool UTF8ToWide(const std::string& utf8, std::wstring* utf16) { - // Use the stored length of the string which allows embedded NULL characters - // to be converted. - return UTF8ToWide(utf8.c_str(), utf8.length(), utf16); -} - -static bool isDriveLetter(wchar_t c) { - return (c >= L'a' && c <= L'z') || (c >= L'A' && c <= L'Z'); -} - -bool UTF8PathToWindowsLongPath(const char* utf8, std::wstring* utf16) { - if (!UTF8ToWide(utf8, utf16)) { - return false; - } - // Note: Although most Win32 File I/O API are limited to MAX_PATH (260 - // characters), the CreateDirectory API is limited to 248 characters. - if (utf16->length() >= 248) { - // If path is of the form "x:\" or "x:/" - if (isDriveLetter((*utf16)[0]) && (*utf16)[1] == L':' && - ((*utf16)[2] == L'\\' || (*utf16)[2] == L'/')) { - // Append long path prefix, and make sure there are no unix-style - // separators to ensure a fully compliant Win32 long path string. - utf16->insert(0, LR"(\\?\)"); - std::replace(utf16->begin(), utf16->end(), L'/', L'\\'); - } - } - return true; -} - -// Versions of standard library APIs that support UTF-8 strings. -namespace utf8 { - -FILE* fopen(const char* name, const char* mode) { - std::wstring name_utf16; - if (!UTF8PathToWindowsLongPath(name, &name_utf16)) { - return nullptr; - } - - std::wstring mode_utf16; - if (!UTF8ToWide(mode, &mode_utf16)) { - return nullptr; - } - - return _wfopen(name_utf16.c_str(), mode_utf16.c_str()); -} - -int mkdir(const char* name, mode_t) { - std::wstring name_utf16; - if (!UTF8PathToWindowsLongPath(name, &name_utf16)) { - return -1; - } - - return _wmkdir(name_utf16.c_str()); -} - -int open(const char* name, int flags, ...) { - std::wstring name_utf16; - if (!UTF8PathToWindowsLongPath(name, &name_utf16)) { - return -1; - } - - int mode = 0; - if ((flags & O_CREAT) != 0) { - va_list args; - va_start(args, flags); - mode = va_arg(args, int); - va_end(args); - } - - return _wopen(name_utf16.c_str(), flags, mode); -} - -int unlink(const char* name) { - std::wstring name_utf16; - if (!UTF8PathToWindowsLongPath(name, &name_utf16)) { - return -1; - } - - return _wunlink(name_utf16.c_str()); -} - -} // namespace utf8 -} // namespace base -} // namespace android diff --git a/base/utf8_test.cpp b/base/utf8_test.cpp deleted file mode 100644 index 472e82c63..000000000 --- a/base/utf8_test.cpp +++ /dev/null @@ -1,488 +0,0 @@ -/* -* Copyright (C) 2015 The Android Open Source Project -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -#include "android-base/utf8.h" - -#include <gtest/gtest.h> - -#include <fcntl.h> -#include <stdlib.h> - -#include "android-base/file.h" -#include "android-base/macros.h" -#include "android-base/unique_fd.h" - -namespace android { -namespace base { - -TEST(UTFStringConversionsTest, ConvertInvalidUTF8) { - std::wstring wide; - - errno = 0; - - // Standalone \xa2 is an invalid UTF-8 sequence, so this should return an - // error. Concatenate two C/C++ literal string constants to prevent the - // compiler from giving an error about "\xa2af" containing a "hex escape - // sequence out of range". - EXPECT_FALSE(android::base::UTF8ToWide("before\xa2" "after", &wide)); - - EXPECT_EQ(EILSEQ, errno); - - // Even if an invalid character is encountered, UTF8ToWide() should still do - // its best to convert the rest of the string. sysdeps_win32.cpp: - // _console_write_utf8() depends on this behavior. - // - // Thus, we verify that the valid characters are converted, but we ignore the - // specific replacement character that UTF8ToWide() may replace the invalid - // UTF-8 characters with because we want to allow that to change if the - // implementation changes. - EXPECT_EQ(0U, wide.find(L"before")); - const wchar_t after_wide[] = L"after"; - EXPECT_EQ(wide.length() - (arraysize(after_wide) - 1), wide.find(after_wide)); -} - -// Below is adapted from https://chromium.googlesource.com/chromium/src/+/master/base/strings/utf_string_conversions_unittest.cc - -// Copyright (c) 2010 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// The tests below from utf_string_conversions_unittest.cc check for this -// preprocessor symbol, so define it, as it is appropriate for Windows. -#define WCHAR_T_IS_UTF16 -static_assert(sizeof(wchar_t) == 2, "wchar_t is not 2 bytes"); - -// The tests below from utf_string_conversions_unittest.cc call versions of -// UTF8ToWide() and WideToUTF8() that don't return success/failure, so these are -// stub implementations with that signature. These are just for testing and -// should not be moved to base because they assert/expect no errors which is -// probably not a good idea (or at least it is something that should be left -// up to the caller, not a base library). - -static std::wstring UTF8ToWide(const std::string& utf8) { - std::wstring utf16; - EXPECT_TRUE(UTF8ToWide(utf8, &utf16)); - return utf16; -} - -static std::string WideToUTF8(const std::wstring& utf16) { - std::string utf8; - EXPECT_TRUE(WideToUTF8(utf16, &utf8)); - return utf8; -} - -namespace { - -const wchar_t* const kConvertRoundtripCases[] = { - L"Google Video", - // "网页 图片 资讯更多 »" - L"\x7f51\x9875\x0020\x56fe\x7247\x0020\x8d44\x8baf\x66f4\x591a\x0020\x00bb", - // "Παγκόσμιος Ιστός" - L"\x03a0\x03b1\x03b3\x03ba\x03cc\x03c3\x03bc\x03b9" - L"\x03bf\x03c2\x0020\x0399\x03c3\x03c4\x03cc\x03c2", - // "Поиск страниц на русском" - L"\x041f\x043e\x0438\x0441\x043a\x0020\x0441\x0442" - L"\x0440\x0430\x043d\x0438\x0446\x0020\x043d\x0430" - L"\x0020\x0440\x0443\x0441\x0441\x043a\x043e\x043c", - // "전체서비스" - L"\xc804\xccb4\xc11c\xbe44\xc2a4", - - // Test characters that take more than 16 bits. This will depend on whether - // wchar_t is 16 or 32 bits. -#if defined(WCHAR_T_IS_UTF16) - L"\xd800\xdf00", - // ????? (Mathematical Alphanumeric Symbols (U+011d40 - U+011d44 : A,B,C,D,E) - L"\xd807\xdd40\xd807\xdd41\xd807\xdd42\xd807\xdd43\xd807\xdd44", -#elif defined(WCHAR_T_IS_UTF32) - L"\x10300", - // ????? (Mathematical Alphanumeric Symbols (U+011d40 - U+011d44 : A,B,C,D,E) - L"\x11d40\x11d41\x11d42\x11d43\x11d44", -#endif -}; - -} // namespace - -TEST(UTFStringConversionsTest, ConvertUTF8AndWide) { - // we round-trip all the wide strings through UTF-8 to make sure everything - // agrees on the conversion. This uses the stream operators to test them - // simultaneously. - for (size_t i = 0; i < arraysize(kConvertRoundtripCases); ++i) { - std::ostringstream utf8; - utf8 << WideToUTF8(kConvertRoundtripCases[i]); - std::wostringstream wide; - wide << UTF8ToWide(utf8.str()); - - EXPECT_EQ(kConvertRoundtripCases[i], wide.str()); - } -} - -TEST(UTFStringConversionsTest, ConvertUTF8AndWideEmptyString) { - // An empty std::wstring should be converted to an empty std::string, - // and vice versa. - std::wstring wempty; - std::string empty; - EXPECT_EQ(empty, WideToUTF8(wempty)); - EXPECT_EQ(wempty, UTF8ToWide(empty)); -} - -TEST(UTFStringConversionsTest, ConvertUTF8ToWide) { - struct UTF8ToWideCase { - const char* utf8; - const wchar_t* wide; - bool success; - } convert_cases[] = { - // Regular UTF-8 input. - {"\xe4\xbd\xa0\xe5\xa5\xbd", L"\x4f60\x597d", true}, - // Non-character is passed through. - {"\xef\xbf\xbfHello", L"\xffffHello", true}, - // Truncated UTF-8 sequence. - {"\xe4\xa0\xe5\xa5\xbd", L"\xfffd\x597d", false}, - // Truncated off the end. - {"\xe5\xa5\xbd\xe4\xa0", L"\x597d\xfffd", false}, - // Non-shortest-form UTF-8. - {"\xf0\x84\xbd\xa0\xe5\xa5\xbd", L"\xfffd\x597d", false}, - // This UTF-8 character decodes to a UTF-16 surrogate, which is illegal. - // Note that for whatever reason, this test fails on Windows XP. - {"\xed\xb0\x80", L"\xfffd", false}, - // Non-BMP characters. The second is a non-character regarded as valid. - // The result will either be in UTF-16 or UTF-32. -#if defined(WCHAR_T_IS_UTF16) - {"A\xF0\x90\x8C\x80z", L"A\xd800\xdf00z", true}, - {"A\xF4\x8F\xBF\xBEz", L"A\xdbff\xdffez", true}, -#elif defined(WCHAR_T_IS_UTF32) - {"A\xF0\x90\x8C\x80z", L"A\x10300z", true}, - {"A\xF4\x8F\xBF\xBEz", L"A\x10fffez", true}, -#endif - }; - - for (size_t i = 0; i < arraysize(convert_cases); i++) { - std::wstring converted; - errno = 0; - const bool success = UTF8ToWide(convert_cases[i].utf8, - strlen(convert_cases[i].utf8), - &converted); - EXPECT_EQ(convert_cases[i].success, success); - // The original test always compared expected and converted, but don't do - // that because our implementation of UTF8ToWide() does not guarantee to - // produce the same output in error situations. - if (success) { - std::wstring expected(convert_cases[i].wide); - EXPECT_EQ(expected, converted); - } else { - EXPECT_EQ(EILSEQ, errno); - } - } - - // Manually test an embedded NULL. - std::wstring converted; - EXPECT_TRUE(UTF8ToWide("\00Z\t", 3, &converted)); - ASSERT_EQ(3U, converted.length()); - EXPECT_EQ(static_cast<wchar_t>(0), converted[0]); - EXPECT_EQ('Z', converted[1]); - EXPECT_EQ('\t', converted[2]); - - // Make sure that conversion replaces, not appends. - EXPECT_TRUE(UTF8ToWide("B", 1, &converted)); - ASSERT_EQ(1U, converted.length()); - EXPECT_EQ('B', converted[0]); -} - -#if defined(WCHAR_T_IS_UTF16) -// This test is only valid when wchar_t == UTF-16. -TEST(UTFStringConversionsTest, ConvertUTF16ToUTF8) { - struct WideToUTF8Case { - const wchar_t* utf16; - const char* utf8; - bool success; - } convert_cases[] = { - // Regular UTF-16 input. - {L"\x4f60\x597d", "\xe4\xbd\xa0\xe5\xa5\xbd", true}, - // Test a non-BMP character. - {L"\xd800\xdf00", "\xF0\x90\x8C\x80", true}, - // Non-characters are passed through. - {L"\xffffHello", "\xEF\xBF\xBFHello", true}, - {L"\xdbff\xdffeHello", "\xF4\x8F\xBF\xBEHello", true}, - // The first character is a truncated UTF-16 character. - // Note that for whatever reason, this test fails on Windows XP. - {L"\xd800\x597d", "\xef\xbf\xbd\xe5\xa5\xbd", -#if (WINVER >= 0x0600) - // Only Vista and later has a new API/flag that correctly returns false. - false -#else - true -#endif - }, - // Truncated at the end. - // Note that for whatever reason, this test fails on Windows XP. - {L"\x597d\xd800", "\xe5\xa5\xbd\xef\xbf\xbd", -#if (WINVER >= 0x0600) - // Only Vista and later has a new API/flag that correctly returns false. - false -#else - true -#endif - }, - }; - - for (size_t i = 0; i < arraysize(convert_cases); i++) { - std::string converted; - errno = 0; - const bool success = WideToUTF8(convert_cases[i].utf16, - wcslen(convert_cases[i].utf16), - &converted); - EXPECT_EQ(convert_cases[i].success, success); - // The original test always compared expected and converted, but don't do - // that because our implementation of WideToUTF8() does not guarantee to - // produce the same output in error situations. - if (success) { - std::string expected(convert_cases[i].utf8); - EXPECT_EQ(expected, converted); - } else { - EXPECT_EQ(EILSEQ, errno); - } - } -} - -#elif defined(WCHAR_T_IS_UTF32) -// This test is only valid when wchar_t == UTF-32. -TEST(UTFStringConversionsTest, ConvertUTF32ToUTF8) { - struct WideToUTF8Case { - const wchar_t* utf32; - const char* utf8; - bool success; - } convert_cases[] = { - // Regular 16-bit input. - {L"\x4f60\x597d", "\xe4\xbd\xa0\xe5\xa5\xbd", true}, - // Test a non-BMP character. - {L"A\x10300z", "A\xF0\x90\x8C\x80z", true}, - // Non-characters are passed through. - {L"\xffffHello", "\xEF\xBF\xBFHello", true}, - {L"\x10fffeHello", "\xF4\x8F\xBF\xBEHello", true}, - // Invalid Unicode code points. - {L"\xfffffffHello", "\xEF\xBF\xBDHello", false}, - // The first character is a truncated UTF-16 character. - {L"\xd800\x597d", "\xef\xbf\xbd\xe5\xa5\xbd", false}, - {L"\xdc01Hello", "\xef\xbf\xbdHello", false}, - }; - - for (size_t i = 0; i < arraysize(convert_cases); i++) { - std::string converted; - EXPECT_EQ(convert_cases[i].success, - WideToUTF8(convert_cases[i].utf32, - wcslen(convert_cases[i].utf32), - &converted)); - std::string expected(convert_cases[i].utf8); - EXPECT_EQ(expected, converted); - } -} -#endif // defined(WCHAR_T_IS_UTF32) - -// The test below uses these types and functions, so just do enough to get the -// test running. -typedef wchar_t char16; -typedef std::wstring string16; - -template<typename T> -static void* WriteInto(T* t, size_t size) { - // std::(w)string::resize() already includes space for a NULL terminator. - t->resize(size - 1); - return &(*t)[0]; -} - -// A stub implementation that calls a helper from above, just to get the test -// below working. This is just for testing and should not be moved to base -// because this ignores errors which is probably not a good idea, plus it takes -// a string16 type which we don't really have. -static std::string UTF16ToUTF8(const string16& utf16) { - return WideToUTF8(utf16); -} - -TEST(UTFStringConversionsTest, ConvertMultiString) { - static char16 multi16[] = { - 'f', 'o', 'o', '\0', - 'b', 'a', 'r', '\0', - 'b', 'a', 'z', '\0', - '\0' - }; - static char multi[] = { - 'f', 'o', 'o', '\0', - 'b', 'a', 'r', '\0', - 'b', 'a', 'z', '\0', - '\0' - }; - string16 multistring16; - memcpy(WriteInto(&multistring16, arraysize(multi16)), multi16, - sizeof(multi16)); - EXPECT_EQ(arraysize(multi16) - 1, multistring16.length()); - std::string expected; - memcpy(WriteInto(&expected, arraysize(multi)), multi, sizeof(multi)); - EXPECT_EQ(arraysize(multi) - 1, expected.length()); - const std::string& converted = UTF16ToUTF8(multistring16); - EXPECT_EQ(arraysize(multi) - 1, converted.length()); - EXPECT_EQ(expected, converted); -} - -// The tests below from sys_string_conversions_unittest.cc call SysWideToUTF8() -// and SysUTF8ToWide(), so these are stub implementations that call the helpers -// above. These are just for testing and should not be moved to base because -// they ignore errors which is probably not a good idea. - -static std::string SysWideToUTF8(const std::wstring& utf16) { - return WideToUTF8(utf16); -} - -static std::wstring SysUTF8ToWide(const std::string& utf8) { - return UTF8ToWide(utf8); -} - -// Below is adapted from https://chromium.googlesource.com/chromium/src/+/master/base/strings/sys_string_conversions_unittest.cc - -// Copyright (c) 2011 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifdef WCHAR_T_IS_UTF32 -static const std::wstring kSysWideOldItalicLetterA = L"\x10300"; -#else -static const std::wstring kSysWideOldItalicLetterA = L"\xd800\xdf00"; -#endif - -TEST(SysStrings, SysWideToUTF8) { - EXPECT_EQ("Hello, world", SysWideToUTF8(L"Hello, world")); - EXPECT_EQ("\xe4\xbd\xa0\xe5\xa5\xbd", SysWideToUTF8(L"\x4f60\x597d")); - - // >16 bits - EXPECT_EQ("\xF0\x90\x8C\x80", SysWideToUTF8(kSysWideOldItalicLetterA)); - - // Error case. When Windows finds a UTF-16 character going off the end of - // a string, it just converts that literal value to UTF-8, even though this - // is invalid. - // - // This is what XP does, but Vista has different behavior, so we don't bother - // verifying it: - // EXPECT_EQ("\xE4\xBD\xA0\xED\xA0\x80zyxw", - // SysWideToUTF8(L"\x4f60\xd800zyxw")); - - // Test embedded NULLs. - std::wstring wide_null(L"a"); - wide_null.push_back(0); - wide_null.push_back('b'); - - std::string expected_null("a"); - expected_null.push_back(0); - expected_null.push_back('b'); - - EXPECT_EQ(expected_null, SysWideToUTF8(wide_null)); -} - -TEST(SysStrings, SysUTF8ToWide) { - EXPECT_EQ(L"Hello, world", SysUTF8ToWide("Hello, world")); - EXPECT_EQ(L"\x4f60\x597d", SysUTF8ToWide("\xe4\xbd\xa0\xe5\xa5\xbd")); - // >16 bits - EXPECT_EQ(kSysWideOldItalicLetterA, SysUTF8ToWide("\xF0\x90\x8C\x80")); - - // Error case. When Windows finds an invalid UTF-8 character, it just skips - // it. This seems weird because it's inconsistent with the reverse conversion. - // - // This is what XP does, but Vista has different behavior, so we don't bother - // verifying it: - // EXPECT_EQ(L"\x4f60zyxw", SysUTF8ToWide("\xe4\xbd\xa0\xe5\xa5zyxw")); - - // Test embedded NULLs. - std::string utf8_null("a"); - utf8_null.push_back(0); - utf8_null.push_back('b'); - - std::wstring expected_null(L"a"); - expected_null.push_back(0); - expected_null.push_back('b'); - - EXPECT_EQ(expected_null, SysUTF8ToWide(utf8_null)); -} - -TEST(UTF8PathToWindowsLongPathTest, DontAddPrefixIfShorterThanMaxPath) { - std::string utf8 = "c:\\mypath\\myfile.txt"; - - std::wstring wide; - EXPECT_TRUE(UTF8PathToWindowsLongPath(utf8.c_str(), &wide)); - - EXPECT_EQ(std::string::npos, wide.find(LR"(\\?\)")); -} - -TEST(UTF8PathToWindowsLongPathTest, AddPrefixIfLongerThanMaxPath) { - std::string utf8 = "c:\\mypath"; - while (utf8.length() < 300 /* MAX_PATH is 260 */) { - utf8 += "\\mypathsegment"; - } - - std::wstring wide; - EXPECT_TRUE(UTF8PathToWindowsLongPath(utf8.c_str(), &wide)); - - EXPECT_EQ(0U, wide.find(LR"(\\?\)")); - EXPECT_EQ(std::string::npos, wide.find(L"/")); -} - -TEST(UTF8PathToWindowsLongPathTest, AddPrefixAndFixSeparatorsIfLongerThanMaxPath) { - std::string utf8 = "c:/mypath"; - while (utf8.length() < 300 /* MAX_PATH is 260 */) { - utf8 += "/mypathsegment"; - } - - std::wstring wide; - EXPECT_TRUE(UTF8PathToWindowsLongPath(utf8.c_str(), &wide)); - - EXPECT_EQ(0U, wide.find(LR"(\\?\)")); - EXPECT_EQ(std::string::npos, wide.find(L"/")); -} - -namespace utf8 { - -TEST(Utf8FilesTest, CanCreateOpenAndDeleteFileWithLongPath) { - TemporaryDir td; - - // Create long directory path - std::string utf8 = td.path; - while (utf8.length() < 300 /* MAX_PATH is 260 */) { - utf8 += "\\mypathsegment"; - EXPECT_EQ(0, mkdir(utf8.c_str(), 0)); - } - - // Create file - utf8 += "\\test-file.bin"; - int flags = O_WRONLY | O_CREAT | O_TRUNC | O_BINARY; - int mode = 0666; - android::base::unique_fd fd(open(utf8.c_str(), flags, mode)); - EXPECT_NE(-1, fd.get()); - - // Close file - fd.reset(); - EXPECT_EQ(-1, fd.get()); - - // Open file with fopen - FILE* file = fopen(utf8.c_str(), "rb"); - EXPECT_NE(nullptr, file); - - if (file) { - fclose(file); - } - - // Delete file - EXPECT_EQ(0, unlink(utf8.c_str())); -} - -} // namespace utf8 -} // namespace base -} // namespace android diff --git a/cpio/mkbootfs.c b/cpio/mkbootfs.c index e52762e9b..58153f3f5 100644 --- a/cpio/mkbootfs.c +++ b/cpio/mkbootfs.c @@ -13,6 +13,7 @@ #include <fcntl.h> #include <private/android_filesystem_config.h> +#include <private/fs_config.h> /* NOTES ** diff --git a/debuggerd/Android.bp b/debuggerd/Android.bp index c7bd1a83b..31c2d5d64 100644 --- a/debuggerd/Android.bp +++ b/debuggerd/Android.bp @@ -169,6 +169,7 @@ cc_library_static { "libdebuggerd/backtrace.cpp", "libdebuggerd/gwp_asan.cpp", "libdebuggerd/open_files_list.cpp", + "libdebuggerd/scudo.cpp", "libdebuggerd/tombstone.cpp", "libdebuggerd/utility.cpp", ], @@ -176,8 +177,13 @@ cc_library_static { local_include_dirs: ["libdebuggerd/include"], export_include_dirs: ["libdebuggerd/include"], - // Needed for private/bionic_fdsan.h - include_dirs: ["bionic/libc"], + include_dirs: [ + // Needed for private/bionic_fdsan.h + "bionic/libc", + + // Needed for scudo/interface.h + "external/scudo/standalone/include", + ], header_libs: [ "bionic_libc_platform_headers", "gwp_asan_headers", @@ -192,7 +198,10 @@ cc_library_static { "liblog", ], - whole_static_libs: ["gwp_asan_crash_handler"], + whole_static_libs: [ + "gwp_asan_crash_handler", + "libscudo", + ], target: { recovery: { @@ -206,6 +215,9 @@ cc_library_static { debuggable: { cflags: ["-DROOT_POSSIBLE"], }, + experimental_mte: { + cflags: ["-DANDROID_EXPERIMENTAL_MTE"], + }, }, } @@ -256,6 +268,10 @@ cc_test { "gwp_asan_headers", ], + include_dirs: [ + "external/scudo/standalone/include", + ], + local_include_dirs: [ "libdebuggerd", ], @@ -271,6 +287,12 @@ cc_test { }, test_suites: ["device-tests"], + + product_variables: { + experimental_mte: { + cflags: ["-DANDROID_EXPERIMENTAL_MTE"], + }, + }, } cc_benchmark { @@ -317,6 +339,10 @@ cc_binary { "libprocinfo", "libunwindstack", ], + + apex_available: [ + "com.android.runtime", + ], } cc_binary { diff --git a/debuggerd/crash_dump.cpp b/debuggerd/crash_dump.cpp index 3e998802b..d7cb9725f 100644 --- a/debuggerd/crash_dump.cpp +++ b/debuggerd/crash_dump.cpp @@ -254,9 +254,7 @@ static void ParseArgs(int argc, char** argv, pid_t* pseudothread_tid, DebuggerdD } static void ReadCrashInfo(unique_fd& fd, siginfo_t* siginfo, - std::unique_ptr<unwindstack::Regs>* regs, uintptr_t* abort_msg_address, - uintptr_t* fdsan_table_address, uintptr_t* gwp_asan_state, - uintptr_t* gwp_asan_metadata) { + std::unique_ptr<unwindstack::Regs>* regs, ProcessInfo* process_info) { std::aligned_storage<sizeof(CrashInfo) + 1, alignof(CrashInfo)>::type buf; CrashInfo* crash_info = reinterpret_cast<CrashInfo*>(&buf); ssize_t rc = TEMP_FAILURE_RETRY(read(fd.get(), &buf, sizeof(buf))); @@ -266,15 +264,13 @@ static void ReadCrashInfo(unique_fd& fd, siginfo_t* siginfo, ssize_t expected_size = 0; switch (crash_info->header.version) { case 1: - expected_size = sizeof(CrashInfoHeader) + sizeof(CrashInfoDataV1); - break; - case 2: - expected_size = sizeof(CrashInfoHeader) + sizeof(CrashInfoDataV2); + case 3: + expected_size = sizeof(CrashInfoHeader) + sizeof(CrashInfoDataStatic); break; - case 3: - expected_size = sizeof(CrashInfoHeader) + sizeof(CrashInfoDataV3); + case 4: + expected_size = sizeof(CrashInfoHeader) + sizeof(CrashInfoDataDynamic); break; default: @@ -282,28 +278,34 @@ static void ReadCrashInfo(unique_fd& fd, siginfo_t* siginfo, break; }; - if (rc != expected_size) { + if (rc < expected_size) { LOG(FATAL) << "read " << rc << " bytes when reading target crash information, expected " << expected_size; } } - *fdsan_table_address = 0; - *gwp_asan_state = 0; - *gwp_asan_metadata = 0; switch (crash_info->header.version) { - case 3: - *gwp_asan_state = crash_info->data.v3.gwp_asan_state; - *gwp_asan_metadata = crash_info->data.v3.gwp_asan_metadata; - FALLTHROUGH_INTENDED; - case 2: - *fdsan_table_address = crash_info->data.v2.fdsan_table_address; + case 4: + process_info->fdsan_table_address = crash_info->data.d.fdsan_table_address; + process_info->gwp_asan_state = crash_info->data.d.gwp_asan_state; + process_info->gwp_asan_metadata = crash_info->data.d.gwp_asan_metadata; + process_info->scudo_stack_depot = crash_info->data.d.scudo_stack_depot; + process_info->scudo_region_info = crash_info->data.d.scudo_region_info; FALLTHROUGH_INTENDED; case 1: - *abort_msg_address = crash_info->data.v1.abort_msg_address; - *siginfo = crash_info->data.v1.siginfo; + case 2: + case 3: + process_info->abort_msg_address = crash_info->data.s.abort_msg_address; + *siginfo = crash_info->data.s.siginfo; + if (signal_has_si_addr(siginfo)) { + // Make a copy of the ucontext field because otherwise it is not aligned enough (due to + // being in a packed struct) and clang complains about that. + ucontext_t ucontext = crash_info->data.s.ucontext; + process_info->has_fault_address = true; + process_info->fault_address = get_fault_address(siginfo, &ucontext); + } regs->reset(unwindstack::Regs::CreateFromUcontext(unwindstack::Regs::CurrentArch(), - &crash_info->data.v1.ucontext)); + &crash_info->data.s.ucontext)); break; default: @@ -425,10 +427,7 @@ int main(int argc, char** argv) { ATRACE_NAME("after reparent"); pid_t pseudothread_tid; DebuggerdDumpType dump_type; - uintptr_t abort_msg_address = 0; - uintptr_t fdsan_table_address = 0; - uintptr_t gwp_asan_state = 0; - uintptr_t gwp_asan_metadata = 0; + ProcessInfo process_info; Initialize(argv); ParseArgs(argc, argv, &pseudothread_tid, &dump_type); @@ -489,8 +488,7 @@ int main(int argc, char** argv) { if (thread == g_target_thread) { // Read the thread's registers along with the rest of the crash info out of the pipe. - ReadCrashInfo(input_pipe, &siginfo, &info.registers, &abort_msg_address, - &fdsan_table_address, &gwp_asan_state, &gwp_asan_metadata); + ReadCrashInfo(input_pipe, &siginfo, &info.registers, &process_info); info.siginfo = &siginfo; info.signo = info.siginfo->si_signo; } else { @@ -599,14 +597,14 @@ int main(int argc, char** argv) { } else { { ATRACE_NAME("fdsan table dump"); - populate_fdsan_table(&open_files, unwinder.GetProcessMemory(), fdsan_table_address); + populate_fdsan_table(&open_files, unwinder.GetProcessMemory(), + process_info.fdsan_table_address); } { ATRACE_NAME("engrave_tombstone"); - engrave_tombstone(std::move(g_output_fd), &unwinder, thread_info, g_target_thread, - abort_msg_address, &open_files, &amfd_data, gwp_asan_state, - gwp_asan_metadata); + engrave_tombstone(std::move(g_output_fd), &unwinder, thread_info, g_target_thread, process_info, + &open_files, &amfd_data); } } diff --git a/debuggerd/crasher/Android.bp b/debuggerd/crasher/Android.bp index e86f499bd..61c539580 100644 --- a/debuggerd/crasher/Android.bp +++ b/debuggerd/crasher/Android.bp @@ -24,12 +24,6 @@ cc_defaults { arm64: { srcs: ["arm64/crashglue.S"], }, - mips: { - srcs: ["mips/crashglue.S"], - }, - mips64: { - srcs: ["mips64/crashglue.S"], - }, x86: { srcs: ["x86/crashglue.S"], }, diff --git a/debuggerd/crasher/crasher.cpp b/debuggerd/crasher/crasher.cpp index 304166472..a2b13a36c 100644 --- a/debuggerd/crasher/crasher.cpp +++ b/debuggerd/crasher/crasher.cpp @@ -349,7 +349,7 @@ noinline int do_action(const char* arg) { int main(int argc, char** argv) { #if defined(STATIC_CRASHER) debuggerd_callbacks_t callbacks = { - .get_abort_message = []() { + .get_process_info = []() { static struct { size_t size; char msg[32]; @@ -357,7 +357,9 @@ int main(int argc, char** argv) { msg.size = strlen("dummy abort message"); memcpy(msg.msg, "dummy abort message", strlen("dummy abort message")); - return reinterpret_cast<abort_msg_t*>(&msg); + return debugger_process_info{ + .abort_msg = reinterpret_cast<void*>(&msg), + }; }, .post_dump = nullptr }; diff --git a/debuggerd/crasher/mips/crashglue.S b/debuggerd/crasher/mips/crashglue.S deleted file mode 100644 index 70a664127..000000000 --- a/debuggerd/crasher/mips/crashglue.S +++ /dev/null @@ -1,48 +0,0 @@ - .set noat - - .globl crash1 - .globl crashnostack - -crash1: - li $0,0xdead0000+0 - li $1,0xdead0000+1 - li $2,0xdead0000+2 - li $3,0xdead0000+3 - li $4,0xdead0000+4 - li $5,0xdead0000+5 - li $6,0xdead0000+6 - li $7,0xdead0000+7 - li $8,0xdead0000+8 - li $9,0xdead0000+9 - li $10,0xdead0000+10 - li $11,0xdead0000+11 - li $12,0xdead0000+12 - li $13,0xdead0000+13 - li $14,0xdead0000+14 - li $15,0xdead0000+15 - li $16,0xdead0000+16 - li $17,0xdead0000+17 - li $18,0xdead0000+18 - li $19,0xdead0000+19 - li $20,0xdead0000+20 - li $21,0xdead0000+21 - li $22,0xdead0000+22 - li $23,0xdead0000+23 - li $24,0xdead0000+24 - li $25,0xdead0000+25 - li $26,0xdead0000+26 - li $27,0xdead0000+27 - li $28,0xdead0000+28 - # don't trash the stack otherwise the signal handler won't run - #li $29,0xdead0000+29 - li $30,0xdead0000+30 - li $31,0xdead0000+31 - - lw $zero,($0) - b . - - -crashnostack: - li $sp, 0 - lw $zero,($0) - b . diff --git a/debuggerd/crasher/mips64/crashglue.S b/debuggerd/crasher/mips64/crashglue.S deleted file mode 100644 index 70a664127..000000000 --- a/debuggerd/crasher/mips64/crashglue.S +++ /dev/null @@ -1,48 +0,0 @@ - .set noat - - .globl crash1 - .globl crashnostack - -crash1: - li $0,0xdead0000+0 - li $1,0xdead0000+1 - li $2,0xdead0000+2 - li $3,0xdead0000+3 - li $4,0xdead0000+4 - li $5,0xdead0000+5 - li $6,0xdead0000+6 - li $7,0xdead0000+7 - li $8,0xdead0000+8 - li $9,0xdead0000+9 - li $10,0xdead0000+10 - li $11,0xdead0000+11 - li $12,0xdead0000+12 - li $13,0xdead0000+13 - li $14,0xdead0000+14 - li $15,0xdead0000+15 - li $16,0xdead0000+16 - li $17,0xdead0000+17 - li $18,0xdead0000+18 - li $19,0xdead0000+19 - li $20,0xdead0000+20 - li $21,0xdead0000+21 - li $22,0xdead0000+22 - li $23,0xdead0000+23 - li $24,0xdead0000+24 - li $25,0xdead0000+25 - li $26,0xdead0000+26 - li $27,0xdead0000+27 - li $28,0xdead0000+28 - # don't trash the stack otherwise the signal handler won't run - #li $29,0xdead0000+29 - li $30,0xdead0000+30 - li $31,0xdead0000+31 - - lw $zero,($0) - b . - - -crashnostack: - li $sp, 0 - lw $zero,($0) - b . diff --git a/debuggerd/debuggerd_test.cpp b/debuggerd/debuggerd_test.cpp index 6a8cc563d..9d7658eb0 100644 --- a/debuggerd/debuggerd_test.cpp +++ b/debuggerd/debuggerd_test.cpp @@ -31,6 +31,9 @@ #include <android/fdsan.h> #include <android/set_abort_message.h> +#include <bionic/malloc.h> +#include <bionic/mte.h> +#include <bionic/mte_kernel.h> #include <bionic/reserved_signals.h> #include <android-base/cmsg.h> @@ -305,6 +308,210 @@ TEST_F(CrasherTest, smoke) { ASSERT_MATCH(result, R"(signal 11 \(SIGSEGV\), code 1 \(SEGV_MAPERR\), fault addr 0xdead)"); } +TEST_F(CrasherTest, tagged_fault_addr) { +#if !defined(__aarch64__) + GTEST_SKIP() << "Requires aarch64"; +#endif + int intercept_result; + unique_fd output_fd; + StartProcess([]() { + *reinterpret_cast<volatile char*>(0x100000000000dead) = '1'; + }); + + StartIntercept(&output_fd); + FinishCrasher(); + AssertDeath(SIGSEGV); + FinishIntercept(&intercept_result); + + ASSERT_EQ(1, intercept_result) << "tombstoned reported failure"; + + std::string result; + ConsumeFd(std::move(output_fd), &result); + + // The address can either be tagged (new kernels) or untagged (old kernels). + ASSERT_MATCH( + result, + R"(signal 11 \(SIGSEGV\), code 1 \(SEGV_MAPERR\), fault addr (0x100000000000dead|0xdead))"); +} + +#if defined(__aarch64__) && defined(ANDROID_EXPERIMENTAL_MTE) +static void SetTagCheckingLevelSync() { + int tagged_addr_ctrl = prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0); + if (tagged_addr_ctrl < 0) { + abort(); + } + + tagged_addr_ctrl = (tagged_addr_ctrl & ~PR_MTE_TCF_MASK) | PR_MTE_TCF_SYNC; + if (prctl(PR_SET_TAGGED_ADDR_CTRL, tagged_addr_ctrl, 0, 0, 0) != 0) { + abort(); + } + + HeapTaggingLevel heap_tagging_level = M_HEAP_TAGGING_LEVEL_SYNC; + if (!android_mallopt(M_SET_HEAP_TAGGING_LEVEL, &heap_tagging_level, sizeof(heap_tagging_level))) { + abort(); + } +} +#endif + +TEST_F(CrasherTest, mte_uaf) { +#if defined(__aarch64__) && defined(ANDROID_EXPERIMENTAL_MTE) + if (!mte_supported()) { + GTEST_SKIP() << "Requires MTE"; + } + + int intercept_result; + unique_fd output_fd; + StartProcess([]() { + SetTagCheckingLevelSync(); + volatile int* p = (volatile int*)malloc(16); + free((void *)p); + p[0] = 42; + }); + + StartIntercept(&output_fd); + FinishCrasher(); + AssertDeath(SIGSEGV); + FinishIntercept(&intercept_result); + + ASSERT_EQ(1, intercept_result) << "tombstoned reported failure"; + + std::string result; + ConsumeFd(std::move(output_fd), &result); + + ASSERT_MATCH(result, R"(signal 11 \(SIGSEGV\), code 9 \(SEGV_MTESERR\))"); + ASSERT_MATCH(result, R"(Cause: \[MTE\]: Use After Free, 0 bytes into a 16-byte allocation.* + +allocated by thread .* + #00 pc)"); + ASSERT_MATCH(result, R"(deallocated by thread .* + #00 pc)"); +#else + GTEST_SKIP() << "Requires aarch64 + ANDROID_EXPERIMENTAL_MTE"; +#endif +} + +TEST_F(CrasherTest, mte_overflow) { +#if defined(__aarch64__) && defined(ANDROID_EXPERIMENTAL_MTE) + if (!mte_supported()) { + GTEST_SKIP() << "Requires MTE"; + } + + int intercept_result; + unique_fd output_fd; + StartProcess([]() { + SetTagCheckingLevelSync(); + volatile int* p = (volatile int*)malloc(16); + p[4] = 42; + }); + + StartIntercept(&output_fd); + FinishCrasher(); + AssertDeath(SIGSEGV); + FinishIntercept(&intercept_result); + + ASSERT_EQ(1, intercept_result) << "tombstoned reported failure"; + + std::string result; + ConsumeFd(std::move(output_fd), &result); + + ASSERT_MATCH(result, R"(signal 11 \(SIGSEGV\))"); + ASSERT_MATCH(result, R"(Cause: \[MTE\]: Buffer Overflow, 0 bytes right of a 16-byte allocation.* + +allocated by thread .* + #00 pc)"); +#else + GTEST_SKIP() << "Requires aarch64 + ANDROID_EXPERIMENTAL_MTE"; +#endif +} + +TEST_F(CrasherTest, mte_underflow) { +#if defined(__aarch64__) && defined(ANDROID_EXPERIMENTAL_MTE) + if (!mte_supported()) { + GTEST_SKIP() << "Requires MTE"; + } + + int intercept_result; + unique_fd output_fd; + StartProcess([]() { + SetTagCheckingLevelSync(); + volatile int* p = (volatile int*)malloc(16); + p[-1] = 42; + }); + + StartIntercept(&output_fd); + FinishCrasher(); + AssertDeath(SIGSEGV); + FinishIntercept(&intercept_result); + + ASSERT_EQ(1, intercept_result) << "tombstoned reported failure"; + + std::string result; + ConsumeFd(std::move(output_fd), &result); + + ASSERT_MATCH(result, R"(signal 11 \(SIGSEGV\), code 9 \(SEGV_MTESERR\))"); + ASSERT_MATCH(result, R"(Cause: \[MTE\]: Buffer Underflow, 4 bytes left of a 16-byte allocation.* + +allocated by thread .* + #00 pc)"); +#else + GTEST_SKIP() << "Requires aarch64 + ANDROID_EXPERIMENTAL_MTE"; +#endif +} + +TEST_F(CrasherTest, mte_multiple_causes) { +#if defined(__aarch64__) && defined(ANDROID_EXPERIMENTAL_MTE) + if (!mte_supported()) { + GTEST_SKIP() << "Requires MTE"; + } + + int intercept_result; + unique_fd output_fd; + StartProcess([]() { + SetTagCheckingLevelSync(); + + // Make two allocations with the same tag and close to one another. Check for both properties + // with a bounds check -- this relies on the fact that only if the allocations have the same tag + // would they be measured as closer than 128 bytes to each other. Otherwise they would be about + // (some non-zero value << 56) apart. + // + // The out-of-bounds access will be considered either an overflow of one or an underflow of the + // other. + std::set<uintptr_t> allocs; + for (int i = 0; i != 4096; ++i) { + uintptr_t alloc = reinterpret_cast<uintptr_t>(malloc(16)); + auto it = allocs.insert(alloc).first; + if (it != allocs.begin() && *std::prev(it) + 128 > alloc) { + *reinterpret_cast<int*>(*std::prev(it) + 16) = 42; + } + if (std::next(it) != allocs.end() && alloc + 128 > *std::next(it)) { + *reinterpret_cast<int*>(alloc + 16) = 42; + } + } + }); + + StartIntercept(&output_fd); + FinishCrasher(); + AssertDeath(SIGSEGV); + FinishIntercept(&intercept_result); + + ASSERT_EQ(1, intercept_result) << "tombstoned reported failure"; + + std::string result; + ConsumeFd(std::move(output_fd), &result); + + ASSERT_MATCH(result, R"(signal 11 \(SIGSEGV\))"); + ASSERT_MATCH( + result, + R"(Note: multiple potential causes for this crash were detected, listing them in decreasing order of probability.)"); + + // Adjacent untracked allocations may cause us to see the wrong underflow here (or only + // overflows), so we can't match explicitly for an underflow message. + ASSERT_MATCH(result, R"(Cause: \[MTE\]: Buffer Overflow, 0 bytes right of a 16-byte allocation)"); +#else + GTEST_SKIP() << "Requires aarch64 + ANDROID_EXPERIMENTAL_MTE"; +#endif +} + TEST_F(CrasherTest, LD_PRELOAD) { int intercept_result; unique_fd output_fd; diff --git a/debuggerd/handler/debuggerd_handler.cpp b/debuggerd/handler/debuggerd_handler.cpp index 8b4b6304f..121a074d5 100644 --- a/debuggerd/handler/debuggerd_handler.cpp +++ b/debuggerd/handler/debuggerd_handler.cpp @@ -83,7 +83,7 @@ using unique_fd = android::base::unique_fd_impl<FdsanBypassCloser>; #define CRASH_DUMP_NAME "crash_dump32" #endif -#define CRASH_DUMP_PATH "/system/bin/" CRASH_DUMP_NAME +#define CRASH_DUMP_PATH "/apex/com.android.runtime/bin/" CRASH_DUMP_NAME // Wrappers that directly invoke the respective syscalls, in case the cached values are invalid. #pragma GCC poison getpid gettid @@ -167,7 +167,7 @@ static bool get_main_thread_name(char* buf, size_t len) { * mutex is being held, so we don't want to use any libc functions that * could allocate memory or hold a lock. */ -static void log_signal_summary(const siginfo_t* info) { +static void log_signal_summary(const siginfo_t* info, const ucontext_t* ucontext) { char thread_name[MAX_TASK_NAME_LEN + 1]; // one more for termination if (prctl(PR_GET_NAME, reinterpret_cast<unsigned long>(thread_name), 0, 0, 0) != 0) { strcpy(thread_name, "<name unknown>"); @@ -186,7 +186,8 @@ static void log_signal_summary(const siginfo_t* info) { // Many signals don't have an address or sender. char addr_desc[32] = ""; // ", fault addr 0x1234" if (signal_has_si_addr(info)) { - async_safe_format_buffer(addr_desc, sizeof(addr_desc), ", fault addr %p", info->si_addr); + async_safe_format_buffer(addr_desc, sizeof(addr_desc), ", fault addr %p", + reinterpret_cast<void*>(get_fault_address(info, ucontext))); } pid_t self_pid = __getpid(); char sender_desc[32] = {}; // " from pid 1234, uid 666" @@ -297,10 +298,7 @@ struct debugger_thread_info { pid_t pseudothread_tid; siginfo_t* siginfo; void* ucontext; - uintptr_t abort_msg; - uintptr_t fdsan_table; - uintptr_t gwp_asan_state; - uintptr_t gwp_asan_metadata; + debugger_process_info process_info; }; // Logging and contacting debuggerd requires free file descriptors, which we might not have. @@ -344,25 +342,36 @@ static int debuggerd_dispatch_pseudothread(void* arg) { fatal_errno("failed to create pipe"); } + uint32_t version; + ssize_t expected; + // ucontext_t is absurdly large on AArch64, so piece it together manually with writev. - uint32_t version = 3; - constexpr size_t expected = sizeof(CrashInfoHeader) + sizeof(CrashInfoDataV3); + struct iovec iovs[4] = { + {.iov_base = &version, .iov_len = sizeof(version)}, + {.iov_base = thread_info->siginfo, .iov_len = sizeof(siginfo_t)}, + {.iov_base = thread_info->ucontext, .iov_len = sizeof(ucontext_t)}, + }; + + if (thread_info->process_info.fdsan_table) { + // Dynamic executables always use version 4. There is no need to increment the version number if + // the format changes, because the sender (linker) and receiver (crash_dump) are version locked. + version = 4; + expected = sizeof(CrashInfoHeader) + sizeof(CrashInfoDataDynamic); + + iovs[3] = {.iov_base = &thread_info->process_info, + .iov_len = sizeof(thread_info->process_info)}; + } else { + // Static executables always use version 1. + version = 1; + expected = sizeof(CrashInfoHeader) + sizeof(CrashInfoDataStatic); + iovs[3] = {.iov_base = &thread_info->process_info.abort_msg, .iov_len = sizeof(uintptr_t)}; + } errno = 0; if (fcntl(output_write.get(), F_SETPIPE_SZ, expected) < static_cast<int>(expected)) { fatal_errno("failed to set pipe buffer size"); } - struct iovec iovs[] = { - {.iov_base = &version, .iov_len = sizeof(version)}, - {.iov_base = thread_info->siginfo, .iov_len = sizeof(siginfo_t)}, - {.iov_base = thread_info->ucontext, .iov_len = sizeof(ucontext_t)}, - {.iov_base = &thread_info->abort_msg, .iov_len = sizeof(uintptr_t)}, - {.iov_base = &thread_info->fdsan_table, .iov_len = sizeof(uintptr_t)}, - {.iov_base = &thread_info->gwp_asan_state, .iov_len = sizeof(uintptr_t)}, - {.iov_base = &thread_info->gwp_asan_metadata, .iov_len = sizeof(uintptr_t)}, - }; - ssize_t rc = TEMP_FAILURE_RETRY(writev(output_write.get(), iovs, arraysize(iovs))); if (rc == -1) { fatal_errno("failed to write crash info"); @@ -408,23 +417,28 @@ static int debuggerd_dispatch_pseudothread(void* arg) { // us to fork off a process to read memory from. char buf[4]; rc = TEMP_FAILURE_RETRY(read(input_read.get(), &buf, sizeof(buf))); - if (rc == -1) { - async_safe_format_log(ANDROID_LOG_FATAL, "libc", "read of IPC pipe failed: %s", strerror(errno)); - return 1; - } else if (rc == 0) { - async_safe_format_log(ANDROID_LOG_FATAL, "libc", "crash_dump helper failed to exec"); - return 1; - } else if (rc != 1) { - async_safe_format_log(ANDROID_LOG_FATAL, "libc", - "read of IPC pipe returned unexpected value: %zd", rc); - return 1; - } else if (buf[0] != '\1') { - async_safe_format_log(ANDROID_LOG_FATAL, "libc", "crash_dump helper reported failure"); - return 1; - } - // crash_dump is ptracing us, fork off a copy of our address space for it to use. - create_vm_process(); + bool success = false; + if (rc == 1 && buf[0] == '\1') { + // crash_dump successfully started, and is ptracing us. + // Fork off a copy of our address space for it to use. + create_vm_process(); + success = true; + } else { + // Something went wrong, log it. + if (rc == -1) { + async_safe_format_log(ANDROID_LOG_FATAL, "libc", "read of IPC pipe failed: %s", + strerror(errno)); + } else if (rc == 0) { + async_safe_format_log(ANDROID_LOG_FATAL, "libc", + "crash_dump helper failed to exec, or was killed"); + } else if (rc != 1) { + async_safe_format_log(ANDROID_LOG_FATAL, "libc", + "read of IPC pipe returned unexpected value: %zd", rc); + } else if (buf[0] != '\1') { + async_safe_format_log(ANDROID_LOG_FATAL, "libc", "crash_dump helper reported failure"); + } + } // Don't leave a zombie child. int status; @@ -435,14 +449,16 @@ static int debuggerd_dispatch_pseudothread(void* arg) { async_safe_format_log(ANDROID_LOG_FATAL, "libc", "crash_dump helper crashed or stopped"); } - if (thread_info->siginfo->si_signo != BIONIC_SIGNAL_DEBUGGER) { - // For crashes, we don't need to minimize pause latency. - // Wait for the dump to complete before having the process exit, to avoid being murdered by - // ActivityManager or init. - TEMP_FAILURE_RETRY(read(input_read, &buf, sizeof(buf))); + if (success) { + if (thread_info->siginfo->si_signo != BIONIC_SIGNAL_DEBUGGER) { + // For crashes, we don't need to minimize pause latency. + // Wait for the dump to complete before having the process exit, to avoid being murdered by + // ActivityManager or init. + TEMP_FAILURE_RETRY(read(input_read, &buf, sizeof(buf))); + } } - return 0; + return success ? 0 : 1; } static void resend_signal(siginfo_t* info) { @@ -468,6 +484,8 @@ static void debuggerd_signal_handler(int signal_number, siginfo_t* info, void* c // making a syscall and checking errno. ErrnoRestorer restorer; + auto *ucontext = static_cast<ucontext_t*>(context); + // It's possible somebody cleared the SA_SIGINFO flag, which would mean // our "info" arg holds an undefined value. if (!have_siginfo(signal_number)) { @@ -489,29 +507,19 @@ static void debuggerd_signal_handler(int signal_number, siginfo_t* info, void* c // check to allow all si_code values in calls coming from inside the house. } - void* abort_message = nullptr; - const gwp_asan::AllocatorState* gwp_asan_state = nullptr; - const gwp_asan::AllocationMetadata* gwp_asan_metadata = nullptr; + debugger_process_info process_info = {}; uintptr_t si_val = reinterpret_cast<uintptr_t>(info->si_ptr); if (signal_number == BIONIC_SIGNAL_DEBUGGER) { if (info->si_code == SI_QUEUE && info->si_pid == __getpid()) { // Allow for the abort message to be explicitly specified via the sigqueue value. // Keep the bottom bit intact for representing whether we want a backtrace or a tombstone. if (si_val != kDebuggerdFallbackSivalUintptrRequestDump) { - abort_message = reinterpret_cast<void*>(si_val & ~1); + process_info.abort_msg = reinterpret_cast<void*>(si_val & ~1); info->si_ptr = reinterpret_cast<void*>(si_val & 1); } } - } else { - if (g_callbacks.get_abort_message) { - abort_message = g_callbacks.get_abort_message(); - } - if (g_callbacks.get_gwp_asan_state) { - gwp_asan_state = g_callbacks.get_gwp_asan_state(); - } - if (g_callbacks.get_gwp_asan_metadata) { - gwp_asan_metadata = g_callbacks.get_gwp_asan_metadata(); - } + } else if (g_callbacks.get_process_info) { + process_info = g_callbacks.get_process_info(); } // If sival_int is ~0, it means that the fallback handler has been called @@ -524,7 +532,7 @@ static void debuggerd_signal_handler(int signal_number, siginfo_t* info, void* c // This check might be racy if another thread sets NO_NEW_PRIVS, but this should be unlikely, // you can only set NO_NEW_PRIVS to 1, and the effect should be at worst a single missing // ANR trace. - debuggerd_fallback_handler(info, static_cast<ucontext_t*>(context), abort_message); + debuggerd_fallback_handler(info, ucontext, process_info.abort_msg); resend_signal(info); return; } @@ -536,17 +544,14 @@ static void debuggerd_signal_handler(int signal_number, siginfo_t* info, void* c return; } - log_signal_summary(info); + log_signal_summary(info, ucontext); debugger_thread_info thread_info = { .crashing_tid = __gettid(), .pseudothread_tid = -1, .siginfo = info, .ucontext = context, - .abort_msg = reinterpret_cast<uintptr_t>(abort_message), - .fdsan_table = reinterpret_cast<uintptr_t>(android_fdsan_get_fd_table()), - .gwp_asan_state = reinterpret_cast<uintptr_t>(gwp_asan_state), - .gwp_asan_metadata = reinterpret_cast<uintptr_t>(gwp_asan_metadata), + .process_info = process_info, }; // Set PR_SET_DUMPABLE to 1, so that crash_dump can ptrace us. diff --git a/debuggerd/include/debuggerd/handler.h b/debuggerd/include/debuggerd/handler.h index 665d24a7f..254ed4f7b 100644 --- a/debuggerd/include/debuggerd/handler.h +++ b/debuggerd/include/debuggerd/handler.h @@ -33,13 +33,22 @@ struct AllocatorState; struct AllocationMetadata; }; // namespace gwp_asan +// When updating this data structure, CrashInfoDataDynamic and the code in +// ReadCrashInfo() must also be updated. +struct debugger_process_info { + void* abort_msg; + void* fdsan_table; + const gwp_asan::AllocatorState* gwp_asan_state; + const gwp_asan::AllocationMetadata* gwp_asan_metadata; + const char* scudo_stack_depot; + const char* scudo_region_info; +}; + // These callbacks are called in a signal handler, and thus must be async signal safe. // If null, the callbacks will not be called. typedef struct { - struct abort_msg_t* (*get_abort_message)(); + debugger_process_info (*get_process_info)(); void (*post_dump)(); - const struct gwp_asan::AllocatorState* (*get_gwp_asan_state)(); - const struct gwp_asan::AllocationMetadata* (*get_gwp_asan_metadata)(); } debuggerd_callbacks_t; void debuggerd_init(debuggerd_callbacks_t* callbacks); diff --git a/debuggerd/libdebuggerd/gwp_asan.cpp b/debuggerd/libdebuggerd/gwp_asan.cpp index 53df783dc..f27136558 100644 --- a/debuggerd/libdebuggerd/gwp_asan.cpp +++ b/debuggerd/libdebuggerd/gwp_asan.cpp @@ -63,12 +63,11 @@ static const gwp_asan::AllocationMetadata* retrieve_gwp_asan_metadata( } GwpAsanCrashData::GwpAsanCrashData(unwindstack::Memory* process_memory, - uintptr_t gwp_asan_state_ptr, uintptr_t gwp_asan_metadata_ptr, - const ThreadInfo& thread_info) { - if (!process_memory || !gwp_asan_metadata_ptr || !gwp_asan_state_ptr) return; + const ProcessInfo& process_info, const ThreadInfo& thread_info) { + if (!process_memory || !process_info.gwp_asan_metadata || !process_info.gwp_asan_state) return; // Extract the GWP-ASan regions from the dead process. - if (!retrieve_gwp_asan_state(process_memory, gwp_asan_state_ptr, &state_)) return; - metadata_.reset(retrieve_gwp_asan_metadata(process_memory, state_, gwp_asan_metadata_ptr)); + if (!retrieve_gwp_asan_state(process_memory, process_info.gwp_asan_state, &state_)) return; + metadata_.reset(retrieve_gwp_asan_metadata(process_memory, state_, process_info.gwp_asan_metadata)); if (!metadata_.get()) return; // Get the external crash address from the thread info. @@ -158,63 +157,6 @@ void GwpAsanCrashData::DumpCause(log_t* log) const { error_string_, diff, byte_suffix, location_str, alloc_size, alloc_address); } -// Build a frame for symbolization using the maps from the provided unwinder. -// The constructed frame contains just enough information to be used to -// symbolize a GWP-ASan stack trace. -static unwindstack::FrameData BuildFrame(unwindstack::Unwinder* unwinder, uintptr_t pc, - size_t frame_num) { - unwindstack::FrameData frame; - frame.num = frame_num; - - unwindstack::Maps* maps = unwinder->GetMaps(); - unwindstack::MapInfo* map_info = maps->Find(pc); - if (!map_info) { - frame.rel_pc = pc; - return frame; - } - - unwindstack::Elf* elf = - map_info->GetElf(unwinder->GetProcessMemory(), unwindstack::Regs::CurrentArch()); - - uint64_t relative_pc = elf->GetRelPc(pc, map_info); - - // Create registers just to get PC adjustment. Doesn't matter what they point - // to. - unwindstack::Regs* regs = unwindstack::Regs::CreateFromLocal(); - uint64_t pc_adjustment = regs->GetPcAdjustment(relative_pc, elf); - relative_pc -= pc_adjustment; - // The debug PC may be different if the PC comes from the JIT. - uint64_t debug_pc = relative_pc; - - // If we don't have a valid ELF file, check the JIT. - if (!elf->valid()) { - unwindstack::JitDebug jit_debug(unwinder->GetProcessMemory()); - uint64_t jit_pc = pc - pc_adjustment; - unwindstack::Elf* jit_elf = jit_debug.GetElf(maps, jit_pc); - if (jit_elf != nullptr) { - debug_pc = jit_pc; - elf = jit_elf; - } - } - - // Copy all the things we need into the frame for symbolization. - frame.rel_pc = relative_pc; - frame.pc = pc - pc_adjustment; - frame.map_name = map_info->name; - frame.map_elf_start_offset = map_info->elf_start_offset; - frame.map_exact_offset = map_info->offset; - frame.map_start = map_info->start; - frame.map_end = map_info->end; - frame.map_flags = map_info->flags; - frame.map_load_bias = elf->GetLoadBias(); - - if (!elf->GetFunctionName(relative_pc, &frame.function_name, &frame.function_offset)) { - frame.function_name = ""; - frame.function_offset = 0; - } - return frame; -} - constexpr size_t kMaxTraceLength = gwp_asan::AllocationMetadata::kMaxTraceLengthToCollect; bool GwpAsanCrashData::HasDeallocationTrace() const { @@ -241,7 +183,8 @@ void GwpAsanCrashData::DumpDeallocationTrace(log_t* log, unwindstack::Unwinder* unwinder->SetDisplayBuildID(true); for (size_t i = 0; i < num_frames; ++i) { - unwindstack::FrameData frame_data = BuildFrame(unwinder, frames.get()[i], i); + unwindstack::FrameData frame_data = unwinder->BuildFrameFromPcOnly(frames.get()[i]); + frame_data.num = i; _LOG(log, logtype::BACKTRACE, " %s\n", unwinder->FormatFrame(frame_data).c_str()); } } @@ -267,7 +210,8 @@ void GwpAsanCrashData::DumpAllocationTrace(log_t* log, unwindstack::Unwinder* un unwinder->SetDisplayBuildID(true); for (size_t i = 0; i < num_frames; ++i) { - unwindstack::FrameData frame_data = BuildFrame(unwinder, frames.get()[i], i); + unwindstack::FrameData frame_data = unwinder->BuildFrameFromPcOnly(frames.get()[i]); + frame_data.num = i; _LOG(log, logtype::BACKTRACE, " %s\n", unwinder->FormatFrame(frame_data).c_str()); } } diff --git a/debuggerd/libdebuggerd/include/libdebuggerd/gwp_asan.h b/debuggerd/libdebuggerd/include/libdebuggerd/gwp_asan.h index aef4c62aa..6c8873341 100644 --- a/debuggerd/libdebuggerd/include/libdebuggerd/gwp_asan.h +++ b/debuggerd/libdebuggerd/include/libdebuggerd/gwp_asan.h @@ -38,8 +38,8 @@ class GwpAsanCrashData { // still be responsible, as it terminates when it detects an internal error // (double free, invalid free). In these cases, we will retrieve the fault // address from the GWP-ASan allocator's state. - GwpAsanCrashData(unwindstack::Memory* process_memory, uintptr_t gwp_asan_state_ptr, - uintptr_t gwp_asan_metadata_ptr, const ThreadInfo& thread_info); + GwpAsanCrashData(unwindstack::Memory* process_memory, const ProcessInfo& process_info, + const ThreadInfo& thread_info); // Is GWP-ASan responsible for this crash. bool CrashIsMine() const; diff --git a/debuggerd/libdebuggerd/include/libdebuggerd/scudo.h b/debuggerd/libdebuggerd/include/libdebuggerd/scudo.h new file mode 100644 index 000000000..4d00ecede --- /dev/null +++ b/debuggerd/libdebuggerd/include/libdebuggerd/scudo.h @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "types.h" +#include "utility.h" + +#include <memory.h> + +#include "scudo/interface.h" + +class ScudoCrashData { + public: + ScudoCrashData() = delete; + ~ScudoCrashData() = default; + ScudoCrashData(unwindstack::Memory* process_memory, const ProcessInfo& process_info); + + bool CrashIsMine() const; + + void DumpCause(log_t* log, unwindstack::Unwinder* unwinder) const; + + private: + scudo_error_info error_info_ = {}; + uintptr_t untagged_fault_addr_; + + void DumpReport(const scudo_error_report* report, log_t* log, + unwindstack::Unwinder* unwinder) const; +}; diff --git a/debuggerd/libdebuggerd/include/libdebuggerd/tombstone.h b/debuggerd/libdebuggerd/include/libdebuggerd/tombstone.h index 291d994b3..3ff7d629d 100644 --- a/debuggerd/libdebuggerd/include/libdebuggerd/tombstone.h +++ b/debuggerd/libdebuggerd/include/libdebuggerd/tombstone.h @@ -44,18 +44,13 @@ constexpr size_t kMaxFrames = 256; int open_tombstone(std::string* path); /* Creates a tombstone file and writes the crash dump to it. */ -void engrave_tombstone(int tombstone_fd, unwindstack::Unwinder* unwinder, - const OpenFilesList* open_files, pid_t pid, pid_t tid, - const std::string& process_name, const std::map<pid_t, std::string>& threads, - uint64_t abort_msg_address, std::string* amfd_data); +void engrave_tombstone(android::base::unique_fd output_fd, unwindstack::Unwinder* unwinder, + const std::map<pid_t, ThreadInfo>& thread_info, pid_t target_thread, + const ProcessInfo& process_info, OpenFilesList* open_files, + std::string* amfd_data); void engrave_tombstone_ucontext(int tombstone_fd, uint64_t abort_msg_address, siginfo_t* siginfo, ucontext_t* ucontext); -void engrave_tombstone(android::base::unique_fd output_fd, unwindstack::Unwinder* unwinder, - const std::map<pid_t, ThreadInfo>& thread_info, pid_t target_thread, - uint64_t abort_msg_address, OpenFilesList* open_files, - std::string* amfd_data, uintptr_t gwp_asan_state, - uintptr_t gwp_asan_metadata); #endif // _DEBUGGERD_TOMBSTONE_H diff --git a/debuggerd/libdebuggerd/include/libdebuggerd/types.h b/debuggerd/libdebuggerd/include/libdebuggerd/types.h index eb4b1b844..04c4b5c58 100644 --- a/debuggerd/libdebuggerd/include/libdebuggerd/types.h +++ b/debuggerd/libdebuggerd/include/libdebuggerd/types.h @@ -35,3 +35,15 @@ struct ThreadInfo { int signo = 0; siginfo_t* siginfo = nullptr; }; + +struct ProcessInfo { + uintptr_t abort_msg_address = 0; + uintptr_t fdsan_table_address = 0; + uintptr_t gwp_asan_state = 0; + uintptr_t gwp_asan_metadata = 0; + uintptr_t scudo_stack_depot = 0; + uintptr_t scudo_region_info = 0; + + bool has_fault_address = false; + uintptr_t fault_address = 0; +}; diff --git a/debuggerd/libdebuggerd/include/libdebuggerd/utility.h b/debuggerd/libdebuggerd/include/libdebuggerd/utility.h index 75bac87d3..7bfcf5d6e 100644 --- a/debuggerd/libdebuggerd/include/libdebuggerd/utility.h +++ b/debuggerd/libdebuggerd/include/libdebuggerd/utility.h @@ -93,4 +93,6 @@ void get_signal_sender(char* buf, size_t n, const siginfo_t*); const char* get_signame(const siginfo_t*); const char* get_sigcode(const siginfo_t*); +uintptr_t get_fault_address(const siginfo_t* siginfo, const ucontext_t* ucontext); + #endif // _DEBUGGERD_UTILITY_H diff --git a/debuggerd/libdebuggerd/scudo.cpp b/debuggerd/libdebuggerd/scudo.cpp new file mode 100644 index 000000000..f8bfe07ce --- /dev/null +++ b/debuggerd/libdebuggerd/scudo.cpp @@ -0,0 +1,159 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "libdebuggerd/scudo.h" +#include "libdebuggerd/gwp_asan.h" + +#include "unwindstack/Memory.h" +#include "unwindstack/Unwinder.h" + +#include <bionic/macros.h> + +std::unique_ptr<char[]> AllocAndReadFully(unwindstack::Memory* process_memory, uint64_t addr, + size_t size) { + auto buf = std::make_unique<char[]>(size); + if (!process_memory->ReadFully(addr, buf.get(), size)) { + return std::unique_ptr<char[]>(); + } + return buf; +} + +static const uintptr_t kTagGranuleSize = 16; + +ScudoCrashData::ScudoCrashData(unwindstack::Memory* process_memory, + const ProcessInfo& process_info) { + if (!process_info.has_fault_address) { + return; + } + + auto stack_depot = AllocAndReadFully(process_memory, process_info.scudo_stack_depot, + __scudo_get_stack_depot_size()); + auto region_info = AllocAndReadFully(process_memory, process_info.scudo_region_info, + __scudo_get_region_info_size()); + + untagged_fault_addr_ = untag_address(process_info.fault_address); + uintptr_t fault_page = untagged_fault_addr_ & ~(PAGE_SIZE - 1); + + uintptr_t memory_begin = fault_page - PAGE_SIZE * 16; + if (memory_begin > fault_page) { + return; + } + + uintptr_t memory_end = fault_page + PAGE_SIZE * 16; + if (memory_end < fault_page) { + return; + } + + auto memory = std::make_unique<char[]>(memory_end - memory_begin); + for (auto i = memory_begin; i != memory_end; i += PAGE_SIZE) { + process_memory->ReadFully(i, memory.get() + i - memory_begin, PAGE_SIZE); + } + + auto memory_tags = std::make_unique<char[]>((memory_end - memory_begin) / kTagGranuleSize); + for (auto i = memory_begin; i != memory_end; i += kTagGranuleSize) { + memory_tags[(i - memory_begin) / kTagGranuleSize] = process_memory->ReadTag(i); + } + + __scudo_get_error_info(&error_info_, process_info.fault_address, stack_depot.get(), + region_info.get(), memory.get(), memory_tags.get(), memory_begin, + memory_end - memory_begin); +} + +bool ScudoCrashData::CrashIsMine() const { + return error_info_.reports[0].error_type != UNKNOWN; +} + +void ScudoCrashData::DumpCause(log_t* log, unwindstack::Unwinder* unwinder) const { + if (error_info_.reports[1].error_type != UNKNOWN) { + _LOG(log, logtype::HEADER, + "\nNote: multiple potential causes for this crash were detected, listing them in " + "decreasing order of probability.\n"); + } + + size_t report_num = 0; + while (report_num < sizeof(error_info_.reports) / sizeof(error_info_.reports[0]) && + error_info_.reports[report_num].error_type != UNKNOWN) { + DumpReport(&error_info_.reports[report_num++], log, unwinder); + } +} + +void ScudoCrashData::DumpReport(const scudo_error_report* report, log_t* log, + unwindstack::Unwinder* unwinder) const { + const char *error_type_str; + switch (report->error_type) { + case USE_AFTER_FREE: + error_type_str = "Use After Free"; + break; + case BUFFER_OVERFLOW: + error_type_str = "Buffer Overflow"; + break; + case BUFFER_UNDERFLOW: + error_type_str = "Buffer Underflow"; + break; + default: + error_type_str = "Unknown"; + break; + } + + uintptr_t diff; + const char* location_str; + + if (untagged_fault_addr_ < report->allocation_address) { + // Buffer Underflow, 6 bytes left of a 41-byte allocation at 0xdeadbeef. + location_str = "left of"; + diff = report->allocation_address - untagged_fault_addr_; + } else if (untagged_fault_addr_ - report->allocation_address < report->allocation_size) { + // Use After Free, 40 bytes into a 41-byte allocation at 0xdeadbeef. + location_str = "into"; + diff = untagged_fault_addr_ - report->allocation_address; + } else { + // Buffer Overflow, 6 bytes right of a 41-byte allocation at 0xdeadbeef. + location_str = "right of"; + diff = untagged_fault_addr_ - report->allocation_address - report->allocation_size; + } + + // Suffix of 'bytes', i.e. 4 bytes' vs. '1 byte'. + const char* byte_suffix = "s"; + if (diff == 1) { + byte_suffix = ""; + } + _LOG(log, logtype::HEADER, + "\nCause: [MTE]: %s, %" PRIuPTR " byte%s %s a %zu-byte allocation at 0x%" PRIxPTR "\n", + error_type_str, diff, byte_suffix, location_str, report->allocation_size, + report->allocation_address); + + if (report->allocation_trace[0]) { + _LOG(log, logtype::BACKTRACE, "\nallocated by thread %u:\n", report->allocation_tid); + unwinder->SetDisplayBuildID(true); + for (size_t i = 0; i < 64 && report->allocation_trace[i]; ++i) { + unwindstack::FrameData frame_data = + unwinder->BuildFrameFromPcOnly(report->allocation_trace[i]); + frame_data.num = i; + _LOG(log, logtype::BACKTRACE, " %s\n", unwinder->FormatFrame(frame_data).c_str()); + } + } + + if (report->deallocation_trace[0]) { + _LOG(log, logtype::BACKTRACE, "\ndeallocated by thread %u:\n", report->deallocation_tid); + unwinder->SetDisplayBuildID(true); + for (size_t i = 0; i < 64 && report->deallocation_trace[i]; ++i) { + unwindstack::FrameData frame_data = + unwinder->BuildFrameFromPcOnly(report->deallocation_trace[i]); + frame_data.num = i; + _LOG(log, logtype::BACKTRACE, " %s\n", unwinder->FormatFrame(frame_data).c_str()); + } + } +} diff --git a/debuggerd/libdebuggerd/test/tombstone_test.cpp b/debuggerd/libdebuggerd/test/tombstone_test.cpp index eed95bc39..aec8c6030 100644 --- a/debuggerd/libdebuggerd/test/tombstone_test.cpp +++ b/debuggerd/libdebuggerd/test/tombstone_test.cpp @@ -371,7 +371,7 @@ public: GwpAsanCrashDataTest( gwp_asan::Error error, const gwp_asan::AllocationMetadata *responsible_allocation) : - GwpAsanCrashData(nullptr, 0u, 0u, ThreadInfo{}) { + GwpAsanCrashData(nullptr, ProcessInfo{}, ThreadInfo{}) { is_gwp_asan_responsible_ = true; error_ = error; responsible_allocation_ = responsible_allocation; diff --git a/debuggerd/libdebuggerd/tombstone.cpp b/debuggerd/libdebuggerd/tombstone.cpp index fd52e8113..ab65dd15a 100644 --- a/debuggerd/libdebuggerd/tombstone.cpp +++ b/debuggerd/libdebuggerd/tombstone.cpp @@ -43,6 +43,7 @@ #include <android-base/unique_fd.h> #include <android/log.h> #include <log/log.h> +#include <log/log_read.h> #include <log/logprint.h> #include <private/android_filesystem_config.h> #include <unwindstack/DexFiles.h> @@ -55,6 +56,7 @@ #include "libdebuggerd/backtrace.h" #include "libdebuggerd/gwp_asan.h" #include "libdebuggerd/open_files_list.h" +#include "libdebuggerd/scudo.h" #include "libdebuggerd/utility.h" #include "gwp_asan/common.h" @@ -154,16 +156,16 @@ static void dump_probable_cause(log_t* log, const siginfo_t* si, unwindstack::Ma } static void dump_signal_info(log_t* log, const ThreadInfo& thread_info, - unwindstack::Memory* process_memory) { + const ProcessInfo& process_info, unwindstack::Memory* process_memory) { char addr_desc[64]; // ", fault addr 0x1234" - if (signal_has_si_addr(thread_info.siginfo)) { - void* addr = thread_info.siginfo->si_addr; + if (process_info.has_fault_address) { + size_t addr = process_info.fault_address; if (thread_info.siginfo->si_signo == SIGILL) { uint32_t instruction = {}; - process_memory->Read(reinterpret_cast<uint64_t>(addr), &instruction, sizeof(instruction)); - snprintf(addr_desc, sizeof(addr_desc), "%p (*pc=%#08x)", addr, instruction); + process_memory->Read(addr, &instruction, sizeof(instruction)); + snprintf(addr_desc, sizeof(addr_desc), "0x%zx (*pc=%#08x)", addr, instruction); } else { - snprintf(addr_desc, sizeof(addr_desc), "%p", addr); + snprintf(addr_desc, sizeof(addr_desc), "0x%zx", addr); } } else { snprintf(addr_desc, sizeof(addr_desc), "--------"); @@ -376,8 +378,7 @@ void dump_memory_and_code(log_t* log, unwindstack::Maps* maps, unwindstack::Memo } static bool dump_thread(log_t* log, unwindstack::Unwinder* unwinder, const ThreadInfo& thread_info, - uint64_t abort_msg_address, bool primary_thread, - const GwpAsanCrashData& gwp_asan_crash_data) { + const ProcessInfo& process_info, bool primary_thread) { log->current_tid = thread_info.tid; if (!primary_thread) { _LOG(log, logtype::THREAD, "--- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---\n"); @@ -385,18 +386,27 @@ static bool dump_thread(log_t* log, unwindstack::Unwinder* unwinder, const Threa dump_thread_info(log, thread_info); if (thread_info.siginfo) { - dump_signal_info(log, thread_info, unwinder->GetProcessMemory().get()); + dump_signal_info(log, thread_info, process_info, unwinder->GetProcessMemory().get()); } - if (primary_thread && gwp_asan_crash_data.CrashIsMine()) { - gwp_asan_crash_data.DumpCause(log); - } else if (thread_info.siginfo) { + std::unique_ptr<GwpAsanCrashData> gwp_asan_crash_data; + std::unique_ptr<ScudoCrashData> scudo_crash_data; + if (primary_thread) { + gwp_asan_crash_data = std::make_unique<GwpAsanCrashData>(unwinder->GetProcessMemory().get(), + process_info, thread_info); + scudo_crash_data = + std::make_unique<ScudoCrashData>(unwinder->GetProcessMemory().get(), process_info); + } + + if (primary_thread && gwp_asan_crash_data->CrashIsMine()) { + gwp_asan_crash_data->DumpCause(log); + } else if (thread_info.siginfo && !(primary_thread && scudo_crash_data->CrashIsMine())) { dump_probable_cause(log, thread_info.siginfo, unwinder->GetMaps(), thread_info.registers.get()); } if (primary_thread) { - dump_abort_message(log, unwinder->GetProcessMemory().get(), abort_msg_address); + dump_abort_message(log, unwinder->GetProcessMemory().get(), process_info.abort_msg_address); } dump_registers(log, thread_info.registers.get()); @@ -413,14 +423,16 @@ static bool dump_thread(log_t* log, unwindstack::Unwinder* unwinder, const Threa } if (primary_thread) { - if (gwp_asan_crash_data.HasDeallocationTrace()) { - gwp_asan_crash_data.DumpDeallocationTrace(log, unwinder); + if (gwp_asan_crash_data->HasDeallocationTrace()) { + gwp_asan_crash_data->DumpDeallocationTrace(log, unwinder); } - if (gwp_asan_crash_data.HasAllocationTrace()) { - gwp_asan_crash_data.DumpAllocationTrace(log, unwinder); + if (gwp_asan_crash_data->HasAllocationTrace()) { + gwp_asan_crash_data->DumpAllocationTrace(log, unwinder); } + scudo_crash_data->DumpCause(log, unwinder); + unwindstack::Maps* maps = unwinder->GetMaps(); dump_memory_and_code(log, maps, unwinder->GetProcessMemory().get(), thread_info.registers.get()); @@ -442,8 +454,6 @@ static bool dump_thread(log_t* log, unwindstack::Unwinder* unwinder, const Threa // that don't match the specified pid, and writes them to the tombstone file. // // If "tail" is non-zero, log the last "tail" number of lines. -static EventTagMap* g_eventTagMap = NULL; - static void dump_log_file(log_t* log, pid_t pid, const char* filename, unsigned int tail) { bool first = true; logger_list* logger_list; @@ -452,8 +462,8 @@ static void dump_log_file(log_t* log, pid_t pid, const char* filename, unsigned return; } - logger_list = android_logger_list_open( - android_name_to_log_id(filename), ANDROID_LOG_RDONLY | ANDROID_LOG_NONBLOCK, tail, pid); + logger_list = + android_logger_list_open(android_name_to_log_id(filename), ANDROID_LOG_NONBLOCK, tail, pid); if (!logger_list) { ALOGE("Unable to open %s: %s\n", filename, strerror(errno)); @@ -502,21 +512,6 @@ static void dump_log_file(log_t* log, pid_t pid, const char* filename, unsigned ptm = localtime_r(&sec, &tmBuf); strftime(timeBuf, sizeof(timeBuf), "%m-%d %H:%M:%S", ptm); - if (log_entry.id() == LOG_ID_EVENTS) { - if (!g_eventTagMap) { - g_eventTagMap = android_openEventTagMap(nullptr); - } - AndroidLogEntry e; - char buf[512]; - if (android_log_processBinaryLogBuffer(&log_entry.entry, &e, g_eventTagMap, buf, - sizeof(buf)) == 0) { - _LOG(log, logtype::LOGS, "%s.%03d %5d %5d %c %-8.*s: %s\n", timeBuf, - log_entry.entry.nsec / 1000000, log_entry.entry.pid, log_entry.entry.tid, 'I', - (int)e.tagLen, e.tag, e.message); - } - continue; - } - char* msg = log_entry.msg(); if (msg == nullptr) { continue; @@ -601,15 +596,16 @@ void engrave_tombstone_ucontext(int tombstone_fd, uint64_t abort_msg_address, si LOG(FATAL) << "Failed to init unwinder object."; } - engrave_tombstone(unique_fd(dup(tombstone_fd)), &unwinder, threads, tid, abort_msg_address, - nullptr, nullptr, 0u, 0u); + ProcessInfo process_info; + process_info.abort_msg_address = abort_msg_address; + engrave_tombstone(unique_fd(dup(tombstone_fd)), &unwinder, threads, tid, process_info, nullptr, + nullptr); } void engrave_tombstone(unique_fd output_fd, unwindstack::Unwinder* unwinder, const std::map<pid_t, ThreadInfo>& threads, pid_t target_thread, - uint64_t abort_msg_address, OpenFilesList* open_files, - std::string* amfd_data, uintptr_t gwp_asan_state_ptr, - uintptr_t gwp_asan_metadata_ptr) { + const ProcessInfo& process_info, OpenFilesList* open_files, + std::string* amfd_data) { // don't copy log messages to tombstone unless this is a dev device bool want_logs = android::base::GetBoolProperty("ro.debuggable", false); @@ -628,12 +624,7 @@ void engrave_tombstone(unique_fd output_fd, unwindstack::Unwinder* unwinder, LOG(FATAL) << "failed to find target thread"; } - GwpAsanCrashData gwp_asan_crash_data(unwinder->GetProcessMemory().get(), - gwp_asan_state_ptr, - gwp_asan_metadata_ptr, it->second); - - dump_thread(&log, unwinder, it->second, abort_msg_address, true, - gwp_asan_crash_data); + dump_thread(&log, unwinder, it->second, process_info, true); if (want_logs) { dump_logs(&log, it->second.pid, 50); @@ -644,7 +635,7 @@ void engrave_tombstone(unique_fd output_fd, unwindstack::Unwinder* unwinder, continue; } - dump_thread(&log, unwinder, thread_info, 0, false, gwp_asan_crash_data); + dump_thread(&log, unwinder, thread_info, process_info, false); } if (open_files) { diff --git a/debuggerd/libdebuggerd/utility.cpp b/debuggerd/libdebuggerd/utility.cpp index 0a1d2a4da..c8a3431b7 100644 --- a/debuggerd/libdebuggerd/utility.cpp +++ b/debuggerd/libdebuggerd/utility.cpp @@ -35,6 +35,7 @@ #include <android-base/stringprintf.h> #include <android-base/strings.h> #include <android-base/unique_fd.h> +#include <bionic/mte_kernel.h> #include <bionic/reserved_signals.h> #include <debuggerd/handler.h> #include <log/log.h> @@ -374,6 +375,12 @@ const char* get_sigcode(const siginfo_t* si) { return "SEGV_ADIDERR"; case SEGV_ADIPERR: return "SEGV_ADIPERR"; +#if defined(ANDROID_EXPERIMENTAL_MTE) + case SEGV_MTEAERR: + return "SEGV_MTEAERR"; + case SEGV_MTESERR: + return "SEGV_MTESERR"; +#endif } static_assert(NSIGSEGV == SEGV_ADIPERR, "missing SEGV_* si_code"); break; @@ -449,3 +456,40 @@ void log_backtrace(log_t* log, unwindstack::Unwinder* unwinder, const char* pref _LOG(log, logtype::BACKTRACE, "%s%s\n", prefix, unwinder->FormatFrame(i).c_str()); } } + +#if defined(__aarch64__) +#define FAR_MAGIC 0x46415201 + +struct far_context { + struct _aarch64_ctx head; + __u64 far; +}; +#endif + +uintptr_t get_fault_address(const siginfo_t* siginfo, const ucontext_t* ucontext) { + (void)ucontext; +#if defined(__aarch64__) + // This relies on a kernel patch: + // https://patchwork.kernel.org/patch/11435077/ + // that hasn't been accepted into the kernel yet. TODO(pcc): Update this to + // use the official interface once it lands. + auto* begin = reinterpret_cast<const char*>(ucontext->uc_mcontext.__reserved); + auto* end = begin + sizeof(ucontext->uc_mcontext.__reserved); + auto* ptr = begin; + while (1) { + auto* ctx = reinterpret_cast<const _aarch64_ctx*>(ptr); + if (ctx->magic == 0) { + break; + } + if (ctx->magic == FAR_MAGIC) { + auto* far_ctx = reinterpret_cast<const far_context*>(ctx); + return far_ctx->far; + } + ptr += ctx->size; + if (ctx->size % sizeof(void*) != 0 || ptr < begin || ptr >= end) { + break; + } + } +#endif + return reinterpret_cast<uintptr_t>(siginfo->si_addr); +} diff --git a/debuggerd/protocol.h b/debuggerd/protocol.h index bf5386412..53a76ea71 100644 --- a/debuggerd/protocol.h +++ b/debuggerd/protocol.h @@ -85,26 +85,24 @@ struct __attribute__((__packed__)) CrashInfoHeader { uint32_t version; }; -struct __attribute__((__packed__)) CrashInfoDataV1 { +struct __attribute__((__packed__)) CrashInfoDataStatic { siginfo_t siginfo; ucontext_t ucontext; uintptr_t abort_msg_address; }; -struct __attribute__((__packed__)) CrashInfoDataV2 : public CrashInfoDataV1 { +struct __attribute__((__packed__)) CrashInfoDataDynamic : public CrashInfoDataStatic { uintptr_t fdsan_table_address; -}; - -struct __attribute__((__packed__)) CrashInfoDataV3 : public CrashInfoDataV2 { uintptr_t gwp_asan_state; uintptr_t gwp_asan_metadata; + uintptr_t scudo_stack_depot; + uintptr_t scudo_region_info; }; struct __attribute__((__packed__)) CrashInfo { CrashInfoHeader header; union { - CrashInfoDataV1 v1; - CrashInfoDataV2 v2; - CrashInfoDataV3 v3; + CrashInfoDataStatic s; + CrashInfoDataDynamic d; } data; }; diff --git a/fastboot/Android.bp b/fastboot/Android.bp index cf0f1ac93..bdb786c49 100644 --- a/fastboot/Android.bp +++ b/fastboot/Android.bp @@ -148,6 +148,7 @@ cc_binary { "libgtest_prod", "libhealthhalutils", "libsnapshot_nobinder", + "update_metadata-protos", ], header_libs: [ diff --git a/fastboot/device/flashing.cpp b/fastboot/device/flashing.cpp index a5f12231e..1bf4c9c9c 100644 --- a/fastboot/device/flashing.cpp +++ b/fastboot/device/flashing.cpp @@ -163,7 +163,9 @@ int Flash(FastbootDevice* device, const std::string& partition_name) { CopyAVBFooter(&data, block_device_size); } WipeOverlayfsForPartition(device, partition_name); - return FlashBlockDevice(handle.fd(), data); + int result = FlashBlockDevice(handle.fd(), data); + sync(); + return result; } bool UpdateSuper(FastbootDevice* device, const std::string& super_name, bool wipe) { @@ -193,6 +195,7 @@ bool UpdateSuper(FastbootDevice* device, const std::string& super_name, bool wip return device->WriteFail("Unable to flash new partition table"); } fs_mgr_overlayfs_teardown(); + sync(); return device->WriteOkay("Successfully flashed partition table"); } @@ -232,5 +235,6 @@ bool UpdateSuper(FastbootDevice* device, const std::string& super_name, bool wip return device->WriteFail("Unable to write new partition table"); } fs_mgr_overlayfs_teardown(); + sync(); return device->WriteOkay("Successfully updated partition table"); } diff --git a/fastboot/device/usb_client.cpp b/fastboot/device/usb_client.cpp index 9c8076528..c65316707 100644 --- a/fastboot/device/usb_client.cpp +++ b/fastboot/device/usb_client.cpp @@ -146,7 +146,7 @@ static struct SsFuncDesc ss_descriptors = { }, }; -#define STR_INTERFACE_ "fastboot" +#define STR_INTERFACE_ "fastbootd" static const struct { struct usb_functionfs_strings_head header; diff --git a/fastboot/fastboot.cpp b/fastboot/fastboot.cpp index 5307a0081..86cf30df2 100644 --- a/fastboot/fastboot.cpp +++ b/fastboot/fastboot.cpp @@ -200,8 +200,10 @@ static std::string find_item(const std::string& item) { double last_start_time; static void Status(const std::string& message) { - static constexpr char kStatusFormat[] = "%-50s "; - fprintf(stderr, kStatusFormat, message.c_str()); + if (!message.empty()) { + static constexpr char kStatusFormat[] = "%-50s "; + fprintf(stderr, kStatusFormat, message.c_str()); + } last_start_time = now(); } @@ -258,6 +260,10 @@ static int match_fastboot(usb_ifc_info* info) { static int list_devices_callback(usb_ifc_info* info) { if (match_fastboot_with_serial(info, nullptr) == 0) { std::string serial = info->serial_number; + std::string interface = info->interface; + if (interface.empty()) { + interface = "fastboot"; + } if (!info->writable) { serial = UsbNoPermissionsShortHelpText(); } @@ -266,9 +272,9 @@ static int list_devices_callback(usb_ifc_info* info) { } // output compatible with "adb devices" if (!g_long_listing) { - printf("%s\tfastboot", serial.c_str()); + printf("%s\t%s", serial.c_str(), interface.c_str()); } else { - printf("%-22s fastboot", serial.c_str()); + printf("%-22s %s", serial.c_str(), interface.c_str()); if (strlen(info->device_path) > 0) printf(" %s", info->device_path); } putchar('\n'); @@ -1284,7 +1290,7 @@ static void reboot_to_userspace_fastboot() { static void CancelSnapshotIfNeeded() { std::string merge_status = "none"; if (fb->GetVar(FB_VAR_SNAPSHOT_UPDATE_STATUS, &merge_status) == fastboot::SUCCESS && - merge_status != "none") { + !merge_status.empty() && merge_status != "none") { fb->SnapshotUpdateCommand("cancel"); } } diff --git a/fastboot/fuzzy_fastboot/fixtures.cpp b/fastboot/fuzzy_fastboot/fixtures.cpp index bd76ff4ee..9b5e5f7bd 100644 --- a/fastboot/fuzzy_fastboot/fixtures.cpp +++ b/fastboot/fuzzy_fastboot/fixtures.cpp @@ -45,6 +45,7 @@ #include <vector> #include <android-base/stringprintf.h> +#include <android-base/strings.h> #include <gtest/gtest.h> #include "fastboot_driver.h" @@ -76,8 +77,7 @@ int FastBootTest::MatchFastboot(usb_ifc_info* info, const std::string& local_ser } bool FastBootTest::IsFastbootOverTcp() { - // serial contains ":" is treated as host ip and port number - return (device_serial.find(":") != std::string::npos); + return android::base::StartsWith(device_serial, "tcp:"); } bool FastBootTest::UsbStillAvailible() { @@ -182,19 +182,14 @@ void FastBootTest::TearDownSerial() { } void FastBootTest::ConnectTcpFastbootDevice() { - std::size_t found = device_serial.find(":"); - if (found != std::string::npos) { - for (int i = 0; i < MAX_TCP_TRIES && !transport; i++) { - std::string error; - std::unique_ptr<Transport> tcp( - tcp::Connect(device_serial.substr(0, found), tcp::kDefaultPort, &error) - .release()); - if (tcp) - transport = - std::unique_ptr<TransportSniffer>(new TransportSniffer(std::move(tcp), 0)); - if (transport != nullptr) break; - std::this_thread::sleep_for(std::chrono::milliseconds(10)); - } + for (int i = 0; i < MAX_TCP_TRIES && !transport; i++) { + std::string error; + std::unique_ptr<Transport> tcp( + tcp::Connect(device_serial.substr(4), tcp::kDefaultPort, &error).release()); + if (tcp) + transport = std::unique_ptr<TransportSniffer>(new TransportSniffer(std::move(tcp), 0)); + if (transport != nullptr) break; + std::this_thread::sleep_for(std::chrono::milliseconds(10)); } } diff --git a/fastboot/usb.h b/fastboot/usb.h index 7ca44c414..e5f56e20a 100644 --- a/fastboot/usb.h +++ b/fastboot/usb.h @@ -50,6 +50,8 @@ struct usb_ifc_info { char serial_number[256]; char device_path[256]; + + char interface[256]; }; class UsbTransport : public Transport { diff --git a/fastboot/usb_linux.cpp b/fastboot/usb_linux.cpp index 6363aa547..964488cc8 100644 --- a/fastboot/usb_linux.cpp +++ b/fastboot/usb_linux.cpp @@ -43,6 +43,8 @@ #include <linux/version.h> #include <linux/usb/ch9.h> +#include <android-base/file.h> +#include <android-base/stringprintf.h> #include <chrono> #include <memory> #include <thread> @@ -263,6 +265,13 @@ static int filter_usb_device(char* sysfs_name, info.has_bulk_in = (in != -1); info.has_bulk_out = (out != -1); + std::string interface; + auto path = android::base::StringPrintf("/sys/bus/usb/devices/%s/%s:1.%d/interface", + sysfs_name, sysfs_name, ifc->bInterfaceNumber); + if (android::base::ReadFileToString(path, &interface)) { + snprintf(info.interface, sizeof(info.interface), "%s", interface.c_str()); + } + if(callback(&info) == 0) { *ept_in_id = in; *ept_out_id = out; diff --git a/fastboot/usb_osx.cpp b/fastboot/usb_osx.cpp index 8a3c21331..610eebfb2 100644 --- a/fastboot/usb_osx.cpp +++ b/fastboot/usb_osx.cpp @@ -368,6 +368,7 @@ static int try_device(io_service_t device, usb_handle *handle) { // device has no serial number handle->info.serial_number[0] = 0; } + handle->info.interface[0] = 0; handle->info.writable = 1; if (try_interfaces(dev, handle)) { diff --git a/fastboot/usb_windows.cpp b/fastboot/usb_windows.cpp index bf840f888..67bf8a30b 100644 --- a/fastboot/usb_windows.cpp +++ b/fastboot/usb_windows.cpp @@ -319,6 +319,7 @@ int recognized_device(usb_handle* handle, ifc_match_func callback) { &serial_number_len, true)) { info.serial_number[0] = 0; } + info.interface[0] = 0; info.device_path[0] = 0; diff --git a/fs_mgr/Android.bp b/fs_mgr/Android.bp index f5daf9174..cd6459989 100644 --- a/fs_mgr/Android.bp +++ b/fs_mgr/Android.bp @@ -162,10 +162,13 @@ cc_binary { defaults: ["fs_mgr_defaults"], static_libs: [ "libavb_user", + "libutils", + "libvold_binder", ], shared_libs: [ "libbootloader_message", "libbase", + "libbinder", "libcutils", "libcrypto", "libext4_utils", diff --git a/fs_mgr/README.overlayfs.md b/fs_mgr/README.overlayfs.md index f57907862..ca782b927 100644 --- a/fs_mgr/README.overlayfs.md +++ b/fs_mgr/README.overlayfs.md @@ -42,7 +42,7 @@ Then enter one of the following sequences: $ adb push <source> <destination> $ adb reboot -Note that you can replace these two lines: +Note that you can replace these two lines in the above sequence: $ adb disable-verity $ adb reboot @@ -51,7 +51,7 @@ with this line: $ adb remount -R -**Note:** _adb reboot -R_ won’t reboot if the device is already in the adb remount state. +**Note:** _adb remount -R_ won’t reboot if the device is already in the adb remount state. None of this changes if OverlayFS needs to be engaged. The decisions whether to use traditional direct file-system remount, diff --git a/fs_mgr/TEST_MAPPING b/fs_mgr/TEST_MAPPING index 676f446e7..6cd043013 100644 --- a/fs_mgr/TEST_MAPPING +++ b/fs_mgr/TEST_MAPPING @@ -14,6 +14,9 @@ }, { "name": "vts_libsnapshot_test" + }, + { + "name": "libsnapshot_fuzzer_test" } ] } diff --git a/fs_mgr/fs_mgr.cpp b/fs_mgr/fs_mgr.cpp index 956147198..10c1eba6f 100644 --- a/fs_mgr/fs_mgr.cpp +++ b/fs_mgr/fs_mgr.cpp @@ -609,9 +609,8 @@ static void tune_metadata_csum(const std::string& blk_device, const FstabEntry& // requires to give last_fsck_time to current to avoid insane time. // otherwise, tune2fs won't enable metadata_csum. - std::string now = std::to_string(time(0)); const char* tune2fs_args[] = {TUNE2FS_BIN, "-O", "metadata_csum,64bit,extent", - "-T", now.c_str(), blk_device.c_str()}; + "-T", "now", blk_device.c_str()}; const char* resize2fs_args[] = {RESIZE2FS_BIN, "-b", blk_device.c_str()}; if (!run_command(tune2fs_args, ARRAY_SIZE(tune2fs_args))) { diff --git a/fs_mgr/fs_mgr_format.cpp b/fs_mgr/fs_mgr_format.cpp index fd7386db3..301c90755 100644 --- a/fs_mgr/fs_mgr_format.cpp +++ b/fs_mgr/fs_mgr_format.cpp @@ -121,7 +121,7 @@ static int format_ext4(const std::string& fs_blkdev, const std::string& fs_mnt_p } static int format_f2fs(const std::string& fs_blkdev, uint64_t dev_sz, bool crypt_footer, - bool needs_projid, bool needs_casefold) { + bool needs_projid, bool needs_casefold, bool fs_compress) { if (!dev_sz) { int rc = get_dev_sz(fs_blkdev, &dev_sz); if (rc) { @@ -147,6 +147,12 @@ static int format_f2fs(const std::string& fs_blkdev, uint64_t dev_sz, bool crypt args.push_back("-C"); args.push_back("utf8"); } + if (fs_compress) { + args.push_back("-O"); + args.push_back("compression"); + args.push_back("-O"); + args.push_back("extra_attr"); + } args.push_back(fs_blkdev.c_str()); args.push_back(size_str.c_str()); @@ -166,7 +172,7 @@ int fs_mgr_do_format(const FstabEntry& entry, bool crypt_footer) { if (entry.fs_type == "f2fs") { return format_f2fs(entry.blk_device, entry.length, crypt_footer, needs_projid, - needs_casefold); + needs_casefold, entry.fs_mgr_flags.fs_compress); } else if (entry.fs_type == "ext4") { return format_ext4(entry.blk_device, entry.mount_point, crypt_footer, needs_projid, entry.fs_mgr_flags.ext_meta_csum); diff --git a/fs_mgr/fs_mgr_fstab.cpp b/fs_mgr/fs_mgr_fstab.cpp index fa4ac3904..0ae558355 100644 --- a/fs_mgr/fs_mgr_fstab.cpp +++ b/fs_mgr/fs_mgr_fstab.cpp @@ -178,6 +178,7 @@ void ParseFsMgrFlags(const std::string& flags, FstabEntry* entry) { CheckFlag("slotselect_other", slot_select_other); CheckFlag("fsverity", fs_verity); CheckFlag("metadata_csum", ext_meta_csum); + CheckFlag("fscompress", fs_compress); #undef CheckFlag @@ -825,6 +826,20 @@ std::set<std::string> GetBootDevices() { return std::set<std::string>(boot_devices.begin(), boot_devices.end()); } + std::string cmdline; + if (android::base::ReadFileToString("/proc/cmdline", &cmdline)) { + std::set<std::string> boot_devices; + const std::string cmdline_key = "androidboot.boot_device"; + for (const auto& [key, value] : fs_mgr_parse_boot_config(cmdline)) { + if (key == cmdline_key) { + boot_devices.emplace(value); + } + } + if (!boot_devices.empty()) { + return boot_devices; + } + } + // Fallback to extract boot devices from fstab. Fstab fstab; if (!ReadDefaultFstab(&fstab)) { diff --git a/fs_mgr/fs_mgr_remount.cpp b/fs_mgr/fs_mgr_remount.cpp index 24cbad7c9..b8b074e0f 100644 --- a/fs_mgr/fs_mgr_remount.cpp +++ b/fs_mgr/fs_mgr_remount.cpp @@ -24,6 +24,7 @@ #include <unistd.h> #include <string> +#include <thread> #include <utility> #include <vector> @@ -31,6 +32,8 @@ #include <android-base/logging.h> #include <android-base/properties.h> #include <android-base/strings.h> +#include <android/os/IVold.h> +#include <binder/IServiceManager.h> #include <bootloader_message/bootloader_message.h> #include <cutils/android_reboot.h> #include <fec/io.h> @@ -103,22 +106,42 @@ void MyLogger(android::base::LogId id, android::base::LogSeverity severity, cons ::exit(0); // SUCCESS } +static android::sp<android::os::IVold> GetVold() { + while (true) { + if (auto sm = android::defaultServiceManager()) { + if (auto binder = sm->getService(android::String16("vold"))) { + if (auto vold = android::interface_cast<android::os::IVold>(binder)) { + return vold; + } + } + } + std::this_thread::sleep_for(2s); + } +} + } // namespace +using namespace std::chrono_literals; + +enum RemountStatus { + REMOUNT_SUCCESS = 0, + NOT_USERDEBUG, + BADARG, + NOT_ROOT, + NO_FSTAB, + UNKNOWN_PARTITION, + INVALID_PARTITION, + VERITY_PARTITION, + BAD_OVERLAY, + NO_MOUNTS, + REMOUNT_FAILED, + MUST_REBOOT, + BINDER_ERROR, + CHECKPOINTING +}; + static int do_remount(int argc, char* argv[]) { - enum { - SUCCESS = 0, - NOT_USERDEBUG, - BADARG, - NOT_ROOT, - NO_FSTAB, - UNKNOWN_PARTITION, - INVALID_PARTITION, - VERITY_PARTITION, - BAD_OVERLAY, - NO_MOUNTS, - REMOUNT_FAILED, - } retval = SUCCESS; + RemountStatus retval = REMOUNT_SUCCESS; // If somehow this executable is delivered on a "user" build, it can // not function, so providing a clear message to the caller rather than @@ -191,6 +214,22 @@ static int do_remount(int argc, char* argv[]) { return NO_FSTAB; } + if (android::base::GetBoolProperty("ro.virtual_ab.enabled", false) && + !android::base::GetBoolProperty("ro.virtual_ab.retrofit", false)) { + // Virtual A/B devices can use /data as backing storage; make sure we're + // not checkpointing. + auto vold = GetVold(); + bool checkpointing = false; + if (!vold->isCheckpointing(&checkpointing).isOk()) { + LOG(ERROR) << "Could not determine checkpointing status."; + return BINDER_ERROR; + } + if (checkpointing) { + LOG(ERROR) << "Cannot use remount when a checkpoint is in progress."; + return CHECKPOINTING; + } + } + // Generate the list of supported overlayfs mount points. auto overlayfs_candidates = fs_mgr_overlayfs_candidate_list(fstab); @@ -304,8 +343,7 @@ static int do_remount(int argc, char* argv[]) { if (partitions.empty() || just_disabled_verity) { if (reboot_later) reboot(setup_overlayfs); if (user_please_reboot_later) { - LOG(INFO) << "Now reboot your device for settings to take effect"; - return 0; + return MUST_REBOOT; } LOG(WARNING) << "No partitions to remount"; return retval; @@ -394,6 +432,12 @@ static int do_remount(int argc, char* argv[]) { int main(int argc, char* argv[]) { android::base::InitLogging(argv, MyLogger); int result = do_remount(argc, argv); - printf("remount %s\n", result ? "failed" : "succeeded"); + if (result == MUST_REBOOT) { + LOG(INFO) << "Now reboot your device for settings to take effect"; + } else if (result == REMOUNT_SUCCESS) { + printf("remount succeeded\n"); + } else { + printf("remount failed\n"); + } return result; } diff --git a/fs_mgr/include_fstab/fstab/fstab.h b/fs_mgr/include_fstab/fstab/fstab.h index 7cf4f8997..40934450c 100644 --- a/fs_mgr/include_fstab/fstab/fstab.h +++ b/fs_mgr/include_fstab/fstab/fstab.h @@ -83,6 +83,7 @@ struct FstabEntry { bool slot_select_other : 1; bool fs_verity : 1; bool ext_meta_csum : 1; + bool fs_compress : 1; } fs_mgr_flags = {}; bool is_encryptable() const { diff --git a/fs_mgr/libfiemap/Android.bp b/fs_mgr/libfiemap/Android.bp index 976cd9033..cae43e6c2 100644 --- a/fs_mgr/libfiemap/Android.bp +++ b/fs_mgr/libfiemap/Android.bp @@ -45,6 +45,7 @@ cc_defaults { whole_static_libs: [ "gsi_aidl_interface-cpp", "libgsi", + "libgsid", ], shared_libs: [ "libbinder", @@ -90,6 +91,7 @@ cc_test { cc_test { name: "fiemap_image_test", static_libs: [ + "libcrypto_utils", "libdm", "libext4_utils", "libfs_mgr", @@ -98,7 +100,6 @@ cc_test { shared_libs: [ "libbase", "libcrypto", - "libcrypto_utils", "libcutils", "liblog", ], @@ -117,6 +118,7 @@ cc_test { "-DSKIP_TEST_IN_PRESUBMIT", ], static_libs: [ + "libcrypto_utils", "libdm", "libext4_utils", "libfs_mgr", @@ -125,7 +127,6 @@ cc_test { shared_libs: [ "libbase", "libcrypto", - "libcrypto_utils", "libcutils", "liblog", ], diff --git a/fs_mgr/libfiemap/binder.cpp b/fs_mgr/libfiemap/binder.cpp index 5e29d4e26..c8516ab2a 100644 --- a/fs_mgr/libfiemap/binder.cpp +++ b/fs_mgr/libfiemap/binder.cpp @@ -19,9 +19,9 @@ #include <android-base/properties.h> #include <android/gsi/BnProgressCallback.h> #include <android/gsi/IGsiService.h> -#include <binder/IServiceManager.h> #include <libfiemap/image_manager.h> #include <libgsi/libgsi.h> +#include <libgsi/libgsid.h> namespace android { namespace fiemap { @@ -224,19 +224,9 @@ bool ImageManagerBinder::MapAllImages(const std::function<bool(std::set<std::str return false; } -static sp<IGsiService> GetGsiService() { - auto sm = android::defaultServiceManager(); - auto name = android::String16(kGsiServiceName); - android::sp<android::IBinder> res = sm->waitForService(name); - if (res) { - return android::interface_cast<IGsiService>(res); - } - return nullptr; -} - std::unique_ptr<IImageManager> IImageManager::Open( const std::string& dir, const std::chrono::milliseconds& /*timeout_ms*/) { - android::sp<IGsiService> service = GetGsiService(); + android::sp<IGsiService> service = android::gsi::GetGsiService(); android::sp<IImageService> manager; auto status = service->openImageService(dir, &manager); diff --git a/fs_mgr/liblp/builder.cpp b/fs_mgr/liblp/builder.cpp index 2f516fa9b..c37d70e05 100644 --- a/fs_mgr/liblp/builder.cpp +++ b/fs_mgr/liblp/builder.cpp @@ -19,6 +19,7 @@ #include <string.h> #include <algorithm> +#include <limits> #include <android-base/unique_fd.h> @@ -369,7 +370,10 @@ bool MetadataBuilder::Init(const std::vector<BlockDeviceInfo>& block_devices, } // Align the metadata size up to the nearest sector. - metadata_max_size = AlignTo(metadata_max_size, LP_SECTOR_SIZE); + if (!AlignTo(metadata_max_size, LP_SECTOR_SIZE, &metadata_max_size)) { + LERROR << "Max metadata size " << metadata_max_size << " is too large."; + return false; + } // Validate and build the block device list. uint32_t logical_block_size = 0; @@ -401,10 +405,15 @@ bool MetadataBuilder::Init(const std::vector<BlockDeviceInfo>& block_devices, // untouched to be compatible code that looks for an MBR. Thus we // start counting free sectors at sector 1, not 0. uint64_t free_area_start = LP_SECTOR_SIZE; - if (out.alignment || out.alignment_offset) { - free_area_start = AlignTo(free_area_start, out.alignment, out.alignment_offset); + bool ok; + if (out.alignment) { + ok = AlignTo(free_area_start, out.alignment, &free_area_start); } else { - free_area_start = AlignTo(free_area_start, logical_block_size); + ok = AlignTo(free_area_start, logical_block_size, &free_area_start); + } + if (!ok) { + LERROR << "Integer overflow computing free area start"; + return false; } out.first_logical_sector = free_area_start / LP_SECTOR_SIZE; @@ -441,10 +450,15 @@ bool MetadataBuilder::Init(const std::vector<BlockDeviceInfo>& block_devices, // Compute the first free sector, factoring in alignment. uint64_t free_area_start = total_reserved; + bool ok; if (super.alignment || super.alignment_offset) { - free_area_start = AlignTo(free_area_start, super.alignment, super.alignment_offset); + ok = AlignTo(free_area_start, super.alignment, &free_area_start); } else { - free_area_start = AlignTo(free_area_start, logical_block_size); + ok = AlignTo(free_area_start, logical_block_size, &free_area_start); + } + if (!ok) { + LERROR << "Integer overflow computing free area start"; + return false; } super.first_logical_sector = free_area_start / LP_SECTOR_SIZE; @@ -544,7 +558,11 @@ void MetadataBuilder::ExtentsToFreeList(const std::vector<Interval>& extents, const Interval& current = extents[i]; DCHECK(previous.device_index == current.device_index); - uint64_t aligned = AlignSector(block_devices_[current.device_index], previous.end); + uint64_t aligned; + if (!AlignSector(block_devices_[current.device_index], previous.end, &aligned)) { + LERROR << "Sector " << previous.end << " caused integer overflow."; + continue; + } if (aligned >= current.start) { // There is no gap between these two extents, try the next one. // Note that we check with >= instead of >, since alignment may @@ -730,7 +748,10 @@ std::vector<Interval> MetadataBuilder::PrioritizeSecondHalfOfSuper( // Choose an aligned sector for the midpoint. This could lead to one half // being slightly larger than the other, but this will not restrict the // size of partitions (it might lead to one extra extent if "B" overflows). - midpoint = AlignSector(super, midpoint); + if (!AlignSector(super, midpoint, &midpoint)) { + LERROR << "Unexpected integer overflow aligning midpoint " << midpoint; + return free_list; + } std::vector<Interval> first_half; std::vector<Interval> second_half; @@ -768,7 +789,11 @@ std::unique_ptr<LinearExtent> MetadataBuilder::ExtendFinalExtent( // If the sector ends where the next aligned chunk begins, then there's // no missing gap to try and allocate. const auto& block_device = block_devices_[extent->device_index()]; - uint64_t next_aligned_sector = AlignSector(block_device, extent->end_sector()); + uint64_t next_aligned_sector; + if (!AlignSector(block_device, extent->end_sector(), &next_aligned_sector)) { + LERROR << "Integer overflow aligning sector " << extent->end_sector(); + return nullptr; + } if (extent->end_sector() == next_aligned_sector) { return nullptr; } @@ -925,13 +950,19 @@ uint64_t MetadataBuilder::UsedSpace() const { return size; } -uint64_t MetadataBuilder::AlignSector(const LpMetadataBlockDevice& block_device, - uint64_t sector) const { +bool MetadataBuilder::AlignSector(const LpMetadataBlockDevice& block_device, uint64_t sector, + uint64_t* out) const { // Note: when reading alignment info from the Kernel, we don't assume it // is aligned to the sector size, so we round up to the nearest sector. uint64_t lba = sector * LP_SECTOR_SIZE; - uint64_t aligned = AlignTo(lba, block_device.alignment, block_device.alignment_offset); - return AlignTo(aligned, LP_SECTOR_SIZE) / LP_SECTOR_SIZE; + if (!AlignTo(lba, block_device.alignment, out)) { + return false; + } + if (!AlignTo(*out, LP_SECTOR_SIZE, out)) { + return false; + } + *out /= LP_SECTOR_SIZE; + return true; } bool MetadataBuilder::FindBlockDeviceByName(const std::string& partition_name, @@ -1005,7 +1036,12 @@ bool MetadataBuilder::UpdateBlockDeviceInfo(size_t index, const BlockDeviceInfo& bool MetadataBuilder::ResizePartition(Partition* partition, uint64_t requested_size, const std::vector<Interval>& free_region_hint) { // Align the space needed up to the nearest sector. - uint64_t aligned_size = AlignTo(requested_size, geometry_.logical_block_size); + uint64_t aligned_size; + if (!AlignTo(requested_size, geometry_.logical_block_size, &aligned_size)) { + LERROR << "Cannot resize partition " << partition->name() << " to " << requested_size + << " bytes; integer overflow."; + return false; + } uint64_t old_size = partition->size(); if (!ValidatePartitionSizeChange(partition, old_size, aligned_size, false)) { diff --git a/fs_mgr/liblp/builder_test.cpp b/fs_mgr/liblp/builder_test.cpp index 977ebe3c0..1a3250ae5 100644 --- a/fs_mgr/liblp/builder_test.cpp +++ b/fs_mgr/liblp/builder_test.cpp @@ -174,7 +174,7 @@ TEST_F(BuilderTest, InternalAlignment) { ASSERT_NE(exported, nullptr); super_device = GetMetadataSuperBlockDevice(*exported.get()); ASSERT_NE(super_device, nullptr); - EXPECT_EQ(super_device->first_logical_sector, 1472); + EXPECT_EQ(super_device->first_logical_sector, 1536); // Alignment offset without alignment doesn't mean anything. device_info.alignment = 0; @@ -190,7 +190,7 @@ TEST_F(BuilderTest, InternalAlignment) { ASSERT_NE(exported, nullptr); super_device = GetMetadataSuperBlockDevice(*exported.get()); ASSERT_NE(super_device, nullptr); - EXPECT_EQ(super_device->first_logical_sector, 174); + EXPECT_EQ(super_device->first_logical_sector, 168); // Test a small alignment with no alignment offset. device_info.alignment = 11 * 1024; @@ -200,7 +200,7 @@ TEST_F(BuilderTest, InternalAlignment) { ASSERT_NE(exported, nullptr); super_device = GetMetadataSuperBlockDevice(*exported.get()); ASSERT_NE(super_device, nullptr); - EXPECT_EQ(super_device->first_logical_sector, 160); + EXPECT_EQ(super_device->first_logical_sector, 154); } TEST_F(BuilderTest, InternalPartitionAlignment) { @@ -228,13 +228,14 @@ TEST_F(BuilderTest, InternalPartitionAlignment) { ASSERT_EQ(extent.target_type, LP_TARGET_TYPE_LINEAR); EXPECT_EQ(extent.num_sectors, 80); + uint64_t aligned_lba; uint64_t lba = extent.target_data * LP_SECTOR_SIZE; - uint64_t aligned_lba = AlignTo(lba, device_info.alignment, device_info.alignment_offset); + ASSERT_TRUE(AlignTo(lba, device_info.alignment, &aligned_lba)); EXPECT_EQ(lba, aligned_lba); } // Sanity check one extent. - EXPECT_EQ(exported->extents.back().target_data, 3008); + EXPECT_EQ(exported->extents.back().target_data, 3072); } TEST_F(BuilderTest, UseAllDiskSpace) { @@ -652,7 +653,7 @@ TEST_F(BuilderTest, MultipleBlockDevices) { }; unique_ptr<MetadataBuilder> builder = MetadataBuilder::New(partitions, "system_a", 65536, 2); ASSERT_NE(builder, nullptr); - EXPECT_EQ(builder->AllocatableSpace(), 467238912); + EXPECT_EQ(builder->AllocatableSpace(), 467402752); // Create a partition that spans 3 devices. Partition* p = builder->AddPartition("system_a", 0); @@ -675,17 +676,17 @@ TEST_F(BuilderTest, MultipleBlockDevices) { EXPECT_EQ(metadata->block_devices[2].alignment, 786432); EXPECT_EQ(metadata->block_devices[2].alignment_offset, 753664); ASSERT_EQ(metadata->extents.size(), 3); - EXPECT_EQ(metadata->extents[0].num_sectors, 522304); + EXPECT_EQ(metadata->extents[0].num_sectors, 522752); EXPECT_EQ(metadata->extents[0].target_type, LP_TARGET_TYPE_LINEAR); - EXPECT_EQ(metadata->extents[0].target_data, 1984); + EXPECT_EQ(metadata->extents[0].target_data, 1536); EXPECT_EQ(metadata->extents[0].target_source, 0); - EXPECT_EQ(metadata->extents[1].num_sectors, 260672); + EXPECT_EQ(metadata->extents[1].num_sectors, 260608); EXPECT_EQ(metadata->extents[1].target_type, LP_TARGET_TYPE_LINEAR); - EXPECT_EQ(metadata->extents[1].target_data, 1472); + EXPECT_EQ(metadata->extents[1].target_data, 1536); EXPECT_EQ(metadata->extents[1].target_source, 1); - EXPECT_EQ(metadata->extents[2].num_sectors, 129088); + EXPECT_EQ(metadata->extents[2].num_sectors, 128704); EXPECT_EQ(metadata->extents[2].target_type, LP_TARGET_TYPE_LINEAR); - EXPECT_EQ(metadata->extents[2].target_data, 1472); + EXPECT_EQ(metadata->extents[2].target_data, 1536); EXPECT_EQ(metadata->extents[2].target_source, 2); } @@ -1019,3 +1020,49 @@ TEST_F(BuilderTest, LinearExtentOverlap) { EXPECT_FALSE(extent.OverlapsWith(LinearExtent{20, 1, 15})); EXPECT_FALSE(extent.OverlapsWith(LinearExtent{20, 1, 10})); } + +TEST_F(BuilderTest, AlignFreeRegion) { + BlockDeviceInfo super("super", 8_GiB, 786432, 0, 4096); + std::vector<BlockDeviceInfo> block_devices = {super}; + + unique_ptr<MetadataBuilder> builder = MetadataBuilder::New(block_devices, "super", 65536, 2); + ASSERT_NE(builder, nullptr); + + Partition* p = builder->AddPartition("system", "default", 0); + ASSERT_NE(p, nullptr); + ASSERT_TRUE(builder->AddLinearExtent(p, "super", 64, (super.alignment + 4096) / 512)); + + p = builder->AddPartition("vendor", "default", 0); + ASSERT_NE(p, nullptr); + ASSERT_TRUE(builder->ResizePartition(p, 2_GiB)); + + const auto& extents = p->extents(); + ASSERT_EQ(extents.size(), 2); + + LinearExtent* e1 = extents[0]->AsLinearExtent(); + ASSERT_NE(e1, nullptr); + LinearExtent* e2 = extents[1]->AsLinearExtent(); + ASSERT_NE(e2, nullptr); + + // The misaligned partition starting at sector 1544 should not cause any + // overlap with previous extents. We should see vendor punch a hole where + // "system" is, extending the hole up to the next aligned block. + EXPECT_EQ(e1->physical_sector(), 1536); + EXPECT_EQ(e1->end_sector(), 1544); + EXPECT_EQ(e2->physical_sector(), 3072); + EXPECT_EQ(e2->end_sector(), 4197368); +} + +TEST_F(BuilderTest, ResizeOverflow) { + BlockDeviceInfo super("super", 8_GiB, 786432, 229376, 4096); + std::vector<BlockDeviceInfo> block_devices = {super}; + + unique_ptr<MetadataBuilder> builder = MetadataBuilder::New(block_devices, "super", 65536, 2); + ASSERT_NE(builder, nullptr); + + ASSERT_TRUE(builder->AddGroup("group", 0)); + + Partition* p = builder->AddPartition("system", "default", 0); + ASSERT_NE(p, nullptr); + ASSERT_FALSE(builder->ResizePartition(p, 18446744073709551615ULL)); +} diff --git a/fs_mgr/liblp/device_test.cpp b/fs_mgr/liblp/device_test.cpp index 382d53d52..6af9d94db 100644 --- a/fs_mgr/liblp/device_test.cpp +++ b/fs_mgr/liblp/device_test.cpp @@ -50,12 +50,7 @@ TEST_F(DeviceTest, BlockDeviceInfo) { // Sanity check that the device doesn't give us some weird inefficient // alignment. EXPECT_EQ(device_info.alignment % LP_SECTOR_SIZE, 0); - EXPECT_EQ(device_info.alignment_offset % LP_SECTOR_SIZE, 0); - EXPECT_LE(device_info.alignment_offset, INT_MAX); EXPECT_EQ(device_info.logical_block_size % LP_SECTOR_SIZE, 0); - - // Having an alignment offset > alignment doesn't really make sense. - EXPECT_LT(device_info.alignment_offset, device_info.alignment); } TEST_F(DeviceTest, ReadSuperPartitionCurrentSlot) { diff --git a/fs_mgr/liblp/images.cpp b/fs_mgr/liblp/images.cpp index e4d92ca90..0b1e522d6 100644 --- a/fs_mgr/liblp/images.cpp +++ b/fs_mgr/liblp/images.cpp @@ -124,7 +124,7 @@ bool WriteToImageFile(borrowed_fd fd, const LpMetadata& input) { } bool WriteToImageFile(const std::string& file, const LpMetadata& input) { - unique_fd fd(open(file.c_str(), O_CREAT | O_RDWR | O_TRUNC | O_CLOEXEC, 0644)); + unique_fd fd(open(file.c_str(), O_CREAT | O_RDWR | O_TRUNC | O_CLOEXEC | O_BINARY, 0644)); if (fd < 0) { PERROR << __PRETTY_FUNCTION__ << " open failed: " << file; return false; @@ -184,7 +184,7 @@ bool ImageBuilder::IsValid() const { } bool ImageBuilder::Export(const std::string& file) { - unique_fd fd(open(file.c_str(), O_CREAT | O_RDWR | O_TRUNC | O_CLOEXEC, 0644)); + unique_fd fd(open(file.c_str(), O_CREAT | O_RDWR | O_TRUNC | O_CLOEXEC | O_BINARY, 0644)); if (fd < 0) { PERROR << "open failed: " << file; return false; @@ -208,7 +208,7 @@ bool ImageBuilder::ExportFiles(const std::string& output_dir) { std::string file_name = "super_" + name + ".img"; std::string file_path = output_dir + "/" + file_name; - static const int kOpenFlags = O_CREAT | O_RDWR | O_TRUNC | O_CLOEXEC | O_NOFOLLOW; + static const int kOpenFlags = O_CREAT | O_RDWR | O_TRUNC | O_CLOEXEC | O_NOFOLLOW | O_BINARY; unique_fd fd(open(file_path.c_str(), kOpenFlags, 0644)); if (fd < 0) { PERROR << "open failed: " << file_path; @@ -443,7 +443,7 @@ bool ImageBuilder::CheckExtentOrdering() { } int ImageBuilder::OpenImageFile(const std::string& file) { - android::base::unique_fd source_fd = GetControlFileOrOpen(file.c_str(), O_RDONLY | O_CLOEXEC); + unique_fd source_fd = GetControlFileOrOpen(file.c_str(), O_RDONLY | O_CLOEXEC | O_BINARY); if (source_fd < 0) { PERROR << "open image file failed: " << file; return -1; diff --git a/fs_mgr/liblp/include/liblp/builder.h b/fs_mgr/liblp/include/liblp/builder.h index bd3915061..89a47b112 100644 --- a/fs_mgr/liblp/include/liblp/builder.h +++ b/fs_mgr/liblp/include/liblp/builder.h @@ -144,7 +144,6 @@ class Partition final { std::vector<std::unique_ptr<Extent>> extents_; uint32_t attributes_; uint64_t size_; - bool disabled_; }; // An interval in the metadata. This is similar to a LinearExtent with one difference. @@ -321,9 +320,6 @@ class MetadataBuilder { // Set the LP_HEADER_FLAG_VIRTUAL_AB_DEVICE flag. void SetVirtualABDeviceFlag(); - // If set, checks for slot suffixes will be ignored internally. - void IgnoreSlotSuffixing(); - bool GetBlockDeviceInfo(const std::string& partition_name, BlockDeviceInfo* info) const; bool UpdateBlockDeviceInfo(const std::string& partition_name, const BlockDeviceInfo& info); @@ -359,7 +355,7 @@ class MetadataBuilder { bool GrowPartition(Partition* partition, uint64_t aligned_size, const std::vector<Interval>& free_region_hint); void ShrinkPartition(Partition* partition, uint64_t aligned_size); - uint64_t AlignSector(const LpMetadataBlockDevice& device, uint64_t sector) const; + bool AlignSector(const LpMetadataBlockDevice& device, uint64_t sector, uint64_t* out) const; uint64_t TotalSizeOfGroup(PartitionGroup* group) const; bool UpdateBlockDeviceInfo(size_t index, const BlockDeviceInfo& info); bool FindBlockDeviceByName(const std::string& partition_name, uint32_t* index) const; diff --git a/fs_mgr/liblp/utility.h b/fs_mgr/liblp/utility.h index 06617696c..c4fe3edfc 100644 --- a/fs_mgr/liblp/utility.h +++ b/fs_mgr/liblp/utility.h @@ -21,6 +21,7 @@ #include <stdint.h> #include <sys/types.h> +#include <limits> #include <string> #include <string_view> @@ -66,27 +67,26 @@ int64_t SeekFile64(int fd, int64_t offset, int whence); void SHA256(const void* data, size_t length, uint8_t out[32]); // Align |base| such that it is evenly divisible by |alignment|, which does not -// have to be a power of two. -constexpr uint64_t AlignTo(uint64_t base, uint32_t alignment) { +// have to be a power of two. Return false on overflow. +template <typename T> +bool AlignTo(T base, uint32_t alignment, T* out) { + static_assert(std::numeric_limits<T>::is_integer); + static_assert(!std::numeric_limits<T>::is_signed); if (!alignment) { - return base; + *out = base; + return true; } - uint64_t remainder = base % alignment; + T remainder = base % alignment; if (remainder == 0) { - return base; + *out = base; + return true; } - return base + (alignment - remainder); -} - -// Same as the above |AlignTo|, except that |base| is only aligned when added to -// |alignment_offset|. -constexpr uint64_t AlignTo(uint64_t base, uint32_t alignment, uint32_t alignment_offset) { - uint64_t aligned = AlignTo(base, alignment) + alignment_offset; - if (aligned - alignment >= base) { - // We overaligned (base < alignment_offset). - return aligned - alignment; + T to_add = alignment - remainder; + if (to_add > std::numeric_limits<T>::max() - base) { + return false; } - return aligned; + *out = base + to_add; + return true; } // Update names from C++ strings. diff --git a/fs_mgr/liblp/utility_test.cpp b/fs_mgr/liblp/utility_test.cpp index cac3989e2..fc90872a5 100644 --- a/fs_mgr/liblp/utility_test.cpp +++ b/fs_mgr/liblp/utility_test.cpp @@ -14,6 +14,8 @@ * limitations under the License. */ +#include <optional> + #include <gtest/gtest.h> #include <liblp/builder.h> #include <liblp/liblp.h> @@ -58,15 +60,28 @@ TEST(liblp, GetMetadataOffset) { EXPECT_EQ(GetBackupMetadataOffset(geometry, 0), backup_start + 16384 * 0); } +std::optional<uint64_t> AlignTo(uint64_t base, uint32_t alignment) { + uint64_t r; + if (!AlignTo(base, alignment, &r)) { + return {}; + } + return {r}; +} + TEST(liblp, AlignTo) { - EXPECT_EQ(AlignTo(37, 0), 37); - EXPECT_EQ(AlignTo(1024, 1024), 1024); - EXPECT_EQ(AlignTo(555, 1024), 1024); - EXPECT_EQ(AlignTo(555, 1000), 1000); - EXPECT_EQ(AlignTo(0, 1024), 0); - EXPECT_EQ(AlignTo(54, 32, 30), 62); - EXPECT_EQ(AlignTo(32, 32, 30), 62); - EXPECT_EQ(AlignTo(17, 32, 30), 30); + EXPECT_EQ(AlignTo(37, 0), std::optional<uint64_t>(37)); + EXPECT_EQ(AlignTo(1024, 1024), std::optional<uint64_t>(1024)); + EXPECT_EQ(AlignTo(555, 1024), std::optional<uint64_t>(1024)); + EXPECT_EQ(AlignTo(555, 1000), std::optional<uint64_t>(1000)); + EXPECT_EQ(AlignTo(0, 1024), std::optional<uint64_t>(0)); + EXPECT_EQ(AlignTo(54, 32), std::optional<uint64_t>(64)); + EXPECT_EQ(AlignTo(32, 32), std::optional<uint64_t>(32)); + EXPECT_EQ(AlignTo(17, 32), std::optional<uint64_t>(32)); + + auto u32limit = std::numeric_limits<uint32_t>::max(); + auto u64limit = std::numeric_limits<uint64_t>::max(); + EXPECT_EQ(AlignTo(u64limit - u32limit + 1, u32limit), std::optional<uint64_t>{u64limit}); + EXPECT_EQ(AlignTo(std::numeric_limits<uint64_t>::max(), 2), std::optional<uint64_t>{}); } TEST(liblp, GetPartitionSlotSuffix) { diff --git a/fs_mgr/libsnapshot/Android.bp b/fs_mgr/libsnapshot/Android.bp index c11878806..95301ff71 100644 --- a/fs_mgr/libsnapshot/Android.bp +++ b/fs_mgr/libsnapshot/Android.bp @@ -72,6 +72,7 @@ filegroup { "device_info.cpp", "snapshot.cpp", "snapshot_stats.cpp", + "snapshot_stub.cpp", "snapshot_metadata_updater.cpp", "partition_cow_creator.cpp", "return.cpp", @@ -99,6 +100,7 @@ cc_library_static { cc_library_static { name: "libsnapshot_init", + native_coverage : true, defaults: ["libsnapshot_defaults"], srcs: [":libsnapshot_sources"], recovery_available: true, @@ -208,6 +210,7 @@ cc_binary { static_libs: [ "libfstab", "libsnapshot", + "update_metadata-protos", ], shared_libs: [ "android.hardware.boot@1.0", @@ -225,3 +228,85 @@ cc_binary { "libutils", ], } + +cc_test { + name: "snapshot_power_test", + srcs: [ + "power_test.cpp", + ], + static_libs: [ + "libsnapshot", + "update_metadata-protos", + ], + shared_libs: [ + "libbase", + "libfs_mgr_binder", + "liblog", + ], + gtest: false, +} + +cc_defaults { + name: "libsnapshot_fuzzer_defaults", + native_coverage : true, + srcs: [ + // Compile the protobuf definition again with type full. + "android/snapshot/snapshot_fuzz.proto", + "update_engine/update_metadata.proto", + "fuzz_utils.cpp", + "snapshot_fuzz.cpp", + "snapshot_fuzz_utils.cpp", + + // Compile libsnapshot sources directly to avoid dependency + // to update_metadata-protos + ":libsnapshot_sources", + ], + static_libs: [ + "libbase", + "libcrypto_static", + "libcutils", + "libext2_uuid", + "libext4_utils", + "libfstab", + "libfs_mgr", + "libgtest", // from libsnapshot_test_helpers + "libgmock", // from libsnapshot_test_helpers + "liblog", + "liblp", + "libsnapshot_test_helpers", + "libprotobuf-mutator", + ], + header_libs: [ + "libfiemap_headers", + "libstorage_literals_headers", + ], + proto: { + type: "full", + canonical_path_from_root: false, + local_include_dirs: ["."], + }, +} + +cc_fuzz { + name: "libsnapshot_fuzzer", + defaults: ["libsnapshot_fuzzer_defaults"], + corpus: ["corpus/*"], + fuzz_config: { + cc: ["android-virtual-ab+bugs@google.com"], + componentid: 30545, + hotlists: ["1646452"], + fuzz_on_haiku_host: false, + fuzz_on_haiku_device: true, + }, +} + +cc_test { + name: "libsnapshot_fuzzer_test", + defaults: ["libsnapshot_fuzzer_defaults"], + data: ["corpus/*"], + test_suites: [ + "device-tests", + ], + auto_gen_config: true, + require_root: true, +} diff --git a/fs_mgr/libsnapshot/PowerTest.md b/fs_mgr/libsnapshot/PowerTest.md new file mode 100644 index 000000000..0b0cb5dbf --- /dev/null +++ b/fs_mgr/libsnapshot/PowerTest.md @@ -0,0 +1,40 @@ +snapshot\_power\_test +--------------------- + +snapshot\_power\_test is a standalone test to simulate power failures during a snapshot-merge operation. + +### Test Setup + +Start by creating two large files that will be used as the pre-merge and post-merge state. You can take two different partition images (for example, a product.img from two separate builds), or just create random data: + + dd if=/dev/urandom of=pre-merge count=1024 bs=1048576 + dd if=/dev/urandom of=post-merge count=1024 bs=1048576 + +Next, push these files to an unencrypted directory on the device: + + adb push pre-merge /data/local/unencrypted + adb push post-merge /data/local/unencrypted + +Next, run the test setup: + + adb sync data + adb shell /data/nativetest64/snapshot_power_test/snapshot_power_test \ + /data/local/unencrypted/pre-merge \ + /data/local/unencrypted/post-merge + +This will create the necessary fiemap-based images. + +### Running +The actual test can be run via `run_power_test.sh`. Its syntax is: + + run_power_test.sh <POST_MERGE_FILE> + +`POST_MERGE_FILE` should be the path on the device of the image to validate the merge against. Example: + + run_power_test.sh /data/local/unencrypted/post-merge + +The device will begin the merge with a 5% chance of injecting a kernel crash every 10ms. The device should be capable of rebooting normally without user intervention. Once the merge has completed, the test will run a final check command to validate the contents of the snapshot against the post-merge file. It will error if there are any incorrect blocks. + +Two environment variables can be passed to `run_power_test.sh`: +1. `FAIL_RATE` - A fraction between 0 and 100 (inclusive) indicating the probability the device should inject a kernel crash every 10ms. +2. `DEVICE_SERIAL` - If multiple devices are attached to adb, this argument is passed as the serial to select (to `adb -s`). diff --git a/fs_mgr/libsnapshot/android/snapshot/snapshot_fuzz.proto b/fs_mgr/libsnapshot/android/snapshot/snapshot_fuzz.proto new file mode 100644 index 000000000..a55b42acb --- /dev/null +++ b/fs_mgr/libsnapshot/android/snapshot/snapshot_fuzz.proto @@ -0,0 +1,110 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package android.snapshot; + +import "update_engine/update_metadata.proto"; + +// Controls the behavior of IDeviceInfo. +// Next: 6 +message FuzzDeviceInfoData { + bool slot_suffix_is_a = 1; + bool is_overlayfs_setup = 2; + bool allow_set_boot_control_merge_status = 3; + bool allow_set_slot_as_unbootable = 4; + bool is_recovery = 5; +} + +// Controls the behavior of the test SnapshotManager. +// Next: 2 +message FuzzSnapshotManagerData { + bool is_local_image_manager = 1; +} + +// A simplified version of CreateLogicalPartitionParams for fuzzing. +// Next: 9 +message CreateLogicalPartitionParamsProto { + bool use_correct_super = 1; + string block_device = 2; + bool has_metadata_slot = 3; + uint32 metadata_slot = 4; + string partition_name = 5; + bool force_writable = 6; + int64 timeout_millis = 7; + string device_name = 8; +} + +// Mimics the API of ISnapshotManager. Defines one action on the snapshot +// manager. +// Next: 18 +message SnapshotManagerActionProto { + message NoArgs {} + message ProcessUpdateStateArgs { + bool has_before_cancel = 1; + bool fail_before_cancel = 2; + } + message CreateLogicalAndSnapshotPartitionsArgs { + bool use_correct_super = 1; + string super = 2; + int64 timeout_millis = 3; + } + message RecoveryCreateSnapshotDevicesArgs { + bool has_metadata_device_object = 1; + bool metadata_mounted = 2; + } + reserved 18 to 9999; + oneof value { + NoArgs begin_update = 1; + NoArgs cancel_update = 2; + bool finished_snapshot_writes = 3; + NoArgs initiate_merge = 4; + ProcessUpdateStateArgs process_update_state = 5; + bool get_update_state = 6; + chromeos_update_engine.DeltaArchiveManifest create_update_snapshots = 7; + CreateLogicalPartitionParamsProto map_update_snapshot = 8; + string unmap_update_snapshot = 9; + NoArgs need_snapshots_in_first_stage_mount = 10; + CreateLogicalAndSnapshotPartitionsArgs create_logical_and_snapshot_partitions = 11; + bool handle_imminent_data_wipe = 12; + NoArgs recovery_create_snapshot_devices = 13; + RecoveryCreateSnapshotDevicesArgs recovery_create_snapshot_devices_with_metadata = 14; + NoArgs dump = 15; + NoArgs ensure_metadata_mounted = 16; + NoArgs get_snapshot_merge_stats_instance = 17; + + // Test directives that has nothing to do with ISnapshotManager API surface. + NoArgs switch_slot = 10000; + } +} + +// Includes all data that needs to be fuzzed. +message SnapshotFuzzData { + FuzzDeviceInfoData device_info_data = 1; + FuzzSnapshotManagerData manager_data = 2; + + // If true: + // - if super_data is empty, create empty super partition metadata. + // - otherwise, create super partition metadata accordingly. + // If false, no valid super partition metadata (it is zeroed) + bool is_super_metadata_valid = 3; + chromeos_update_engine.DeltaArchiveManifest super_data = 4; + + // Whether the directory that mocks /metadata/ota/snapshot is created. + bool has_metadata_snapshots_dir = 5; + + // More data used to prep the test before running actions. + reserved 6 to 9999; + repeated SnapshotManagerActionProto actions = 10000; +} diff --git a/fs_mgr/libsnapshot/corpus/launch_device.txt b/fs_mgr/libsnapshot/corpus/launch_device.txt new file mode 100644 index 000000000..55a7f2ccd --- /dev/null +++ b/fs_mgr/libsnapshot/corpus/launch_device.txt @@ -0,0 +1,161 @@ +device_info_data { + slot_suffix_is_a: true + is_overlayfs_setup: false + allow_set_boot_control_merge_status: true + allow_set_slot_as_unbootable: true + is_recovery: false +} +manager_data { + is_local_image_manager: false +} +is_super_metadata_valid: true +super_data { + partitions { + partition_name: "sys_a" + new_partition_info { + size: 3145728 + } + } + partitions { + partition_name: "vnd_a" + new_partition_info { + size: 3145728 + } + } + partitions { + partition_name: "prd_a" + new_partition_info { + size: 3145728 + } + } + dynamic_partition_metadata { + groups { + name: "group_google_dp_a" + size: 15728640 + partition_names: "sys_a" + partition_names: "vnd_a" + partition_names: "prd_a" + } + } +} +has_metadata_snapshots_dir: true +actions { + begin_update { + } +} +actions { + create_update_snapshots { + partitions { + partition_name: "sys" + new_partition_info { + size: 3878912 + } + operations { + type: ZERO, + dst_extents { + start_block: 0 + num_blocks: 947 + } + } + } + partitions { + partition_name: "vnd" + new_partition_info { + size: 3878912 + } + operations { + type: ZERO, + dst_extents { + start_block: 0 + num_blocks: 947 + } + } + } + partitions { + partition_name: "prd" + new_partition_info { + size: 3878912 + } + operations { + type: ZERO, + dst_extents { + start_block: 0 + num_blocks: 947 + } + } + } + dynamic_partition_metadata { + groups { + name: "group_google_dp" + size: 15728640 + partition_names: "sys" + partition_names: "vnd" + partition_names: "prd" + } + } + } +} +actions { + map_update_snapshot { + use_correct_super: true + has_metadata_slot: true + metadata_slot: 1 + partition_name: "sys_b" + force_writable: true + timeout_millis: 3000 + } +} +actions { + map_update_snapshot { + use_correct_super: true + has_metadata_slot: true + metadata_slot: 1 + partition_name: "vnd_b" + force_writable: true + timeout_millis: 3000 + } +} +actions { + map_update_snapshot { + use_correct_super: true + has_metadata_slot: true + metadata_slot: 1 + partition_name: "prd_b" + force_writable: true + timeout_millis: 3000 + } +} +actions { + finished_snapshot_writes: false +} +actions { + unmap_update_snapshot: "sys_b" +} +actions { + unmap_update_snapshot: "vnd_b" +} +actions { + unmap_update_snapshot: "prd_b" +} +actions { + switch_slot { + } +} +actions { + need_snapshots_in_first_stage_mount { + } +} +actions { + create_logical_and_snapshot_partitions { + use_correct_super: true + timeout_millis: 5000 + } +} +actions { + initiate_merge { + } +} +actions { + process_update_state { + } +} diff --git a/fs_mgr/libsnapshot/fuzz.sh b/fs_mgr/libsnapshot/fuzz.sh new file mode 100755 index 000000000..5995cefde --- /dev/null +++ b/fs_mgr/libsnapshot/fuzz.sh @@ -0,0 +1,90 @@ +#!/bin/bash +PROJECT_PATH=system/core/fs_mgr/libsnapshot +FUZZ_TARGET=libsnapshot_fuzzer +TARGET_ARCH=$(get_build_var TARGET_ARCH) +FUZZ_BINARY=/data/fuzz/${TARGET_ARCH}/${FUZZ_TARGET}/${FUZZ_TARGET} +DEVICE_INIT_CORPUS_DIR=/data/fuzz/${TARGET_ARCH}/${FUZZ_TARGET}/corpus +DEVICE_GENERATED_CORPUS_DIR=/data/local/tmp/${FUZZ_TARGET}/corpus +DEVICE_GCOV_DIR=/data/local/tmp/${FUZZ_TARGET}/gcov +HOST_SCRATCH_DIR=/tmp/${FUZZ_TARGET} +GCOV_TOOL=${HOST_SCRATCH_DIR}/llvm-gcov + +build_normal() ( + pushd $(gettop) + NATIVE_COVERAGE="" NATIVE_LINE_COVERAGE="" NATIVE_COVERAGE_PATHS="" m ${FUZZ_TARGET} + ret=$? + popd + return ${ret} +) + +build_cov() { + pushd $(gettop) + NATIVE_COVERAGE="true" NATIVE_LINE_COVERAGE="true" NATIVE_COVERAGE_PATHS="${PROJECT_PATH}" m ${FUZZ_TARGET} + ret=$? + popd + return ${ret} +} + +prepare_device() { + adb root && adb remount && + adb shell mkdir -p ${DEVICE_GENERATED_CORPUS_DIR} && + adb shell rm -rf ${DEVICE_GCOV_DIR} && + adb shell mkdir -p ${DEVICE_GCOV_DIR} +} + +push_binary() { + adb push ${ANDROID_PRODUCT_OUT}/${FUZZ_BINARY} ${FUZZ_BINARY} && + adb push ${ANDROID_PRODUCT_OUT}/${DEVICE_INIT_CORPUS_DIR} $(dirname ${FUZZ_BINARY}) +} + +prepare_host() { + which lcov || { + echo "please run:"; + echo " sudo apt-get install lcov "; + return 1; + } + rm -rf ${HOST_SCRATCH_DIR} && + mkdir -p ${HOST_SCRATCH_DIR} +} + +# run_snapshot_fuzz -runs=10000 +generate_corpus() { + [[ "$@" ]] || { echo "run with -runs=X"; return 1; } + + prepare_device && + build_normal && + push_binary && + adb shell ${FUZZ_BINARY} "$@" ${DEVICE_INIT_CORPUS_DIR} ${DEVICE_GENERATED_CORPUS_DIR} +} + +run_snapshot_fuzz() { + prepare_device && + build_cov && + push_binary && + adb shell GCOV_PREFIX=${DEVICE_GCOV_DIR} GCOV_PREFIX_STRIP=3 \ + ${FUZZ_BINARY} \ + -runs=0 \ + ${DEVICE_INIT_CORPUS_DIR} ${DEVICE_GENERATED_CORPUS_DIR} +} + +show_fuzz_result() { + prepare_host && + unzip -o -j -d ${HOST_SCRATCH_DIR} ${ANDROID_PRODUCT_OUT}/coverage/data/fuzz/${TARGET_ARCH}/${FUZZ_TARGET}/${FUZZ_TARGET}.zip && + adb shell find ${DEVICE_GCOV_DIR} -type f | xargs -I {} adb pull {} ${HOST_SCRATCH_DIR} && + ls ${HOST_SCRATCH_DIR} && + cat > ${GCOV_TOOL} <<< ' +#!/bin/bash +exec llvm-cov gcov "$@" +' && + chmod +x ${GCOV_TOOL} && + lcov --directory ${HOST_SCRATCH_DIR} --base-directory $(gettop) --gcov-tool ${GCOV_TOOL} --capture -o ${HOST_SCRATCH_DIR}/report.cov && + genhtml ${HOST_SCRATCH_DIR}/report.cov -o ${HOST_SCRATCH_DIR}/html && + echo file://$(realpath ${HOST_SCRATCH_DIR}/html/index.html) +} + +# run_snapshot_fuzz -runs=10000 +run_snapshot_fuzz_all() { + generate_corpus "$@" && + run_snapshot_fuzz && + show_fuzz_result +} diff --git a/fs_mgr/libsnapshot/fuzz_utils.cpp b/fs_mgr/libsnapshot/fuzz_utils.cpp new file mode 100644 index 000000000..0263f7e46 --- /dev/null +++ b/fs_mgr/libsnapshot/fuzz_utils.cpp @@ -0,0 +1,38 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fuzz_utils.h" + +#include <android-base/logging.h> + +namespace android::fuzz { + +void CheckInternal(bool value, std::string_view msg) { + CHECK(value) << msg; +} + +const google::protobuf::OneofDescriptor* GetProtoValueDescriptor( + const google::protobuf::Descriptor* action_desc) { + CHECK(action_desc); + CHECK(action_desc->oneof_decl_count() == 1) + << action_desc->oneof_decl_count() << " oneof fields found in " << action_desc->name() + << "; only one is expected."; + auto* oneof_value_desc = action_desc->oneof_decl(0); + CHECK(oneof_value_desc); + CHECK(oneof_value_desc->name() == "value") + << "oneof field has name " << oneof_value_desc->name(); + return oneof_value_desc; +} + +} // namespace android::fuzz diff --git a/fs_mgr/libsnapshot/fuzz_utils.h b/fs_mgr/libsnapshot/fuzz_utils.h new file mode 100644 index 000000000..20b13b2fa --- /dev/null +++ b/fs_mgr/libsnapshot/fuzz_utils.h @@ -0,0 +1,285 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include <map> +#include <string> +#include <string_view> + +#include <google/protobuf/descriptor.h> +#include <google/protobuf/message.h> +#include <google/protobuf/repeated_field.h> + +// Utilities for using a protobuf definition to fuzz APIs in a class. +// Terms: +// The "fuzzed class" is the C++ class definition whose functions are fuzzed. +// The "fuzzed object" is an instantiated object of the fuzzed class. It is +// typically created and destroyed for each test run. +// An "action" is an operation on the fuzzed object that may mutate its state. +// This typically involves one function call into the fuzzed object. + +namespace android::fuzz { + +// CHECK(value) << msg +void CheckInternal(bool value, std::string_view msg); + +// Get the oneof descriptor inside Action +const google::protobuf::OneofDescriptor* GetProtoValueDescriptor( + const google::protobuf::Descriptor* action_desc); + +template <typename Class> +using FunctionMapImpl = + std::map<int, std::function<void(Class*, const google::protobuf::Message& action_proto, + const google::protobuf::FieldDescriptor* field_desc)>>; + +template <typename Class> +class FunctionMap : public FunctionMapImpl<Class> { + public: + void CheckEmplace(typename FunctionMapImpl<Class>::key_type key, + typename FunctionMapImpl<Class>::mapped_type&& value) { + auto [it, inserted] = this->emplace(key, std::move(value)); + CheckInternal(inserted, + "Multiple implementation registered for tag number " + std::to_string(key)); + } +}; + +template <typename Action> +int CheckConsistency() { + const auto* function_map = Action::GetFunctionMap(); + const auto* action_value_desc = GetProtoValueDescriptor(Action::Proto::GetDescriptor()); + + for (int field_index = 0; field_index < action_value_desc->field_count(); ++field_index) { + const auto* field_desc = action_value_desc->field(field_index); + CheckInternal(function_map->find(field_desc->number()) != function_map->end(), + "Missing impl for function " + field_desc->camelcase_name()); + } + return 0; +} + +// Get the field descriptor for the oneof field in the action message. If no oneof field is set, +// return nullptr. +template <typename Action> +const google::protobuf::FieldDescriptor* GetValueFieldDescriptor( + const typename Action::Proto& action_proto) { + static auto* action_value_desc = GetProtoValueDescriptor(Action::Proto::GetDescriptor()); + + auto* action_refl = Action::Proto::GetReflection(); + if (!action_refl->HasOneof(action_proto, action_value_desc)) { + return nullptr; + } + return action_refl->GetOneofFieldDescriptor(action_proto, action_value_desc); +} + +template <typename Action> +void ExecuteActionProto(typename Action::ClassType* module, + const typename Action::Proto& action_proto) { + const auto* field_desc = GetValueFieldDescriptor<Action>(action_proto); + if (field_desc == nullptr) return; + auto number = field_desc->number(); + const auto& map = *Action::GetFunctionMap(); + auto it = map.find(number); + CheckInternal(it != map.end(), "Missing impl for function " + field_desc->camelcase_name()); + const auto& func = it->second; + func(module, action_proto, field_desc); +} + +template <typename Action> +void ExecuteAllActionProtos( + typename Action::ClassType* module, + const google::protobuf::RepeatedPtrField<typename Action::Proto>& action_protos) { + for (const auto& proto : action_protos) { + ExecuteActionProto<Action>(module, proto); + } +} + +// Safely cast message to T. Returns a pointer to message if cast successfully, otherwise nullptr. +template <typename T> +const T* SafeCast(const google::protobuf::Message& message) { + if (message.GetDescriptor() != T::GetDescriptor()) { + return nullptr; + } + return static_cast<const T*>(&message); +} + +// Cast message to const T&. Abort if type mismatch. +template <typename T> +const T& CheckedCast(const google::protobuf::Message& message) { + const auto* ptr = SafeCast<T>(message); + CheckInternal(ptr, "Cannot cast " + message.GetDescriptor()->name() + " to " + + T::GetDescriptor()->name()); + return *ptr; +} + +// A templated way to a primitive field from a message using reflection. +template <typename T> +struct PrimitiveGetter; +#define FUZZ_DEFINE_PRIMITIVE_GETTER(type, func_name) \ + template <> \ + struct PrimitiveGetter<type> { \ + static constexpr const auto fp = &google::protobuf::Reflection::func_name; \ + } + +FUZZ_DEFINE_PRIMITIVE_GETTER(bool, GetBool); +FUZZ_DEFINE_PRIMITIVE_GETTER(uint32_t, GetUInt32); +FUZZ_DEFINE_PRIMITIVE_GETTER(int32_t, GetInt32); +FUZZ_DEFINE_PRIMITIVE_GETTER(uint64_t, GetUInt64); +FUZZ_DEFINE_PRIMITIVE_GETTER(int64_t, GetInt64); +FUZZ_DEFINE_PRIMITIVE_GETTER(double, GetDouble); +FUZZ_DEFINE_PRIMITIVE_GETTER(float, GetFloat); + +// ActionPerformer extracts arguments from the protobuf message, and then call FuzzFunction +// with these arguments. +template <typename FuzzFunction, typename Signature, typename Enabled = void> +struct ActionPerformerImpl; // undefined + +template <typename FuzzFunction, typename MessageProto> +struct ActionPerformerImpl< + FuzzFunction, void(const MessageProto&), + typename std::enable_if_t<std::is_base_of_v<google::protobuf::Message, MessageProto>>> { + static typename FuzzFunction::ReturnType Invoke( + typename FuzzFunction::ClassType* module, const google::protobuf::Message& action_proto, + const google::protobuf::FieldDescriptor* field_desc) { + const MessageProto& arg = CheckedCast<std::remove_reference_t<MessageProto>>( + action_proto.GetReflection()->GetMessage(action_proto, field_desc)); + return FuzzFunction::ImplBody(module, arg); + } +}; + +template <typename FuzzFunction, typename Primitive> +struct ActionPerformerImpl<FuzzFunction, void(Primitive), + typename std::enable_if_t<std::is_arithmetic_v<Primitive>>> { + static typename FuzzFunction::ReturnType Invoke( + typename FuzzFunction::ClassType* module, const google::protobuf::Message& action_proto, + const google::protobuf::FieldDescriptor* field_desc) { + Primitive arg = std::invoke(PrimitiveGetter<Primitive>::fp, action_proto.GetReflection(), + action_proto, field_desc); + return FuzzFunction::ImplBody(module, arg); + } +}; + +template <typename FuzzFunction> +struct ActionPerformerImpl<FuzzFunction, void()> { + static typename FuzzFunction::ReturnType Invoke(typename FuzzFunction::ClassType* module, + const google::protobuf::Message&, + const google::protobuf::FieldDescriptor*) { + return FuzzFunction::ImplBody(module); + } +}; + +template <typename FuzzFunction> +struct ActionPerformerImpl<FuzzFunction, void(const std::string&)> { + static typename FuzzFunction::ReturnType Invoke( + typename FuzzFunction::ClassType* module, const google::protobuf::Message& action_proto, + const google::protobuf::FieldDescriptor* field_desc) { + std::string scratch; + const std::string& arg = action_proto.GetReflection()->GetStringReference( + action_proto, field_desc, &scratch); + return FuzzFunction::ImplBody(module, arg); + } +}; + +template <typename FuzzFunction> +struct ActionPerformer : ActionPerformerImpl<FuzzFunction, typename FuzzFunction::Signature> {}; + +} // namespace android::fuzz + +// Fuzz existing C++ class, ClassType, with a collection of functions under the name Action. +// +// Prerequisite: ActionProto must be defined in Protobuf to describe possible actions: +// message FooActionProto { +// message NoArgs {} +// oneof value { +// bool do_foo = 1; +// NoArgs do_bar = 1; +// } +// } +// Use it to fuzz a C++ class Foo by doing the following: +// FUZZ_CLASS(Foo, FooAction) +// After linking functions of Foo to FooAction, execute all actions by: +// FooAction::ExecuteAll(foo_object, action_protos) +#define FUZZ_CLASS(Class, Action) \ + class Action { \ + public: \ + using Proto = Action##Proto; \ + using ClassType = Class; \ + using FunctionMap = android::fuzz::FunctionMap<Class>; \ + static FunctionMap* GetFunctionMap() { \ + static Action::FunctionMap map; \ + return ↦ \ + } \ + static void ExecuteAll(Class* module, \ + const google::protobuf::RepeatedPtrField<Proto>& action_protos) { \ + [[maybe_unused]] static int consistent = android::fuzz::CheckConsistency<Action>(); \ + android::fuzz::ExecuteAllActionProtos<Action>(module, action_protos); \ + } \ + } + +#define FUZZ_FUNCTION_CLASS_NAME(Action, FunctionName) Action##_##FunctionName +#define FUZZ_FUNCTION_TAG_NAME(FunctionName) k##FunctionName + +// Implement an action defined in protobuf. Example: +// message FooActionProto { +// oneof value { +// bool do_foo = 1; +// } +// } +// class Foo { public: void DoAwesomeFoo(bool arg); }; +// FUZZ_OBJECT(FooAction, Foo); +// FUZZ_FUNCTION(FooAction, DoFoo, void, IFoo* module, bool arg) { +// module->DoAwesomeFoo(arg); +// } +// The name DoFoo is the camel case name of the action in protobuf definition of FooActionProto. +#define FUZZ_FUNCTION(Action, FunctionName, Return, ModuleArg, ...) \ + class FUZZ_FUNCTION_CLASS_NAME(Action, FunctionName) { \ + public: \ + using ActionType = Action; \ + using ClassType = Action::ClassType; \ + using ReturnType = Return; \ + using Signature = void(__VA_ARGS__); \ + static constexpr const char name[] = #FunctionName; \ + static constexpr const auto tag = \ + Action::Proto::ValueCase::FUZZ_FUNCTION_TAG_NAME(FunctionName); \ + static ReturnType ImplBody(ModuleArg, ##__VA_ARGS__); \ + \ + private: \ + static bool registered_; \ + }; \ + auto FUZZ_FUNCTION_CLASS_NAME(Action, FunctionName)::registered_ = ([] { \ + auto tag = FUZZ_FUNCTION_CLASS_NAME(Action, FunctionName)::tag; \ + auto func = &::android::fuzz::ActionPerformer<FUZZ_FUNCTION_CLASS_NAME( \ + Action, FunctionName)>::Invoke; \ + Action::GetFunctionMap()->CheckEmplace(tag, func); \ + return true; \ + })(); \ + Return FUZZ_FUNCTION_CLASS_NAME(Action, FunctionName)::ImplBody(ModuleArg, ##__VA_ARGS__) + +// Implement a simple action by linking it to the function with the same name. Example: +// message FooActionProto { +// message NoArgs {} +// oneof value { +// NoArgs do_bar = 1; +// } +// } +// class Foo { public void DoBar(); }; +// FUZZ_OBJECT(FooAction, Foo); +// FUZZ_FUNCTION(FooAction, DoBar); +// The name DoBar is the camel case name of the action in protobuf definition of FooActionProto, and +// also the name of the function of Foo. +#define FUZZ_SIMPLE_FUNCTION(Action, FunctionName) \ + FUZZ_FUNCTION(Action, FunctionName, \ + decltype(std::declval<Action::ClassType>().FunctionName()), \ + Action::ClassType* module) { \ + return module->FunctionName(); \ + } diff --git a/fs_mgr/libsnapshot/include/libsnapshot/mock_device_info.h b/fs_mgr/libsnapshot/include/libsnapshot/mock_device_info.h new file mode 100644 index 000000000..ef9d64849 --- /dev/null +++ b/fs_mgr/libsnapshot/include/libsnapshot/mock_device_info.h @@ -0,0 +1,37 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include <libsnapshot/snapshot.h> + +#include <gmock/gmock.h> + +namespace android::snapshot { + +class MockDeviceInfo : public SnapshotManager::IDeviceInfo { + public: + MOCK_METHOD(std::string, GetGsidDir, (), (const, override)); + MOCK_METHOD(std::string, GetMetadataDir, (), (const, override)); + MOCK_METHOD(std::string, GetSlotSuffix, (), (const, override)); + MOCK_METHOD(std::string, GetOtherSlotSuffix, (), (const, override)); + MOCK_METHOD(std::string, GetSuperDevice, (uint32_t slot), (const, override)); + MOCK_METHOD(const android::fs_mgr::IPartitionOpener&, GetPartitionOpener, (), (const)); + MOCK_METHOD(bool, IsOverlayfsSetup, (), (const, override)); + MOCK_METHOD(bool, SetBootControlMergeStatus, (MergeStatus status), (override)); + MOCK_METHOD(bool, SetSlotAsUnbootable, (unsigned int slot), (override)); + MOCK_METHOD(bool, IsRecovery, (), (const, override)); +}; + +} // namespace android::snapshot diff --git a/fs_mgr/libsnapshot/include/libsnapshot/mock_snapshot.h b/fs_mgr/libsnapshot/include/libsnapshot/mock_snapshot.h new file mode 100644 index 000000000..4457de398 --- /dev/null +++ b/fs_mgr/libsnapshot/include/libsnapshot/mock_snapshot.h @@ -0,0 +1,55 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include <libsnapshot/snapshot.h> + +#include <gmock/gmock.h> + +namespace android::snapshot { + +class MockSnapshotManager : public ISnapshotManager { + public: + MOCK_METHOD(bool, BeginUpdate, (), (override)); + MOCK_METHOD(bool, CancelUpdate, (), (override)); + MOCK_METHOD(bool, FinishedSnapshotWrites, (bool wipe), (override)); + MOCK_METHOD(bool, InitiateMerge, (uint64_t * cow_file_size), (override)); + + MOCK_METHOD(UpdateState, ProcessUpdateState, + (const std::function<bool()>& callback, const std::function<bool()>& before_cancel), + (override)); + MOCK_METHOD(UpdateState, GetUpdateState, (double* progress), (override)); + MOCK_METHOD(Return, CreateUpdateSnapshots, + (const chromeos_update_engine::DeltaArchiveManifest& manifest), (override)); + MOCK_METHOD(bool, MapUpdateSnapshot, + (const android::fs_mgr::CreateLogicalPartitionParams& params, + std::string* snapshot_path), + (override)); + MOCK_METHOD(bool, UnmapUpdateSnapshot, (const std::string& target_partition_name), (override)); + MOCK_METHOD(bool, NeedSnapshotsInFirstStageMount, (), (override)); + MOCK_METHOD(bool, CreateLogicalAndSnapshotPartitions, + (const std::string& super_device, const std::chrono::milliseconds& timeout_ms), + (override)); + MOCK_METHOD(bool, HandleImminentDataWipe, (const std::function<void()>& callback), (override)); + MOCK_METHOD(bool, FinishMergeInRecovery, (), (override)); + MOCK_METHOD(CreateResult, RecoveryCreateSnapshotDevices, (), (override)); + MOCK_METHOD(CreateResult, RecoveryCreateSnapshotDevices, + (const std::unique_ptr<AutoDevice>& metadata_device), (override)); + MOCK_METHOD(bool, Dump, (std::ostream & os), (override)); + MOCK_METHOD(std::unique_ptr<AutoDevice>, EnsureMetadataMounted, (), (override)); + MOCK_METHOD(ISnapshotMergeStats*, GetSnapshotMergeStatsInstance, (), (override)); +}; + +} // namespace android::snapshot diff --git a/fs_mgr/libsnapshot/include/libsnapshot/snapshot.h b/fs_mgr/libsnapshot/include/libsnapshot/snapshot.h index b20797870..3c2c77683 100644 --- a/fs_mgr/libsnapshot/include/libsnapshot/snapshot.h +++ b/fs_mgr/libsnapshot/include/libsnapshot/snapshot.h @@ -70,6 +70,8 @@ struct AutoDeleteCowImage; struct AutoDeleteSnapshot; struct AutoDeviceList; struct PartitionCowCreator; +class ISnapshotMergeStats; +class SnapshotMergeStats; class SnapshotStatus; static constexpr const std::string_view kCowGroupName = "cow"; @@ -83,17 +85,7 @@ enum class CreateResult : unsigned int { NOT_CREATED, }; -class SnapshotManager final { - using CreateLogicalPartitionParams = android::fs_mgr::CreateLogicalPartitionParams; - using IPartitionOpener = android::fs_mgr::IPartitionOpener; - using LpMetadata = android::fs_mgr::LpMetadata; - using MetadataBuilder = android::fs_mgr::MetadataBuilder; - using DeltaArchiveManifest = chromeos_update_engine::DeltaArchiveManifest; - using MergeStatus = android::hardware::boot::V1_1::MergeStatus; - using FiemapStatus = android::fiemap::FiemapStatus; - - friend class SnapshotMergeStats; - +class ISnapshotManager { public: // Dependency injection for testing. class IDeviceInfo { @@ -104,39 +96,23 @@ class SnapshotManager final { virtual std::string GetSlotSuffix() const = 0; virtual std::string GetOtherSlotSuffix() const = 0; virtual std::string GetSuperDevice(uint32_t slot) const = 0; - virtual const IPartitionOpener& GetPartitionOpener() const = 0; + virtual const android::fs_mgr::IPartitionOpener& GetPartitionOpener() const = 0; virtual bool IsOverlayfsSetup() const = 0; - virtual bool SetBootControlMergeStatus(MergeStatus status) = 0; + virtual bool SetBootControlMergeStatus( + android::hardware::boot::V1_1::MergeStatus status) = 0; virtual bool SetSlotAsUnbootable(unsigned int slot) = 0; virtual bool IsRecovery() const = 0; }; - - ~SnapshotManager(); - - // Return a new SnapshotManager instance, or null on error. The device - // pointer is owned for the lifetime of SnapshotManager. If null, a default - // instance will be created. - static std::unique_ptr<SnapshotManager> New(IDeviceInfo* device = nullptr); - - // This is similar to New(), except designed specifically for first-stage - // init or recovery. - static std::unique_ptr<SnapshotManager> NewForFirstStageMount(IDeviceInfo* device = nullptr); - - // Helper function for first-stage init to check whether a SnapshotManager - // might be needed to perform first-stage mounts. - static bool IsSnapshotManagerNeeded(); - - // Helper function for second stage init to restorecon on the rollback indicator. - static std::string GetGlobalRollbackIndicatorPath(); + virtual ~ISnapshotManager() = default; // Begin an update. This must be called before creating any snapshots. It // will fail if GetUpdateState() != None. - bool BeginUpdate(); + virtual bool BeginUpdate() = 0; // Cancel an update; any snapshots will be deleted. This is allowed if the // state == Initiated, None, or Unverified (before rebooting to the new // slot). - bool CancelUpdate(); + virtual bool CancelUpdate() = 0; // Mark snapshot writes as having completed. After this, new snapshots cannot // be created, and the device must either cancel the OTA (either before @@ -144,11 +120,11 @@ class SnapshotManager final { // Before calling this function, all snapshots must be mapped. // If |wipe| is set to true, wipe is scheduled after reboot, and snapshots // may need to be merged before wiping. - bool FinishedSnapshotWrites(bool wipe); + virtual bool FinishedSnapshotWrites(bool wipe) = 0; // Initiate a merge on all snapshot devices. This should only be used after an // update has been marked successful after booting. - bool InitiateMerge(uint64_t* cow_file_size = nullptr); + virtual bool InitiateMerge(uint64_t* cow_file_size = nullptr) = 0; // Perform any necessary post-boot actions. This should be run soon after // /data is mounted. @@ -178,8 +154,8 @@ class SnapshotManager final { // // The optional callback allows the caller to periodically check the // progress with GetUpdateState(). - UpdateState ProcessUpdateState(const std::function<bool()>& callback = {}, - const std::function<bool()>& before_cancel = {}); + virtual UpdateState ProcessUpdateState(const std::function<bool()>& callback = {}, + const std::function<bool()>& before_cancel = {}) = 0; // Find the status of the current update, if any. // @@ -187,28 +163,31 @@ class SnapshotManager final { // Merging: Value in the range [0, 100] // MergeCompleted: 100 // Other: 0 - UpdateState GetUpdateState(double* progress = nullptr); + virtual UpdateState GetUpdateState(double* progress = nullptr) = 0; // Create necessary COW device / files for OTA clients. New logical partitions will be added to // group "cow" in target_metadata. Regions of partitions of current_metadata will be // "write-protected" and snapshotted. - Return CreateUpdateSnapshots(const DeltaArchiveManifest& manifest); + virtual Return CreateUpdateSnapshots( + const chromeos_update_engine::DeltaArchiveManifest& manifest) = 0; // Map a snapshotted partition for OTA clients to write to. Write-protected regions are // determined previously in CreateSnapshots. - bool MapUpdateSnapshot(const CreateLogicalPartitionParams& params, std::string* snapshot_path); + // |snapshot_path| must not be nullptr. + virtual bool MapUpdateSnapshot(const android::fs_mgr::CreateLogicalPartitionParams& params, + std::string* snapshot_path) = 0; // Unmap a snapshot device that's previously mapped with MapUpdateSnapshot. - bool UnmapUpdateSnapshot(const std::string& target_partition_name); + virtual bool UnmapUpdateSnapshot(const std::string& target_partition_name) = 0; // If this returns true, first-stage mount must call // CreateLogicalAndSnapshotPartitions rather than CreateLogicalPartitions. - bool NeedSnapshotsInFirstStageMount(); + virtual bool NeedSnapshotsInFirstStageMount() = 0; // Perform first-stage mapping of snapshot targets. This replaces init's // call to CreateLogicalPartitions when snapshots are present. - bool CreateLogicalAndSnapshotPartitions(const std::string& super_device, - const std::chrono::milliseconds& timeout_ms = {}); + virtual bool CreateLogicalAndSnapshotPartitions( + const std::string& super_device, const std::chrono::milliseconds& timeout_ms = {}) = 0; // This method should be called preceding any wipe or flash of metadata or // userdata. It is only valid in recovery or fastbootd, and it ensures that @@ -221,11 +200,11 @@ class SnapshotManager final { // // Returns true on success (or nothing to do), false on failure. The // optional callback fires periodically to query progress via GetUpdateState. - bool HandleImminentDataWipe(const std::function<void()>& callback = {}); + virtual bool HandleImminentDataWipe(const std::function<void()>& callback = {}) = 0; // Force a merge to complete in recovery. This is similar to HandleImminentDataWipe // but does not expect a data wipe after. - bool FinishMergeInRecovery(); + virtual bool FinishMergeInRecovery() = 0; // This method is only allowed in recovery and is used as a helper to // initialize the snapshot devices as a requirement to mount a snapshotted @@ -238,14 +217,15 @@ class SnapshotManager final { // be aborted. // This function mounts /metadata when called, and unmounts /metadata upon // return. - CreateResult RecoveryCreateSnapshotDevices(); + virtual CreateResult RecoveryCreateSnapshotDevices() = 0; // Same as RecoveryCreateSnapshotDevices(), but does not auto mount/umount // /metadata. - CreateResult RecoveryCreateSnapshotDevices(const std::unique_ptr<AutoDevice>& metadata_device); + virtual CreateResult RecoveryCreateSnapshotDevices( + const std::unique_ptr<AutoDevice>& metadata_device) = 0; // Dump debug information. - bool Dump(std::ostream& os); + virtual bool Dump(std::ostream& os) = 0; // Ensure metadata directory is mounted in recovery. When the returned // AutoDevice is destroyed, the metadata directory is automatically @@ -261,7 +241,66 @@ class SnapshotManager final { // auto b = mgr->EnsureMetadataMounted(); // does nothing // b.reset() // unmounts // a.reset() // does nothing - std::unique_ptr<AutoDevice> EnsureMetadataMounted(); + virtual std::unique_ptr<AutoDevice> EnsureMetadataMounted() = 0; + + // Return the associated ISnapshotMergeStats instance. Never null. + virtual ISnapshotMergeStats* GetSnapshotMergeStatsInstance() = 0; +}; + +class SnapshotManager final : public ISnapshotManager { + using CreateLogicalPartitionParams = android::fs_mgr::CreateLogicalPartitionParams; + using IPartitionOpener = android::fs_mgr::IPartitionOpener; + using LpMetadata = android::fs_mgr::LpMetadata; + using MetadataBuilder = android::fs_mgr::MetadataBuilder; + using DeltaArchiveManifest = chromeos_update_engine::DeltaArchiveManifest; + using MergeStatus = android::hardware::boot::V1_1::MergeStatus; + using FiemapStatus = android::fiemap::FiemapStatus; + + friend class SnapshotMergeStats; + + public: + ~SnapshotManager(); + + // Return a new SnapshotManager instance, or null on error. The device + // pointer is owned for the lifetime of SnapshotManager. If null, a default + // instance will be created. + static std::unique_ptr<SnapshotManager> New(IDeviceInfo* device = nullptr); + + // This is similar to New(), except designed specifically for first-stage + // init or recovery. + static std::unique_ptr<SnapshotManager> NewForFirstStageMount(IDeviceInfo* device = nullptr); + + // Helper function for first-stage init to check whether a SnapshotManager + // might be needed to perform first-stage mounts. + static bool IsSnapshotManagerNeeded(); + + // Helper function for second stage init to restorecon on the rollback indicator. + static std::string GetGlobalRollbackIndicatorPath(); + + // ISnapshotManager overrides. + bool BeginUpdate() override; + bool CancelUpdate() override; + bool FinishedSnapshotWrites(bool wipe) override; + bool InitiateMerge(uint64_t* cow_file_size = nullptr) override; + UpdateState ProcessUpdateState(const std::function<bool()>& callback = {}, + const std::function<bool()>& before_cancel = {}) override; + UpdateState GetUpdateState(double* progress = nullptr) override; + Return CreateUpdateSnapshots(const DeltaArchiveManifest& manifest) override; + bool MapUpdateSnapshot(const CreateLogicalPartitionParams& params, + std::string* snapshot_path) override; + bool UnmapUpdateSnapshot(const std::string& target_partition_name) override; + bool NeedSnapshotsInFirstStageMount() override; + bool CreateLogicalAndSnapshotPartitions( + const std::string& super_device, + const std::chrono::milliseconds& timeout_ms = {}) override; + bool HandleImminentDataWipe(const std::function<void()>& callback = {}) override; + bool FinishMergeInRecovery() override; + CreateResult RecoveryCreateSnapshotDevices() override; + CreateResult RecoveryCreateSnapshotDevices( + const std::unique_ptr<AutoDevice>& metadata_device) override; + bool Dump(std::ostream& os) override; + std::unique_ptr<AutoDevice> EnsureMetadataMounted() override; + ISnapshotMergeStats* GetSnapshotMergeStatsInstance() override; private: FRIEND_TEST(SnapshotTest, CleanFirstStageMount); @@ -285,6 +324,7 @@ class SnapshotManager final { friend class SnapshotUpdateTest; friend class FlashAfterUpdateTest; friend class LockTestConsumer; + friend class SnapshotFuzzEnv; friend struct AutoDeleteCowImage; friend struct AutoDeleteSnapshot; friend struct PartitionCowCreator; diff --git a/fs_mgr/libsnapshot/include/libsnapshot/snapshot_stats.h b/fs_mgr/libsnapshot/include/libsnapshot/snapshot_stats.h index bdc3ea325..d691d4f31 100644 --- a/fs_mgr/libsnapshot/include/libsnapshot/snapshot_stats.h +++ b/fs_mgr/libsnapshot/include/libsnapshot/snapshot_stats.h @@ -23,16 +23,14 @@ namespace android { namespace snapshot { -class SnapshotMergeStats { +class ISnapshotMergeStats { public: - // Not thread safe. - static SnapshotMergeStats* GetInstance(SnapshotManager& manager); - + virtual ~ISnapshotMergeStats() = default; // Called when merge starts or resumes. - bool Start(); - void set_state(android::snapshot::UpdateState state); - virtual void set_cow_file_size(uint64_t cow_file_size); - virtual uint64_t cow_file_size(); + virtual bool Start() = 0; + virtual void set_state(android::snapshot::UpdateState state) = 0; + virtual void set_cow_file_size(uint64_t cow_file_size) = 0; + virtual uint64_t cow_file_size() = 0; // Called when merge ends. Properly clean up permanent storage. class Result { @@ -42,11 +40,23 @@ class SnapshotMergeStats { // Time between successful Start() / Resume() to Finish(). virtual std::chrono::steady_clock::duration merge_time() const = 0; }; - std::unique_ptr<Result> Finish(); + // Return nullptr if any failure. + virtual std::unique_ptr<Result> Finish() = 0; +}; - private: - virtual ~SnapshotMergeStats() {} +class SnapshotMergeStats : public ISnapshotMergeStats { + public: + // Not thread safe. + static SnapshotMergeStats* GetInstance(SnapshotManager& manager); + // ISnapshotMergeStats overrides + bool Start() override; + void set_state(android::snapshot::UpdateState state) override; + void set_cow_file_size(uint64_t cow_file_size) override; + uint64_t cow_file_size() override; + std::unique_ptr<Result> Finish() override; + + private: bool ReadState(); bool WriteState(); bool DeleteState(); diff --git a/fs_mgr/libsnapshot/include/libsnapshot/snapshot_stub.h b/fs_mgr/libsnapshot/include/libsnapshot/snapshot_stub.h new file mode 100644 index 000000000..7a27fadca --- /dev/null +++ b/fs_mgr/libsnapshot/include/libsnapshot/snapshot_stub.h @@ -0,0 +1,53 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include <libsnapshot/snapshot.h> + +namespace android::snapshot { + +class SnapshotManagerStub : public ISnapshotManager { + public: + // Create a stubbed snapshot manager. All calls into the stub fails. + static std::unique_ptr<ISnapshotManager> New(); + + // ISnapshotManager overrides. + bool BeginUpdate() override; + bool CancelUpdate() override; + bool FinishedSnapshotWrites(bool wipe) override; + bool InitiateMerge(uint64_t* cow_file_size = nullptr) override; + UpdateState ProcessUpdateState(const std::function<bool()>& callback = {}, + const std::function<bool()>& before_cancel = {}) override; + UpdateState GetUpdateState(double* progress = nullptr) override; + Return CreateUpdateSnapshots( + const chromeos_update_engine::DeltaArchiveManifest& manifest) override; + bool MapUpdateSnapshot(const android::fs_mgr::CreateLogicalPartitionParams& params, + std::string* snapshot_path) override; + bool UnmapUpdateSnapshot(const std::string& target_partition_name) override; + bool NeedSnapshotsInFirstStageMount() override; + bool CreateLogicalAndSnapshotPartitions( + const std::string& super_device, + const std::chrono::milliseconds& timeout_ms = {}) override; + bool HandleImminentDataWipe(const std::function<void()>& callback = {}) override; + bool FinishMergeInRecovery() override; + CreateResult RecoveryCreateSnapshotDevices() override; + CreateResult RecoveryCreateSnapshotDevices( + const std::unique_ptr<AutoDevice>& metadata_device) override; + bool Dump(std::ostream& os) override; + std::unique_ptr<AutoDevice> EnsureMetadataMounted() override; + ISnapshotMergeStats* GetSnapshotMergeStatsInstance() override; +}; + +} // namespace android::snapshot diff --git a/fs_mgr/libsnapshot/partition_cow_creator_test.cpp b/fs_mgr/libsnapshot/partition_cow_creator_test.cpp index 526f8749a..adfb97519 100644 --- a/fs_mgr/libsnapshot/partition_cow_creator_test.cpp +++ b/fs_mgr/libsnapshot/partition_cow_creator_test.cpp @@ -46,20 +46,20 @@ class PartitionCowCreatorTest : public ::testing::Test { }; TEST_F(PartitionCowCreatorTest, IntersectSelf) { - constexpr uint64_t initial_size = 1_MiB; - constexpr uint64_t final_size = 40_KiB; + constexpr uint64_t super_size = 1_MiB; + constexpr uint64_t partition_size = 40_KiB; - auto builder_a = MetadataBuilder::New(initial_size, 1_KiB, 2); + auto builder_a = MetadataBuilder::New(super_size, 1_KiB, 2); ASSERT_NE(builder_a, nullptr); auto system_a = builder_a->AddPartition("system_a", LP_PARTITION_ATTR_READONLY); ASSERT_NE(system_a, nullptr); - ASSERT_TRUE(builder_a->ResizePartition(system_a, final_size)); + ASSERT_TRUE(builder_a->ResizePartition(system_a, partition_size)); - auto builder_b = MetadataBuilder::New(initial_size, 1_KiB, 2); + auto builder_b = MetadataBuilder::New(super_size, 1_KiB, 2); ASSERT_NE(builder_b, nullptr); auto system_b = builder_b->AddPartition("system_b", LP_PARTITION_ATTR_READONLY); ASSERT_NE(system_b, nullptr); - ASSERT_TRUE(builder_b->ResizePartition(system_b, final_size)); + ASSERT_TRUE(builder_b->ResizePartition(system_b, partition_size)); PartitionCowCreator creator{.target_metadata = builder_b.get(), .target_suffix = "_b", @@ -68,8 +68,8 @@ TEST_F(PartitionCowCreatorTest, IntersectSelf) { .current_suffix = "_a"}; auto ret = creator.Run(); ASSERT_TRUE(ret.has_value()); - ASSERT_EQ(final_size, ret->snapshot_status.device_size()); - ASSERT_EQ(final_size, ret->snapshot_status.snapshot_size()); + ASSERT_EQ(partition_size, ret->snapshot_status.device_size()); + ASSERT_EQ(partition_size, ret->snapshot_status.snapshot_size()); } TEST_F(PartitionCowCreatorTest, Holes) { @@ -118,20 +118,20 @@ TEST_F(PartitionCowCreatorTest, CowSize) { using RepeatedInstallOperationPtr = google::protobuf::RepeatedPtrField<InstallOperation>; using Extent = chromeos_update_engine::Extent; - constexpr uint64_t initial_size = 50_MiB; - constexpr uint64_t final_size = 40_MiB; + constexpr uint64_t super_size = 50_MiB; + constexpr uint64_t partition_size = 40_MiB; - auto builder_a = MetadataBuilder::New(initial_size, 1_KiB, 2); + auto builder_a = MetadataBuilder::New(super_size, 1_KiB, 2); ASSERT_NE(builder_a, nullptr); auto system_a = builder_a->AddPartition("system_a", LP_PARTITION_ATTR_READONLY); ASSERT_NE(system_a, nullptr); - ASSERT_TRUE(builder_a->ResizePartition(system_a, final_size)); + ASSERT_TRUE(builder_a->ResizePartition(system_a, partition_size)); - auto builder_b = MetadataBuilder::New(initial_size, 1_KiB, 2); + auto builder_b = MetadataBuilder::New(super_size, 1_KiB, 2); ASSERT_NE(builder_b, nullptr); auto system_b = builder_b->AddPartition("system_b", LP_PARTITION_ATTR_READONLY); ASSERT_NE(system_b, nullptr); - ASSERT_TRUE(builder_b->ResizePartition(system_b, final_size)); + ASSERT_TRUE(builder_b->ResizePartition(system_b, partition_size)); const uint64_t block_size = builder_b->logical_block_size(); const uint64_t chunk_size = kSnapshotChunkSize * dm::kSectorSize; @@ -197,6 +197,31 @@ TEST_F(PartitionCowCreatorTest, CowSize) { ASSERT_EQ(6 * chunk_size, cow_device_size(iopv, builder_a.get(), builder_b.get(), system_b)); } +TEST_F(PartitionCowCreatorTest, Zero) { + constexpr uint64_t super_size = 1_MiB; + auto builder_a = MetadataBuilder::New(super_size, 1_KiB, 2); + ASSERT_NE(builder_a, nullptr); + + auto builder_b = MetadataBuilder::New(super_size, 1_KiB, 2); + ASSERT_NE(builder_b, nullptr); + auto system_b = builder_b->AddPartition("system_b", LP_PARTITION_ATTR_READONLY); + ASSERT_NE(system_b, nullptr); + + PartitionCowCreator creator{.target_metadata = builder_b.get(), + .target_suffix = "_b", + .target_partition = system_b, + .current_metadata = builder_a.get(), + .current_suffix = "_a", + .operations = nullptr}; + + auto ret = creator.Run(); + + ASSERT_EQ(0u, ret->snapshot_status.device_size()); + ASSERT_EQ(0u, ret->snapshot_status.snapshot_size()); + ASSERT_EQ(0u, ret->snapshot_status.cow_file_size()); + ASSERT_EQ(0u, ret->snapshot_status.cow_partition_size()); +} + TEST(DmSnapshotInternals, CowSizeCalculator) { DmSnapCowSizeCalculator cc(512, 8); unsigned long int b; diff --git a/fs_mgr/libsnapshot/power_test.cpp b/fs_mgr/libsnapshot/power_test.cpp new file mode 100644 index 000000000..4d2548ae9 --- /dev/null +++ b/fs_mgr/libsnapshot/power_test.cpp @@ -0,0 +1,559 @@ +// +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include <errno.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> + +#include <chrono> +#include <iostream> +#include <random> +#include <string> +#include <thread> +#include <vector> + +#include <android-base/file.h> +#include <android-base/parsedouble.h> +#include <android-base/stringprintf.h> +#include <android-base/strings.h> +#include <android-base/unique_fd.h> +#include <ext4_utils/ext4_utils.h> +#include <fstab/fstab.h> +#include <libdm/dm.h> +#include <libfiemap/image_manager.h> + +using namespace std::chrono_literals; +using namespace std::string_literals; +using android::base::borrowed_fd; +using android::base::unique_fd; +using android::dm::DeviceMapper; +using android::dm::DmDeviceState; +using android::dm::DmTable; +using android::dm::DmTargetSnapshot; +using android::dm::SnapshotStorageMode; +using android::fiemap::ImageManager; +using android::fs_mgr::Fstab; + +namespace android { +namespace snapshot { + +static void usage() { + std::cerr << "Usage:\n"; + std::cerr << " create <orig-payload> <new-payload>\n"; + std::cerr << "\n"; + std::cerr << " Create a snapshot device containing the contents of\n"; + std::cerr << " orig-payload, and then write the contents of new-payload.\n"; + std::cerr << " The original files are not modified.\n"; + std::cerr << "\n"; + std::cerr << " merge <fail-rate>\n"; + std::cerr << "\n"; + std::cerr << " Merge the snapshot previously started by create, and wait\n"; + std::cerr << " for it to complete. Once done, it is compared to the\n"; + std::cerr << " new-payload for consistency. The original files are not \n"; + std::cerr << " modified. If a fail-rate is passed (as a fraction between 0\n"; + std::cerr << " and 100), every 10ms the device has that percent change of\n"; + std::cerr << " injecting a kernel crash.\n"; + std::cerr << "\n"; + std::cerr << " check <new-payload>\n"; + std::cerr << " Verify that all artifacts are correct after a merge\n"; + std::cerr << " completes.\n"; + std::cerr << "\n"; + std::cerr << " cleanup\n"; + std::cerr << " Remove all ImageManager artifacts from create/merge.\n"; +} + +class PowerTest final { + public: + PowerTest(); + bool Run(int argc, char** argv); + + private: + bool OpenImageManager(); + bool Create(int argc, char** argv); + bool Merge(int argc, char** argv); + bool Check(int argc, char** argv); + bool Cleanup(); + bool CleanupImage(const std::string& name); + bool SetupImages(const std::string& first_file, borrowed_fd second_fd); + bool MapImages(); + bool MapSnapshot(SnapshotStorageMode mode); + bool GetMergeStatus(DmTargetSnapshot::Status* status); + + static constexpr char kSnapshotName[] = "snapshot-power-test"; + static constexpr char kSnapshotImageName[] = "snapshot-power-test-image"; + static constexpr char kSnapshotCowName[] = "snapshot-power-test-cow"; + + DeviceMapper& dm_; + std::unique_ptr<ImageManager> images_; + std::string image_path_; + std::string cow_path_; + std::string snapshot_path_; +}; + +PowerTest::PowerTest() : dm_(DeviceMapper::Instance()) {} + +bool PowerTest::Run([[maybe_unused]] int argc, [[maybe_unused]] char** argv) { + if (!OpenImageManager()) { + return false; + } + + if (argc < 2) { + usage(); + return false; + } + if (argv[1] == "create"s) { + return Create(argc, argv); + } else if (argv[1] == "merge"s) { + return Merge(argc, argv); + } else if (argv[1] == "check"s) { + return Check(argc, argv); + } else if (argv[1] == "cleanup"s) { + return Cleanup(); + } else { + usage(); + return false; + } +} + +bool PowerTest::OpenImageManager() { + std::vector<std::string> dirs = { + "/data/gsi/test", + "/metadata/gsi/test", + }; + for (const auto& dir : dirs) { + if (mkdir(dir.c_str(), 0700) && errno != EEXIST) { + std::cerr << "mkdir " << dir << ": " << strerror(errno) << "\n"; + return false; + } + } + + images_ = ImageManager::Open("/metadata/gsi/test", "/data/gsi/test"); + if (!images_) { + std::cerr << "Could not open ImageManager\n"; + return false; + } + return true; +} + +bool PowerTest::Create(int argc, char** argv) { + if (argc < 4) { + usage(); + return false; + } + + std::string first = argv[2]; + std::string second = argv[3]; + + unique_fd second_fd(open(second.c_str(), O_RDONLY)); + if (second_fd < 0) { + std::cerr << "open " << second << ": " << strerror(errno) << "\n"; + return false; + } + + if (!Cleanup()) { + return false; + } + if (!SetupImages(first, second_fd)) { + return false; + } + if (!MapSnapshot(SnapshotStorageMode::Persistent)) { + return false; + } + + struct stat s; + if (fstat(second_fd, &s)) { + std::cerr << "fstat " << second << ": " << strerror(errno) << "\n"; + return false; + } + + unique_fd snap_fd(open(snapshot_path_.c_str(), O_WRONLY)); + if (snap_fd < 0) { + std::cerr << "open " << snapshot_path_ << ": " << strerror(errno) << "\n"; + return false; + } + + uint8_t chunk[4096]; + uint64_t written = 0; + while (written < s.st_size) { + uint64_t remaining = s.st_size - written; + size_t bytes = (size_t)std::min((uint64_t)sizeof(chunk), remaining); + if (!android::base::ReadFully(second_fd, chunk, bytes)) { + std::cerr << "read " << second << ": " << strerror(errno) << "\n"; + return false; + } + if (!android::base::WriteFully(snap_fd, chunk, bytes)) { + std::cerr << "write " << snapshot_path_ << ": " << strerror(errno) << "\n"; + return false; + } + written += bytes; + } + if (fsync(snap_fd)) { + std::cerr << "fsync: " << strerror(errno) << "\n"; + return false; + } + + sync(); + + snap_fd = {}; + if (!dm_.DeleteDeviceIfExists(kSnapshotName)) { + std::cerr << "could not delete dm device " << kSnapshotName << "\n"; + return false; + } + if (!images_->UnmapImageIfExists(kSnapshotImageName)) { + std::cerr << "failed to unmap " << kSnapshotImageName << "\n"; + return false; + } + if (!images_->UnmapImageIfExists(kSnapshotCowName)) { + std::cerr << "failed to unmap " << kSnapshotImageName << "\n"; + return false; + } + return true; +} + +bool PowerTest::Cleanup() { + if (!dm_.DeleteDeviceIfExists(kSnapshotName)) { + std::cerr << "could not delete dm device " << kSnapshotName << "\n"; + return false; + } + if (!CleanupImage(kSnapshotImageName) || !CleanupImage(kSnapshotCowName)) { + return false; + } + return true; +} + +bool PowerTest::CleanupImage(const std::string& name) { + if (!images_->UnmapImageIfExists(name)) { + std::cerr << "failed to unmap " << name << "\n"; + return false; + } + if (images_->BackingImageExists(name) && !images_->DeleteBackingImage(name)) { + std::cerr << "failed to delete " << name << "\n"; + return false; + } + return true; +} + +bool PowerTest::SetupImages(const std::string& first, borrowed_fd second_fd) { + unique_fd first_fd(open(first.c_str(), O_RDONLY)); + if (first_fd < 0) { + std::cerr << "open " << first << ": " << strerror(errno) << "\n"; + return false; + } + + struct stat s1, s2; + if (fstat(first_fd.get(), &s1)) { + std::cerr << "first stat: " << strerror(errno) << "\n"; + return false; + } + if (fstat(second_fd.get(), &s2)) { + std::cerr << "second stat: " << strerror(errno) << "\n"; + return false; + } + + // Pick the bigger size of both images, rounding up to the nearest block. + uint64_t s1_size = (s1.st_size + 4095) & ~uint64_t(4095); + uint64_t s2_size = (s2.st_size + 4095) & ~uint64_t(4095); + uint64_t image_size = std::max(s1_size, s2_size) + (1024 * 1024 * 128); + if (!images_->CreateBackingImage(kSnapshotImageName, image_size, 0, nullptr)) { + std::cerr << "failed to create " << kSnapshotImageName << "\n"; + return false; + } + // Use the same size for the cow. + if (!images_->CreateBackingImage(kSnapshotCowName, image_size, 0, nullptr)) { + std::cerr << "failed to create " << kSnapshotCowName << "\n"; + return false; + } + if (!MapImages()) { + return false; + } + + unique_fd image_fd(open(image_path_.c_str(), O_WRONLY)); + if (image_fd < 0) { + std::cerr << "open: " << image_path_ << ": " << strerror(errno) << "\n"; + return false; + } + + uint8_t chunk[4096]; + uint64_t written = 0; + while (written < s1.st_size) { + uint64_t remaining = s1.st_size - written; + size_t bytes = (size_t)std::min((uint64_t)sizeof(chunk), remaining); + if (!android::base::ReadFully(first_fd, chunk, bytes)) { + std::cerr << "read: " << strerror(errno) << "\n"; + return false; + } + if (!android::base::WriteFully(image_fd, chunk, bytes)) { + std::cerr << "write: " << strerror(errno) << "\n"; + return false; + } + written += bytes; + } + if (fsync(image_fd)) { + std::cerr << "fsync: " << strerror(errno) << "\n"; + return false; + } + + // Zero the first block of the COW. + unique_fd cow_fd(open(cow_path_.c_str(), O_WRONLY)); + if (cow_fd < 0) { + std::cerr << "open: " << cow_path_ << ": " << strerror(errno) << "\n"; + return false; + } + + memset(chunk, 0, sizeof(chunk)); + if (!android::base::WriteFully(cow_fd, chunk, sizeof(chunk))) { + std::cerr << "read: " << strerror(errno) << "\n"; + return false; + } + if (fsync(cow_fd)) { + std::cerr << "fsync: " << strerror(errno) << "\n"; + return false; + } + return true; +} + +bool PowerTest::MapImages() { + if (!images_->MapImageDevice(kSnapshotImageName, 10s, &image_path_)) { + std::cerr << "failed to map " << kSnapshotImageName << "\n"; + return false; + } + if (!images_->MapImageDevice(kSnapshotCowName, 10s, &cow_path_)) { + std::cerr << "failed to map " << kSnapshotCowName << "\n"; + return false; + } + return true; +} + +bool PowerTest::MapSnapshot(SnapshotStorageMode mode) { + uint64_t sectors; + { + unique_fd fd(open(image_path_.c_str(), O_RDONLY)); + if (fd < 0) { + std::cerr << "open: " << image_path_ << ": " << strerror(errno) << "\n"; + return false; + } + sectors = get_block_device_size(fd) / 512; + } + + DmTable table; + table.Emplace<DmTargetSnapshot>(0, sectors, image_path_, cow_path_, mode, 8); + if (!dm_.CreateDevice(kSnapshotName, table, &snapshot_path_, 10s)) { + std::cerr << "failed to create snapshot device\n"; + return false; + } + return true; +} + +bool PowerTest::GetMergeStatus(DmTargetSnapshot::Status* status) { + std::vector<DeviceMapper::TargetInfo> targets; + if (!dm_.GetTableStatus(kSnapshotName, &targets)) { + std::cerr << "failed to get merge status\n"; + return false; + } + if (targets.size() != 1) { + std::cerr << "merge device has wrong number of targets\n"; + return false; + } + if (!DmTargetSnapshot::ParseStatusText(targets[0].data, status)) { + std::cerr << "could not parse merge target status text\n"; + return false; + } + return true; +} + +static std::string GetUserdataBlockDeviceName() { + Fstab fstab; + if (!ReadFstabFromFile("/proc/mounts", &fstab)) { + return {}; + } + + auto entry = android::fs_mgr::GetEntryForMountPoint(&fstab, "/data"); + if (!entry) { + return {}; + } + + auto prefix = "/dev/block/"s; + if (!android::base::StartsWith(entry->blk_device, prefix)) { + return {}; + } + return entry->blk_device.substr(prefix.size()); +} + +bool PowerTest::Merge(int argc, char** argv) { + // Start an f2fs GC to really stress things. :TODO: figure out data device + auto userdata_dev = GetUserdataBlockDeviceName(); + if (userdata_dev.empty()) { + std::cerr << "could not locate userdata block device\n"; + return false; + } + + auto cmd = + android::base::StringPrintf("echo 1 > /sys/fs/f2fs/%s/gc_urgent", userdata_dev.c_str()); + system(cmd.c_str()); + + if (dm_.GetState(kSnapshotName) == DmDeviceState::INVALID) { + if (!MapImages()) { + return false; + } + if (!MapSnapshot(SnapshotStorageMode::Merge)) { + return false; + } + } + + std::random_device r; + std::default_random_engine re(r()); + std::uniform_real_distribution<double> dist(0.0, 100.0); + + std::optional<double> failure_rate; + if (argc >= 3) { + double d; + if (!android::base::ParseDouble(argv[2], &d)) { + std::cerr << "Could not parse failure rate as double: " << argv[2] << "\n"; + return false; + } + failure_rate = d; + } + + while (true) { + DmTargetSnapshot::Status status; + if (!GetMergeStatus(&status)) { + return false; + } + if (!status.error.empty()) { + std::cerr << "merge reported error: " << status.error << "\n"; + return false; + } + if (status.sectors_allocated == status.metadata_sectors) { + break; + } + + std::cerr << status.sectors_allocated << " / " << status.metadata_sectors << "\n"; + + if (failure_rate && *failure_rate >= dist(re)) { + system("echo 1 > /proc/sys/kernel/sysrq"); + system("echo c > /proc/sysrq-trigger"); + } + + std::this_thread::sleep_for(10ms); + } + + std::cout << "Merge completed.\n"; + return true; +} + +bool PowerTest::Check([[maybe_unused]] int argc, [[maybe_unused]] char** argv) { + if (argc < 3) { + std::cerr << "Expected argument: <new-image-path>\n"; + return false; + } + std::string md_path, image_path; + std::string canonical_path = argv[2]; + + if (!dm_.GetDmDevicePathByName(kSnapshotName, &md_path)) { + std::cerr << "could not get dm-path for merge device\n"; + return false; + } + if (!images_->GetMappedImageDevice(kSnapshotImageName, &image_path)) { + std::cerr << "could not get image path\n"; + return false; + } + + unique_fd md_fd(open(md_path.c_str(), O_RDONLY)); + if (md_fd < 0) { + std::cerr << "open: " << md_path << ": " << strerror(errno) << "\n"; + return false; + } + unique_fd image_fd(open(image_path.c_str(), O_RDONLY)); + if (image_fd < 0) { + std::cerr << "open: " << image_path << ": " << strerror(errno) << "\n"; + return false; + } + unique_fd canonical_fd(open(canonical_path.c_str(), O_RDONLY)); + if (canonical_fd < 0) { + std::cerr << "open: " << canonical_path << ": " << strerror(errno) << "\n"; + return false; + } + + struct stat s; + if (fstat(canonical_fd, &s)) { + std::cerr << "fstat: " << canonical_path << ": " << strerror(errno) << "\n"; + return false; + } + uint64_t canonical_size = s.st_size; + uint64_t md_size = get_block_device_size(md_fd); + uint64_t image_size = get_block_device_size(image_fd); + if (image_size != md_size) { + std::cerr << "image size does not match merge device size\n"; + return false; + } + if (canonical_size > image_size) { + std::cerr << "canonical size " << canonical_size << " is greater than image size " + << image_size << "\n"; + return false; + } + + constexpr size_t kBlockSize = 4096; + uint8_t canonical_buffer[kBlockSize]; + uint8_t image_buffer[kBlockSize]; + uint8_t md_buffer[kBlockSize]; + + uint64_t remaining = canonical_size; + uint64_t blockno = 0; + while (remaining) { + size_t bytes = (size_t)std::min((uint64_t)kBlockSize, remaining); + if (!android::base::ReadFully(canonical_fd, canonical_buffer, bytes)) { + std::cerr << "read: " << canonical_buffer << ": " << strerror(errno) << "\n"; + return false; + } + if (!android::base::ReadFully(image_fd, image_buffer, bytes)) { + std::cerr << "read: " << image_buffer << ": " << strerror(errno) << "\n"; + return false; + } + if (!android::base::ReadFully(md_fd, md_buffer, bytes)) { + std::cerr << "read: " << md_buffer << ": " << strerror(errno) << "\n"; + return false; + } + if (memcmp(canonical_buffer, image_buffer, bytes)) { + std::cerr << "canonical and image differ at block " << blockno << "\n"; + return false; + } + if (memcmp(canonical_buffer, md_buffer, bytes)) { + std::cerr << "canonical and image differ at block " << blockno << "\n"; + return false; + } + + remaining -= bytes; + blockno++; + } + + std::cout << "Images all match.\n"; + return true; +} + +} // namespace snapshot +} // namespace android + +int main(int argc, char** argv) { + android::snapshot::PowerTest test; + + if (!test.Run(argc, argv)) { + std::cerr << "Unexpected error running test." << std::endl; + return 1; + } + fflush(stdout); + return 0; +} diff --git a/fs_mgr/libsnapshot/run_power_test.sh b/fs_mgr/libsnapshot/run_power_test.sh new file mode 100755 index 000000000..dc03dc975 --- /dev/null +++ b/fs_mgr/libsnapshot/run_power_test.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +set -e + +if [ -z "$FAIL_RATE" ]; then + FAIL_RATE=5.0 +fi +if [ ! -z "$ANDROID_SERIAL" ]; then + DEVICE_ARGS=-s $ANDROID_SERIAL +else + DEVICE_ARGS= +fi + +TEST_BIN=/data/nativetest64/snapshot_power_test/snapshot_power_test + +while : +do + adb $DEVICE_ARGS wait-for-device + adb $DEVICE_ARGS root + adb $DEVICE_ARGS shell rm $TEST_BIN + adb $DEVICE_ARGS sync data + set +e + output=$(adb $DEVICE_ARGS shell $TEST_BIN merge $FAIL_RATE 2>&1) + set -e + if [[ "$output" == *"Merge completed"* ]]; then + echo "Merge completed." + break + fi + if [[ "$output" == *"Unexpected error"* ]]; then + echo "Unexpected error." + exit 1 + fi +done + +adb $DEVICE_ARGS shell $TEST_BIN check $1 diff --git a/fs_mgr/libsnapshot/snapshot.cpp b/fs_mgr/libsnapshot/snapshot.cpp index 0739fab74..55214f545 100644 --- a/fs_mgr/libsnapshot/snapshot.cpp +++ b/fs_mgr/libsnapshot/snapshot.cpp @@ -2277,6 +2277,10 @@ Return SnapshotManager::CreateUpdateSnapshotsInternal( auto operations_it = install_operation_map.find(target_partition->name()); if (operations_it != install_operation_map.end()) { cow_creator->operations = operations_it->second; + } else { + LOG(INFO) << target_partition->name() + << " isn't included in the payload, skipping the cow creation."; + continue; } cow_creator->extra_extents.clear(); @@ -2748,6 +2752,10 @@ bool SnapshotManager::UpdateForwardMergeIndicator(bool wipe) { return true; } +ISnapshotMergeStats* SnapshotManager::GetSnapshotMergeStatsInstance() { + return SnapshotMergeStats::GetInstance(*this); +} + bool SnapshotManager::GetMappedImageDeviceStringOrPath(const std::string& device_name, std::string* device_string_or_mapped_path) { auto& dm = DeviceMapper::Instance(); diff --git a/fs_mgr/libsnapshot/snapshot_fuzz.cpp b/fs_mgr/libsnapshot/snapshot_fuzz.cpp new file mode 100644 index 000000000..5b145c31f --- /dev/null +++ b/fs_mgr/libsnapshot/snapshot_fuzz.cpp @@ -0,0 +1,352 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <stddef.h> +#include <stdint.h> +#include <sysexits.h> + +#include <functional> +#include <sstream> +#include <tuple> + +#include <android-base/logging.h> +#include <android-base/properties.h> +#include <android-base/result.h> +#include <gtest/gtest.h> +#include <src/libfuzzer/libfuzzer_macro.h> +#include <storage_literals/storage_literals.h> + +#include "fuzz_utils.h" +#include "snapshot_fuzz_utils.h" + +using android::base::Error; +using android::base::GetBoolProperty; +using android::base::LogId; +using android::base::LogSeverity; +using android::base::ReadFileToString; +using android::base::Result; +using android::base::SetLogger; +using android::base::StderrLogger; +using android::base::StdioLogger; +using android::fs_mgr::CreateLogicalPartitionParams; +using android::fuzz::CheckedCast; +using android::snapshot::SnapshotFuzzData; +using android::snapshot::SnapshotFuzzEnv; +using chromeos_update_engine::DeltaArchiveManifest; +using google::protobuf::FieldDescriptor; +using google::protobuf::Message; +using google::protobuf::RepeatedPtrField; + +// Avoid linking to libgsi since it needs disk I/O. +namespace android::gsi { +bool IsGsiRunning() { + LOG(FATAL) << "Called IsGsiRunning"; + __builtin_unreachable(); +} +std::string GetDsuSlot(const std::string& install_dir) { + LOG(FATAL) << "Called GetDsuSlot(" << install_dir << ")"; + __builtin_unreachable(); +} +} // namespace android::gsi + +namespace android::snapshot { + +const SnapshotFuzzData* current_data = nullptr; +const SnapshotTestModule* current_module = nullptr; + +SnapshotFuzzEnv* GetSnapshotFuzzEnv(); + +FUZZ_CLASS(ISnapshotManager, SnapshotManagerAction); + +using ProcessUpdateStateArgs = SnapshotManagerAction::Proto::ProcessUpdateStateArgs; +using CreateLogicalAndSnapshotPartitionsArgs = + SnapshotManagerAction::Proto::CreateLogicalAndSnapshotPartitionsArgs; +using RecoveryCreateSnapshotDevicesArgs = + SnapshotManagerAction::Proto::RecoveryCreateSnapshotDevicesArgs; + +FUZZ_SIMPLE_FUNCTION(SnapshotManagerAction, BeginUpdate); +FUZZ_SIMPLE_FUNCTION(SnapshotManagerAction, CancelUpdate); +FUZZ_SIMPLE_FUNCTION(SnapshotManagerAction, InitiateMerge); +FUZZ_SIMPLE_FUNCTION(SnapshotManagerAction, NeedSnapshotsInFirstStageMount); +FUZZ_SIMPLE_FUNCTION(SnapshotManagerAction, RecoveryCreateSnapshotDevices); +FUZZ_SIMPLE_FUNCTION(SnapshotManagerAction, EnsureMetadataMounted); +FUZZ_SIMPLE_FUNCTION(SnapshotManagerAction, GetSnapshotMergeStatsInstance); + +#define SNAPSHOT_FUZZ_FUNCTION(FunctionName, ReturnType, ...) \ + FUZZ_FUNCTION(SnapshotManagerAction, FunctionName, ReturnType, ISnapshotManager* snapshot, \ + ##__VA_ARGS__) + +SNAPSHOT_FUZZ_FUNCTION(FinishedSnapshotWrites, bool, bool wipe) { + return snapshot->FinishedSnapshotWrites(wipe); +} + +SNAPSHOT_FUZZ_FUNCTION(ProcessUpdateState, bool, const ProcessUpdateStateArgs& args) { + std::function<bool()> before_cancel; + if (args.has_before_cancel()) { + before_cancel = [&]() { return args.fail_before_cancel(); }; + } + return snapshot->ProcessUpdateState({}, before_cancel); +} + +SNAPSHOT_FUZZ_FUNCTION(GetUpdateState, UpdateState, bool has_progress_arg) { + double progress; + return snapshot->GetUpdateState(has_progress_arg ? &progress : nullptr); +} + +SNAPSHOT_FUZZ_FUNCTION(HandleImminentDataWipe, bool, bool has_callback) { + std::function<void()> callback; + if (has_callback) { + callback = []() {}; + } + return snapshot->HandleImminentDataWipe(callback); +} + +SNAPSHOT_FUZZ_FUNCTION(Dump, bool) { + std::stringstream ss; + return snapshot->Dump(ss); +} + +SNAPSHOT_FUZZ_FUNCTION(CreateUpdateSnapshots, bool, const DeltaArchiveManifest& manifest) { + return snapshot->CreateUpdateSnapshots(manifest); +} + +SNAPSHOT_FUZZ_FUNCTION(UnmapUpdateSnapshot, bool, const std::string& name) { + return snapshot->UnmapUpdateSnapshot(name); +} + +SNAPSHOT_FUZZ_FUNCTION(CreateLogicalAndSnapshotPartitions, bool, + const CreateLogicalAndSnapshotPartitionsArgs& args) { + const std::string* super; + if (args.use_correct_super()) { + super = &GetSnapshotFuzzEnv()->super(); + } else { + super = &args.super(); + } + return snapshot->CreateLogicalAndSnapshotPartitions( + *super, std::chrono::milliseconds(args.timeout_millis())); +} + +SNAPSHOT_FUZZ_FUNCTION(RecoveryCreateSnapshotDevicesWithMetadata, CreateResult, + const RecoveryCreateSnapshotDevicesArgs& args) { + std::unique_ptr<AutoDevice> device; + if (args.has_metadata_device_object()) { + device = std::make_unique<DummyAutoDevice>(args.metadata_mounted()); + } + return snapshot->RecoveryCreateSnapshotDevices(device); +} + +SNAPSHOT_FUZZ_FUNCTION(MapUpdateSnapshot, bool, + const CreateLogicalPartitionParamsProto& params_proto) { + auto partition_opener = std::make_unique<TestPartitionOpener>(GetSnapshotFuzzEnv()->super()); + CreateLogicalPartitionParams params; + if (params_proto.use_correct_super()) { + params.block_device = GetSnapshotFuzzEnv()->super(); + } else { + params.block_device = params_proto.block_device(); + } + if (params_proto.has_metadata_slot()) { + params.metadata_slot = params_proto.metadata_slot(); + } + params.partition_name = params_proto.partition_name(); + params.force_writable = params_proto.force_writable(); + params.timeout_ms = std::chrono::milliseconds(params_proto.timeout_millis()); + params.device_name = params_proto.device_name(); + params.partition_opener = partition_opener.get(); + std::string path; + return snapshot->MapUpdateSnapshot(params, &path); +} + +SNAPSHOT_FUZZ_FUNCTION(SwitchSlot, void) { + (void)snapshot; + CHECK(current_module != nullptr); + CHECK(current_module->device_info != nullptr); + current_module->device_info->SwitchSlot(); +} + +// During global init, log all messages to stdio. This is only done once. +int AllowLoggingDuringGlobalInit() { + SetLogger(&StdioLogger); + return 0; +} + +// Only log fatal messages during tests. +void FatalOnlyLogger(LogId logid, LogSeverity severity, const char* tag, const char* file, + unsigned int line, const char* message) { + if (severity == LogSeverity::FATAL) { + StderrLogger(logid, severity, tag, file, line, message); + + // If test fails by a LOG(FATAL) or CHECK(), log the corpus. If it abort()'s, there's + // nothing else we can do. + StderrLogger(logid, severity, tag, __FILE__, __LINE__, + "Attempting to dump current corpus:"); + if (current_data == nullptr) { + StderrLogger(logid, severity, tag, __FILE__, __LINE__, "Current corpus is nullptr."); + } else { + std::string content; + if (!google::protobuf::TextFormat::PrintToString(*current_data, &content)) { + StderrLogger(logid, severity, tag, __FILE__, __LINE__, + "Failed to print corpus to string."); + } else { + StderrLogger(logid, severity, tag, __FILE__, __LINE__, content.c_str()); + } + } + } +} +// Stop logging (except fatal messages) after global initialization. This is only done once. +int StopLoggingAfterGlobalInit() { + (void)GetSnapshotFuzzEnv(); + [[maybe_unused]] static protobuf_mutator::protobuf::LogSilencer log_silencer; + SetLogger(&FatalOnlyLogger); + return 0; +} + +SnapshotFuzzEnv* GetSnapshotFuzzEnv() { + [[maybe_unused]] static auto allow_logging = AllowLoggingDuringGlobalInit(); + static SnapshotFuzzEnv env; + return &env; +} + +SnapshotTestModule SetUpTest(const SnapshotFuzzData& snapshot_fuzz_data) { + current_data = &snapshot_fuzz_data; + + auto env = GetSnapshotFuzzEnv(); + env->CheckSoftReset(); + + auto test_module = env->CheckCreateSnapshotManager(snapshot_fuzz_data); + current_module = &test_module; + CHECK(test_module.snapshot); + return test_module; +} + +void TearDownTest() { + current_module = nullptr; + current_data = nullptr; +} + +} // namespace android::snapshot + +DEFINE_PROTO_FUZZER(const SnapshotFuzzData& snapshot_fuzz_data) { + using namespace android::snapshot; + + [[maybe_unused]] static auto stop_logging = StopLoggingAfterGlobalInit(); + auto test_module = SetUpTest(snapshot_fuzz_data); + SnapshotManagerAction::ExecuteAll(test_module.snapshot.get(), snapshot_fuzz_data.actions()); + TearDownTest(); +} + +namespace android::snapshot { + +// Work-around to cast a 'void' value to Result<void>. +template <typename T> +struct GoodResult { + template <typename F> + static Result<T> Cast(F&& f) { + return f(); + } +}; + +template <> +struct GoodResult<void> { + template <typename F> + static Result<void> Cast(F&& f) { + f(); + return {}; + } +}; + +class LibsnapshotFuzzerTest : public ::testing::Test { + protected: + static void SetUpTestCase() { + // Do initialization once. + (void)GetSnapshotFuzzEnv(); + } + void SetUp() override { + bool is_virtual_ab = GetBoolProperty("ro.virtual_ab.enabled", false); + if (!is_virtual_ab) GTEST_SKIP() << "Test only runs on Virtual A/B devices."; + } + void SetUpFuzzData(const std::string& fn) { + auto path = android::base::GetExecutableDirectory() + "/corpus/"s + fn; + std::string proto_text; + ASSERT_TRUE(ReadFileToString(path, &proto_text)); + snapshot_fuzz_data_ = std::make_unique<SnapshotFuzzData>(); + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(proto_text, + snapshot_fuzz_data_.get())); + test_module_ = android::snapshot::SetUpTest(*snapshot_fuzz_data_); + } + void TearDown() override { android::snapshot::TearDownTest(); } + template <typename FuzzFunction> + Result<typename FuzzFunction::ReturnType> Execute(int action_index) { + if (action_index >= snapshot_fuzz_data_->actions_size()) { + return Error() << "Index " << action_index << " is out of bounds (" + << snapshot_fuzz_data_->actions_size() << " actions in corpus"; + } + const auto& action_proto = snapshot_fuzz_data_->actions(action_index); + const auto* field_desc = + android::fuzz::GetValueFieldDescriptor<typename FuzzFunction::ActionType>( + action_proto); + if (field_desc == nullptr) { + return Error() << "Action at index " << action_index << " has no value defined."; + } + if (FuzzFunction::tag != field_desc->number()) { + return Error() << "Action at index " << action_index << " is expected to be " + << FuzzFunction::name << ", but it is " << field_desc->name() + << " in corpus."; + } + return GoodResult<typename FuzzFunction::ReturnType>::Cast([&]() { + return android::fuzz::ActionPerformer<FuzzFunction>::Invoke(test_module_.snapshot.get(), + action_proto, field_desc); + }); + } + + std::unique_ptr<SnapshotFuzzData> snapshot_fuzz_data_; + SnapshotTestModule test_module_; +}; + +#define SNAPSHOT_FUZZ_FN_NAME(name) FUZZ_FUNCTION_CLASS_NAME(SnapshotManagerAction, name) + +MATCHER_P(ResultIs, expected, "") { + if (!arg.ok()) { + *result_listener << arg.error(); + return false; + } + *result_listener << "expected: " << expected; + return arg.value() == expected; +} + +#define ASSERT_RESULT_TRUE(actual) ASSERT_THAT(actual, ResultIs(true)) + +// Check that launch_device.txt is executed correctly. +TEST_F(LibsnapshotFuzzerTest, LaunchDevice) { + SetUpFuzzData("launch_device.txt"); + + int i = 0; + ASSERT_RESULT_TRUE(Execute<SNAPSHOT_FUZZ_FN_NAME(BeginUpdate)>(i++)); + ASSERT_RESULT_TRUE(Execute<SNAPSHOT_FUZZ_FN_NAME(CreateUpdateSnapshots)>(i++)); + ASSERT_RESULT_TRUE(Execute<SNAPSHOT_FUZZ_FN_NAME(MapUpdateSnapshot)>(i++)) << "sys_b"; + ASSERT_RESULT_TRUE(Execute<SNAPSHOT_FUZZ_FN_NAME(MapUpdateSnapshot)>(i++)) << "vnd_b"; + ASSERT_RESULT_TRUE(Execute<SNAPSHOT_FUZZ_FN_NAME(MapUpdateSnapshot)>(i++)) << "prd_b"; + ASSERT_RESULT_TRUE(Execute<SNAPSHOT_FUZZ_FN_NAME(FinishedSnapshotWrites)>(i++)); + ASSERT_RESULT_TRUE(Execute<SNAPSHOT_FUZZ_FN_NAME(UnmapUpdateSnapshot)>(i++)) << "sys_b"; + ASSERT_RESULT_TRUE(Execute<SNAPSHOT_FUZZ_FN_NAME(UnmapUpdateSnapshot)>(i++)) << "vnd_b"; + ASSERT_RESULT_TRUE(Execute<SNAPSHOT_FUZZ_FN_NAME(UnmapUpdateSnapshot)>(i++)) << "prd_b"; + ASSERT_RESULT_OK(Execute<SNAPSHOT_FUZZ_FN_NAME(SwitchSlot)>(i++)); + ASSERT_EQ("_b", test_module_.device_info->GetSlotSuffix()); + ASSERT_RESULT_TRUE(Execute<SNAPSHOT_FUZZ_FN_NAME(NeedSnapshotsInFirstStageMount)>(i++)); + ASSERT_RESULT_TRUE(Execute<SNAPSHOT_FUZZ_FN_NAME(CreateLogicalAndSnapshotPartitions)>(i++)); + ASSERT_RESULT_TRUE(Execute<SNAPSHOT_FUZZ_FN_NAME(InitiateMerge)>(i++)); + ASSERT_RESULT_TRUE(Execute<SNAPSHOT_FUZZ_FN_NAME(ProcessUpdateState)>(i++)); + ASSERT_EQ(i, snapshot_fuzz_data_->actions_size()) << "Not all actions are executed."; +} + +} // namespace android::snapshot diff --git a/fs_mgr/libsnapshot/snapshot_fuzz_utils.cpp b/fs_mgr/libsnapshot/snapshot_fuzz_utils.cpp new file mode 100644 index 000000000..892653585 --- /dev/null +++ b/fs_mgr/libsnapshot/snapshot_fuzz_utils.cpp @@ -0,0 +1,516 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <ftw.h> +#include <inttypes.h> +#include <sys/mman.h> +#include <sys/mount.h> +#include <sys/stat.h> +#include <sysexits.h> + +#include <chrono> +#include <string> + +#include <android-base/file.h> +#include <android-base/logging.h> +#include <android-base/properties.h> +#include <android-base/stringprintf.h> +#include <android-base/strings.h> +#include <fs_mgr.h> +#include <libsnapshot/auto_device.h> +#include <libsnapshot/snapshot.h> +#include <storage_literals/storage_literals.h> + +#include "snapshot_fuzz_utils.h" +#include "utility.h" + +// Prepends the errno string, but it is good enough. +#ifndef PCHECK +#define PCHECK(x) CHECK(x) << strerror(errno) << ": " +#endif + +using namespace android::storage_literals; +using namespace std::chrono_literals; +using namespace std::string_literals; + +using android::base::Basename; +using android::base::ReadFileToString; +using android::base::SetProperty; +using android::base::Split; +using android::base::StartsWith; +using android::base::StringPrintf; +using android::base::unique_fd; +using android::base::WriteStringToFile; +using android::dm::DeviceMapper; +using android::dm::DmTarget; +using android::dm::LoopControl; +using android::fiemap::IImageManager; +using android::fiemap::ImageManager; +using android::fs_mgr::BlockDeviceInfo; +using android::fs_mgr::FstabEntry; +using android::fs_mgr::IPartitionOpener; +using chromeos_update_engine::DynamicPartitionMetadata; + +static const char MNT_DIR[] = "/mnt"; +static const char BLOCK_SYSFS[] = "/sys/block"; + +static const char FAKE_ROOT_NAME[] = "snapshot_fuzz"; +static const auto SUPER_IMAGE_SIZE = 16_MiB; +static const auto DATA_IMAGE_SIZE = 16_MiB; +static const auto FAKE_ROOT_SIZE = 64_MiB; + +namespace android::snapshot { + +bool Mkdir(const std::string& path) { + if (mkdir(path.c_str(), 0750) == -1 && errno != EEXIST) { + PLOG(ERROR) << "Cannot create " << path; + return false; + } + return true; +} + +bool RmdirRecursive(const std::string& path) { + auto callback = [](const char* child, const struct stat*, int file_type, struct FTW*) -> int { + switch (file_type) { + case FTW_D: + case FTW_DP: + case FTW_DNR: + if (rmdir(child) == -1) { + PLOG(ERROR) << "rmdir " << child; + return -1; + } + return 0; + case FTW_NS: + default: + if (rmdir(child) != -1) break; + [[fallthrough]]; + case FTW_F: + case FTW_SL: + case FTW_SLN: + if (unlink(child) == -1) { + PLOG(ERROR) << "unlink " << child; + return -1; + } + return 0; + } + return 0; + }; + + return nftw(path.c_str(), callback, 128, FTW_DEPTH | FTW_MOUNT | FTW_PHYS) == 0; +} + +std::string GetLinearBaseDeviceString(const DeviceMapper::TargetInfo& target) { + if (target.spec.target_type != "linear"s) return {}; + auto tokens = Split(target.data, " "); + CHECK_EQ(2, tokens.size()); + return tokens[0]; +} + +std::vector<std::string> GetSnapshotBaseDeviceStrings(const DeviceMapper::TargetInfo& target) { + if (target.spec.target_type != "snapshot"s && target.spec.target_type != "snapshot-merge"s) + return {}; + auto tokens = Split(target.data, " "); + CHECK_EQ(4, tokens.size()); + return {tokens[0], tokens[1]}; +} + +bool ShouldDeleteLoopDevice(const std::string& node) { + std::string backing_file; + if (ReadFileToString(StringPrintf("%s/loop/backing_file", node.data()), &backing_file)) { + if (StartsWith(backing_file, std::string(MNT_DIR) + "/" + FAKE_ROOT_NAME)) { + return true; + } + } + return false; +} + +std::vector<DeviceMapper::TargetInfo> GetTableInfoIfExists(const std::string& dev_name) { + auto& dm = DeviceMapper::Instance(); + std::vector<DeviceMapper::TargetInfo> table; + if (!dm.GetTableInfo(dev_name, &table)) { + PCHECK(errno == ENODEV); + return {}; + } + return table; +} + +std::set<std::string> GetAllBaseDeviceStrings(const std::string& child_dev) { + std::set<std::string> ret; + for (const auto& child_target : GetTableInfoIfExists(child_dev)) { + auto snapshot_bases = GetSnapshotBaseDeviceStrings(child_target); + ret.insert(snapshot_bases.begin(), snapshot_bases.end()); + + auto linear_base = GetLinearBaseDeviceString(child_target); + if (!linear_base.empty()) { + ret.insert(linear_base); + } + } + return ret; +} + +using PropertyList = std::set<std::string>; +void InsertProperty(const char* key, const char* /*name*/, void* cookie) { + reinterpret_cast<PropertyList*>(cookie)->insert(key); +} + +// Attempt to delete all devices that is based on dev_name, including itself. +void CheckDeleteDeviceMapperTree(const std::string& dev_name, bool known_allow_delete = false, + uint64_t depth = 100) { + CHECK(depth > 0) << "Reaching max depth when deleting " << dev_name + << ". There may be devices referencing itself. Check `dmctl list devices -v`."; + + auto& dm = DeviceMapper::Instance(); + auto table = GetTableInfoIfExists(dev_name); + if (table.empty()) { + PCHECK(dm.DeleteDeviceIfExists(dev_name)) << dev_name; + return; + } + + if (!known_allow_delete) { + for (const auto& target : table) { + auto base_device_string = GetLinearBaseDeviceString(target); + if (base_device_string.empty()) continue; + if (ShouldDeleteLoopDevice( + StringPrintf("/sys/dev/block/%s", base_device_string.data()))) { + known_allow_delete = true; + break; + } + } + } + if (!known_allow_delete) { + return; + } + + std::string dev_string; + PCHECK(dm.GetDeviceString(dev_name, &dev_string)); + + std::vector<DeviceMapper::DmBlockDevice> devices; + PCHECK(dm.GetAvailableDevices(&devices)); + for (const auto& child_dev : devices) { + auto child_bases = GetAllBaseDeviceStrings(child_dev.name()); + if (child_bases.find(dev_string) != child_bases.end()) { + CheckDeleteDeviceMapperTree(child_dev.name(), true /* known_allow_delete */, depth - 1); + } + } + + PCHECK(dm.DeleteDeviceIfExists(dev_name)) << dev_name; +} + +// Attempt to clean up residues from previous runs. +void CheckCleanupDeviceMapperDevices() { + auto& dm = DeviceMapper::Instance(); + std::vector<DeviceMapper::DmBlockDevice> devices; + PCHECK(dm.GetAvailableDevices(&devices)); + + for (const auto& dev : devices) { + CheckDeleteDeviceMapperTree(dev.name()); + } +} + +void CheckUmount(const std::string& path) { + PCHECK(TEMP_FAILURE_RETRY(umount(path.data()) == 0) || errno == ENOENT || errno == EINVAL) + << path; +} + +void CheckDetachLoopDevices(const std::set<std::string>& exclude_names = {}) { + // ~SnapshotFuzzEnv automatically does the following. + std::unique_ptr<DIR, decltype(&closedir)> dir(opendir(BLOCK_SYSFS), closedir); + PCHECK(dir != nullptr) << BLOCK_SYSFS; + LoopControl loop_control; + dirent* dp; + while ((dp = readdir(dir.get())) != nullptr) { + if (exclude_names.find(dp->d_name) != exclude_names.end()) { + continue; + } + if (!ShouldDeleteLoopDevice(StringPrintf("%s/%s", BLOCK_SYSFS, dp->d_name).data())) { + continue; + } + PCHECK(loop_control.Detach(StringPrintf("/dev/block/%s", dp->d_name).data())); + } +} + +void CheckUmountAll() { + CheckUmount(std::string(MNT_DIR) + "/snapshot_fuzz_data"); + CheckUmount(std::string(MNT_DIR) + "/" + FAKE_ROOT_NAME); +} + +class AutoDeleteDir : public AutoDevice { + public: + static std::unique_ptr<AutoDeleteDir> New(const std::string& path) { + if (!Mkdir(path)) { + return std::unique_ptr<AutoDeleteDir>(new AutoDeleteDir("")); + } + return std::unique_ptr<AutoDeleteDir>(new AutoDeleteDir(path)); + } + ~AutoDeleteDir() { + if (!HasDevice()) return; + PCHECK(rmdir(name_.c_str()) == 0 || errno == ENOENT) << name_; + } + + private: + AutoDeleteDir(const std::string& path) : AutoDevice(path) {} +}; + +class AutoUnmount : public AutoDevice { + public: + ~AutoUnmount() { + if (!HasDevice()) return; + CheckUmount(name_); + } + AutoUnmount(const std::string& path) : AutoDevice(path) {} +}; + +class AutoUnmountTmpfs : public AutoUnmount { + public: + static std::unique_ptr<AutoUnmount> New(const std::string& path, uint64_t size) { + if (mount("tmpfs", path.c_str(), "tmpfs", 0, + (void*)StringPrintf("size=%" PRIu64, size).data()) == -1) { + PLOG(ERROR) << "Cannot mount " << path; + return std::unique_ptr<AutoUnmount>(new AutoUnmount("")); + } + return std::unique_ptr<AutoUnmount>(new AutoUnmount(path)); + } + private: + using AutoUnmount::AutoUnmount; +}; + +// A directory on tmpfs. Upon destruct, it is unmounted and deleted. +class AutoMemBasedDir : public AutoDevice { + public: + static std::unique_ptr<AutoMemBasedDir> New(const std::string& name, uint64_t size) { + auto ret = std::unique_ptr<AutoMemBasedDir>(new AutoMemBasedDir(name)); + ret->auto_delete_mount_dir_ = AutoDeleteDir::New(ret->mount_path()); + if (!ret->auto_delete_mount_dir_->HasDevice()) { + return std::unique_ptr<AutoMemBasedDir>(new AutoMemBasedDir("")); + } + ret->auto_umount_mount_point_ = AutoUnmountTmpfs::New(ret->mount_path(), size); + if (!ret->auto_umount_mount_point_->HasDevice()) { + return std::unique_ptr<AutoMemBasedDir>(new AutoMemBasedDir("")); + } + // tmp_path() and persist_path does not need to be deleted upon destruction, hence it is + // not wrapped with AutoDeleteDir. + if (!Mkdir(ret->tmp_path())) { + return std::unique_ptr<AutoMemBasedDir>(new AutoMemBasedDir("")); + } + if (!Mkdir(ret->persist_path())) { + return std::unique_ptr<AutoMemBasedDir>(new AutoMemBasedDir("")); + } + return ret; + } + // Return the temporary scratch directory. + std::string tmp_path() const { + CHECK(HasDevice()); + return mount_path() + "/tmp"; + } + // Return the temporary scratch directory. + std::string persist_path() const { + CHECK(HasDevice()); + return mount_path() + "/persist"; + } + // Delete all contents in tmp_path() and start over. tmp_path() itself is re-created. + void CheckSoftReset() { + PCHECK(RmdirRecursive(tmp_path())); + PCHECK(Mkdir(tmp_path())); + } + + private: + AutoMemBasedDir(const std::string& name) : AutoDevice(name) {} + std::string mount_path() const { + CHECK(HasDevice()); + return MNT_DIR + "/"s + name_; + } + std::unique_ptr<AutoDeleteDir> auto_delete_mount_dir_; + std::unique_ptr<AutoUnmount> auto_umount_mount_point_; +}; + +SnapshotFuzzEnv::SnapshotFuzzEnv() { + CheckCleanupDeviceMapperDevices(); + CheckDetachLoopDevices(); + CheckUmountAll(); + + fake_root_ = AutoMemBasedDir::New(FAKE_ROOT_NAME, FAKE_ROOT_SIZE); + CHECK(fake_root_ != nullptr); + CHECK(fake_root_->HasDevice()); + loop_control_ = std::make_unique<LoopControl>(); + + fake_data_mount_point_ = MNT_DIR + "/snapshot_fuzz_data"s; + auto_delete_data_mount_point_ = AutoDeleteDir::New(fake_data_mount_point_); + CHECK(auto_delete_data_mount_point_ != nullptr); + CHECK(auto_delete_data_mount_point_->HasDevice()); + + const auto& fake_persist_path = fake_root_->persist_path(); + mapped_super_ = CheckMapImage(fake_persist_path + "/super.img", SUPER_IMAGE_SIZE, + loop_control_.get(), &fake_super_); + mapped_data_ = CheckMapImage(fake_persist_path + "/data.img", DATA_IMAGE_SIZE, + loop_control_.get(), &fake_data_block_device_); + mounted_data_ = CheckMountFormatData(fake_data_block_device_, fake_data_mount_point_); +} + +SnapshotFuzzEnv::~SnapshotFuzzEnv() { + CheckCleanupDeviceMapperDevices(); + mounted_data_ = nullptr; + auto_delete_data_mount_point_ = nullptr; + mapped_data_ = nullptr; + mapped_super_ = nullptr; + CheckDetachLoopDevices(); + loop_control_ = nullptr; + fake_root_ = nullptr; + CheckUmountAll(); +} + +void CheckZeroFill(const std::string& file, size_t size) { + std::string zeros(size, '\0'); + PCHECK(WriteStringToFile(zeros, file)) << "Cannot write zeros to " << file; +} + +void SnapshotFuzzEnv::CheckSoftReset() { + fake_root_->CheckSoftReset(); + CheckZeroFill(super(), SUPER_IMAGE_SIZE); + CheckCleanupDeviceMapperDevices(); + CheckDetachLoopDevices({Basename(fake_super_), Basename(fake_data_block_device_)}); +} + +std::unique_ptr<IImageManager> SnapshotFuzzEnv::CheckCreateFakeImageManager( + const std::string& metadata_dir, const std::string& data_dir) { + PCHECK(Mkdir(metadata_dir)); + PCHECK(Mkdir(data_dir)); + return SnapshotFuzzImageManager::Open(metadata_dir, data_dir); +} + +// Helper to create a loop device for a file. +static void CheckCreateLoopDevice(LoopControl* control, const std::string& file, + const std::chrono::milliseconds& timeout_ms, std::string* path) { + static constexpr int kOpenFlags = O_RDWR | O_NOFOLLOW | O_CLOEXEC; + android::base::unique_fd file_fd(open(file.c_str(), kOpenFlags)); + PCHECK(file_fd >= 0) << "Could not open file: " << file; + CHECK(control->Attach(file_fd, timeout_ms, path)) + << "Could not create loop device for: " << file; +} + +class AutoDetachLoopDevice : public AutoDevice { + public: + AutoDetachLoopDevice(LoopControl* control, const std::string& device) + : AutoDevice(device), control_(control) {} + ~AutoDetachLoopDevice() { PCHECK(control_->Detach(name_)) << name_; } + + private: + LoopControl* control_; +}; + +std::unique_ptr<AutoDevice> SnapshotFuzzEnv::CheckMapImage(const std::string& img_path, + uint64_t size, LoopControl* control, + std::string* mapped_path) { + CheckZeroFill(img_path, size); + CheckCreateLoopDevice(control, img_path, 1s, mapped_path); + + return std::make_unique<AutoDetachLoopDevice>(control, *mapped_path); +} + +SnapshotTestModule SnapshotFuzzEnv::CheckCreateSnapshotManager(const SnapshotFuzzData& data) { + SnapshotTestModule ret; + auto partition_opener = std::make_unique<TestPartitionOpener>(super()); + ret.opener = partition_opener.get(); + CheckWriteSuperMetadata(data, *partition_opener); + auto metadata_dir = fake_root_->tmp_path() + "/snapshot_metadata"; + PCHECK(Mkdir(metadata_dir)); + if (data.has_metadata_snapshots_dir()) { + PCHECK(Mkdir(metadata_dir + "/snapshots")); + } + + ret.device_info = new SnapshotFuzzDeviceInfo(data.device_info_data(), + std::move(partition_opener), metadata_dir); + auto snapshot = SnapshotManager::New(ret.device_info /* takes ownership */); + snapshot->images_ = + CheckCreateFakeImageManager(fake_root_->tmp_path() + "/images_manager_metadata", + fake_data_mount_point_ + "/image_manager_data"); + snapshot->has_local_image_manager_ = data.manager_data().is_local_image_manager(); + ret.snapshot = std::move(snapshot); + + return ret; +} + +const std::string& SnapshotFuzzEnv::super() const { + return fake_super_; +} + +void SnapshotFuzzEnv::CheckWriteSuperMetadata(const SnapshotFuzzData& data, + const IPartitionOpener& opener) { + if (!data.is_super_metadata_valid()) { + // Leave it zero. + return; + } + + BlockDeviceInfo super_device("super", SUPER_IMAGE_SIZE, 0, 0, 4096); + std::vector<BlockDeviceInfo> devices = {super_device}; + auto builder = MetadataBuilder::New(devices, "super", 65536, 2); + CHECK(builder != nullptr); + + // Attempt to create a super partition metadata using proto. All errors are ignored. + for (const auto& group_proto : data.super_data().dynamic_partition_metadata().groups()) { + (void)builder->AddGroup(group_proto.name(), group_proto.size()); + for (const auto& partition_name : group_proto.partition_names()) { + (void)builder->AddPartition(partition_name, group_proto.name(), + LP_PARTITION_ATTR_READONLY); + } + } + + for (const auto& partition_proto : data.super_data().partitions()) { + auto p = builder->FindPartition(partition_proto.partition_name()); + if (p == nullptr) continue; + (void)builder->ResizePartition(p, partition_proto.new_partition_info().size()); + } + + auto metadata = builder->Export(); + // metadata may be nullptr if it is not valid (e.g. partition name too long). + // In this case, just use empty super partition data. + if (metadata == nullptr) { + builder = MetadataBuilder::New(devices, "super", 65536, 2); + CHECK(builder != nullptr); + metadata = builder->Export(); + CHECK(metadata != nullptr); + } + CHECK(FlashPartitionTable(opener, super(), *metadata.get())); +} + +std::unique_ptr<AutoDevice> SnapshotFuzzEnv::CheckMountFormatData(const std::string& blk_device, + const std::string& mount_point) { + FstabEntry entry{ + .blk_device = blk_device, + .length = static_cast<off64_t>(DATA_IMAGE_SIZE), + .fs_type = "ext4", + .mount_point = mount_point, + }; + CHECK(0 == fs_mgr_do_format(entry, false /* crypt_footer */)); + CHECK(0 == fs_mgr_do_mount_one(entry)); + return std::make_unique<AutoUnmount>(mount_point); +} + +SnapshotFuzzImageManager::~SnapshotFuzzImageManager() { + // Remove relevant gsid.mapped_images.* props. + for (const auto& name : mapped_) { + CHECK(UnmapImageIfExists(name)) << "Cannot unmap " << name; + } +} + +bool SnapshotFuzzImageManager::MapImageDevice(const std::string& name, + const std::chrono::milliseconds& timeout_ms, + std::string* path) { + if (impl_->MapImageDevice(name, timeout_ms, path)) { + mapped_.insert(name); + return true; + } + return false; +} + +} // namespace android::snapshot diff --git a/fs_mgr/libsnapshot/snapshot_fuzz_utils.h b/fs_mgr/libsnapshot/snapshot_fuzz_utils.h new file mode 100644 index 000000000..fa327b8a7 --- /dev/null +++ b/fs_mgr/libsnapshot/snapshot_fuzz_utils.h @@ -0,0 +1,203 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <memory> +#include <set> +#include <string> + +#include <android-base/file.h> +#include <android-base/stringprintf.h> +#include <android/snapshot/snapshot_fuzz.pb.h> +#include <libdm/loop_control.h> +#include <libfiemap/image_manager.h> +#include <liblp/liblp.h> +#include <libsnapshot/auto_device.h> +#include <libsnapshot/test_helpers.h> + +// libsnapshot-specific code for fuzzing. Defines fake classes that are depended +// by SnapshotManager. + +#include "android/snapshot/snapshot_fuzz.pb.h" + +namespace android::snapshot { + +class AutoMemBasedDir; +class SnapshotFuzzDeviceInfo; + +class DummyAutoDevice : public AutoDevice { + public: + DummyAutoDevice(bool mounted) : AutoDevice(mounted ? "dummy" : "") {} +}; + +struct SnapshotTestModule { + std::unique_ptr<ISnapshotManager> snapshot; + SnapshotFuzzDeviceInfo* device_info = nullptr; + TestPartitionOpener* opener = nullptr; +}; + +// Prepare test environment. This has a heavy overhead and should be done once. +class SnapshotFuzzEnv { + public: + // Check if test should run at all. + static bool ShouldSkipTest(); + + // Initialize the environment. + SnapshotFuzzEnv(); + ~SnapshotFuzzEnv(); + + // Soft reset part of the environment before running the next test. + // Abort if fails. + void CheckSoftReset(); + + // Create a snapshot manager for this test run. + // Client is responsible for maintaining the lifetime of |data| over the life time of + // ISnapshotManager. + SnapshotTestModule CheckCreateSnapshotManager(const SnapshotFuzzData& data); + + // Return path to super partition. + const std::string& super() const; + + private: + std::unique_ptr<AutoMemBasedDir> fake_root_; + std::unique_ptr<android::dm::LoopControl> loop_control_; + std::string fake_data_mount_point_; + std::unique_ptr<AutoDevice> auto_delete_data_mount_point_; + std::unique_ptr<AutoDevice> mapped_super_; + std::string fake_super_; + std::unique_ptr<AutoDevice> mapped_data_; + std::string fake_data_block_device_; + std::unique_ptr<AutoDevice> mounted_data_; + + static std::unique_ptr<android::fiemap::IImageManager> CheckCreateFakeImageManager( + const std::string& metadata_dir, const std::string& data_dir); + static std::unique_ptr<AutoDevice> CheckMapImage(const std::string& fake_persist_path, + uint64_t size, + android::dm::LoopControl* control, + std::string* mapped_path); + static std::unique_ptr<AutoDevice> CheckMountFormatData(const std::string& blk_device, + const std::string& mount_point); + + void CheckWriteSuperMetadata(const SnapshotFuzzData& proto, + const android::fs_mgr::IPartitionOpener& opener); +}; + +class SnapshotFuzzDeviceInfo : public ISnapshotManager::IDeviceInfo { + public: + // Client is responsible for maintaining the lifetime of |data|. + SnapshotFuzzDeviceInfo(const FuzzDeviceInfoData& data, + std::unique_ptr<TestPartitionOpener>&& partition_opener, + const std::string& metadata_dir) + : data_(&data), + partition_opener_(std::move(partition_opener)), + metadata_dir_(metadata_dir) {} + + // Following APIs are mocked. + std::string GetGsidDir() const override { return "fuzz_ota"; } + std::string GetMetadataDir() const override { return metadata_dir_; } + std::string GetSuperDevice(uint32_t) const override { + // TestPartitionOpener can recognize this. + return "super"; + } + const android::fs_mgr::IPartitionOpener& GetPartitionOpener() const override { + return *partition_opener_; + } + + // Following APIs are fuzzed. + std::string GetSlotSuffix() const override { return CurrentSlotIsA() ? "_a" : "_b"; } + std::string GetOtherSlotSuffix() const override { return CurrentSlotIsA() ? "_b" : "_a"; } + bool IsOverlayfsSetup() const override { return data_->is_overlayfs_setup(); } + bool SetBootControlMergeStatus(android::hardware::boot::V1_1::MergeStatus) override { + return data_->allow_set_boot_control_merge_status(); + } + bool SetSlotAsUnbootable(unsigned int) override { + return data_->allow_set_slot_as_unbootable(); + } + bool IsRecovery() const override { return data_->is_recovery(); } + + void SwitchSlot() { switched_slot_ = !switched_slot_; } + + private: + const FuzzDeviceInfoData* data_; + std::unique_ptr<TestPartitionOpener> partition_opener_; + std::string metadata_dir_; + bool switched_slot_ = false; + + bool CurrentSlotIsA() const { return data_->slot_suffix_is_a() != switched_slot_; } +}; + +// A spy class on ImageManager implementation. Upon destruction, unmaps all images +// map through this object. +class SnapshotFuzzImageManager : public android::fiemap::IImageManager { + public: + static std::unique_ptr<SnapshotFuzzImageManager> Open(const std::string& metadata_dir, + const std::string& data_dir) { + auto impl = android::fiemap::ImageManager::Open(metadata_dir, data_dir); + if (impl == nullptr) return nullptr; + return std::unique_ptr<SnapshotFuzzImageManager>( + new SnapshotFuzzImageManager(std::move(impl))); + } + + ~SnapshotFuzzImageManager(); + + // Spied APIs. + bool MapImageDevice(const std::string& name, const std::chrono::milliseconds& timeout_ms, + std::string* path) override; + + // Other functions call through. + android::fiemap::FiemapStatus CreateBackingImage( + const std::string& name, uint64_t size, int flags, + std::function<bool(uint64_t, uint64_t)>&& on_progress) override { + return impl_->CreateBackingImage(name, size, flags, std::move(on_progress)); + } + bool DeleteBackingImage(const std::string& name) override { + return impl_->DeleteBackingImage(name); + } + bool UnmapImageDevice(const std::string& name) override { + return impl_->UnmapImageDevice(name); + } + bool BackingImageExists(const std::string& name) override { + return impl_->BackingImageExists(name); + } + bool IsImageMapped(const std::string& name) override { return impl_->IsImageMapped(name); } + bool MapImageWithDeviceMapper(const IPartitionOpener& opener, const std::string& name, + std::string* dev) override { + return impl_->MapImageWithDeviceMapper(opener, name, dev); + } + bool GetMappedImageDevice(const std::string& name, std::string* device) override { + return impl_->GetMappedImageDevice(name, device); + } + bool MapAllImages(const std::function<bool(std::set<std::string>)>& init) override { + return impl_->MapAllImages(init); + } + bool DisableImage(const std::string& name) override { return impl_->DisableImage(name); } + bool RemoveDisabledImages() override { return impl_->RemoveDisabledImages(); } + std::vector<std::string> GetAllBackingImages() override { return impl_->GetAllBackingImages(); } + android::fiemap::FiemapStatus ZeroFillNewImage(const std::string& name, + uint64_t bytes) override { + return impl_->ZeroFillNewImage(name, bytes); + } + bool RemoveAllImages() override { return impl_->RemoveAllImages(); } + bool UnmapImageIfExists(const std::string& name) override { + return impl_->UnmapImageIfExists(name); + } + + private: + std::unique_ptr<android::fiemap::IImageManager> impl_; + std::set<std::string> mapped_; + + SnapshotFuzzImageManager(std::unique_ptr<android::fiemap::IImageManager>&& impl) + : impl_(std::move(impl)) {} +}; + +} // namespace android::snapshot diff --git a/fs_mgr/libsnapshot/snapshot_metadata_updater.cpp b/fs_mgr/libsnapshot/snapshot_metadata_updater.cpp index 60bf79647..051584c34 100644 --- a/fs_mgr/libsnapshot/snapshot_metadata_updater.cpp +++ b/fs_mgr/libsnapshot/snapshot_metadata_updater.cpp @@ -62,6 +62,8 @@ SnapshotMetadataUpdater::SnapshotMetadataUpdater(MetadataBuilder* builder, uint3 std::string(it->second) + target_suffix_, &p}); } } + + partial_update_ = manifest.partial_update(); } bool SnapshotMetadataUpdater::ShrinkPartitions() const { @@ -82,6 +84,18 @@ bool SnapshotMetadataUpdater::ShrinkPartitions() const { } bool SnapshotMetadataUpdater::DeletePartitions() const { + // For partial update, not all dynamic partitions are included in the payload. + // TODO(xunchang) delete the untouched partitions whose group is in the payload. + // e.g. Delete vendor in the following scenario + // On device: + // Group A: system, vendor + // In payload: + // Group A: system + if (partial_update_) { + LOG(INFO) << "Skip deleting partitions for partial update"; + return true; + } + std::vector<std::string> partitions_to_delete; // Don't delete partitions in groups where the group name doesn't have target_suffix, // e.g. default. @@ -139,6 +153,11 @@ bool SnapshotMetadataUpdater::ShrinkGroups() const { } bool SnapshotMetadataUpdater::DeleteGroups() const { + if (partial_update_) { + LOG(INFO) << "Skip deleting groups for partial update"; + return true; + } + std::vector<std::string> existing_groups = builder_->ListGroups(); for (const auto& existing_group_name : existing_groups) { // Don't delete groups without target suffix, e.g. default. diff --git a/fs_mgr/libsnapshot/snapshot_metadata_updater.h b/fs_mgr/libsnapshot/snapshot_metadata_updater.h index 83c94607d..5b1cbf91a 100644 --- a/fs_mgr/libsnapshot/snapshot_metadata_updater.h +++ b/fs_mgr/libsnapshot/snapshot_metadata_updater.h @@ -79,6 +79,7 @@ class SnapshotMetadataUpdater { const std::string target_suffix_; std::vector<Group> groups_; std::vector<Partition> partitions_; + bool partial_update_{false}; }; } // namespace snapshot diff --git a/fs_mgr/libsnapshot/snapshot_stub.cpp b/fs_mgr/libsnapshot/snapshot_stub.cpp new file mode 100644 index 000000000..9b6f758cc --- /dev/null +++ b/fs_mgr/libsnapshot/snapshot_stub.cpp @@ -0,0 +1,132 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <libsnapshot/snapshot_stub.h> + +#include <android-base/logging.h> + +#include <libsnapshot/snapshot_stats.h> + +using android::fs_mgr::CreateLogicalPartitionParams; +using chromeos_update_engine::DeltaArchiveManifest; + +namespace android::snapshot { + +std::unique_ptr<ISnapshotManager> SnapshotManagerStub::New() { + return std::make_unique<SnapshotManagerStub>(); +} + +bool SnapshotManagerStub::BeginUpdate() { + LOG(ERROR) << __FUNCTION__ << " should never be called."; + return false; +} + +bool SnapshotManagerStub::CancelUpdate() { + LOG(ERROR) << __FUNCTION__ << " should never be called."; + return false; +} + +bool SnapshotManagerStub::FinishedSnapshotWrites(bool) { + LOG(ERROR) << __FUNCTION__ << " should never be called."; + return false; +} + +bool SnapshotManagerStub::InitiateMerge(uint64_t*) { + LOG(ERROR) << __FUNCTION__ << " should never be called."; + return false; +} + +UpdateState SnapshotManagerStub::ProcessUpdateState(const std::function<bool()>&, + const std::function<bool()>&) { + LOG(ERROR) << __FUNCTION__ << " should never be called."; + return UpdateState::None; +} + +UpdateState SnapshotManagerStub::GetUpdateState(double*) { + LOG(ERROR) << __FUNCTION__ << " should never be called."; + return UpdateState::None; +} + +Return SnapshotManagerStub::CreateUpdateSnapshots(const DeltaArchiveManifest&) { + LOG(ERROR) << __FUNCTION__ << " should never be called."; + return Return::Error(); +} + +bool SnapshotManagerStub::MapUpdateSnapshot(const CreateLogicalPartitionParams&, std::string*) { + LOG(ERROR) << __FUNCTION__ << " should never be called."; + return false; +} + +bool SnapshotManagerStub::UnmapUpdateSnapshot(const std::string&) { + LOG(ERROR) << __FUNCTION__ << " should never be called."; + return false; +} + +bool SnapshotManagerStub::NeedSnapshotsInFirstStageMount() { + LOG(ERROR) << __FUNCTION__ << " should never be called."; + return false; +} + +bool SnapshotManagerStub::CreateLogicalAndSnapshotPartitions(const std::string&, + const std::chrono::milliseconds&) { + LOG(ERROR) << __FUNCTION__ << " should never be called."; + return false; +} + +bool SnapshotManagerStub::HandleImminentDataWipe(const std::function<void()>&) { + LOG(ERROR) << __FUNCTION__ << " should never be called."; + return false; +} + +bool SnapshotManagerStub::FinishMergeInRecovery() { + LOG(ERROR) << __FUNCTION__ << " should never be called."; + return false; +} + +CreateResult SnapshotManagerStub::RecoveryCreateSnapshotDevices() { + LOG(ERROR) << __FUNCTION__ << " should never be called."; + return CreateResult::ERROR; +} + +CreateResult SnapshotManagerStub::RecoveryCreateSnapshotDevices( + const std::unique_ptr<AutoDevice>&) { + LOG(ERROR) << __FUNCTION__ << " should never be called."; + return CreateResult::ERROR; +} + +bool SnapshotManagerStub::Dump(std::ostream&) { + LOG(ERROR) << __FUNCTION__ << " should never be called."; + return false; +} + +std::unique_ptr<AutoDevice> SnapshotManagerStub::EnsureMetadataMounted() { + LOG(ERROR) << __FUNCTION__ << " should never be called."; + return nullptr; +} + +class SnapshotMergeStatsStub : public ISnapshotMergeStats { + bool Start() override { return false; } + void set_state(android::snapshot::UpdateState) override {} + void set_cow_file_size(uint64_t) override {} + uint64_t cow_file_size() override { return 0; } + std::unique_ptr<Result> Finish() override { return nullptr; } +}; + +ISnapshotMergeStats* SnapshotManagerStub::GetSnapshotMergeStatsInstance() { + static SnapshotMergeStatsStub snapshot_merge_stats; + LOG(ERROR) << __FUNCTION__ << " should never be called."; + return &snapshot_merge_stats; +} + +} // namespace android::snapshot diff --git a/fs_mgr/libsnapshot/snapshot_test.cpp b/fs_mgr/libsnapshot/snapshot_test.cpp index 9ca2412df..2bd013599 100644 --- a/fs_mgr/libsnapshot/snapshot_test.cpp +++ b/fs_mgr/libsnapshot/snapshot_test.cpp @@ -41,6 +41,11 @@ #include <libsnapshot/test_helpers.h> #include "utility.h" +// Mock classes are not used. Header included to ensure mocked class definition aligns with the +// class itself. +#include <libsnapshot/mock_device_info.h> +#include <libsnapshot/mock_snapshot.h> + namespace android { namespace snapshot { @@ -1799,7 +1804,6 @@ class ImageManagerTest : public SnapshotTest, public WithParamInterface<uint64_t protected: void SetUp() override { if (!is_virtual_ab_) GTEST_SKIP() << "Test for Virtual A/B devices only"; - GTEST_SKIP() << "WIP failure b/149738928"; SnapshotTest::SetUp(); userdata_ = std::make_unique<LowSpaceUserdata>(); diff --git a/fs_mgr/libsnapshot/update_engine/update_metadata.proto b/fs_mgr/libsnapshot/update_engine/update_metadata.proto new file mode 100644 index 000000000..202e39be1 --- /dev/null +++ b/fs_mgr/libsnapshot/update_engine/update_metadata.proto @@ -0,0 +1,81 @@ +// +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// A subset of system/update_engine/update_metadata.proto. A separate file is +// used here because: +// - The original file is optimized for LITE_RUNTIME, but fuzzing needs +// reflection. +// - The definition here has less fields. libsnapshot only uses fields declared +// here, and all fields declared here are fuzzed by libsnapshot_fuzzer. If +// libsnapshot uses more fields in system/update_engine/update_metadata.proto +// in the future, they must be added here too, otherwise it will fail to +// compile. +// +// It is okay that this file is older than +// system/update_engine/update_metadata.proto as long as the messages defined +// here can also be parsed by protobuf defined there. However, it is not +// okay to add fields here without adding them to +// system/update_engine/update_metadata.proto. Doing so will cause a compiler +// error when libsnapshot code starts to use these dangling fields. + +syntax = "proto2"; + +package chromeos_update_engine; + +message Extent { + optional uint64 start_block = 1; + optional uint64 num_blocks = 2; +} + +message PartitionInfo { + optional uint64 size = 1; +} + +message InstallOperation { + enum Type { + SOURCE_COPY = 4; + // Not used by libsnapshot. Declared here so that the fuzzer has an + // alternative value to use for |type|. + ZERO = 6; + } + required Type type = 1; + repeated Extent src_extents = 4; + repeated Extent dst_extents = 6; +} + +message PartitionUpdate { + required string partition_name = 1; + optional PartitionInfo new_partition_info = 7; + repeated InstallOperation operations = 8; + optional Extent hash_tree_extent = 11; + optional Extent fec_extent = 15; +} + +message DynamicPartitionGroup { + required string name = 1; + optional uint64 size = 2; + repeated string partition_names = 3; +} + +message DynamicPartitionMetadata { + repeated DynamicPartitionGroup groups = 1; +} + +message DeltaArchiveManifest { + repeated PartitionUpdate partitions = 13; + optional DynamicPartitionMetadata dynamic_partition_metadata = 15; + optional bool partial_update = 16; +} diff --git a/fs_mgr/tests/AndroidTest.xml b/fs_mgr/tests/AndroidTest.xml index 0ff89957a..de835b3ad 100644 --- a/fs_mgr/tests/AndroidTest.xml +++ b/fs_mgr/tests/AndroidTest.xml @@ -21,6 +21,9 @@ <option name="push" value="CtsFsMgrTestCases->/data/local/tmp/CtsFsMgrTestCases" /> <option name="append-bitness" value="true" /> </target_preparer> + <target_preparer class="com.android.tradefed.targetprep.RootTargetPreparer"> + <option name="throw-on-error" value="false" /> + </target_preparer> <test class="com.android.tradefed.testtype.GTest" > <option name="native-test-device-path" value="/data/local/tmp" /> <option name="module-name" value="CtsFsMgrTestCases" /> diff --git a/gatekeeperd/gatekeeperd.cpp b/gatekeeperd/gatekeeperd.cpp index 1d65b1ced..c81a80ef3 100644 --- a/gatekeeperd/gatekeeperd.cpp +++ b/gatekeeperd/gatekeeperd.cpp @@ -159,8 +159,8 @@ public: #define GK_ERROR *gkResponse = GKResponse::error(), Status::ok() - Status enroll(int32_t uid, const std::unique_ptr<std::vector<uint8_t>>& currentPasswordHandle, - const std::unique_ptr<std::vector<uint8_t>>& currentPassword, + Status enroll(int32_t uid, const std::optional<std::vector<uint8_t>>& currentPasswordHandle, + const std::optional<std::vector<uint8_t>>& currentPassword, const std::vector<uint8_t>& desiredPassword, GKResponse* gkResponse) override { IPCThreadState* ipc = IPCThreadState::self(); const int calling_pid = ipc->getCallingPid(); diff --git a/init/Android.bp b/init/Android.bp index 827a8293f..edf9099c0 100644 --- a/init/Android.bp +++ b/init/Android.bp @@ -131,6 +131,7 @@ cc_defaults { "libpropertyinfoparser", "libsnapshot_init", "lib_apex_manifest_proto_lite", + "update_metadata-protos", ], shared_libs: [ "libbacktrace", diff --git a/init/Android.mk b/init/Android.mk index 416b732d8..da94daf44 100644 --- a/init/Android.mk +++ b/init/Android.mk @@ -113,6 +113,7 @@ LOCAL_STATIC_LIBRARIES := \ libext2_uuid \ libprotobuf-cpp-lite \ libsnapshot_init \ + update_metadata-protos \ LOCAL_SANITIZE := signed-integer-overflow # First stage init is weird: it may start without stdout/stderr, and no /proc. diff --git a/init/AndroidTest.xml b/init/AndroidTest.xml index 920dc6c58..17f509ae5 100644 --- a/init/AndroidTest.xml +++ b/init/AndroidTest.xml @@ -24,6 +24,9 @@ <option name="push" value="CtsInitTestCases->/data/local/tmp/CtsInitTestCases" /> <option name="append-bitness" value="true" /> </target_preparer> + <target_preparer class="com.android.tradefed.targetprep.RootTargetPreparer"> + <option name="throw-on-error" value="false" /> + </target_preparer> <test class="com.android.tradefed.testtype.GTest" > <option name="native-test-device-path" value="/data/local/tmp" /> <option name="module-name" value="CtsInitTestCases" /> diff --git a/init/README.md b/init/README.md index 6f2c01f05..188f19b29 100644 --- a/init/README.md +++ b/init/README.md @@ -564,9 +564,11 @@ provides the `aidl_lazy_test_1` interface. _options_ include "barrier=1", "noauto\_da\_alloc", "discard", ... as a comma separated string, e.g. barrier=1,noauto\_da\_alloc -`parse_apex_configs` -> Parses config file(s) from the mounted APEXes. Intended to be used only once - when apexd notifies the mount event by setting apexd.status to ready. +`perform_apex_config` +> Performs tasks after APEXes are mounted. For example, creates data directories + for the mounted APEXes, parses config file(s) from them, and updates linker + configurations. Intended to be used only once when apexd notifies the mount + event by setting `apexd.status` to ready. `restart <service>` > Stops and restarts a running service, does nothing if the service is currently @@ -762,7 +764,7 @@ The _commands_ are listed below. These are equivalent to using the `start`, `restart`, and `stop` commands on the service specified by the _value_ of the property. -`oneshot_one` and `oneshot_off` will turn on or off the _oneshot_ +`oneshot_on` and `oneshot_off` will turn on or off the _oneshot_ flag for the service specified by the _value_ of the property. This is particularly intended for services that are conditionally lazy HALs. When they are lazy HALs, oneshot must be on, otherwise oneshot should be off. diff --git a/init/builtins.cpp b/init/builtins.cpp index 0ac66f272..0b456e70a 100644 --- a/init/builtins.cpp +++ b/init/builtins.cpp @@ -1221,6 +1221,20 @@ static Result<void> GenerateLinkerConfiguration() { return {}; } +static Result<void> MountLinkerConfigForDefaultNamespace() { + // No need to mount linkerconfig for default mount namespace if the path does not exist (which + // would mean it is already mounted) + if (access("/linkerconfig/default", 0) != 0) { + return {}; + } + + if (mount("/linkerconfig/default", "/linkerconfig", nullptr, MS_BIND | MS_REC, nullptr) != 0) { + return ErrnoError() << "Failed to mount linker configuration for default mount namespace."; + } + + return {}; +} + static bool IsApexUpdatable() { static bool updatable = android::sysprop::ApexProperties::updatable().value_or(false); return updatable; @@ -1319,11 +1333,14 @@ static Result<void> do_perform_apex_config(const BuiltinArguments& args) { } static Result<void> do_enter_default_mount_ns(const BuiltinArguments& args) { - if (SwitchToDefaultMountNamespace()) { - return {}; - } else { - return Error() << "Failed to enter into default mount namespace"; + if (auto result = SwitchToMountNamespaceIfNeeded(NS_DEFAULT); !result.ok()) { + return result.error(); + } + if (auto result = MountLinkerConfigForDefaultNamespace(); !result.ok()) { + return result.error(); } + LOG(INFO) << "Switched to default mount namespace"; + return {}; } // Builtin-function-map start diff --git a/init/host_init_verifier.cpp b/init/host_init_verifier.cpp index 0bd4df463..ef9a4514c 100644 --- a/init/host_init_verifier.cpp +++ b/init/host_init_verifier.cpp @@ -32,6 +32,7 @@ #include <android-base/logging.h> #include <android-base/parseint.h> #include <android-base/strings.h> +#include <generated_android_ids.h> #include <hidl/metadata.h> #include <property_info_serializer/property_info_serializer.h> @@ -48,9 +49,6 @@ #include "service_list.h" #include "service_parser.h" -#define EXCLUDE_FS_CONFIG_STRUCTURES -#include "generated_android_ids.h" - using namespace std::literals; using android::base::ParseInt; diff --git a/init/init.cpp b/init/init.cpp index 29859c5f5..631db8e7f 100644 --- a/init/init.cpp +++ b/init/init.cpp @@ -46,6 +46,7 @@ #include <android-base/properties.h> #include <android-base/stringprintf.h> #include <android-base/strings.h> +#include <backtrace/Backtrace.h> #include <fs_avb/fs_avb.h> #include <fs_mgr_vendor_overlay.h> #include <keyutils.h> @@ -231,18 +232,45 @@ static class ShutdownState { std::optional<std::string> CheckShutdown() { auto lock = std::lock_guard{shutdown_command_lock_}; if (do_shutdown_ && !IsShuttingDown()) { - do_shutdown_ = false; return shutdown_command_; } return {}; } + bool do_shutdown() const { return do_shutdown_; } + void set_do_shutdown(bool value) { do_shutdown_ = value; } + private: std::mutex shutdown_command_lock_; std::string shutdown_command_; bool do_shutdown_ = false; } shutdown_state; +static void UnwindMainThreadStack() { + std::unique_ptr<Backtrace> backtrace(Backtrace::Create(BACKTRACE_CURRENT_PROCESS, 1)); + if (!backtrace->Unwind(0)) { + LOG(ERROR) << __FUNCTION__ << "sys.powerctl: Failed to unwind callstack."; + } + for (size_t i = 0; i < backtrace->NumFrames(); i++) { + LOG(ERROR) << "sys.powerctl: " << backtrace->FormatFrameData(i); + } +} + +void DebugRebootLogging() { + LOG(INFO) << "sys.powerctl: do_shutdown: " << shutdown_state.do_shutdown() + << " IsShuttingDown: " << IsShuttingDown(); + if (shutdown_state.do_shutdown()) { + LOG(ERROR) << "sys.powerctl set while a previous shutdown command has not been handled"; + UnwindMainThreadStack(); + DumpShutdownDebugInformation(); + } + if (IsShuttingDown()) { + LOG(ERROR) << "sys.powerctl set while init is already shutting down"; + UnwindMainThreadStack(); + DumpShutdownDebugInformation(); + } +} + void DumpState() { ServiceList::GetInstance().DumpState(); ActionManager::GetInstance().DumpState(); @@ -696,7 +724,7 @@ int SecondStageMain(int argc, char** argv) { trigger_shutdown = [](const std::string& command) { shutdown_state.TriggerShutdown(command); }; SetStdioToDevNull(argv); - InitKernelLogging(argv); + InitSecondStageLogging(argv); LOG(INFO) << "init second stage started!"; // Init should not crash because of a dependence on any other process, therefore we ignore @@ -846,7 +874,10 @@ int SecondStageMain(int argc, char** argv) { auto shutdown_command = shutdown_state.CheckShutdown(); if (shutdown_command) { + LOG(INFO) << "Got shutdown_command '" << *shutdown_command + << "' Calling HandlePowerctlMessage()"; HandlePowerctlMessage(*shutdown_command); + shutdown_state.set_do_shutdown(false); } if (!(prop_waiter_state.MightBeWaiting() || Service::is_exec_service_running())) { diff --git a/init/init.h b/init/init.h index 27f64e297..4f686cb5b 100644 --- a/init/init.h +++ b/init/init.h @@ -42,6 +42,8 @@ void SendLoadPersistentPropertiesMessage(); void PropertyChanged(const std::string& name, const std::string& value); bool QueueControlMessage(const std::string& message, const std::string& name, pid_t pid, int fd); +void DebugRebootLogging(); + int SecondStageMain(int argc, char** argv); } // namespace init diff --git a/init/mount_namespace.cpp b/init/mount_namespace.cpp index f3b584c4c..b9d5d674c 100644 --- a/init/mount_namespace.cpp +++ b/init/mount_namespace.cpp @@ -176,20 +176,6 @@ static bool ActivateFlattenedApexesIfPossible() { return true; } -static Result<void> MountLinkerConfigForDefaultNamespace() { - // No need to mount linkerconfig for default mount namespace if the path does not exist (which - // would mean it is already mounted) - if (access("/linkerconfig/default", 0) != 0) { - return {}; - } - - if (mount("/linkerconfig/default", "/linkerconfig", nullptr, MS_BIND | MS_REC, nullptr) != 0) { - return ErrnoError() << "Failed to mount linker configuration for default mount namespace."; - } - - return {}; -} - static android::base::unique_fd bootstrap_ns_fd; static android::base::unique_fd default_ns_fd; @@ -290,40 +276,20 @@ bool SetupMountNamespaces() { return success; } -bool SwitchToDefaultMountNamespace() { - if (IsRecoveryMode()) { - // we don't have multiple namespaces in recovery mode - return true; - } - if (default_ns_id != GetMountNamespaceId()) { - if (setns(default_ns_fd.get(), CLONE_NEWNS) == -1) { - PLOG(ERROR) << "Failed to switch back to the default mount namespace."; - return false; - } - - if (auto result = MountLinkerConfigForDefaultNamespace(); !result.ok()) { - LOG(ERROR) << result.error(); - return false; - } - } - - LOG(INFO) << "Switched to default mount namespace"; - return true; -} - -bool SwitchToBootstrapMountNamespaceIfNeeded() { - if (IsRecoveryMode()) { - // we don't have multiple namespaces in recovery mode - return true; +Result<void> SwitchToMountNamespaceIfNeeded(MountNamespace target_mount_namespace) { + if (IsRecoveryMode() || !IsApexUpdatable()) { + // we don't have multiple namespaces in recovery mode or if apex is not updatable + return {}; } - if (bootstrap_ns_id != GetMountNamespaceId() && bootstrap_ns_fd.get() != -1 && - IsApexUpdatable()) { - if (setns(bootstrap_ns_fd.get(), CLONE_NEWNS) == -1) { - PLOG(ERROR) << "Failed to switch to bootstrap mount namespace."; - return false; + const auto& ns_id = target_mount_namespace == NS_BOOTSTRAP ? bootstrap_ns_id : default_ns_id; + const auto& ns_fd = target_mount_namespace == NS_BOOTSTRAP ? bootstrap_ns_fd : default_ns_fd; + const auto& ns_name = target_mount_namespace == NS_BOOTSTRAP ? "bootstrap" : "default"; + if (ns_id != GetMountNamespaceId() && ns_fd.get() != -1) { + if (setns(ns_fd.get(), CLONE_NEWNS) == -1) { + return ErrnoError() << "Failed to switch to " << ns_name << " mount namespace."; } } - return true; + return {}; } } // namespace init diff --git a/init/mount_namespace.h b/init/mount_namespace.h index c41a449f3..d4d6f82da 100644 --- a/init/mount_namespace.h +++ b/init/mount_namespace.h @@ -16,12 +16,15 @@ #pragma once +#include <android-base/result.h> + namespace android { namespace init { +enum MountNamespace { NS_BOOTSTRAP, NS_DEFAULT }; + bool SetupMountNamespaces(); -bool SwitchToDefaultMountNamespace(); -bool SwitchToBootstrapMountNamespaceIfNeeded(); +base::Result<void> SwitchToMountNamespaceIfNeeded(MountNamespace target_mount_namespace); } // namespace init } // namespace android diff --git a/init/parser.cpp b/init/parser.cpp index 507ee4a5b..5c18551e1 100644 --- a/init/parser.cpp +++ b/init/parser.cpp @@ -41,7 +41,7 @@ void Parser::AddSingleLineParser(const std::string& prefix, LineCallback callbac } void Parser::ParseData(const std::string& filename, std::string* data) { - data->push_back('\n'); // TODO: fix tokenizer + data->push_back('\n'); data->push_back('\0'); parse_state state; diff --git a/init/property_service.cpp b/init/property_service.cpp index a89504e59..612854d80 100644 --- a/init/property_service.cpp +++ b/init/property_service.cpp @@ -491,6 +491,9 @@ uint32_t HandlePropertySet(const std::string& name, const std::string& value, } LOG(INFO) << "Received sys.powerctl='" << value << "' from pid: " << cr.pid << process_log_string; + if (!value.empty()) { + DebugRebootLogging(); + } if (value == "reboot,userspace" && !is_userspace_reboot_supported().value_or(false)) { *error = "Userspace reboot is not supported by this device"; return PROP_ERROR_INVALID_VALUE; @@ -708,8 +711,8 @@ static void LoadProperties(char* data, const char* filter, const char* filename, if (it == properties->end()) { (*properties)[key] = value; } else if (it->second != value) { - LOG(WARNING) << "Overriding previous 'ro.' property '" << key << "':'" - << it->second << "' with new value '" << value << "'"; + LOG(WARNING) << "Overriding previous property '" << key << "':'" << it->second + << "' with new value '" << value << "'"; it->second = value; } } else { @@ -874,28 +877,32 @@ static void property_derive_build_fingerprint() { } void PropertyLoadBootDefaults() { - // TODO(b/117892318): merge prop.default and build.prop files into one // We read the properties and their values into a map, in order to always allow properties // loaded in the later property files to override the properties in loaded in the earlier // property files, regardless of if they are "ro." properties or not. std::map<std::string, std::string> properties; - if (!load_properties_from_file("/system/etc/prop.default", nullptr, &properties)) { - // Try recovery path - if (!load_properties_from_file("/prop.default", nullptr, &properties)) { - // Try legacy path - load_properties_from_file("/default.prop", nullptr, &properties); - } + + if (IsRecoveryMode()) { + load_properties_from_file("/prop.default", nullptr, &properties); } + load_properties_from_file("/system/build.prop", nullptr, &properties); load_properties_from_file("/system_ext/build.prop", nullptr, &properties); - load_properties_from_file("/vendor/default.prop", nullptr, &properties); + + // TODO(b/117892318): uncomment the following condition when vendor.imgs for + // aosp_* targets are all updated. +// if (SelinuxGetVendorAndroidVersion() <= __ANDROID_API_R__) { + load_properties_from_file("/vendor/default.prop", nullptr, &properties); +// } load_properties_from_file("/vendor/build.prop", nullptr, &properties); + if (SelinuxGetVendorAndroidVersion() >= __ANDROID_API_Q__) { load_properties_from_file("/odm/etc/build.prop", nullptr, &properties); } else { load_properties_from_file("/odm/default.prop", nullptr, &properties); load_properties_from_file("/odm/build.prop", nullptr, &properties); } + load_properties_from_file("/product/build.prop", nullptr, &properties); load_properties_from_file("/factory/factory.prop", "ro.*", &properties); diff --git a/init/reboot.cpp b/init/reboot.cpp index 23a07aa64..5f8b99191 100644 --- a/init/reboot.cpp +++ b/init/reboot.cpp @@ -544,6 +544,18 @@ static int StopServicesAndLogViolations(const std::vector<Service*>& services, return still_running; } +static Result<void> UnmountAllApexes() { + const char* args[] = {"/system/bin/apexd", "--unmount-all"}; + int status; + if (logwrap_fork_execvp(arraysize(args), args, &status, false, LOG_KLOG, true, nullptr) != 0) { + return ErrnoError() << "Failed to call '/system/bin/apexd --unmount-all'"; + } + if (WIFEXITED(status) && WEXITSTATUS(status) == 0) { + return {}; + } + return Error() << "'/system/bin/apexd --unmount-all' failed : " << status; +} + //* Reboot / shutdown the system. // cmd ANDROID_RB_* as defined in android_reboot.h // reason Reason string like "reboot", "shutdown,userrequested" @@ -701,6 +713,11 @@ static void DoReboot(unsigned int cmd, const std::string& reason, const std::str // 5. drop caches and disable zram backing device, if exist KillZramBackingDevice(); + LOG(INFO) << "Ready to unmount apexes. So far shutdown sequence took " << t; + // 6. unmount active apexes, otherwise they might prevent clean unmount of /data. + if (auto ret = UnmountAllApexes(); !ret.ok()) { + LOG(ERROR) << ret.error(); + } UmountStat stat = TryUmountAndFsck(cmd, run_fsck, shutdown_timeout - t.duration(), &reboot_semaphore); // Follow what linux shutdown is doing: one more sync with little bit delay @@ -739,18 +756,6 @@ static void LeaveShutdown() { StartSendingMessages(); } -static Result<void> UnmountAllApexes() { - const char* args[] = {"/system/bin/apexd", "--unmount-all"}; - int status; - if (logwrap_fork_execvp(arraysize(args), args, &status, false, LOG_KLOG, true, nullptr) != 0) { - return ErrnoError() << "Failed to call '/system/bin/apexd --unmount-all'"; - } - if (WIFEXITED(status) && WEXITSTATUS(status) == 0) { - return {}; - } - return Error() << "'/system/bin/apexd --unmount-all' failed : " << status; -} - static std::chrono::milliseconds GetMillisProperty(const std::string& name, std::chrono::milliseconds default_value) { auto value = GetUintProperty(name, static_cast<uint64_t>(default_value.count())); @@ -834,7 +839,7 @@ static Result<void> DoUserspaceReboot() { sub_reason = "apex"; return result; } - if (!SwitchToBootstrapMountNamespaceIfNeeded()) { + if (!SwitchToMountNamespaceIfNeeded(NS_BOOTSTRAP)) { sub_reason = "ns_switch"; return Error() << "Failed to switch to bootstrap namespace"; } diff --git a/init/security.cpp b/init/security.cpp index 6cbe642c0..2450d654b 100644 --- a/init/security.cpp +++ b/init/security.cpp @@ -128,8 +128,7 @@ static bool SetHighestAvailableOptionValue(const std::string& path, int min, int #define MMAP_RND_PATH "/proc/sys/vm/mmap_rnd_bits" #define MMAP_RND_COMPAT_PATH "/proc/sys/vm/mmap_rnd_compat_bits" -// __attribute__((unused)) due to lack of mips support: see mips block in SetMmapRndBitsAction -static bool __attribute__((unused)) SetMmapRndBitsMin(int start, int min, bool compat) { +static bool SetMmapRndBitsMin(int start, int min, bool compat) { std::string path; if (compat) { path = MMAP_RND_COMPAT_PATH; @@ -174,9 +173,6 @@ Result<void> SetMmapRndBitsAction(const BuiltinArguments&) { if (SetMmapRndBitsMin(16, 16, h64)) { return {}; } -#elif defined(__mips__) || defined(__mips64__) - // TODO: add mips support b/27788820 - return {}; #else LOG(ERROR) << "Unknown architecture"; #endif diff --git a/init/service.cpp b/init/service.cpp index 69f944eb2..68365b3c3 100644 --- a/init/service.cpp +++ b/init/service.cpp @@ -90,7 +90,9 @@ static Result<std::string> ComputeContextFromExecutable(const std::string& servi << "\") has incorrect label or no domain transition from " << mycon.get() << " to another SELinux domain defined. Have you configured your " "service correctly? https://source.android.com/security/selinux/" - "device-policy#label_new_services_and_address_denials"; + "device-policy#label_new_services_and_address_denials. Note: this " + "error shows up even in permissive mode in order to make auditing " + "denials possible."; } if (rc < 0) { return Error() << "Could not get process context"; @@ -463,6 +465,16 @@ Result<void> Service::Start() { pre_apexd_ = true; } + // For pre-apexd services, override mount namespace as "bootstrap" one before starting. + // Note: "ueventd" is supposed to be run in "default" mount namespace even if it's pre-apexd + // to support loading firmwares from APEXes. + std::optional<MountNamespace> override_mount_namespace; + if (name_ == "ueventd") { + override_mount_namespace = NS_DEFAULT; + } else if (pre_apexd_) { + override_mount_namespace = NS_BOOTSTRAP; + } + post_data_ = ServiceList::GetInstance().IsPostData(); LOG(INFO) << "starting service '" << name_ << "'..."; @@ -494,7 +506,8 @@ Result<void> Service::Start() { if (pid == 0) { umask(077); - if (auto result = EnterNamespaces(namespaces_, name_, pre_apexd_); !result.ok()) { + if (auto result = EnterNamespaces(namespaces_, name_, override_mount_namespace); + !result.ok()) { LOG(FATAL) << "Service '" << name_ << "' failed to set up namespaces: " << result.error(); } diff --git a/init/service_utils.cpp b/init/service_utils.cpp index 484c2c89f..05e632b68 100644 --- a/init/service_utils.cpp +++ b/init/service_utils.cpp @@ -194,7 +194,8 @@ Result<Descriptor> FileDescriptor::Create() const { return Descriptor(ANDROID_FILE_ENV_PREFIX + name, std::move(fd)); } -Result<void> EnterNamespaces(const NamespaceInfo& info, const std::string& name, bool pre_apexd) { +Result<void> EnterNamespaces(const NamespaceInfo& info, const std::string& name, + std::optional<MountNamespace> override_mount_namespace) { for (const auto& [nstype, path] : info.namespaces_to_enter) { if (auto result = EnterNamespace(nstype, path.c_str()); !result.ok()) { return result; @@ -202,9 +203,10 @@ Result<void> EnterNamespaces(const NamespaceInfo& info, const std::string& name, } #if defined(__ANDROID__) - if (pre_apexd) { - if (!SwitchToBootstrapMountNamespaceIfNeeded()) { - return Error() << "could not enter into the bootstrap mount namespace"; + if (override_mount_namespace.has_value()) { + if (auto result = SwitchToMountNamespaceIfNeeded(override_mount_namespace.value()); + !result.ok()) { + return result; } } #endif diff --git a/init/service_utils.h b/init/service_utils.h index 3f1071e5b..e74f8c17e 100644 --- a/init/service_utils.h +++ b/init/service_utils.h @@ -19,12 +19,14 @@ #include <sys/resource.h> #include <sys/types.h> +#include <optional> #include <string> #include <vector> #include <android-base/unique_fd.h> #include <cutils/iosched_policy.h> +#include "mount_namespace.h" #include "result.h" namespace android { @@ -66,7 +68,8 @@ struct NamespaceInfo { // Pair of namespace type, path to name. std::vector<std::pair<int, std::string>> namespaces_to_enter; }; -Result<void> EnterNamespaces(const NamespaceInfo& info, const std::string& name, bool pre_apexd); +Result<void> EnterNamespaces(const NamespaceInfo& info, const std::string& name, + std::optional<MountNamespace> override_mount_namespace); struct ProcessAttributes { std::string console; diff --git a/init/test_kill_services/Android.bp b/init/test_kill_services/Android.bp new file mode 100644 index 000000000..d59e548a4 --- /dev/null +++ b/init/test_kill_services/Android.bp @@ -0,0 +1,11 @@ +cc_test { + name: "init_kill_services_test", + srcs: ["init_kill_services_test.cpp"], + shared_libs: ["libbase"], + test_suites: ["general-tests"], + + // TODO(b/153565474): switch back to auto-generation + // and add back: + // require_root: true, + auto_gen_config: false, +} diff --git a/init/test_kill_services/AndroidTest.xml b/init/test_kill_services/AndroidTest.xml new file mode 100644 index 000000000..8018efafb --- /dev/null +++ b/init/test_kill_services/AndroidTest.xml @@ -0,0 +1,43 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2020 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration description="Runs init_kill_services_test."> + <option name="test-suite-tag" value="apct" /> + <option name="test-suite-tag" value="apct-native" /> + + <!-- cannot be autogenerated: b/153565474 --> + <target_preparer class="com.android.tradefed.targetprep.RebootTargetPreparer"> + <!-- flake mitigation, in case device is in bad state--> + <option name="pre-reboot" value="true" /> + <!-- sometimes device gets into bad state, and we don't detect it in this test, + so the test succeeds and the next test fails. This is a really bad result, so + to avoid that, making sure we reboot the device again before running any more + tests. + TODO(b/152556737): add metrics for successful device recovery --> + <option name="post-reboot" value="true" /> + </target_preparer> + + <target_preparer class="com.android.tradefed.targetprep.RootTargetPreparer"/> + + <target_preparer class="com.android.tradefed.targetprep.PushFilePreparer"> + <option name="cleanup" value="true" /> + <option name="push" value="init_kill_services_test->/data/local/tmp/init_kill_services_test" /> + </target_preparer> + + <test class="com.android.tradefed.testtype.GTest" > + <option name="native-test-device-path" value="/data/local/tmp" /> + <option name="module-name" value="init_kill_services_test" /> + </test> +</configuration> diff --git a/init/test_kill_services/init_kill_services_test.cpp b/init/test_kill_services/init_kill_services_test.cpp new file mode 100644 index 000000000..7e543f232 --- /dev/null +++ b/init/test_kill_services/init_kill_services_test.cpp @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <gtest/gtest.h> + +#include <android-base/properties.h> + +#include <iostream> + +using ::android::base::GetProperty; +using ::android::base::SetProperty; + +void ExpectKillingServiceRecovers(const std::string& service_name) { + const std::string status_prop = "init.svc." + service_name; + const std::string pid_prop = "init.svc_debug_pid." + service_name; + + const std::string initial_pid = GetProperty(pid_prop, ""); + + EXPECT_EQ("running", GetProperty(status_prop, "")) << status_prop; + EXPECT_NE("", initial_pid) << pid_prop; + + EXPECT_EQ(0, system(("kill -9 " + initial_pid).c_str())); + + constexpr size_t kMaxWaitMilliseconds = 10000; + constexpr size_t kRetryWaitMilliseconds = 100; + + constexpr size_t kRetryTimes = kMaxWaitMilliseconds / kRetryWaitMilliseconds; + + for (size_t retry = 0; retry < kRetryTimes; retry++) { + const std::string& pid = GetProperty(pid_prop, ""); + if (pid != initial_pid && pid != "") break; + usleep(kRetryWaitMilliseconds * 1000); + } + + // svc_debug_pid is set after svc property + EXPECT_EQ("running", GetProperty(status_prop, "")); +} + +class InitKillServicesTest : public ::testing::TestWithParam<std::string> {}; + +TEST_P(InitKillServicesTest, KillCriticalProcesses) { + ExpectKillingServiceRecovers(GetParam()); + + // sanity check init is still responding + EXPECT_TRUE(SetProperty("test.death.test", "asdf")); + EXPECT_EQ(GetProperty("test.death.test", ""), "asdf"); + EXPECT_TRUE(SetProperty("test.death.test", "")); +} + +static inline std::string PrintName(const testing::TestParamInfo<std::string>& info) { + return info.param; +} + +INSTANTIATE_TEST_CASE_P(DeathTest, InitKillServicesTest, + ::testing::Values("lmkd", "ueventd", "hwservicemanager", "servicemanager"), + PrintName); diff --git a/init/tokenizer_test.cpp b/init/tokenizer_test.cpp index 6b31683e0..012288458 100644 --- a/init/tokenizer_test.cpp +++ b/init/tokenizer_test.cpp @@ -28,7 +28,7 @@ namespace { void RunTest(const std::string& data, const std::vector<std::vector<std::string>>& expected_tokens) { auto data_copy = std::string{data}; - data_copy.push_back('\n'); // TODO: fix tokenizer + data_copy.push_back('\n'); data_copy.push_back('\0'); parse_state state; diff --git a/init/ueventd.cpp b/init/ueventd.cpp index d2b503b4f..7514b6184 100644 --- a/init/ueventd.cpp +++ b/init/ueventd.cpp @@ -285,7 +285,6 @@ int ueventd_main(int argc, char** argv) { // Keep the current product name base configuration so we remain backwards compatible and // allow it to override everything. - // TODO: cleanup platform ueventd.rc to remove vendor specific device node entries (b/34968103) auto hardware = android::base::GetProperty("ro.hardware", ""); auto ueventd_configuration = ParseConfig({"/system/etc/ueventd.rc", "/vendor/ueventd.rc", diff --git a/init/ueventd_test.cpp b/init/ueventd_test.cpp index 2d7d2f819..fc3cdfb7d 100644 --- a/init/ueventd_test.cpp +++ b/init/ueventd_test.cpp @@ -53,11 +53,7 @@ void WriteFromMultipleThreads(std::vector<std::pair<std::string, T>>& files_and_ }; std::vector<std::thread> threads; - // TODO(b/63712782): Structured bindings + templated containers are broken in clang :( - // for (const auto& [file, parameter] : files_and_parameters) { - for (const auto& pair : files_and_parameters) { - const auto& file = pair.first; - const auto& parameter = pair.second; + for (const auto& [file, parameter] : files_and_parameters) { threads.emplace_back(std::thread(make_thread_function(file, parameter))); } diff --git a/init/util.cpp b/init/util.cpp index 255434a1b..aec3173d3 100644 --- a/init/util.cpp +++ b/init/util.cpp @@ -30,7 +30,9 @@ #include <time.h> #include <unistd.h> +#include <mutex> #include <thread> +#include <vector> #include <android-base/file.h> #include <android-base/logging.h> @@ -722,5 +724,50 @@ bool IsRecoveryMode() { return access("/system/bin/recovery", F_OK) == 0; } +// TODO(b/155203339): remove this +// Devices in the lab seem to be stuck during shutdown, but the logs don't capture the last actions +// before shutdown started, so we record those lines, ignoring requests to shutdown, and replay them +// if we identify that the device is stuck. +constexpr size_t kRecordedLogsSize = 30; +std::string recorded_logs[kRecordedLogsSize]; +size_t recorded_log_position = 0; +std::mutex recorded_logs_lock; + +void InitSecondStageLogging(char** argv) { + SetFatalRebootTarget(); + auto second_stage_logger = [](android::base::LogId log_id, android::base::LogSeverity severity, + const char* tag, const char* file, unsigned int line, + const char* message) { + // We only store logs for init, not its children, and only if they're not related to + // sys.powerctl. + if (getpid() == 1 && strstr(message, "sys.powerctl") == nullptr) { + auto lock = std::lock_guard{recorded_logs_lock}; + recorded_logs[recorded_log_position++] = message; + if (recorded_log_position == kRecordedLogsSize) { + recorded_log_position = 0; + } + } + android::base::KernelLogger(log_id, severity, tag, file, line, message); + }; + android::base::InitLogging(argv, second_stage_logger, InitAborter); +} + +void DumpShutdownDebugInformation() { + auto lock = std::lock_guard{recorded_logs_lock}; + android::base::KernelLogger( + android::base::MAIN, android::base::ERROR, "init", nullptr, 0, + "===================== Dumping previous init lines ====================="); + for (size_t i = recorded_log_position; i < kRecordedLogsSize; ++i) { + android::base::KernelLogger(android::base::MAIN, android::base::ERROR, "init", nullptr, 0, + recorded_logs[i].c_str()); + } + for (size_t i = 0; i < recorded_log_position; ++i) { + android::base::KernelLogger(android::base::MAIN, android::base::ERROR, "init", nullptr, 0, + recorded_logs[i].c_str()); + } + android::base::KernelLogger(android::base::MAIN, android::base::ERROR, "init", nullptr, 0, + "===================== End of dump ====================="); +} + } // namespace init } // namespace android diff --git a/init/util.h b/init/util.h index 3cdc9f408..8a6aa6053 100644 --- a/init/util.h +++ b/init/util.h @@ -98,6 +98,8 @@ Result<std::string> ParseUmountAll(const std::vector<std::string>& args); void SetStdioToDevNull(char** argv); void InitKernelLogging(char** argv); +void InitSecondStageLogging(char** argv); +void DumpShutdownDebugInformation(); bool IsRecoveryMode(); } // namespace init } // namespace android diff --git a/init/util_test.cpp b/init/util_test.cpp index 96a5b5511..565e7d4f0 100644 --- a/init/util_test.cpp +++ b/init/util_test.cpp @@ -61,8 +61,8 @@ TEST(util, ReadFileWorldWiteable) { TEST(util, ReadFileSymbolicLink) { errno = 0; - // lrw------- 1 root root 23 2008-12-31 19:00 default.prop -> system/etc/prop.default - auto file_contents = ReadFile("/default.prop"); + // lrwxr-xr-x 1 root shell 6 2009-01-01 09:00 /system/bin/ps -> toybox + auto file_contents = ReadFile("/system/bin/ps"); EXPECT_EQ(ELOOP, errno); ASSERT_FALSE(file_contents.ok()); EXPECT_EQ("open() failed: Too many symbolic links encountered", diff --git a/libbacktrace/Android.bp b/libbacktrace/Android.bp index c4cfa6fbe..f75e8dfd4 100644 --- a/libbacktrace/Android.bp +++ b/libbacktrace/Android.bp @@ -126,7 +126,10 @@ cc_library { // Static library without DEX support to avoid dependencies on the ART APEX. cc_library_static { name: "libbacktrace_no_dex", - visibility: ["//system/core/debuggerd"], + visibility: [ + "//system/core/debuggerd", + "//system/core/init", + ], defaults: ["libbacktrace_defaults"], cflags: ["-DNO_LIBDEXFILE_SUPPORT"], target: { diff --git a/libcutils/Android.bp b/libcutils/Android.bp index 24110eeae..04b8f66f6 100644 --- a/libcutils/Android.bp +++ b/libcutils/Android.bp @@ -152,6 +152,7 @@ cc_library { "iosched_policy.cpp", "load_file.cpp", "native_handle.cpp", + "properties.cpp", "record_stream.cpp", "strlcpy.c", "threads.cpp", @@ -187,7 +188,6 @@ cc_library { "fs_config.cpp", "klog.cpp", "partition_utils.cpp", - "properties.cpp", "qtaguid.cpp", "trace-dev.cpp", "uevent.cpp", @@ -258,6 +258,7 @@ cc_defaults { name: "libcutils_test_default", srcs: [ "native_handle_test.cpp", + "properties_test.cpp", "sockets_test.cpp", ], @@ -269,7 +270,6 @@ cc_defaults { "ashmem_test.cpp", "fs_config_test.cpp", "multiuser_test.cpp", - "properties_test.cpp", "sched_policy_test.cpp", "str_parms_test.cpp", "trace-dev_test.cpp", diff --git a/libcutils/include/cutils/ashmem.h b/libcutils/include/cutils/ashmem.h index d80caa698..1913c1e4e 100644 --- a/libcutils/include/cutils/ashmem.h +++ b/libcutils/include/cutils/ashmem.h @@ -1,14 +1,20 @@ -/* cutils/ashmem.h - ** - ** Copyright 2008 The Android Open Source Project - ** - ** This file is dual licensed. It may be redistributed and/or modified - ** under the terms of the Apache 2.0 License OR version 2 of the GNU - ** General Public License. +/* + * Copyright (C) 2008 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ -#ifndef _CUTILS_ASHMEM_H -#define _CUTILS_ASHMEM_H +#pragma once #include <stddef.h> @@ -30,5 +36,3 @@ int ashmem_get_size_region(int fd); #ifdef __cplusplus } #endif - -#endif /* _CUTILS_ASHMEM_H */ diff --git a/libcutils/include/cutils/properties.h b/libcutils/include/cutils/properties.h index d2e08712a..78d8bc654 100644 --- a/libcutils/include/cutils/properties.h +++ b/libcutils/include/cutils/properties.h @@ -14,27 +14,30 @@ * limitations under the License. */ -#ifndef __CUTILS_PROPERTIES_H -#define __CUTILS_PROPERTIES_H +#pragma once #include <sys/cdefs.h> #include <stddef.h> -#include <sys/system_properties.h> #include <stdint.h> +#if __has_include(<sys/system_properties.h>) +#include <sys/system_properties.h> +#else +#define PROP_VALUE_MAX 92 +#endif + #ifdef __cplusplus extern "C" { #endif -/* System properties are *small* name value pairs managed by the -** property service. If your data doesn't fit in the provided -** space it is not appropriate for a system property. -** -** WARNING: system/bionic/include/sys/system_properties.h also defines -** these, but with different names. (TODO: fix that) -*/ -#define PROPERTY_KEY_MAX PROP_NAME_MAX -#define PROPERTY_VALUE_MAX PROP_VALUE_MAX +// +// Deprecated. +// +// See <android-base/properties.h> for better API. +// + +#define PROPERTY_KEY_MAX PROP_NAME_MAX +#define PROPERTY_VALUE_MAX PROP_VALUE_MAX /* property_get: returns the length of the value which will never be ** greater than PROPERTY_VALUE_MAX - 1 and will always be zero terminated. @@ -146,5 +149,3 @@ int property_get(const char *key, char *value, const char *default_value) { #ifdef __cplusplus } #endif - -#endif diff --git a/libcutils/include/private/android_filesystem_config.h b/libcutils/include/private/android_filesystem_config.h index b73a29b22..e4f45a85e 100644 --- a/libcutils/include/private/android_filesystem_config.h +++ b/libcutils/include/private/android_filesystem_config.h @@ -34,14 +34,7 @@ * partition, from which the system reads passwd and group files. */ -#ifndef _ANDROID_FILESYSTEM_CONFIG_H_ -#define _ANDROID_FILESYSTEM_CONFIG_H_ - -#include <sys/types.h> - -#if !defined(__ANDROID_VNDK__) && !defined(EXCLUDE_FS_CONFIG_STRUCTURES) -#include <private/fs_config.h> -#endif +#pragma once /* This is the master Users and Groups config for the platform. * DO NOT EVER RENUMBER @@ -224,5 +217,3 @@ * documented at the top of this header file. * Also see build/tools/fs_config for more details. */ - -#endif diff --git a/libcutils/include/private/canned_fs_config.h b/libcutils/include/private/canned_fs_config.h index 135b91c6c..ad4de4ce6 100644 --- a/libcutils/include/private/canned_fs_config.h +++ b/libcutils/include/private/canned_fs_config.h @@ -14,10 +14,10 @@ * limitations under the License. */ -#ifndef _CANNED_FS_CONFIG_H -#define _CANNED_FS_CONFIG_H +#pragma once #include <inttypes.h> +#include <sys/cdefs.h> __BEGIN_DECLS @@ -26,5 +26,3 @@ void canned_fs_config(const char* path, int dir, const char* target_out_path, un unsigned* gid, unsigned* mode, uint64_t* capabilities); __END_DECLS - -#endif diff --git a/libcutils/properties.cpp b/libcutils/properties.cpp index 5dbbeba48..03f04961c 100644 --- a/libcutils/properties.cpp +++ b/libcutils/properties.cpp @@ -16,27 +16,19 @@ #include <cutils/properties.h> -#define LOG_TAG "properties" -// #define LOG_NDEBUG 0 - -#include <assert.h> -#include <ctype.h> #include <errno.h> #include <inttypes.h> #include <stdlib.h> #include <string.h> #include <unistd.h> -#include <cutils/sockets.h> -#include <log/log.h> +#include <android-base/properties.h> -int8_t property_get_bool(const char *key, int8_t default_value) { - if (!key) { - return default_value; - } +int8_t property_get_bool(const char* key, int8_t default_value) { + if (!key) return default_value; int8_t result = default_value; - char buf[PROPERTY_VALUE_MAX] = {'\0'}; + char buf[PROPERTY_VALUE_MAX] = {}; int len = property_get(key, buf, ""); if (len == 1) { @@ -57,73 +49,53 @@ int8_t property_get_bool(const char *key, int8_t default_value) { return result; } -// Convert string property to int (default if fails); return default value if out of bounds -static intmax_t property_get_imax(const char *key, intmax_t lower_bound, intmax_t upper_bound, - intmax_t default_value) { - if (!key) { - return default_value; +template <typename T> +static T property_get_int(const char* key, T default_value) { + if (!key) return default_value; + + char value[PROPERTY_VALUE_MAX] = {}; + if (property_get(key, value, "") < 1) return default_value; + + // libcutils unwisely allows octal, which libbase doesn't. + T result = default_value; + int saved_errno = errno; + errno = 0; + char* end = nullptr; + intmax_t v = strtoimax(value, &end, 0); + if (errno != ERANGE && end != value && v >= std::numeric_limits<T>::min() && + v <= std::numeric_limits<T>::max()) { + result = v; } - - intmax_t result = default_value; - char buf[PROPERTY_VALUE_MAX] = {'\0'}; - char *end = NULL; - - int len = property_get(key, buf, ""); - if (len > 0) { - int tmp = errno; - errno = 0; - - // Infer base automatically - result = strtoimax(buf, &end, /*base*/ 0); - if ((result == INTMAX_MIN || result == INTMAX_MAX) && errno == ERANGE) { - // Over or underflow - result = default_value; - ALOGV("%s(%s,%" PRIdMAX ") - overflow", __FUNCTION__, key, default_value); - } else if (result < lower_bound || result > upper_bound) { - // Out of range of requested bounds - result = default_value; - ALOGV("%s(%s,%" PRIdMAX ") - out of range", __FUNCTION__, key, default_value); - } else if (end == buf) { - // Numeric conversion failed - result = default_value; - ALOGV("%s(%s,%" PRIdMAX ") - numeric conversion failed", __FUNCTION__, key, - default_value); - } - - errno = tmp; - } - + errno = saved_errno; return result; } -int64_t property_get_int64(const char *key, int64_t default_value) { - return (int64_t)property_get_imax(key, INT64_MIN, INT64_MAX, default_value); +int64_t property_get_int64(const char* key, int64_t default_value) { + return property_get_int<int64_t>(key, default_value); } -int32_t property_get_int32(const char *key, int32_t default_value) { - return (int32_t)property_get_imax(key, INT32_MIN, INT32_MAX, default_value); +int32_t property_get_int32(const char* key, int32_t default_value) { + return property_get_int<int32_t>(key, default_value); } -#define _REALLY_INCLUDE_SYS__SYSTEM_PROPERTIES_H_ -#include <sys/_system_properties.h> - -int property_set(const char *key, const char *value) { +int property_set(const char* key, const char* value) { return __system_property_set(key, value); } -int property_get(const char *key, char *value, const char *default_value) { +int property_get(const char* key, char* value, const char* default_value) { int len = __system_property_get(key, value); - if (len > 0) { - return len; - } - if (default_value) { - len = strnlen(default_value, PROPERTY_VALUE_MAX - 1); - memcpy(value, default_value, len); - value[len] = '\0'; + if (len < 1 && default_value) { + snprintf(value, PROPERTY_VALUE_MAX, "%s", default_value); + return strlen(value); } return len; } +#if __has_include(<sys/system_properties.h>) + +#define _REALLY_INCLUDE_SYS__SYSTEM_PROPERTIES_H_ +#include <sys/_system_properties.h> + struct callback_data { void (*callback)(const char* name, const char* value, void* cookie); void* cookie; @@ -139,6 +111,8 @@ static void property_list_callback(const prop_info* pi, void* data) { } int property_list(void (*fn)(const char* name, const char* value, void* cookie), void* cookie) { - callback_data data = { fn, cookie }; + callback_data data = {fn, cookie}; return __system_property_foreach(property_list_callback, &data); } + +#endif diff --git a/libcutils/properties_test.cpp b/libcutils/properties_test.cpp index 79219720e..efc01833a 100644 --- a/libcutils/properties_test.cpp +++ b/libcutils/properties_test.cpp @@ -93,160 +93,179 @@ protected: } }; -TEST_F(PropertiesTest, SetString) { - +TEST_F(PropertiesTest, property_set_null_key) { // Null key -> unsuccessful set - { - // Null key -> fails - EXPECT_GT(0, property_set(/*key*/NULL, PROPERTY_TEST_VALUE_DEFAULT)); - } + EXPECT_GT(0, property_set(/*key*/ NULL, PROPERTY_TEST_VALUE_DEFAULT)); +} - // Null value -> returns default value - { - // Null value -> OK , and it clears the value - EXPECT_OK(property_set(PROPERTY_TEST_KEY, /*value*/NULL)); - ResetValue(); +TEST_F(PropertiesTest, property_set_null_value) { + // Null value -> OK, and it clears the value + EXPECT_OK(property_set(PROPERTY_TEST_KEY, /*value*/ NULL)); + ResetValue(); - // Since the value is null, default value will be returned - size_t len = property_get(PROPERTY_TEST_KEY, mValue, PROPERTY_TEST_VALUE_DEFAULT); - EXPECT_EQ(strlen(PROPERTY_TEST_VALUE_DEFAULT), len); - EXPECT_STREQ(PROPERTY_TEST_VALUE_DEFAULT, mValue); - } + // Since the value is null, default value will be returned + size_t len = property_get(PROPERTY_TEST_KEY, mValue, PROPERTY_TEST_VALUE_DEFAULT); + EXPECT_EQ(strlen(PROPERTY_TEST_VALUE_DEFAULT), len); + EXPECT_STREQ(PROPERTY_TEST_VALUE_DEFAULT, mValue); +} +TEST_F(PropertiesTest, property_set) { // Trivial case => get returns what was set - { - size_t len = SetAndGetProperty("hello_world"); - EXPECT_EQ(strlen("hello_world"), len) << "hello_world key"; - EXPECT_STREQ("hello_world", mValue); - ResetValue(); - } + size_t len = SetAndGetProperty("hello_world"); + EXPECT_EQ(strlen("hello_world"), len) << "hello_world key"; + EXPECT_STREQ("hello_world", mValue); + ResetValue(); +} +TEST_F(PropertiesTest, property_set_empty) { // Set to empty string => get returns default always - { - const char* EMPTY_STRING_DEFAULT = "EMPTY_STRING"; - size_t len = SetAndGetProperty("", EMPTY_STRING_DEFAULT); - EXPECT_EQ(strlen(EMPTY_STRING_DEFAULT), len) << "empty key"; - EXPECT_STREQ(EMPTY_STRING_DEFAULT, mValue); - ResetValue(); - } + const char* EMPTY_STRING_DEFAULT = "EMPTY_STRING"; + size_t len = SetAndGetProperty("", EMPTY_STRING_DEFAULT); + EXPECT_EQ(strlen(EMPTY_STRING_DEFAULT), len) << "empty key"; + EXPECT_STREQ(EMPTY_STRING_DEFAULT, mValue); + ResetValue(); +} +TEST_F(PropertiesTest, property_set_max_length) { // Set to max length => get returns what was set - { - std::string maxLengthString = std::string(PROPERTY_VALUE_MAX-1, 'a'); + std::string maxLengthString = std::string(PROPERTY_VALUE_MAX - 1, 'a'); - int len = SetAndGetProperty(maxLengthString.c_str()); - EXPECT_EQ(PROPERTY_VALUE_MAX-1, len) << "max length key"; - EXPECT_STREQ(maxLengthString.c_str(), mValue); - ResetValue(); - } + int len = SetAndGetProperty(maxLengthString.c_str()); + EXPECT_EQ(PROPERTY_VALUE_MAX - 1, len) << "max length key"; + EXPECT_STREQ(maxLengthString.c_str(), mValue); + ResetValue(); +} +TEST_F(PropertiesTest, property_set_too_long) { // Set to max length + 1 => set fails - { - const char* VALID_TEST_VALUE = "VALID_VALUE"; - ASSERT_OK(property_set(PROPERTY_TEST_KEY, VALID_TEST_VALUE)); + const char* VALID_TEST_VALUE = "VALID_VALUE"; + ASSERT_OK(property_set(PROPERTY_TEST_KEY, VALID_TEST_VALUE)); - std::string oneLongerString = std::string(PROPERTY_VALUE_MAX, 'a'); + std::string oneLongerString = std::string(PROPERTY_VALUE_MAX, 'a'); - // Expect that the value set fails since it's too long - EXPECT_GT(0, property_set(PROPERTY_TEST_KEY, oneLongerString.c_str())); - size_t len = property_get(PROPERTY_TEST_KEY, mValue, PROPERTY_TEST_VALUE_DEFAULT); + // Expect that the value set fails since it's too long + EXPECT_GT(0, property_set(PROPERTY_TEST_KEY, oneLongerString.c_str())); + size_t len = property_get(PROPERTY_TEST_KEY, mValue, PROPERTY_TEST_VALUE_DEFAULT); - EXPECT_EQ(strlen(VALID_TEST_VALUE), len) << "set should've failed"; - EXPECT_STREQ(VALID_TEST_VALUE, mValue); - ResetValue(); - } + EXPECT_EQ(strlen(VALID_TEST_VALUE), len) << "set should've failed"; + EXPECT_STREQ(VALID_TEST_VALUE, mValue); + ResetValue(); } -TEST_F(PropertiesTest, GetString) { - +TEST_F(PropertiesTest, property_get_too_long) { // Try to use a default value that's too long => get truncates the value - { - ASSERT_OK(property_set(PROPERTY_TEST_KEY, "")); + ASSERT_OK(property_set(PROPERTY_TEST_KEY, "")); - std::string maxLengthString = std::string(PROPERTY_VALUE_MAX - 1, 'a'); - std::string oneLongerString = std::string(PROPERTY_VALUE_MAX, 'a'); + std::string maxLengthString = std::string(PROPERTY_VALUE_MAX - 1, 'a'); + std::string oneLongerString = std::string(PROPERTY_VALUE_MAX, 'a'); - // Expect that the value is truncated since it's too long (by 1) - int len = property_get(PROPERTY_TEST_KEY, mValue, oneLongerString.c_str()); - EXPECT_EQ(PROPERTY_VALUE_MAX - 1, len); - EXPECT_STREQ(maxLengthString.c_str(), mValue); - ResetValue(); - } + // Expect that the value is truncated since it's too long (by 1) + int len = property_get(PROPERTY_TEST_KEY, mValue, oneLongerString.c_str()); + EXPECT_EQ(PROPERTY_VALUE_MAX - 1, len); + EXPECT_STREQ(maxLengthString.c_str(), mValue); + ResetValue(); +} +TEST_F(PropertiesTest, property_get_default_too_long) { // Try to use a default value that's the max length => get succeeds - { - ASSERT_OK(property_set(PROPERTY_TEST_KEY, "")); + ASSERT_OK(property_set(PROPERTY_TEST_KEY, "")); - std::string maxLengthString = std::string(PROPERTY_VALUE_MAX - 1, 'b'); + std::string maxLengthString = std::string(PROPERTY_VALUE_MAX - 1, 'b'); - // Expect that the value matches maxLengthString - int len = property_get(PROPERTY_TEST_KEY, mValue, maxLengthString.c_str()); - EXPECT_EQ(PROPERTY_VALUE_MAX - 1, len); - EXPECT_STREQ(maxLengthString.c_str(), mValue); - ResetValue(); - } + // Expect that the value matches maxLengthString + int len = property_get(PROPERTY_TEST_KEY, mValue, maxLengthString.c_str()); + EXPECT_EQ(PROPERTY_VALUE_MAX - 1, len); + EXPECT_STREQ(maxLengthString.c_str(), mValue); + ResetValue(); +} +TEST_F(PropertiesTest, property_get_default_okay) { // Try to use a default value of length one => get succeeds - { - ASSERT_OK(property_set(PROPERTY_TEST_KEY, "")); + ASSERT_OK(property_set(PROPERTY_TEST_KEY, "")); - std::string oneCharString = std::string(1, 'c'); + std::string oneCharString = std::string(1, 'c'); - // Expect that the value matches oneCharString - int len = property_get(PROPERTY_TEST_KEY, mValue, oneCharString.c_str()); - EXPECT_EQ(1, len); - EXPECT_STREQ(oneCharString.c_str(), mValue); - ResetValue(); - } + // Expect that the value matches oneCharString + int len = property_get(PROPERTY_TEST_KEY, mValue, oneCharString.c_str()); + EXPECT_EQ(1, len); + EXPECT_STREQ(oneCharString.c_str(), mValue); + ResetValue(); +} +TEST_F(PropertiesTest, property_get_default_empty) { // Try to use a default value of length zero => get succeeds - { - ASSERT_OK(property_set(PROPERTY_TEST_KEY, "")); + ASSERT_OK(property_set(PROPERTY_TEST_KEY, "")); - std::string zeroCharString = std::string(0, 'd'); + std::string zeroCharString = std::string(0, 'd'); - // Expect that the value matches oneCharString - int len = property_get(PROPERTY_TEST_KEY, mValue, zeroCharString.c_str()); - EXPECT_EQ(0, len); - EXPECT_STREQ(zeroCharString.c_str(), mValue); - ResetValue(); - } + // Expect that the value matches oneCharString + int len = property_get(PROPERTY_TEST_KEY, mValue, zeroCharString.c_str()); + EXPECT_EQ(0, len); + EXPECT_STREQ(zeroCharString.c_str(), mValue); + ResetValue(); +} +TEST_F(PropertiesTest, property_get_default_NULL) { // Try to use a NULL default value => get returns 0 - { - ASSERT_OK(property_set(PROPERTY_TEST_KEY, "")); + ASSERT_OK(property_set(PROPERTY_TEST_KEY, "")); - // Expect a return value of 0 - int len = property_get(PROPERTY_TEST_KEY, mValue, NULL); - EXPECT_EQ(0, len); - ResetValue(); - } + // Expect a return value of 0 + int len = property_get(PROPERTY_TEST_KEY, mValue, NULL); + EXPECT_EQ(0, len); + ResetValue(); } -TEST_F(PropertiesTest, GetBool) { - /** - * TRUE - */ - const char *valuesTrue[] = { "1", "true", "y", "yes", "on", }; - for (size_t i = 0; i < arraysize(valuesTrue); ++i) { - ASSERT_OK(property_set(PROPERTY_TEST_KEY, valuesTrue[i])); - bool val = property_get_bool(PROPERTY_TEST_KEY, /*default_value*/false); - EXPECT_TRUE(val) << "Property should've been TRUE for value: '" << valuesTrue[i] << "'"; - } +TEST_F(PropertiesTest, property_get_bool_0) { + ASSERT_OK(property_set(PROPERTY_TEST_KEY, "0")); + ASSERT_FALSE(property_get_bool(PROPERTY_TEST_KEY, true)); +} - /** - * FALSE - */ - const char *valuesFalse[] = { "0", "false", "n", "no", "off", }; - for (size_t i = 0; i < arraysize(valuesFalse); ++i) { - ASSERT_OK(property_set(PROPERTY_TEST_KEY, valuesFalse[i])); - bool val = property_get_bool(PROPERTY_TEST_KEY, /*default_value*/true); - EXPECT_FALSE(val) << "Property shoud've been FALSE For string value: '" << valuesFalse[i] << "'"; - } +TEST_F(PropertiesTest, property_get_bool_1) { + ASSERT_OK(property_set(PROPERTY_TEST_KEY, "1")); + ASSERT_TRUE(property_get_bool(PROPERTY_TEST_KEY, false)); +} + +TEST_F(PropertiesTest, property_get_bool_false) { + ASSERT_OK(property_set(PROPERTY_TEST_KEY, "false")); + ASSERT_FALSE(property_get_bool(PROPERTY_TEST_KEY, true)); +} + +TEST_F(PropertiesTest, property_get_bool_n) { + ASSERT_OK(property_set(PROPERTY_TEST_KEY, "n")); + ASSERT_FALSE(property_get_bool(PROPERTY_TEST_KEY, true)); +} + +TEST_F(PropertiesTest, property_get_bool_no) { + ASSERT_OK(property_set(PROPERTY_TEST_KEY, "no")); + ASSERT_FALSE(property_get_bool(PROPERTY_TEST_KEY, true)); +} + +TEST_F(PropertiesTest, property_get_bool_off) { + ASSERT_OK(property_set(PROPERTY_TEST_KEY, "off")); + ASSERT_FALSE(property_get_bool(PROPERTY_TEST_KEY, true)); +} + +TEST_F(PropertiesTest, property_get_bool_on) { + ASSERT_OK(property_set(PROPERTY_TEST_KEY, "on")); + ASSERT_TRUE(property_get_bool(PROPERTY_TEST_KEY, false)); +} + +TEST_F(PropertiesTest, property_get_bool_true) { + ASSERT_OK(property_set(PROPERTY_TEST_KEY, "true")); + ASSERT_TRUE(property_get_bool(PROPERTY_TEST_KEY, false)); +} + +TEST_F(PropertiesTest, property_get_bool_y) { + ASSERT_OK(property_set(PROPERTY_TEST_KEY, "y")); + ASSERT_TRUE(property_get_bool(PROPERTY_TEST_KEY, false)); +} + +TEST_F(PropertiesTest, property_get_bool_yes) { + ASSERT_OK(property_set(PROPERTY_TEST_KEY, "yes")); + ASSERT_TRUE(property_get_bool(PROPERTY_TEST_KEY, false)); +} - /** - * NEITHER - */ +TEST_F(PropertiesTest, property_get_bool_neither) { const char *valuesNeither[] = { "x0", "x1", "2", "-2", "True", "False", "garbage", "", " ", "+1", " 1 ", " true", " true ", " y ", " yes", "yes ", "+0", "-0", "00", " 00 ", " false", "false ", @@ -263,7 +282,7 @@ TEST_F(PropertiesTest, GetBool) { } } -TEST_F(PropertiesTest, GetInt64) { +TEST_F(PropertiesTest, property_get_int64) { const int64_t DEFAULT_VALUE = INT64_C(0xDEADBEEFBEEFDEAD); const std::string longMaxString = ToString(INT64_MAX); @@ -310,7 +329,7 @@ TEST_F(PropertiesTest, GetInt64) { } } -TEST_F(PropertiesTest, GetInt32) { +TEST_F(PropertiesTest, property_get_int32) { const int32_t DEFAULT_VALUE = INT32_C(0xDEADBEEF); const std::string intMaxString = ToString(INT32_MAX); diff --git a/liblog/README.md b/liblog/README.md index 871399a57..74a2cd746 100644 --- a/liblog/README.md +++ b/liblog/README.md @@ -60,8 +60,6 @@ Public Functions and Macros LOG_EVENT_INT(tag, value) LOG_EVENT_LONG(tag, value) - clockid_t android_log_clockid() - log_id_t android_logger_get_id(struct logger *logger) int android_logger_clear(struct logger *logger) int android_logger_get_log_size(struct logger *logger) @@ -118,10 +116,9 @@ at a time in time sorted order, optionally limited to a specific pid and tail of finally a call closing the logs. A single log can be opened with `android_logger_list_open()`; or multiple logs can be opened with `android_logger_list_alloc()`, calling in turn the `android_logger_open()` for each log id. Each entry can be retrieved with -`android_logger_list_read()`. The log(s) can be closed with `android_logger_list_free()`. The logs -should be opened with an `ANDROID_LOG_RDONLY` mode. `ANDROID_LOG_NONBLOCK` mode will report when -the log reading is done with an `EAGAIN` error return code, otherwise the -`android_logger_list_read()` call will block for new entries. +`android_logger_list_read()`. The log(s) can be closed with `android_logger_list_free()`. +`ANDROID_LOG_NONBLOCK` mode will report when the log reading is done with an `EAGAIN` error return +code, otherwise the `android_logger_list_read()` call will block for new entries. The `ANDROID_LOG_WRAP` mode flag to the `android_logger_list_alloc_time()` signals logd to quiesce the reader until the buffer is about to prune at the start time then proceed to dumping content. @@ -130,14 +127,12 @@ The `ANDROID_LOG_PSTORE` mode flag to the `android_logger_open()` is used to swi logs to the persistent logs from before the last reboot. The value returned by `android_logger_open()` can be used as a parameter to the -`android_logger_clear()` function to empty the sub-log. It is recommended to only open log -`ANDROID_LOG_WRONLY` in that case. +`android_logger_clear()` function to empty the sub-log. The value returned by `android_logger_open()` can be used as a parameter to the `android_logger_get_log_(size|readable_size|version)` to retrieve the sub-log maximum size, readable size and log buffer format protocol version respectively. `android_logger_get_id()` returns the id -that was used when opening the sub-log. It is recommended to open the log `ANDROID_LOG_RDONLY` in -these cases. +that was used when opening the sub-log. Errors ------ diff --git a/liblog/README.protocol.md b/liblog/README.protocol.md index fef29c921..f247b2817 100644 --- a/liblog/README.protocol.md +++ b/liblog/README.protocol.md @@ -17,6 +17,49 @@ The data that liblog sends to logd is represented below. }; }; +where the embedded structs are defined as: + + struct android_log_header_t { + uint8_t id; + uint16_t tid; + log_time realtime; + }; + + struct log_time { + uint32_t tv_sec = 0; + uint32_t tv_nsec = 0; + } + + struct android_event_header_t { + int32_t tag; + }; + + struct android_event_list_t { + int8_t type; // EVENT_TYPE_LIST + int8_t element_count; + }; + + struct android_event_float_t { + int8_t type; // EVENT_TYPE_FLOAT + float data; + }; + + struct android_event_int_t { + int8_t type; // EVENT_TYPE_INT + int32_t data; + } android_event_int_t; + + struct android_event_long_t { + int8_t type; // EVENT_TYPE_LONG + int64_t data; + }; + + struct android_event_string_t { + int8_t type; // EVENT_TYPE_STRING; + int32_t length; + char data[]; + }; + The payload, excluding the header, has a max size of LOGGER_ENTRY_MAX_PAYLOAD. ## header diff --git a/liblog/include/android/log.h b/liblog/include/android/log.h index 512c7cd7b..8a0ebf22f 100644 --- a/liblog/include/android/log.h +++ b/liblog/include/android/log.h @@ -185,14 +185,26 @@ int __android_log_buf_print(int bufID, int prio, const char* tag, const char* fm * and sending log messages to user defined loggers specified in __android_log_set_logger(). */ struct __android_log_message { - size_t - struct_size; /** Must be set to sizeof(__android_log_message) and is used for versioning. */ - int32_t buffer_id; /** {@link log_id_t} values. */ - int32_t priority; /** {@link android_LogPriority} values. */ - const char* tag; /** The tag for the log message. */ - const char* file; /** Optional file name, may be set to nullptr. */ - uint32_t line; /** Optional line number, ignore if file is nullptr. */ - const char* message; /** The log message itself. */ + /** Must be set to sizeof(__android_log_message) and is used for versioning. */ + size_t struct_size; + + /** {@link log_id_t} values. */ + int32_t buffer_id; + + /** {@link android_LogPriority} values. */ + int32_t priority; + + /** The tag for the log message. */ + const char* tag; + + /** Optional file name, may be set to nullptr. */ + const char* file; + + /** Optional line number, ignore if file is nullptr. */ + uint32_t line; + + /** The log message itself. */ + const char* message; }; /** @@ -215,7 +227,7 @@ typedef void (*__android_aborter_function)(const char* abort_message); * buffers, then pass the message to liblog via this function, and therefore we do not want to * duplicate the loggability check here. * - * @param log_message the log message itself, see {@link __android_log_message}. + * @param log_message the log message itself, see __android_log_message. * * Available since API level 30. */ @@ -237,7 +249,7 @@ void __android_log_set_logger(__android_logger_function logger) __INTRODUCED_IN( * Writes the log message to logd. This is an __android_logger_function and can be provided to * __android_log_set_logger(). It is the default logger when running liblog on a device. * - * @param log_message the log message to write, see {@link __android_log_message}. + * @param log_message the log message to write, see __android_log_message. * * Available since API level 30. */ @@ -247,7 +259,7 @@ void __android_log_logd_logger(const struct __android_log_message* log_message) * Writes the log message to stderr. This is an __android_logger_function and can be provided to * __android_log_set_logger(). It is the default logger when running liblog on host. * - * @param log_message the log message to write, see {@link __android_log_message}. + * @param log_message the log message to write, see __android_log_message. * * Available since API level 30. */ @@ -259,7 +271,7 @@ void __android_log_stderr_logger(const struct __android_log_message* log_message * user defined aborter function is highly recommended to abort and be noreturn, but is not strictly * required to. * - * @param aborter the new aborter function, see {@link __android_aborter_function}. + * @param aborter the new aborter function, see __android_aborter_function. * * Available since API level 30. */ @@ -297,23 +309,41 @@ __INTRODUCED_IN(30); * minimum priority needed to log. If only one is set, then that value is used to determine the * minimum priority needed. If none are set, then default_priority is used. * - * @param prio the priority to test, takes {@link android_LogPriority} values. + * @param prio the priority to test, takes android_LogPriority values. * @param tag the tag to test. - * @param len the length of the tag. * @param default_prio the default priority to use if no properties or minimum priority are set. * @return an integer where 1 indicates that the message is loggable and 0 indicates that it is not. * * Available since API level 30. */ int __android_log_is_loggable(int prio, const char* tag, int default_prio) __INTRODUCED_IN(30); + +/** + * Use the per-tag properties "log.tag.<tagname>" along with the minimum priority from + * __android_log_set_minimum_priority() to determine if a log message with a given prio and tag will + * be printed. A non-zero result indicates yes, zero indicates false. + * + * If both a priority for a tag and a minimum priority are set by + * __android_log_set_minimum_priority(), then the lowest of the two values are to determine the + * minimum priority needed to log. If only one is set, then that value is used to determine the + * minimum priority needed. If none are set, then default_priority is used. + * + * @param prio the priority to test, takes android_LogPriority values. + * @param tag the tag to test. + * @param len the length of the tag. + * @param default_prio the default priority to use if no properties or minimum priority are set. + * @return an integer where 1 indicates that the message is loggable and 0 indicates that it is not. + * + * Available since API level 30. + */ int __android_log_is_loggable_len(int prio, const char* tag, size_t len, int default_prio) __INTRODUCED_IN(30); /** * Sets the minimum priority that will be logged for this process. * - * @param priority the new minimum priority to set, takes @{link android_LogPriority} values. - * @return the previous set minimum priority as @{link android_LogPriority} values, or + * @param priority the new minimum priority to set, takes android_LogPriority values. + * @return the previous set minimum priority as android_LogPriority values, or * ANDROID_LOG_DEFAULT if none was set. * * Available since API level 30. @@ -324,7 +354,7 @@ int32_t __android_log_set_minimum_priority(int32_t priority) __INTRODUCED_IN(30) * Gets the minimum priority that will be logged for this process. If none has been set by a * previous __android_log_set_minimum_priority() call, this returns ANDROID_LOG_DEFAULT. * - * @return the current minimum priority as @{link android_LogPriority} values, or + * @return the current minimum priority as android_LogPriority values, or * ANDROID_LOG_DEFAULT if none is set. * * Available since API level 30. diff --git a/liblog/include/log/log.h b/liblog/include/log/log.h index c116add4c..d7e9b7dab 100644 --- a/liblog/include/log/log.h +++ b/liblog/include/log/log.h @@ -29,7 +29,6 @@ #include <log/log_id.h> #include <log/log_main.h> #include <log/log_radio.h> -#include <log/log_read.h> #include <log/log_safetynet.h> #include <log/log_system.h> #include <log/log_time.h> @@ -65,6 +64,13 @@ extern "C" { #endif /* + * The maximum size of the log entry payload that can be + * written to the logger. An attempt to write more than + * this amount will result in a truncated log entry. + */ +#define LOGGER_ENTRY_MAX_PAYLOAD 4068 + +/* * Event logging. */ @@ -87,8 +93,6 @@ int __android_log_stats_bwrite(int32_t tag, const void* payload, size_t len); /* * Event log entry types. */ -#ifndef __AndroidEventLogType_defined -#define __AndroidEventLogType_defined typedef enum { /* Special markers for android_log_list_element type */ EVENT_TYPE_LIST_STOP = '\n', /* declare end of list */ @@ -101,9 +105,6 @@ typedef enum { EVENT_TYPE_LIST = 3, EVENT_TYPE_FLOAT = 4, } AndroidEventLogType; -#endif -#define sizeof_AndroidEventLogType sizeof(typeof_AndroidEventLogType) -#define typeof_AndroidEventLogType unsigned char #ifndef LOG_EVENT_INT #define LOG_EVENT_INT(_tag, _value) \ @@ -132,12 +133,6 @@ typedef enum { (void)__android_log_bswrite(_tag, _value); #endif -#ifdef __linux__ - -clockid_t android_log_clockid(void); - -#endif /* __linux__ */ - /* --------------------------------------------------------------------- */ /* diff --git a/liblog/include/log/log_id.h b/liblog/include/log/log_id.h index c8fafe760..8e4faeb06 100644 --- a/liblog/include/log/log_id.h +++ b/liblog/include/log/log_id.h @@ -23,14 +23,6 @@ extern "C" { #endif /* - * Send a simple string to the log. - */ -int __android_log_buf_write(int bufID, int prio, const char* tag, - const char* text); -int __android_log_buf_print(int bufID, int prio, const char* tag, const char* fmt, ...) - __attribute__((__format__(printf, 4, 5))); - -/* * log_id_t helpers */ log_id_t android_name_to_log_id(const char* logName); diff --git a/liblog/include/log/log_properties.h b/liblog/include/log/log_properties.h index 3a8af6deb..3497d63dc 100644 --- a/liblog/include/log/log_properties.h +++ b/liblog/include/log/log_properties.h @@ -1,11 +1,18 @@ /* -** -** Copyright 2017, The Android Open Source Project -** -** This file is dual licensed. It may be redistributed and/or modified -** under the terms of the Apache 2.0 License OR version 2 of the GNU -** General Public License. -*/ + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #pragma once diff --git a/liblog/include/log/log_radio.h b/liblog/include/log/log_radio.h index 8b8a3625f..f5525c134 100644 --- a/liblog/include/log/log_radio.h +++ b/liblog/include/log/log_radio.h @@ -17,7 +17,6 @@ #pragma once #include <android/log.h> -#include <log/log_id.h> /* * Normally we strip the effects of ALOGV (VERBOSE messages), diff --git a/liblog/include/log/log_read.h b/liblog/include/log/log_read.h index f9c1d691e..23d76f485 100644 --- a/liblog/include/log/log_read.h +++ b/liblog/include/log/log_read.h @@ -16,30 +16,18 @@ #pragma once -#include <sys/types.h> - -/* deal with possible sys/cdefs.h conflict with fcntl.h */ -#ifdef __unused -#define __unused_defined __unused -#undef __unused -#endif - -#include <fcntl.h> /* Pick up O_* macros */ - -/* restore definitions from above */ -#ifdef __unused_defined -#define __unused __attribute__((__unused__)) -#endif - #include <stdint.h> +#include <sys/types.h> -#include <log/log_id.h> +#include <android/log.h> #include <log/log_time.h> #ifdef __cplusplus extern "C" { #endif +#define ANDROID_LOG_WRAP_DEFAULT_TIMEOUT 7200 /* 2 hour default */ + /* * Native log reading interface section. See logcat for sample code. * @@ -60,13 +48,6 @@ struct logger_entry { }; /* - * The maximum size of the log entry payload that can be - * written to the logger. An attempt to write more than - * this amount will result in a truncated log entry. - */ -#define LOGGER_ENTRY_MAX_PAYLOAD 4068 - -/* * The maximum size of a log entry which can be read. * An attempt to read less than this amount may result * in read() returning EINVAL. @@ -79,32 +60,9 @@ struct log_msg { struct logger_entry entry; } __attribute__((aligned(4))); #ifdef __cplusplus - /* Matching log_time operators */ - bool operator==(const log_msg& T) const { - return (entry.sec == T.entry.sec) && (entry.nsec == T.entry.nsec); - } - bool operator!=(const log_msg& T) const { - return !(*this == T); - } - bool operator<(const log_msg& T) const { - return (entry.sec < T.entry.sec) || - ((entry.sec == T.entry.sec) && (entry.nsec < T.entry.nsec)); - } - bool operator>=(const log_msg& T) const { - return !(*this < T); - } - bool operator>(const log_msg& T) const { - return (entry.sec > T.entry.sec) || - ((entry.sec == T.entry.sec) && (entry.nsec > T.entry.nsec)); - } - bool operator<=(const log_msg& T) const { - return !(*this > T); - } uint64_t nsec() const { return static_cast<uint64_t>(entry.sec) * NS_PER_SEC + entry.nsec; } - - /* packet methods */ log_id_t id() { return static_cast<log_id_t>(entry.lid); } @@ -137,17 +95,10 @@ ssize_t android_logger_get_prune_list(struct logger_list* logger_list, char* buf, size_t len); int android_logger_set_prune_list(struct logger_list* logger_list, const char* buf, size_t len); -#define ANDROID_LOG_RDONLY O_RDONLY -#define ANDROID_LOG_WRONLY O_WRONLY -#define ANDROID_LOG_RDWR O_RDWR -#define ANDROID_LOG_ACCMODE O_ACCMODE -#ifndef O_NONBLOCK +/* The below values are used for the `mode` argument of the below functions. */ +/* Note that 0x00000003 were previously used and should be considered reserved. */ #define ANDROID_LOG_NONBLOCK 0x00000800 -#else -#define ANDROID_LOG_NONBLOCK O_NONBLOCK -#endif #define ANDROID_LOG_WRAP 0x40000000 /* Block until buffer about to wrap */ -#define ANDROID_LOG_WRAP_DEFAULT_TIMEOUT 7200 /* 2 hour default */ #define ANDROID_LOG_PSTORE 0x80000000 struct logger_list* android_logger_list_alloc(int mode, unsigned int tail, @@ -161,7 +112,6 @@ int android_logger_list_read(struct logger_list* logger_list, /* Multiple log_id_t opens */ struct logger* android_logger_open(struct logger_list* logger_list, log_id_t id); -#define android_logger_close android_logger_free /* Single log_id_t open */ struct logger_list* android_logger_list_open(log_id_t id, int mode, unsigned int tail, pid_t pid); diff --git a/liblog/include/log/log_safetynet.h b/liblog/include/log/log_safetynet.h index d3e9b1954..b2604b554 100644 --- a/liblog/include/log/log_safetynet.h +++ b/liblog/include/log/log_safetynet.h @@ -1,11 +1,18 @@ /* -** -** Copyright 2017, The Android Open Source Project -** -** This file is dual licensed. It may be redistributed and/or modified -** under the terms of the Apache 2.0 License OR version 2 of the GNU -** General Public License. -*/ + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #pragma once diff --git a/liblog/include/log/log_system.h b/liblog/include/log/log_system.h index eaec741ea..6f40515e8 100644 --- a/liblog/include/log/log_system.h +++ b/liblog/include/log/log_system.h @@ -17,7 +17,6 @@ #pragma once #include <android/log.h> -#include <log/log_id.h> /* * Normally we strip the effects of ALOGV (VERBOSE messages), diff --git a/liblog/include/log/log_time.h b/liblog/include/log/log_time.h index 6b4458cae..f50764de3 100644 --- a/liblog/include/log/log_time.h +++ b/liblog/include/log/log_time.h @@ -37,9 +37,7 @@ struct log_time { uint32_t tv_sec = 0; /* good to Feb 5 2106 */ uint32_t tv_nsec = 0; - static const uint32_t tv_sec_max = 0xFFFFFFFFUL; - static const uint32_t tv_nsec_max = 999999999UL; - static const timespec EPOCH; + static constexpr timespec EPOCH = {0, 0}; log_time() {} explicit log_time(const timespec& T) @@ -55,16 +53,6 @@ struct log_time { tv_nsec = static_cast<uint32_t>(T.tv_nsec); } #endif - explicit log_time(const char* T) { - const uint8_t* c = reinterpret_cast<const uint8_t*>(T); - tv_sec = c[0] | (static_cast<uint32_t>(c[1]) << 8) | - (static_cast<uint32_t>(c[2]) << 16) | - (static_cast<uint32_t>(c[3]) << 24); - tv_nsec = c[4] | (static_cast<uint32_t>(c[5]) << 8) | - (static_cast<uint32_t>(c[6]) << 16) | - (static_cast<uint32_t>(c[7]) << 24); - } - /* timespec */ bool operator==(const timespec& T) const { return (tv_sec == static_cast<uint32_t>(T.tv_sec)) && @@ -90,17 +78,6 @@ struct log_time { return !(*this > T); } - log_time operator-=(const timespec& T); - log_time operator-(const timespec& T) const { - log_time local(*this); - return local -= T; - } - log_time operator+=(const timespec& T); - log_time operator+(const timespec& T) const { - log_time local(*this); - return local += T; - } - /* log_time */ bool operator==(const log_time& T) const { return (tv_sec == T.tv_sec) && (tv_nsec == T.tv_nsec); @@ -123,12 +100,36 @@ struct log_time { return !(*this > T); } - log_time operator-=(const log_time& T); + log_time operator-=(const log_time& T) { + // No concept of negative time, clamp to EPOCH + if (*this <= T) { + return *this = log_time(EPOCH); + } + + if (this->tv_nsec < T.tv_nsec) { + --this->tv_sec; + this->tv_nsec = NS_PER_SEC + this->tv_nsec - T.tv_nsec; + } else { + this->tv_nsec -= T.tv_nsec; + } + this->tv_sec -= T.tv_sec; + + return *this; + } log_time operator-(const log_time& T) const { log_time local(*this); return local -= T; } - log_time operator+=(const log_time& T); + log_time operator+=(const log_time& T) { + this->tv_nsec += T.tv_nsec; + if (this->tv_nsec >= NS_PER_SEC) { + this->tv_nsec -= NS_PER_SEC; + ++this->tv_sec; + } + this->tv_sec += T.tv_sec; + + return *this; + } log_time operator+(const log_time& T) const { log_time local(*this); return local += T; @@ -146,10 +147,8 @@ struct log_time { tv_nsec / (NS_PER_SEC / MS_PER_SEC); } - static const char default_format[]; - /* Add %#q for the fraction of a second to the standard library functions */ - char* strptime(const char* s, const char* format = default_format); + char* strptime(const char* s, const char* format); } __attribute__((__packed__)); } diff --git a/liblog/include_vndk/log/log.h b/liblog/include_vndk/log/log.h index a79beecd4..ab4adc49a 100644 --- a/liblog/include_vndk/log/log.h +++ b/liblog/include_vndk/log/log.h @@ -3,6 +3,9 @@ #ifndef _LIBS_LOG_LOG_H #define _LIBS_LOG_LOG_H +/* Historically vendors have depended on this header being included. */ +#include <fcntl.h> + #include <android/log.h> #include <log/log_id.h> #include <log/log_main.h> diff --git a/liblog/liblog.map.txt b/liblog/liblog.map.txt index 6ca1a164d..161fcf1a9 100644 --- a/liblog/liblog.map.txt +++ b/liblog/liblog.map.txt @@ -1,6 +1,6 @@ LIBLOG { global: - android_name_to_log_id; # llndk + android_name_to_log_id; # apex llndk android_log_id_to_name; # llndk __android_log_assert; __android_log_buf_print; @@ -22,7 +22,7 @@ LIBLOG_L { android_logger_list_alloc; # apex llndk android_logger_list_alloc_time; # apex llndk android_logger_list_free; # apex llndk - android_logger_list_open; # llndk + android_logger_list_open; # apex llndk android_logger_list_read; # apex llndk android_logger_open; # apex llndk android_logger_set_log_size; # llndk diff --git a/liblog/log_time.cpp b/liblog/log_time.cpp index 3fbe1cbc5..14c408c1c 100644 --- a/liblog/log_time.cpp +++ b/liblog/log_time.cpp @@ -21,11 +21,7 @@ #include <private/android_logger.h> -const char log_time::default_format[] = "%m-%d %H:%M:%S.%q"; -const timespec log_time::EPOCH = {0, 0}; - // Add %#q for fractional seconds to standard strptime function - char* log_time::strptime(const char* s, const char* format) { time_t now; #ifdef __linux__ @@ -131,59 +127,3 @@ char* log_time::strptime(const char* s, const char* format) { #endif return ret; } - -log_time log_time::operator-=(const timespec& T) { - // No concept of negative time, clamp to EPOCH - if (*this <= T) { - return *this = log_time(EPOCH); - } - - if (this->tv_nsec < (unsigned long int)T.tv_nsec) { - --this->tv_sec; - this->tv_nsec = NS_PER_SEC + this->tv_nsec - T.tv_nsec; - } else { - this->tv_nsec -= T.tv_nsec; - } - this->tv_sec -= T.tv_sec; - - return *this; -} - -log_time log_time::operator+=(const timespec& T) { - this->tv_nsec += (unsigned long int)T.tv_nsec; - if (this->tv_nsec >= NS_PER_SEC) { - this->tv_nsec -= NS_PER_SEC; - ++this->tv_sec; - } - this->tv_sec += T.tv_sec; - - return *this; -} - -log_time log_time::operator-=(const log_time& T) { - // No concept of negative time, clamp to EPOCH - if (*this <= T) { - return *this = log_time(EPOCH); - } - - if (this->tv_nsec < T.tv_nsec) { - --this->tv_sec; - this->tv_nsec = NS_PER_SEC + this->tv_nsec - T.tv_nsec; - } else { - this->tv_nsec -= T.tv_nsec; - } - this->tv_sec -= T.tv_sec; - - return *this; -} - -log_time log_time::operator+=(const log_time& T) { - this->tv_nsec += T.tv_nsec; - if (this->tv_nsec >= NS_PER_SEC) { - this->tv_nsec -= NS_PER_SEC; - ++this->tv_sec; - } - this->tv_sec += T.tv_sec; - - return *this; -} diff --git a/liblog/logger_write.cpp b/liblog/logger_write.cpp index b1bed80e9..22c7ecaff 100644 --- a/liblog/logger_write.cpp +++ b/liblog/logger_write.cpp @@ -192,7 +192,7 @@ static int write_to_log(log_id_t log_id, struct iovec* vec, size_t nr) { return -EINVAL; } - clock_gettime(android_log_clockid(), &ts); + clock_gettime(CLOCK_REALTIME, &ts); if (log_id == LOG_ID_SECURITY) { if (vec[0].iov_len < 4) { @@ -334,7 +334,7 @@ int __android_log_vprint(int prio, const char* tag, const char* fmt, va_list ap) return -EPERM; } - char buf[LOG_BUF_SIZE]; + __attribute__((uninitialized)) char buf[LOG_BUF_SIZE]; vsnprintf(buf, LOG_BUF_SIZE, fmt, ap); @@ -352,7 +352,7 @@ int __android_log_print(int prio, const char* tag, const char* fmt, ...) { } va_list ap; - char buf[LOG_BUF_SIZE]; + __attribute__((uninitialized)) char buf[LOG_BUF_SIZE]; va_start(ap, fmt); vsnprintf(buf, LOG_BUF_SIZE, fmt, ap); @@ -372,7 +372,7 @@ int __android_log_buf_print(int bufID, int prio, const char* tag, const char* fm } va_list ap; - char buf[LOG_BUF_SIZE]; + __attribute__((uninitialized)) char buf[LOG_BUF_SIZE]; va_start(ap, fmt); vsnprintf(buf, LOG_BUF_SIZE, fmt, ap); @@ -385,7 +385,7 @@ int __android_log_buf_print(int bufID, int prio, const char* tag, const char* fm } void __android_log_assert(const char* cond, const char* tag, const char* fmt, ...) { - char buf[LOG_BUF_SIZE]; + __attribute__((uninitialized)) char buf[LOG_BUF_SIZE]; if (fmt) { va_list ap; diff --git a/liblog/logprint.cpp b/liblog/logprint.cpp index 5c69bf863..238431f91 100644 --- a/liblog/logprint.cpp +++ b/liblog/logprint.cpp @@ -19,6 +19,8 @@ #define HAVE_STRSEP #endif +#include <log/logprint.h> + #include <assert.h> #include <ctype.h> #include <errno.h> @@ -37,7 +39,7 @@ #include <cutils/list.h> #include <log/log.h> -#include <log/logprint.h> +#include <log/log_read.h> #include <private/android_logger.h> #define MS_PER_NSEC 1000000 @@ -214,11 +216,7 @@ AndroidLogFormat* android_log_format_new() { p_ret->year_output = false; p_ret->zone_output = false; p_ret->epoch_output = false; -#ifdef __ANDROID__ - p_ret->monotonic_output = android_log_clockid() == CLOCK_MONOTONIC; -#else p_ret->monotonic_output = false; -#endif p_ret->uid_output = false; p_ret->descriptive_output = false; descriptive_output = false; @@ -1463,13 +1461,10 @@ char* android_log_formatLogLine(AndroidLogFormat* p_format, char* defaultBuffer, nsec = entry->tv_nsec; #if __ANDROID__ if (p_format->monotonic_output) { - /* prevent convertMonotonic from being called if logd is monotonic */ - if (android_log_clockid() != CLOCK_MONOTONIC) { - struct timespec time; - convertMonotonic(&time, entry); - now = time.tv_sec; - nsec = time.tv_nsec; - } + struct timespec time; + convertMonotonic(&time, entry); + now = time.tv_sec; + nsec = time.tv_nsec; } #endif if (now < 0) { diff --git a/liblog/pmsg_reader.cpp b/liblog/pmsg_reader.cpp index 3bdc30f09..564090087 100644 --- a/liblog/pmsg_reader.cpp +++ b/liblog/pmsg_reader.cpp @@ -185,7 +185,7 @@ ssize_t __android_log_pmsg_file_read(log_id_t logId, char prio, const char* pref /* Add just enough clues in logger_list and transp to make API function */ memset(&logger_list, 0, sizeof(logger_list)); - logger_list.mode = ANDROID_LOG_PSTORE | ANDROID_LOG_NONBLOCK | ANDROID_LOG_RDONLY; + logger_list.mode = ANDROID_LOG_PSTORE | ANDROID_LOG_NONBLOCK; logger_list.log_mask = (unsigned)-1; if (logId != LOG_ID_ANY) { logger_list.log_mask = (1 << logId); diff --git a/liblog/pmsg_writer.cpp b/liblog/pmsg_writer.cpp index 0751e2c53..8e676bdbf 100644 --- a/liblog/pmsg_writer.cpp +++ b/liblog/pmsg_writer.cpp @@ -188,7 +188,7 @@ ssize_t __android_log_pmsg_file_write(log_id_t logId, char prio, const char* fil return -EINVAL; } - clock_gettime(android_log_clockid(), &ts); + clock_gettime(CLOCK_REALTIME, &ts); cp = strdup(filename); if (!cp) { diff --git a/liblog/properties.cpp b/liblog/properties.cpp index 37670ecd6..f5e060c1f 100644 --- a/liblog/properties.cpp +++ b/liblog/properties.cpp @@ -365,29 +365,6 @@ static inline unsigned char do_cache2_char(struct cache2_char* self) { return c; } -static unsigned char evaluate_persist_ro(const struct cache2_char* self) { - unsigned char c = self->cache_persist.c; - - if (c) { - return c; - } - - return self->cache_ro.c; -} - -/* - * Timestamp state generally remains constant, but can change at any time - * to handle developer requirements. - */ -clockid_t android_log_clockid() { - static struct cache2_char clockid = {PTHREAD_MUTEX_INITIALIZER, 0, - "persist.logd.timestamp", {{NULL, 0xFFFFFFFF}, '\0'}, - "ro.logd.timestamp", {{NULL, 0xFFFFFFFF}, '\0'}, - evaluate_persist_ro}; - - return (tolower(do_cache2_char(&clockid)) == 'm') ? CLOCK_MONOTONIC : CLOCK_REALTIME; -} - /* * Security state generally remains constant, but the DO must be able * to turn off logging should it become spammy after an attack is detected. diff --git a/liblog/tests/Android.bp b/liblog/tests/Android.bp index 50800c543..2a6424b05 100644 --- a/liblog/tests/Android.bp +++ b/liblog/tests/Android.bp @@ -72,6 +72,7 @@ cc_defaults { ], static_libs: ["liblog"], isolated: true, + require_root: true, } // Build tests for the device (with .so). Run with: diff --git a/liblog/tests/libc_test.cpp b/liblog/tests/libc_test.cpp index 3534eb8d3..1f2626390 100644 --- a/liblog/tests/libc_test.cpp +++ b/liblog/tests/libc_test.cpp @@ -22,6 +22,10 @@ TEST(libc, __pstore_append) { #ifdef __ANDROID__ #ifndef NO_PSTORE + if (access("/dev/pmsg0", W_OK) != 0) { + GTEST_SKIP() << "pmsg0 not found, skipping test"; + } + FILE* fp; ASSERT_TRUE(NULL != (fp = fopen("/dev/pmsg0", "ae"))); static const char message[] = "libc.__pstore_append\n"; diff --git a/liblog/tests/liblog_benchmark.cpp b/liblog/tests/liblog_benchmark.cpp index 39ac7a5f3..3bd5cf20f 100644 --- a/liblog/tests/liblog_benchmark.cpp +++ b/liblog/tests/liblog_benchmark.cpp @@ -27,9 +27,11 @@ #include <unordered_set> #include <android-base/file.h> +#include <android-base/properties.h> #include <benchmark/benchmark.h> #include <cutils/sockets.h> #include <log/event_tag_map.h> +#include <log/log_read.h> #include <private/android_logger.h> BENCHMARK_MAIN(); @@ -182,7 +184,7 @@ static void BM_pmsg_short(benchmark::State& state) { */ struct timespec ts; - clock_gettime(android_log_clockid(), &ts); + clock_gettime(CLOCK_REALTIME, &ts); android_pmsg_log_header_t pmsg_header; pmsg_header.magic = LOGGER_MAGIC; @@ -258,7 +260,7 @@ static void BM_pmsg_short_aligned(benchmark::State& state) { */ struct timespec ts; - clock_gettime(android_log_clockid(), &ts); + clock_gettime(CLOCK_REALTIME, &ts); struct packet { android_pmsg_log_header_t pmsg_header; @@ -333,7 +335,7 @@ static void BM_pmsg_short_unaligned1(benchmark::State& state) { */ struct timespec ts; - clock_gettime(android_log_clockid(), &ts); + clock_gettime(CLOCK_REALTIME, &ts); struct packet { android_pmsg_log_header_t pmsg_header; @@ -408,7 +410,7 @@ static void BM_pmsg_long_aligned(benchmark::State& state) { */ struct timespec ts; - clock_gettime(android_log_clockid(), &ts); + clock_gettime(CLOCK_REALTIME, &ts); struct packet { android_pmsg_log_header_t pmsg_header; @@ -481,7 +483,7 @@ static void BM_pmsg_long_unaligned1(benchmark::State& state) { */ struct timespec ts; - clock_gettime(android_log_clockid(), &ts); + clock_gettime(CLOCK_REALTIME, &ts); struct packet { android_pmsg_log_header_t pmsg_header; @@ -647,8 +649,7 @@ static const int alarm_time = 3; static void BM_log_latency(benchmark::State& state) { pid_t pid = getpid(); - struct logger_list* logger_list = - android_logger_list_open(LOG_ID_EVENTS, ANDROID_LOG_RDONLY, 0, pid); + struct logger_list* logger_list = android_logger_list_open(LOG_ID_EVENTS, 0, 0, pid); if (!logger_list) { fprintf(stderr, "Unable to open events log: %s\n", strerror(errno)); @@ -683,8 +684,8 @@ static void BM_log_latency(benchmark::State& state) { if (!eventData || (eventData[4] != EVENT_TYPE_LONG)) { continue; } - log_time tx(eventData + 4 + 1); - if (ts != tx) { + log_time* tx = reinterpret_cast<log_time*>(eventData + 4 + 1); + if (ts != *tx) { if (0xDEADBEEFA55A5AA5ULL == caught_convert(eventData + 4 + 1)) { state.SkipWithError("signal"); break; @@ -722,8 +723,7 @@ static void caught_delay(int /*signum*/) { static void BM_log_delay(benchmark::State& state) { pid_t pid = getpid(); - struct logger_list* logger_list = - android_logger_list_open(LOG_ID_EVENTS, ANDROID_LOG_RDONLY, 0, pid); + struct logger_list* logger_list = android_logger_list_open(LOG_ID_EVENTS, 0, 0, pid); if (!logger_list) { fprintf(stderr, "Unable to open events log: %s\n", strerror(errno)); @@ -757,8 +757,8 @@ static void BM_log_delay(benchmark::State& state) { if (!eventData || (eventData[4] != EVENT_TYPE_LONG)) { continue; } - log_time tx(eventData + 4 + 1); - if (ts != tx) { + log_time* tx = reinterpret_cast<log_time*>(eventData + 4 + 1); + if (ts != *tx) { if (0xDEADBEEFA55A5AA6ULL == caught_convert(eventData + 4 + 1)) { state.SkipWithError("signal"); break; @@ -792,16 +792,6 @@ static void BM_is_loggable(benchmark::State& state) { BENCHMARK(BM_is_loggable); /* - * Measure the time it takes for android_log_clockid. - */ -static void BM_clockid(benchmark::State& state) { - while (state.KeepRunning()) { - android_log_clockid(); - } -} -BENCHMARK(BM_clockid); - -/* * Measure the time it takes for __android_log_security. */ static void BM_security(benchmark::State& state) { @@ -1025,3 +1015,14 @@ static void BM_lookupEventTagNum_logd_existing(benchmark::State& state) { } } BENCHMARK(BM_lookupEventTagNum_logd_existing); + +static void BM_log_verbose_overhead(benchmark::State& state) { + std::string test_log_tag = "liblog_verbose_tag"; + android::base::SetProperty("log.tag." + test_log_tag, "I"); + for (auto _ : state) { + __android_log_print(ANDROID_LOG_VERBOSE, test_log_tag.c_str(), "%s test log message %d %d", + "test test", 123, 456); + } + android::base::SetProperty("log.tag." + test_log_tag, ""); +} +BENCHMARK(BM_log_verbose_overhead); diff --git a/liblog/tests/liblog_test.cpp b/liblog/tests/liblog_test.cpp index a60d2df31..fbc3d7a5a 100644 --- a/liblog/tests/liblog_test.cpp +++ b/liblog/tests/liblog_test.cpp @@ -40,6 +40,7 @@ #include <gtest/gtest.h> #include <log/log_event_list.h> #include <log/log_properties.h> +#include <log/log_read.h> #include <log/logprint.h> #include <private/android_filesystem_config.h> #include <private/android_logger.h> @@ -82,7 +83,7 @@ static void RunLogTests(log_id_t log_buffer, FWrite write_messages, FCheck check pid_t pid = getpid(); auto logger_list = std::unique_ptr<struct logger_list, ListCloser>{ - android_logger_list_open(log_buffer, ANDROID_LOG_RDONLY, 1000, pid)}; + android_logger_list_open(log_buffer, 0, 1000, pid)}; ASSERT_TRUE(logger_list); write_messages(); @@ -97,16 +98,13 @@ static void RunLogTests(log_id_t log_buffer, FWrite write_messages, FCheck check ASSERT_EQ(log_buffer, log_msg.id()); ASSERT_EQ(pid, log_msg.entry.pid); - // TODO: Should this be an assert? - if (log_msg.msg() == nullptr) { - continue; - } + ASSERT_NE(nullptr, log_msg.msg()); check_message(log_msg, &found); } auto logger_list_non_block = std::unique_ptr<struct logger_list, ListCloser>{ - android_logger_list_open(log_buffer, ANDROID_LOG_RDONLY | ANDROID_LOG_NONBLOCK, 1000, pid)}; + android_logger_list_open(log_buffer, ANDROID_LOG_NONBLOCK, 1000, pid)}; ASSERT_TRUE(logger_list_non_block); size_t count = 0; @@ -121,10 +119,7 @@ static void RunLogTests(log_id_t log_buffer, FWrite write_messages, FCheck check ASSERT_EQ(log_buffer, log_msg.id()); ASSERT_EQ(pid, log_msg.entry.pid); - // TODO: Should this be an assert? - if (log_msg.msg() == nullptr) { - continue; - } + ASSERT_NE(nullptr, log_msg.msg()); found = false; check_message(log_msg, &found); @@ -160,7 +155,6 @@ static std::string popenToString(const std::string& command) { return ret; } -#ifndef NO_PSTORE static bool isPmsgActive() { pid_t pid = getpid(); @@ -170,7 +164,6 @@ static bool isPmsgActive() { return std::string::npos != myPidFds.find(" -> /dev/pmsg0"); } -#endif /* NO_PSTORE */ static bool isLogdwActive() { std::string logdwSignature = @@ -222,16 +215,18 @@ TEST(liblog, __android_log_btwrite__android_logger_list_read) { log_time ts(CLOCK_MONOTONIC); log_time ts1(ts); + bool has_pstore = access("/dev/pmsg0", W_OK) == 0; + auto write_function = [&] { EXPECT_LT(0, __android_log_btwrite(0, EVENT_TYPE_LONG, &ts, sizeof(ts))); // Check that we can close and reopen the logger bool logdwActiveAfter__android_log_btwrite; if (getuid() == AID_ROOT) { tested__android_log_close = true; -#ifndef NO_PSTORE - bool pmsgActiveAfter__android_log_btwrite = isPmsgActive(); - EXPECT_TRUE(pmsgActiveAfter__android_log_btwrite); -#endif /* NO_PSTORE */ + if (has_pstore) { + bool pmsgActiveAfter__android_log_btwrite = isPmsgActive(); + EXPECT_TRUE(pmsgActiveAfter__android_log_btwrite); + } logdwActiveAfter__android_log_btwrite = isLogdwActive(); EXPECT_TRUE(logdwActiveAfter__android_log_btwrite); } else if (!tested__android_log_close) { @@ -239,10 +234,10 @@ TEST(liblog, __android_log_btwrite__android_logger_list_read) { } __android_log_close(); if (getuid() == AID_ROOT) { -#ifndef NO_PSTORE - bool pmsgActiveAfter__android_log_close = isPmsgActive(); - EXPECT_FALSE(pmsgActiveAfter__android_log_close); -#endif /* NO_PSTORE */ + if (has_pstore) { + bool pmsgActiveAfter__android_log_close = isPmsgActive(); + EXPECT_FALSE(pmsgActiveAfter__android_log_close); + } bool logdwActiveAfter__android_log_close = isLogdwActive(); EXPECT_FALSE(logdwActiveAfter__android_log_close); } @@ -250,10 +245,10 @@ TEST(liblog, __android_log_btwrite__android_logger_list_read) { ts1 = log_time(CLOCK_MONOTONIC); EXPECT_LT(0, __android_log_btwrite(0, EVENT_TYPE_LONG, &ts1, sizeof(ts1))); if (getuid() == AID_ROOT) { -#ifndef NO_PSTORE - bool pmsgActiveAfter__android_log_btwrite = isPmsgActive(); - EXPECT_TRUE(pmsgActiveAfter__android_log_btwrite); -#endif /* NO_PSTORE */ + if (has_pstore) { + bool pmsgActiveAfter__android_log_btwrite = isPmsgActive(); + EXPECT_TRUE(pmsgActiveAfter__android_log_btwrite); + } logdwActiveAfter__android_log_btwrite = isLogdwActive(); EXPECT_TRUE(logdwActiveAfter__android_log_btwrite); } @@ -275,10 +270,10 @@ TEST(liblog, __android_log_btwrite__android_logger_list_read) { return; } - log_time tx(reinterpret_cast<char*>(&eventData->payload.data)); - if (ts == tx) { + log_time* tx = reinterpret_cast<log_time*>(&eventData->payload.data); + if (ts == *tx) { ++count; - } else if (ts1 == tx) { + } else if (ts1 == *tx) { ++second_count; } @@ -334,8 +329,6 @@ static void bswrite_test(const char* message) { #ifdef __ANDROID__ pid_t pid = getpid(); - log_time ts(android_log_clockid()); - size_t num_lines = 1, size = 0, length = 0, total = 0; const char* cp = message; while (*cp) { @@ -437,7 +430,6 @@ static void buf_write_test(const char* message) { pid_t pid = getpid(); static const char tag[] = "TEST__android_log_buf_write"; - log_time ts(android_log_clockid()); auto write_function = [&] { EXPECT_LT(0, __android_log_buf_write(LOG_ID_MAIN, ANDROID_LOG_INFO, tag, message)); @@ -572,8 +564,7 @@ TEST(liblog, android_logger_list_read__cpu_signal) { v += pid & 0xFFFF; - ASSERT_TRUE(NULL != (logger_list = android_logger_list_open( - LOG_ID_EVENTS, ANDROID_LOG_RDONLY, 1000, pid))); + ASSERT_TRUE(NULL != (logger_list = android_logger_list_open(LOG_ID_EVENTS, 0, 1000, pid))); int count = 0; @@ -728,8 +719,7 @@ TEST(liblog, android_logger_list_read__cpu_thread) { v += pid & 0xFFFF; - ASSERT_TRUE(NULL != (logger_list = android_logger_list_open( - LOG_ID_EVENTS, ANDROID_LOG_RDONLY, 1000, pid))); + ASSERT_TRUE(NULL != (logger_list = android_logger_list_open(LOG_ID_EVENTS, 0, 1000, pid))); int count = 0; @@ -1033,7 +1023,7 @@ TEST(liblog, __android_log_buf_print__maxtag) { #endif } -// TODO: This test is tautological. android_logger_list_read() calls recv() with +// Note: This test is tautological. android_logger_list_read() calls recv() with // LOGGER_ENTRY_MAX_PAYLOAD as its size argument, so it's not possible for this test to read a // payload larger than that size. TEST(liblog, too_big_payload) { @@ -1093,11 +1083,11 @@ TEST(liblog, dual_reader) { pid_t pid = getpid(); auto logger_list1 = std::unique_ptr<struct logger_list, ListCloser>{ - android_logger_list_open(LOG_ID_MAIN, ANDROID_LOG_RDONLY, expected_count1, pid)}; + android_logger_list_open(LOG_ID_MAIN, 0, expected_count1, pid)}; ASSERT_TRUE(logger_list1); auto logger_list2 = std::unique_ptr<struct logger_list, ListCloser>{ - android_logger_list_open(LOG_ID_MAIN, ANDROID_LOG_RDONLY, expected_count2, pid)}; + android_logger_list_open(LOG_ID_MAIN, 0, expected_count2, pid)}; ASSERT_TRUE(logger_list2); for (int i = 25; i > 0; --i) { @@ -1128,14 +1118,12 @@ TEST(liblog, dual_reader) { } // Test again with the nonblocking reader. - auto logger_list_non_block1 = - std::unique_ptr<struct logger_list, ListCloser>{android_logger_list_open( - LOG_ID_MAIN, ANDROID_LOG_RDONLY | ANDROID_LOG_NONBLOCK, expected_count1, pid)}; + auto logger_list_non_block1 = std::unique_ptr<struct logger_list, ListCloser>{ + android_logger_list_open(LOG_ID_MAIN, ANDROID_LOG_NONBLOCK, expected_count1, pid)}; ASSERT_TRUE(logger_list_non_block1); - auto logger_list_non_block2 = - std::unique_ptr<struct logger_list, ListCloser>{android_logger_list_open( - LOG_ID_MAIN, ANDROID_LOG_RDONLY | ANDROID_LOG_NONBLOCK, expected_count2, pid)}; + auto logger_list_non_block2 = std::unique_ptr<struct logger_list, ListCloser>{ + android_logger_list_open(LOG_ID_MAIN, ANDROID_LOG_NONBLOCK, expected_count2, pid)}; ASSERT_TRUE(logger_list_non_block2); count1 = 0; count2 = 0; @@ -1542,8 +1530,8 @@ static int count_matching_ts(log_time ts) { pid_t pid = getpid(); - struct logger_list* logger_list = android_logger_list_open( - LOG_ID_EVENTS, ANDROID_LOG_RDONLY | ANDROID_LOG_NONBLOCK, 1000, pid); + struct logger_list* logger_list = + android_logger_list_open(LOG_ID_EVENTS, ANDROID_LOG_NONBLOCK, 1000, pid); int count = 0; if (logger_list == NULL) return count; @@ -1832,10 +1820,8 @@ TEST(liblog, __security_buffer) { gid = getgid(); pid_t pid = getpid(); - ASSERT_TRUE(NULL != - (logger_list = android_logger_list_open( - LOG_ID_SECURITY, ANDROID_LOG_RDONLY | ANDROID_LOG_NONBLOCK, - 1000, pid))); + ASSERT_TRUE(NULL != (logger_list = android_logger_list_open(LOG_ID_SECURITY, ANDROID_LOG_NONBLOCK, + 1000, pid))); log_time ts(CLOCK_MONOTONIC); @@ -1988,7 +1974,6 @@ TEST(liblog, #endif } -// TODO: Do we need to check that we didn't actually write anything if we return a failure here? TEST(liblog, android_errorWriteWithInfoLog__android_logger_list_read__null_data) { #ifdef __ANDROID__ diff --git a/liblog/tests/log_read_test.cpp b/liblog/tests/log_read_test.cpp index 1be99aa80..3e0961749 100644 --- a/liblog/tests/log_read_test.cpp +++ b/liblog/tests/log_read_test.cpp @@ -34,8 +34,7 @@ TEST(liblog, android_logger_get_) { // This test assumes the log buffers are filled with noise from // normal operations. It will fail if done immediately after a // logcat -c. - struct logger_list* logger_list = - android_logger_list_alloc(ANDROID_LOG_WRONLY, 0, 0); + struct logger_list* logger_list = android_logger_list_alloc(0, 0, 0); for (int i = LOG_ID_MIN; i < LOG_ID_MAX; ++i) { log_id_t id = static_cast<log_id_t>(i); diff --git a/liblog/tests/log_wrap_test.cpp b/liblog/tests/log_wrap_test.cpp index e06964f72..755898a41 100644 --- a/liblog/tests/log_wrap_test.cpp +++ b/liblog/tests/log_wrap_test.cpp @@ -32,7 +32,7 @@ static void read_with_wrap() { // Read the last line in the log to get a starting timestamp. We're assuming // the log is not empty. - const int mode = ANDROID_LOG_RDONLY | ANDROID_LOG_NONBLOCK; + const int mode = ANDROID_LOG_NONBLOCK; struct logger_list* logger_list = android_logger_list_open(LOG_ID_MAIN, mode, 1000, 0); diff --git a/libmodprobe/libmodprobe.cpp b/libmodprobe/libmodprobe.cpp index 07504c158..5a6ae8be1 100644 --- a/libmodprobe/libmodprobe.cpp +++ b/libmodprobe/libmodprobe.cpp @@ -198,8 +198,7 @@ bool Modprobe::ParseBlocklistCallback(const std::vector<std::string>& args) { auto it = args.begin(); const std::string& type = *it++; - // +Legacy - if ((type != "blocklist") && (type != "blacklist")) { + if (type != "blocklist") { LOG(ERROR) << "non-blocklist line encountered in modules.blocklist"; return false; } @@ -334,8 +333,6 @@ Modprobe::Modprobe(const std::vector<std::string>& base_paths, const std::string auto blocklist_callback = std::bind(&Modprobe::ParseBlocklistCallback, this, _1); ParseCfg(base_path + "/modules.blocklist", blocklist_callback); - // Legacy - ParseCfg(base_path + "/modules.blacklist", blocklist_callback); } ParseKernelCmdlineOptions(); diff --git a/libmodprobe/libmodprobe_ext.cpp b/libmodprobe/libmodprobe_ext.cpp index 84f71506a..fb1f5e73e 100644 --- a/libmodprobe/libmodprobe_ext.cpp +++ b/libmodprobe/libmodprobe_ext.cpp @@ -35,7 +35,7 @@ bool Modprobe::Insmod(const std::string& path_name, const std::string& parameter android::base::unique_fd fd( TEMP_FAILURE_RETRY(open(path_name.c_str(), O_RDONLY | O_NOFOLLOW | O_CLOEXEC))); if (fd == -1) { - LOG(ERROR) << "Could not open module '" << path_name << "'"; + PLOG(ERROR) << "Could not open module '" << path_name << "'"; return false; } @@ -49,7 +49,7 @@ bool Modprobe::Insmod(const std::string& path_name, const std::string& parameter options = options + " " + parameters; } - LOG(INFO) << "Loading module " << path_name << " with args \"" << options << "\""; + LOG(INFO) << "Loading module " << path_name << " with args '" << options << "'"; int ret = syscall(__NR_finit_module, fd.get(), options.c_str(), 0); if (ret != 0) { if (errno == EEXIST) { @@ -57,7 +57,7 @@ bool Modprobe::Insmod(const std::string& path_name, const std::string& parameter module_loaded_.emplace(canonical_name); return true; } - LOG(ERROR) << "Failed to insmod '" << path_name << "' with args '" << options << "'"; + PLOG(ERROR) << "Failed to insmod '" << path_name << "' with args '" << options << "'"; return false; } diff --git a/libprocessgroup/cgroup_map.cpp b/libprocessgroup/cgroup_map.cpp index 2fc920b54..b82b0ab63 100644 --- a/libprocessgroup/cgroup_map.cpp +++ b/libprocessgroup/cgroup_map.cpp @@ -115,7 +115,13 @@ bool CgroupController::GetTaskGroup(int tid, std::string* group) const { return true; } - std::string cg_tag = StringPrintf(":%s:", name()); + std::string cg_tag; + + if (version() == 2) { + cg_tag = "0::"; + } else { + cg_tag = StringPrintf(":%s:", name()); + } size_t start_pos = content.find(cg_tag); if (start_pos == std::string::npos) { return false; diff --git a/libprocessgroup/cgrouprc/include/android/cgrouprc.h b/libprocessgroup/cgrouprc/include/android/cgrouprc.h index 0ce51237a..7e74432f7 100644 --- a/libprocessgroup/cgrouprc/include/android/cgrouprc.h +++ b/libprocessgroup/cgrouprc/include/android/cgrouprc.h @@ -69,6 +69,7 @@ __attribute__((warn_unused_result)) uint32_t ACgroupController_getVersion(const * Flag bitmask used in ACgroupController_getFlags */ #define CGROUPRC_CONTROLLER_FLAG_MOUNTED 0x1 +#define CGROUPRC_CONTROLLER_FLAG_NEEDS_ACTIVATION 0x2 #if __ANDROID_API__ >= __ANDROID_API_R__ diff --git a/libprocessgroup/setup/cgroup_descriptor.h b/libprocessgroup/setup/cgroup_descriptor.h index f029c4f12..699c03cec 100644 --- a/libprocessgroup/setup/cgroup_descriptor.h +++ b/libprocessgroup/setup/cgroup_descriptor.h @@ -25,7 +25,7 @@ namespace cgrouprc { class CgroupDescriptor { public: CgroupDescriptor(uint32_t version, const std::string& name, const std::string& path, - mode_t mode, const std::string& uid, const std::string& gid); + mode_t mode, const std::string& uid, const std::string& gid, uint32_t flags); const format::CgroupController* controller() const { return &controller_; } mode_t mode() const { return mode_; } diff --git a/libprocessgroup/setup/cgroup_map_write.cpp b/libprocessgroup/setup/cgroup_map_write.cpp index 17ea06e09..25f16a6e9 100644 --- a/libprocessgroup/setup/cgroup_map_write.cpp +++ b/libprocessgroup/setup/cgroup_map_write.cpp @@ -17,6 +17,7 @@ //#define LOG_NDEBUG 0 #define LOG_TAG "libprocessgroup" +#include <dirent.h> #include <errno.h> #include <fcntl.h> #include <grp.h> @@ -54,64 +55,120 @@ namespace cgrouprc { static constexpr const char* CGROUPS_DESC_FILE = "/etc/cgroups.json"; static constexpr const char* CGROUPS_DESC_VENDOR_FILE = "/vendor/etc/cgroups.json"; +static bool ChangeDirModeAndOwner(const std::string& path, mode_t mode, const std::string& uid, + const std::string& gid, bool permissive_mode = false) { + uid_t pw_uid = -1; + gid_t gr_gid = -1; + + if (!uid.empty()) { + passwd* uid_pwd = getpwnam(uid.c_str()); + if (!uid_pwd) { + PLOG(ERROR) << "Unable to decode UID for '" << uid << "'"; + return false; + } + + pw_uid = uid_pwd->pw_uid; + gr_gid = -1; + + if (!gid.empty()) { + group* gid_pwd = getgrnam(gid.c_str()); + if (!gid_pwd) { + PLOG(ERROR) << "Unable to decode GID for '" << gid << "'"; + return false; + } + gr_gid = gid_pwd->gr_gid; + } + } + + auto dir = std::unique_ptr<DIR, decltype(&closedir)>(opendir(path.c_str()), closedir); + + if (dir == NULL) { + PLOG(ERROR) << "opendir failed for " << path; + return false; + } + + struct dirent* dir_entry; + while ((dir_entry = readdir(dir.get()))) { + if (!strcmp("..", dir_entry->d_name)) { + continue; + } + + std::string file_path = path + "/" + dir_entry->d_name; + + if (pw_uid != -1 && lchown(file_path.c_str(), pw_uid, gr_gid) < 0) { + PLOG(ERROR) << "lchown() failed for " << file_path; + return false; + } + + if (fchmodat(AT_FDCWD, file_path.c_str(), mode, AT_SYMLINK_NOFOLLOW) != 0 && + (errno != EROFS || !permissive_mode)) { + PLOG(ERROR) << "fchmodat() failed for " << path; + return false; + } + } + + return true; +} + static bool Mkdir(const std::string& path, mode_t mode, const std::string& uid, const std::string& gid) { + bool permissive_mode = false; + if (mode == 0) { + /* Allow chmod to fail */ + permissive_mode = true; mode = 0755; } if (mkdir(path.c_str(), mode) != 0) { - /* chmod in case the directory already exists */ - if (errno == EEXIST) { - if (fchmodat(AT_FDCWD, path.c_str(), mode, AT_SYMLINK_NOFOLLOW) != 0) { - // /acct is a special case when the directory already exists - // TODO: check if file mode is already what we want instead of using EROFS - if (errno != EROFS) { - PLOG(ERROR) << "fchmodat() failed for " << path; - return false; - } - } - } else { + // /acct is a special case when the directory already exists + if (errno != EEXIST) { PLOG(ERROR) << "mkdir() failed for " << path; return false; + } else { + permissive_mode = true; } } - if (uid.empty()) { + if (uid.empty() && permissive_mode) { return true; } - passwd* uid_pwd = getpwnam(uid.c_str()); - if (!uid_pwd) { - PLOG(ERROR) << "Unable to decode UID for '" << uid << "'"; + if (!ChangeDirModeAndOwner(path, mode, uid, gid, permissive_mode)) { + PLOG(ERROR) << "change of ownership or mode failed for " << path; return false; } - uid_t pw_uid = uid_pwd->pw_uid; - gid_t gr_gid = -1; - if (!gid.empty()) { - group* gid_pwd = getgrnam(gid.c_str()); - if (!gid_pwd) { - PLOG(ERROR) << "Unable to decode GID for '" << gid << "'"; - return false; - } - gr_gid = gid_pwd->gr_gid; - } + return true; +} - if (lchown(path.c_str(), pw_uid, gr_gid) < 0) { - PLOG(ERROR) << "lchown() failed for " << path; - return false; +static void MergeCgroupToDescriptors(std::map<std::string, CgroupDescriptor>* descriptors, + const Json::Value& cgroup, const std::string& name, + const std::string& root_path, int cgroups_version) { + std::string path; + + if (!root_path.empty()) { + path = root_path + "/" + cgroup["Path"].asString(); + } else { + path = cgroup["Path"].asString(); } - /* chown may have cleared S_ISUID and S_ISGID, chmod again */ - if (mode & (S_ISUID | S_ISGID)) { - if (fchmodat(AT_FDCWD, path.c_str(), mode, AT_SYMLINK_NOFOLLOW) != 0) { - PLOG(ERROR) << "fchmodat() failed for " << path; - return false; - } + uint32_t controller_flags = 0; + + if (cgroup["NeedsActivation"].isBool() && cgroup["NeedsActivation"].asBool()) { + controller_flags |= CGROUPRC_CONTROLLER_FLAG_NEEDS_ACTIVATION; } - return true; + CgroupDescriptor descriptor( + cgroups_version, name, path, std::strtoul(cgroup["Mode"].asString().c_str(), 0, 8), + cgroup["UID"].asString(), cgroup["GID"].asString(), controller_flags); + + auto iter = descriptors->find(name); + if (iter == descriptors->end()) { + descriptors->emplace(name, descriptor); + } else { + iter->second = descriptor; + } } static bool ReadDescriptorsFromFile(const std::string& file_name, @@ -135,36 +192,19 @@ static bool ReadDescriptorsFromFile(const std::string& file_name, const Json::Value& cgroups = root["Cgroups"]; for (Json::Value::ArrayIndex i = 0; i < cgroups.size(); ++i) { std::string name = cgroups[i]["Controller"].asString(); - auto iter = descriptors->find(name); - if (iter == descriptors->end()) { - descriptors->emplace( - name, CgroupDescriptor( - 1, name, cgroups[i]["Path"].asString(), - std::strtoul(cgroups[i]["Mode"].asString().c_str(), 0, 8), - cgroups[i]["UID"].asString(), cgroups[i]["GID"].asString())); - } else { - iter->second = CgroupDescriptor( - 1, name, cgroups[i]["Path"].asString(), - std::strtoul(cgroups[i]["Mode"].asString().c_str(), 0, 8), - cgroups[i]["UID"].asString(), cgroups[i]["GID"].asString()); - } + MergeCgroupToDescriptors(descriptors, cgroups[i], name, "", 1); } } if (root.isMember("Cgroups2")) { const Json::Value& cgroups2 = root["Cgroups2"]; - auto iter = descriptors->find(CGROUPV2_CONTROLLER_NAME); - if (iter == descriptors->end()) { - descriptors->emplace( - CGROUPV2_CONTROLLER_NAME, - CgroupDescriptor(2, CGROUPV2_CONTROLLER_NAME, cgroups2["Path"].asString(), - std::strtoul(cgroups2["Mode"].asString().c_str(), 0, 8), - cgroups2["UID"].asString(), cgroups2["GID"].asString())); - } else { - iter->second = - CgroupDescriptor(2, CGROUPV2_CONTROLLER_NAME, cgroups2["Path"].asString(), - std::strtoul(cgroups2["Mode"].asString().c_str(), 0, 8), - cgroups2["UID"].asString(), cgroups2["GID"].asString()); + std::string root_path = cgroups2["Path"].asString(); + MergeCgroupToDescriptors(descriptors, cgroups2, CGROUPV2_CONTROLLER_NAME, "", 2); + + const Json::Value& childGroups = cgroups2["Controllers"]; + for (Json::Value::ArrayIndex i = 0; i < childGroups.size(); ++i) { + std::string name = childGroups[i]["Controller"].asString(); + MergeCgroupToDescriptors(descriptors, childGroups[i], name, root_path, 2); } } @@ -192,17 +232,51 @@ static bool ReadDescriptors(std::map<std::string, CgroupDescriptor>* descriptors static bool SetupCgroup(const CgroupDescriptor& descriptor) { const format::CgroupController* controller = descriptor.controller(); - // mkdir <path> [mode] [owner] [group] - if (!Mkdir(controller->path(), descriptor.mode(), descriptor.uid(), descriptor.gid())) { - LOG(ERROR) << "Failed to create directory for " << controller->name() << " cgroup"; - return false; - } - int result; if (controller->version() == 2) { - result = mount("none", controller->path(), "cgroup2", MS_NODEV | MS_NOEXEC | MS_NOSUID, - nullptr); + result = 0; + if (!strcmp(controller->name(), CGROUPV2_CONTROLLER_NAME)) { + // /sys/fs/cgroup is created by cgroup2 with specific selinux permissions, + // try to create again in case the mount point is changed + if (!Mkdir(controller->path(), 0, "", "")) { + LOG(ERROR) << "Failed to create directory for " << controller->name() << " cgroup"; + return false; + } + + result = mount("none", controller->path(), "cgroup2", MS_NODEV | MS_NOEXEC | MS_NOSUID, + nullptr); + + // selinux permissions change after mounting, so it's ok to change mode and owner now + if (!ChangeDirModeAndOwner(controller->path(), descriptor.mode(), descriptor.uid(), + descriptor.gid())) { + LOG(ERROR) << "Failed to create directory for " << controller->name() << " cgroup"; + result = -1; + } else { + LOG(ERROR) << "restored ownership for " << controller->name() << " cgroup"; + } + } else { + if (!Mkdir(controller->path(), descriptor.mode(), descriptor.uid(), descriptor.gid())) { + LOG(ERROR) << "Failed to create directory for " << controller->name() << " cgroup"; + return false; + } + + if (controller->flags() & CGROUPRC_CONTROLLER_FLAG_NEEDS_ACTIVATION) { + std::string str = std::string("+") + controller->name(); + std::string path = std::string(controller->path()) + "/cgroup.subtree_control"; + + if (!base::WriteStringToFile(str, path)) { + LOG(ERROR) << "Failed to activate controller " << controller->name(); + return false; + } + } + } } else { + // mkdir <path> [mode] [owner] [group] + if (!Mkdir(controller->path(), descriptor.mode(), descriptor.uid(), descriptor.gid())) { + LOG(ERROR) << "Failed to create directory for " << controller->name() << " cgroup"; + return false; + } + // Unfortunately historically cpuset controller was mounted using a mount command // different from all other controllers. This results in controller attributes not // to be prepended with controller name. For example this way instead of @@ -267,8 +341,8 @@ static bool WriteRcFile(const std::map<std::string, CgroupDescriptor>& descripto CgroupDescriptor::CgroupDescriptor(uint32_t version, const std::string& name, const std::string& path, mode_t mode, const std::string& uid, - const std::string& gid) - : controller_(version, 0, name, path), mode_(mode), uid_(uid), gid_(gid) {} + const std::string& gid, uint32_t flags = 0) + : controller_(version, flags, name, path), mode_(mode), uid_(uid), gid_(gid) {} void CgroupDescriptor::set_mounted(bool mounted) { uint32_t flags = controller_.flags(); diff --git a/libsparse/output_file.cpp b/libsparse/output_file.cpp index e35cb0d0e..b883c13f2 100644 --- a/libsparse/output_file.cpp +++ b/libsparse/output_file.cpp @@ -35,8 +35,9 @@ #include "sparse_crc32.h" #include "sparse_format.h" +#include <android-base/mapped_file.h> + #ifndef _WIN32 -#include <sys/mman.h> #define O_BINARY 0 #else #define ftruncate64 ftruncate @@ -45,7 +46,6 @@ #if defined(__APPLE__) && defined(__MACH__) #define lseek64 lseek #define ftruncate64 ftruncate -#define mmap64 mmap #define off64_t off_t #endif @@ -649,52 +649,10 @@ int write_fill_chunk(struct output_file* out, unsigned int len, uint32_t fill_va } int write_fd_chunk(struct output_file* out, unsigned int len, int fd, int64_t offset) { - int ret; - int64_t aligned_offset; - int aligned_diff; - uint64_t buffer_size; - char* ptr; - - aligned_offset = offset & ~(4096 - 1); - aligned_diff = offset - aligned_offset; - buffer_size = (uint64_t)len + (uint64_t)aligned_diff; - -#ifndef _WIN32 - if (buffer_size > SIZE_MAX) return -E2BIG; - char* data = - reinterpret_cast<char*>(mmap64(nullptr, buffer_size, PROT_READ, MAP_SHARED, fd, aligned_offset)); - if (data == MAP_FAILED) { - return -errno; - } - ptr = data + aligned_diff; -#else - off64_t pos; - char* data = reinterpret_cast<char*>(malloc(len)); - if (!data) { - return -errno; - } - pos = lseek64(fd, offset, SEEK_SET); - if (pos < 0) { - free(data); - return -errno; - } - ret = read_all(fd, data, len); - if (ret < 0) { - free(data); - return ret; - } - ptr = data; -#endif - - ret = out->sparse_ops->write_data_chunk(out, len, ptr); + auto m = android::base::MappedFile::FromFd(fd, offset, len, PROT_READ); + if (!m) return -errno; -#ifndef _WIN32 - munmap(data, buffer_size); -#else - free(data); -#endif - - return ret; + return out->sparse_ops->write_data_chunk(out, m->size(), m->data()); } /* Write a contiguous region of data blocks from a file */ diff --git a/libsync/Android.bp b/libsync/Android.bp index c996e1bf7..bad623054 100644 --- a/libsync/Android.bp +++ b/libsync/Android.bp @@ -25,6 +25,12 @@ cc_library { recovery_available: true, native_bridge_supported: true, defaults: ["libsync_defaults"], + stubs: { + symbol_file: "libsync.map.txt", + versions: [ + "26", + ], + }, } llndk_library { diff --git a/libsync/OWNERS b/libsync/OWNERS index dc61733a7..e75b15b8f 100644 --- a/libsync/OWNERS +++ b/libsync/OWNERS @@ -1,3 +1,3 @@ -ghackmann@google.com +chrisforbes@google.com +hridya@google.com jessehall@google.com -marissaw@google.com diff --git a/libsync/libsync.map.txt b/libsync/libsync.map.txt index 91c3528d1..aac6b570f 100644 --- a/libsync/libsync.map.txt +++ b/libsync/libsync.map.txt @@ -19,7 +19,7 @@ LIBSYNC { sync_merge; # introduced=26 sync_file_info; # introduced=26 sync_file_info_free; # introduced=26 - sync_wait; # llndk + sync_wait; # llndk apex sync_fence_info; # llndk sync_pt_info; # llndk sync_fence_info_free; # llndk diff --git a/libsystem/OWNERS b/libsystem/OWNERS index fdea80467..4f800d4ce 100644 --- a/libsystem/OWNERS +++ b/libsystem/OWNERS @@ -1,7 +1,6 @@ # graphics/composer adyabr@google.com lpy@google.com -marissaw@google.com stoza@google.com vhau@google.com diff --git a/libunwindstack/Android.bp b/libunwindstack/Android.bp index 563c2c296..3c44534d0 100644 --- a/libunwindstack/Android.bp +++ b/libunwindstack/Android.bp @@ -57,6 +57,7 @@ cc_defaults { "MapInfo.cpp", "Maps.cpp", "Memory.cpp", + "MemoryMte.cpp", "LocalUnwinder.cpp", "Regs.cpp", "RegsArm.cpp", @@ -74,13 +75,29 @@ cc_defaults { ], target: { - // Always disable optimizations for host to make it easier to debug. host: { + // Always disable optimizations for host to make it easier to debug. cflags: [ "-O0", "-g", ], }, + android: { + header_libs: ["bionic_libc_platform_headers"], + product_variables: { + experimental_mte: { + cflags: ["-DANDROID_EXPERIMENTAL_MTE"], + }, + }, + }, + linux_bionic: { + header_libs: ["bionic_libc_platform_headers"], + product_variables: { + experimental_mte: { + cflags: ["-DANDROID_EXPERIMENTAL_MTE"], + }, + }, + }, }, arch: { @@ -90,12 +107,6 @@ cc_defaults { x86_64: { srcs: ["AsmGetRegsX86_64.S"], }, - mips: { - srcs: ["AsmGetRegsMips.S"], - }, - mips64: { - srcs: ["AsmGetRegsMips64.S"], - }, }, static_libs: [ @@ -225,6 +236,7 @@ cc_defaults { "tests/MemoryRangesTest.cpp", "tests/MemoryRemoteTest.cpp", "tests/MemoryTest.cpp", + "tests/MemoryMteTest.cpp", "tests/RegsInfoTest.cpp", "tests/RegsIterateTest.cpp", "tests/RegsStepIfSignalHandlerTest.cpp", @@ -280,6 +292,25 @@ cc_defaults { "tests/files/offline/straddle_arm/*", "tests/files/offline/straddle_arm64/*", ], + + target: { + android: { + header_libs: ["bionic_libc_platform_headers"], + product_variables: { + experimental_mte: { + cflags: ["-DANDROID_EXPERIMENTAL_MTE"], + }, + }, + }, + linux_bionic: { + header_libs: ["bionic_libc_platform_headers"], + product_variables: { + experimental_mte: { + cflags: ["-DANDROID_EXPERIMENTAL_MTE"], + }, + }, + }, + }, } cc_test { @@ -385,12 +416,29 @@ cc_benchmark { srcs: [ "benchmarks/unwind_benchmarks.cpp", + "benchmarks/ElfBenchmark.cpp", + "benchmarks/MapsBenchmark.cpp", + "benchmarks/SymbolBenchmark.cpp", + "benchmarks/Utils.cpp", + ], + + data: [ + "benchmarks/files/*", ], shared_libs: [ "libbase", "libunwindstack", ], + + target: { + android: { + static_libs: [ + "libmeminfo", + "libprocinfo", + ], + }, + }, } // Generates the elf data for use in the tests for .gnu_debugdata frames. diff --git a/libunwindstack/AsmGetRegsMips.S b/libunwindstack/AsmGetRegsMips.S deleted file mode 100644 index 183d0a928..000000000 --- a/libunwindstack/AsmGetRegsMips.S +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (C) 2017 The Android Open Source Project - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS - * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - .text - .type AsmGetRegs, %function - .globl AsmGetRegs - .ent AsmGetRegs - .balign 16 -AsmGetRegs: - .cfi_startproc - .cfi_def_cfa $sp, 0 - .set push - .set noreorder - .cpload $t9 - sw $zero, 0($a0) - .set noat - sw $at, 4($a0) - .set at - sw $v0, 8($a0) - sw $v1, 12($a0) - sw $a0, 16($a0) - sw $a1, 20($a0) - sw $a2, 24($a0) - sw $a3, 28($a0) - sw $t0, 32($a0) - sw $t1, 36($a0) - sw $t2, 40($a0) - sw $t3, 44($a0) - sw $t4, 48($a0) - sw $t5, 52($a0) - sw $t6, 56($a0) - sw $t7, 60($a0) - sw $s0, 64($a0) - sw $s1, 68($a0) - sw $s2, 72($a0) - sw $s3, 76($a0) - sw $s4, 80($a0) - sw $s5, 84($a0) - sw $s6, 88($a0) - sw $s7, 92($a0) - sw $t8, 96($a0) - sw $t9, 100($a0) - sw $k0, 104($a0) - sw $k1, 108($a0) - sw $gp, 112($a0) - sw $sp, 116($a0) - sw $s8, 120($a0) - sw $ra, 124($a0) - jalr $zero, $ra - sw $ra, 128($a0) // set PC to the calling function - - .set pop - .cfi_endproc - .size AsmGetRegs, .-AsmGetRegs - .end AsmGetRegs diff --git a/libunwindstack/AsmGetRegsMips64.S b/libunwindstack/AsmGetRegsMips64.S deleted file mode 100644 index 7a244f6f8..000000000 --- a/libunwindstack/AsmGetRegsMips64.S +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (C) 2017 The Android Open Source Project - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS - * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - .text - .type AsmGetRegs, %function - .globl AsmGetRegs - .ent AsmGetRegs - .balign 16 -AsmGetRegs: - .cfi_startproc - .cfi_def_cfa $sp, 0 - .set push - .set noreorder - .cpload $t9 - sd $zero, 0($a0) - .set noat - sd $at, 8($a0) - .set at - sd $v0, 16($a0) - sd $v1, 24($a0) - sd $a0, 32($a0) - sd $a1, 40($a0) - sd $a2, 48($a0) - sd $a3, 56($a0) - sd $a4, 64($a0) - sd $a5, 72($a0) - sd $a6, 80($a0) - sd $a7, 88($a0) - sd $t0, 96($a0) - sd $t1, 104($a0) - sd $t2, 112($a0) - sd $t3, 120($a0) - sd $s0, 128($a0) - sd $s1, 136($a0) - sd $s2, 144($a0) - sd $s3, 152($a0) - sd $s4, 160($a0) - sd $s5, 168($a0) - sd $s6, 176($a0) - sd $s7, 184($a0) - sd $t8, 192($a0) - sd $t9, 200($a0) - sd $k0, 208($a0) - sd $k1, 216($a0) - sd $gp, 224($a0) - sd $sp, 232($a0) - sd $s8, 240($a0) - sd $ra, 248($a0) - jalr $zero, $ra - sd $ra, 256($a0) // set PC to the calling function - - .set pop - .cfi_endproc - .size AsmGetRegs, .-AsmGetRegs - .end AsmGetRegs diff --git a/libunwindstack/DexFile.cpp b/libunwindstack/DexFile.cpp index bf63abf1c..8fc3d23ec 100644 --- a/libunwindstack/DexFile.cpp +++ b/libunwindstack/DexFile.cpp @@ -50,6 +50,22 @@ static bool HasDexSupport() { std::unique_ptr<DexFile> DexFile::Create(uint64_t dex_file_offset_in_memory, Memory* memory, MapInfo* info) { + if (UNLIKELY(!HasDexSupport())) { + return nullptr; + } + + size_t max_size = info->end - dex_file_offset_in_memory; + if (memory->IsLocal()) { + size_t size = max_size; + + std::string err_msg; + std::unique_ptr<art_api::dex::DexFile> art_dex_file = DexFile::OpenFromMemory( + reinterpret_cast<void const*>(dex_file_offset_in_memory), &size, info->name, &err_msg); + if (art_dex_file != nullptr && size <= max_size) { + return std::unique_ptr<DexFile>(new DexFile(art_dex_file)); + } + } + if (!info->name.empty()) { std::unique_ptr<DexFile> dex_file = DexFileFromFile::Create(dex_file_offset_in_memory - info->start + info->offset, info->name); @@ -57,7 +73,7 @@ std::unique_ptr<DexFile> DexFile::Create(uint64_t dex_file_offset_in_memory, Mem return dex_file; } } - return DexFileFromMemory::Create(dex_file_offset_in_memory, memory, info->name); + return DexFileFromMemory::Create(dex_file_offset_in_memory, memory, info->name, max_size); } bool DexFile::GetMethodInformation(uint64_t dex_offset, std::string* method_name, @@ -94,7 +110,8 @@ std::unique_ptr<DexFileFromFile> DexFileFromFile::Create(uint64_t dex_file_offse std::unique_ptr<DexFileFromMemory> DexFileFromMemory::Create(uint64_t dex_file_offset_in_memory, Memory* memory, - const std::string& name) { + const std::string& name, + size_t max_size) { if (UNLIKELY(!HasDexSupport())) { return nullptr; } @@ -105,6 +122,9 @@ std::unique_ptr<DexFileFromMemory> DexFileFromMemory::Create(uint64_t dex_file_o std::string error_msg; std::unique_ptr<art_api::dex::DexFile> art_dex_file = OpenFromMemory(backing_memory.data(), &size, name, &error_msg); + if (size > max_size) { + return nullptr; + } if (art_dex_file != nullptr) { return std::unique_ptr<DexFileFromMemory>( diff --git a/libunwindstack/DexFile.h b/libunwindstack/DexFile.h index 4e8369f84..fe185dae0 100644 --- a/libunwindstack/DexFile.h +++ b/libunwindstack/DexFile.h @@ -55,7 +55,8 @@ class DexFileFromFile : public DexFile { class DexFileFromMemory : public DexFile { public: static std::unique_ptr<DexFileFromMemory> Create(uint64_t dex_file_offset_in_memory, - Memory* memory, const std::string& name); + Memory* memory, const std::string& name, + size_t max_size); private: DexFileFromMemory(std::unique_ptr<art_api::dex::DexFile>& art_dex_file, diff --git a/libunwindstack/Elf.cpp b/libunwindstack/Elf.cpp index f01b0926b..286febcdc 100644 --- a/libunwindstack/Elf.cpp +++ b/libunwindstack/Elf.cpp @@ -124,6 +124,12 @@ bool Elf::GetGlobalVariableOffset(const std::string& name, uint64_t* memory_offs return false; } + if (arch() == ARCH_ARM64) { + // Tagged pointer after Android R would lead top byte to have random values + // https://source.android.com/devices/tech/debug/tagged-pointers + vaddr &= (1ULL << 56) - 1; + } + // Check the .data section. uint64_t vaddr_start = interface_->data_vaddr_start(); if (vaddr >= vaddr_start && vaddr < interface_->data_vaddr_end()) { diff --git a/libunwindstack/ElfInterface.cpp b/libunwindstack/ElfInterface.cpp index 341275dbb..17470fd3e 100644 --- a/libunwindstack/ElfInterface.cpp +++ b/libunwindstack/ElfInterface.cpp @@ -371,7 +371,7 @@ void ElfInterface::ReadSectionHeaders(const EhdrType& ehdr) { // Look for the .debug_frame and .gnu_debugdata. if (shdr.sh_name < sec_size) { std::string name; - if (memory_->ReadString(sec_offset + shdr.sh_name, &name)) { + if (memory_->ReadString(sec_offset + shdr.sh_name, &name, sec_size - shdr.sh_name)) { if (name == ".debug_frame") { debug_frame_offset_ = shdr.sh_offset; debug_frame_size_ = shdr.sh_size; @@ -405,7 +405,7 @@ void ElfInterface::ReadSectionHeaders(const EhdrType& ehdr) { } else if (shdr.sh_type == SHT_NOTE) { if (shdr.sh_name < sec_size) { std::string name; - if (memory_->ReadString(sec_offset + shdr.sh_name, &name) && + if (memory_->ReadString(sec_offset + shdr.sh_name, &name, sec_size - shdr.sh_name) && name == ".note.gnu.build-id") { gnu_build_id_offset_ = shdr.sh_offset; gnu_build_id_size_ = shdr.sh_size; @@ -456,10 +456,11 @@ std::string ElfInterface::GetSonameWithTemplate() { for (const auto& entry : strtabs_) { if (entry.first == strtab_addr) { soname_offset = entry.second + soname_offset; - if (soname_offset >= entry.second + strtab_size) { + uint64_t soname_max = entry.second + strtab_size; + if (soname_offset >= soname_max) { return ""; } - if (!memory_->ReadString(soname_offset, &soname_)) { + if (!memory_->ReadString(soname_offset, &soname_, soname_max - soname_offset)) { return ""; } soname_type_ = SONAME_VALID; @@ -608,7 +609,8 @@ bool GetBuildIDInfo(Memory* memory, uint64_t* build_id_offset, uint64_t* build_i } std::string name; if (shdr.sh_type == SHT_NOTE && shdr.sh_name < sec_size && - memory->ReadString(sec_offset + shdr.sh_name, &name) && name == ".note.gnu.build-id") { + memory->ReadString(sec_offset + shdr.sh_name, &name, sec_size - shdr.sh_name) && + name == ".note.gnu.build-id") { *build_id_offset = shdr.sh_offset; *build_id_size = shdr.sh_size; return true; @@ -662,7 +664,7 @@ std::string ElfInterface::ReadBuildIDFromMemory(Memory* memory) { if (note_size - offset < hdr.n_descsz || hdr.n_descsz == 0) { return ""; } - std::string build_id(hdr.n_descsz - 1, '\0'); + std::string build_id(hdr.n_descsz, '\0'); if (memory->ReadFully(note_offset + offset, &build_id[0], hdr.n_descsz)) { return build_id; } diff --git a/libunwindstack/LocalUnwinder.cpp b/libunwindstack/LocalUnwinder.cpp index 5d8120036..05650fb2e 100644 --- a/libunwindstack/LocalUnwinder.cpp +++ b/libunwindstack/LocalUnwinder.cpp @@ -106,7 +106,7 @@ bool LocalUnwinder::Unwind(std::vector<LocalFrameData>* frame_info, size_t max_f uint64_t step_pc = rel_pc; uint64_t pc_adjustment; if (adjust_pc) { - pc_adjustment = regs->GetPcAdjustment(rel_pc, elf); + pc_adjustment = GetPcAdjustment(rel_pc, elf, arch); } else { pc_adjustment = 0; } diff --git a/libunwindstack/Memory.cpp b/libunwindstack/Memory.cpp index 8de3d9808..b4623faac 100644 --- a/libunwindstack/Memory.cpp +++ b/libunwindstack/Memory.cpp @@ -158,20 +158,30 @@ bool Memory::ReadFully(uint64_t addr, void* dst, size_t size) { return rc == size; } -bool Memory::ReadString(uint64_t addr, std::string* string, uint64_t max_read) { - string->clear(); - uint64_t bytes_read = 0; - while (bytes_read < max_read) { - uint8_t value; - if (!ReadFully(addr, &value, sizeof(value))) { - return false; +bool Memory::ReadString(uint64_t addr, std::string* dst, size_t max_read) { + char buffer[256]; // Large enough for 99% of symbol names. + size_t size = 0; // Number of bytes which were read into the buffer. + for (size_t offset = 0; offset < max_read; offset += size) { + // Look for null-terminator first, so we can allocate string of exact size. + // If we know the end of valid memory range, do the reads in larger blocks. + size_t read = std::min(sizeof(buffer), max_read - offset); + size = Read(addr + offset, buffer, read); + if (size == 0) { + return false; // We have not found end of string yet and we can not read more data. } - if (value == '\0') { - return true; + size_t length = strnlen(buffer, size); // Index of the null-terminator. + if (length < size) { + // We found the null-terminator. Allocate the string and set its content. + if (offset == 0) { + // We did just single read, so the buffer already contains the whole string. + dst->assign(buffer, length); + return true; + } else { + // The buffer contains only the last block. Read the whole string again. + dst->assign(offset + length, '\0'); + return ReadFully(addr, dst->data(), dst->size()); + } } - string->push_back(value); - addr++; - bytes_read++; } return false; } @@ -324,6 +334,16 @@ size_t MemoryLocal::Read(uint64_t addr, void* dst, size_t size) { return ProcessVmRead(getpid(), addr, dst, size); } +#if !defined(ANDROID_EXPERIMENTAL_MTE) +long MemoryRemote::ReadTag(uint64_t) { + return -1; +} + +long MemoryLocal::ReadTag(uint64_t) { + return -1; +} +#endif + MemoryRange::MemoryRange(const std::shared_ptr<Memory>& memory, uint64_t begin, uint64_t length, uint64_t offset) : memory_(memory), begin_(begin), length_(length), offset_(offset) {} diff --git a/libunwindstack/MemoryCache.h b/libunwindstack/MemoryCache.h index 769d90746..d97640d8b 100644 --- a/libunwindstack/MemoryCache.h +++ b/libunwindstack/MemoryCache.h @@ -33,6 +33,7 @@ class MemoryCache : public Memory { virtual ~MemoryCache() = default; size_t Read(uint64_t addr, void* dst, size_t size) override; + long ReadTag(uint64_t addr) override { return impl_->ReadTag(addr); } void Clear() override { cache_.clear(); } diff --git a/libunwindstack/MemoryLocal.h b/libunwindstack/MemoryLocal.h index 29aaf128b..741f1077c 100644 --- a/libunwindstack/MemoryLocal.h +++ b/libunwindstack/MemoryLocal.h @@ -28,7 +28,10 @@ class MemoryLocal : public Memory { MemoryLocal() = default; virtual ~MemoryLocal() = default; + bool IsLocal() const override { return true; } + size_t Read(uint64_t addr, void* dst, size_t size) override; + long ReadTag(uint64_t addr) override; }; } // namespace unwindstack diff --git a/libunwindstack/MemoryMte.cpp b/libunwindstack/MemoryMte.cpp new file mode 100644 index 000000000..46a546e1c --- /dev/null +++ b/libunwindstack/MemoryMte.cpp @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if defined(ANDROID_EXPERIMENTAL_MTE) + +#include <sys/ptrace.h> +#include <sys/uio.h> + +#include <bionic/mte.h> +#include <bionic/mte_kernel.h> + +#include "MemoryLocal.h" +#include "MemoryRemote.h" + +namespace unwindstack { + +long MemoryRemote::ReadTag(uint64_t addr) { +#if defined(__aarch64__) + char tag; + iovec iov = {&tag, 1}; + if (ptrace(PTRACE_PEEKMTETAGS, pid_, reinterpret_cast<void*>(addr), &iov) != 0 || + iov.iov_len != 1) { + return -1; + } + return tag; +#else + (void)addr; + return -1; +#endif +} + +long MemoryLocal::ReadTag(uint64_t addr) { +#if defined(__aarch64__) + // Check that the memory is readable first. This is racy with the ldg but there's not much + // we can do about it. + char data; + if (!mte_supported() || !Read(addr, &data, 1)) { + return -1; + } + + __asm__ __volatile__(".arch_extension mte; ldg %0, [%0]" : "+r"(addr) : : "memory"); + return (addr >> 56) & 0xf; +#else + (void)addr; + return -1; +#endif +} + +} // namespace unwindstack + +#endif diff --git a/libunwindstack/MemoryRemote.h b/libunwindstack/MemoryRemote.h index db367d649..dd09c8838 100644 --- a/libunwindstack/MemoryRemote.h +++ b/libunwindstack/MemoryRemote.h @@ -32,6 +32,7 @@ class MemoryRemote : public Memory { virtual ~MemoryRemote() = default; size_t Read(uint64_t addr, void* dst, size_t size) override; + long ReadTag(uint64_t addr) override; pid_t pid() { return pid_; } diff --git a/libunwindstack/Regs.cpp b/libunwindstack/Regs.cpp index c7dec5259..03aa6c205 100644 --- a/libunwindstack/Regs.cpp +++ b/libunwindstack/Regs.cpp @@ -100,10 +100,6 @@ ArchEnum Regs::CurrentArch() { return ARCH_X86; #elif defined(__x86_64__) return ARCH_X86_64; -#elif defined(__mips__) && !defined(__LP64__) - return ARCH_MIPS; -#elif defined(__mips__) && defined(__LP64__) - return ARCH_MIPS64; #else abort(); #endif @@ -119,14 +115,68 @@ Regs* Regs::CreateFromLocal() { regs = new RegsX86(); #elif defined(__x86_64__) regs = new RegsX86_64(); -#elif defined(__mips__) && !defined(__LP64__) - regs = new RegsMips(); -#elif defined(__mips__) && defined(__LP64__) - regs = new RegsMips64(); #else abort(); #endif return regs; } +uint64_t GetPcAdjustment(uint64_t rel_pc, Elf* elf, ArchEnum arch) { + switch (arch) { + case ARCH_ARM: { + if (!elf->valid()) { + return 2; + } + + uint64_t load_bias = elf->GetLoadBias(); + if (rel_pc < load_bias) { + if (rel_pc < 2) { + return 0; + } + return 2; + } + uint64_t adjusted_rel_pc = rel_pc - load_bias; + if (adjusted_rel_pc < 5) { + if (adjusted_rel_pc < 2) { + return 0; + } + return 2; + } + + if (adjusted_rel_pc & 1) { + // This is a thumb instruction, it could be 2 or 4 bytes. + uint32_t value; + if (!elf->memory()->ReadFully(adjusted_rel_pc - 5, &value, sizeof(value)) || + (value & 0xe000f000) != 0xe000f000) { + return 2; + } + } + return 4; + } + case ARCH_ARM64: { + if (rel_pc < 4) { + return 0; + } + return 4; + } + case ARCH_MIPS: + case ARCH_MIPS64: { + if (rel_pc < 8) { + return 0; + } + // For now, just assume no compact branches + return 8; + } + case ARCH_X86: + case ARCH_X86_64: { + if (rel_pc == 0) { + return 0; + } + return 1; + } + case ARCH_UNKNOWN: + return 0; + } +} + } // namespace unwindstack diff --git a/libunwindstack/RegsArm.cpp b/libunwindstack/RegsArm.cpp index 1b1f7ebbc..1aaa08f56 100644 --- a/libunwindstack/RegsArm.cpp +++ b/libunwindstack/RegsArm.cpp @@ -51,37 +51,6 @@ void RegsArm::set_sp(uint64_t sp) { regs_[ARM_REG_SP] = sp; } -uint64_t RegsArm::GetPcAdjustment(uint64_t rel_pc, Elf* elf) { - if (!elf->valid()) { - return 2; - } - - uint64_t load_bias = elf->GetLoadBias(); - if (rel_pc < load_bias) { - if (rel_pc < 2) { - return 0; - } - return 2; - } - uint64_t adjusted_rel_pc = rel_pc - load_bias; - if (adjusted_rel_pc < 5) { - if (adjusted_rel_pc < 2) { - return 0; - } - return 2; - } - - if (adjusted_rel_pc & 1) { - // This is a thumb instruction, it could be 2 or 4 bytes. - uint32_t value; - if (!elf->memory()->ReadFully(adjusted_rel_pc - 5, &value, sizeof(value)) || - (value & 0xe000f000) != 0xe000f000) { - return 2; - } - } - return 4; -} - bool RegsArm::SetPcFromReturnAddress(Memory*) { uint32_t lr = regs_[ARM_REG_LR]; if (regs_[ARM_REG_PC] == lr) { diff --git a/libunwindstack/RegsArm64.cpp b/libunwindstack/RegsArm64.cpp index 00b336737..5b7431a57 100644 --- a/libunwindstack/RegsArm64.cpp +++ b/libunwindstack/RegsArm64.cpp @@ -52,13 +52,6 @@ void RegsArm64::set_sp(uint64_t sp) { regs_[ARM64_REG_SP] = sp; } -uint64_t RegsArm64::GetPcAdjustment(uint64_t rel_pc, Elf*) { - if (rel_pc < 4) { - return 0; - } - return 4; -} - bool RegsArm64::SetPcFromReturnAddress(Memory*) { uint64_t lr = regs_[ARM64_REG_LR]; if (regs_[ARM64_REG_PC] == lr) { diff --git a/libunwindstack/RegsMips.cpp b/libunwindstack/RegsMips.cpp index ebefe429b..ab8469114 100644 --- a/libunwindstack/RegsMips.cpp +++ b/libunwindstack/RegsMips.cpp @@ -52,14 +52,6 @@ void RegsMips::set_sp(uint64_t sp) { regs_[MIPS_REG_SP] = static_cast<uint32_t>(sp); } -uint64_t RegsMips::GetPcAdjustment(uint64_t rel_pc, Elf*) { - if (rel_pc < 8) { - return 0; - } - // For now, just assume no compact branches - return 8; -} - bool RegsMips::SetPcFromReturnAddress(Memory*) { uint32_t ra = regs_[MIPS_REG_RA]; if (regs_[MIPS_REG_PC] == ra) { diff --git a/libunwindstack/RegsMips64.cpp b/libunwindstack/RegsMips64.cpp index be2fd22f3..7f600d362 100644 --- a/libunwindstack/RegsMips64.cpp +++ b/libunwindstack/RegsMips64.cpp @@ -52,14 +52,6 @@ void RegsMips64::set_sp(uint64_t sp) { regs_[MIPS64_REG_SP] = sp; } -uint64_t RegsMips64::GetPcAdjustment(uint64_t rel_pc, Elf*) { - if (rel_pc < 8) { - return 0; - } - // For now, just assume no compact branches - return 8; -} - bool RegsMips64::SetPcFromReturnAddress(Memory*) { uint64_t ra = regs_[MIPS64_REG_RA]; if (regs_[MIPS64_REG_PC] == ra) { diff --git a/libunwindstack/RegsX86.cpp b/libunwindstack/RegsX86.cpp index 5538fc0be..4d3c246a4 100644 --- a/libunwindstack/RegsX86.cpp +++ b/libunwindstack/RegsX86.cpp @@ -50,13 +50,6 @@ void RegsX86::set_sp(uint64_t sp) { regs_[X86_REG_SP] = static_cast<uint32_t>(sp); } -uint64_t RegsX86::GetPcAdjustment(uint64_t rel_pc, Elf*) { - if (rel_pc == 0) { - return 0; - } - return 1; -} - bool RegsX86::SetPcFromReturnAddress(Memory* process_memory) { // Attempt to get the return address from the top of the stack. uint32_t new_pc; diff --git a/libunwindstack/RegsX86_64.cpp b/libunwindstack/RegsX86_64.cpp index 5b9aa58c4..c9e245d2f 100644 --- a/libunwindstack/RegsX86_64.cpp +++ b/libunwindstack/RegsX86_64.cpp @@ -51,13 +51,6 @@ void RegsX86_64::set_sp(uint64_t sp) { regs_[X86_64_REG_SP] = sp; } -uint64_t RegsX86_64::GetPcAdjustment(uint64_t rel_pc, Elf*) { - if (rel_pc == 0) { - return 0; - } - return 1; -} - bool RegsX86_64::SetPcFromReturnAddress(Memory* process_memory) { // Attempt to get the return address from the top of the stack. uint64_t new_pc; diff --git a/libunwindstack/Symbols.cpp b/libunwindstack/Symbols.cpp index e3c15a298..2117ebd3e 100644 --- a/libunwindstack/Symbols.cpp +++ b/libunwindstack/Symbols.cpp @@ -19,6 +19,7 @@ #include <algorithm> #include <string> +#include <vector> #include <unwindstack/Memory.h> @@ -29,23 +30,55 @@ namespace unwindstack { Symbols::Symbols(uint64_t offset, uint64_t size, uint64_t entry_size, uint64_t str_offset, uint64_t str_size) - : cur_offset_(offset), - offset_(offset), - end_(offset + size), + : offset_(offset), + count_(entry_size != 0 ? size / entry_size : 0), entry_size_(entry_size), str_offset_(str_offset), str_end_(str_offset_ + str_size) {} -const Symbols::Info* Symbols::GetInfoFromCache(uint64_t addr) { - // Binary search the table. +template <typename SymType> +static bool IsFunc(const SymType* entry) { + return entry->st_shndx != SHN_UNDEF && ELF32_ST_TYPE(entry->st_info) == STT_FUNC; +} + +// Read symbol entry from memory and cache it so we don't have to read it again. +template <typename SymType> +inline __attribute__((__always_inline__)) const Symbols::Info* Symbols::ReadFuncInfo( + uint32_t symbol_index, Memory* elf_memory) { + auto it = symbols_.find(symbol_index); + if (it != symbols_.end()) { + return &it->second; + } + SymType sym; + if (!elf_memory->ReadFully(offset_ + symbol_index * entry_size_, &sym, sizeof(sym))) { + return nullptr; + } + if (!IsFunc(&sym)) { + // We need the address for binary search, but we don't want it to be matched. + sym.st_size = 0; + } + Info info{.addr = sym.st_value, .size = static_cast<uint32_t>(sym.st_size), .name = sym.st_name}; + return &symbols_.emplace(symbol_index, info).first->second; +} + +// Binary search the symbol table to find function containing the given address. +// Without remap, the symbol table is assumed to be sorted and accessed directly. +// If the symbol table is not sorted this method might fail but should not crash. +// When the indices are remapped, they are guaranteed to be sorted by address. +template <typename SymType, bool RemapIndices> +const Symbols::Info* Symbols::BinarySearch(uint64_t addr, Memory* elf_memory) { size_t first = 0; - size_t last = symbols_.size(); + size_t last = RemapIndices ? remap_->size() : count_; while (first < last) { size_t current = first + (last - first) / 2; - const Info* info = &symbols_[current]; - if (addr < info->start_offset) { + size_t symbol_index = RemapIndices ? remap_.value()[current] : current; + const Info* info = ReadFuncInfo<SymType>(symbol_index, elf_memory); + if (info == nullptr) { + return nullptr; + } + if (addr < info->addr) { last = current; - } else if (addr < info->end_offset) { + } else if (addr < info->addr + info->size) { return info; } else { first = current + 1; @@ -54,64 +87,72 @@ const Symbols::Info* Symbols::GetInfoFromCache(uint64_t addr) { return nullptr; } +// Create remapping table which allows us to access symbols as if they were sorted by address. template <typename SymType> -bool Symbols::GetName(uint64_t addr, Memory* elf_memory, std::string* name, uint64_t* func_offset) { - if (symbols_.size() != 0) { - const Info* info = GetInfoFromCache(addr); - if (info) { - CHECK(addr >= info->start_offset && addr <= info->end_offset); - *func_offset = addr - info->start_offset; - return elf_memory->ReadString(info->str_offset, name, str_end_ - info->str_offset); +void Symbols::BuildRemapTable(Memory* elf_memory) { + std::vector<uint64_t> addrs; // Addresses of all symbols (addrs[i] == symbols[i].st_value). + addrs.reserve(count_); + remap_.emplace(); // Construct the optional remap table. + remap_->reserve(count_); + for (size_t symbol_idx = 0; symbol_idx < count_;) { + // Read symbols from memory. We intentionally bypass the cache to save memory. + // Do the reads in batches so that we minimize the number of memory read calls. + uint8_t buffer[1024]; + size_t read = std::min<size_t>(sizeof(buffer), (count_ - symbol_idx) * entry_size_); + size_t size = elf_memory->Read(offset_ + symbol_idx * entry_size_, buffer, read); + if (size < sizeof(SymType)) { + break; // Stop processing, something looks like it is corrupted. } - } - - bool symbol_added = false; - bool return_value = false; - while (cur_offset_ + entry_size_ <= end_) { - SymType entry; - if (!elf_memory->ReadFully(cur_offset_, &entry, sizeof(entry))) { - // Stop all processing, something looks like it is corrupted. - cur_offset_ = UINT64_MAX; - return false; - } - cur_offset_ += entry_size_; - - if (entry.st_shndx != SHN_UNDEF && ELF32_ST_TYPE(entry.st_info) == STT_FUNC) { - // Treat st_value as virtual address. - uint64_t start_offset = entry.st_value; - uint64_t end_offset = start_offset + entry.st_size; - - // Cache the value. - symbols_.emplace_back(start_offset, end_offset, str_offset_ + entry.st_name); - symbol_added = true; - - if (addr >= start_offset && addr < end_offset) { - *func_offset = addr - start_offset; - uint64_t offset = str_offset_ + entry.st_name; - if (offset < str_end_) { - return_value = elf_memory->ReadString(offset, name, str_end_ - offset); - } - break; + for (size_t offset = 0; offset + sizeof(SymType) <= size; offset += entry_size_, symbol_idx++) { + SymType sym; + memcpy(&sym, &buffer[offset], sizeof(SymType)); // Copy to ensure alignment. + addrs.push_back(sym.st_value); // Always insert so it is indexable by symbol index. + if (IsFunc(&sym)) { + remap_->push_back(symbol_idx); // Indices of function symbols only. } } } + // Sort by address to make the remap list binary searchable (stable due to the a<b tie break). + auto comp = [&addrs](auto a, auto b) { return std::tie(addrs[a], a) < std::tie(addrs[b], b); }; + std::sort(remap_->begin(), remap_->end(), comp); + // Remove duplicate entries (methods de-duplicated by the linker). + auto pred = [&addrs](auto a, auto b) { return addrs[a] == addrs[b]; }; + remap_->erase(std::unique(remap_->begin(), remap_->end(), pred), remap_->end()); + remap_->shrink_to_fit(); +} - if (symbol_added) { - std::sort(symbols_.begin(), symbols_.end(), - [](const Info& a, const Info& b) { return a.start_offset < b.start_offset; }); +template <typename SymType> +bool Symbols::GetName(uint64_t addr, Memory* elf_memory, std::string* name, uint64_t* func_offset) { + const Info* info; + if (!remap_.has_value()) { + // Assume the symbol table is sorted. If it is not, this will gracefully fail. + info = BinarySearch<SymType, false>(addr, elf_memory); + if (info == nullptr) { + // Create the remapping table and retry the search. + BuildRemapTable<SymType>(elf_memory); + symbols_.clear(); // Remove cached symbols since the access pattern will be different. + info = BinarySearch<SymType, true>(addr, elf_memory); + } + } else { + // Fast search using the previously created remap table. + info = BinarySearch<SymType, true>(addr, elf_memory); + } + if (info == nullptr) { + return false; } - return return_value; + // Read the function name from the string table. + *func_offset = addr - info->addr; + uint64_t str = str_offset_ + info->name; + return str < str_end_ && elf_memory->ReadString(str, name, str_end_ - str); } template <typename SymType> bool Symbols::GetGlobal(Memory* elf_memory, const std::string& name, uint64_t* memory_address) { - uint64_t cur_offset = offset_; - while (cur_offset + entry_size_ <= end_) { + for (uint32_t i = 0; i < count_; i++) { SymType entry; - if (!elf_memory->ReadFully(cur_offset, &entry, sizeof(entry))) { + if (!elf_memory->ReadFully(offset_ + i * entry_size_, &entry, sizeof(entry))) { return false; } - cur_offset += entry_size_; if (entry.st_shndx != SHN_UNDEF && ELF32_ST_TYPE(entry.st_info) == STT_OBJECT && ELF32_ST_BIND(entry.st_info) == STB_GLOBAL) { diff --git a/libunwindstack/Symbols.h b/libunwindstack/Symbols.h index 7fcd067d3..3b3f20b1c 100644 --- a/libunwindstack/Symbols.h +++ b/libunwindstack/Symbols.h @@ -19,8 +19,9 @@ #include <stdint.h> +#include <optional> #include <string> -#include <vector> +#include <unordered_map> namespace unwindstack { @@ -29,11 +30,9 @@ class Memory; class Symbols { struct Info { - Info(uint64_t start_offset, uint64_t end_offset, uint64_t str_offset) - : start_offset(start_offset), end_offset(end_offset), str_offset(str_offset) {} - uint64_t start_offset; - uint64_t end_offset; - uint64_t str_offset; + uint64_t addr; // Symbol address. + uint32_t size; // Symbol size in bytes. Zero if not a function. + uint32_t name; // Offset in .strtab. }; public: @@ -41,8 +40,6 @@ class Symbols { uint64_t str_size); virtual ~Symbols() = default; - const Info* GetInfoFromCache(uint64_t addr); - template <typename SymType> bool GetName(uint64_t addr, Memory* elf_memory, std::string* name, uint64_t* func_offset); @@ -51,18 +48,27 @@ class Symbols { void ClearCache() { symbols_.clear(); - cur_offset_ = offset_; + remap_.reset(); } private: - uint64_t cur_offset_; - uint64_t offset_; - uint64_t end_; - uint64_t entry_size_; - uint64_t str_offset_; - uint64_t str_end_; - - std::vector<Info> symbols_; + template <typename SymType> + const Info* ReadFuncInfo(uint32_t symbol_index, Memory* elf_memory); + + template <typename SymType, bool RemapIndices> + const Info* BinarySearch(uint64_t addr, Memory* elf_memory); + + template <typename SymType> + void BuildRemapTable(Memory* elf_memory); + + const uint64_t offset_; + const uint64_t count_; + const uint64_t entry_size_; + const uint64_t str_offset_; + const uint64_t str_end_; + + std::unordered_map<uint32_t, Info> symbols_; // Cache of read symbols (keyed by symbol index). + std::optional<std::vector<uint32_t>> remap_; // Indices of function symbols sorted by address. }; } // namespace unwindstack diff --git a/libunwindstack/Unwinder.cpp b/libunwindstack/Unwinder.cpp index 1bb031947..2d867cd5e 100644 --- a/libunwindstack/Unwinder.cpp +++ b/libunwindstack/Unwinder.cpp @@ -181,7 +181,7 @@ void Unwinder::Unwind(const std::vector<std::string>* initial_map_names_to_skip, step_pc = rel_pc; } if (adjust_pc) { - pc_adjustment = regs_->GetPcAdjustment(rel_pc, elf); + pc_adjustment = GetPcAdjustment(rel_pc, elf, arch); } else { pc_adjustment = 0; } @@ -395,4 +395,54 @@ bool UnwinderFromPid::Init(ArchEnum arch) { return true; } +FrameData Unwinder::BuildFrameFromPcOnly(uint64_t pc) { + FrameData frame; + + Maps* maps = GetMaps(); + MapInfo* map_info = maps->Find(pc); + if (!map_info) { + frame.rel_pc = pc; + return frame; + } + + ArchEnum arch = Regs::CurrentArch(); + Elf* elf = map_info->GetElf(GetProcessMemory(), arch); + + uint64_t relative_pc = elf->GetRelPc(pc, map_info); + + uint64_t pc_adjustment = GetPcAdjustment(relative_pc, elf, arch); + relative_pc -= pc_adjustment; + // The debug PC may be different if the PC comes from the JIT. + uint64_t debug_pc = relative_pc; + + // If we don't have a valid ELF file, check the JIT. + if (!elf->valid()) { + JitDebug jit_debug(GetProcessMemory()); + uint64_t jit_pc = pc - pc_adjustment; + Elf* jit_elf = jit_debug.GetElf(maps, jit_pc); + if (jit_elf != nullptr) { + debug_pc = jit_pc; + elf = jit_elf; + } + } + + // Copy all the things we need into the frame for symbolization. + frame.rel_pc = relative_pc; + frame.pc = pc - pc_adjustment; + frame.map_name = map_info->name; + frame.map_elf_start_offset = map_info->elf_start_offset; + frame.map_exact_offset = map_info->offset; + frame.map_start = map_info->start; + frame.map_end = map_info->end; + frame.map_flags = map_info->flags; + frame.map_load_bias = elf->GetLoadBias(); + + if (!resolve_names_ || + !elf->GetFunctionName(relative_pc, &frame.function_name, &frame.function_offset)) { + frame.function_name = ""; + frame.function_offset = 0; + } + return frame; +} + } // namespace unwindstack diff --git a/libunwindstack/benchmarks/ElfBenchmark.cpp b/libunwindstack/benchmarks/ElfBenchmark.cpp new file mode 100644 index 000000000..a46bd7a24 --- /dev/null +++ b/libunwindstack/benchmarks/ElfBenchmark.cpp @@ -0,0 +1,142 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <err.h> +#include <malloc.h> +#include <stdint.h> + +#include <string> + +#include <benchmark/benchmark.h> + +#include <unwindstack/Elf.h> +#include <unwindstack/Maps.h> +#include <unwindstack/Memory.h> +#include <unwindstack/Regs.h> + +#include "Utils.h" + +static void BenchmarkElfCreate(benchmark::State& state, const std::string& elf_file) { +#if defined(__BIONIC__) + uint64_t rss_bytes = 0; +#endif + uint64_t alloc_bytes = 0; + for (auto _ : state) { + state.PauseTiming(); +#if defined(__BIONIC__) + mallopt(M_PURGE, 0); + uint64_t rss_bytes_before = 0; + GatherRss(&rss_bytes_before); +#endif + uint64_t alloc_bytes_before = mallinfo().uordblks; + auto file_memory = unwindstack::Memory::CreateFileMemory(elf_file, 0); + state.ResumeTiming(); + + unwindstack::Elf elf(file_memory.release()); + if (!elf.Init() || !elf.valid()) { + errx(1, "Internal Error: Cannot open elf."); + } + + state.PauseTiming(); +#if defined(__BIONIC__) + mallopt(M_PURGE, 0); +#endif + alloc_bytes += mallinfo().uordblks - alloc_bytes_before; +#if defined(__BIONIC__) + GatherRss(&rss_bytes); + rss_bytes -= rss_bytes_before; +#endif + state.ResumeTiming(); + } + +#if defined(__BIONIC__) + state.counters["RSS_BYTES"] = rss_bytes / static_cast<double>(state.iterations()); +#endif + state.counters["ALLOCATED_BYTES"] = alloc_bytes / static_cast<double>(state.iterations()); +} + +void BM_elf_create(benchmark::State& state) { + BenchmarkElfCreate(state, GetElfFile()); +} +BENCHMARK(BM_elf_create); + +void BM_elf_create_compressed(benchmark::State& state) { + BenchmarkElfCreate(state, GetCompressedElfFile()); +} +BENCHMARK(BM_elf_create_compressed); + +static void InitializeBuildId(benchmark::State& state, unwindstack::Maps& maps, + unwindstack::MapInfo** build_id_map_info) { + if (!maps.Parse()) { + state.SkipWithError("Failed to parse local maps."); + return; + } + + // Find the libc.so share library and use that for benchmark purposes. + *build_id_map_info = nullptr; + for (auto& map_info : maps) { + if (map_info->offset == 0 && map_info->GetBuildID() != "") { + *build_id_map_info = map_info.get(); + break; + } + } + + if (*build_id_map_info == nullptr) { + state.SkipWithError("Failed to find a map with a BuildID."); + } +} + +static void BM_elf_get_build_id_from_object(benchmark::State& state) { + unwindstack::LocalMaps maps; + unwindstack::MapInfo* build_id_map_info; + InitializeBuildId(state, maps, &build_id_map_info); + + unwindstack::Elf* elf = build_id_map_info->GetElf(std::shared_ptr<unwindstack::Memory>(), + unwindstack::Regs::CurrentArch()); + if (!elf->valid()) { + state.SkipWithError("Cannot get valid elf from map."); + } + + for (auto _ : state) { + state.PauseTiming(); + uintptr_t id = build_id_map_info->build_id; + if (id != 0) { + delete reinterpret_cast<std::string*>(id); + build_id_map_info->build_id = 0; + } + state.ResumeTiming(); + benchmark::DoNotOptimize(build_id_map_info->GetBuildID()); + } +} +BENCHMARK(BM_elf_get_build_id_from_object); + +static void BM_elf_get_build_id_from_file(benchmark::State& state) { + unwindstack::LocalMaps maps; + unwindstack::MapInfo* build_id_map_info; + InitializeBuildId(state, maps, &build_id_map_info); + + for (auto _ : state) { + state.PauseTiming(); + uintptr_t id = build_id_map_info->build_id; + if (id != 0) { + delete reinterpret_cast<std::string*>(id); + build_id_map_info->build_id = 0; + } + state.ResumeTiming(); + benchmark::DoNotOptimize(build_id_map_info->GetBuildID()); + } +} +BENCHMARK(BM_elf_get_build_id_from_file); diff --git a/libunwindstack/benchmarks/MapsBenchmark.cpp b/libunwindstack/benchmarks/MapsBenchmark.cpp new file mode 100644 index 000000000..5df149168 --- /dev/null +++ b/libunwindstack/benchmarks/MapsBenchmark.cpp @@ -0,0 +1,160 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <err.h> +#include <stdint.h> + +#include <string> + +#include <android-base/file.h> +#include <android-base/stringprintf.h> + +#include <benchmark/benchmark.h> + +#include <unwindstack/Maps.h> + +class BenchmarkLocalUpdatableMaps : public unwindstack::LocalUpdatableMaps { + public: + BenchmarkLocalUpdatableMaps() : unwindstack::LocalUpdatableMaps() {} + virtual ~BenchmarkLocalUpdatableMaps() = default; + + const std::string GetMapsFile() const override { return maps_file_; } + + void BenchmarkSetMapsFile(const std::string& maps_file) { maps_file_ = maps_file; } + + private: + std::string maps_file_; +}; + +static constexpr size_t kNumSmallMaps = 100; +static constexpr size_t kNumLargeMaps = 10000; + +static void CreateMap(const char* filename, size_t num_maps, size_t increment = 1) { + std::string maps; + for (size_t i = 0; i < num_maps; i += increment) { + maps += android::base::StringPrintf("%zu-%zu r-xp 0000 00:00 0 name%zu\n", i * 1000, + (i + 1) * 1000 * increment, i * increment); + } + if (!android::base::WriteStringToFile(maps, filename)) { + errx(1, "WriteStringToFile failed"); + } +} + +static void ReparseBenchmark(benchmark::State& state, const char* maps1, size_t maps1_total, + const char* maps2, size_t maps2_total) { + for (auto _ : state) { + BenchmarkLocalUpdatableMaps maps; + maps.BenchmarkSetMapsFile(maps1); + if (!maps.Reparse()) { + errx(1, "Internal Error: reparse of initial maps filed."); + } + if (maps.Total() != maps1_total) { + errx(1, "Internal Error: Incorrect total number of maps %zu, expected %zu.", maps.Total(), + maps1_total); + } + maps.BenchmarkSetMapsFile(maps2); + if (!maps.Reparse()) { + errx(1, "Internal Error: reparse of second set of maps filed."); + } + if (maps.Total() != maps2_total) { + errx(1, "Internal Error: Incorrect total number of maps %zu, expected %zu.", maps.Total(), + maps2_total); + } + } +} + +void BM_local_updatable_maps_reparse_double_initial_small(benchmark::State& state) { + TemporaryFile initial_maps; + CreateMap(initial_maps.path, kNumSmallMaps, 2); + + TemporaryFile reparse_maps; + CreateMap(reparse_maps.path, kNumSmallMaps); + + ReparseBenchmark(state, initial_maps.path, kNumSmallMaps / 2, reparse_maps.path, kNumSmallMaps); +} +BENCHMARK(BM_local_updatable_maps_reparse_double_initial_small); + +void BM_local_updatable_maps_reparse_double_initial_large(benchmark::State& state) { + TemporaryFile initial_maps; + CreateMap(initial_maps.path, kNumLargeMaps, 2); + + TemporaryFile reparse_maps; + CreateMap(reparse_maps.path, kNumLargeMaps); + + ReparseBenchmark(state, initial_maps.path, kNumLargeMaps / 2, reparse_maps.path, kNumLargeMaps); +} +BENCHMARK(BM_local_updatable_maps_reparse_double_initial_large); + +void BM_local_updatable_maps_reparse_same_maps_small(benchmark::State& state) { + static constexpr size_t kNumSmallMaps = 100; + TemporaryFile maps; + CreateMap(maps.path, kNumSmallMaps); + + ReparseBenchmark(state, maps.path, kNumSmallMaps, maps.path, kNumSmallMaps); +} +BENCHMARK(BM_local_updatable_maps_reparse_same_maps_small); + +void BM_local_updatable_maps_reparse_same_maps_large(benchmark::State& state) { + TemporaryFile maps; + CreateMap(maps.path, kNumLargeMaps); + + ReparseBenchmark(state, maps.path, kNumLargeMaps, maps.path, kNumLargeMaps); +} +BENCHMARK(BM_local_updatable_maps_reparse_same_maps_large); + +void BM_local_updatable_maps_reparse_few_extra_small(benchmark::State& state) { + TemporaryFile maps1; + CreateMap(maps1.path, kNumSmallMaps - 4); + + TemporaryFile maps2; + CreateMap(maps2.path, kNumSmallMaps); + + ReparseBenchmark(state, maps1.path, kNumSmallMaps - 4, maps2.path, kNumSmallMaps); +} +BENCHMARK(BM_local_updatable_maps_reparse_few_extra_small); + +void BM_local_updatable_maps_reparse_few_extra_large(benchmark::State& state) { + TemporaryFile maps1; + CreateMap(maps1.path, kNumLargeMaps - 4); + + TemporaryFile maps2; + CreateMap(maps2.path, kNumLargeMaps); + + ReparseBenchmark(state, maps1.path, kNumLargeMaps - 4, maps2.path, kNumLargeMaps); +} +BENCHMARK(BM_local_updatable_maps_reparse_few_extra_large); + +void BM_local_updatable_maps_reparse_few_less_small(benchmark::State& state) { + TemporaryFile maps1; + CreateMap(maps1.path, kNumSmallMaps); + + TemporaryFile maps2; + CreateMap(maps2.path, kNumSmallMaps - 4); + + ReparseBenchmark(state, maps1.path, kNumSmallMaps, maps2.path, kNumSmallMaps - 4); +} +BENCHMARK(BM_local_updatable_maps_reparse_few_less_small); + +void BM_local_updatable_maps_reparse_few_less_large(benchmark::State& state) { + TemporaryFile maps1; + CreateMap(maps1.path, kNumLargeMaps); + + TemporaryFile maps2; + CreateMap(maps2.path, kNumLargeMaps - 4); + + ReparseBenchmark(state, maps1.path, kNumLargeMaps, maps2.path, kNumLargeMaps - 4); +} +BENCHMARK(BM_local_updatable_maps_reparse_few_less_large); diff --git a/libunwindstack/benchmarks/SymbolBenchmark.cpp b/libunwindstack/benchmarks/SymbolBenchmark.cpp new file mode 100644 index 000000000..73088dab0 --- /dev/null +++ b/libunwindstack/benchmarks/SymbolBenchmark.cpp @@ -0,0 +1,129 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <err.h> +#include <inttypes.h> +#include <malloc.h> +#include <stdint.h> + +#include <string> +#include <vector> + +#include <benchmark/benchmark.h> + +#include <unwindstack/Elf.h> +#include <unwindstack/Memory.h> + +#include "Utils.h" + +static void BenchmarkSymbolLookup(benchmark::State& state, std::vector<uint64_t> offsets, + std::string elf_file, bool expect_found) { +#if defined(__BIONIC__) + uint64_t rss_bytes = 0; +#endif + uint64_t alloc_bytes = 0; + for (auto _ : state) { + state.PauseTiming(); + unwindstack::Elf elf(unwindstack::Memory::CreateFileMemory(elf_file, 0).release()); + if (!elf.Init() || !elf.valid()) { + errx(1, "Internal Error: Cannot open elf."); + } + +#if defined(__BIONIC__) + mallopt(M_PURGE, 0); + uint64_t rss_bytes_before = 0; + GatherRss(&rss_bytes_before); +#endif + uint64_t alloc_bytes_before = mallinfo().uordblks; + state.ResumeTiming(); + + for (auto pc : offsets) { + std::string name; + uint64_t offset; + bool found = elf.GetFunctionName(pc, &name, &offset); + if (expect_found && !found) { + errx(1, "expected pc 0x%" PRIx64 " present, but not found.", pc); + } else if (!expect_found && found) { + errx(1, "expected pc 0x%" PRIx64 " not present, but found.", pc); + } + } + + state.PauseTiming(); +#if defined(__BIONIC__) + mallopt(M_PURGE, 0); +#endif + alloc_bytes += mallinfo().uordblks - alloc_bytes_before; +#if defined(__BIONIC__) + GatherRss(&rss_bytes); + rss_bytes -= rss_bytes_before; +#endif + state.ResumeTiming(); + } + +#if defined(__BIONIC__) + state.counters["RSS_BYTES"] = rss_bytes / static_cast<double>(state.iterations()); +#endif + state.counters["ALLOCATED_BYTES"] = alloc_bytes / static_cast<double>(state.iterations()); +} + +static void BenchmarkSymbolLookup(benchmark::State& state, uint64_t pc, std::string elf_file, + bool expect_found) { + BenchmarkSymbolLookup(state, std::vector<uint64_t>{pc}, elf_file, expect_found); +} + +void BM_symbol_not_present(benchmark::State& state) { + BenchmarkSymbolLookup(state, 0, GetElfFile(), false); +} +BENCHMARK(BM_symbol_not_present); + +void BM_symbol_find_single(benchmark::State& state) { + BenchmarkSymbolLookup(state, 0x22b2bc, GetElfFile(), true); +} +BENCHMARK(BM_symbol_find_single); + +void BM_symbol_find_single_many_times(benchmark::State& state) { + BenchmarkSymbolLookup(state, std::vector<uint64_t>(15, 0x22b2bc), GetElfFile(), true); +} +BENCHMARK(BM_symbol_find_single_many_times); + +void BM_symbol_find_multiple(benchmark::State& state) { + BenchmarkSymbolLookup(state, + std::vector<uint64_t>{0x22b2bc, 0xd5d30, 0x1312e8, 0x13582e, 0x1389c8}, + GetElfFile(), true); +} +BENCHMARK(BM_symbol_find_multiple); + +void BM_symbol_not_present_from_sorted(benchmark::State& state) { + BenchmarkSymbolLookup(state, 0, GetSymbolSortedElfFile(), false); +} +BENCHMARK(BM_symbol_not_present_from_sorted); + +void BM_symbol_find_single_from_sorted(benchmark::State& state) { + BenchmarkSymbolLookup(state, 0x138638, GetSymbolSortedElfFile(), true); +} +BENCHMARK(BM_symbol_find_single_from_sorted); + +void BM_symbol_find_single_many_times_from_sorted(benchmark::State& state) { + BenchmarkSymbolLookup(state, std::vector<uint64_t>(15, 0x138638), GetSymbolSortedElfFile(), true); +} +BENCHMARK(BM_symbol_find_single_many_times_from_sorted); + +void BM_symbol_find_multiple_from_sorted(benchmark::State& state) { + BenchmarkSymbolLookup(state, + std::vector<uint64_t>{0x138638, 0x84350, 0x14df18, 0x1f3a38, 0x1f3ca8}, + GetSymbolSortedElfFile(), true); +} +BENCHMARK(BM_symbol_find_multiple_from_sorted); diff --git a/libunwindstack/benchmarks/Utils.cpp b/libunwindstack/benchmarks/Utils.cpp new file mode 100644 index 000000000..c92f1099d --- /dev/null +++ b/libunwindstack/benchmarks/Utils.cpp @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <err.h> +#include <stdint.h> + +#include <string> +#include <vector> + +#include <android-base/file.h> +#include <android-base/strings.h> +#include <benchmark/benchmark.h> + +#include <unwindstack/Elf.h> +#include <unwindstack/Memory.h> + +std::string GetElfFile() { + return android::base::GetExecutableDirectory() + "/benchmarks/files/libart_arm.so"; +} + +std::string GetSymbolSortedElfFile() { + return android::base::GetExecutableDirectory() + "/benchmarks/files/boot_arm.oat"; +} + +std::string GetCompressedElfFile() { + // Both are the same right now. + return GetSymbolSortedElfFile(); +} + +#if defined(__BIONIC__) + +#include <meminfo/procmeminfo.h> +#include <procinfo/process_map.h> + +void GatherRss(uint64_t* rss_bytes) { + android::meminfo::ProcMemInfo proc_mem(getpid()); + const std::vector<android::meminfo::Vma>& maps = proc_mem.MapsWithoutUsageStats(); + for (auto& vma : maps) { + if (vma.name == "[anon:libc_malloc]" || android::base::StartsWith(vma.name, "[anon:scudo:") || + android::base::StartsWith(vma.name, "[anon:GWP-ASan")) { + android::meminfo::Vma update_vma(vma); + if (!proc_mem.FillInVmaStats(update_vma)) { + err(1, "FillInVmaStats failed\n"); + } + *rss_bytes += update_vma.usage.rss; + } + } +} +#endif diff --git a/base/include/android-base/threads.h b/libunwindstack/benchmarks/Utils.h index dba1fc620..bee6efcad 100644 --- a/base/include/android-base/threads.h +++ b/libunwindstack/benchmarks/Utils.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018 The Android Open Source Project + * Copyright (C) 2020 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,17 +14,26 @@ * limitations under the License. */ -#pragma once +#ifndef _LIBUNWINDSTACK_UTILS_H +#define _LIBUNWINDSTACK_UTILS_H #include <stdint.h> -namespace android { -namespace base { -uint64_t GetThreadId(); -} -} // namespace android +#include <string> + +std::string GetElfFile(); + +std::string GetSymbolSortedElfFile(); + +std::string GetCompressedElfFile(); + +#if defined(__BIONIC__) + +#include <meminfo/procmeminfo.h> +#include <procinfo/process_map.h> + +void GatherRss(uint64_t* rss_bytes); -#if defined(__GLIBC__) -// bionic has this Linux-specifix call, but glibc doesn't. -extern "C" int tgkill(int tgid, int tid, int sig); #endif + +#endif // _LIBUNWINDSTACK_UTILS_h diff --git a/libunwindstack/benchmarks/files/boot_arm.oat b/libunwindstack/benchmarks/files/boot_arm.oat Binary files differnew file mode 100644 index 000000000..51188ebd3 --- /dev/null +++ b/libunwindstack/benchmarks/files/boot_arm.oat diff --git a/libunwindstack/benchmarks/files/libart_arm.so b/libunwindstack/benchmarks/files/libart_arm.so Binary files differnew file mode 100644 index 000000000..2201faf8f --- /dev/null +++ b/libunwindstack/benchmarks/files/libart_arm.so diff --git a/libunwindstack/benchmarks/unwind_benchmarks.cpp b/libunwindstack/benchmarks/unwind_benchmarks.cpp index de9137aa4..0bee6ef78 100644 --- a/libunwindstack/benchmarks/unwind_benchmarks.cpp +++ b/libunwindstack/benchmarks/unwind_benchmarks.cpp @@ -22,7 +22,6 @@ #include <android-base/strings.h> -#include <unwindstack/Elf.h> #include <unwindstack/Maps.h> #include <unwindstack/Memory.h> #include <unwindstack/Regs.h> @@ -83,63 +82,4 @@ static void BM_cached_unwind(benchmark::State& state) { } BENCHMARK(BM_cached_unwind); -static void Initialize(benchmark::State& state, unwindstack::Maps& maps, - unwindstack::MapInfo** build_id_map_info) { - if (!maps.Parse()) { - state.SkipWithError("Failed to parse local maps."); - return; - } - - // Find the libc.so share library and use that for benchmark purposes. - *build_id_map_info = nullptr; - for (auto& map_info : maps) { - if (map_info->offset == 0 && map_info->GetBuildID() != "") { - *build_id_map_info = map_info.get(); - break; - } - } - - if (*build_id_map_info == nullptr) { - state.SkipWithError("Failed to find a map with a BuildID."); - } -} - -static void BM_get_build_id_from_elf(benchmark::State& state) { - unwindstack::LocalMaps maps; - unwindstack::MapInfo* build_id_map_info; - Initialize(state, maps, &build_id_map_info); - - unwindstack::Elf* elf = build_id_map_info->GetElf(std::shared_ptr<unwindstack::Memory>(), - unwindstack::Regs::CurrentArch()); - if (!elf->valid()) { - state.SkipWithError("Cannot get valid elf from map."); - } - - for (auto _ : state) { - uintptr_t id = build_id_map_info->build_id; - if (id != 0) { - delete reinterpret_cast<std::string*>(id); - build_id_map_info->build_id = 0; - } - benchmark::DoNotOptimize(build_id_map_info->GetBuildID()); - } -} -BENCHMARK(BM_get_build_id_from_elf); - -static void BM_get_build_id_from_file(benchmark::State& state) { - unwindstack::LocalMaps maps; - unwindstack::MapInfo* build_id_map_info; - Initialize(state, maps, &build_id_map_info); - - for (auto _ : state) { - uintptr_t id = build_id_map_info->build_id; - if (id != 0) { - delete reinterpret_cast<std::string*>(id); - build_id_map_info->build_id = 0; - } - benchmark::DoNotOptimize(build_id_map_info->GetBuildID()); - } -} -BENCHMARK(BM_get_build_id_from_file); - BENCHMARK_MAIN(); diff --git a/libunwindstack/include/unwindstack/Memory.h b/libunwindstack/include/unwindstack/Memory.h index 310656497..3d8187856 100644 --- a/libunwindstack/include/unwindstack/Memory.h +++ b/libunwindstack/include/unwindstack/Memory.h @@ -37,11 +37,14 @@ class Memory { uint64_t end); static std::unique_ptr<Memory> CreateFileMemory(const std::string& path, uint64_t offset); - virtual bool ReadString(uint64_t addr, std::string* string, uint64_t max_read = UINT64_MAX); + virtual bool ReadString(uint64_t addr, std::string* dst, size_t max_read); virtual void Clear() {} + virtual bool IsLocal() const { return false; } + virtual size_t Read(uint64_t addr, void* dst, size_t size) = 0; + virtual long ReadTag(uint64_t) { return -1; } bool ReadFully(uint64_t addr, void* dst, size_t size); diff --git a/libunwindstack/include/unwindstack/Regs.h b/libunwindstack/include/unwindstack/Regs.h index 4f761b4ef..a367e6cf1 100644 --- a/libunwindstack/include/unwindstack/Regs.h +++ b/libunwindstack/include/unwindstack/Regs.h @@ -64,8 +64,6 @@ class Regs { uint64_t dex_pc() { return dex_pc_; } void set_dex_pc(uint64_t dex_pc) { dex_pc_ = dex_pc; } - virtual uint64_t GetPcAdjustment(uint64_t rel_pc, Elf* elf) = 0; - virtual bool StepIfSignalHandler(uint64_t elf_offset, Elf* elf, Memory* process_memory) = 0; virtual bool SetPcFromReturnAddress(Memory* process_memory) = 0; @@ -110,6 +108,8 @@ class RegsImpl : public Regs { std::vector<AddressType> regs_; }; +uint64_t GetPcAdjustment(uint64_t rel_pc, Elf* elf, ArchEnum arch); + } // namespace unwindstack #endif // _LIBUNWINDSTACK_REGS_H diff --git a/libunwindstack/include/unwindstack/RegsArm.h b/libunwindstack/include/unwindstack/RegsArm.h index aa029be64..fbb34e7f8 100644 --- a/libunwindstack/include/unwindstack/RegsArm.h +++ b/libunwindstack/include/unwindstack/RegsArm.h @@ -36,8 +36,6 @@ class RegsArm : public RegsImpl<uint32_t> { ArchEnum Arch() override final; - uint64_t GetPcAdjustment(uint64_t rel_pc, Elf* elf) override; - bool SetPcFromReturnAddress(Memory* process_memory) override; bool StepIfSignalHandler(uint64_t elf_offset, Elf* elf, Memory* process_memory) override; diff --git a/libunwindstack/include/unwindstack/RegsArm64.h b/libunwindstack/include/unwindstack/RegsArm64.h index 5cd7e5ba9..2b3ddeb77 100644 --- a/libunwindstack/include/unwindstack/RegsArm64.h +++ b/libunwindstack/include/unwindstack/RegsArm64.h @@ -36,8 +36,6 @@ class RegsArm64 : public RegsImpl<uint64_t> { ArchEnum Arch() override final; - uint64_t GetPcAdjustment(uint64_t rel_pc, Elf* elf) override; - bool SetPcFromReturnAddress(Memory* process_memory) override; bool StepIfSignalHandler(uint64_t elf_offset, Elf* elf, Memory* process_memory) override; diff --git a/libunwindstack/include/unwindstack/RegsGetLocal.h b/libunwindstack/include/unwindstack/RegsGetLocal.h index f0b5e3a1d..300a3ecc5 100644 --- a/libunwindstack/include/unwindstack/RegsGetLocal.h +++ b/libunwindstack/include/unwindstack/RegsGetLocal.h @@ -81,7 +81,7 @@ inline __attribute__((__always_inline__)) void AsmGetRegs(void* reg_data) { : "x12", "x13", "memory"); } -#elif defined(__i386__) || defined(__x86_64__) || defined(__mips__) +#elif defined(__i386__) || defined(__x86_64__) extern "C" void AsmGetRegs(void* regs); diff --git a/libunwindstack/include/unwindstack/RegsMips.h b/libunwindstack/include/unwindstack/RegsMips.h index 8164a1515..dc09b83b8 100644 --- a/libunwindstack/include/unwindstack/RegsMips.h +++ b/libunwindstack/include/unwindstack/RegsMips.h @@ -36,8 +36,6 @@ class RegsMips : public RegsImpl<uint32_t> { ArchEnum Arch() override final; - uint64_t GetPcAdjustment(uint64_t rel_pc, Elf* elf) override; - bool SetPcFromReturnAddress(Memory* process_memory) override; bool StepIfSignalHandler(uint64_t elf_offset, Elf* elf, Memory* process_memory) override; diff --git a/libunwindstack/include/unwindstack/RegsMips64.h b/libunwindstack/include/unwindstack/RegsMips64.h index c98254259..64a80dc0b 100644 --- a/libunwindstack/include/unwindstack/RegsMips64.h +++ b/libunwindstack/include/unwindstack/RegsMips64.h @@ -36,8 +36,6 @@ class RegsMips64 : public RegsImpl<uint64_t> { ArchEnum Arch() override final; - uint64_t GetPcAdjustment(uint64_t rel_pc, Elf* elf) override; - bool SetPcFromReturnAddress(Memory* process_memory) override; bool StepIfSignalHandler(uint64_t elf_offset, Elf* elf, Memory* process_memory) override; diff --git a/libunwindstack/include/unwindstack/RegsX86.h b/libunwindstack/include/unwindstack/RegsX86.h index 2323a4fc0..cfbdda661 100644 --- a/libunwindstack/include/unwindstack/RegsX86.h +++ b/libunwindstack/include/unwindstack/RegsX86.h @@ -37,8 +37,6 @@ class RegsX86 : public RegsImpl<uint32_t> { ArchEnum Arch() override final; - uint64_t GetPcAdjustment(uint64_t rel_pc, Elf* elf) override; - bool SetPcFromReturnAddress(Memory* process_memory) override; bool StepIfSignalHandler(uint64_t elf_offset, Elf* elf, Memory* process_memory) override; diff --git a/libunwindstack/include/unwindstack/RegsX86_64.h b/libunwindstack/include/unwindstack/RegsX86_64.h index 3e919a4f8..a11aef09f 100644 --- a/libunwindstack/include/unwindstack/RegsX86_64.h +++ b/libunwindstack/include/unwindstack/RegsX86_64.h @@ -37,8 +37,6 @@ class RegsX86_64 : public RegsImpl<uint64_t> { ArchEnum Arch() override final; - uint64_t GetPcAdjustment(uint64_t rel_pc, Elf* elf) override; - bool SetPcFromReturnAddress(Memory* process_memory) override; bool StepIfSignalHandler(uint64_t elf_offset, Elf* elf, Memory* process_memory) override; diff --git a/libunwindstack/include/unwindstack/Unwinder.h b/libunwindstack/include/unwindstack/Unwinder.h index 67762c0bd..4d49f236a 100644 --- a/libunwindstack/include/unwindstack/Unwinder.h +++ b/libunwindstack/include/unwindstack/Unwinder.h @@ -114,6 +114,13 @@ class Unwinder { ErrorCode LastErrorCode() { return last_error_.code; } uint64_t LastErrorAddress() { return last_error_.address; } + // Builds a frame for symbolization using the maps from this unwinder. The + // constructed frame contains just enough information to be used to symbolize + // frames collected by frame-pointer unwinding that's done outside of + // libunwindstack. This is used by tombstoned to symbolize frame pointer-based + // stack traces that are collected by tools such as GWP-ASan and MTE. + FrameData BuildFrameFromPcOnly(uint64_t pc); + protected: Unwinder(size_t max_frames) : max_frames_(max_frames) { frames_.reserve(max_frames); } diff --git a/libunwindstack/tests/DexFileTest.cpp b/libunwindstack/tests/DexFileTest.cpp index dc935a36e..1deba0145 100644 --- a/libunwindstack/tests/DexFileTest.cpp +++ b/libunwindstack/tests/DexFileTest.cpp @@ -21,6 +21,7 @@ #include <unordered_map> +#include <MemoryLocal.h> #include <android-base/file.h> #include <gtest/gtest.h> #include <unwindstack/MapInfo.h> @@ -109,7 +110,7 @@ TEST(DexFileTest, from_memory_fail_too_small_for_header) { memory.SetMemory(0x1000, kDexData, 10); - EXPECT_TRUE(DexFileFromMemory::Create(0x1000, &memory, "") == nullptr); + EXPECT_TRUE(DexFileFromMemory::Create(0x1000, &memory, "", sizeof(kDexData)) == nullptr); } TEST(DexFileTest, from_memory_fail_too_small_for_data) { @@ -117,7 +118,7 @@ TEST(DexFileTest, from_memory_fail_too_small_for_data) { memory.SetMemory(0x1000, kDexData, sizeof(kDexData) - 2); - EXPECT_TRUE(DexFileFromMemory::Create(0x1000, &memory, "") == nullptr); + EXPECT_TRUE(DexFileFromMemory::Create(0x1000, &memory, "", sizeof(kDexData)) == nullptr); } TEST(DexFileTest, from_memory_open) { @@ -125,7 +126,7 @@ TEST(DexFileTest, from_memory_open) { memory.SetMemory(0x1000, kDexData, sizeof(kDexData)); - EXPECT_TRUE(DexFileFromMemory::Create(0x1000, &memory, "") != nullptr); + EXPECT_TRUE(DexFileFromMemory::Create(0x1000, &memory, "", sizeof(kDexData)) != nullptr); } TEST(DexFileTest, from_memory_no_leak) { @@ -136,7 +137,7 @@ TEST(DexFileTest, from_memory_no_leak) { size_t first_allocated_bytes = 0; size_t last_allocated_bytes = 0; for (size_t i = 0; i < kNumLeakLoops; i++) { - EXPECT_TRUE(DexFileFromMemory::Create(0x1000, &memory, "") != nullptr); + EXPECT_TRUE(DexFileFromMemory::Create(0x1000, &memory, "", sizeof(kDexData)) != nullptr); ASSERT_NO_FATAL_FAILURE(CheckForLeak(i, &first_allocated_bytes, &last_allocated_bytes)); } } @@ -213,6 +214,43 @@ TEST(DexFileTest, create_using_memory_file_is_malformed) { EXPECT_TRUE(dex_file == nullptr); } +TEST(DexFileTest, create_using_memory_size_too_small) { + MemoryFake memory; + memory.SetMemory(0x4000, kDexData, sizeof(kDexData)); + MapInfo info(nullptr, nullptr, 0x100, sizeof(kDexData) - 2, 0x200, 0x5, "/does/not/exist"); + EXPECT_TRUE(DexFile::Create(0x4000, &memory, &info) != nullptr); +} + +class MemoryLocalFake : public MemoryLocal { + public: + MemoryLocalFake(size_t memory_size) : backing_(memory_size) {} + virtual ~MemoryLocalFake() = default; + + void* Data() { return backing_.data(); } + + private: + std::vector<void*> backing_; +}; + +TEST(DexFileTest, create_using_local_memory) { + MemoryLocalFake memory(sizeof(kDexData)); + + memcpy(memory.Data(), kDexData, sizeof(kDexData)); + uint64_t start = reinterpret_cast<uint64_t>(memory.Data()); + MapInfo info(nullptr, nullptr, start, start + 0x1000, 0x200, 0x5, "/does/not/exist"); + EXPECT_TRUE(DexFile::Create(start, &memory, &info) != nullptr); +} + +TEST(DexFileTest, create_using_local_memory_size_too_small) { + MemoryLocalFake memory(sizeof(kDexData)); + + memcpy(memory.Data(), kDexData, sizeof(kDexData)); + uint64_t start = reinterpret_cast<uint64_t>(memory.Data()); + MapInfo info(nullptr, nullptr, start, start + sizeof(kDexData) - 2, 0x200, 0x5, + "/does/not/exist"); + EXPECT_TRUE(DexFile::Create(start, &memory, &info) == nullptr); +} + TEST(DexFileTest, get_method) { MemoryFake memory; memory.SetMemory(0x4000, kDexData, sizeof(kDexData)); diff --git a/libunwindstack/tests/ElfFake.h b/libunwindstack/tests/ElfFake.h index fc90dab8e..3b6cb8032 100644 --- a/libunwindstack/tests/ElfFake.h +++ b/libunwindstack/tests/ElfFake.h @@ -55,6 +55,8 @@ class ElfFake : public Elf { void FakeSetLoadBias(uint64_t load_bias) { load_bias_ = load_bias; } + void FakeSetArch(ArchEnum arch) { arch_ = arch; } + void FakeSetInterface(ElfInterface* interface) { interface_.reset(interface); } void FakeSetGnuDebugdataInterface(ElfInterface* interface) { gnu_debugdata_interface_.reset(interface); diff --git a/libunwindstack/tests/ElfTest.cpp b/libunwindstack/tests/ElfTest.cpp index 1f3ed8190..f0852a4ee 100644 --- a/libunwindstack/tests/ElfTest.cpp +++ b/libunwindstack/tests/ElfTest.cpp @@ -438,6 +438,48 @@ TEST_F(ElfTest, get_global_vaddr_in_dynamic_section) { EXPECT_EQ(0xc080U, offset); } +TEST_F(ElfTest, get_global_vaddr_with_tagged_pointer) { + ElfFake elf(memory_); + elf.FakeSetValid(true); + elf.FakeSetArch(ARCH_ARM64); + + ElfInterfaceMock* interface = new ElfInterfaceMock(memory_); + elf.FakeSetInterface(interface); + interface->MockSetDataVaddrStart(0x500); + interface->MockSetDataVaddrEnd(0x600); + interface->MockSetDataOffset(0xa000); + + std::string global("something"); + EXPECT_CALL(*interface, GetGlobalVariable(global, ::testing::_)) + .WillOnce(::testing::DoAll(::testing::SetArgPointee<1>(0x8800000000000580), + ::testing::Return(true))); + + uint64_t offset; + ASSERT_TRUE(elf.GetGlobalVariableOffset(global, &offset)); + EXPECT_EQ(0xa080U, offset); +} + +TEST_F(ElfTest, get_global_vaddr_without_tagged_pointer) { + ElfFake elf(memory_); + elf.FakeSetValid(true); + elf.FakeSetArch(ARCH_X86_64); + + ElfInterfaceMock* interface = new ElfInterfaceMock(memory_); + elf.FakeSetInterface(interface); + interface->MockSetDataVaddrStart(0x8800000000000500); + interface->MockSetDataVaddrEnd(0x8800000000000600); + interface->MockSetDataOffset(0x880000000000a000); + + std::string global("something"); + EXPECT_CALL(*interface, GetGlobalVariable(global, ::testing::_)) + .WillOnce(::testing::DoAll(::testing::SetArgPointee<1>(0x8800000000000580), + ::testing::Return(true))); + + uint64_t offset; + ASSERT_TRUE(elf.GetGlobalVariableOffset(global, &offset)); + EXPECT_EQ(0x880000000000a080U, offset); +} + TEST_F(ElfTest, is_valid_pc_elf_invalid) { ElfFake elf(memory_); elf.FakeSetValid(false); diff --git a/libunwindstack/tests/MapInfoGetBuildIDTest.cpp b/libunwindstack/tests/MapInfoGetBuildIDTest.cpp index 6953e26f9..70e136bce 100644 --- a/libunwindstack/tests/MapInfoGetBuildIDTest.cpp +++ b/libunwindstack/tests/MapInfoGetBuildIDTest.cpp @@ -142,15 +142,14 @@ static void InitElfData(int fd) { char note_section[128]; Elf32_Nhdr note_header = {}; - note_header.n_namesz = 4; // "GNU" - note_header.n_descsz = 12; // "ELF_BUILDID" + note_header.n_namesz = sizeof("GNU"); + note_header.n_descsz = sizeof("ELF_BUILDID") - 1; note_header.n_type = NT_GNU_BUILD_ID; memcpy(¬e_section, ¬e_header, sizeof(note_header)); size_t note_offset = sizeof(note_header); - memcpy(¬e_section[note_offset], "GNU", sizeof("GNU")); - note_offset += sizeof("GNU"); - memcpy(¬e_section[note_offset], "ELF_BUILDID", sizeof("ELF_BUILDID")); - note_offset += sizeof("ELF_BUILDID"); + memcpy(¬e_section[note_offset], "GNU", note_header.n_namesz); + note_offset += note_header.n_namesz; + memcpy(¬e_section[note_offset], "ELF_BUILDID", note_header.n_descsz); Elf32_Shdr shdr = {}; shdr.sh_type = SHT_NOTE; @@ -195,4 +194,10 @@ TEST_F(MapInfoGetBuildIDTest, multiple_thread_elf_exists_in_memory) { MultipleThreadTest("ELF_BUILDID"); } +TEST_F(MapInfoGetBuildIDTest, real_elf) { + MapInfo map_info(nullptr, nullptr, 0x1000, 0x20000, 0, PROT_READ | PROT_WRITE, + TestGetFileDirectory() + "offline/empty_arm64/libc.so"); + EXPECT_EQ("6df0590c4920f4c7b9f34fe833f37d54", map_info.GetPrintableBuildID()); +} + } // namespace unwindstack diff --git a/libunwindstack/tests/MemoryMteTest.cpp b/libunwindstack/tests/MemoryMteTest.cpp new file mode 100644 index 000000000..3ae322e35 --- /dev/null +++ b/libunwindstack/tests/MemoryMteTest.cpp @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if defined(ANDROID_EXPERIMENTAL_MTE) + +#include <sys/mman.h> +#include <sys/types.h> + +#include <gtest/gtest.h> + +#include <bionic/mte.h> + +#include "MemoryLocal.h" +#include "MemoryRemote.h" +#include "TestUtils.h" + +namespace unwindstack { + +static uintptr_t CreateTagMapping() { + uintptr_t mapping = + reinterpret_cast<uintptr_t>(mmap(nullptr, getpagesize(), PROT_READ | PROT_WRITE | PROT_MTE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); + if (reinterpret_cast<void*>(mapping) == MAP_FAILED) { + return 0; + } +#if defined(__aarch64__) + __asm__ __volatile__(".arch_extension mte; stg %0, [%0]" + : + : "r"(mapping + (1ULL << 56)) + : "memory"); +#endif + return mapping; +} + +TEST(MemoryMteTest, remote_read_tag) { +#if !defined(__aarch64__) + GTEST_SKIP() << "Requires aarch64"; +#else + if (!mte_supported()) { + GTEST_SKIP() << "Requires MTE"; + } +#endif + + uintptr_t mapping = CreateTagMapping(); + ASSERT_NE(0U, mapping); + + pid_t pid; + if ((pid = fork()) == 0) { + while (true) + ; + exit(1); + } + ASSERT_LT(0, pid); + TestScopedPidReaper reap(pid); + + ASSERT_TRUE(TestAttach(pid)); + + MemoryRemote remote(pid); + + EXPECT_EQ(1, remote.ReadTag(mapping)); + EXPECT_EQ(0, remote.ReadTag(mapping + 16)); + + ASSERT_TRUE(TestDetach(pid)); +} + +TEST(MemoryMteTest, local_read_tag) { +#if !defined(__aarch64__) + GTEST_SKIP() << "Requires aarch64"; +#else + if (!mte_supported()) { + GTEST_SKIP() << "Requires MTE"; + } +#endif + + uintptr_t mapping = CreateTagMapping(); + ASSERT_NE(0U, mapping); + + MemoryLocal local; + + EXPECT_EQ(1, local.ReadTag(mapping)); + EXPECT_EQ(0, local.ReadTag(mapping + 16)); +} + +} // namespace unwindstack + +#endif diff --git a/libunwindstack/tests/MemoryRemoteTest.cpp b/libunwindstack/tests/MemoryRemoteTest.cpp index c90dedcec..621893b1f 100644 --- a/libunwindstack/tests/MemoryRemoteTest.cpp +++ b/libunwindstack/tests/MemoryRemoteTest.cpp @@ -26,8 +26,8 @@ #include <vector> -#include <android-base/test_utils.h> #include <android-base/file.h> +#include <android-base/test_utils.h> #include <gtest/gtest.h> #include "MemoryRemote.h" @@ -37,24 +37,7 @@ namespace unwindstack { -class MemoryRemoteTest : public ::testing::Test { - protected: - static bool Attach(pid_t pid) { - if (ptrace(PTRACE_ATTACH, pid, 0, 0) == -1) { - return false; - } - - return TestQuiescePid(pid); - } - - static bool Detach(pid_t pid) { - return ptrace(PTRACE_DETACH, pid, 0, 0) == 0; - } - - static constexpr size_t NS_PER_SEC = 1000000000ULL; -}; - -TEST_F(MemoryRemoteTest, read) { +TEST(MemoryRemoteTest, read) { std::vector<uint8_t> src(1024); memset(src.data(), 0x4c, 1024); @@ -66,7 +49,7 @@ TEST_F(MemoryRemoteTest, read) { ASSERT_LT(0, pid); TestScopedPidReaper reap(pid); - ASSERT_TRUE(Attach(pid)); + ASSERT_TRUE(TestAttach(pid)); MemoryRemote remote(pid); @@ -76,10 +59,10 @@ TEST_F(MemoryRemoteTest, read) { ASSERT_EQ(0x4cU, dst[i]) << "Failed at byte " << i; } - ASSERT_TRUE(Detach(pid)); + ASSERT_TRUE(TestDetach(pid)); } -TEST_F(MemoryRemoteTest, read_large) { +TEST(MemoryRemoteTest, read_large) { static constexpr size_t kTotalPages = 245; std::vector<uint8_t> src(kTotalPages * getpagesize()); for (size_t i = 0; i < kTotalPages; i++) { @@ -95,7 +78,7 @@ TEST_F(MemoryRemoteTest, read_large) { ASSERT_LT(0, pid); TestScopedPidReaper reap(pid); - ASSERT_TRUE(Attach(pid)); + ASSERT_TRUE(TestAttach(pid)); MemoryRemote remote(pid); @@ -105,10 +88,10 @@ TEST_F(MemoryRemoteTest, read_large) { ASSERT_EQ(i / getpagesize(), dst[i]) << "Failed at byte " << i; } - ASSERT_TRUE(Detach(pid)); + ASSERT_TRUE(TestDetach(pid)); } -TEST_F(MemoryRemoteTest, read_partial) { +TEST(MemoryRemoteTest, read_partial) { char* mapping = static_cast<char*>( mmap(nullptr, 4 * getpagesize(), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); ASSERT_NE(MAP_FAILED, mapping); @@ -128,7 +111,7 @@ TEST_F(MemoryRemoteTest, read_partial) { // Unmap from our process. ASSERT_EQ(0, munmap(mapping, 3 * getpagesize())); - ASSERT_TRUE(Attach(pid)); + ASSERT_TRUE(TestAttach(pid)); MemoryRemote remote(pid); @@ -149,10 +132,10 @@ TEST_F(MemoryRemoteTest, read_partial) { ASSERT_EQ(0x4cU, dst[i]) << "Failed at byte " << i; } - ASSERT_TRUE(Detach(pid)); + ASSERT_TRUE(TestDetach(pid)); } -TEST_F(MemoryRemoteTest, read_fail) { +TEST(MemoryRemoteTest, read_fail) { int pagesize = getpagesize(); void* src = mmap(nullptr, pagesize * 2, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE,-1, 0); memset(src, 0x4c, pagesize * 2); @@ -169,7 +152,7 @@ TEST_F(MemoryRemoteTest, read_fail) { ASSERT_LT(0, pid); TestScopedPidReaper reap(pid); - ASSERT_TRUE(Attach(pid)); + ASSERT_TRUE(TestAttach(pid)); MemoryRemote remote(pid); @@ -188,10 +171,10 @@ TEST_F(MemoryRemoteTest, read_fail) { ASSERT_EQ(0, munmap(src, pagesize)); - ASSERT_TRUE(Detach(pid)); + ASSERT_TRUE(TestDetach(pid)); } -TEST_F(MemoryRemoteTest, read_overflow) { +TEST(MemoryRemoteTest, read_overflow) { pid_t pid; if ((pid = fork()) == 0) { while (true) @@ -201,7 +184,7 @@ TEST_F(MemoryRemoteTest, read_overflow) { ASSERT_LT(0, pid); TestScopedPidReaper reap(pid); - ASSERT_TRUE(Attach(pid)); + ASSERT_TRUE(TestAttach(pid)); MemoryRemote remote(pid); @@ -209,10 +192,10 @@ TEST_F(MemoryRemoteTest, read_overflow) { std::vector<uint8_t> dst(200); ASSERT_FALSE(remote.ReadFully(UINT64_MAX - 100, dst.data(), 200)); - ASSERT_TRUE(Detach(pid)); + ASSERT_TRUE(TestDetach(pid)); } -TEST_F(MemoryRemoteTest, read_illegal) { +TEST(MemoryRemoteTest, read_illegal) { pid_t pid; if ((pid = fork()) == 0) { while (true); @@ -221,7 +204,7 @@ TEST_F(MemoryRemoteTest, read_illegal) { ASSERT_LT(0, pid); TestScopedPidReaper reap(pid); - ASSERT_TRUE(Attach(pid)); + ASSERT_TRUE(TestAttach(pid)); MemoryRemote remote(pid); @@ -229,10 +212,10 @@ TEST_F(MemoryRemoteTest, read_illegal) { ASSERT_FALSE(remote.ReadFully(0, dst.data(), 1)); ASSERT_FALSE(remote.ReadFully(0, dst.data(), 100)); - ASSERT_TRUE(Detach(pid)); + ASSERT_TRUE(TestDetach(pid)); } -TEST_F(MemoryRemoteTest, read_mprotect_hole) { +TEST(MemoryRemoteTest, read_mprotect_hole) { size_t page_size = getpagesize(); void* mapping = mmap(nullptr, 3 * getpagesize(), PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); @@ -250,7 +233,7 @@ TEST_F(MemoryRemoteTest, read_mprotect_hole) { ASSERT_EQ(0, munmap(mapping, 3 * page_size)); - ASSERT_TRUE(Attach(pid)); + ASSERT_TRUE(TestAttach(pid)); MemoryRemote remote(pid); std::vector<uint8_t> dst(getpagesize() * 4, 0xCC); @@ -263,9 +246,11 @@ TEST_F(MemoryRemoteTest, read_mprotect_hole) { for (size_t i = read_size; i < dst.size(); ++i) { ASSERT_EQ(0xCC, dst[i]); } + + ASSERT_TRUE(TestDetach(pid)); } -TEST_F(MemoryRemoteTest, read_munmap_hole) { +TEST(MemoryRemoteTest, read_munmap_hole) { size_t page_size = getpagesize(); void* mapping = mmap(nullptr, 3 * getpagesize(), PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); @@ -285,7 +270,7 @@ TEST_F(MemoryRemoteTest, read_munmap_hole) { ASSERT_EQ(0, munmap(mapping, page_size)); ASSERT_EQ(0, munmap(static_cast<char*>(mapping) + 2 * page_size, page_size)); - ASSERT_TRUE(Attach(pid)); + ASSERT_TRUE(TestAttach(pid)); MemoryRemote remote(pid); std::vector<uint8_t> dst(getpagesize() * 4, 0xCC); @@ -297,11 +282,13 @@ TEST_F(MemoryRemoteTest, read_munmap_hole) { for (size_t i = read_size; i < dst.size(); ++i) { ASSERT_EQ(0xCC, dst[i]); } + + ASSERT_TRUE(TestDetach(pid)); } // Verify that the memory remote object chooses a memory read function // properly. Either process_vm_readv or ptrace. -TEST_F(MemoryRemoteTest, read_choose_correctly) { +TEST(MemoryRemoteTest, read_choose_correctly) { size_t page_size = getpagesize(); void* mapping = mmap(nullptr, 2 * getpagesize(), PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); @@ -320,7 +307,7 @@ TEST_F(MemoryRemoteTest, read_choose_correctly) { ASSERT_EQ(0, munmap(mapping, 2 * page_size)); - ASSERT_TRUE(Attach(pid)); + ASSERT_TRUE(TestAttach(pid)); // We know that process_vm_readv of a mprotect'd PROT_NONE region will fail. // Read from the PROT_NONE area first to force the choice of ptrace. @@ -348,6 +335,8 @@ TEST_F(MemoryRemoteTest, read_choose_correctly) { bytes = remote_readv.Read(reinterpret_cast<uint64_t>(mapping) + page_size, &value, sizeof(value)); ASSERT_EQ(sizeof(value), bytes); ASSERT_EQ(0xfcfcfcfcU, value); + + ASSERT_TRUE(TestDetach(pid)); } } // namespace unwindstack diff --git a/libunwindstack/tests/MemoryTest.cpp b/libunwindstack/tests/MemoryTest.cpp index 365598499..8a8eb24f5 100644 --- a/libunwindstack/tests/MemoryTest.cpp +++ b/libunwindstack/tests/MemoryTest.cpp @@ -59,10 +59,10 @@ TEST(MemoryTest, read_string) { memory.SetMemory(100, name.c_str(), name.size() + 1); std::string dst_name; - ASSERT_TRUE(memory.ReadString(100, &dst_name)); + ASSERT_TRUE(memory.ReadString(100, &dst_name, 100)); ASSERT_EQ("string_in_memory", dst_name); - ASSERT_TRUE(memory.ReadString(107, &dst_name)); + ASSERT_TRUE(memory.ReadString(107, &dst_name, 100)); ASSERT_EQ("in_memory", dst_name); // Set size greater than string. @@ -82,15 +82,56 @@ TEST(MemoryTest, read_string_error) { std::string dst_name; // Read from a non-existant address. - ASSERT_FALSE(memory.ReadString(100, &dst_name)); + ASSERT_FALSE(memory.ReadString(100, &dst_name, 100)); // This should fail because there is no terminating '\0'. - ASSERT_FALSE(memory.ReadString(0, &dst_name)); + ASSERT_FALSE(memory.ReadString(0, &dst_name, 100)); // This should pass because there is a terminating '\0'. memory.SetData8(name.size(), '\0'); - ASSERT_TRUE(memory.ReadString(0, &dst_name)); + ASSERT_TRUE(memory.ReadString(0, &dst_name, 100)); ASSERT_EQ("short", dst_name); } +TEST(MemoryTest, read_string_long) { + // This string should be greater than 768 characters long (greater than 3 times + // the buffer in the ReadString function) to read multiple blocks. + static constexpr char kLongString[] = + "one two three four five six seven eight nine ten eleven twelve thirteen fourteen fifteen " + "sixteen seventeen eightteen nineteen twenty twenty-one twenty-two twenty-three twenty-four " + "twenty-five twenty-six twenty-seven twenty-eight twenty-nine thirty thirty-one thirty-two " + "thirty-three thirty-four thirty-five thirty-six thirty-seven thirty-eight thirty-nine forty " + "forty-one forty-two forty-three forty-four forty-five forty-size forty-seven forty-eight " + "forty-nine fifty fifty-one fifty-two fifty-three fifty-four fifty-five fifty-six " + "fifty-seven fifty-eight fifty-nine sixty sixty-one sixty-two sixty-three sixty-four " + "sixty-five sixty-six sixty-seven sixty-eight sixty-nine seventy seventy-one seventy-two " + "seventy-three seventy-four seventy-five seventy-six seventy-seven seventy-eight " + "seventy-nine eighty"; + + MemoryFake memory; + + memory.SetMemory(100, kLongString, sizeof(kLongString)); + + std::string dst_name; + ASSERT_TRUE(memory.ReadString(100, &dst_name, sizeof(kLongString))); + ASSERT_EQ(kLongString, dst_name); + + std::string expected_str(kLongString, 255); + memory.SetMemory(100, expected_str.data(), expected_str.length() + 1); + ASSERT_TRUE(memory.ReadString(100, &dst_name, 256)); + ASSERT_EQ(expected_str, dst_name); + ASSERT_FALSE(memory.ReadString(100, &dst_name, 255)); + + expected_str = std::string(kLongString, 256); + memory.SetMemory(100, expected_str.data(), expected_str.length() + 1); + ASSERT_TRUE(memory.ReadString(100, &dst_name, 257)); + ASSERT_EQ(expected_str, dst_name); + ASSERT_FALSE(memory.ReadString(100, &dst_name, 256)); + + expected_str = std::string(kLongString, 299); + memory.SetMemory(100, expected_str.data(), expected_str.length() + 1); + ASSERT_TRUE(memory.ReadString(100, &dst_name, 300)); + ASSERT_EQ(expected_str, dst_name); +} + } // namespace unwindstack diff --git a/libunwindstack/tests/RegsFake.h b/libunwindstack/tests/RegsFake.h index 207d46efe..75fc9d02d 100644 --- a/libunwindstack/tests/RegsFake.h +++ b/libunwindstack/tests/RegsFake.h @@ -54,8 +54,6 @@ class RegsFake : public Regs { return fake_arch_ == ARCH_ARM || fake_arch_ == ARCH_X86 || fake_arch_ == ARCH_MIPS; } - uint64_t GetPcAdjustment(uint64_t, Elf*) override { return 2; } - bool StepIfSignalHandler(uint64_t, Elf*, Memory*) override { return false; } void FakeSetArch(ArchEnum arch) { fake_arch_ = arch; } @@ -86,7 +84,6 @@ class RegsImplFake : public RegsImpl<TypeParam> { void set_pc(uint64_t pc) override { fake_pc_ = pc; } void set_sp(uint64_t sp) override { fake_sp_ = sp; } - uint64_t GetPcAdjustment(uint64_t, Elf*) override { return 0; } bool SetPcFromReturnAddress(Memory*) override { return false; } bool StepIfSignalHandler(uint64_t, Elf*, Memory*) override { return false; } diff --git a/libunwindstack/tests/RegsTest.cpp b/libunwindstack/tests/RegsTest.cpp index 0a33e2ff4..e4fc6f070 100644 --- a/libunwindstack/tests/RegsTest.cpp +++ b/libunwindstack/tests/RegsTest.cpp @@ -93,123 +93,104 @@ TEST_F(RegsTest, regs64) { } TEST_F(RegsTest, rel_pc) { - RegsArm64 arm64; - EXPECT_EQ(4U, arm64.GetPcAdjustment(0x10, elf_.get())); - EXPECT_EQ(4U, arm64.GetPcAdjustment(0x4, elf_.get())); - EXPECT_EQ(0U, arm64.GetPcAdjustment(0x3, elf_.get())); - EXPECT_EQ(0U, arm64.GetPcAdjustment(0x2, elf_.get())); - EXPECT_EQ(0U, arm64.GetPcAdjustment(0x1, elf_.get())); - EXPECT_EQ(0U, arm64.GetPcAdjustment(0x0, elf_.get())); - - RegsX86 x86; - EXPECT_EQ(1U, x86.GetPcAdjustment(0x100, elf_.get())); - EXPECT_EQ(1U, x86.GetPcAdjustment(0x2, elf_.get())); - EXPECT_EQ(1U, x86.GetPcAdjustment(0x1, elf_.get())); - EXPECT_EQ(0U, x86.GetPcAdjustment(0x0, elf_.get())); - - RegsX86_64 x86_64; - EXPECT_EQ(1U, x86_64.GetPcAdjustment(0x100, elf_.get())); - EXPECT_EQ(1U, x86_64.GetPcAdjustment(0x2, elf_.get())); - EXPECT_EQ(1U, x86_64.GetPcAdjustment(0x1, elf_.get())); - EXPECT_EQ(0U, x86_64.GetPcAdjustment(0x0, elf_.get())); - - RegsMips mips; - EXPECT_EQ(8U, mips.GetPcAdjustment(0x10, elf_.get())); - EXPECT_EQ(8U, mips.GetPcAdjustment(0x8, elf_.get())); - EXPECT_EQ(0U, mips.GetPcAdjustment(0x7, elf_.get())); - EXPECT_EQ(0U, mips.GetPcAdjustment(0x6, elf_.get())); - EXPECT_EQ(0U, mips.GetPcAdjustment(0x5, elf_.get())); - EXPECT_EQ(0U, mips.GetPcAdjustment(0x4, elf_.get())); - EXPECT_EQ(0U, mips.GetPcAdjustment(0x3, elf_.get())); - EXPECT_EQ(0U, mips.GetPcAdjustment(0x2, elf_.get())); - EXPECT_EQ(0U, mips.GetPcAdjustment(0x1, elf_.get())); - EXPECT_EQ(0U, mips.GetPcAdjustment(0x0, elf_.get())); - - RegsMips64 mips64; - EXPECT_EQ(8U, mips64.GetPcAdjustment(0x10, elf_.get())); - EXPECT_EQ(8U, mips64.GetPcAdjustment(0x8, elf_.get())); - EXPECT_EQ(0U, mips64.GetPcAdjustment(0x7, elf_.get())); - EXPECT_EQ(0U, mips64.GetPcAdjustment(0x6, elf_.get())); - EXPECT_EQ(0U, mips64.GetPcAdjustment(0x5, elf_.get())); - EXPECT_EQ(0U, mips64.GetPcAdjustment(0x4, elf_.get())); - EXPECT_EQ(0U, mips64.GetPcAdjustment(0x3, elf_.get())); - EXPECT_EQ(0U, mips64.GetPcAdjustment(0x2, elf_.get())); - EXPECT_EQ(0U, mips64.GetPcAdjustment(0x1, elf_.get())); - EXPECT_EQ(0U, mips64.GetPcAdjustment(0x0, elf_.get())); + EXPECT_EQ(4U, GetPcAdjustment(0x10, elf_.get(), ARCH_ARM64)); + EXPECT_EQ(4U, GetPcAdjustment(0x4, elf_.get(), ARCH_ARM64)); + EXPECT_EQ(0U, GetPcAdjustment(0x3, elf_.get(), ARCH_ARM64)); + EXPECT_EQ(0U, GetPcAdjustment(0x2, elf_.get(), ARCH_ARM64)); + EXPECT_EQ(0U, GetPcAdjustment(0x1, elf_.get(), ARCH_ARM64)); + EXPECT_EQ(0U, GetPcAdjustment(0x0, elf_.get(), ARCH_ARM64)); + + EXPECT_EQ(1U, GetPcAdjustment(0x100, elf_.get(), ARCH_X86)); + EXPECT_EQ(1U, GetPcAdjustment(0x2, elf_.get(), ARCH_X86)); + EXPECT_EQ(1U, GetPcAdjustment(0x1, elf_.get(), ARCH_X86)); + EXPECT_EQ(0U, GetPcAdjustment(0x0, elf_.get(), ARCH_X86)); + + EXPECT_EQ(1U, GetPcAdjustment(0x100, elf_.get(), ARCH_X86_64)); + EXPECT_EQ(1U, GetPcAdjustment(0x2, elf_.get(), ARCH_X86_64)); + EXPECT_EQ(1U, GetPcAdjustment(0x1, elf_.get(), ARCH_X86_64)); + EXPECT_EQ(0U, GetPcAdjustment(0x0, elf_.get(), ARCH_X86_64)); + + EXPECT_EQ(8U, GetPcAdjustment(0x10, elf_.get(), ARCH_MIPS)); + EXPECT_EQ(8U, GetPcAdjustment(0x8, elf_.get(), ARCH_MIPS)); + EXPECT_EQ(0U, GetPcAdjustment(0x7, elf_.get(), ARCH_MIPS)); + EXPECT_EQ(0U, GetPcAdjustment(0x6, elf_.get(), ARCH_MIPS)); + EXPECT_EQ(0U, GetPcAdjustment(0x5, elf_.get(), ARCH_MIPS)); + EXPECT_EQ(0U, GetPcAdjustment(0x4, elf_.get(), ARCH_MIPS)); + EXPECT_EQ(0U, GetPcAdjustment(0x3, elf_.get(), ARCH_MIPS)); + EXPECT_EQ(0U, GetPcAdjustment(0x2, elf_.get(), ARCH_MIPS)); + EXPECT_EQ(0U, GetPcAdjustment(0x1, elf_.get(), ARCH_MIPS)); + EXPECT_EQ(0U, GetPcAdjustment(0x0, elf_.get(), ARCH_MIPS)); + + EXPECT_EQ(8U, GetPcAdjustment(0x10, elf_.get(), ARCH_MIPS64)); + EXPECT_EQ(8U, GetPcAdjustment(0x8, elf_.get(), ARCH_MIPS64)); + EXPECT_EQ(0U, GetPcAdjustment(0x7, elf_.get(), ARCH_MIPS64)); + EXPECT_EQ(0U, GetPcAdjustment(0x6, elf_.get(), ARCH_MIPS64)); + EXPECT_EQ(0U, GetPcAdjustment(0x5, elf_.get(), ARCH_MIPS64)); + EXPECT_EQ(0U, GetPcAdjustment(0x4, elf_.get(), ARCH_MIPS64)); + EXPECT_EQ(0U, GetPcAdjustment(0x3, elf_.get(), ARCH_MIPS64)); + EXPECT_EQ(0U, GetPcAdjustment(0x2, elf_.get(), ARCH_MIPS64)); + EXPECT_EQ(0U, GetPcAdjustment(0x1, elf_.get(), ARCH_MIPS64)); + EXPECT_EQ(0U, GetPcAdjustment(0x0, elf_.get(), ARCH_MIPS64)); } TEST_F(RegsTest, rel_pc_arm) { - RegsArm arm; - // Check fence posts. elf_->FakeSetLoadBias(0); - EXPECT_EQ(2U, arm.GetPcAdjustment(0x5, elf_.get())); - EXPECT_EQ(2U, arm.GetPcAdjustment(0x4, elf_.get())); - EXPECT_EQ(2U, arm.GetPcAdjustment(0x3, elf_.get())); - EXPECT_EQ(2U, arm.GetPcAdjustment(0x2, elf_.get())); - EXPECT_EQ(0U, arm.GetPcAdjustment(0x1, elf_.get())); - EXPECT_EQ(0U, arm.GetPcAdjustment(0x0, elf_.get())); + EXPECT_EQ(2U, GetPcAdjustment(0x5, elf_.get(), ARCH_ARM)); + EXPECT_EQ(2U, GetPcAdjustment(0x4, elf_.get(), ARCH_ARM)); + EXPECT_EQ(2U, GetPcAdjustment(0x3, elf_.get(), ARCH_ARM)); + EXPECT_EQ(2U, GetPcAdjustment(0x2, elf_.get(), ARCH_ARM)); + EXPECT_EQ(0U, GetPcAdjustment(0x1, elf_.get(), ARCH_ARM)); + EXPECT_EQ(0U, GetPcAdjustment(0x0, elf_.get(), ARCH_ARM)); elf_->FakeSetLoadBias(0x100); - EXPECT_EQ(0U, arm.GetPcAdjustment(0x1, elf_.get())); - EXPECT_EQ(2U, arm.GetPcAdjustment(0x2, elf_.get())); - EXPECT_EQ(2U, arm.GetPcAdjustment(0xff, elf_.get())); - EXPECT_EQ(2U, arm.GetPcAdjustment(0x105, elf_.get())); - EXPECT_EQ(2U, arm.GetPcAdjustment(0x104, elf_.get())); - EXPECT_EQ(2U, arm.GetPcAdjustment(0x103, elf_.get())); - EXPECT_EQ(2U, arm.GetPcAdjustment(0x102, elf_.get())); - EXPECT_EQ(0U, arm.GetPcAdjustment(0x101, elf_.get())); - EXPECT_EQ(0U, arm.GetPcAdjustment(0x100, elf_.get())); + EXPECT_EQ(0U, GetPcAdjustment(0x1, elf_.get(), ARCH_ARM)); + EXPECT_EQ(2U, GetPcAdjustment(0x2, elf_.get(), ARCH_ARM)); + EXPECT_EQ(2U, GetPcAdjustment(0xff, elf_.get(), ARCH_ARM)); + EXPECT_EQ(2U, GetPcAdjustment(0x105, elf_.get(), ARCH_ARM)); + EXPECT_EQ(2U, GetPcAdjustment(0x104, elf_.get(), ARCH_ARM)); + EXPECT_EQ(2U, GetPcAdjustment(0x103, elf_.get(), ARCH_ARM)); + EXPECT_EQ(2U, GetPcAdjustment(0x102, elf_.get(), ARCH_ARM)); + EXPECT_EQ(0U, GetPcAdjustment(0x101, elf_.get(), ARCH_ARM)); + EXPECT_EQ(0U, GetPcAdjustment(0x100, elf_.get(), ARCH_ARM)); // Check thumb instructions handling. elf_->FakeSetLoadBias(0); memory_->SetData32(0x2000, 0); - EXPECT_EQ(2U, arm.GetPcAdjustment(0x2005, elf_.get())); + EXPECT_EQ(2U, GetPcAdjustment(0x2005, elf_.get(), ARCH_ARM)); memory_->SetData32(0x2000, 0xe000f000); - EXPECT_EQ(4U, arm.GetPcAdjustment(0x2005, elf_.get())); + EXPECT_EQ(4U, GetPcAdjustment(0x2005, elf_.get(), ARCH_ARM)); elf_->FakeSetLoadBias(0x400); memory_->SetData32(0x2100, 0); - EXPECT_EQ(2U, arm.GetPcAdjustment(0x2505, elf_.get())); + EXPECT_EQ(2U, GetPcAdjustment(0x2505, elf_.get(), ARCH_ARM)); memory_->SetData32(0x2100, 0xf111f111); - EXPECT_EQ(4U, arm.GetPcAdjustment(0x2505, elf_.get())); + EXPECT_EQ(4U, GetPcAdjustment(0x2505, elf_.get(), ARCH_ARM)); } TEST_F(RegsTest, elf_invalid) { - RegsArm regs_arm; - RegsArm64 regs_arm64; - RegsX86 regs_x86; - RegsX86_64 regs_x86_64; - RegsMips regs_mips; - RegsMips64 regs_mips64; MapInfo map_info(nullptr, nullptr, 0x1000, 0x2000, 0, 0, ""); Elf* invalid_elf = new Elf(nullptr); map_info.elf.reset(invalid_elf); - regs_arm.set_pc(0x1500); - EXPECT_EQ(0x500U, invalid_elf->GetRelPc(regs_arm.pc(), &map_info)); - EXPECT_EQ(2U, regs_arm.GetPcAdjustment(0x500U, invalid_elf)); - EXPECT_EQ(2U, regs_arm.GetPcAdjustment(0x511U, invalid_elf)); + EXPECT_EQ(0x500U, invalid_elf->GetRelPc(0x1500, &map_info)); + EXPECT_EQ(2U, GetPcAdjustment(0x500U, invalid_elf, ARCH_ARM)); + EXPECT_EQ(2U, GetPcAdjustment(0x511U, invalid_elf, ARCH_ARM)); - regs_arm64.set_pc(0x1600); - EXPECT_EQ(0x600U, invalid_elf->GetRelPc(regs_arm64.pc(), &map_info)); - EXPECT_EQ(4U, regs_arm64.GetPcAdjustment(0x600U, invalid_elf)); + EXPECT_EQ(0x600U, invalid_elf->GetRelPc(0x1600, &map_info)); + EXPECT_EQ(4U, GetPcAdjustment(0x600U, invalid_elf, ARCH_ARM64)); - regs_x86.set_pc(0x1700); - EXPECT_EQ(0x700U, invalid_elf->GetRelPc(regs_x86.pc(), &map_info)); - EXPECT_EQ(1U, regs_x86.GetPcAdjustment(0x700U, invalid_elf)); + EXPECT_EQ(0x700U, invalid_elf->GetRelPc(0x1700, &map_info)); + EXPECT_EQ(1U, GetPcAdjustment(0x700U, invalid_elf, ARCH_X86)); - regs_x86_64.set_pc(0x1800); - EXPECT_EQ(0x800U, invalid_elf->GetRelPc(regs_x86_64.pc(), &map_info)); - EXPECT_EQ(1U, regs_x86_64.GetPcAdjustment(0x800U, invalid_elf)); + EXPECT_EQ(0x800U, invalid_elf->GetRelPc(0x1800, &map_info)); + EXPECT_EQ(1U, GetPcAdjustment(0x800U, invalid_elf, ARCH_X86_64)); - regs_mips.set_pc(0x1900); - EXPECT_EQ(0x900U, invalid_elf->GetRelPc(regs_mips.pc(), &map_info)); - EXPECT_EQ(8U, regs_mips.GetPcAdjustment(0x900U, invalid_elf)); + EXPECT_EQ(0x900U, invalid_elf->GetRelPc(0x1900, &map_info)); + EXPECT_EQ(8U, GetPcAdjustment(0x900U, invalid_elf, ARCH_MIPS)); - regs_mips64.set_pc(0x1a00); - EXPECT_EQ(0xa00U, invalid_elf->GetRelPc(regs_mips64.pc(), &map_info)); - EXPECT_EQ(8U, regs_mips64.GetPcAdjustment(0xa00U, invalid_elf)); + EXPECT_EQ(0xa00U, invalid_elf->GetRelPc(0x1a00, &map_info)); + EXPECT_EQ(8U, GetPcAdjustment(0xa00U, invalid_elf, ARCH_MIPS64)); } TEST_F(RegsTest, arm_verify_sp_pc) { diff --git a/libunwindstack/tests/SymbolsTest.cpp b/libunwindstack/tests/SymbolsTest.cpp index c58aeff71..9afbeec55 100644 --- a/libunwindstack/tests/SymbolsTest.cpp +++ b/libunwindstack/tests/SymbolsTest.cpp @@ -185,18 +185,21 @@ TYPED_TEST_P(SymbolsTest, multiple_entries_nonstandard_size) { std::string fake_name; this->InitSym(&sym, 0x5000, 0x10, 0x40); + this->memory_.SetMemoryBlock(offset, entry_size, 0); this->memory_.SetMemory(offset, &sym, sizeof(sym)); fake_name = "function_one"; this->memory_.SetMemory(0x2040, fake_name.c_str(), fake_name.size() + 1); offset += entry_size; this->InitSym(&sym, 0x3004, 0x200, 0x100); + this->memory_.SetMemoryBlock(offset, entry_size, 0); this->memory_.SetMemory(offset, &sym, sizeof(sym)); fake_name = "function_two"; this->memory_.SetMemory(0x2100, fake_name.c_str(), fake_name.size() + 1); offset += entry_size; this->InitSym(&sym, 0xa010, 0x20, 0x230); + this->memory_.SetMemoryBlock(offset, entry_size, 0); this->memory_.SetMemory(offset, &sym, sizeof(sym)); fake_name = "function_three"; this->memory_.SetMemory(0x2230, fake_name.c_str(), fake_name.size() + 1); @@ -274,7 +277,9 @@ TYPED_TEST_P(SymbolsTest, symtab_read_cached) { // Do call that should cache all of the entries (except the string data). std::string name; uint64_t func_offset; - ASSERT_FALSE(symbols.GetName<TypeParam>(0x6000, &this->memory_, &name, &func_offset)); + ASSERT_FALSE(symbols.GetName<TypeParam>(0x5000, &this->memory_, &name, &func_offset)); + ASSERT_FALSE(symbols.GetName<TypeParam>(0x2000, &this->memory_, &name, &func_offset)); + ASSERT_FALSE(symbols.GetName<TypeParam>(0x1000, &this->memory_, &name, &func_offset)); this->memory_.Clear(); ASSERT_FALSE(symbols.GetName<TypeParam>(0x6000, &this->memory_, &name, &func_offset)); diff --git a/libunwindstack/tests/TestUtils.h b/libunwindstack/tests/TestUtils.h index a4d7b9b33..0685006bf 100644 --- a/libunwindstack/tests/TestUtils.h +++ b/libunwindstack/tests/TestUtils.h @@ -21,6 +21,7 @@ #include <sys/ptrace.h> #include <sys/types.h> #include <sys/wait.h> +#include <unistd.h> namespace unwindstack { @@ -50,6 +51,18 @@ inline bool TestQuiescePid(pid_t pid) { return ready; } +inline bool TestAttach(pid_t pid) { + if (ptrace(PTRACE_ATTACH, pid, 0, 0) == -1) { + return false; + } + + return TestQuiescePid(pid); +} + +inline bool TestDetach(pid_t pid) { + return ptrace(PTRACE_DETACH, pid, 0, 0) == 0; +} + void TestCheckForLeaks(void (*unwind_func)(void*), void* data); } // namespace unwindstack diff --git a/libunwindstack/tests/UnwinderTest.cpp b/libunwindstack/tests/UnwinderTest.cpp index ef1950ccc..dd33aa9d7 100644 --- a/libunwindstack/tests/UnwinderTest.cpp +++ b/libunwindstack/tests/UnwinderTest.cpp @@ -161,8 +161,8 @@ TEST_F(UnwinderTest, multiple_frames) { regs_.set_pc(0x1000); regs_.set_sp(0x10000); - ElfInterfaceFake::FakePushStepData(StepData(0x1102, 0x10010, false)); - ElfInterfaceFake::FakePushStepData(StepData(0x1202, 0x10020, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x1104, 0x10010, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x1204, 0x10020, false)); ElfInterfaceFake::FakePushStepData(StepData(0, 0, true)); Unwinder unwinder(64, maps_.get(), ®s_, process_memory_); @@ -225,8 +225,8 @@ TEST_F(UnwinderTest, multiple_frames_dont_resolve_names) { regs_.set_pc(0x1000); regs_.set_sp(0x10000); - ElfInterfaceFake::FakePushStepData(StepData(0x1102, 0x10010, false)); - ElfInterfaceFake::FakePushStepData(StepData(0x1202, 0x10020, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x1104, 0x10010, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x1204, 0x10020, false)); ElfInterfaceFake::FakePushStepData(StepData(0, 0, true)); Unwinder unwinder(64, maps_.get(), ®s_, process_memory_); @@ -445,7 +445,7 @@ TEST_F(UnwinderTest, no_frames_after_finished) { TEST_F(UnwinderTest, max_frames) { for (size_t i = 0; i < 30; i++) { ElfInterfaceFake::FakePushFunctionData(FunctionData("Frame" + std::to_string(i), i)); - ElfInterfaceFake::FakePushStepData(StepData(0x1102 + i * 0x100, 0x10010 + i * 0x10, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x1104 + i * 0x100, 0x10010 + i * 0x10, false)); } regs_.set_pc(0x1000); @@ -484,12 +484,12 @@ TEST_F(UnwinderTest, verify_frames_skipped) { regs_.set_pc(0x20000); regs_.set_sp(0x10000); - ElfInterfaceFake::FakePushStepData(StepData(0x23002, 0x10010, false)); - ElfInterfaceFake::FakePushStepData(StepData(0x23102, 0x10020, false)); - ElfInterfaceFake::FakePushStepData(StepData(0x20002, 0x10030, false)); - ElfInterfaceFake::FakePushStepData(StepData(0x21002, 0x10040, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x23004, 0x10010, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x23104, 0x10020, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x20004, 0x10030, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x21004, 0x10040, false)); ElfInterfaceFake::FakePushStepData(StepData(0x1002, 0x10050, false)); - ElfInterfaceFake::FakePushStepData(StepData(0x21002, 0x10060, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x21004, 0x10060, false)); ElfInterfaceFake::FakePushStepData(StepData(0x23002, 0x10070, false)); ElfInterfaceFake::FakePushStepData(StepData(0, 0, true)); @@ -553,7 +553,7 @@ TEST_F(UnwinderTest, sp_not_in_map) { regs_.set_pc(0x1000); regs_.set_sp(0x63000); - ElfInterfaceFake::FakePushStepData(StepData(0x21002, 0x50020, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x21004, 0x50020, false)); ElfInterfaceFake::FakePushStepData(StepData(0, 0, true)); Unwinder unwinder(64, maps_.get(), ®s_, process_memory_); @@ -670,10 +670,10 @@ TEST_F(UnwinderTest, speculative_frame) { // Fake as if code called a nullptr function. regs_.set_pc(0); regs_.set_sp(0x10000); - regs_.FakeSetReturnAddress(0x1202); + regs_.FakeSetReturnAddress(0x1204); regs_.FakeSetReturnAddressValid(true); - ElfInterfaceFake::FakePushStepData(StepData(0x23102, 0x10020, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x23104, 0x10020, false)); ElfInterfaceFake::FakePushStepData(StepData(0, 0, true)); Unwinder unwinder(64, maps_.get(), ®s_, process_memory_); @@ -789,7 +789,7 @@ TEST_F(UnwinderTest, speculative_frame_not_removed_pc_bad) { // Fake as if code called a nullptr function. regs_.set_pc(0); regs_.set_sp(0x10000); - regs_.FakeSetReturnAddress(0x1202); + regs_.FakeSetReturnAddress(0x1204); regs_.FakeSetReturnAddressValid(true); Unwinder unwinder(64, maps_.get(), ®s_, process_memory_); @@ -858,8 +858,8 @@ TEST_F(UnwinderTest, map_ignore_suffixes) { // Fake as if code called a nullptr function. regs_.set_pc(0x1000); regs_.set_sp(0x10000); - ElfInterfaceFake::FakePushStepData(StepData(0x43402, 0x10010, false)); - ElfInterfaceFake::FakePushStepData(StepData(0x53502, 0x10020, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x43404, 0x10010, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x53504, 0x10020, false)); ElfInterfaceFake::FakePushStepData(StepData(0, 0, true)); Unwinder unwinder(64, maps_.get(), ®s_, process_memory_); @@ -915,11 +915,11 @@ TEST_F(UnwinderTest, sp_pc_do_not_change) { regs_.set_pc(0x1000); regs_.set_sp(0x10000); - ElfInterfaceFake::FakePushStepData(StepData(0x33402, 0x10010, false)); - ElfInterfaceFake::FakePushStepData(StepData(0x33502, 0x10020, false)); - ElfInterfaceFake::FakePushStepData(StepData(0x33502, 0x10020, false)); - ElfInterfaceFake::FakePushStepData(StepData(0x33502, 0x10020, false)); - ElfInterfaceFake::FakePushStepData(StepData(0x33502, 0x10020, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x33404, 0x10010, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x33504, 0x10020, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x33504, 0x10020, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x33504, 0x10020, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x33504, 0x10020, false)); ElfInterfaceFake::FakePushStepData(StepData(0, 0, true)); Unwinder unwinder(64, maps_.get(), ®s_, process_memory_); @@ -1113,7 +1113,7 @@ TEST_F(UnwinderTest, dex_pc_multiple_frames) { regs_.set_pc(0x1000); regs_.set_sp(0x10000); regs_.FakeSetDexPc(0xa3400); - ElfInterfaceFake::FakePushStepData(StepData(0x33402, 0x10010, false)); + ElfInterfaceFake::FakePushStepData(StepData(0x33404, 0x10010, false)); ElfInterfaceFake::FakePushStepData(StepData(0, 0, true)); Unwinder unwinder(64, maps_.get(), ®s_, process_memory_); diff --git a/libusbhost/usbhost.c b/libusbhost/usbhost.c index 415488fc0..3bed0e367 100644 --- a/libusbhost/usbhost.c +++ b/libusbhost/usbhost.c @@ -597,6 +597,11 @@ struct usb_descriptor_header *usb_descriptor_iter_next(struct usb_descriptor_ite if (iter->curr_desc >= iter->config_end) return NULL; next = (struct usb_descriptor_header*)iter->curr_desc; + // Corrupt descriptor with zero length, cannot continue iterating + if (next->bLength == 0) { + D("usb_descriptor_iter_next got zero length USB descriptor, ending iteration\n"); + return NULL; + } iter->curr_desc += next->bLength; return next; } diff --git a/libutils/Android.bp b/libutils/Android.bp index 3a30a9e77..ea39d3449 100644 --- a/libutils/Android.bp +++ b/libutils/Android.bp @@ -79,12 +79,6 @@ cc_defaults { "liblog", ], - arch: { - mips: { - cflags: ["-DALIGN_DOUBLE"], - }, - }, - target: { android: { cflags: ["-fvisibility=protected"], @@ -180,15 +174,9 @@ cc_library { "CallStack.cpp", ], - arch: { - mips: { - cflags: ["-DALIGN_DOUBLE"], - }, - }, - shared_libs: [ - "libutils", - "libbacktrace", + "libutils", + "libbacktrace", ], target: { @@ -206,6 +194,45 @@ cc_library { }, } +cc_defaults { + name: "libutils_fuzz_defaults", + host_supported: true, + shared_libs: [ + "libutils", + "libbase", + ], +} + +cc_fuzz { + name: "libutils_fuzz_bitset", + defaults: ["libutils_fuzz_defaults"], + srcs: ["BitSet_fuzz.cpp"], +} + +cc_fuzz { + name: "libutils_fuzz_filemap", + defaults: ["libutils_fuzz_defaults"], + srcs: ["FileMap_fuzz.cpp"], +} + +cc_fuzz { + name: "libutils_fuzz_string8", + defaults: ["libutils_fuzz_defaults"], + srcs: ["String8_fuzz.cpp"], +} + +cc_fuzz { + name: "libutils_fuzz_string16", + defaults: ["libutils_fuzz_defaults"], + srcs: ["String16_fuzz.cpp"], +} + +cc_fuzz { + name: "libutils_fuzz_vector", + defaults: ["libutils_fuzz_defaults"], + srcs: ["Vector_fuzz.cpp"], +} + cc_test { name: "libutils_test", host_supported: true, @@ -219,6 +246,7 @@ cc_test { "String8_test.cpp", "String16_test.cpp", "StrongPointer_test.cpp", + "Timers_test.cpp", "Unicode_test.cpp", "Vector_test.cpp", ], diff --git a/libutils/BitSet_fuzz.cpp b/libutils/BitSet_fuzz.cpp new file mode 100644 index 000000000..2e6043cf0 --- /dev/null +++ b/libutils/BitSet_fuzz.cpp @@ -0,0 +1,70 @@ +/* + * Copyright 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include <functional> + +#include "fuzzer/FuzzedDataProvider.h" +#include "utils/BitSet.h" +static constexpr uint8_t MAX_OPERATIONS = 50; + +// We need to handle both 32 and 64 bit bitsets, so we use a function template +// here. Sadly, std::function can't be generic, so we generate a vector of +// std::functions using this function. +template <typename T> +std::vector<std::function<void(T, uint32_t)>> getOperationsForType() { + return { + [](T bs, uint32_t val) -> void { bs.markBit(val); }, + [](T bs, uint32_t val) -> void { bs.valueForBit(val); }, + [](T bs, uint32_t val) -> void { bs.hasBit(val); }, + [](T bs, uint32_t val) -> void { bs.clearBit(val); }, + [](T bs, uint32_t val) -> void { bs.getIndexOfBit(val); }, + [](T bs, uint32_t) -> void { bs.clearFirstMarkedBit(); }, + [](T bs, uint32_t) -> void { bs.markFirstUnmarkedBit(); }, + [](T bs, uint32_t) -> void { bs.clearLastMarkedBit(); }, + [](T bs, uint32_t) -> void { bs.clear(); }, + [](T bs, uint32_t) -> void { bs.count(); }, + [](T bs, uint32_t) -> void { bs.isEmpty(); }, + [](T bs, uint32_t) -> void { bs.isFull(); }, + [](T bs, uint32_t) -> void { bs.firstMarkedBit(); }, + [](T bs, uint32_t) -> void { bs.lastMarkedBit(); }, + }; +} + +// Our operations for 32 and 64 bit bitsets +static const std::vector<std::function<void(android::BitSet32, uint32_t)>> thirtyTwoBitOps = + getOperationsForType<android::BitSet32>(); +static const std::vector<std::function<void(android::BitSet64, uint32_t)>> sixtyFourBitOps = + getOperationsForType<android::BitSet64>(); + +void runOperationFor32Bit(android::BitSet32 bs, uint32_t bit, uint8_t operation) { + thirtyTwoBitOps[operation](bs, bit); +} + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + FuzzedDataProvider dataProvider(data, size); + uint32_t thirty_two_base = dataProvider.ConsumeIntegral<uint32_t>(); + uint64_t sixty_four_base = dataProvider.ConsumeIntegral<uint64_t>(); + android::BitSet32 b1 = android::BitSet32(thirty_two_base); + android::BitSet64 b2 = android::BitSet64(sixty_four_base); + + size_t opsRun = 0; + while (dataProvider.remaining_bytes() > 0 && opsRun++ < MAX_OPERATIONS) { + uint32_t bit = dataProvider.ConsumeIntegral<uint32_t>(); + uint8_t op = dataProvider.ConsumeIntegral<uint8_t>(); + thirtyTwoBitOps[op % thirtyTwoBitOps.size()](b1, bit); + sixtyFourBitOps[op % sixtyFourBitOps.size()](b2, bit); + } + return 0; +} diff --git a/libutils/FileMap_fuzz.cpp b/libutils/FileMap_fuzz.cpp new file mode 100644 index 000000000..d800564f1 --- /dev/null +++ b/libutils/FileMap_fuzz.cpp @@ -0,0 +1,50 @@ +/* + * Copyright 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include <iostream> + +#include "android-base/file.h" +#include "fuzzer/FuzzedDataProvider.h" +#include "utils/FileMap.h" + +static constexpr uint16_t MAX_STR_SIZE = 256; +static constexpr uint8_t MAX_FILENAME_SIZE = 32; + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + FuzzedDataProvider dataProvider(data, size); + TemporaryFile tf; + // Generate file contents + std::string contents = dataProvider.ConsumeRandomLengthString(MAX_STR_SIZE); + // If we have string contents, dump them into the file. + // Otherwise, just leave it as an empty file. + if (contents.length() > 0) { + const char* bytes = contents.c_str(); + android::base::WriteStringToFd(bytes, tf.fd); + } + android::FileMap m; + // Generate create() params + std::string orig_name = dataProvider.ConsumeRandomLengthString(MAX_FILENAME_SIZE); + size_t length = dataProvider.ConsumeIntegralInRange<size_t>(1, SIZE_MAX); + off64_t offset = dataProvider.ConsumeIntegralInRange<off64_t>(1, INT64_MAX); + bool read_only = dataProvider.ConsumeBool(); + m.create(orig_name.c_str(), tf.fd, offset, length, read_only); + m.getDataOffset(); + m.getFileName(); + m.getDataLength(); + m.getDataPtr(); + int enum_index = dataProvider.ConsumeIntegral<int>(); + m.advise(static_cast<android::FileMap::MapAdvice>(enum_index)); + return 0; +} diff --git a/libutils/String16.cpp b/libutils/String16.cpp index e2a8c59a0..d514f29fb 100644 --- a/libutils/String16.cpp +++ b/libutils/String16.cpp @@ -90,19 +90,6 @@ String16::String16() { } -String16::String16(StaticLinkage) - : mString(nullptr) -{ - // this constructor is used when we can't rely on the static-initializers - // having run. In this case we always allocate an empty string. It's less - // efficient than using getEmptyString(), but we assume it's uncommon. - - SharedBuffer* buf = static_cast<SharedBuffer*>(alloc(sizeof(char16_t))); - char16_t* data = static_cast<char16_t*>(buf->data()); - data[0] = 0; - mString = data; -} - String16::String16(const String16& o) : mString(o.mString) { diff --git a/libutils/String16_fuzz.cpp b/libutils/String16_fuzz.cpp new file mode 100644 index 000000000..63c280071 --- /dev/null +++ b/libutils/String16_fuzz.cpp @@ -0,0 +1,122 @@ +/* + * Copyright 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include <iostream> + +#include "fuzzer/FuzzedDataProvider.h" +#include "utils/String16.h" +static constexpr int MAX_STRING_BYTES = 256; +static constexpr uint8_t MAX_OPERATIONS = 50; + +std::vector<std::function<void(FuzzedDataProvider&, android::String16, android::String16)>> + operations = { + + // Bytes and size + ([](FuzzedDataProvider&, android::String16 str1, android::String16) -> void { + str1.string(); + }), + ([](FuzzedDataProvider&, android::String16 str1, android::String16) -> void { + str1.isStaticString(); + }), + ([](FuzzedDataProvider&, android::String16 str1, android::String16) -> void { + str1.size(); + }), + + // Casing + ([](FuzzedDataProvider&, android::String16 str1, android::String16) -> void { + str1.makeLower(); + }), + + // Comparison + ([](FuzzedDataProvider&, android::String16 str1, android::String16 str2) -> void { + str1.startsWith(str2); + }), + ([](FuzzedDataProvider&, android::String16 str1, android::String16 str2) -> void { + str1.contains(str2.string()); + }), + ([](FuzzedDataProvider&, android::String16 str1, android::String16 str2) -> void { + str1.compare(str2); + }), + + // Append and format + ([](FuzzedDataProvider&, android::String16 str1, android::String16 str2) -> void { + str1.append(str2); + }), + ([](FuzzedDataProvider& dataProvider, android::String16 str1, + android::String16 str2) -> void { + int pos = dataProvider.ConsumeIntegralInRange<int>(0, str1.size()); + str1.insert(pos, str2.string()); + }), + + // Find and replace operations + ([](FuzzedDataProvider& dataProvider, android::String16 str1, + android::String16) -> void { + char16_t findChar = dataProvider.ConsumeIntegral<char16_t>(); + str1.findFirst(findChar); + }), + ([](FuzzedDataProvider& dataProvider, android::String16 str1, + android::String16) -> void { + char16_t findChar = dataProvider.ConsumeIntegral<char16_t>(); + str1.findLast(findChar); + }), + ([](FuzzedDataProvider& dataProvider, android::String16 str1, + android::String16) -> void { + char16_t findChar = dataProvider.ConsumeIntegral<char16_t>(); + char16_t replaceChar = dataProvider.ConsumeIntegral<char16_t>(); + str1.replaceAll(findChar, replaceChar); + }), + ([](FuzzedDataProvider& dataProvider, android::String16 str1, + android::String16) -> void { + size_t len = dataProvider.ConsumeIntegral<size_t>(); + size_t begin = dataProvider.ConsumeIntegral<size_t>(); + str1.remove(len, begin); + }), +}; + +void callFunc(uint8_t index, FuzzedDataProvider& dataProvider, android::String16 str1, + android::String16 str2) { + operations[index](dataProvider, str1, str2); +} + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + FuzzedDataProvider dataProvider(data, size); + // We're generating two char vectors. + // First, generate lengths. + const size_t kVecOneLen = dataProvider.ConsumeIntegralInRange<size_t>(1, MAX_STRING_BYTES); + const size_t kVecTwoLen = dataProvider.ConsumeIntegralInRange<size_t>(1, MAX_STRING_BYTES); + + // Next, populate the vectors + std::vector<char> vec = dataProvider.ConsumeBytesWithTerminator<char>(kVecOneLen); + std::vector<char> vec_two = dataProvider.ConsumeBytesWithTerminator<char>(kVecTwoLen); + + // Get pointers to their data + char* char_one = vec.data(); + char* char_two = vec_two.data(); + + // Create UTF16 representations + android::String16 str_one_utf16 = android::String16(char_one); + android::String16 str_two_utf16 = android::String16(char_two); + + // Run operations against strings + int opsRun = 0; + while (dataProvider.remaining_bytes() > 0 && opsRun++ < MAX_OPERATIONS) { + uint8_t op = dataProvider.ConsumeIntegralInRange<uint8_t>(0, operations.size() - 1); + callFunc(op, dataProvider, str_one_utf16, str_two_utf16); + } + + str_one_utf16.remove(0, str_one_utf16.size()); + str_two_utf16.remove(0, str_two_utf16.size()); + return 0; +} diff --git a/libutils/String8.cpp b/libutils/String8.cpp index d13548e4c..d00e39c56 100644 --- a/libutils/String8.cpp +++ b/libutils/String8.cpp @@ -125,19 +125,6 @@ String8::String8() { } -String8::String8(StaticLinkage) - : mString(nullptr) -{ - // this constructor is used when we can't rely on the static-initializers - // having run. In this case we always allocate an empty string. It's less - // efficient than using getEmptyString(), but we assume it's uncommon. - - char* data = static_cast<char*>( - SharedBuffer::alloc(sizeof(char))->data()); - data[0] = 0; - mString = data; -} - String8::String8(const String8& o) : mString(o.mString) { diff --git a/libutils/String8_fuzz.cpp b/libutils/String8_fuzz.cpp new file mode 100644 index 000000000..2adfe98b0 --- /dev/null +++ b/libutils/String8_fuzz.cpp @@ -0,0 +1,136 @@ +/* + * Copyright 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include <functional> +#include <iostream> + +#include "fuzzer/FuzzedDataProvider.h" +#include "utils/String8.h" + +static constexpr int MAX_STRING_BYTES = 256; +static constexpr uint8_t MAX_OPERATIONS = 50; + +std::vector<std::function<void(FuzzedDataProvider&, android::String8, android::String8)>> + operations = { + + // Bytes and size + [](FuzzedDataProvider&, android::String8 str1, android::String8) -> void { + str1.bytes(); + }, + [](FuzzedDataProvider&, android::String8 str1, android::String8) -> void { + str1.isEmpty(); + }, + [](FuzzedDataProvider&, android::String8 str1, android::String8) -> void { + str1.length(); + }, + [](FuzzedDataProvider&, android::String8 str1, android::String8) -> void { + str1.size(); + }, + + // Casing + [](FuzzedDataProvider&, android::String8 str1, android::String8) -> void { + str1.toUpper(); + }, + [](FuzzedDataProvider&, android::String8 str1, android::String8) -> void { + str1.toLower(); + }, + + [](FuzzedDataProvider&, android::String8 str1, android::String8 str2) -> void { + str1.removeAll(str2.c_str()); + }, + [](FuzzedDataProvider&, android::String8 str1, android::String8 str2) -> void { + str1.compare(str2); + }, + + // Append and format + [](FuzzedDataProvider&, android::String8 str1, android::String8 str2) -> void { + str1.append(str2); + }, + [](FuzzedDataProvider&, android::String8 str1, android::String8 str2) -> void { + str1.appendFormat(str1.c_str(), str2.c_str()); + }, + [](FuzzedDataProvider&, android::String8 str1, android::String8 str2) -> void { + str1.format(str1.c_str(), str2.c_str()); + }, + + // Find operation + [](FuzzedDataProvider& dataProvider, android::String8 str1, + android::String8) -> void { + // We need to get a value from our fuzzer here. + int start_index = dataProvider.ConsumeIntegralInRange<int>(0, str1.size()); + str1.find(str1.c_str(), start_index); + }, + + // Path handling + [](FuzzedDataProvider&, android::String8 str1, android::String8) -> void { + str1.getBasePath(); + }, + [](FuzzedDataProvider&, android::String8 str1, android::String8) -> void { + str1.getPathExtension(); + }, + [](FuzzedDataProvider&, android::String8 str1, android::String8) -> void { + str1.getPathLeaf(); + }, + [](FuzzedDataProvider&, android::String8 str1, android::String8) -> void { + str1.getPathDir(); + }, + [](FuzzedDataProvider&, android::String8 str1, android::String8) -> void { + str1.convertToResPath(); + }, + [](FuzzedDataProvider&, android::String8 str1, android::String8) -> void { + android::String8 path_out_str = android::String8(); + str1.walkPath(&path_out_str); + path_out_str.clear(); + }, + [](FuzzedDataProvider& dataProvider, android::String8 str1, + android::String8) -> void { + str1.setPathName(dataProvider.ConsumeBytesWithTerminator<char>(5).data()); + }, + [](FuzzedDataProvider& dataProvider, android::String8 str1, + android::String8) -> void { + str1.appendPath(dataProvider.ConsumeBytesWithTerminator<char>(5).data()); + }, +}; + +void callFunc(uint8_t index, FuzzedDataProvider& dataProvider, android::String8 str1, + android::String8 str2) { + operations[index](dataProvider, str1, str2); +} + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + FuzzedDataProvider dataProvider(data, size); + // Generate vector lengths + const size_t kVecOneLen = dataProvider.ConsumeIntegralInRange<size_t>(1, MAX_STRING_BYTES); + const size_t kVecTwoLen = dataProvider.ConsumeIntegralInRange<size_t>(1, MAX_STRING_BYTES); + // Populate vectors + std::vector<char> vec = dataProvider.ConsumeBytesWithTerminator<char>(kVecOneLen); + std::vector<char> vec_two = dataProvider.ConsumeBytesWithTerminator<char>(kVecTwoLen); + // Create UTF-8 pointers + android::String8 str_one_utf8 = android::String8(vec.data()); + android::String8 str_two_utf8 = android::String8(vec_two.data()); + + // Run operations against strings + int opsRun = 0; + while (dataProvider.remaining_bytes() > 0 && opsRun++ < MAX_OPERATIONS) { + uint8_t op = dataProvider.ConsumeIntegralInRange<uint8_t>(0, operations.size() - 1); + callFunc(op, dataProvider, str_one_utf8, str_two_utf8); + } + + // Just to be extra sure these can be freed, we're going to explicitly clear + // them + str_one_utf8.clear(); + str_two_utf8.clear(); + return 0; +} diff --git a/libutils/StrongPointer_test.cpp b/libutils/StrongPointer_test.cpp index 7b2e37f27..d37c1de6f 100644 --- a/libutils/StrongPointer_test.cpp +++ b/libutils/StrongPointer_test.cpp @@ -36,10 +36,8 @@ private: TEST(StrongPointer, move) { bool isDeleted; - SPFoo* foo = new SPFoo(&isDeleted); - ASSERT_EQ(0, foo->getStrongCount()); - ASSERT_FALSE(isDeleted) << "Already deleted...?"; - sp<SPFoo> sp1(foo); + sp<SPFoo> sp1 = sp<SPFoo>::make(&isDeleted); + SPFoo* foo = sp1.get(); ASSERT_EQ(1, foo->getStrongCount()); { sp<SPFoo> sp2 = std::move(sp1); @@ -65,7 +63,7 @@ TEST(StrongPointer, NullptrComparison) { TEST(StrongPointer, PointerComparison) { bool isDeleted; - sp<SPFoo> foo = new SPFoo(&isDeleted); + sp<SPFoo> foo = sp<SPFoo>::make(&isDeleted); ASSERT_EQ(foo.get(), foo); ASSERT_EQ(foo, foo.get()); ASSERT_NE(nullptr, foo); diff --git a/libutils/Threads.cpp b/libutils/Threads.cpp index 540dcf49d..147db542d 100644 --- a/libutils/Threads.cpp +++ b/libutils/Threads.cpp @@ -302,8 +302,7 @@ void androidSetCreateThreadFunc(android_create_thread_fn func) } #if defined(__ANDROID__) -int androidSetThreadPriority(pid_t tid, int pri) -{ +int androidSetThreadPriority(pid_t tid, int pri, bool change_policy) { int rc = 0; int lasterr = 0; int curr_pri = getpriority(PRIO_PROCESS, tid); @@ -312,17 +311,19 @@ int androidSetThreadPriority(pid_t tid, int pri) return rc; } - if (pri >= ANDROID_PRIORITY_BACKGROUND) { - rc = SetTaskProfiles(tid, {"SCHED_SP_BACKGROUND"}, true) ? 0 : -1; - } else if (curr_pri >= ANDROID_PRIORITY_BACKGROUND) { - SchedPolicy policy = SP_FOREGROUND; - // Change to the sched policy group of the process. - get_sched_policy(getpid(), &policy); - rc = SetTaskProfiles(tid, {get_sched_policy_profile_name(policy)}, true) ? 0 : -1; - } + if (change_policy) { + if (pri >= ANDROID_PRIORITY_BACKGROUND) { + rc = SetTaskProfiles(tid, {"SCHED_SP_BACKGROUND"}, true) ? 0 : -1; + } else if (curr_pri >= ANDROID_PRIORITY_BACKGROUND) { + SchedPolicy policy = SP_FOREGROUND; + // Change to the sched policy group of the process. + get_sched_policy(getpid(), &policy); + rc = SetTaskProfiles(tid, {get_sched_policy_profile_name(policy)}, true) ? 0 : -1; + } - if (rc) { - lasterr = errno; + if (rc) { + lasterr = errno; + } } if (setpriority(PRIO_PROCESS, tid, pri) < 0) { diff --git a/libutils/Timers.cpp b/libutils/Timers.cpp index 1172ae784..fd3f4a957 100644 --- a/libutils/Timers.cpp +++ b/libutils/Timers.cpp @@ -20,31 +20,37 @@ #include <utils/Timers.h> #include <limits.h> +#include <stdlib.h> #include <time.h> -// host linux support requires Linux 2.6.39+ +#include <android-base/macros.h> + +static constexpr size_t clock_id_max = 5; + +static void checkClockId(int clock) { + if (clock < 0 || clock >= clock_id_max) abort(); +} + #if defined(__linux__) -nsecs_t systemTime(int clock) -{ - static const clockid_t clocks[] = { - CLOCK_REALTIME, - CLOCK_MONOTONIC, - CLOCK_PROCESS_CPUTIME_ID, - CLOCK_THREAD_CPUTIME_ID, - CLOCK_BOOTTIME - }; - struct timespec t; - t.tv_sec = t.tv_nsec = 0; +nsecs_t systemTime(int clock) { + checkClockId(clock); + static constexpr clockid_t clocks[] = {CLOCK_REALTIME, CLOCK_MONOTONIC, + CLOCK_PROCESS_CPUTIME_ID, CLOCK_THREAD_CPUTIME_ID, + CLOCK_BOOTTIME}; + static_assert(clock_id_max == arraysize(clocks)); + timespec t = {}; clock_gettime(clocks[clock], &t); return nsecs_t(t.tv_sec)*1000000000LL + t.tv_nsec; } #else -nsecs_t systemTime(int /*clock*/) -{ +nsecs_t systemTime(int clock) { + // TODO: is this ever called with anything but REALTIME on mac/windows? + checkClockId(clock); + // Clock support varies widely across hosts. Mac OS doesn't support - // CLOCK_BOOTTIME, and Windows is windows. - struct timeval t; - t.tv_sec = t.tv_usec = 0; + // CLOCK_BOOTTIME (and doesn't even have clock_gettime until 10.12). + // Windows is windows. + timeval t = {}; gettimeofday(&t, nullptr); return nsecs_t(t.tv_sec)*1000000000LL + nsecs_t(t.tv_usec)*1000LL; } diff --git a/base/include/android-base/errno_restorer.h b/libutils/Timers_test.cpp index 1c8597c96..ec0051e67 100644 --- a/base/include/android-base/errno_restorer.h +++ b/libutils/Timers_test.cpp @@ -14,29 +14,16 @@ * limitations under the License. */ -#pragma once - -#include "errno.h" - -#include "android-base/macros.h" - -namespace android { -namespace base { - -class ErrnoRestorer { - public: - ErrnoRestorer() : saved_errno_(errno) {} - - ~ErrnoRestorer() { errno = saved_errno_; } - - // Allow this object to be used as part of && operation. - operator bool() const { return true; } - - private: - const int saved_errno_; - - DISALLOW_COPY_AND_ASSIGN(ErrnoRestorer); -}; - -} // namespace base -} // namespace android +#include <utils/Timers.h> + +#include <gtest/gtest.h> + +TEST(Timers, systemTime_invalid) { + EXPECT_EXIT(systemTime(-1), testing::KilledBySignal(SIGABRT), ""); + systemTime(SYSTEM_TIME_REALTIME); + systemTime(SYSTEM_TIME_MONOTONIC); + systemTime(SYSTEM_TIME_PROCESS); + systemTime(SYSTEM_TIME_THREAD); + systemTime(SYSTEM_TIME_BOOTTIME); + EXPECT_EXIT(systemTime(SYSTEM_TIME_BOOTTIME + 1), testing::KilledBySignal(SIGABRT), ""); +} diff --git a/libutils/Vector_fuzz.cpp b/libutils/Vector_fuzz.cpp new file mode 100644 index 000000000..f6df05135 --- /dev/null +++ b/libutils/Vector_fuzz.cpp @@ -0,0 +1,83 @@ +/* + * Copyright 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "fuzzer/FuzzedDataProvider.h" +#include "utils/Vector.h" +static constexpr uint16_t MAX_VEC_SIZE = 5000; + +void runVectorFuzz(const uint8_t* data, size_t size) { + FuzzedDataProvider dataProvider(data, size); + android::Vector<uint8_t> vec = android::Vector<uint8_t>(); + // We want to test handling of sizeof as well. + android::Vector<uint32_t> vec32 = android::Vector<uint32_t>(); + + // We're going to generate two vectors of this size + size_t vectorSize = dataProvider.ConsumeIntegralInRange<size_t>(0, MAX_VEC_SIZE); + vec.setCapacity(vectorSize); + vec32.setCapacity(vectorSize); + for (size_t i = 0; i < vectorSize; i++) { + uint8_t count = dataProvider.ConsumeIntegralInRange<uint8_t>(1, 5); + vec.insertAt((uint8_t)i, i, count); + vec32.insertAt((uint32_t)i, i, count); + vec.push_front(i); + vec32.push(i); + } + + // Now we'll perform some test operations with any remaining data + // Index to perform operations at + size_t index = dataProvider.ConsumeIntegralInRange<size_t>(0, vec.size()); + std::vector<uint8_t> remainingVec = dataProvider.ConsumeRemainingBytes<uint8_t>(); + // Insert an array and vector + vec.insertArrayAt(remainingVec.data(), index, remainingVec.size()); + android::Vector<uint8_t> vecCopy = android::Vector<uint8_t>(vec); + vec.insertVectorAt(vecCopy, index); + // Same thing for 32 bit vector + android::Vector<uint32_t> vec32Copy = android::Vector<uint32_t>(vec32); + vec32.insertArrayAt(vec32Copy.array(), index, vec32.size()); + vec32.insertVectorAt(vec32Copy, index); + // Replace single character + if (remainingVec.size() > 0) { + vec.replaceAt(remainingVec[0], index); + vec32.replaceAt(static_cast<uint32_t>(remainingVec[0]), index); + } else { + vec.replaceAt(0, index); + vec32.replaceAt(0, index); + } + // Add any remaining bytes + for (uint8_t i : remainingVec) { + vec.add(i); + vec32.add(static_cast<uint32_t>(i)); + } + // Shrink capactiy + vec.setCapacity(remainingVec.size()); + vec32.setCapacity(remainingVec.size()); + // Iterate through each pointer + size_t sum = 0; + for (auto& it : vec) { + sum += it; + } + for (auto& it : vec32) { + sum += it; + } + // Cleanup + vec.clear(); + vecCopy.clear(); + vec32.clear(); + vec32Copy.clear(); +} +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + runVectorFuzz(data, size); + return 0; +} diff --git a/libutils/include/utils/AndroidThreads.h b/libutils/include/utils/AndroidThreads.h index a8d785175..3c30a2a85 100644 --- a/libutils/include/utils/AndroidThreads.h +++ b/libutils/include/utils/AndroidThreads.h @@ -78,7 +78,9 @@ extern void androidSetCreateThreadFunc(android_create_thread_fn func); // should be one of the ANDROID_PRIORITY constants. Returns INVALID_OPERATION // if the priority set failed, else another value if just the group set failed; // in either case errno is set. Thread ID zero means current thread. -extern int androidSetThreadPriority(pid_t tid, int prio); +// Parameter "change_policy" indicates if sched policy should be changed. It needs +// not be checked again if the change is done elsewhere like activity manager. +extern int androidSetThreadPriority(pid_t tid, int prio, bool change_policy = true); // Get the current priority of a particular thread. Returns one of the // ANDROID_PRIORITY constants or a negative result in case of error. diff --git a/libutils/include/utils/Looper.h b/libutils/include/utils/Looper.h index c439c5ce0..466fbb726 100644 --- a/libutils/include/utils/Looper.h +++ b/libutils/include/utils/Looper.h @@ -26,6 +26,8 @@ #include <android-base/unique_fd.h> +#include <utility> + namespace android { /* @@ -438,9 +440,8 @@ private: struct MessageEnvelope { MessageEnvelope() : uptime(0) { } - MessageEnvelope(nsecs_t u, const sp<MessageHandler> h, - const Message& m) : uptime(u), handler(h), message(m) { - } + MessageEnvelope(nsecs_t u, sp<MessageHandler> h, const Message& m) + : uptime(u), handler(std::move(h)), message(m) {} nsecs_t uptime; sp<MessageHandler> handler; diff --git a/libutils/include/utils/RefBase.h b/libutils/include/utils/RefBase.h index 89f048db8..e7acd17d2 100644 --- a/libutils/include/utils/RefBase.h +++ b/libutils/include/utils/RefBase.h @@ -297,6 +297,11 @@ public: } protected: + // When constructing these objects, prefer using sp::make<>. Using a RefBase + // object on the stack or with other refcount mechanisms (e.g. + // std::shared_ptr) is inherently wrong. RefBase types have an implicit + // ownership model and cannot be safely used with other ownership models. + RefBase(); virtual ~RefBase(); diff --git a/libutils/include/utils/String16.h b/libutils/include/utils/String16.h index c0e3f1eba..1a4b47eef 100644 --- a/libutils/include/utils/String16.h +++ b/libutils/include/utils/String16.h @@ -39,17 +39,7 @@ class StaticString16; class String16 { public: - /* - * Use String16(StaticLinkage) if you're statically linking against - * libutils and declaring an empty static String16, e.g.: - * - * static String16 sAStaticEmptyString(String16::kEmptyString); - * static String16 sAnotherStaticEmptyString(sAStaticEmptyString); - */ - enum StaticLinkage { kEmptyString }; - String16(); - explicit String16(StaticLinkage); String16(const String16& o); String16(const String16& o, size_t len, @@ -197,7 +187,7 @@ public: ANDROID_TRIVIAL_MOVE_TRAIT(String16) static inline std::ostream& operator<<(std::ostream& os, const String16& str) { - os << String8(str).c_str(); + os << String8(str); return os; } diff --git a/libutils/include/utils/String8.h b/libutils/include/utils/String8.h index 0ddcbb2be..0bcb716af 100644 --- a/libutils/include/utils/String8.h +++ b/libutils/include/utils/String8.h @@ -17,7 +17,8 @@ #ifndef ANDROID_STRING8_H #define ANDROID_STRING8_H -#include <string> // for std::string +#include <iostream> +#include <string> #include <utils/Errors.h> #include <utils/Unicode.h> @@ -39,16 +40,7 @@ class String16; class String8 { public: - /* use String8(StaticLinkage) if you're statically linking against - * libutils and declaring an empty static String8, e.g.: - * - * static String8 sAStaticEmptyString(String8::kEmptyString); - * static String8 sAnotherStaticEmptyString(sAStaticEmptyString); - */ - enum StaticLinkage { kEmptyString }; - String8(); - explicit String8(StaticLinkage); String8(const String8& o); explicit String8(const char* o); explicit String8(const char* o, size_t numChars); @@ -241,6 +233,11 @@ private: // require any change to the underlying SharedBuffer contents or reference count. ANDROID_TRIVIAL_MOVE_TRAIT(String8) +static inline std::ostream& operator<<(std::ostream& os, const String8& str) { + os << str.c_str(); + return os; +} + // --------------------------------------------------------------------------- // No user servicable parts below. diff --git a/libutils/include/utils/StrongPointer.h b/libutils/include/utils/StrongPointer.h index 6f4fb4788..11128f227 100644 --- a/libutils/include/utils/StrongPointer.h +++ b/libutils/include/utils/StrongPointer.h @@ -32,6 +32,12 @@ class sp { public: inline sp() : m_ptr(nullptr) { } + // TODO: switch everyone to using this over new, and make RefBase operator + // new private to that class so that we can avoid RefBase being used with + // other memory management mechanisms. + template <typename... Args> + static inline sp<T> make(Args&&... args); + sp(T* other); // NOLINT(implicit) sp(const sp<T>& other); sp(sp<T>&& other) noexcept; @@ -160,9 +166,6 @@ void sp_report_stack_pointer(); // It does not appear safe to broaden this check to include adjacent pages; apparently this code // is used in environments where there may not be a guard page below (at higher addresses than) // the bottom of the stack. -// -// TODO: Consider adding make_sp<T>() to allocate an object and wrap the resulting pointer safely -// without checking overhead. template <typename T> void sp<T>::check_not_on_stack(const void* ptr) { static constexpr int MIN_PAGE_SIZE = 0x1000; // 4K. Safer than including sys/user.h. @@ -174,6 +177,18 @@ void sp<T>::check_not_on_stack(const void* ptr) { } } +// TODO: Ideally we should find a way to increment the reference count before running the +// constructor, so that generating an sp<> to this in the constructor is no longer dangerous. +template <typename T> +template <typename... Args> +sp<T> sp<T>::make(Args&&... args) { + T* t = new T(std::forward<Args>(args)...); + sp<T> result; + result.m_ptr = t; + t->incStrong(t); // bypass check_not_on_stack for heap allocation + return result; +} + template<typename T> sp<T>::sp(T* other) : m_ptr(other) { diff --git a/libutils/include/utils/Timers.h b/libutils/include/utils/Timers.h index 54ec47489..197fc26b8 100644 --- a/libutils/include/utils/Timers.h +++ b/libutils/include/utils/Timers.h @@ -14,11 +14,7 @@ * limitations under the License. */ -// -// Timer functions. -// -#ifndef _LIBS_UTILS_TIMERS_H -#define _LIBS_UTILS_TIMERS_H +#pragma once #include <stdint.h> #include <sys/types.h> @@ -77,11 +73,11 @@ static CONSTEXPR inline nsecs_t milliseconds(nsecs_t v) { return ms2ns(v); } static CONSTEXPR inline nsecs_t microseconds(nsecs_t v) { return us2ns(v); } enum { - SYSTEM_TIME_REALTIME = 0, // system-wide realtime clock - SYSTEM_TIME_MONOTONIC = 1, // monotonic time since unspecified starting point - SYSTEM_TIME_PROCESS = 2, // high-resolution per-process clock - SYSTEM_TIME_THREAD = 3, // high-resolution per-thread clock - SYSTEM_TIME_BOOTTIME = 4 // same as SYSTEM_TIME_MONOTONIC, but including CPU suspend time + SYSTEM_TIME_REALTIME = 0, // system-wide realtime clock + SYSTEM_TIME_MONOTONIC = 1, // monotonic time since unspecified starting point + SYSTEM_TIME_PROCESS = 2, // high-resolution per-process clock + SYSTEM_TIME_THREAD = 3, // high-resolution per-thread clock + SYSTEM_TIME_BOOTTIME = 4, // same as SYSTEM_TIME_MONOTONIC, but including CPU suspend time }; // return the system-time according to the specified clock @@ -104,5 +100,3 @@ int toMillisecondTimeoutDelay(nsecs_t referenceTime, nsecs_t timeoutTime); #ifdef __cplusplus } // extern "C" #endif - -#endif // _LIBS_UTILS_TIMERS_H diff --git a/libziparchive/.clang-format b/libziparchive/.clang-format deleted file mode 120000 index fd0645fdf..000000000 --- a/libziparchive/.clang-format +++ /dev/null @@ -1 +0,0 @@ -../.clang-format-2
\ No newline at end of file diff --git a/libziparchive/Android.bp b/libziparchive/Android.bp deleted file mode 100644 index 553136af8..000000000 --- a/libziparchive/Android.bp +++ /dev/null @@ -1,218 +0,0 @@ -// -// Copyright (C) 2013 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -cc_defaults { - name: "libziparchive_flags", - cflags: [ - // ZLIB_CONST turns on const for input buffers, which is pretty standard. - "-DZLIB_CONST", - "-Werror", - "-Wall", - "-D_FILE_OFFSET_BITS=64", - ], - cppflags: [ - // Incorrectly warns when C++11 empty brace {} initializer is used. - // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61489 - "-Wno-missing-field-initializers", - "-Wconversion", - "-Wno-sign-conversion", - ], - - // Enable -Wold-style-cast only for non-Windows targets. _islower_l, - // _isupper_l etc. in MinGW locale_win32.h (included from - // libcxx/include/__locale) has an old-style-cast. - target: { - not_windows: { - cppflags: [ - "-Wold-style-cast", - ], - }, - }, - sanitize: { - misc_undefined: [ - "signed-integer-overflow", - "unsigned-integer-overflow", - "shift", - "integer-divide-by-zero", - "implicit-signed-integer-truncation", - // TODO: Fix crash when we enable this option - // "implicit-unsigned-integer-truncation", - // TODO: not tested yet. - // "implicit-integer-sign-change", - ], - }, -} - -cc_defaults { - name: "libziparchive_defaults", - srcs: [ - "zip_archive.cc", - "zip_archive_stream_entry.cc", - "zip_writer.cc", - ], - - target: { - windows: { - cflags: ["-mno-ms-bitfields"], - - enabled: true, - }, - }, - - shared_libs: [ - "libbase", - "liblog", - ], - - // for FRIEND_TEST - static_libs: ["libgtest_prod"], - export_static_lib_headers: ["libgtest_prod"], - - export_include_dirs: ["include"], -} - -cc_library { - name: "libziparchive", - host_supported: true, - vendor_available: true, - recovery_available: true, - native_bridge_supported: true, - vndk: { - enabled: true, - }, - double_loadable: true, - export_shared_lib_headers: ["libbase"], - - defaults: [ - "libziparchive_defaults", - "libziparchive_flags", - ], - shared_libs: [ - "liblog", - "libbase", - "libz", - ], - target: { - linux_bionic: { - enabled: true, - }, - }, - - apex_available: [ - "//apex_available:platform", - "com.android.art.debug", - "com.android.art.release", - ], -} - -// Tests. -cc_test { - name: "ziparchive-tests", - host_supported: true, - defaults: ["libziparchive_flags"], - - data: [ - "testdata/**/*", - ], - - srcs: [ - "entry_name_utils_test.cc", - "zip_archive_test.cc", - "zip_writer_test.cc", - ], - shared_libs: [ - "libbase", - "liblog", - ], - - static_libs: [ - "libziparchive", - "libz", - "libutils", - ], - - target: { - host: { - cppflags: ["-Wno-unnamed-type-template-args"], - }, - windows: { - enabled: true, - }, - }, - test_suites: ["device-tests"], -} - -// Performance benchmarks. -cc_benchmark { - name: "ziparchive-benchmarks", - defaults: ["libziparchive_flags"], - - srcs: [ - "zip_archive_benchmark.cpp", - ], - shared_libs: [ - "libbase", - "liblog", - ], - - static_libs: [ - "libziparchive", - "libz", - "libutils", - ], - - target: { - host: { - cppflags: ["-Wno-unnamed-type-template-args"], - }, - }, -} - -cc_binary { - name: "ziptool", - defaults: ["libziparchive_flags"], - srcs: ["ziptool.cpp"], - shared_libs: [ - "libbase", - "libziparchive", - ], - recovery_available: true, - host_supported: true, - target: { - android: { - symlinks: ["unzip", "zipinfo"], - }, - }, -} - -cc_fuzz { - name: "libziparchive_fuzzer", - srcs: ["libziparchive_fuzzer.cpp"], - static_libs: ["libziparchive", "libbase", "libz", "liblog"], - host_supported: true, - corpus: ["testdata/*"], -} - -sh_test { - name: "ziptool-tests", - src: "run-ziptool-tests-on-android.sh", - filename: "run-ziptool-tests-on-android.sh", - test_suites: ["general-tests"], - host_supported: true, - device_supported: false, - test_config: "ziptool-tests.xml", - data: ["cli-tests/**/*"], - target_required: ["cli-test", "ziptool"], -} diff --git a/libziparchive/OWNERS b/libziparchive/OWNERS deleted file mode 100644 index fcc567aa2..000000000 --- a/libziparchive/OWNERS +++ /dev/null @@ -1 +0,0 @@ -narayan@google.com diff --git a/libziparchive/cli-tests/files/example.zip b/libziparchive/cli-tests/files/example.zip Binary files differdeleted file mode 100644 index c3292e929..000000000 --- a/libziparchive/cli-tests/files/example.zip +++ /dev/null diff --git a/libziparchive/cli-tests/unzip.test b/libziparchive/cli-tests/unzip.test deleted file mode 100755 index 6e5cbf213..000000000 --- a/libziparchive/cli-tests/unzip.test +++ /dev/null @@ -1,148 +0,0 @@ -# unzip tests. - -# Note: since "master key", Android uses libziparchive for all zip file -# handling, and that scans the whole central directory immediately. Not only -# lookups by name but also iteration is implemented using the resulting hash -# table, meaning that any test that makes assumptions about iteration order -# will fail on Android. - -name: unzip -l -command: unzip -l $FILES/example.zip d1/d2/x.txt -after: [ ! -f d1/d2/x.txt ] -expected-stdout: - Archive: $FILES/example.zip - Length Date Time Name - --------- ---------- ----- ---- - 1024 2017-06-04 08:45 d1/d2/x.txt - --------- ------- - 1024 1 file ---- - -name: unzip -lq -command: unzip -lq $FILES/example.zip d1/d2/x.txt -after: [ ! -f d1/d2/x.txt ] -expected-stdout: - Length Date Time Name - --------- ---------- ----- ---- - 1024 2017-06-04 08:45 d1/d2/x.txt - --------- ------- - 1024 1 file ---- - -name: unzip -lv -command: unzip -lv $FILES/example.zip d1/d2/x.txt -after: [ ! -f d1/d2/file ] -expected-stdout: - Archive: $FILES/example.zip - Length Method Size Cmpr Date Time CRC-32 Name - -------- ------ ------- ---- ---------- ----- -------- ---- - 1024 Defl:N 11 99% 2017-06-04 08:45 48d7f063 d1/d2/x.txt - -------- ------- --- ------- - 1024 11 99% 1 file ---- - -name: unzip -v -command: unzip -v $FILES/example.zip d1/d2/x.txt -after: [ ! -f d1/d2/file ] -expected-stdout: - Archive: $FILES/example.zip - Length Method Size Cmpr Date Time CRC-32 Name - -------- ------ ------- ---- ---------- ----- -------- ---- - 1024 Defl:N 11 99% 2017-06-04 08:45 48d7f063 d1/d2/x.txt - -------- ------- --- ------- - 1024 11 99% 1 file ---- - -name: unzip one file -command: unzip -q $FILES/example.zip d1/d2/a.txt && cat d1/d2/a.txt -after: [ ! -f d1/d2/b.txt ] -expected-stdout: - a ---- - -name: unzip all files -command: unzip -q $FILES/example.zip -after: [ -f d1/d2/a.txt ] -after: [ -f d1/d2/b.txt ] -after: [ -f d1/d2/c.txt ] -after: [ -f d1/d2/empty.txt ] -after: [ -f d1/d2/x.txt ] -after: [ -d d1/d2/dir ] -expected-stdout: ---- - -name: unzip -o -before: mkdir -p d1/d2 -before: echo b > d1/d2/a.txt -command: unzip -q -o $FILES/example.zip d1/d2/a.txt && cat d1/d2/a.txt -expected-stdout: - a ---- - -name: unzip -n -before: mkdir -p d1/d2 -before: echo b > d1/d2/a.txt -command: unzip -q -n $FILES/example.zip d1/d2/a.txt && cat d1/d2/a.txt -expected-stdout: - b ---- - -# The reference implementation will create *one* level of missing directories, -# so this succeeds. -name: unzip -d shallow non-existent -command: unzip -q -d will-be-created $FILES/example.zip d1/d2/a.txt -after: [ -d will-be-created ] -after: [ -f will-be-created/d1/d2/a.txt ] ---- - -# The reference implementation will *only* create one level of missing -# directories, so this fails. -name: unzip -d deep non-existent -command: unzip -q -d oh-no/will-not-be-created $FILES/example.zip d1/d2/a.txt 2> stderr ; echo $? > status -after: [ ! -d oh-no ] -after: [ ! -d oh-no/will-not-be-created ] -after: [ ! -f oh-no/will-not-be-created/d1/d2/a.txt ] -after: grep -q "oh-no/will-not-be-created" stderr -after: grep -q "No such file or directory" stderr -# The reference implementation has *lots* of non-zero exit values, but we stick to 0 and 1. -after: [ $(cat status) -gt 0 ] ---- - -name: unzip -d exists -before: mkdir dir -command: unzip -q -d dir $FILES/example.zip d1/d2/a.txt && cat dir/d1/d2/a.txt -after: [ ! -f d1/d2/a.txt ] -expected-stdout: - a ---- - -name: unzip -p -command: unzip -p $FILES/example.zip d1/d2/a.txt -after: [ ! -f d1/d2/a.txt ] -expected-stdout: - a ---- - -name: unzip -x FILE... -# Note: the RI ignores -x DIR for some reason, but it's not obvious we should. -command: unzip -q $FILES/example.zip -x d1/d2/a.txt d1/d2/b.txt d1/d2/empty.txt d1/d2/x.txt && cat d1/d2/c.txt -after: [ ! -f d1/d2/a.txt ] -after: [ ! -f d1/d2/b.txt ] -after: [ ! -f d1/d2/empty.txt ] -after: [ ! -f d1/d2/x.txt ] -after: [ -d d1/d2/dir ] -expected-stdout: - ccc ---- - -name: unzip FILE -x FILE... -command: unzip -q $FILES/example.zip d1/d2/a.txt d1/d2/b.txt -x d1/d2/a.txt && cat d1/d2/b.txt -after: [ ! -f d1/d2/a.txt ] -after: [ -f d1/d2/b.txt ] -after: [ ! -f d1/d2/c.txt ] -after: [ ! -f d1/d2/empty.txt ] -after: [ ! -f d1/d2/x.txt ] -after: [ ! -d d1/d2/dir ] -expected-stdout: - bb ---- diff --git a/libziparchive/cli-tests/zipinfo.test b/libziparchive/cli-tests/zipinfo.test deleted file mode 100755 index d5bce1c01..000000000 --- a/libziparchive/cli-tests/zipinfo.test +++ /dev/null @@ -1,53 +0,0 @@ -# zipinfo tests. - -# Note: since "master key", Android uses libziparchive for all zip file -# handling, and that scans the whole central directory immediately. Not only -# lookups by name but also iteration is implemented using the resulting hash -# table, meaning that any test that makes assumptions about iteration order -# will fail on Android. - -name: zipinfo -1 -command: zipinfo -1 $FILES/example.zip | sort -expected-stdout: - d1/ - d1/d2/a.txt - d1/d2/b.txt - d1/d2/c.txt - d1/d2/dir/ - d1/d2/empty.txt - d1/d2/x.txt ---- - -name: zipinfo header -command: zipinfo $FILES/example.zip | head -2 -expected-stdout: - Archive: $FILES/example.zip - Zip file size: 1082 bytes, number of entries: 7 ---- - -name: zipinfo footer -command: zipinfo $FILES/example.zip | tail -1 -expected-stdout: - 7 files, 1033 bytes uncompressed, 20 bytes compressed: 98.1% ---- - -name: zipinfo directory -# The RI doesn't use ISO dates. -command: zipinfo $FILES/example.zip d1/ | sed s/17-Jun-/2017-06-/ -expected-stdout: - drwxr-x--- 3.0 unx 0 bx stor 2017-06-04 08:40 d1/ ---- - -name: zipinfo stored -# The RI doesn't use ISO dates. -command: zipinfo $FILES/example.zip d1/d2/empty.txt | sed s/17-Jun-/2017-06-/ -expected-stdout: - -rw-r----- 3.0 unx 0 bx stor 2017-06-04 08:43 d1/d2/empty.txt ---- - -name: zipinfo deflated -# The RI doesn't use ISO dates. -command: zipinfo $FILES/example.zip d1/d2/x.txt | sed s/17-Jun-/2017-06-/ -expected-stdout: - -rw-r----- 3.0 unx 1024 tx defN 2017-06-04 08:45 d1/d2/x.txt ---- diff --git a/libziparchive/entry_name_utils-inl.h b/libziparchive/entry_name_utils-inl.h deleted file mode 100644 index 10311b565..000000000 --- a/libziparchive/entry_name_utils-inl.h +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef LIBZIPARCHIVE_ENTRY_NAME_UTILS_INL_H_ -#define LIBZIPARCHIVE_ENTRY_NAME_UTILS_INL_H_ - -#include <stddef.h> -#include <stdint.h> - -#include <limits> - -// Check if |length| bytes at |entry_name| constitute a valid entry name. -// Entry names must be valid UTF-8 and must not contain '0'. They also must -// fit into the central directory record. -inline bool IsValidEntryName(const uint8_t* entry_name, const size_t length) { - if (length > std::numeric_limits<uint16_t>::max()) { - return false; - } - for (size_t i = 0; i < length; ++i) { - const uint8_t byte = entry_name[i]; - if (byte == 0) { - return false; - } else if ((byte & 0x80) == 0) { - // Single byte sequence. - continue; - } else if ((byte & 0xc0) == 0x80 || (byte & 0xfe) == 0xfe) { - // Invalid sequence. - return false; - } else { - // 2-5 byte sequences. - for (uint8_t first = static_cast<uint8_t>((byte & 0x7f) << 1); first & 0x80; - first = static_cast<uint8_t>((first & 0x7f) << 1)) { - ++i; - - // Missing continuation byte.. - if (i == length) { - return false; - } - - // Invalid continuation byte. - const uint8_t continuation_byte = entry_name[i]; - if ((continuation_byte & 0xc0) != 0x80) { - return false; - } - } - } - } - - return true; -} - -#endif // LIBZIPARCHIVE_ENTRY_NAME_UTILS_INL_H_ diff --git a/libziparchive/entry_name_utils_test.cc b/libziparchive/entry_name_utils_test.cc deleted file mode 100644 index d83d8540c..000000000 --- a/libziparchive/entry_name_utils_test.cc +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "entry_name_utils-inl.h" - -#include <gtest/gtest.h> - -TEST(entry_name_utils, NullChars) { - // 'A', 'R', '\0', 'S', 'E' - const uint8_t zeroes[] = {0x41, 0x52, 0x00, 0x53, 0x45}; - ASSERT_FALSE(IsValidEntryName(zeroes, sizeof(zeroes))); - - const uint8_t zeroes_continuation_chars[] = {0xc2, 0xa1, 0xc2, 0x00}; - ASSERT_FALSE(IsValidEntryName(zeroes_continuation_chars, sizeof(zeroes_continuation_chars))); -} - -TEST(entry_name_utils, InvalidSequence) { - // 0xfe is an invalid start byte - const uint8_t invalid[] = {0x41, 0xfe}; - ASSERT_FALSE(IsValidEntryName(invalid, sizeof(invalid))); - - // 0x91 is an invalid start byte (it's a valid continuation byte). - const uint8_t invalid2[] = {0x41, 0x91}; - ASSERT_FALSE(IsValidEntryName(invalid2, sizeof(invalid2))); -} - -TEST(entry_name_utils, TruncatedContinuation) { - // Malayalam script with truncated bytes. There should be 2 bytes - // after 0xe0 - const uint8_t truncated[] = {0xe0, 0xb4, 0x85, 0xe0, 0xb4}; - ASSERT_FALSE(IsValidEntryName(truncated, sizeof(truncated))); - - // 0xc2 is the start of a 2 byte sequence that we've subsequently - // dropped. - const uint8_t truncated2[] = {0xc2, 0xc2, 0xa1}; - ASSERT_FALSE(IsValidEntryName(truncated2, sizeof(truncated2))); -} - -TEST(entry_name_utils, BadContinuation) { - // 0x41 is an invalid continuation char, since it's MSBs - // aren't "10..." (are 01). - const uint8_t bad[] = {0xc2, 0xa1, 0xc2, 0x41}; - ASSERT_FALSE(IsValidEntryName(bad, sizeof(bad))); - - // 0x41 is an invalid continuation char, since it's MSBs - // aren't "10..." (are 11). - const uint8_t bad2[] = {0xc2, 0xa1, 0xc2, 0xfe}; - ASSERT_FALSE(IsValidEntryName(bad2, sizeof(bad2))); -} diff --git a/libziparchive/include/ziparchive/zip_archive.h b/libziparchive/include/ziparchive/zip_archive.h deleted file mode 100644 index fbc47dbc4..000000000 --- a/libziparchive/include/ziparchive/zip_archive.h +++ /dev/null @@ -1,289 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -/* - * Read-only access to Zip archives, with minimal heap allocation. - */ - -#include <stdint.h> -#include <string.h> -#include <sys/cdefs.h> -#include <sys/types.h> - -#include <string> -#include <string_view> - -#include "android-base/off64_t.h" - -/* Zip compression methods we support */ -enum { - kCompressStored = 0, // no compression - kCompressDeflated = 8, // standard deflate -}; - -/* - * Represents information about a zip entry in a zip file. - */ -struct ZipEntry { - // Compression method. One of kCompressStored or kCompressDeflated. - // See also `gpbf` for deflate subtypes. - uint16_t method; - - // Modification time. The zipfile format specifies - // that the first two little endian bytes contain the time - // and the last two little endian bytes contain the date. - // See `GetModificationTime`. Use signed integer to avoid the - // sub-overflow. - // TODO: should be overridden by extra time field, if present. - int32_t mod_time; - - // Returns `mod_time` as a broken-down struct tm. - struct tm GetModificationTime() const; - - // Suggested Unix mode for this entry, from the zip archive if created on - // Unix, or a default otherwise. See also `external_file_attributes`. - mode_t unix_mode; - - // 1 if this entry contains a data descriptor segment, 0 - // otherwise. - uint8_t has_data_descriptor; - - // Crc32 value of this ZipEntry. This information might - // either be stored in the local file header or in a special - // Data descriptor footer at the end of the file entry. - uint32_t crc32; - - // Compressed length of this ZipEntry. Might be present - // either in the local file header or in the data descriptor - // footer. - uint32_t compressed_length; - - // Uncompressed length of this ZipEntry. Might be present - // either in the local file header or in the data descriptor - // footer. - uint32_t uncompressed_length; - - // The offset to the start of data for this ZipEntry. - off64_t offset; - - // The version of zip and the host file system this came from (for zipinfo). - uint16_t version_made_by; - - // The raw attributes, whose interpretation depends on the host - // file system in `version_made_by` (for zipinfo). See also `unix_mode`. - uint32_t external_file_attributes; - - // Specifics about the deflation (for zipinfo). - uint16_t gpbf; - // Whether this entry is believed to be text or binary (for zipinfo). - bool is_text; -}; - -struct ZipArchive; -typedef ZipArchive* ZipArchiveHandle; - -/* - * Open a Zip archive, and sets handle to the value of the opaque - * handle for the file. This handle must be released by calling - * CloseArchive with this handle. - * - * Returns 0 on success, and negative values on failure. - */ -int32_t OpenArchive(const char* fileName, ZipArchiveHandle* handle); - -/* - * Like OpenArchive, but takes a file descriptor open for reading - * at the start of the file. The descriptor must be mappable (this does - * not allow access to a stream). - * - * Sets handle to the value of the opaque handle for this file descriptor. - * This handle must be released by calling CloseArchive with this handle. - * - * If assume_ownership parameter is 'true' calling CloseArchive will close - * the file. - * - * This function maps and scans the central directory and builds a table - * of entries for future lookups. - * - * "debugFileName" will appear in error messages, but is not otherwise used. - * - * Returns 0 on success, and negative values on failure. - */ -int32_t OpenArchiveFd(const int fd, const char* debugFileName, ZipArchiveHandle* handle, - bool assume_ownership = true); - -int32_t OpenArchiveFdRange(const int fd, const char* debugFileName, ZipArchiveHandle* handle, - off64_t length, off64_t offset, bool assume_ownership = true); - -int32_t OpenArchiveFromMemory(const void* address, size_t length, const char* debugFileName, - ZipArchiveHandle* handle); -/* - * Close archive, releasing resources associated with it. This will - * unmap the central directory of the zipfile and free all internal - * data structures associated with the file. It is an error to use - * this handle for any further operations without an intervening - * call to one of the OpenArchive variants. - */ -void CloseArchive(ZipArchiveHandle archive); - -/** See GetArchiveInfo(). */ -struct ZipArchiveInfo { - /** The size in bytes of the archive itself. Used by zipinfo. */ - off64_t archive_size; - /** The number of entries in the archive. */ - size_t entry_count; -}; - -/** - * Returns information about the given archive. - */ -ZipArchiveInfo GetArchiveInfo(ZipArchiveHandle archive); - -/* - * Find an entry in the Zip archive, by name. |data| must be non-null. - * - * Returns 0 if an entry is found, and populates |data| with information - * about this entry. Returns negative values otherwise. - * - * It's important to note that |data->crc32|, |data->compLen| and - * |data->uncompLen| might be set to values from the central directory - * if this file entry contains a data descriptor footer. To verify crc32s - * and length, a call to VerifyCrcAndLengths must be made after entry data - * has been processed. - * - * On non-Windows platforms this method does not modify internal state and - * can be called concurrently. - */ -int32_t FindEntry(const ZipArchiveHandle archive, const std::string_view entryName, ZipEntry* data); - -/* - * Start iterating over all entries of a zip file. The order of iteration - * is not guaranteed to be the same as the order of elements - * in the central directory but is stable for a given zip file. |cookie| will - * contain the value of an opaque cookie which can be used to make one or more - * calls to Next. All calls to StartIteration must be matched by a call to - * EndIteration to free any allocated memory. - * - * This method also accepts optional prefix and suffix to restrict iteration to - * entry names that start with |optional_prefix| or end with |optional_suffix|. - * - * Returns 0 on success and negative values on failure. - */ -int32_t StartIteration(ZipArchiveHandle archive, void** cookie_ptr, - const std::string_view optional_prefix = "", - const std::string_view optional_suffix = ""); - -/* - * Advance to the next element in the zipfile in iteration order. - * - * Returns 0 on success, -1 if there are no more elements in this - * archive and lower negative values on failure. - */ -int32_t Next(void* cookie, ZipEntry* data, std::string* name); -int32_t Next(void* cookie, ZipEntry* data, std::string_view* name); - -/* - * End iteration over all entries of a zip file and frees the memory allocated - * in StartIteration. - */ -void EndIteration(void* cookie); - -/* - * Uncompress and write an entry to an open file identified by |fd|. - * |entry->uncompressed_length| bytes will be written to the file at - * its current offset, and the file will be truncated at the end of - * the uncompressed data (no truncation if |fd| references a block - * device). - * - * Returns 0 on success and negative values on failure. - */ -int32_t ExtractEntryToFile(ZipArchiveHandle archive, ZipEntry* entry, int fd); - -/** - * Uncompress a given zip entry to the memory region at |begin| and of - * size |size|. This size is expected to be the same as the *declared* - * uncompressed length of the zip entry. It is an error if the *actual* - * number of uncompressed bytes differs from this number. - * - * Returns 0 on success and negative values on failure. - */ -int32_t ExtractToMemory(ZipArchiveHandle archive, ZipEntry* entry, uint8_t* begin, uint32_t size); - -int GetFileDescriptor(const ZipArchiveHandle archive); - -/** - * Returns the offset of the zip archive in the backing file descriptor, or 0 if the zip archive is - * not backed by a file descriptor. - */ -off64_t GetFileDescriptorOffset(const ZipArchiveHandle archive); - -const char* ErrorCodeString(int32_t error_code); - -#if !defined(_WIN32) -typedef bool (*ProcessZipEntryFunction)(const uint8_t* buf, size_t buf_size, void* cookie); - -/* - * Stream the uncompressed data through the supplied function, - * passing cookie to it each time it gets called. - */ -int32_t ProcessZipEntryContents(ZipArchiveHandle archive, ZipEntry* entry, - ProcessZipEntryFunction func, void* cookie); -#endif - -namespace zip_archive { - -class Writer { - public: - virtual bool Append(uint8_t* buf, size_t buf_size) = 0; - virtual ~Writer(); - - protected: - Writer() = default; - - private: - Writer(const Writer&) = delete; - void operator=(const Writer&) = delete; -}; - -class Reader { - public: - virtual bool ReadAtOffset(uint8_t* buf, size_t len, uint32_t offset) const = 0; - virtual ~Reader(); - - protected: - Reader() = default; - - private: - Reader(const Reader&) = delete; - void operator=(const Reader&) = delete; -}; - -/* - * Inflates the first |compressed_length| bytes of |reader| to a given |writer|. - * |crc_out| is set to the CRC32 checksum of the uncompressed data. - * - * Returns 0 on success and negative values on failure, for example if |reader| - * cannot supply the right amount of data, or if the number of bytes written to - * data does not match |uncompressed_length|. - * - * If |crc_out| is not nullptr, it is set to the crc32 checksum of the - * uncompressed data. - */ -int32_t Inflate(const Reader& reader, const uint32_t compressed_length, - const uint32_t uncompressed_length, Writer* writer, uint64_t* crc_out); -} // namespace zip_archive diff --git a/libziparchive/include/ziparchive/zip_archive_stream_entry.h b/libziparchive/include/ziparchive/zip_archive_stream_entry.h deleted file mode 100644 index 8c6ca795a..000000000 --- a/libziparchive/include/ziparchive/zip_archive_stream_entry.h +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Read-only stream access to Zip archives entries. -#pragma once - -#include <ziparchive/zip_archive.h> - -#include <vector> - -#include "android-base/off64_t.h" - -class ZipArchiveStreamEntry { - public: - virtual ~ZipArchiveStreamEntry() {} - - virtual const std::vector<uint8_t>* Read() = 0; - - virtual bool Verify() = 0; - - static ZipArchiveStreamEntry* Create(ZipArchiveHandle handle, const ZipEntry& entry); - static ZipArchiveStreamEntry* CreateRaw(ZipArchiveHandle handle, const ZipEntry& entry); - - protected: - ZipArchiveStreamEntry(ZipArchiveHandle handle) : handle_(handle) {} - - virtual bool Init(const ZipEntry& entry); - - ZipArchiveHandle handle_; - - off64_t offset_ = 0; - uint32_t crc32_ = 0u; -}; diff --git a/libziparchive/include/ziparchive/zip_writer.h b/libziparchive/include/ziparchive/zip_writer.h deleted file mode 100644 index d68683dfe..000000000 --- a/libziparchive/include/ziparchive/zip_writer.h +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include <cstdio> -#include <ctime> - -#include <gtest/gtest_prod.h> -#include <memory> -#include <string> -#include <string_view> -#include <vector> - -#include "android-base/macros.h" -#include "android-base/off64_t.h" - -struct z_stream_s; -typedef struct z_stream_s z_stream; - -/** - * Writes a Zip file via a stateful interface. - * - * Example: - * - * FILE* file = fopen("path/to/zip.zip", "wb"); - * - * ZipWriter writer(file); - * - * writer.StartEntry("test.txt", ZipWriter::kCompress | ZipWriter::kAlign); - * writer.WriteBytes(buffer, bufferLen); - * writer.WriteBytes(buffer2, bufferLen2); - * writer.FinishEntry(); - * - * writer.StartEntry("empty.txt", 0); - * writer.FinishEntry(); - * - * writer.Finish(); - * - * fclose(file); - */ -class ZipWriter { - public: - enum { - /** - * Flag to compress the zip entry using deflate. - */ - kCompress = 0x01, - - /** - * Flag to align the zip entry data on a 32bit boundary. Useful for - * mmapping the data at runtime. - */ - kAlign32 = 0x02, - }; - - /** - * A struct representing a zip file entry. - */ - struct FileEntry { - std::string path; - uint16_t compression_method; - uint32_t crc32; - uint32_t compressed_size; - uint32_t uncompressed_size; - uint16_t last_mod_time; - uint16_t last_mod_date; - uint16_t padding_length; - off64_t local_file_header_offset; - }; - - static const char* ErrorCodeString(int32_t error_code); - - /** - * Create a ZipWriter that will write into a FILE stream. The file should be opened with - * open mode of "wb" or "w+b". ZipWriter does not take ownership of the file stream. The - * caller is responsible for closing the file. - */ - explicit ZipWriter(FILE* f); - - // Move constructor. - ZipWriter(ZipWriter&& zipWriter) noexcept; - - // Move assignment. - ZipWriter& operator=(ZipWriter&& zipWriter) noexcept; - - /** - * Starts a new zip entry with the given path and flags. - * Flags can be a bitwise OR of ZipWriter::kCompress and ZipWriter::kAlign. - * Subsequent calls to WriteBytes(const void*, size_t) will add data to this entry. - * Returns 0 on success, and an error value < 0 on failure. - */ - int32_t StartEntry(std::string_view path, size_t flags); - - /** - * Starts a new zip entry with the given path and flags, where the - * entry will be aligned to the given alignment. - * Flags can only be ZipWriter::kCompress. Using the flag ZipWriter::kAlign32 - * will result in an error. - * Subsequent calls to WriteBytes(const void*, size_t) will add data to this entry. - * Returns 0 on success, and an error value < 0 on failure. - */ - int32_t StartAlignedEntry(std::string_view path, size_t flags, uint32_t alignment); - - /** - * Same as StartEntry(const char*, size_t), but sets a last modified time for the entry. - */ - int32_t StartEntryWithTime(std::string_view path, size_t flags, time_t time); - - /** - * Same as StartAlignedEntry(const char*, size_t), but sets a last modified time for the entry. - */ - int32_t StartAlignedEntryWithTime(std::string_view path, size_t flags, time_t time, uint32_t alignment); - - /** - * Writes bytes to the zip file for the previously started zip entry. - * Returns 0 on success, and an error value < 0 on failure. - */ - int32_t WriteBytes(const void* data, size_t len); - - /** - * Finish a zip entry started with StartEntry(const char*, size_t) or - * StartEntryWithTime(const char*, size_t, time_t). This must be called before - * any new zip entries are started, or before Finish() is called. - * Returns 0 on success, and an error value < 0 on failure. - */ - int32_t FinishEntry(); - - /** - * Discards the last-written entry. Can only be called after an entry has been written using - * FinishEntry(). - * Returns 0 on success, and an error value < 0 on failure. - */ - int32_t DiscardLastEntry(); - - /** - * Sets `out_entry` to the last entry written after a call to FinishEntry(). - * Returns 0 on success, and an error value < 0 if no entries have been written. - */ - int32_t GetLastEntry(FileEntry* out_entry); - - /** - * Writes the Central Directory Headers and flushes the zip file stream. - * Returns 0 on success, and an error value < 0 on failure. - */ - int32_t Finish(); - - private: - DISALLOW_COPY_AND_ASSIGN(ZipWriter); - - int32_t HandleError(int32_t error_code); - int32_t PrepareDeflate(); - int32_t StoreBytes(FileEntry* file, const void* data, uint32_t len); - int32_t CompressBytes(FileEntry* file, const void* data, uint32_t len); - int32_t FlushCompressedBytes(FileEntry* file); - bool ShouldUseDataDescriptor() const; - - enum class State { - kWritingZip, - kWritingEntry, - kDone, - kError, - }; - - FILE* file_; - bool seekable_; - off64_t current_offset_; - State state_; - std::vector<FileEntry> files_; - FileEntry current_file_entry_; - - std::unique_ptr<z_stream, void (*)(z_stream*)> z_stream_; - std::vector<uint8_t> buffer_; - - FRIEND_TEST(zipwriter, WriteToUnseekableFile); -}; diff --git a/libziparchive/libziparchive_fuzzer.cpp b/libziparchive/libziparchive_fuzzer.cpp deleted file mode 100644 index 75e7939da..000000000 --- a/libziparchive/libziparchive_fuzzer.cpp +++ /dev/null @@ -1,13 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -#include <stddef.h> -#include <stdint.h> - -#include <ziparchive/zip_archive.h> - -extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { - ZipArchiveHandle handle = nullptr; - OpenArchiveFromMemory(data, size, "fuzz", &handle); - CloseArchive(handle); - return 0; -} diff --git a/libziparchive/run-ziptool-tests-on-android.sh b/libziparchive/run-ziptool-tests-on-android.sh deleted file mode 100755 index 3c23d4373..000000000 --- a/libziparchive/run-ziptool-tests-on-android.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -# Copy the tests across. -adb shell rm -rf /data/local/tmp/ziptool-tests/ -adb shell mkdir /data/local/tmp/ziptool-tests/ -adb push cli-tests/ /data/local/tmp/ziptool-tests/ -#adb push cli-test /data/local/tmp/ziptool-tests/ - -if tty -s; then - dash_t="-t" -else - dash_t="" -fi - -exec adb shell $dash_t cli-test /data/local/tmp/ziptool-tests/cli-tests/*.test diff --git a/libziparchive/testdata/bad_crc.zip b/libziparchive/testdata/bad_crc.zip Binary files differdeleted file mode 100644 index e12ba07bd..000000000 --- a/libziparchive/testdata/bad_crc.zip +++ /dev/null diff --git a/libziparchive/testdata/bad_filename.zip b/libziparchive/testdata/bad_filename.zip Binary files differdeleted file mode 100644 index 294eaf562..000000000 --- a/libziparchive/testdata/bad_filename.zip +++ /dev/null diff --git a/libziparchive/testdata/crash.apk b/libziparchive/testdata/crash.apk Binary files differdeleted file mode 100644 index d6dd52dd7..000000000 --- a/libziparchive/testdata/crash.apk +++ /dev/null diff --git a/libziparchive/testdata/declaredlength.zip b/libziparchive/testdata/declaredlength.zip Binary files differdeleted file mode 100644 index 773380c45..000000000 --- a/libziparchive/testdata/declaredlength.zip +++ /dev/null diff --git a/libziparchive/testdata/dummy-update.zip b/libziparchive/testdata/dummy-update.zip Binary files differdeleted file mode 100644 index 6976bf155..000000000 --- a/libziparchive/testdata/dummy-update.zip +++ /dev/null diff --git a/libziparchive/testdata/empty.zip b/libziparchive/testdata/empty.zip Binary files differdeleted file mode 100644 index 15cb0ecb3..000000000 --- a/libziparchive/testdata/empty.zip +++ /dev/null diff --git a/libziparchive/testdata/large.zip b/libziparchive/testdata/large.zip Binary files differdeleted file mode 100644 index 49659c832..000000000 --- a/libziparchive/testdata/large.zip +++ /dev/null diff --git a/libziparchive/testdata/valid.zip b/libziparchive/testdata/valid.zip Binary files differdeleted file mode 100644 index 9e7cb7800..000000000 --- a/libziparchive/testdata/valid.zip +++ /dev/null diff --git a/libziparchive/testdata/zero-size-cd.zip b/libziparchive/testdata/zero-size-cd.zip Binary files differdeleted file mode 100644 index b6c8cbeac..000000000 --- a/libziparchive/testdata/zero-size-cd.zip +++ /dev/null diff --git a/libziparchive/zip_archive.cc b/libziparchive/zip_archive.cc deleted file mode 100644 index 489fcb1aa..000000000 --- a/libziparchive/zip_archive.cc +++ /dev/null @@ -1,1315 +0,0 @@ -/* - * Copyright (C) 2008 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Read-only access to Zip archives, with minimal heap allocation. - */ - -#define LOG_TAG "ziparchive" - -#include "ziparchive/zip_archive.h" - -#include <errno.h> -#include <fcntl.h> -#include <inttypes.h> -#include <limits.h> -#include <stdlib.h> -#include <string.h> -#include <time.h> -#include <unistd.h> - -#include <memory> -#include <vector> - -#if defined(__APPLE__) -#define lseek64 lseek -#endif - -#if defined(__BIONIC__) -#include <android/fdsan.h> -#endif - -#include <android-base/file.h> -#include <android-base/logging.h> -#include <android-base/macros.h> // TEMP_FAILURE_RETRY may or may not be in unistd -#include <android-base/mapped_file.h> -#include <android-base/memory.h> -#include <android-base/strings.h> -#include <android-base/utf8.h> -#include <log/log.h> -#include "zlib.h" - -#include "entry_name_utils-inl.h" -#include "zip_archive_common.h" -#include "zip_archive_private.h" - -using android::base::get_unaligned; - -// Used to turn on crc checks - verify that the content CRC matches the values -// specified in the local file header and the central directory. -static constexpr bool kCrcChecksEnabled = false; - -// The maximum number of bytes to scan backwards for the EOCD start. -static const uint32_t kMaxEOCDSearch = kMaxCommentLen + sizeof(EocdRecord); - -/* - * A Read-only Zip archive. - * - * We want "open" and "find entry by name" to be fast operations, and - * we want to use as little memory as possible. We memory-map the zip - * central directory, and load a hash table with pointers to the filenames - * (which aren't null-terminated). The other fields are at a fixed offset - * from the filename, so we don't need to extract those (but we do need - * to byte-read and endian-swap them every time we want them). - * - * It's possible that somebody has handed us a massive (~1GB) zip archive, - * so we can't expect to mmap the entire file. - * - * To speed comparisons when doing a lookup by name, we could make the mapping - * "private" (copy-on-write) and null-terminate the filenames after verifying - * the record structure. However, this requires a private mapping of - * every page that the Central Directory touches. Easier to tuck a copy - * of the string length into the hash table entry. - */ - -/* - * Round up to the next highest power of 2. - * - * Found on http://graphics.stanford.edu/~seander/bithacks.html. - */ -static uint32_t RoundUpPower2(uint32_t val) { - val--; - val |= val >> 1; - val |= val >> 2; - val |= val >> 4; - val |= val >> 8; - val |= val >> 16; - val++; - - return val; -} - -static uint32_t ComputeHash(std::string_view name) { - return static_cast<uint32_t>(std::hash<std::string_view>{}(name)); -} - -/* - * Convert a ZipEntry to a hash table index, verifying that it's in a - * valid range. - */ -static int64_t EntryToIndex(const ZipStringOffset* hash_table, const uint32_t hash_table_size, - std::string_view name, const uint8_t* start) { - const uint32_t hash = ComputeHash(name); - - // NOTE: (hash_table_size - 1) is guaranteed to be non-negative. - uint32_t ent = hash & (hash_table_size - 1); - while (hash_table[ent].name_offset != 0) { - if (hash_table[ent].ToStringView(start) == name) { - return ent; - } - ent = (ent + 1) & (hash_table_size - 1); - } - - ALOGV("Zip: Unable to find entry %.*s", static_cast<int>(name.size()), name.data()); - return kEntryNotFound; -} - -/* - * Add a new entry to the hash table. - */ -static int32_t AddToHash(ZipStringOffset* hash_table, const uint32_t hash_table_size, - std::string_view name, const uint8_t* start) { - const uint64_t hash = ComputeHash(name); - uint32_t ent = hash & (hash_table_size - 1); - - /* - * We over-allocated the table, so we're guaranteed to find an empty slot. - * Further, we guarantee that the hashtable size is not 0. - */ - while (hash_table[ent].name_offset != 0) { - if (hash_table[ent].ToStringView(start) == name) { - // We've found a duplicate entry. We don't accept duplicates. - ALOGW("Zip: Found duplicate entry %.*s", static_cast<int>(name.size()), name.data()); - return kDuplicateEntry; - } - ent = (ent + 1) & (hash_table_size - 1); - } - - // `name` has already been validated before entry. - const char* start_char = reinterpret_cast<const char*>(start); - hash_table[ent].name_offset = static_cast<uint32_t>(name.data() - start_char); - hash_table[ent].name_length = static_cast<uint16_t>(name.size()); - return 0; -} - -#if defined(__BIONIC__) -uint64_t GetOwnerTag(const ZipArchive* archive) { - return android_fdsan_create_owner_tag(ANDROID_FDSAN_OWNER_TYPE_ZIPARCHIVE, - reinterpret_cast<uint64_t>(archive)); -} -#endif - -ZipArchive::ZipArchive(MappedZipFile&& map, bool assume_ownership) - : mapped_zip(map), - close_file(assume_ownership), - directory_offset(0), - central_directory(), - directory_map(), - num_entries(0), - hash_table_size(0), - hash_table(nullptr) { -#if defined(__BIONIC__) - if (assume_ownership) { - CHECK(mapped_zip.HasFd()); - android_fdsan_exchange_owner_tag(mapped_zip.GetFileDescriptor(), 0, GetOwnerTag(this)); - } -#endif -} - -ZipArchive::ZipArchive(const void* address, size_t length) - : mapped_zip(address, length), - close_file(false), - directory_offset(0), - central_directory(), - directory_map(), - num_entries(0), - hash_table_size(0), - hash_table(nullptr) {} - -ZipArchive::~ZipArchive() { - if (close_file && mapped_zip.GetFileDescriptor() >= 0) { -#if defined(__BIONIC__) - android_fdsan_close_with_tag(mapped_zip.GetFileDescriptor(), GetOwnerTag(this)); -#else - close(mapped_zip.GetFileDescriptor()); -#endif - } - - free(hash_table); -} - -static int32_t MapCentralDirectory0(const char* debug_file_name, ZipArchive* archive, - off64_t file_length, uint32_t read_amount, - uint8_t* scan_buffer) { - const off64_t search_start = file_length - read_amount; - - if (!archive->mapped_zip.ReadAtOffset(scan_buffer, read_amount, search_start)) { - ALOGE("Zip: read %" PRId64 " from offset %" PRId64 " failed", static_cast<int64_t>(read_amount), - static_cast<int64_t>(search_start)); - return kIoError; - } - - /* - * Scan backward for the EOCD magic. In an archive without a trailing - * comment, we'll find it on the first try. (We may want to consider - * doing an initial minimal read; if we don't find it, retry with a - * second read as above.) - */ - CHECK_LE(read_amount, std::numeric_limits<int32_t>::max()); - int32_t i = read_amount - sizeof(EocdRecord); - for (; i >= 0; i--) { - if (scan_buffer[i] == 0x50) { - uint32_t* sig_addr = reinterpret_cast<uint32_t*>(&scan_buffer[i]); - if (get_unaligned<uint32_t>(sig_addr) == EocdRecord::kSignature) { - ALOGV("+++ Found EOCD at buf+%d", i); - break; - } - } - } - if (i < 0) { - ALOGD("Zip: EOCD not found, %s is not zip", debug_file_name); - return kInvalidFile; - } - - const off64_t eocd_offset = search_start + i; - const EocdRecord* eocd = reinterpret_cast<const EocdRecord*>(scan_buffer + i); - /* - * Verify that there's no trailing space at the end of the central directory - * and its comment. - */ - const off64_t calculated_length = eocd_offset + sizeof(EocdRecord) + eocd->comment_length; - if (calculated_length != file_length) { - ALOGW("Zip: %" PRId64 " extraneous bytes at the end of the central directory", - static_cast<int64_t>(file_length - calculated_length)); - return kInvalidFile; - } - - /* - * Grab the CD offset and size, and the number of entries in the - * archive and verify that they look reasonable. - */ - if (static_cast<off64_t>(eocd->cd_start_offset) + eocd->cd_size > eocd_offset) { - ALOGW("Zip: bad offsets (dir %" PRIu32 ", size %" PRIu32 ", eocd %" PRId64 ")", - eocd->cd_start_offset, eocd->cd_size, static_cast<int64_t>(eocd_offset)); - return kInvalidOffset; - } - if (eocd->num_records == 0) { -#if defined(__ANDROID__) - ALOGW("Zip: empty archive?"); -#endif - return kEmptyArchive; - } - - ALOGV("+++ num_entries=%" PRIu32 " dir_size=%" PRIu32 " dir_offset=%" PRIu32, eocd->num_records, - eocd->cd_size, eocd->cd_start_offset); - - // It all looks good. Create a mapping for the CD, and set the fields - // in archive. - if (!archive->InitializeCentralDirectory(static_cast<off64_t>(eocd->cd_start_offset), - static_cast<size_t>(eocd->cd_size))) { - return kMmapFailed; - } - - archive->num_entries = eocd->num_records; - archive->directory_offset = eocd->cd_start_offset; - - return 0; -} - -/* - * Find the zip Central Directory and memory-map it. - * - * On success, returns 0 after populating fields from the EOCD area: - * directory_offset - * directory_ptr - * num_entries - */ -static int32_t MapCentralDirectory(const char* debug_file_name, ZipArchive* archive) { - // Test file length. We use lseek64 to make sure the file - // is small enough to be a zip file (Its size must be less than - // 0xffffffff bytes). - off64_t file_length = archive->mapped_zip.GetFileLength(); - if (file_length == -1) { - return kInvalidFile; - } - - if (file_length > static_cast<off64_t>(0xffffffff)) { - ALOGV("Zip: zip file too long %" PRId64, static_cast<int64_t>(file_length)); - return kInvalidFile; - } - - if (file_length < static_cast<off64_t>(sizeof(EocdRecord))) { - ALOGV("Zip: length %" PRId64 " is too small to be zip", static_cast<int64_t>(file_length)); - return kInvalidFile; - } - - /* - * Perform the traditional EOCD snipe hunt. - * - * We're searching for the End of Central Directory magic number, - * which appears at the start of the EOCD block. It's followed by - * 18 bytes of EOCD stuff and up to 64KB of archive comment. We - * need to read the last part of the file into a buffer, dig through - * it to find the magic number, parse some values out, and use those - * to determine the extent of the CD. - * - * We start by pulling in the last part of the file. - */ - uint32_t read_amount = kMaxEOCDSearch; - if (file_length < read_amount) { - read_amount = static_cast<uint32_t>(file_length); - } - - std::vector<uint8_t> scan_buffer(read_amount); - int32_t result = - MapCentralDirectory0(debug_file_name, archive, file_length, read_amount, scan_buffer.data()); - return result; -} - -/* - * Parses the Zip archive's Central Directory. Allocates and populates the - * hash table. - * - * Returns 0 on success. - */ -static int32_t ParseZipArchive(ZipArchive* archive) { - const uint8_t* const cd_ptr = archive->central_directory.GetBasePtr(); - const size_t cd_length = archive->central_directory.GetMapLength(); - const uint16_t num_entries = archive->num_entries; - - /* - * Create hash table. We have a minimum 75% load factor, possibly as - * low as 50% after we round off to a power of 2. There must be at - * least one unused entry to avoid an infinite loop during creation. - */ - archive->hash_table_size = RoundUpPower2(1 + (num_entries * 4) / 3); - archive->hash_table = - reinterpret_cast<ZipStringOffset*>(calloc(archive->hash_table_size, sizeof(ZipStringOffset))); - if (archive->hash_table == nullptr) { - ALOGW("Zip: unable to allocate the %u-entry hash_table, entry size: %zu", - archive->hash_table_size, sizeof(ZipStringOffset)); - return kAllocationFailed; - } - - /* - * Walk through the central directory, adding entries to the hash - * table and verifying values. - */ - const uint8_t* const cd_end = cd_ptr + cd_length; - const uint8_t* ptr = cd_ptr; - for (uint16_t i = 0; i < num_entries; i++) { - if (ptr > cd_end - sizeof(CentralDirectoryRecord)) { - ALOGW("Zip: ran off the end (item #%" PRIu16 ", %zu bytes of central directory)", i, - cd_length); -#if defined(__ANDROID__) - android_errorWriteLog(0x534e4554, "36392138"); -#endif - return kInvalidFile; - } - - const CentralDirectoryRecord* cdr = reinterpret_cast<const CentralDirectoryRecord*>(ptr); - if (cdr->record_signature != CentralDirectoryRecord::kSignature) { - ALOGW("Zip: missed a central dir sig (at %" PRIu16 ")", i); - return kInvalidFile; - } - - const off64_t local_header_offset = cdr->local_file_header_offset; - if (local_header_offset >= archive->directory_offset) { - ALOGW("Zip: bad LFH offset %" PRId64 " at entry %" PRIu16, - static_cast<int64_t>(local_header_offset), i); - return kInvalidFile; - } - - const uint16_t file_name_length = cdr->file_name_length; - const uint16_t extra_length = cdr->extra_field_length; - const uint16_t comment_length = cdr->comment_length; - const uint8_t* file_name = ptr + sizeof(CentralDirectoryRecord); - - if (file_name + file_name_length > cd_end) { - ALOGW("Zip: file name for entry %" PRIu16 - " exceeds the central directory range, file_name_length: %" PRIu16 ", cd_length: %zu", - i, file_name_length, cd_length); - return kInvalidEntryName; - } - // Check that file name is valid UTF-8 and doesn't contain NUL (U+0000) characters. - if (!IsValidEntryName(file_name, file_name_length)) { - ALOGW("Zip: invalid file name at entry %" PRIu16, i); - return kInvalidEntryName; - } - - // Add the CDE filename to the hash table. - std::string_view entry_name{reinterpret_cast<const char*>(file_name), file_name_length}; - const int add_result = AddToHash(archive->hash_table, archive->hash_table_size, entry_name, - archive->central_directory.GetBasePtr()); - if (add_result != 0) { - ALOGW("Zip: Error adding entry to hash table %d", add_result); - return add_result; - } - - ptr += sizeof(CentralDirectoryRecord) + file_name_length + extra_length + comment_length; - if ((ptr - cd_ptr) > static_cast<int64_t>(cd_length)) { - ALOGW("Zip: bad CD advance (%tu vs %zu) at entry %" PRIu16, ptr - cd_ptr, cd_length, i); - return kInvalidFile; - } - } - - uint32_t lfh_start_bytes; - if (!archive->mapped_zip.ReadAtOffset(reinterpret_cast<uint8_t*>(&lfh_start_bytes), - sizeof(uint32_t), 0)) { - ALOGW("Zip: Unable to read header for entry at offset == 0."); - return kInvalidFile; - } - - if (lfh_start_bytes != LocalFileHeader::kSignature) { - ALOGW("Zip: Entry at offset zero has invalid LFH signature %" PRIx32, lfh_start_bytes); -#if defined(__ANDROID__) - android_errorWriteLog(0x534e4554, "64211847"); -#endif - return kInvalidFile; - } - - ALOGV("+++ zip good scan %" PRIu16 " entries", num_entries); - - return 0; -} - -static int32_t OpenArchiveInternal(ZipArchive* archive, const char* debug_file_name) { - int32_t result = MapCentralDirectory(debug_file_name, archive); - return result != 0 ? result : ParseZipArchive(archive); -} - -int32_t OpenArchiveFd(int fd, const char* debug_file_name, ZipArchiveHandle* handle, - bool assume_ownership) { - ZipArchive* archive = new ZipArchive(MappedZipFile(fd), assume_ownership); - *handle = archive; - return OpenArchiveInternal(archive, debug_file_name); -} - -int32_t OpenArchiveFdRange(int fd, const char* debug_file_name, ZipArchiveHandle* handle, - off64_t length, off64_t offset, bool assume_ownership) { - ZipArchive* archive = new ZipArchive(MappedZipFile(fd, length, offset), assume_ownership); - *handle = archive; - - if (length < 0) { - ALOGW("Invalid zip length %" PRId64, length); - return kIoError; - } - - if (offset < 0) { - ALOGW("Invalid zip offset %" PRId64, offset); - return kIoError; - } - - return OpenArchiveInternal(archive, debug_file_name); -} - -int32_t OpenArchive(const char* fileName, ZipArchiveHandle* handle) { - const int fd = ::android::base::utf8::open(fileName, O_RDONLY | O_BINARY | O_CLOEXEC, 0); - ZipArchive* archive = new ZipArchive(MappedZipFile(fd), true); - *handle = archive; - - if (fd < 0) { - ALOGW("Unable to open '%s': %s", fileName, strerror(errno)); - return kIoError; - } - - return OpenArchiveInternal(archive, fileName); -} - -int32_t OpenArchiveFromMemory(const void* address, size_t length, const char* debug_file_name, - ZipArchiveHandle* handle) { - ZipArchive* archive = new ZipArchive(address, length); - *handle = archive; - return OpenArchiveInternal(archive, debug_file_name); -} - -ZipArchiveInfo GetArchiveInfo(ZipArchiveHandle archive) { - ZipArchiveInfo result; - result.archive_size = archive->mapped_zip.GetFileLength(); - result.entry_count = archive->num_entries; - return result; -} - -/* - * Close a ZipArchive, closing the file and freeing the contents. - */ -void CloseArchive(ZipArchiveHandle archive) { - ALOGV("Closing archive %p", archive); - delete archive; -} - -static int32_t ValidateDataDescriptor(MappedZipFile& mapped_zip, ZipEntry* entry) { - uint8_t ddBuf[sizeof(DataDescriptor) + sizeof(DataDescriptor::kOptSignature)]; - off64_t offset = entry->offset; - if (entry->method != kCompressStored) { - offset += entry->compressed_length; - } else { - offset += entry->uncompressed_length; - } - - if (!mapped_zip.ReadAtOffset(ddBuf, sizeof(ddBuf), offset)) { - return kIoError; - } - - const uint32_t ddSignature = *(reinterpret_cast<const uint32_t*>(ddBuf)); - const uint16_t ddOffset = (ddSignature == DataDescriptor::kOptSignature) ? 4 : 0; - const DataDescriptor* descriptor = reinterpret_cast<const DataDescriptor*>(ddBuf + ddOffset); - - // Validate that the values in the data descriptor match those in the central - // directory. - if (entry->compressed_length != descriptor->compressed_size || - entry->uncompressed_length != descriptor->uncompressed_size || - entry->crc32 != descriptor->crc32) { - ALOGW("Zip: size/crc32 mismatch. expected {%" PRIu32 ", %" PRIu32 ", %" PRIx32 - "}, was {%" PRIu32 ", %" PRIu32 ", %" PRIx32 "}", - entry->compressed_length, entry->uncompressed_length, entry->crc32, - descriptor->compressed_size, descriptor->uncompressed_size, descriptor->crc32); - return kInconsistentInformation; - } - - return 0; -} - -static int32_t FindEntry(const ZipArchive* archive, const int32_t ent, ZipEntry* data) { - const uint16_t nameLen = archive->hash_table[ent].name_length; - - // Recover the start of the central directory entry from the filename - // pointer. The filename is the first entry past the fixed-size data, - // so we can just subtract back from that. - const uint8_t* base_ptr = archive->central_directory.GetBasePtr(); - const uint8_t* ptr = base_ptr + archive->hash_table[ent].name_offset; - ptr -= sizeof(CentralDirectoryRecord); - - // This is the base of our mmapped region, we have to sanity check that - // the name that's in the hash table is a pointer to a location within - // this mapped region. - if (ptr < base_ptr || ptr > base_ptr + archive->central_directory.GetMapLength()) { - ALOGW("Zip: Invalid entry pointer"); - return kInvalidOffset; - } - - const CentralDirectoryRecord* cdr = reinterpret_cast<const CentralDirectoryRecord*>(ptr); - - // The offset of the start of the central directory in the zipfile. - // We keep this lying around so that we can sanity check all our lengths - // and our per-file structures. - const off64_t cd_offset = archive->directory_offset; - - // Fill out the compression method, modification time, crc32 - // and other interesting attributes from the central directory. These - // will later be compared against values from the local file header. - data->method = cdr->compression_method; - data->mod_time = cdr->last_mod_date << 16 | cdr->last_mod_time; - data->crc32 = cdr->crc32; - data->compressed_length = cdr->compressed_size; - data->uncompressed_length = cdr->uncompressed_size; - - // Figure out the local header offset from the central directory. The - // actual file data will begin after the local header and the name / - // extra comments. - const off64_t local_header_offset = cdr->local_file_header_offset; - if (local_header_offset + static_cast<off64_t>(sizeof(LocalFileHeader)) >= cd_offset) { - ALOGW("Zip: bad local hdr offset in zip"); - return kInvalidOffset; - } - - uint8_t lfh_buf[sizeof(LocalFileHeader)]; - if (!archive->mapped_zip.ReadAtOffset(lfh_buf, sizeof(lfh_buf), local_header_offset)) { - ALOGW("Zip: failed reading lfh name from offset %" PRId64, - static_cast<int64_t>(local_header_offset)); - return kIoError; - } - - const LocalFileHeader* lfh = reinterpret_cast<const LocalFileHeader*>(lfh_buf); - - if (lfh->lfh_signature != LocalFileHeader::kSignature) { - ALOGW("Zip: didn't find signature at start of lfh, offset=%" PRId64, - static_cast<int64_t>(local_header_offset)); - return kInvalidOffset; - } - - // Paranoia: Match the values specified in the local file header - // to those specified in the central directory. - - // Warn if central directory and local file header don't agree on the use - // of a trailing Data Descriptor. The reference implementation is inconsistent - // and appears to use the LFH value during extraction (unzip) but the CD value - // while displayng information about archives (zipinfo). The spec remains - // silent on this inconsistency as well. - // - // For now, always use the version from the LFH but make sure that the values - // specified in the central directory match those in the data descriptor. - // - // NOTE: It's also worth noting that unzip *does* warn about inconsistencies in - // bit 11 (EFS: The language encoding flag, marking that filename and comment are - // encoded using UTF-8). This implementation does not check for the presence of - // that flag and always enforces that entry names are valid UTF-8. - if ((lfh->gpb_flags & kGPBDDFlagMask) != (cdr->gpb_flags & kGPBDDFlagMask)) { - ALOGW("Zip: gpb flag mismatch at bit 3. expected {%04" PRIx16 "}, was {%04" PRIx16 "}", - cdr->gpb_flags, lfh->gpb_flags); - } - - // If there is no trailing data descriptor, verify that the central directory and local file - // header agree on the crc, compressed, and uncompressed sizes of the entry. - if ((lfh->gpb_flags & kGPBDDFlagMask) == 0) { - data->has_data_descriptor = 0; - if (data->compressed_length != lfh->compressed_size || - data->uncompressed_length != lfh->uncompressed_size || data->crc32 != lfh->crc32) { - ALOGW("Zip: size/crc32 mismatch. expected {%" PRIu32 ", %" PRIu32 ", %" PRIx32 - "}, was {%" PRIu32 ", %" PRIu32 ", %" PRIx32 "}", - data->compressed_length, data->uncompressed_length, data->crc32, lfh->compressed_size, - lfh->uncompressed_size, lfh->crc32); - return kInconsistentInformation; - } - } else { - data->has_data_descriptor = 1; - } - - // 4.4.2.1: the upper byte of `version_made_by` gives the source OS. Unix is 3. - data->version_made_by = cdr->version_made_by; - data->external_file_attributes = cdr->external_file_attributes; - if ((data->version_made_by >> 8) == 3) { - data->unix_mode = (cdr->external_file_attributes >> 16) & 0xffff; - } else { - data->unix_mode = 0777; - } - - // 4.4.4: general purpose bit flags. - data->gpbf = lfh->gpb_flags; - - // 4.4.14: the lowest bit of the internal file attributes field indicates text. - // Currently only needed to implement zipinfo. - data->is_text = (cdr->internal_file_attributes & 1); - - // Check that the local file header name matches the declared - // name in the central directory. - if (lfh->file_name_length != nameLen) { - ALOGW("Zip: lfh name length did not match central directory"); - return kInconsistentInformation; - } - const off64_t name_offset = local_header_offset + sizeof(LocalFileHeader); - if (name_offset + lfh->file_name_length > cd_offset) { - ALOGW("Zip: lfh name has invalid declared length"); - return kInvalidOffset; - } - std::vector<uint8_t> name_buf(nameLen); - if (!archive->mapped_zip.ReadAtOffset(name_buf.data(), nameLen, name_offset)) { - ALOGW("Zip: failed reading lfh name from offset %" PRId64, static_cast<int64_t>(name_offset)); - return kIoError; - } - const std::string_view entry_name = - archive->hash_table[ent].ToStringView(archive->central_directory.GetBasePtr()); - if (memcmp(entry_name.data(), name_buf.data(), nameLen) != 0) { - ALOGW("Zip: lfh name did not match central directory"); - return kInconsistentInformation; - } - - const off64_t data_offset = local_header_offset + sizeof(LocalFileHeader) + - lfh->file_name_length + lfh->extra_field_length; - if (data_offset > cd_offset) { - ALOGW("Zip: bad data offset %" PRId64 " in zip", static_cast<int64_t>(data_offset)); - return kInvalidOffset; - } - - if (static_cast<off64_t>(data_offset + data->compressed_length) > cd_offset) { - ALOGW("Zip: bad compressed length in zip (%" PRId64 " + %" PRIu32 " > %" PRId64 ")", - static_cast<int64_t>(data_offset), data->compressed_length, - static_cast<int64_t>(cd_offset)); - return kInvalidOffset; - } - - if (data->method == kCompressStored && - static_cast<off64_t>(data_offset + data->uncompressed_length) > cd_offset) { - ALOGW("Zip: bad uncompressed length in zip (%" PRId64 " + %" PRIu32 " > %" PRId64 ")", - static_cast<int64_t>(data_offset), data->uncompressed_length, - static_cast<int64_t>(cd_offset)); - return kInvalidOffset; - } - - data->offset = data_offset; - return 0; -} - -struct IterationHandle { - ZipArchive* archive; - - std::string prefix; - std::string suffix; - - uint32_t position = 0; - - IterationHandle(ZipArchive* archive, std::string_view in_prefix, std::string_view in_suffix) - : archive(archive), prefix(in_prefix), suffix(in_suffix) {} -}; - -int32_t StartIteration(ZipArchiveHandle archive, void** cookie_ptr, - const std::string_view optional_prefix, - const std::string_view optional_suffix) { - if (archive == NULL || archive->hash_table == NULL) { - ALOGW("Zip: Invalid ZipArchiveHandle"); - return kInvalidHandle; - } - - if (optional_prefix.size() > static_cast<size_t>(UINT16_MAX) || - optional_suffix.size() > static_cast<size_t>(UINT16_MAX)) { - ALOGW("Zip: prefix/suffix too long"); - return kInvalidEntryName; - } - - *cookie_ptr = new IterationHandle(archive, optional_prefix, optional_suffix); - return 0; -} - -void EndIteration(void* cookie) { - delete reinterpret_cast<IterationHandle*>(cookie); -} - -int32_t FindEntry(const ZipArchiveHandle archive, const std::string_view entryName, - ZipEntry* data) { - if (entryName.empty() || entryName.size() > static_cast<size_t>(UINT16_MAX)) { - ALOGW("Zip: Invalid filename of length %zu", entryName.size()); - return kInvalidEntryName; - } - - const int64_t ent = EntryToIndex(archive->hash_table, archive->hash_table_size, entryName, - archive->central_directory.GetBasePtr()); - if (ent < 0) { - ALOGV("Zip: Could not find entry %.*s", static_cast<int>(entryName.size()), entryName.data()); - return static_cast<int32_t>(ent); // kEntryNotFound is safe to truncate. - } - // We know there are at most hash_table_size entries, safe to truncate. - return FindEntry(archive, static_cast<uint32_t>(ent), data); -} - -int32_t Next(void* cookie, ZipEntry* data, std::string* name) { - std::string_view sv; - int32_t result = Next(cookie, data, &sv); - if (result == 0 && name) { - *name = std::string(sv); - } - return result; -} - -int32_t Next(void* cookie, ZipEntry* data, std::string_view* name) { - IterationHandle* handle = reinterpret_cast<IterationHandle*>(cookie); - if (handle == NULL) { - ALOGW("Zip: Null ZipArchiveHandle"); - return kInvalidHandle; - } - - ZipArchive* archive = handle->archive; - if (archive == NULL || archive->hash_table == NULL) { - ALOGW("Zip: Invalid ZipArchiveHandle"); - return kInvalidHandle; - } - - const uint32_t currentOffset = handle->position; - const uint32_t hash_table_length = archive->hash_table_size; - const ZipStringOffset* hash_table = archive->hash_table; - for (uint32_t i = currentOffset; i < hash_table_length; ++i) { - const std::string_view entry_name = - hash_table[i].ToStringView(archive->central_directory.GetBasePtr()); - if (hash_table[i].name_offset != 0 && (android::base::StartsWith(entry_name, handle->prefix) && - android::base::EndsWith(entry_name, handle->suffix))) { - handle->position = (i + 1); - const int error = FindEntry(archive, i, data); - if (!error && name) { - *name = entry_name; - } - return error; - } - } - - handle->position = 0; - return kIterationEnd; -} - -// A Writer that writes data to a fixed size memory region. -// The size of the memory region must be equal to the total size of -// the data appended to it. -class MemoryWriter : public zip_archive::Writer { - public: - MemoryWriter(uint8_t* buf, size_t size) : Writer(), buf_(buf), size_(size), bytes_written_(0) {} - - virtual bool Append(uint8_t* buf, size_t buf_size) override { - if (bytes_written_ + buf_size > size_) { - ALOGW("Zip: Unexpected size %zu (declared) vs %zu (actual)", size_, - bytes_written_ + buf_size); - return false; - } - - memcpy(buf_ + bytes_written_, buf, buf_size); - bytes_written_ += buf_size; - return true; - } - - private: - uint8_t* const buf_; - const size_t size_; - size_t bytes_written_; -}; - -// A Writer that appends data to a file |fd| at its current position. -// The file will be truncated to the end of the written data. -class FileWriter : public zip_archive::Writer { - public: - // Creates a FileWriter for |fd| and prepare to write |entry| to it, - // guaranteeing that the file descriptor is valid and that there's enough - // space on the volume to write out the entry completely and that the file - // is truncated to the correct length (no truncation if |fd| references a - // block device). - // - // Returns a valid FileWriter on success, |nullptr| if an error occurred. - static FileWriter Create(int fd, const ZipEntry* entry) { - const uint32_t declared_length = entry->uncompressed_length; - const off64_t current_offset = lseek64(fd, 0, SEEK_CUR); - if (current_offset == -1) { - ALOGW("Zip: unable to seek to current location on fd %d: %s", fd, strerror(errno)); - return FileWriter{}; - } - -#if defined(__linux__) - if (declared_length > 0) { - // Make sure we have enough space on the volume to extract the compressed - // entry. Note that the call to ftruncate below will change the file size but - // will not allocate space on disk and this call to fallocate will not - // change the file size. - // Note: fallocate is only supported by the following filesystems - - // btrfs, ext4, ocfs2, and xfs. Therefore fallocate might fail with - // EOPNOTSUPP error when issued in other filesystems. - // Hence, check for the return error code before concluding that the - // disk does not have enough space. - long result = TEMP_FAILURE_RETRY(fallocate(fd, 0, current_offset, declared_length)); - if (result == -1 && errno == ENOSPC) { - ALOGW("Zip: unable to allocate %" PRId64 " bytes at offset %" PRId64 ": %s", - static_cast<int64_t>(declared_length), static_cast<int64_t>(current_offset), - strerror(errno)); - return FileWriter{}; - } - } -#endif // __linux__ - - struct stat sb; - if (fstat(fd, &sb) == -1) { - ALOGW("Zip: unable to fstat file: %s", strerror(errno)); - return FileWriter{}; - } - - // Block device doesn't support ftruncate(2). - if (!S_ISBLK(sb.st_mode)) { - long result = TEMP_FAILURE_RETRY(ftruncate(fd, declared_length + current_offset)); - if (result == -1) { - ALOGW("Zip: unable to truncate file to %" PRId64 ": %s", - static_cast<int64_t>(declared_length + current_offset), strerror(errno)); - return FileWriter{}; - } - } - - return FileWriter(fd, declared_length); - } - - FileWriter(FileWriter&& other) noexcept - : fd_(other.fd_), - declared_length_(other.declared_length_), - total_bytes_written_(other.total_bytes_written_) { - other.fd_ = -1; - } - - bool IsValid() const { return fd_ != -1; } - - virtual bool Append(uint8_t* buf, size_t buf_size) override { - if (total_bytes_written_ + buf_size > declared_length_) { - ALOGW("Zip: Unexpected size %zu (declared) vs %zu (actual)", declared_length_, - total_bytes_written_ + buf_size); - return false; - } - - const bool result = android::base::WriteFully(fd_, buf, buf_size); - if (result) { - total_bytes_written_ += buf_size; - } else { - ALOGW("Zip: unable to write %zu bytes to file; %s", buf_size, strerror(errno)); - } - - return result; - } - - private: - explicit FileWriter(const int fd = -1, const size_t declared_length = 0) - : Writer(), fd_(fd), declared_length_(declared_length), total_bytes_written_(0) {} - - int fd_; - const size_t declared_length_; - size_t total_bytes_written_; -}; - -class EntryReader : public zip_archive::Reader { - public: - EntryReader(const MappedZipFile& zip_file, const ZipEntry* entry) - : Reader(), zip_file_(zip_file), entry_(entry) {} - - virtual bool ReadAtOffset(uint8_t* buf, size_t len, uint32_t offset) const { - return zip_file_.ReadAtOffset(buf, len, entry_->offset + offset); - } - - virtual ~EntryReader() {} - - private: - const MappedZipFile& zip_file_; - const ZipEntry* entry_; -}; - -// This method is using libz macros with old-style-casts -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wold-style-cast" -static inline int zlib_inflateInit2(z_stream* stream, int window_bits) { - return inflateInit2(stream, window_bits); -} -#pragma GCC diagnostic pop - -namespace zip_archive { - -// Moved out of line to avoid -Wweak-vtables. -Reader::~Reader() {} -Writer::~Writer() {} - -int32_t Inflate(const Reader& reader, const uint32_t compressed_length, - const uint32_t uncompressed_length, Writer* writer, uint64_t* crc_out) { - const size_t kBufSize = 32768; - std::vector<uint8_t> read_buf(kBufSize); - std::vector<uint8_t> write_buf(kBufSize); - z_stream zstream; - int zerr; - - /* - * Initialize the zlib stream struct. - */ - memset(&zstream, 0, sizeof(zstream)); - zstream.zalloc = Z_NULL; - zstream.zfree = Z_NULL; - zstream.opaque = Z_NULL; - zstream.next_in = NULL; - zstream.avail_in = 0; - zstream.next_out = &write_buf[0]; - zstream.avail_out = kBufSize; - zstream.data_type = Z_UNKNOWN; - - /* - * Use the undocumented "negative window bits" feature to tell zlib - * that there's no zlib header waiting for it. - */ - zerr = zlib_inflateInit2(&zstream, -MAX_WBITS); - if (zerr != Z_OK) { - if (zerr == Z_VERSION_ERROR) { - ALOGE("Installed zlib is not compatible with linked version (%s)", ZLIB_VERSION); - } else { - ALOGW("Call to inflateInit2 failed (zerr=%d)", zerr); - } - - return kZlibError; - } - - auto zstream_deleter = [](z_stream* stream) { - inflateEnd(stream); /* free up any allocated structures */ - }; - - std::unique_ptr<z_stream, decltype(zstream_deleter)> zstream_guard(&zstream, zstream_deleter); - - const bool compute_crc = (crc_out != nullptr); - uLong crc = 0; - uint32_t remaining_bytes = compressed_length; - do { - /* read as much as we can */ - if (zstream.avail_in == 0) { - const uint32_t read_size = (remaining_bytes > kBufSize) ? kBufSize : remaining_bytes; - const uint32_t offset = (compressed_length - remaining_bytes); - // Make sure to read at offset to ensure concurrent access to the fd. - if (!reader.ReadAtOffset(read_buf.data(), read_size, offset)) { - ALOGW("Zip: inflate read failed, getSize = %u: %s", read_size, strerror(errno)); - return kIoError; - } - - remaining_bytes -= read_size; - - zstream.next_in = &read_buf[0]; - zstream.avail_in = read_size; - } - - /* uncompress the data */ - zerr = inflate(&zstream, Z_NO_FLUSH); - if (zerr != Z_OK && zerr != Z_STREAM_END) { - ALOGW("Zip: inflate zerr=%d (nIn=%p aIn=%u nOut=%p aOut=%u)", zerr, zstream.next_in, - zstream.avail_in, zstream.next_out, zstream.avail_out); - return kZlibError; - } - - /* write when we're full or when we're done */ - if (zstream.avail_out == 0 || (zerr == Z_STREAM_END && zstream.avail_out != kBufSize)) { - const size_t write_size = zstream.next_out - &write_buf[0]; - if (!writer->Append(&write_buf[0], write_size)) { - return kIoError; - } else if (compute_crc) { - DCHECK_LE(write_size, kBufSize); - crc = crc32(crc, &write_buf[0], static_cast<uint32_t>(write_size)); - } - - zstream.next_out = &write_buf[0]; - zstream.avail_out = kBufSize; - } - } while (zerr == Z_OK); - - CHECK_EQ(zerr, Z_STREAM_END); /* other errors should've been caught */ - - // NOTE: zstream.adler is always set to 0, because we're using the -MAX_WBITS - // "feature" of zlib to tell it there won't be a zlib file header. zlib - // doesn't bother calculating the checksum in that scenario. We just do - // it ourselves above because there are no additional gains to be made by - // having zlib calculate it for us, since they do it by calling crc32 in - // the same manner that we have above. - if (compute_crc) { - *crc_out = crc; - } - - if (zstream.total_out != uncompressed_length || remaining_bytes != 0) { - ALOGW("Zip: size mismatch on inflated file (%lu vs %" PRIu32 ")", zstream.total_out, - uncompressed_length); - return kInconsistentInformation; - } - - return 0; -} -} // namespace zip_archive - -static int32_t InflateEntryToWriter(MappedZipFile& mapped_zip, const ZipEntry* entry, - zip_archive::Writer* writer, uint64_t* crc_out) { - const EntryReader reader(mapped_zip, entry); - - return zip_archive::Inflate(reader, entry->compressed_length, entry->uncompressed_length, writer, - crc_out); -} - -static int32_t CopyEntryToWriter(MappedZipFile& mapped_zip, const ZipEntry* entry, - zip_archive::Writer* writer, uint64_t* crc_out) { - static const uint32_t kBufSize = 32768; - std::vector<uint8_t> buf(kBufSize); - - const uint32_t length = entry->uncompressed_length; - uint32_t count = 0; - uLong crc = 0; - while (count < length) { - uint32_t remaining = length - count; - off64_t offset = entry->offset + count; - - // Safe conversion because kBufSize is narrow enough for a 32 bit signed value. - const uint32_t block_size = (remaining > kBufSize) ? kBufSize : remaining; - - // Make sure to read at offset to ensure concurrent access to the fd. - if (!mapped_zip.ReadAtOffset(buf.data(), block_size, offset)) { - ALOGW("CopyFileToFile: copy read failed, block_size = %u, offset = %" PRId64 ": %s", - block_size, static_cast<int64_t>(offset), strerror(errno)); - return kIoError; - } - - if (!writer->Append(&buf[0], block_size)) { - return kIoError; - } - if (crc_out) { - crc = crc32(crc, &buf[0], block_size); - } - count += block_size; - } - - if (crc_out) { - *crc_out = crc; - } - - return 0; -} - -int32_t ExtractToWriter(ZipArchiveHandle archive, ZipEntry* entry, zip_archive::Writer* writer) { - const uint16_t method = entry->method; - - // this should default to kUnknownCompressionMethod. - int32_t return_value = -1; - uint64_t crc = 0; - if (method == kCompressStored) { - return_value = - CopyEntryToWriter(archive->mapped_zip, entry, writer, kCrcChecksEnabled ? &crc : nullptr); - } else if (method == kCompressDeflated) { - return_value = InflateEntryToWriter(archive->mapped_zip, entry, writer, - kCrcChecksEnabled ? &crc : nullptr); - } - - if (!return_value && entry->has_data_descriptor) { - return_value = ValidateDataDescriptor(archive->mapped_zip, entry); - if (return_value) { - return return_value; - } - } - - // Validate that the CRC matches the calculated value. - if (kCrcChecksEnabled && (entry->crc32 != static_cast<uint32_t>(crc))) { - ALOGW("Zip: crc mismatch: expected %" PRIu32 ", was %" PRIu64, entry->crc32, crc); - return kInconsistentInformation; - } - - return return_value; -} - -int32_t ExtractToMemory(ZipArchiveHandle archive, ZipEntry* entry, uint8_t* begin, uint32_t size) { - MemoryWriter writer(begin, size); - return ExtractToWriter(archive, entry, &writer); -} - -int32_t ExtractEntryToFile(ZipArchiveHandle archive, ZipEntry* entry, int fd) { - auto writer = FileWriter::Create(fd, entry); - if (!writer.IsValid()) { - return kIoError; - } - - return ExtractToWriter(archive, entry, &writer); -} - -const char* ErrorCodeString(int32_t error_code) { - // Make sure that the number of entries in kErrorMessages and ErrorCodes - // match. - static_assert((-kLastErrorCode + 1) == arraysize(kErrorMessages), - "(-kLastErrorCode + 1) != arraysize(kErrorMessages)"); - - const uint32_t idx = -error_code; - if (idx < arraysize(kErrorMessages)) { - return kErrorMessages[idx]; - } - - return "Unknown return code"; -} - -int GetFileDescriptor(const ZipArchiveHandle archive) { - return archive->mapped_zip.GetFileDescriptor(); -} - -off64_t GetFileDescriptorOffset(const ZipArchiveHandle archive) { - return archive->mapped_zip.GetFileOffset(); -} - -#if !defined(_WIN32) -class ProcessWriter : public zip_archive::Writer { - public: - ProcessWriter(ProcessZipEntryFunction func, void* cookie) - : Writer(), proc_function_(func), cookie_(cookie) {} - - virtual bool Append(uint8_t* buf, size_t buf_size) override { - return proc_function_(buf, buf_size, cookie_); - } - - private: - ProcessZipEntryFunction proc_function_; - void* cookie_; -}; - -int32_t ProcessZipEntryContents(ZipArchiveHandle archive, ZipEntry* entry, - ProcessZipEntryFunction func, void* cookie) { - ProcessWriter writer(func, cookie); - return ExtractToWriter(archive, entry, &writer); -} - -#endif //! defined(_WIN32) - -int MappedZipFile::GetFileDescriptor() const { - if (!has_fd_) { - ALOGW("Zip: MappedZipFile doesn't have a file descriptor."); - return -1; - } - return fd_; -} - -const void* MappedZipFile::GetBasePtr() const { - if (has_fd_) { - ALOGW("Zip: MappedZipFile doesn't have a base pointer."); - return nullptr; - } - return base_ptr_; -} - -off64_t MappedZipFile::GetFileOffset() const { - return fd_offset_; -} - -off64_t MappedZipFile::GetFileLength() const { - if (has_fd_) { - if (data_length_ != -1) { - return data_length_; - } - data_length_ = lseek64(fd_, 0, SEEK_END); - if (data_length_ == -1) { - ALOGE("Zip: lseek on fd %d failed: %s", fd_, strerror(errno)); - } - return data_length_; - } else { - if (base_ptr_ == nullptr) { - ALOGE("Zip: invalid file map"); - return -1; - } - return data_length_; - } -} - -// Attempts to read |len| bytes into |buf| at offset |off|. -bool MappedZipFile::ReadAtOffset(uint8_t* buf, size_t len, off64_t off) const { - if (has_fd_) { - if (off < 0) { - ALOGE("Zip: invalid offset %" PRId64, off); - return false; - } - - off64_t read_offset; - if (__builtin_add_overflow(fd_offset_, off, &read_offset)) { - ALOGE("Zip: invalid read offset %" PRId64 " overflows, fd offset %" PRId64, off, fd_offset_); - return false; - } - - if (data_length_ != -1) { - off64_t read_end; - if (len > std::numeric_limits<off64_t>::max() || - __builtin_add_overflow(off, static_cast<off64_t>(len), &read_end)) { - ALOGE("Zip: invalid read length %" PRId64 " overflows, offset %" PRId64, - static_cast<off64_t>(len), off); - return false; - } - - if (read_end > data_length_) { - ALOGE("Zip: invalid read length %" PRId64 " exceeds data length %" PRId64 ", offset %" - PRId64, static_cast<off64_t>(len), data_length_, off); - return false; - } - } - - if (!android::base::ReadFullyAtOffset(fd_, buf, len, read_offset)) { - ALOGE("Zip: failed to read at offset %" PRId64, off); - return false; - } - } else { - if (off < 0 || off > data_length_) { - ALOGE("Zip: invalid offset: %" PRId64 ", data length: %" PRId64, off, data_length_); - return false; - } - memcpy(buf, static_cast<const uint8_t*>(base_ptr_) + off, len); - } - return true; -} - -void CentralDirectory::Initialize(const void* map_base_ptr, off64_t cd_start_offset, - size_t cd_size) { - base_ptr_ = static_cast<const uint8_t*>(map_base_ptr) + cd_start_offset; - length_ = cd_size; -} - -bool ZipArchive::InitializeCentralDirectory(off64_t cd_start_offset, size_t cd_size) { - if (mapped_zip.HasFd()) { - directory_map = android::base::MappedFile::FromFd(mapped_zip.GetFileDescriptor(), - mapped_zip.GetFileOffset() + cd_start_offset, - cd_size, PROT_READ); - if (!directory_map) { - ALOGE("Zip: failed to map central directory (offset %" PRId64 ", size %zu): %s", - cd_start_offset, cd_size, strerror(errno)); - return false; - } - - CHECK_EQ(directory_map->size(), cd_size); - central_directory.Initialize(directory_map->data(), 0 /*offset*/, cd_size); - } else { - if (mapped_zip.GetBasePtr() == nullptr) { - ALOGE("Zip: Failed to map central directory, bad mapped_zip base pointer"); - return false; - } - if (static_cast<off64_t>(cd_start_offset) + static_cast<off64_t>(cd_size) > - mapped_zip.GetFileLength()) { - ALOGE( - "Zip: Failed to map central directory, offset exceeds mapped memory region (" - "start_offset %" PRId64 ", cd_size %zu, mapped_region_size %" PRId64 ")", - static_cast<int64_t>(cd_start_offset), cd_size, mapped_zip.GetFileLength()); - return false; - } - - central_directory.Initialize(mapped_zip.GetBasePtr(), cd_start_offset, cd_size); - } - return true; -} - -// This function returns the embedded timestamp as is; and doesn't perform validations. -tm ZipEntry::GetModificationTime() const { - tm t = {}; - - t.tm_hour = (mod_time >> 11) & 0x1f; - t.tm_min = (mod_time >> 5) & 0x3f; - t.tm_sec = (mod_time & 0x1f) << 1; - - t.tm_year = ((mod_time >> 25) & 0x7f) + 80; - t.tm_mon = ((mod_time >> 21) & 0xf) - 1; - t.tm_mday = (mod_time >> 16) & 0x1f; - - return t; -} diff --git a/libziparchive/zip_archive_benchmark.cpp b/libziparchive/zip_archive_benchmark.cpp deleted file mode 100644 index cfa5912a2..000000000 --- a/libziparchive/zip_archive_benchmark.cpp +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright (C) 2017 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <cstdio> -#include <cstdlib> -#include <cstring> -#include <string> -#include <tuple> -#include <vector> - -#include <android-base/test_utils.h> -#include <benchmark/benchmark.h> -#include <ziparchive/zip_archive.h> -#include <ziparchive/zip_archive_stream_entry.h> -#include <ziparchive/zip_writer.h> - -static std::unique_ptr<TemporaryFile> CreateZip(int size = 4, int count = 1000) { - auto result = std::make_unique<TemporaryFile>(); - FILE* fp = fdopen(result->fd, "w"); - - ZipWriter writer(fp); - std::string lastName = "file"; - for (size_t i = 0; i < count; i++) { - // Make file names longer and longer. - lastName = lastName + std::to_string(i); - writer.StartEntry(lastName.c_str(), ZipWriter::kCompress); - while (size > 0) { - writer.WriteBytes("helo", 4); - size -= 4; - } - writer.FinishEntry(); - } - writer.Finish(); - fclose(fp); - - return result; -} - -static void FindEntry_no_match(benchmark::State& state) { - // Create a temporary zip archive. - std::unique_ptr<TemporaryFile> temp_file(CreateZip()); - ZipArchiveHandle handle; - ZipEntry data; - - // In order to walk through all file names in the archive, look for a name - // that does not exist in the archive. - std::string_view name("thisFileNameDoesNotExist"); - - // Start the benchmark. - for (auto _ : state) { - OpenArchive(temp_file->path, &handle); - FindEntry(handle, name, &data); - CloseArchive(handle); - } -} -BENCHMARK(FindEntry_no_match); - -static void Iterate_all_files(benchmark::State& state) { - std::unique_ptr<TemporaryFile> temp_file(CreateZip()); - ZipArchiveHandle handle; - void* iteration_cookie; - ZipEntry data; - std::string name; - - for (auto _ : state) { - OpenArchive(temp_file->path, &handle); - StartIteration(handle, &iteration_cookie); - while (Next(iteration_cookie, &data, &name) == 0) { - } - EndIteration(iteration_cookie); - CloseArchive(handle); - } -} -BENCHMARK(Iterate_all_files); - -static void StartAlignedEntry(benchmark::State& state) { - TemporaryFile file; - FILE* fp = fdopen(file.fd, "w"); - - ZipWriter writer(fp); - - auto alignment = uint32_t(state.range(0)); - std::string name = "name"; - int counter = 0; - for (auto _ : state) { - writer.StartAlignedEntry(name + std::to_string(counter++), 0, alignment); - state.PauseTiming(); - writer.WriteBytes("hola", 4); - writer.FinishEntry(); - state.ResumeTiming(); - } - - writer.Finish(); - fclose(fp); -} -BENCHMARK(StartAlignedEntry)->Arg(2)->Arg(16)->Arg(1024)->Arg(4096); - -static void ExtractEntry(benchmark::State& state) { - std::unique_ptr<TemporaryFile> temp_file(CreateZip(1024 * 1024, 1)); - - ZipArchiveHandle handle; - ZipEntry data; - if (OpenArchive(temp_file->path, &handle)) { - state.SkipWithError("Failed to open archive"); - } - if (FindEntry(handle, "file0", &data)) { - state.SkipWithError("Failed to find archive entry"); - } - - std::vector<uint8_t> buffer(1024 * 1024); - for (auto _ : state) { - if (ExtractToMemory(handle, &data, buffer.data(), uint32_t(buffer.size()))) { - state.SkipWithError("Failed to extract archive entry"); - break; - } - } - CloseArchive(handle); -} - -BENCHMARK(ExtractEntry)->Arg(2)->Arg(16)->Arg(1024); - -BENCHMARK_MAIN(); diff --git a/libziparchive/zip_archive_common.h b/libziparchive/zip_archive_common.h deleted file mode 100644 index 8b99bde91..000000000 --- a/libziparchive/zip_archive_common.h +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef LIBZIPARCHIVE_ZIPARCHIVECOMMON_H_ -#define LIBZIPARCHIVE_ZIPARCHIVECOMMON_H_ - -#include "android-base/macros.h" - -#include <inttypes.h> - -// The "end of central directory" (EOCD) record. Each archive -// contains exactly once such record which appears at the end of -// the archive. It contains archive wide information like the -// number of entries in the archive and the offset to the central -// directory of the offset. -struct EocdRecord { - static const uint32_t kSignature = 0x06054b50; - - // End of central directory signature, should always be - // |kSignature|. - uint32_t eocd_signature; - // The number of the current "disk", i.e, the "disk" that this - // central directory is on. - // - // This implementation assumes that each archive spans a single - // disk only. i.e, that disk_num == 1. - uint16_t disk_num; - // The disk where the central directory starts. - // - // This implementation assumes that each archive spans a single - // disk only. i.e, that cd_start_disk == 1. - uint16_t cd_start_disk; - // The number of central directory records on this disk. - // - // This implementation assumes that each archive spans a single - // disk only. i.e, that num_records_on_disk == num_records. - uint16_t num_records_on_disk; - // The total number of central directory records. - uint16_t num_records; - // The size of the central directory (in bytes). - uint32_t cd_size; - // The offset of the start of the central directory, relative - // to the start of the file. - uint32_t cd_start_offset; - // Length of the central directory comment. - uint16_t comment_length; - - private: - EocdRecord() = default; - DISALLOW_COPY_AND_ASSIGN(EocdRecord); -} __attribute__((packed)); - -// A structure representing the fixed length fields for a single -// record in the central directory of the archive. In addition to -// the fixed length fields listed here, each central directory -// record contains a variable length "file_name" and "extra_field" -// whose lengths are given by |file_name_length| and |extra_field_length| -// respectively. -struct CentralDirectoryRecord { - static const uint32_t kSignature = 0x02014b50; - - // The start of record signature. Must be |kSignature|. - uint32_t record_signature; - // Source tool version. Top byte gives source OS. - uint16_t version_made_by; - // Tool version. Ignored by this implementation. - uint16_t version_needed; - // The "general purpose bit flags" for this entry. The only - // flag value that we currently check for is the "data descriptor" - // flag. - uint16_t gpb_flags; - // The compression method for this entry, one of |kCompressStored| - // and |kCompressDeflated|. - uint16_t compression_method; - // The file modification time and date for this entry. - uint16_t last_mod_time; - uint16_t last_mod_date; - // The CRC-32 checksum for this entry. - uint32_t crc32; - // The compressed size (in bytes) of this entry. - uint32_t compressed_size; - // The uncompressed size (in bytes) of this entry. - uint32_t uncompressed_size; - // The length of the entry file name in bytes. The file name - // will appear immediately after this record. - uint16_t file_name_length; - // The length of the extra field info (in bytes). This data - // will appear immediately after the entry file name. - uint16_t extra_field_length; - // The length of the entry comment (in bytes). This data will - // appear immediately after the extra field. - uint16_t comment_length; - // The start disk for this entry. Ignored by this implementation). - uint16_t file_start_disk; - // File attributes. Ignored by this implementation. - uint16_t internal_file_attributes; - // File attributes. For archives created on Unix, the top bits are the mode. - uint32_t external_file_attributes; - // The offset to the local file header for this entry, from the - // beginning of this archive. - uint32_t local_file_header_offset; - - private: - CentralDirectoryRecord() = default; - DISALLOW_COPY_AND_ASSIGN(CentralDirectoryRecord); -} __attribute__((packed)); - -// The local file header for a given entry. This duplicates information -// present in the central directory of the archive. It is an error for -// the information here to be different from the central directory -// information for a given entry. -struct LocalFileHeader { - static const uint32_t kSignature = 0x04034b50; - - // The local file header signature, must be |kSignature|. - uint32_t lfh_signature; - // Tool version. Ignored by this implementation. - uint16_t version_needed; - // The "general purpose bit flags" for this entry. The only - // flag value that we currently check for is the "data descriptor" - // flag. - uint16_t gpb_flags; - // The compression method for this entry, one of |kCompressStored| - // and |kCompressDeflated|. - uint16_t compression_method; - // The file modification time and date for this entry. - uint16_t last_mod_time; - uint16_t last_mod_date; - // The CRC-32 checksum for this entry. - uint32_t crc32; - // The compressed size (in bytes) of this entry. - uint32_t compressed_size; - // The uncompressed size (in bytes) of this entry. - uint32_t uncompressed_size; - // The length of the entry file name in bytes. The file name - // will appear immediately after this record. - uint16_t file_name_length; - // The length of the extra field info (in bytes). This data - // will appear immediately after the entry file name. - uint16_t extra_field_length; - - private: - LocalFileHeader() = default; - DISALLOW_COPY_AND_ASSIGN(LocalFileHeader); -} __attribute__((packed)); - -struct DataDescriptor { - // The *optional* data descriptor start signature. - static const uint32_t kOptSignature = 0x08074b50; - - // CRC-32 checksum of the entry. - uint32_t crc32; - // Compressed size of the entry. - uint32_t compressed_size; - // Uncompressed size of the entry. - uint32_t uncompressed_size; - - private: - DataDescriptor() = default; - DISALLOW_COPY_AND_ASSIGN(DataDescriptor); -} __attribute__((packed)); - -// mask value that signifies that the entry has a DD -static const uint32_t kGPBDDFlagMask = 0x0008; - -// The maximum size of a central directory or a file -// comment in bytes. -static const uint32_t kMaxCommentLen = 65535; - -#endif /* LIBZIPARCHIVE_ZIPARCHIVECOMMON_H_ */ diff --git a/libziparchive/zip_archive_private.h b/libziparchive/zip_archive_private.h deleted file mode 100644 index 362503816..000000000 --- a/libziparchive/zip_archive_private.h +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Copyright (C) 2008 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include <ziparchive/zip_archive.h> - -#include <stdint.h> -#include <stdlib.h> -#include <unistd.h> - -#include <memory> -#include <vector> - -#include "android-base/macros.h" -#include "android-base/mapped_file.h" - -static const char* kErrorMessages[] = { - "Success", - "Iteration ended", - "Zlib error", - "Invalid file", - "Invalid handle", - "Duplicate entries in archive", - "Empty archive", - "Entry not found", - "Invalid offset", - "Inconsistent information", - "Invalid entry name", - "I/O error", - "File mapping failed", - "Allocation failed", -}; - -enum ErrorCodes : int32_t { - kIterationEnd = -1, - - // We encountered a Zlib error when inflating a stream from this file. - // Usually indicates file corruption. - kZlibError = -2, - - // The input file cannot be processed as a zip archive. Usually because - // it's too small, too large or does not have a valid signature. - kInvalidFile = -3, - - // An invalid iteration / ziparchive handle was passed in as an input - // argument. - kInvalidHandle = -4, - - // The zip archive contained two (or possibly more) entries with the same - // name. - kDuplicateEntry = -5, - - // The zip archive contains no entries. - kEmptyArchive = -6, - - // The specified entry was not found in the archive. - kEntryNotFound = -7, - - // The zip archive contained an invalid local file header pointer. - kInvalidOffset = -8, - - // The zip archive contained inconsistent entry information. This could - // be because the central directory & local file header did not agree, or - // if the actual uncompressed length or crc32 do not match their declared - // values. - kInconsistentInformation = -9, - - // An invalid entry name was encountered. - kInvalidEntryName = -10, - - // An I/O related system call (read, lseek, ftruncate, map) failed. - kIoError = -11, - - // We were not able to mmap the central directory or entry contents. - kMmapFailed = -12, - - // An allocation failed. - kAllocationFailed = -13, - - kLastErrorCode = kAllocationFailed, -}; - -class MappedZipFile { - public: - explicit MappedZipFile(const int fd) - : has_fd_(true), fd_(fd), fd_offset_(0), base_ptr_(nullptr), data_length_(-1) {} - - explicit MappedZipFile(const int fd, off64_t length, off64_t offset) - : has_fd_(true), fd_(fd), fd_offset_(offset), base_ptr_(nullptr), data_length_(length) {} - - explicit MappedZipFile(const void* address, size_t length) - : has_fd_(false), fd_(-1), fd_offset_(0), base_ptr_(address), - data_length_(static_cast<off64_t>(length)) {} - - bool HasFd() const { return has_fd_; } - - int GetFileDescriptor() const; - - const void* GetBasePtr() const; - - off64_t GetFileOffset() const; - - off64_t GetFileLength() const; - - bool ReadAtOffset(uint8_t* buf, size_t len, off64_t off) const; - - private: - // If has_fd_ is true, fd is valid and we'll read contents of a zip archive - // from the file. Otherwise, we're opening the archive from a memory mapped - // file. In that case, base_ptr_ points to the start of the memory region and - // data_length_ defines the file length. - const bool has_fd_; - - const int fd_; - const off64_t fd_offset_; - - const void* const base_ptr_; - mutable off64_t data_length_; -}; - -class CentralDirectory { - public: - CentralDirectory(void) : base_ptr_(nullptr), length_(0) {} - - const uint8_t* GetBasePtr() const { return base_ptr_; } - - size_t GetMapLength() const { return length_; } - - void Initialize(const void* map_base_ptr, off64_t cd_start_offset, size_t cd_size); - - private: - const uint8_t* base_ptr_; - size_t length_; -}; - -/** - * More space efficient string representation of strings in an mmaped zipped - * file than std::string_view. Using std::string_view as an entry in the - * ZipArchive hash table wastes space. std::string_view stores a pointer to a - * string (on 64 bit, 8 bytes) and the length to read from that pointer, - * 2 bytes. Because of alignment, the structure consumes 16 bytes, wasting - * 6 bytes. - * - * ZipStringOffset stores a 4 byte offset from a fixed location in the memory - * mapped file instead of the entire address, consuming 8 bytes with alignment. - */ -struct ZipStringOffset { - uint32_t name_offset; - uint16_t name_length; - - const std::string_view ToStringView(const uint8_t* start) const { - return std::string_view{reinterpret_cast<const char*>(start + name_offset), name_length}; - } -}; - -struct ZipArchive { - // open Zip archive - mutable MappedZipFile mapped_zip; - const bool close_file; - - // mapped central directory area - off64_t directory_offset; - CentralDirectory central_directory; - std::unique_ptr<android::base::MappedFile> directory_map; - - // number of entries in the Zip archive - uint16_t num_entries; - - // We know how many entries are in the Zip archive, so we can have a - // fixed-size hash table. We define a load factor of 0.75 and over - // allocate so the maximum number entries can never be higher than - // ((4 * UINT16_MAX) / 3 + 1) which can safely fit into a uint32_t. - uint32_t hash_table_size; - ZipStringOffset* hash_table; - - ZipArchive(MappedZipFile&& map, bool assume_ownership); - ZipArchive(const void* address, size_t length); - ~ZipArchive(); - - bool InitializeCentralDirectory(off64_t cd_start_offset, size_t cd_size); -}; diff --git a/libziparchive/zip_archive_stream_entry.cc b/libziparchive/zip_archive_stream_entry.cc deleted file mode 100644 index 1ec95b6eb..000000000 --- a/libziparchive/zip_archive_stream_entry.cc +++ /dev/null @@ -1,323 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define LOG_TAG "ZIPARCHIVE" - -// Read-only stream access to Zip Archive entries. -#include <errno.h> -#include <inttypes.h> -#include <string.h> -#include <sys/types.h> -#include <unistd.h> - -#include <memory> -#include <vector> - -#include <android-base/file.h> -#include <android-base/logging.h> -#include <log/log.h> - -#include <ziparchive/zip_archive.h> -#include <ziparchive/zip_archive_stream_entry.h> -#include <zlib.h> - -#include "zip_archive_private.h" - -static constexpr size_t kBufSize = 65535; - -bool ZipArchiveStreamEntry::Init(const ZipEntry& entry) { - crc32_ = entry.crc32; - offset_ = entry.offset; - return true; -} - -class ZipArchiveStreamEntryUncompressed : public ZipArchiveStreamEntry { - public: - explicit ZipArchiveStreamEntryUncompressed(ZipArchiveHandle handle) - : ZipArchiveStreamEntry(handle) {} - virtual ~ZipArchiveStreamEntryUncompressed() {} - - const std::vector<uint8_t>* Read() override; - - bool Verify() override; - - protected: - bool Init(const ZipEntry& entry) override; - - uint32_t length_ = 0u; - - private: - std::vector<uint8_t> data_; - uint32_t computed_crc32_ = 0u; -}; - -bool ZipArchiveStreamEntryUncompressed::Init(const ZipEntry& entry) { - if (!ZipArchiveStreamEntry::Init(entry)) { - return false; - } - - length_ = entry.uncompressed_length; - - data_.resize(kBufSize); - computed_crc32_ = 0; - - return true; -} - -const std::vector<uint8_t>* ZipArchiveStreamEntryUncompressed::Read() { - // Simple sanity check. The vector should *only* be handled by this code. A caller - // should not const-cast and modify the capacity. This may invalidate next_out. - // - // Note: it would be better to store the results of data() across Read calls. - CHECK_EQ(data_.capacity(), kBufSize); - - if (length_ == 0) { - return nullptr; - } - - size_t bytes = (length_ > data_.size()) ? data_.size() : length_; - ZipArchive* archive = reinterpret_cast<ZipArchive*>(handle_); - errno = 0; - if (!archive->mapped_zip.ReadAtOffset(data_.data(), bytes, offset_)) { - if (errno != 0) { - ALOGE("Error reading from archive fd: %s", strerror(errno)); - } else { - ALOGE("Short read of zip file, possibly corrupted zip?"); - } - length_ = 0; - return nullptr; - } - - if (bytes < data_.size()) { - data_.resize(bytes); - } - computed_crc32_ = static_cast<uint32_t>( - crc32(computed_crc32_, data_.data(), static_cast<uint32_t>(data_.size()))); - length_ -= bytes; - offset_ += bytes; - return &data_; -} - -bool ZipArchiveStreamEntryUncompressed::Verify() { - return length_ == 0 && crc32_ == computed_crc32_; -} - -class ZipArchiveStreamEntryCompressed : public ZipArchiveStreamEntry { - public: - explicit ZipArchiveStreamEntryCompressed(ZipArchiveHandle handle) - : ZipArchiveStreamEntry(handle) {} - virtual ~ZipArchiveStreamEntryCompressed(); - - const std::vector<uint8_t>* Read() override; - - bool Verify() override; - - protected: - bool Init(const ZipEntry& entry) override; - - private: - bool z_stream_init_ = false; - z_stream z_stream_; - std::vector<uint8_t> in_; - std::vector<uint8_t> out_; - uint32_t uncompressed_length_ = 0u; - uint32_t compressed_length_ = 0u; - uint32_t computed_crc32_ = 0u; -}; - -// This method is using libz macros with old-style-casts -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wold-style-cast" -static inline int zlib_inflateInit2(z_stream* stream, int window_bits) { - return inflateInit2(stream, window_bits); -} -#pragma GCC diagnostic pop - -bool ZipArchiveStreamEntryCompressed::Init(const ZipEntry& entry) { - if (!ZipArchiveStreamEntry::Init(entry)) { - return false; - } - - // Initialize the zlib stream struct. - memset(&z_stream_, 0, sizeof(z_stream_)); - z_stream_.zalloc = Z_NULL; - z_stream_.zfree = Z_NULL; - z_stream_.opaque = Z_NULL; - z_stream_.next_in = nullptr; - z_stream_.avail_in = 0; - z_stream_.avail_out = 0; - z_stream_.data_type = Z_UNKNOWN; - - // Use the undocumented "negative window bits" feature to tell zlib - // that there's no zlib header waiting for it. - int zerr = zlib_inflateInit2(&z_stream_, -MAX_WBITS); - if (zerr != Z_OK) { - if (zerr == Z_VERSION_ERROR) { - ALOGE("Installed zlib is not compatible with linked version (%s)", ZLIB_VERSION); - } else { - ALOGE("Call to inflateInit2 failed (zerr=%d)", zerr); - } - - return false; - } - - z_stream_init_ = true; - - uncompressed_length_ = entry.uncompressed_length; - compressed_length_ = entry.compressed_length; - - out_.resize(kBufSize); - in_.resize(kBufSize); - - computed_crc32_ = 0; - - return true; -} - -ZipArchiveStreamEntryCompressed::~ZipArchiveStreamEntryCompressed() { - if (z_stream_init_) { - inflateEnd(&z_stream_); - z_stream_init_ = false; - } -} - -bool ZipArchiveStreamEntryCompressed::Verify() { - return z_stream_init_ && uncompressed_length_ == 0 && compressed_length_ == 0 && - crc32_ == computed_crc32_; -} - -const std::vector<uint8_t>* ZipArchiveStreamEntryCompressed::Read() { - // Simple sanity check. The vector should *only* be handled by this code. A caller - // should not const-cast and modify the capacity. This may invalidate next_out. - // - // Note: it would be better to store the results of data() across Read calls. - CHECK_EQ(out_.capacity(), kBufSize); - - if (z_stream_.avail_out == 0) { - z_stream_.next_out = out_.data(); - z_stream_.avail_out = static_cast<uint32_t>(out_.size()); - ; - } - - while (true) { - if (z_stream_.avail_in == 0) { - if (compressed_length_ == 0) { - return nullptr; - } - DCHECK_LE(in_.size(), std::numeric_limits<uint32_t>::max()); // Should be buf size = 64k. - uint32_t bytes = (compressed_length_ > in_.size()) ? static_cast<uint32_t>(in_.size()) - : compressed_length_; - ZipArchive* archive = reinterpret_cast<ZipArchive*>(handle_); - errno = 0; - if (!archive->mapped_zip.ReadAtOffset(in_.data(), bytes, offset_)) { - if (errno != 0) { - ALOGE("Error reading from archive fd: %s", strerror(errno)); - } else { - ALOGE("Short read of zip file, possibly corrupted zip?"); - } - return nullptr; - } - - compressed_length_ -= bytes; - offset_ += bytes; - z_stream_.next_in = in_.data(); - z_stream_.avail_in = bytes; - } - - int zerr = inflate(&z_stream_, Z_NO_FLUSH); - if (zerr != Z_OK && zerr != Z_STREAM_END) { - ALOGE("inflate zerr=%d (nIn=%p aIn=%u nOut=%p aOut=%u)", zerr, z_stream_.next_in, - z_stream_.avail_in, z_stream_.next_out, z_stream_.avail_out); - return nullptr; - } - - if (z_stream_.avail_out == 0) { - uncompressed_length_ -= out_.size(); - computed_crc32_ = static_cast<uint32_t>( - crc32(computed_crc32_, out_.data(), static_cast<uint32_t>(out_.size()))); - return &out_; - } - if (zerr == Z_STREAM_END) { - if (z_stream_.avail_out != 0) { - // Resize the vector down to the actual size of the data. - out_.resize(out_.size() - z_stream_.avail_out); - computed_crc32_ = static_cast<uint32_t>( - crc32(computed_crc32_, out_.data(), static_cast<uint32_t>(out_.size()))); - uncompressed_length_ -= out_.size(); - return &out_; - } - return nullptr; - } - } - return nullptr; -} - -class ZipArchiveStreamEntryRawCompressed : public ZipArchiveStreamEntryUncompressed { - public: - explicit ZipArchiveStreamEntryRawCompressed(ZipArchiveHandle handle) - : ZipArchiveStreamEntryUncompressed(handle) {} - virtual ~ZipArchiveStreamEntryRawCompressed() {} - - bool Verify() override; - - protected: - bool Init(const ZipEntry& entry) override; -}; - -bool ZipArchiveStreamEntryRawCompressed::Init(const ZipEntry& entry) { - if (!ZipArchiveStreamEntryUncompressed::Init(entry)) { - return false; - } - length_ = entry.compressed_length; - - return true; -} - -bool ZipArchiveStreamEntryRawCompressed::Verify() { - return length_ == 0; -} - -ZipArchiveStreamEntry* ZipArchiveStreamEntry::Create(ZipArchiveHandle handle, - const ZipEntry& entry) { - ZipArchiveStreamEntry* stream = nullptr; - if (entry.method != kCompressStored) { - stream = new ZipArchiveStreamEntryCompressed(handle); - } else { - stream = new ZipArchiveStreamEntryUncompressed(handle); - } - if (stream && !stream->Init(entry)) { - delete stream; - stream = nullptr; - } - - return stream; -} - -ZipArchiveStreamEntry* ZipArchiveStreamEntry::CreateRaw(ZipArchiveHandle handle, - const ZipEntry& entry) { - ZipArchiveStreamEntry* stream = nullptr; - if (entry.method == kCompressStored) { - // Not compressed, don't need to do anything special. - stream = new ZipArchiveStreamEntryUncompressed(handle); - } else { - stream = new ZipArchiveStreamEntryRawCompressed(handle); - } - if (stream && !stream->Init(entry)) { - delete stream; - stream = nullptr; - } - return stream; -} diff --git a/libziparchive/zip_archive_test.cc b/libziparchive/zip_archive_test.cc deleted file mode 100644 index 35fb3fe0f..000000000 --- a/libziparchive/zip_archive_test.cc +++ /dev/null @@ -1,859 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "zip_archive_private.h" - -#include <errno.h> -#include <fcntl.h> -#include <getopt.h> -#include <stdio.h> -#include <string.h> -#include <unistd.h> - -#include <memory> -#include <vector> - -#include <android-base/file.h> -#include <android-base/logging.h> -#include <android-base/mapped_file.h> -#include <android-base/unique_fd.h> -#include <gtest/gtest.h> -#include <ziparchive/zip_archive.h> -#include <ziparchive/zip_archive_stream_entry.h> - -static std::string test_data_dir = android::base::GetExecutableDirectory() + "/testdata"; - -static const std::string kValidZip = "valid.zip"; -static const std::string kLargeZip = "large.zip"; -static const std::string kBadCrcZip = "bad_crc.zip"; - -static const std::vector<uint8_t> kATxtContents{'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'a', - 'b', 'c', 'd', 'e', 'f', 'g', 'h', '\n'}; - -static const std::vector<uint8_t> kATxtContentsCompressed{'K', 'L', 'J', 'N', 'I', 'M', 'K', - 207, 'H', 132, 210, '\\', '\0'}; - -static const std::vector<uint8_t> kBTxtContents{'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', '\n'}; - -static int32_t OpenArchiveWrapper(const std::string& name, ZipArchiveHandle* handle) { - const std::string abs_path = test_data_dir + "/" + name; - return OpenArchive(abs_path.c_str(), handle); -} - -TEST(ziparchive, Open) { - ZipArchiveHandle handle; - ASSERT_EQ(0, OpenArchiveWrapper(kValidZip, &handle)); - CloseArchive(handle); - - ASSERT_EQ(kInvalidEntryName, OpenArchiveWrapper("bad_filename.zip", &handle)); - CloseArchive(handle); -} - -TEST(ziparchive, OutOfBound) { - ZipArchiveHandle handle; - ASSERT_EQ(kInvalidOffset, OpenArchiveWrapper("crash.apk", &handle)); - CloseArchive(handle); -} - -TEST(ziparchive, EmptyArchive) { - ZipArchiveHandle handle; - ASSERT_EQ(kEmptyArchive, OpenArchiveWrapper("empty.zip", &handle)); - CloseArchive(handle); -} - -TEST(ziparchive, ZeroSizeCentralDirectory) { - ZipArchiveHandle handle; - ASSERT_EQ(kInvalidFile, OpenArchiveWrapper("zero-size-cd.zip", &handle)); - CloseArchive(handle); -} - -TEST(ziparchive, OpenMissing) { - ZipArchiveHandle handle; - ASSERT_NE(0, OpenArchiveWrapper("missing.zip", &handle)); - - // Confirm the file descriptor is not going to be mistaken for a valid one. - ASSERT_EQ(-1, GetFileDescriptor(handle)); -} - -TEST(ziparchive, OpenAssumeFdOwnership) { - int fd = open((test_data_dir + "/" + kValidZip).c_str(), O_RDONLY | O_BINARY); - ASSERT_NE(-1, fd); - ZipArchiveHandle handle; - ASSERT_EQ(0, OpenArchiveFd(fd, "OpenWithAssumeFdOwnership", &handle)); - CloseArchive(handle); - ASSERT_EQ(-1, lseek(fd, 0, SEEK_SET)); - ASSERT_EQ(EBADF, errno); -} - -TEST(ziparchive, OpenDoNotAssumeFdOwnership) { - int fd = open((test_data_dir + "/" + kValidZip).c_str(), O_RDONLY | O_BINARY); - ASSERT_NE(-1, fd); - ZipArchiveHandle handle; - ASSERT_EQ(0, OpenArchiveFd(fd, "OpenWithAssumeFdOwnership", &handle, false)); - CloseArchive(handle); - ASSERT_EQ(0, lseek(fd, 0, SEEK_SET)); - close(fd); -} - -TEST(ziparchive, OpenAssumeFdRangeOwnership) { - int fd = open((test_data_dir + "/" + kValidZip).c_str(), O_RDONLY | O_BINARY); - ASSERT_NE(-1, fd); - const off64_t length = lseek64(fd, 0, SEEK_END); - ASSERT_NE(-1, length); - ZipArchiveHandle handle; - ASSERT_EQ(0, OpenArchiveFdRange(fd, "OpenWithAssumeFdOwnership", &handle, - static_cast<size_t>(length), 0)); - CloseArchive(handle); - ASSERT_EQ(-1, lseek(fd, 0, SEEK_SET)); - ASSERT_EQ(EBADF, errno); -} - -TEST(ziparchive, OpenDoNotAssumeFdRangeOwnership) { - int fd = open((test_data_dir + "/" + kValidZip).c_str(), O_RDONLY | O_BINARY); - ASSERT_NE(-1, fd); - const off64_t length = lseek(fd, 0, SEEK_END); - ASSERT_NE(-1, length); - ZipArchiveHandle handle; - ASSERT_EQ(0, OpenArchiveFdRange(fd, "OpenWithAssumeFdOwnership", &handle, - static_cast<size_t>(length), 0, false)); - CloseArchive(handle); - ASSERT_EQ(0, lseek(fd, 0, SEEK_SET)); - close(fd); -} - -TEST(ziparchive, Iteration_std_string_view) { - ZipArchiveHandle handle; - ASSERT_EQ(0, OpenArchiveWrapper(kValidZip, &handle)); - - void* iteration_cookie; - ASSERT_EQ(0, StartIteration(handle, &iteration_cookie)); - - ZipEntry data; - std::vector<std::string_view> names; - std::string_view name; - while (Next(iteration_cookie, &data, &name) == 0) names.push_back(name); - - // Assert that the names are as expected. - std::vector<std::string_view> expected_names{"a.txt", "b.txt", "b/", "b/c.txt", "b/d.txt"}; - std::sort(names.begin(), names.end()); - ASSERT_EQ(expected_names, names); - - CloseArchive(handle); -} - -static void AssertIterationOrder(const std::string_view prefix, const std::string_view suffix, - const std::vector<std::string>& expected_names_sorted) { - ZipArchiveHandle handle; - ASSERT_EQ(0, OpenArchiveWrapper(kValidZip, &handle)); - - void* iteration_cookie; - ASSERT_EQ(0, StartIteration(handle, &iteration_cookie, prefix, suffix)); - - ZipEntry data; - std::vector<std::string> names; - - std::string name; - for (size_t i = 0; i < expected_names_sorted.size(); ++i) { - ASSERT_EQ(0, Next(iteration_cookie, &data, &name)); - names.push_back(name); - } - - // End of iteration. - ASSERT_EQ(-1, Next(iteration_cookie, &data, &name)); - CloseArchive(handle); - - // Assert that the names are as expected. - std::sort(names.begin(), names.end()); - ASSERT_EQ(expected_names_sorted, names); -} - -TEST(ziparchive, Iteration) { - static const std::vector<std::string> kExpectedMatchesSorted = {"a.txt", "b.txt", "b/", "b/c.txt", - "b/d.txt"}; - - AssertIterationOrder("", "", kExpectedMatchesSorted); -} - -TEST(ziparchive, IterationWithPrefix) { - static const std::vector<std::string> kExpectedMatchesSorted = {"b/", "b/c.txt", "b/d.txt"}; - - AssertIterationOrder("b/", "", kExpectedMatchesSorted); -} - -TEST(ziparchive, IterationWithSuffix) { - static const std::vector<std::string> kExpectedMatchesSorted = {"a.txt", "b.txt", "b/c.txt", - "b/d.txt"}; - - AssertIterationOrder("", ".txt", kExpectedMatchesSorted); -} - -TEST(ziparchive, IterationWithPrefixAndSuffix) { - static const std::vector<std::string> kExpectedMatchesSorted = {"b.txt", "b/c.txt", "b/d.txt"}; - - AssertIterationOrder("b", ".txt", kExpectedMatchesSorted); -} - -TEST(ziparchive, IterationWithBadPrefixAndSuffix) { - ZipArchiveHandle handle; - ASSERT_EQ(0, OpenArchiveWrapper(kValidZip, &handle)); - - void* iteration_cookie; - ASSERT_EQ(0, StartIteration(handle, &iteration_cookie, "x", "y")); - - ZipEntry data; - std::string name; - - // End of iteration. - ASSERT_EQ(-1, Next(iteration_cookie, &data, &name)); - - CloseArchive(handle); -} - -TEST(ziparchive, FindEntry) { - ZipArchiveHandle handle; - ASSERT_EQ(0, OpenArchiveWrapper(kValidZip, &handle)); - - ZipEntry data; - ASSERT_EQ(0, FindEntry(handle, "a.txt", &data)); - - // Known facts about a.txt, from zipinfo -v. - ASSERT_EQ(63, data.offset); - ASSERT_EQ(kCompressDeflated, data.method); - ASSERT_EQ(static_cast<uint32_t>(17), data.uncompressed_length); - ASSERT_EQ(static_cast<uint32_t>(13), data.compressed_length); - ASSERT_EQ(0x950821c5, data.crc32); - ASSERT_EQ(static_cast<uint32_t>(0x438a8005), data.mod_time); - - // An entry that doesn't exist. Should be a negative return code. - ASSERT_LT(FindEntry(handle, "this file does not exist", &data), 0); - - CloseArchive(handle); -} - -TEST(ziparchive, FindEntry_empty) { - ZipArchiveHandle handle; - ASSERT_EQ(0, OpenArchiveWrapper(kValidZip, &handle)); - - ZipEntry data; - ASSERT_EQ(kInvalidEntryName, FindEntry(handle, "", &data)); - - CloseArchive(handle); -} - -TEST(ziparchive, FindEntry_too_long) { - ZipArchiveHandle handle; - ASSERT_EQ(0, OpenArchiveWrapper(kValidZip, &handle)); - - std::string very_long_name(65536, 'x'); - ZipEntry data; - ASSERT_EQ(kInvalidEntryName, FindEntry(handle, very_long_name, &data)); - - CloseArchive(handle); -} - -TEST(ziparchive, TestInvalidDeclaredLength) { - ZipArchiveHandle handle; - ASSERT_EQ(0, OpenArchiveWrapper("declaredlength.zip", &handle)); - - void* iteration_cookie; - ASSERT_EQ(0, StartIteration(handle, &iteration_cookie)); - - std::string name; - ZipEntry data; - - ASSERT_EQ(Next(iteration_cookie, &data, &name), 0); - ASSERT_EQ(Next(iteration_cookie, &data, &name), 0); - - CloseArchive(handle); -} - -TEST(ziparchive, OpenArchiveFdRange) { - TemporaryFile tmp_file; - ASSERT_NE(-1, tmp_file.fd); - - const std::string leading_garbage(21, 'x'); - ASSERT_TRUE(android::base::WriteFully(tmp_file.fd, leading_garbage.c_str(), - leading_garbage.size())); - - std::string valid_content; - ASSERT_TRUE(android::base::ReadFileToString(test_data_dir + "/" + kValidZip, &valid_content)); - ASSERT_TRUE(android::base::WriteFully(tmp_file.fd, valid_content.c_str(), valid_content.size())); - - const std::string ending_garbage(42, 'x'); - ASSERT_TRUE(android::base::WriteFully(tmp_file.fd, ending_garbage.c_str(), - ending_garbage.size())); - - ZipArchiveHandle handle; - ASSERT_EQ(0, lseek(tmp_file.fd, 0, SEEK_SET)); - ASSERT_EQ(0, OpenArchiveFdRange(tmp_file.fd, "OpenArchiveFdRange", &handle, - valid_content.size(), - static_cast<off64_t>(leading_garbage.size()))); - - // An entry that's deflated. - ZipEntry data; - ASSERT_EQ(0, FindEntry(handle, "a.txt", &data)); - const uint32_t a_size = data.uncompressed_length; - ASSERT_EQ(a_size, kATxtContents.size()); - auto buffer = std::unique_ptr<uint8_t[]>(new uint8_t[a_size]); - ASSERT_EQ(0, ExtractToMemory(handle, &data, buffer.get(), a_size)); - ASSERT_EQ(0, memcmp(buffer.get(), kATxtContents.data(), a_size)); - - // An entry that's stored. - ASSERT_EQ(0, FindEntry(handle, "b.txt", &data)); - const uint32_t b_size = data.uncompressed_length; - ASSERT_EQ(b_size, kBTxtContents.size()); - buffer = std::unique_ptr<uint8_t[]>(new uint8_t[b_size]); - ASSERT_EQ(0, ExtractToMemory(handle, &data, buffer.get(), b_size)); - ASSERT_EQ(0, memcmp(buffer.get(), kBTxtContents.data(), b_size)); - - CloseArchive(handle); -} - -TEST(ziparchive, ExtractToMemory) { - ZipArchiveHandle handle; - ASSERT_EQ(0, OpenArchiveWrapper(kValidZip, &handle)); - - // An entry that's deflated. - ZipEntry data; - ASSERT_EQ(0, FindEntry(handle, "a.txt", &data)); - const uint32_t a_size = data.uncompressed_length; - ASSERT_EQ(a_size, kATxtContents.size()); - uint8_t* buffer = new uint8_t[a_size]; - ASSERT_EQ(0, ExtractToMemory(handle, &data, buffer, a_size)); - ASSERT_EQ(0, memcmp(buffer, kATxtContents.data(), a_size)); - delete[] buffer; - - // An entry that's stored. - ASSERT_EQ(0, FindEntry(handle, "b.txt", &data)); - const uint32_t b_size = data.uncompressed_length; - ASSERT_EQ(b_size, kBTxtContents.size()); - buffer = new uint8_t[b_size]; - ASSERT_EQ(0, ExtractToMemory(handle, &data, buffer, b_size)); - ASSERT_EQ(0, memcmp(buffer, kBTxtContents.data(), b_size)); - delete[] buffer; - - CloseArchive(handle); -} - -static const uint32_t kEmptyEntriesZip[] = { - 0x04034b50, 0x0000000a, 0x63600000, 0x00004438, 0x00000000, 0x00000000, 0x00090000, - 0x6d65001c, 0x2e797470, 0x55747874, 0x03000954, 0x52e25c13, 0x52e25c24, 0x000b7875, - 0x42890401, 0x88040000, 0x50000013, 0x1e02014b, 0x00000a03, 0x60000000, 0x00443863, - 0x00000000, 0x00000000, 0x09000000, 0x00001800, 0x00000000, 0xa0000000, 0x00000081, - 0x706d6500, 0x742e7974, 0x54557478, 0x13030005, 0x7552e25c, 0x01000b78, 0x00428904, - 0x13880400, 0x4b500000, 0x00000605, 0x00010000, 0x004f0001, 0x00430000, 0x00000000}; - -// This is a zip file containing a single entry (ab.txt) that contains -// 90072 repetitions of the string "ab\n" and has an uncompressed length -// of 270216 bytes. -static const uint16_t kAbZip[] = { - 0x4b50, 0x0403, 0x0014, 0x0000, 0x0008, 0x51d2, 0x4698, 0xc4b0, 0x2cda, 0x011b, 0x0000, 0x1f88, - 0x0004, 0x0006, 0x001c, 0x6261, 0x742e, 0x7478, 0x5455, 0x0009, 0x7c03, 0x3a09, 0x7c55, 0x3a09, - 0x7555, 0x0b78, 0x0100, 0x8904, 0x0042, 0x0400, 0x1388, 0x0000, 0xc2ed, 0x0d31, 0x0000, 0x030c, - 0x7fa0, 0x3b2e, 0x22ff, 0xa2aa, 0x841f, 0x45fc, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, - 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, - 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, - 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, - 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, - 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, - 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, - 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, - 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, - 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, - 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, - 0x5555, 0x5555, 0x5555, 0x5555, 0xdd55, 0x502c, 0x014b, 0x1e02, 0x1403, 0x0000, 0x0800, 0xd200, - 0x9851, 0xb046, 0xdac4, 0x1b2c, 0x0001, 0x8800, 0x041f, 0x0600, 0x1800, 0x0000, 0x0000, 0x0100, - 0x0000, 0xa000, 0x0081, 0x0000, 0x6100, 0x2e62, 0x7874, 0x5574, 0x0554, 0x0300, 0x097c, 0x553a, - 0x7875, 0x000b, 0x0401, 0x4289, 0x0000, 0x8804, 0x0013, 0x5000, 0x054b, 0x0006, 0x0000, 0x0100, - 0x0100, 0x4c00, 0x0000, 0x5b00, 0x0001, 0x0000, 0x0000}; - -static const std::string kAbTxtName("ab.txt"); -static const size_t kAbUncompressedSize = 270216; - -TEST(ziparchive, EmptyEntries) { - TemporaryFile tmp_file; - ASSERT_NE(-1, tmp_file.fd); - ASSERT_TRUE(android::base::WriteFully(tmp_file.fd, kEmptyEntriesZip, sizeof(kEmptyEntriesZip))); - - ZipArchiveHandle handle; - ASSERT_EQ(0, OpenArchiveFd(tmp_file.fd, "EmptyEntriesTest", &handle, false)); - - ZipEntry entry; - ASSERT_EQ(0, FindEntry(handle, "empty.txt", &entry)); - ASSERT_EQ(static_cast<uint32_t>(0), entry.uncompressed_length); - uint8_t buffer[1]; - ASSERT_EQ(0, ExtractToMemory(handle, &entry, buffer, 1)); - - TemporaryFile tmp_output_file; - ASSERT_NE(-1, tmp_output_file.fd); - ASSERT_EQ(0, ExtractEntryToFile(handle, &entry, tmp_output_file.fd)); - - struct stat stat_buf; - ASSERT_EQ(0, fstat(tmp_output_file.fd, &stat_buf)); - ASSERT_EQ(0, stat_buf.st_size); -} - -TEST(ziparchive, EntryLargerThan32K) { - TemporaryFile tmp_file; - ASSERT_NE(-1, tmp_file.fd); - ASSERT_TRUE(android::base::WriteFully(tmp_file.fd, reinterpret_cast<const uint8_t*>(kAbZip), - sizeof(kAbZip) - 1)); - ZipArchiveHandle handle; - ASSERT_EQ(0, OpenArchiveFd(tmp_file.fd, "EntryLargerThan32KTest", &handle, false)); - - ZipEntry entry; - ASSERT_EQ(0, FindEntry(handle, kAbTxtName, &entry)); - ASSERT_EQ(kAbUncompressedSize, entry.uncompressed_length); - - // Extract the entry to memory. - std::vector<uint8_t> buffer(kAbUncompressedSize); - ASSERT_EQ(0, ExtractToMemory(handle, &entry, &buffer[0], static_cast<uint32_t>(buffer.size()))); - - // Extract the entry to a file. - TemporaryFile tmp_output_file; - ASSERT_NE(-1, tmp_output_file.fd); - ASSERT_EQ(0, ExtractEntryToFile(handle, &entry, tmp_output_file.fd)); - - // Make sure the extracted file size is as expected. - struct stat stat_buf; - ASSERT_EQ(0, fstat(tmp_output_file.fd, &stat_buf)); - ASSERT_EQ(kAbUncompressedSize, static_cast<size_t>(stat_buf.st_size)); - - // Read the file back to a buffer and make sure the contents are - // the same as the memory buffer we extracted directly to. - std::vector<uint8_t> file_contents(kAbUncompressedSize); - ASSERT_EQ(0, lseek(tmp_output_file.fd, 0, SEEK_SET)); - ASSERT_TRUE(android::base::ReadFully(tmp_output_file.fd, &file_contents[0], file_contents.size())); - ASSERT_EQ(file_contents, buffer); - - for (int i = 0; i < 90072; ++i) { - const uint8_t* line = &file_contents[0] + (3 * i); - ASSERT_EQ('a', line[0]); - ASSERT_EQ('b', line[1]); - ASSERT_EQ('\n', line[2]); - } -} - -TEST(ziparchive, TrailerAfterEOCD) { - TemporaryFile tmp_file; - ASSERT_NE(-1, tmp_file.fd); - - // Create a file with 8 bytes of random garbage. - static const uint8_t trailer[] = {'A', 'n', 'd', 'r', 'o', 'i', 'd', 'z'}; - ASSERT_TRUE(android::base::WriteFully(tmp_file.fd, kEmptyEntriesZip, sizeof(kEmptyEntriesZip))); - ASSERT_TRUE(android::base::WriteFully(tmp_file.fd, trailer, sizeof(trailer))); - - ZipArchiveHandle handle; - ASSERT_GT(0, OpenArchiveFd(tmp_file.fd, "EmptyEntriesTest", &handle, false)); -} - -TEST(ziparchive, ExtractToFile) { - TemporaryFile tmp_file; - ASSERT_NE(-1, tmp_file.fd); - const uint8_t data[8] = {'1', '2', '3', '4', '5', '6', '7', '8'}; - const size_t data_size = sizeof(data); - - ASSERT_TRUE(android::base::WriteFully(tmp_file.fd, data, data_size)); - - ZipArchiveHandle handle; - ASSERT_EQ(0, OpenArchiveWrapper(kValidZip, &handle)); - - ZipEntry entry; - ASSERT_EQ(0, FindEntry(handle, "a.txt", &entry)); - ASSERT_EQ(0, ExtractEntryToFile(handle, &entry, tmp_file.fd)); - - // Assert that the first 8 bytes of the file haven't been clobbered. - uint8_t read_buffer[data_size]; - ASSERT_EQ(0, lseek(tmp_file.fd, 0, SEEK_SET)); - ASSERT_TRUE(android::base::ReadFully(tmp_file.fd, read_buffer, data_size)); - ASSERT_EQ(0, memcmp(read_buffer, data, data_size)); - - // Assert that the remainder of the file contains the incompressed data. - std::vector<uint8_t> uncompressed_data(entry.uncompressed_length); - ASSERT_TRUE( - android::base::ReadFully(tmp_file.fd, uncompressed_data.data(), entry.uncompressed_length)); - ASSERT_EQ(0, memcmp(&uncompressed_data[0], kATxtContents.data(), kATxtContents.size())); - - // Assert that the total length of the file is sane - ASSERT_EQ(static_cast<ssize_t>(data_size + kATxtContents.size()), - lseek(tmp_file.fd, 0, SEEK_END)); -} - -#if !defined(_WIN32) -TEST(ziparchive, OpenFromMemory) { - const std::string zip_path = test_data_dir + "/dummy-update.zip"; - android::base::unique_fd fd(open(zip_path.c_str(), O_RDONLY | O_BINARY)); - ASSERT_NE(-1, fd); - struct stat sb; - ASSERT_EQ(0, fstat(fd, &sb)); - - // Memory map the file first and open the archive from the memory region. - auto file_map{ - android::base::MappedFile::FromFd(fd, 0, static_cast<size_t>(sb.st_size), PROT_READ)}; - ZipArchiveHandle handle; - ASSERT_EQ(0, - OpenArchiveFromMemory(file_map->data(), file_map->size(), zip_path.c_str(), &handle)); - - // Assert one entry can be found and extracted correctly. - ZipEntry binary_entry; - ASSERT_EQ(0, FindEntry(handle, "META-INF/com/google/android/update-binary", &binary_entry)); - TemporaryFile tmp_binary; - ASSERT_NE(-1, tmp_binary.fd); - ASSERT_EQ(0, ExtractEntryToFile(handle, &binary_entry, tmp_binary.fd)); -} -#endif - -static void ZipArchiveStreamTest(ZipArchiveHandle& handle, const std::string& entry_name, bool raw, - bool verified, ZipEntry* entry, std::vector<uint8_t>* read_data) { - ASSERT_EQ(0, FindEntry(handle, entry_name, entry)); - std::unique_ptr<ZipArchiveStreamEntry> stream; - if (raw) { - stream.reset(ZipArchiveStreamEntry::CreateRaw(handle, *entry)); - if (entry->method == kCompressStored) { - read_data->resize(entry->uncompressed_length); - } else { - read_data->resize(entry->compressed_length); - } - } else { - stream.reset(ZipArchiveStreamEntry::Create(handle, *entry)); - read_data->resize(entry->uncompressed_length); - } - uint8_t* read_data_ptr = read_data->data(); - ASSERT_TRUE(stream.get() != nullptr); - const std::vector<uint8_t>* data; - uint64_t total_size = 0; - while ((data = stream->Read()) != nullptr) { - total_size += data->size(); - memcpy(read_data_ptr, data->data(), data->size()); - read_data_ptr += data->size(); - } - ASSERT_EQ(verified, stream->Verify()); - ASSERT_EQ(total_size, read_data->size()); -} - -static void ZipArchiveStreamTestUsingContents(const std::string& zip_file, - const std::string& entry_name, - const std::vector<uint8_t>& contents, bool raw) { - ZipArchiveHandle handle; - ASSERT_EQ(0, OpenArchiveWrapper(zip_file, &handle)); - - ZipEntry entry; - std::vector<uint8_t> read_data; - ZipArchiveStreamTest(handle, entry_name, raw, true, &entry, &read_data); - - ASSERT_EQ(contents.size(), read_data.size()); - ASSERT_TRUE(memcmp(read_data.data(), contents.data(), read_data.size()) == 0); - - CloseArchive(handle); -} - -static void ZipArchiveStreamTestUsingMemory(const std::string& zip_file, - const std::string& entry_name) { - ZipArchiveHandle handle; - ASSERT_EQ(0, OpenArchiveWrapper(zip_file, &handle)); - - ZipEntry entry; - std::vector<uint8_t> read_data; - ZipArchiveStreamTest(handle, entry_name, false, true, &entry, &read_data); - - std::vector<uint8_t> cmp_data(entry.uncompressed_length); - ASSERT_EQ(entry.uncompressed_length, read_data.size()); - ASSERT_EQ( - 0, ExtractToMemory(handle, &entry, cmp_data.data(), static_cast<uint32_t>(cmp_data.size()))); - ASSERT_TRUE(memcmp(read_data.data(), cmp_data.data(), read_data.size()) == 0); - - CloseArchive(handle); -} - -TEST(ziparchive, StreamCompressed) { - ZipArchiveStreamTestUsingContents(kValidZip, "a.txt", kATxtContents, false); -} - -TEST(ziparchive, StreamUncompressed) { - ZipArchiveStreamTestUsingContents(kValidZip, "b.txt", kBTxtContents, false); -} - -TEST(ziparchive, StreamRawCompressed) { - ZipArchiveStreamTestUsingContents(kValidZip, "a.txt", kATxtContentsCompressed, true); -} - -TEST(ziparchive, StreamRawUncompressed) { - ZipArchiveStreamTestUsingContents(kValidZip, "b.txt", kBTxtContents, true); -} - -TEST(ziparchive, StreamLargeCompressed) { - ZipArchiveStreamTestUsingMemory(kLargeZip, "compress.txt"); -} - -TEST(ziparchive, StreamLargeUncompressed) { - ZipArchiveStreamTestUsingMemory(kLargeZip, "uncompress.txt"); -} - -TEST(ziparchive, StreamCompressedBadCrc) { - ZipArchiveHandle handle; - ASSERT_EQ(0, OpenArchiveWrapper(kBadCrcZip, &handle)); - - ZipEntry entry; - std::vector<uint8_t> read_data; - ZipArchiveStreamTest(handle, "a.txt", false, false, &entry, &read_data); - - CloseArchive(handle); -} - -TEST(ziparchive, StreamUncompressedBadCrc) { - ZipArchiveHandle handle; - ASSERT_EQ(0, OpenArchiveWrapper(kBadCrcZip, &handle)); - - ZipEntry entry; - std::vector<uint8_t> read_data; - ZipArchiveStreamTest(handle, "b.txt", false, false, &entry, &read_data); - - CloseArchive(handle); -} - -// Generated using the following Java program: -// public static void main(String[] foo) throws Exception { -// FileOutputStream fos = new -// FileOutputStream("/tmp/data_descriptor.zip"); -// ZipOutputStream zos = new ZipOutputStream(fos); -// ZipEntry ze = new ZipEntry("name"); -// ze.setMethod(ZipEntry.DEFLATED); -// zos.putNextEntry(ze); -// zos.write("abdcdefghijk".getBytes()); -// zos.closeEntry(); -// zos.close(); -// } -// -// cat /tmp/data_descriptor.zip | xxd -i -// -static const std::vector<uint8_t> kDataDescriptorZipFile{ - 0x50, 0x4b, 0x03, 0x04, 0x14, 0x00, 0x08, 0x08, 0x08, 0x00, 0x30, 0x59, 0xce, 0x4a, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x6e, 0x61, - 0x6d, 0x65, 0x4b, 0x4c, 0x4a, 0x49, 0x4e, 0x49, 0x4d, 0x4b, 0xcf, 0xc8, 0xcc, 0xca, 0x06, 0x00, - //[sig---------------], [crc32---------------], [csize---------------], [size----------------] - 0x50, 0x4b, 0x07, 0x08, 0x3d, 0x4e, 0x0e, 0xf9, 0x0e, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, - 0x50, 0x4b, 0x01, 0x02, 0x14, 0x00, 0x14, 0x00, 0x08, 0x08, 0x08, 0x00, 0x30, 0x59, 0xce, 0x4a, - 0x3d, 0x4e, 0x0e, 0xf9, 0x0e, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6e, 0x61, - 0x6d, 0x65, 0x50, 0x4b, 0x05, 0x06, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x32, 0x00, - 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00}; - -// The offsets of the data descriptor in this file, so we can mess with -// them later in the test. -static constexpr uint32_t kDataDescriptorOffset = 48; -static constexpr uint32_t kCSizeOffset = kDataDescriptorOffset + 8; -static constexpr uint32_t kSizeOffset = kCSizeOffset + 4; - -static void ExtractEntryToMemory(const std::vector<uint8_t>& zip_data, - std::vector<uint8_t>* entry_out, int32_t* error_code_out) { - TemporaryFile tmp_file; - ASSERT_NE(-1, tmp_file.fd); - ASSERT_TRUE(android::base::WriteFully(tmp_file.fd, &zip_data[0], zip_data.size())); - ZipArchiveHandle handle; - ASSERT_EQ(0, OpenArchiveFd(tmp_file.fd, "ExtractEntryToMemory", &handle, false)); - - // This function expects a variant of kDataDescriptorZipFile, for look for - // an entry whose name is "name" and whose size is 12 (contents = - // "abdcdefghijk"). - ZipEntry entry; - ASSERT_EQ(0, FindEntry(handle, "name", &entry)); - ASSERT_EQ(static_cast<uint32_t>(12), entry.uncompressed_length); - - entry_out->resize(12); - (*error_code_out) = ExtractToMemory(handle, &entry, &((*entry_out)[0]), 12); - - CloseArchive(handle); -} - -TEST(ziparchive, ValidDataDescriptors) { - std::vector<uint8_t> entry; - int32_t error_code = 0; - ExtractEntryToMemory(kDataDescriptorZipFile, &entry, &error_code); - - ASSERT_EQ(0, error_code); - ASSERT_EQ(12u, entry.size()); - ASSERT_EQ('a', entry[0]); - ASSERT_EQ('k', entry[11]); -} - -TEST(ziparchive, InvalidDataDescriptors_csize) { - std::vector<uint8_t> invalid_csize = kDataDescriptorZipFile; - invalid_csize[kCSizeOffset] = 0xfe; - - std::vector<uint8_t> entry; - int32_t error_code = 0; - ExtractEntryToMemory(invalid_csize, &entry, &error_code); - - ASSERT_EQ(kInconsistentInformation, error_code); -} - -TEST(ziparchive, InvalidDataDescriptors_size) { - std::vector<uint8_t> invalid_size = kDataDescriptorZipFile; - invalid_size[kSizeOffset] = 0xfe; - - std::vector<uint8_t> entry; - int32_t error_code = 0; - ExtractEntryToMemory(invalid_size, &entry, &error_code); - - ASSERT_EQ(kInconsistentInformation, error_code); -} - -TEST(ziparchive, ErrorCodeString) { - ASSERT_STREQ("Success", ErrorCodeString(0)); - - // Out of bounds. - ASSERT_STREQ("Unknown return code", ErrorCodeString(1)); - ASSERT_STRNE("Unknown return code", ErrorCodeString(kLastErrorCode)); - ASSERT_STREQ("Unknown return code", ErrorCodeString(kLastErrorCode - 1)); - - ASSERT_STREQ("I/O error", ErrorCodeString(kIoError)); -} - -// A zip file whose local file header at offset zero is corrupted. -// -// --------------- -// cat foo > a.txt -// zip a.zip a.txt -// cat a.zip | xxd -i -// -// Manual changes : -// [2] = 0xff // Corrupt the LFH signature of entry 0. -// [3] = 0xff // Corrupt the LFH signature of entry 0. -static const std::vector<uint8_t> kZipFileWithBrokenLfhSignature{ - //[lfh-sig-----------], [lfh contents--------------------------------- - 0x50, 0x4b, 0xff, 0xff, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x77, 0x80, - //-------------------------------------------------------------------- - 0x09, 0x4b, 0xa8, 0x65, 0x32, 0x7e, 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, - //-------------------------------] [file-name-----------------], [--- - 0x00, 0x00, 0x05, 0x00, 0x1c, 0x00, 0x61, 0x2e, 0x74, 0x78, 0x74, 0x55, - // entry-contents------------------------------------------------------ - 0x54, 0x09, 0x00, 0x03, 0x51, 0x24, 0x8b, 0x59, 0x51, 0x24, 0x8b, 0x59, - //-------------------------------------------------------------------- - 0x75, 0x78, 0x0b, 0x00, 0x01, 0x04, 0x89, 0x42, 0x00, 0x00, 0x04, 0x88, - //-------------------------------------], [cd-record-sig-------], [--- - 0x13, 0x00, 0x00, 0x66, 0x6f, 0x6f, 0x0a, 0x50, 0x4b, 0x01, 0x02, 0x1e, - // cd-record----------------------------------------------------------- - 0x03, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x77, 0x80, 0x09, 0x4b, 0xa8, - //-------------------------------------------------------------------- - 0x65, 0x32, 0x7e, 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x05, - //-------------------------------------------------------------------- - 0x00, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xa0, - //-] [lfh-file-header-off-], [file-name-----------------], [extra---- - 0x81, 0x00, 0x00, 0x00, 0x00, 0x61, 0x2e, 0x74, 0x78, 0x74, 0x55, 0x54, - //-------------------------------------------------------------------- - 0x05, 0x00, 0x03, 0x51, 0x24, 0x8b, 0x59, 0x75, 0x78, 0x0b, 0x00, 0x01, - //-------------------------------------------------------], [eocd-sig- - 0x04, 0x89, 0x42, 0x00, 0x00, 0x04, 0x88, 0x13, 0x00, 0x00, 0x50, 0x4b, - //-------], [--------------------------------------------------------- - 0x05, 0x06, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x4b, 0x00, - //-------------------------------------------] - 0x00, 0x00, 0x43, 0x00, 0x00, 0x00, 0x00, 0x00}; - -TEST(ziparchive, BrokenLfhSignature) { - TemporaryFile tmp_file; - ASSERT_NE(-1, tmp_file.fd); - ASSERT_TRUE(android::base::WriteFully(tmp_file.fd, &kZipFileWithBrokenLfhSignature[0], - kZipFileWithBrokenLfhSignature.size())); - ZipArchiveHandle handle; - ASSERT_EQ(kInvalidFile, OpenArchiveFd(tmp_file.fd, "LeadingNonZipBytes", &handle, false)); -} - -class VectorReader : public zip_archive::Reader { - public: - VectorReader(const std::vector<uint8_t>& input) : Reader(), input_(input) {} - - bool ReadAtOffset(uint8_t* buf, size_t len, uint32_t offset) const { - if ((offset + len) < input_.size()) { - return false; - } - - memcpy(buf, &input_[offset], len); - return true; - } - - private: - const std::vector<uint8_t>& input_; -}; - -class VectorWriter : public zip_archive::Writer { - public: - VectorWriter() : Writer() {} - - bool Append(uint8_t* buf, size_t size) { - output_.insert(output_.end(), buf, buf + size); - return true; - } - - std::vector<uint8_t>& GetOutput() { return output_; } - - private: - std::vector<uint8_t> output_; -}; - -class BadReader : public zip_archive::Reader { - public: - BadReader() : Reader() {} - - bool ReadAtOffset(uint8_t*, size_t, uint32_t) const { return false; } -}; - -class BadWriter : public zip_archive::Writer { - public: - BadWriter() : Writer() {} - - bool Append(uint8_t*, size_t) { return false; } -}; - -TEST(ziparchive, Inflate) { - const uint32_t compressed_length = static_cast<uint32_t>(kATxtContentsCompressed.size()); - const uint32_t uncompressed_length = static_cast<uint32_t>(kATxtContents.size()); - - const VectorReader reader(kATxtContentsCompressed); - { - VectorWriter writer; - uint64_t crc_out = 0; - - int32_t ret = - zip_archive::Inflate(reader, compressed_length, uncompressed_length, &writer, &crc_out); - ASSERT_EQ(0, ret); - ASSERT_EQ(kATxtContents, writer.GetOutput()); - ASSERT_EQ(0x950821C5u, crc_out); - } - - { - VectorWriter writer; - int32_t ret = - zip_archive::Inflate(reader, compressed_length, uncompressed_length, &writer, nullptr); - ASSERT_EQ(0, ret); - ASSERT_EQ(kATxtContents, writer.GetOutput()); - } - - { - BadWriter writer; - int32_t ret = - zip_archive::Inflate(reader, compressed_length, uncompressed_length, &writer, nullptr); - ASSERT_EQ(kIoError, ret); - } - - { - BadReader reader; - VectorWriter writer; - int32_t ret = - zip_archive::Inflate(reader, compressed_length, uncompressed_length, &writer, nullptr); - ASSERT_EQ(kIoError, ret); - ASSERT_EQ(0u, writer.GetOutput().size()); - } -} diff --git a/libziparchive/zip_writer.cc b/libziparchive/zip_writer.cc deleted file mode 100644 index 67279a6b7..000000000 --- a/libziparchive/zip_writer.cc +++ /dev/null @@ -1,582 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "ziparchive/zip_writer.h" - -#include <sys/param.h> -#include <sys/stat.h> -#include <zlib.h> -#include <cstdio> -#define DEF_MEM_LEVEL 8 // normally in zutil.h? - -#include <memory> -#include <vector> - -#include "android-base/logging.h" - -#include "entry_name_utils-inl.h" -#include "zip_archive_common.h" - -#undef powerof2 -#define powerof2(x) \ - ({ \ - __typeof__(x) _x = (x); \ - __typeof__(x) _x2; \ - __builtin_add_overflow(_x, -1, &_x2) ? 1 : ((_x2 & _x) == 0); \ - }) - -/* Zip compression methods we support */ -enum { - kCompressStored = 0, // no compression - kCompressDeflated = 8, // standard deflate -}; - -// Size of the output buffer used for compression. -static const size_t kBufSize = 32768u; - -// No error, operation completed successfully. -static const int32_t kNoError = 0; - -// The ZipWriter is in a bad state. -static const int32_t kInvalidState = -1; - -// There was an IO error while writing to disk. -static const int32_t kIoError = -2; - -// The zip entry name was invalid. -static const int32_t kInvalidEntryName = -3; - -// An error occurred in zlib. -static const int32_t kZlibError = -4; - -// The start aligned function was called with the aligned flag. -static const int32_t kInvalidAlign32Flag = -5; - -// The alignment parameter is not a power of 2. -static const int32_t kInvalidAlignment = -6; - -static const char* sErrorCodes[] = { - "Invalid state", "IO error", "Invalid entry name", "Zlib error", -}; - -const char* ZipWriter::ErrorCodeString(int32_t error_code) { - if (error_code < 0 && (-error_code) < static_cast<int32_t>(arraysize(sErrorCodes))) { - return sErrorCodes[-error_code]; - } - return nullptr; -} - -static void DeleteZStream(z_stream* stream) { - deflateEnd(stream); - delete stream; -} - -ZipWriter::ZipWriter(FILE* f) - : file_(f), - seekable_(false), - current_offset_(0), - state_(State::kWritingZip), - z_stream_(nullptr, DeleteZStream), - buffer_(kBufSize) { - // Check if the file is seekable (regular file). If fstat fails, that's fine, subsequent calls - // will fail as well. - struct stat file_stats; - if (fstat(fileno(f), &file_stats) == 0) { - seekable_ = S_ISREG(file_stats.st_mode); - } -} - -ZipWriter::ZipWriter(ZipWriter&& writer) noexcept - : file_(writer.file_), - seekable_(writer.seekable_), - current_offset_(writer.current_offset_), - state_(writer.state_), - files_(std::move(writer.files_)), - z_stream_(std::move(writer.z_stream_)), - buffer_(std::move(writer.buffer_)) { - writer.file_ = nullptr; - writer.state_ = State::kError; -} - -ZipWriter& ZipWriter::operator=(ZipWriter&& writer) noexcept { - file_ = writer.file_; - seekable_ = writer.seekable_; - current_offset_ = writer.current_offset_; - state_ = writer.state_; - files_ = std::move(writer.files_); - z_stream_ = std::move(writer.z_stream_); - buffer_ = std::move(writer.buffer_); - writer.file_ = nullptr; - writer.state_ = State::kError; - return *this; -} - -int32_t ZipWriter::HandleError(int32_t error_code) { - state_ = State::kError; - z_stream_.reset(); - return error_code; -} - -int32_t ZipWriter::StartEntry(std::string_view path, size_t flags) { - uint32_t alignment = 0; - if (flags & kAlign32) { - flags &= ~kAlign32; - alignment = 4; - } - return StartAlignedEntryWithTime(path, flags, time_t(), alignment); -} - -int32_t ZipWriter::StartAlignedEntry(std::string_view path, size_t flags, uint32_t alignment) { - return StartAlignedEntryWithTime(path, flags, time_t(), alignment); -} - -int32_t ZipWriter::StartEntryWithTime(std::string_view path, size_t flags, time_t time) { - uint32_t alignment = 0; - if (flags & kAlign32) { - flags &= ~kAlign32; - alignment = 4; - } - return StartAlignedEntryWithTime(path, flags, time, alignment); -} - -static void ExtractTimeAndDate(time_t when, uint16_t* out_time, uint16_t* out_date) { - /* round up to an even number of seconds */ - when = static_cast<time_t>((static_cast<unsigned long>(when) + 1) & (~1)); - - struct tm* ptm; -#if !defined(_WIN32) - struct tm tm_result; - ptm = localtime_r(&when, &tm_result); -#else - ptm = localtime(&when); -#endif - - int year = ptm->tm_year; - if (year < 80) { - year = 80; - } - - *out_date = static_cast<uint16_t>((year - 80) << 9 | (ptm->tm_mon + 1) << 5 | ptm->tm_mday); - *out_time = static_cast<uint16_t>(ptm->tm_hour << 11 | ptm->tm_min << 5 | ptm->tm_sec >> 1); -} - -static void CopyFromFileEntry(const ZipWriter::FileEntry& src, bool use_data_descriptor, - LocalFileHeader* dst) { - dst->lfh_signature = LocalFileHeader::kSignature; - if (use_data_descriptor) { - // Set this flag to denote that a DataDescriptor struct will appear after the data, - // containing the crc and size fields. - dst->gpb_flags |= kGPBDDFlagMask; - - // The size and crc fields must be 0. - dst->compressed_size = 0u; - dst->uncompressed_size = 0u; - dst->crc32 = 0u; - } else { - dst->compressed_size = src.compressed_size; - dst->uncompressed_size = src.uncompressed_size; - dst->crc32 = src.crc32; - } - dst->compression_method = src.compression_method; - dst->last_mod_time = src.last_mod_time; - dst->last_mod_date = src.last_mod_date; - DCHECK_LE(src.path.size(), std::numeric_limits<uint16_t>::max()); - dst->file_name_length = static_cast<uint16_t>(src.path.size()); - dst->extra_field_length = src.padding_length; -} - -int32_t ZipWriter::StartAlignedEntryWithTime(std::string_view path, size_t flags, time_t time, - uint32_t alignment) { - if (state_ != State::kWritingZip) { - return kInvalidState; - } - - // Can only have 16535 entries because of zip records. - if (files_.size() == std::numeric_limits<uint16_t>::max()) { - return HandleError(kIoError); - } - - if (flags & kAlign32) { - return kInvalidAlign32Flag; - } - - if (powerof2(alignment) == 0) { - return kInvalidAlignment; - } - if (alignment > std::numeric_limits<uint16_t>::max()) { - return kInvalidAlignment; - } - - FileEntry file_entry = {}; - file_entry.local_file_header_offset = current_offset_; - file_entry.path = path; - // No support for larger than 4GB files. - if (file_entry.local_file_header_offset > std::numeric_limits<uint32_t>::max()) { - return HandleError(kIoError); - } - - if (!IsValidEntryName(reinterpret_cast<const uint8_t*>(file_entry.path.data()), - file_entry.path.size())) { - return kInvalidEntryName; - } - - if (flags & ZipWriter::kCompress) { - file_entry.compression_method = kCompressDeflated; - - int32_t result = PrepareDeflate(); - if (result != kNoError) { - return result; - } - } else { - file_entry.compression_method = kCompressStored; - } - - ExtractTimeAndDate(time, &file_entry.last_mod_time, &file_entry.last_mod_date); - - off_t offset = current_offset_ + sizeof(LocalFileHeader) + file_entry.path.size(); - // prepare a pre-zeroed memory page in case when we need to pad some aligned data. - static constexpr auto kPageSize = 4096; - static constexpr char kSmallZeroPadding[kPageSize] = {}; - // use this buffer if our preallocated one is too small - std::vector<char> zero_padding_big; - const char* zero_padding = nullptr; - - if (alignment != 0 && (offset & (alignment - 1))) { - // Pad the extra field so the data will be aligned. - uint16_t padding = static_cast<uint16_t>(alignment - (offset % alignment)); - file_entry.padding_length = padding; - offset += padding; - if (padding <= std::size(kSmallZeroPadding)) { - zero_padding = kSmallZeroPadding; - } else { - zero_padding_big.resize(padding, 0); - zero_padding = zero_padding_big.data(); - } - } - - LocalFileHeader header = {}; - // Always start expecting a data descriptor. When the data has finished being written, - // if it is possible to seek back, the GPB flag will reset and the sizes written. - CopyFromFileEntry(file_entry, true /*use_data_descriptor*/, &header); - - if (fwrite(&header, sizeof(header), 1, file_) != 1) { - return HandleError(kIoError); - } - - if (fwrite(path.data(), 1, path.size(), file_) != path.size()) { - return HandleError(kIoError); - } - - if (file_entry.padding_length != 0 && fwrite(zero_padding, 1, file_entry.padding_length, - file_) != file_entry.padding_length) { - return HandleError(kIoError); - } - - current_file_entry_ = std::move(file_entry); - current_offset_ = offset; - state_ = State::kWritingEntry; - return kNoError; -} - -int32_t ZipWriter::DiscardLastEntry() { - if (state_ != State::kWritingZip || files_.empty()) { - return kInvalidState; - } - - FileEntry& last_entry = files_.back(); - current_offset_ = last_entry.local_file_header_offset; - if (fseeko(file_, current_offset_, SEEK_SET) != 0) { - return HandleError(kIoError); - } - files_.pop_back(); - return kNoError; -} - -int32_t ZipWriter::GetLastEntry(FileEntry* out_entry) { - CHECK(out_entry != nullptr); - - if (files_.empty()) { - return kInvalidState; - } - *out_entry = files_.back(); - return kNoError; -} - -int32_t ZipWriter::PrepareDeflate() { - CHECK(state_ == State::kWritingZip); - - // Initialize the z_stream for compression. - z_stream_ = std::unique_ptr<z_stream, void (*)(z_stream*)>(new z_stream(), DeleteZStream); - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wold-style-cast" - int zerr = deflateInit2(z_stream_.get(), Z_BEST_COMPRESSION, Z_DEFLATED, -MAX_WBITS, - DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY); -#pragma GCC diagnostic pop - - if (zerr != Z_OK) { - if (zerr == Z_VERSION_ERROR) { - LOG(ERROR) << "Installed zlib is not compatible with linked version (" << ZLIB_VERSION << ")"; - return HandleError(kZlibError); - } else { - LOG(ERROR) << "deflateInit2 failed (zerr=" << zerr << ")"; - return HandleError(kZlibError); - } - } - - z_stream_->next_out = buffer_.data(); - DCHECK_EQ(buffer_.size(), kBufSize); - z_stream_->avail_out = static_cast<uint32_t>(buffer_.size()); - return kNoError; -} - -int32_t ZipWriter::WriteBytes(const void* data, size_t len) { - if (state_ != State::kWritingEntry) { - return HandleError(kInvalidState); - } - // Need to be able to mark down data correctly. - if (len + static_cast<uint64_t>(current_file_entry_.uncompressed_size) > - std::numeric_limits<uint32_t>::max()) { - return HandleError(kIoError); - } - uint32_t len32 = static_cast<uint32_t>(len); - - int32_t result = kNoError; - if (current_file_entry_.compression_method & kCompressDeflated) { - result = CompressBytes(¤t_file_entry_, data, len32); - } else { - result = StoreBytes(¤t_file_entry_, data, len32); - } - - if (result != kNoError) { - return result; - } - - current_file_entry_.crc32 = static_cast<uint32_t>( - crc32(current_file_entry_.crc32, reinterpret_cast<const Bytef*>(data), len32)); - current_file_entry_.uncompressed_size += len32; - return kNoError; -} - -int32_t ZipWriter::StoreBytes(FileEntry* file, const void* data, uint32_t len) { - CHECK(state_ == State::kWritingEntry); - - if (fwrite(data, 1, len, file_) != len) { - return HandleError(kIoError); - } - file->compressed_size += len; - current_offset_ += len; - return kNoError; -} - -int32_t ZipWriter::CompressBytes(FileEntry* file, const void* data, uint32_t len) { - CHECK(state_ == State::kWritingEntry); - CHECK(z_stream_); - CHECK(z_stream_->next_out != nullptr); - CHECK(z_stream_->avail_out != 0); - - // Prepare the input. - z_stream_->next_in = reinterpret_cast<const uint8_t*>(data); - z_stream_->avail_in = len; - - while (z_stream_->avail_in > 0) { - // We have more data to compress. - int zerr = deflate(z_stream_.get(), Z_NO_FLUSH); - if (zerr != Z_OK) { - return HandleError(kZlibError); - } - - if (z_stream_->avail_out == 0) { - // The output is full, let's write it to disk. - size_t write_bytes = z_stream_->next_out - buffer_.data(); - if (fwrite(buffer_.data(), 1, write_bytes, file_) != write_bytes) { - return HandleError(kIoError); - } - file->compressed_size += write_bytes; - current_offset_ += write_bytes; - - // Reset the output buffer for the next input. - z_stream_->next_out = buffer_.data(); - DCHECK_EQ(buffer_.size(), kBufSize); - z_stream_->avail_out = static_cast<uint32_t>(buffer_.size()); - } - } - return kNoError; -} - -int32_t ZipWriter::FlushCompressedBytes(FileEntry* file) { - CHECK(state_ == State::kWritingEntry); - CHECK(z_stream_); - CHECK(z_stream_->next_out != nullptr); - CHECK(z_stream_->avail_out != 0); - - // Keep deflating while there isn't enough space in the buffer to - // to complete the compress. - int zerr; - while ((zerr = deflate(z_stream_.get(), Z_FINISH)) == Z_OK) { - CHECK(z_stream_->avail_out == 0); - size_t write_bytes = z_stream_->next_out - buffer_.data(); - if (fwrite(buffer_.data(), 1, write_bytes, file_) != write_bytes) { - return HandleError(kIoError); - } - file->compressed_size += write_bytes; - current_offset_ += write_bytes; - - z_stream_->next_out = buffer_.data(); - DCHECK_EQ(buffer_.size(), kBufSize); - z_stream_->avail_out = static_cast<uint32_t>(buffer_.size()); - } - if (zerr != Z_STREAM_END) { - return HandleError(kZlibError); - } - - size_t write_bytes = z_stream_->next_out - buffer_.data(); - if (write_bytes != 0) { - if (fwrite(buffer_.data(), 1, write_bytes, file_) != write_bytes) { - return HandleError(kIoError); - } - file->compressed_size += write_bytes; - current_offset_ += write_bytes; - } - z_stream_.reset(); - return kNoError; -} - -bool ZipWriter::ShouldUseDataDescriptor() const { - // Only use a trailing "data descriptor" if the output isn't seekable. - return !seekable_; -} - -int32_t ZipWriter::FinishEntry() { - if (state_ != State::kWritingEntry) { - return kInvalidState; - } - - if (current_file_entry_.compression_method & kCompressDeflated) { - int32_t result = FlushCompressedBytes(¤t_file_entry_); - if (result != kNoError) { - return result; - } - } - - if (ShouldUseDataDescriptor()) { - // Some versions of ZIP don't allow STORED data to have a trailing DataDescriptor. - // If this file is not seekable, or if the data is compressed, write a DataDescriptor. - const uint32_t sig = DataDescriptor::kOptSignature; - if (fwrite(&sig, sizeof(sig), 1, file_) != 1) { - return HandleError(kIoError); - } - - DataDescriptor dd = {}; - dd.crc32 = current_file_entry_.crc32; - dd.compressed_size = current_file_entry_.compressed_size; - dd.uncompressed_size = current_file_entry_.uncompressed_size; - if (fwrite(&dd, sizeof(dd), 1, file_) != 1) { - return HandleError(kIoError); - } - current_offset_ += sizeof(DataDescriptor::kOptSignature) + sizeof(dd); - } else { - // Seek back to the header and rewrite to include the size. - if (fseeko(file_, current_file_entry_.local_file_header_offset, SEEK_SET) != 0) { - return HandleError(kIoError); - } - - LocalFileHeader header = {}; - CopyFromFileEntry(current_file_entry_, false /*use_data_descriptor*/, &header); - - if (fwrite(&header, sizeof(header), 1, file_) != 1) { - return HandleError(kIoError); - } - - if (fseeko(file_, current_offset_, SEEK_SET) != 0) { - return HandleError(kIoError); - } - } - - files_.emplace_back(std::move(current_file_entry_)); - state_ = State::kWritingZip; - return kNoError; -} - -int32_t ZipWriter::Finish() { - if (state_ != State::kWritingZip) { - return kInvalidState; - } - - off_t startOfCdr = current_offset_; - for (FileEntry& file : files_) { - CentralDirectoryRecord cdr = {}; - cdr.record_signature = CentralDirectoryRecord::kSignature; - if (ShouldUseDataDescriptor()) { - cdr.gpb_flags |= kGPBDDFlagMask; - } - cdr.compression_method = file.compression_method; - cdr.last_mod_time = file.last_mod_time; - cdr.last_mod_date = file.last_mod_date; - cdr.crc32 = file.crc32; - cdr.compressed_size = file.compressed_size; - cdr.uncompressed_size = file.uncompressed_size; - // Checked in IsValidEntryName. - DCHECK_LE(file.path.size(), std::numeric_limits<uint16_t>::max()); - cdr.file_name_length = static_cast<uint16_t>(file.path.size()); - // Checked in StartAlignedEntryWithTime. - DCHECK_LE(file.local_file_header_offset, std::numeric_limits<uint32_t>::max()); - cdr.local_file_header_offset = static_cast<uint32_t>(file.local_file_header_offset); - if (fwrite(&cdr, sizeof(cdr), 1, file_) != 1) { - return HandleError(kIoError); - } - - if (fwrite(file.path.data(), 1, file.path.size(), file_) != file.path.size()) { - return HandleError(kIoError); - } - - current_offset_ += sizeof(cdr) + file.path.size(); - } - - EocdRecord er = {}; - er.eocd_signature = EocdRecord::kSignature; - er.disk_num = 0; - er.cd_start_disk = 0; - // Checked when adding entries. - DCHECK_LE(files_.size(), std::numeric_limits<uint16_t>::max()); - er.num_records_on_disk = static_cast<uint16_t>(files_.size()); - er.num_records = static_cast<uint16_t>(files_.size()); - if (current_offset_ > std::numeric_limits<uint32_t>::max()) { - return HandleError(kIoError); - } - er.cd_size = static_cast<uint32_t>(current_offset_ - startOfCdr); - er.cd_start_offset = static_cast<uint32_t>(startOfCdr); - - if (fwrite(&er, sizeof(er), 1, file_) != 1) { - return HandleError(kIoError); - } - - current_offset_ += sizeof(er); - - // Since we can BackUp() and potentially finish writing at an offset less than one we had - // already written at, we must truncate the file. - - if (ftruncate(fileno(file_), current_offset_) != 0) { - return HandleError(kIoError); - } - - if (fflush(file_) != 0) { - return HandleError(kIoError); - } - - state_ = State::kDone; - return kNoError; -} diff --git a/libziparchive/zip_writer_test.cc b/libziparchive/zip_writer_test.cc deleted file mode 100644 index d324d4bfd..000000000 --- a/libziparchive/zip_writer_test.cc +++ /dev/null @@ -1,428 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "ziparchive/zip_writer.h" -#include "ziparchive/zip_archive.h" - -#include <android-base/test_utils.h> -#include <gtest/gtest.h> -#include <time.h> -#include <memory> -#include <vector> - -static ::testing::AssertionResult AssertFileEntryContentsEq(const std::string& expected, - ZipArchiveHandle handle, - ZipEntry* zip_entry); - -struct zipwriter : public ::testing::Test { - TemporaryFile* temp_file_; - int fd_; - FILE* file_; - - void SetUp() override { - temp_file_ = new TemporaryFile(); - fd_ = temp_file_->fd; - file_ = fdopen(fd_, "w"); - ASSERT_NE(file_, nullptr); - } - - void TearDown() override { - fclose(file_); - delete temp_file_; - } -}; - -TEST_F(zipwriter, WriteUncompressedZipWithOneFile) { - ZipWriter writer(file_); - - const char* expected = "hello"; - - ASSERT_EQ(0, writer.StartEntry("file.txt", 0)); - ASSERT_EQ(0, writer.WriteBytes("he", 2)); - ASSERT_EQ(0, writer.WriteBytes("llo", 3)); - ASSERT_EQ(0, writer.FinishEntry()); - ASSERT_EQ(0, writer.Finish()); - - ASSERT_GE(0, lseek(fd_, 0, SEEK_SET)); - - ZipArchiveHandle handle; - ASSERT_EQ(0, OpenArchiveFd(fd_, "temp", &handle, false)); - - ZipEntry data; - ASSERT_EQ(0, FindEntry(handle, "file.txt", &data)); - EXPECT_EQ(kCompressStored, data.method); - EXPECT_EQ(0u, data.has_data_descriptor); - EXPECT_EQ(strlen(expected), data.compressed_length); - ASSERT_EQ(strlen(expected), data.uncompressed_length); - ASSERT_TRUE(AssertFileEntryContentsEq(expected, handle, &data)); - - CloseArchive(handle); -} - -TEST_F(zipwriter, WriteUncompressedZipWithMultipleFiles) { - ZipWriter writer(file_); - - ASSERT_EQ(0, writer.StartEntry("file.txt", 0)); - ASSERT_EQ(0, writer.WriteBytes("he", 2)); - ASSERT_EQ(0, writer.FinishEntry()); - - ASSERT_EQ(0, writer.StartEntry("file/file.txt", 0)); - ASSERT_EQ(0, writer.WriteBytes("llo", 3)); - ASSERT_EQ(0, writer.FinishEntry()); - - ASSERT_EQ(0, writer.StartEntry("file/file2.txt", 0)); - ASSERT_EQ(0, writer.FinishEntry()); - - ASSERT_EQ(0, writer.Finish()); - - ASSERT_GE(0, lseek(fd_, 0, SEEK_SET)); - - ZipArchiveHandle handle; - ASSERT_EQ(0, OpenArchiveFd(fd_, "temp", &handle, false)); - - ZipEntry data; - - ASSERT_EQ(0, FindEntry(handle, "file.txt", &data)); - EXPECT_EQ(kCompressStored, data.method); - EXPECT_EQ(2u, data.compressed_length); - ASSERT_EQ(2u, data.uncompressed_length); - ASSERT_TRUE(AssertFileEntryContentsEq("he", handle, &data)); - - ASSERT_EQ(0, FindEntry(handle, "file/file.txt", &data)); - EXPECT_EQ(kCompressStored, data.method); - EXPECT_EQ(3u, data.compressed_length); - ASSERT_EQ(3u, data.uncompressed_length); - ASSERT_TRUE(AssertFileEntryContentsEq("llo", handle, &data)); - - ASSERT_EQ(0, FindEntry(handle, "file/file2.txt", &data)); - EXPECT_EQ(kCompressStored, data.method); - EXPECT_EQ(0u, data.compressed_length); - EXPECT_EQ(0u, data.uncompressed_length); - - CloseArchive(handle); -} - -TEST_F(zipwriter, WriteUncompressedZipFileWithAlignedFlag) { - ZipWriter writer(file_); - - ASSERT_EQ(0, writer.StartEntry("align.txt", ZipWriter::kAlign32)); - ASSERT_EQ(0, writer.WriteBytes("he", 2)); - ASSERT_EQ(0, writer.FinishEntry()); - ASSERT_EQ(0, writer.Finish()); - - ASSERT_GE(0, lseek(fd_, 0, SEEK_SET)); - - ZipArchiveHandle handle; - ASSERT_EQ(0, OpenArchiveFd(fd_, "temp", &handle, false)); - - ZipEntry data; - ASSERT_EQ(0, FindEntry(handle, "align.txt", &data)); - EXPECT_EQ(0, data.offset & 0x03); - - CloseArchive(handle); -} - -static struct tm MakeTm() { - struct tm tm; - memset(&tm, 0, sizeof(struct tm)); - tm.tm_year = 2001 - 1900; - tm.tm_mon = 1; - tm.tm_mday = 12; - tm.tm_hour = 18; - tm.tm_min = 30; - tm.tm_sec = 20; - return tm; -} - -TEST_F(zipwriter, WriteUncompressedZipFileWithAlignedFlagAndTime) { - ZipWriter writer(file_); - - struct tm tm = MakeTm(); - time_t time = mktime(&tm); - ASSERT_EQ(0, writer.StartEntryWithTime("align.txt", ZipWriter::kAlign32, time)); - ASSERT_EQ(0, writer.WriteBytes("he", 2)); - ASSERT_EQ(0, writer.FinishEntry()); - ASSERT_EQ(0, writer.Finish()); - - ASSERT_GE(0, lseek(fd_, 0, SEEK_SET)); - - ZipArchiveHandle handle; - ASSERT_EQ(0, OpenArchiveFd(fd_, "temp", &handle, false)); - - ZipEntry data; - ASSERT_EQ(0, FindEntry(handle, "align.txt", &data)); - EXPECT_EQ(0, data.offset & 0x03); - - struct tm mod = data.GetModificationTime(); - EXPECT_EQ(tm.tm_sec, mod.tm_sec); - EXPECT_EQ(tm.tm_min, mod.tm_min); - EXPECT_EQ(tm.tm_hour, mod.tm_hour); - EXPECT_EQ(tm.tm_mday, mod.tm_mday); - EXPECT_EQ(tm.tm_mon, mod.tm_mon); - EXPECT_EQ(tm.tm_year, mod.tm_year); - - CloseArchive(handle); -} - -TEST_F(zipwriter, WriteUncompressedZipFileWithAlignedValue) { - ZipWriter writer(file_); - - ASSERT_EQ(0, writer.StartAlignedEntry("align.txt", 0, 4096)); - ASSERT_EQ(0, writer.WriteBytes("he", 2)); - ASSERT_EQ(0, writer.FinishEntry()); - ASSERT_EQ(0, writer.Finish()); - - ASSERT_GE(0, lseek(fd_, 0, SEEK_SET)); - - ZipArchiveHandle handle; - ASSERT_EQ(0, OpenArchiveFd(fd_, "temp", &handle, false)); - - ZipEntry data; - ASSERT_EQ(0, FindEntry(handle, "align.txt", &data)); - EXPECT_EQ(0, data.offset & 0xfff); - - CloseArchive(handle); -} - -TEST_F(zipwriter, WriteUncompressedZipFileWithAlignedValueAndTime) { - ZipWriter writer(file_); - - struct tm tm = MakeTm(); - time_t time = mktime(&tm); - ASSERT_EQ(0, writer.StartAlignedEntryWithTime("align.txt", 0, time, 4096)); - ASSERT_EQ(0, writer.WriteBytes("he", 2)); - ASSERT_EQ(0, writer.FinishEntry()); - ASSERT_EQ(0, writer.Finish()); - - ASSERT_GE(0, lseek(fd_, 0, SEEK_SET)); - - ZipArchiveHandle handle; - ASSERT_EQ(0, OpenArchiveFd(fd_, "temp", &handle, false)); - - ZipEntry data; - ASSERT_EQ(0, FindEntry(handle, "align.txt", &data)); - EXPECT_EQ(0, data.offset & 0xfff); - - struct tm mod = data.GetModificationTime(); - EXPECT_EQ(tm.tm_sec, mod.tm_sec); - EXPECT_EQ(tm.tm_min, mod.tm_min); - EXPECT_EQ(tm.tm_hour, mod.tm_hour); - EXPECT_EQ(tm.tm_mday, mod.tm_mday); - EXPECT_EQ(tm.tm_mon, mod.tm_mon); - EXPECT_EQ(tm.tm_year, mod.tm_year); - - CloseArchive(handle); -} - -TEST_F(zipwriter, WriteCompressedZipWithOneFile) { - ZipWriter writer(file_); - - ASSERT_EQ(0, writer.StartEntry("file.txt", ZipWriter::kCompress)); - ASSERT_EQ(0, writer.WriteBytes("helo", 4)); - ASSERT_EQ(0, writer.FinishEntry()); - ASSERT_EQ(0, writer.Finish()); - - ASSERT_GE(0, lseek(fd_, 0, SEEK_SET)); - - ZipArchiveHandle handle; - ASSERT_EQ(0, OpenArchiveFd(fd_, "temp", &handle, false)); - - ZipEntry data; - ASSERT_EQ(0, FindEntry(handle, "file.txt", &data)); - EXPECT_EQ(kCompressDeflated, data.method); - EXPECT_EQ(0u, data.has_data_descriptor); - ASSERT_EQ(4u, data.uncompressed_length); - ASSERT_TRUE(AssertFileEntryContentsEq("helo", handle, &data)); - - CloseArchive(handle); -} - -TEST_F(zipwriter, WriteCompressedZipFlushFull) { - // This exact data will cause the Finish() to require multiple calls - // to deflate() because the ZipWriter buffer isn't big enough to hold - // the entire compressed data buffer. - constexpr size_t kBufSize = 10000000; - std::vector<uint8_t> buffer(kBufSize); - size_t prev = 1; - for (size_t i = 0; i < kBufSize; i++) { - buffer[i] = static_cast<uint8_t>(i + prev); - prev = i; - } - - ZipWriter writer(file_); - ASSERT_EQ(0, writer.StartEntry("file.txt", ZipWriter::kCompress)); - ASSERT_EQ(0, writer.WriteBytes(buffer.data(), buffer.size())); - ASSERT_EQ(0, writer.FinishEntry()); - ASSERT_EQ(0, writer.Finish()); - - ASSERT_GE(0, lseek(fd_, 0, SEEK_SET)); - - ZipArchiveHandle handle; - ASSERT_EQ(0, OpenArchiveFd(fd_, "temp", &handle, false)); - - ZipEntry data; - ASSERT_EQ(0, FindEntry(handle, "file.txt", &data)); - EXPECT_EQ(kCompressDeflated, data.method); - EXPECT_EQ(kBufSize, data.uncompressed_length); - - std::vector<uint8_t> decompress(kBufSize); - memset(decompress.data(), 0, kBufSize); - ASSERT_EQ(0, ExtractToMemory(handle, &data, decompress.data(), - static_cast<uint32_t>(decompress.size()))); - EXPECT_EQ(0, memcmp(decompress.data(), buffer.data(), kBufSize)) - << "Input buffer and output buffer are different."; - - CloseArchive(handle); -} - -TEST_F(zipwriter, CheckStartEntryErrors) { - ZipWriter writer(file_); - - ASSERT_EQ(-5, writer.StartAlignedEntry("align.txt", ZipWriter::kAlign32, 4096)); - ASSERT_EQ(-6, writer.StartAlignedEntry("align.txt", 0, 3)); -} - -TEST_F(zipwriter, BackupRemovesTheLastFile) { - ZipWriter writer(file_); - - const char* kKeepThis = "keep this"; - const char* kDropThis = "drop this"; - const char* kReplaceWithThis = "replace with this"; - - ZipWriter::FileEntry entry; - EXPECT_LT(writer.GetLastEntry(&entry), 0); - - ASSERT_EQ(0, writer.StartEntry("keep.txt", 0)); - ASSERT_EQ(0, writer.WriteBytes(kKeepThis, strlen(kKeepThis))); - ASSERT_EQ(0, writer.FinishEntry()); - - ASSERT_EQ(0, writer.GetLastEntry(&entry)); - EXPECT_EQ("keep.txt", entry.path); - - ASSERT_EQ(0, writer.StartEntry("drop.txt", 0)); - ASSERT_EQ(0, writer.WriteBytes(kDropThis, strlen(kDropThis))); - ASSERT_EQ(0, writer.FinishEntry()); - - ASSERT_EQ(0, writer.GetLastEntry(&entry)); - EXPECT_EQ("drop.txt", entry.path); - - ASSERT_EQ(0, writer.DiscardLastEntry()); - - ASSERT_EQ(0, writer.GetLastEntry(&entry)); - EXPECT_EQ("keep.txt", entry.path); - - ASSERT_EQ(0, writer.StartEntry("replace.txt", 0)); - ASSERT_EQ(0, writer.WriteBytes(kReplaceWithThis, strlen(kReplaceWithThis))); - ASSERT_EQ(0, writer.FinishEntry()); - - ASSERT_EQ(0, writer.GetLastEntry(&entry)); - EXPECT_EQ("replace.txt", entry.path); - - ASSERT_EQ(0, writer.Finish()); - - // Verify that "drop.txt" does not exist. - - ASSERT_GE(0, lseek(fd_, 0, SEEK_SET)); - - ZipArchiveHandle handle; - ASSERT_EQ(0, OpenArchiveFd(fd_, "temp", &handle, false)); - - ZipEntry data; - ASSERT_EQ(0, FindEntry(handle, "keep.txt", &data)); - ASSERT_TRUE(AssertFileEntryContentsEq(kKeepThis, handle, &data)); - - ASSERT_NE(0, FindEntry(handle, "drop.txt", &data)); - - ASSERT_EQ(0, FindEntry(handle, "replace.txt", &data)); - ASSERT_TRUE(AssertFileEntryContentsEq(kReplaceWithThis, handle, &data)); - - CloseArchive(handle); -} - -TEST_F(zipwriter, WriteToUnseekableFile) { - const char* expected = "hello"; - ZipWriter writer(file_); - writer.seekable_ = false; - - ASSERT_EQ(0, writer.StartEntry("file.txt", 0)); - ASSERT_EQ(0, writer.WriteBytes(expected, strlen(expected))); - ASSERT_EQ(0, writer.FinishEntry()); - ASSERT_EQ(0, writer.Finish()); - ASSERT_GE(0, lseek(fd_, 0, SEEK_SET)); - - ZipArchiveHandle handle; - ASSERT_EQ(0, OpenArchiveFd(fd_, "temp", &handle, false)); - ZipEntry data; - ASSERT_EQ(0, FindEntry(handle, "file.txt", &data)); - EXPECT_EQ(kCompressStored, data.method); - EXPECT_EQ(1u, data.has_data_descriptor); - EXPECT_EQ(strlen(expected), data.compressed_length); - ASSERT_EQ(strlen(expected), data.uncompressed_length); - ASSERT_TRUE(AssertFileEntryContentsEq(expected, handle, &data)); - CloseArchive(handle); -} - -TEST_F(zipwriter, TruncateFileAfterBackup) { - ZipWriter writer(file_); - - const char* kSmall = "small"; - - ASSERT_EQ(0, writer.StartEntry("small.txt", 0)); - ASSERT_EQ(0, writer.WriteBytes(kSmall, strlen(kSmall))); - ASSERT_EQ(0, writer.FinishEntry()); - - ASSERT_EQ(0, writer.StartEntry("large.txt", 0)); - std::vector<uint8_t> data; - data.resize(1024 * 1024, 0xef); - ASSERT_EQ(0, writer.WriteBytes(data.data(), data.size())); - ASSERT_EQ(0, writer.FinishEntry()); - - off_t before_len = ftello(file_); - - ZipWriter::FileEntry entry; - ASSERT_EQ(0, writer.GetLastEntry(&entry)); - ASSERT_EQ(0, writer.DiscardLastEntry()); - - ASSERT_EQ(0, writer.Finish()); - - off_t after_len = ftello(file_); - - ASSERT_GT(before_len, after_len); -} - -static ::testing::AssertionResult AssertFileEntryContentsEq(const std::string& expected, - ZipArchiveHandle handle, - ZipEntry* zip_entry) { - if (expected.size() != zip_entry->uncompressed_length) { - return ::testing::AssertionFailure() - << "uncompressed entry size " << zip_entry->uncompressed_length - << " does not match expected size " << expected.size(); - } - - std::string actual; - actual.resize(expected.size()); - - uint8_t* buffer = reinterpret_cast<uint8_t*>(&*actual.begin()); - if (ExtractToMemory(handle, zip_entry, buffer, static_cast<uint32_t>(actual.size())) != 0) { - return ::testing::AssertionFailure() << "failed to extract entry"; - } - - if (expected != actual) { - return ::testing::AssertionFailure() << "actual zip_entry data '" << actual - << "' does not match expected '" << expected << "'"; - } - return ::testing::AssertionSuccess(); -} diff --git a/libziparchive/ziptool-tests.xml b/libziparchive/ziptool-tests.xml deleted file mode 100644 index 211119fe3..000000000 --- a/libziparchive/ziptool-tests.xml +++ /dev/null @@ -1,26 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?> -<!-- Copyright (C) 2019 The Android Open Source Project - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<configuration description="Config for running ziptool-tests through Atest or in Infra"> - <option name="test-suite-tag" value="ziptool-tests" /> - <!-- This test requires a device, so it's not annotated with a null-device. --> - <test class="com.android.tradefed.testtype.binary.ExecutableHostTest" > - <option name="binary" value="run-ziptool-tests-on-android.sh" /> - <!-- Test script assumes a relative path with the cli-tests/ folders. --> - <option name="relative-path-execution" value="true" /> - <!-- Tests shouldn't be that long but set 15m to be safe. --> - <option name="per-binary-timeout" value="15m" /> - </test> -</configuration> diff --git a/libziparchive/ziptool.cpp b/libziparchive/ziptool.cpp deleted file mode 100644 index dd42e90b9..000000000 --- a/libziparchive/ziptool.cpp +++ /dev/null @@ -1,528 +0,0 @@ -/* - * Copyright (C) 2017 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <errno.h> -#include <fcntl.h> -#include <fnmatch.h> -#include <getopt.h> -#include <inttypes.h> -#include <libgen.h> -#include <stdarg.h> -#include <stdio.h> -#include <stdlib.h> -#include <sys/stat.h> -#include <sys/types.h> -#include <time.h> -#include <unistd.h> - -#include <set> -#include <string> - -#include <android-base/file.h> -#include <android-base/strings.h> -#include <ziparchive/zip_archive.h> - -using android::base::EndsWith; -using android::base::StartsWith; - -enum OverwriteMode { - kAlways, - kNever, - kPrompt, -}; - -enum Role { - kUnzip, - kZipinfo, -}; - -static Role role; -static OverwriteMode overwrite_mode = kPrompt; -static bool flag_1 = false; -static std::string flag_d; -static bool flag_l = false; -static bool flag_p = false; -static bool flag_q = false; -static bool flag_v = false; -static bool flag_x = false; -static const char* archive_name = nullptr; -static std::set<std::string> includes; -static std::set<std::string> excludes; -static uint64_t total_uncompressed_length = 0; -static uint64_t total_compressed_length = 0; -static size_t file_count = 0; - -static const char* g_progname; - -static void die(int error, const char* fmt, ...) { - va_list ap; - - va_start(ap, fmt); - fprintf(stderr, "%s: ", g_progname); - vfprintf(stderr, fmt, ap); - if (error != 0) fprintf(stderr, ": %s", strerror(error)); - fprintf(stderr, "\n"); - va_end(ap); - exit(1); -} - -static bool ShouldInclude(const std::string& name) { - // Explicitly excluded? - if (!excludes.empty()) { - for (const auto& exclude : excludes) { - if (!fnmatch(exclude.c_str(), name.c_str(), 0)) return false; - } - } - - // Implicitly included? - if (includes.empty()) return true; - - // Explicitly included? - for (const auto& include : includes) { - if (!fnmatch(include.c_str(), name.c_str(), 0)) return true; - } - return false; -} - -static bool MakeDirectoryHierarchy(const std::string& path) { - // stat rather than lstat because a symbolic link to a directory is fine too. - struct stat sb; - if (stat(path.c_str(), &sb) != -1 && S_ISDIR(sb.st_mode)) return true; - - // Ensure the parent directories exist first. - if (!MakeDirectoryHierarchy(android::base::Dirname(path))) return false; - - // Then try to create this directory. - return (mkdir(path.c_str(), 0777) != -1); -} - -static float CompressionRatio(int64_t uncompressed, int64_t compressed) { - if (uncompressed == 0) return 0; - return static_cast<float>(100LL * (uncompressed - compressed)) / - static_cast<float>(uncompressed); -} - -static void MaybeShowHeader(ZipArchiveHandle zah) { - if (role == kUnzip) { - // unzip has three formats. - if (!flag_q) printf("Archive: %s\n", archive_name); - if (flag_v) { - printf( - " Length Method Size Cmpr Date Time CRC-32 Name\n" - "-------- ------ ------- ---- ---------- ----- -------- ----\n"); - } else if (flag_l) { - printf( - " Length Date Time Name\n" - "--------- ---------- ----- ----\n"); - } - } else { - // zipinfo. - if (!flag_1 && includes.empty() && excludes.empty()) { - ZipArchiveInfo info{GetArchiveInfo(zah)}; - printf("Archive: %s\n", archive_name); - printf("Zip file size: %" PRId64 " bytes, number of entries: %zu\n", info.archive_size, - info.entry_count); - } - } -} - -static void MaybeShowFooter() { - if (role == kUnzip) { - if (flag_v) { - printf( - "-------- ------- --- -------\n" - "%8" PRId64 " %8" PRId64 " %3.0f%% %zu file%s\n", - total_uncompressed_length, total_compressed_length, - CompressionRatio(total_uncompressed_length, total_compressed_length), file_count, - (file_count == 1) ? "" : "s"); - } else if (flag_l) { - printf( - "--------- -------\n" - "%9" PRId64 " %zu file%s\n", - total_uncompressed_length, file_count, (file_count == 1) ? "" : "s"); - } - } else { - if (!flag_1 && includes.empty() && excludes.empty()) { - printf("%zu files, %" PRId64 " bytes uncompressed, %" PRId64 " bytes compressed: %.1f%%\n", - file_count, total_uncompressed_length, total_compressed_length, - CompressionRatio(total_uncompressed_length, total_compressed_length)); - } - } -} - -static bool PromptOverwrite(const std::string& dst) { - // TODO: [r]ename not implemented because it doesn't seem useful. - printf("replace %s? [y]es, [n]o, [A]ll, [N]one: ", dst.c_str()); - fflush(stdout); - while (true) { - char* line = nullptr; - size_t n; - if (getline(&line, &n, stdin) == -1) { - die(0, "(EOF/read error; assuming [N]one...)"); - overwrite_mode = kNever; - return false; - } - if (n == 0) continue; - char cmd = line[0]; - free(line); - switch (cmd) { - case 'y': - return true; - case 'n': - return false; - case 'A': - overwrite_mode = kAlways; - return true; - case 'N': - overwrite_mode = kNever; - return false; - } - } -} - -static void ExtractToPipe(ZipArchiveHandle zah, ZipEntry& entry, const std::string& name) { - // We need to extract to memory because ExtractEntryToFile insists on - // being able to seek and truncate, and you can't do that with stdout. - uint8_t* buffer = new uint8_t[entry.uncompressed_length]; - int err = ExtractToMemory(zah, &entry, buffer, entry.uncompressed_length); - if (err < 0) { - die(0, "failed to extract %s: %s", name.c_str(), ErrorCodeString(err)); - } - if (!android::base::WriteFully(1, buffer, entry.uncompressed_length)) { - die(errno, "failed to write %s to stdout", name.c_str()); - } - delete[] buffer; -} - -static void ExtractOne(ZipArchiveHandle zah, ZipEntry& entry, const std::string& name) { - // Bad filename? - if (StartsWith(name, "/") || StartsWith(name, "../") || name.find("/../") != std::string::npos) { - die(0, "bad filename %s", name.c_str()); - } - - // Where are we actually extracting to (for human-readable output)? - // flag_d is the empty string if -d wasn't used, or has a trailing '/' - // otherwise. - std::string dst = flag_d + name; - - // Ensure the directory hierarchy exists. - if (!MakeDirectoryHierarchy(android::base::Dirname(name))) { - die(errno, "couldn't create directory hierarchy for %s", dst.c_str()); - } - - // An entry in a zip file can just be a directory itself. - if (EndsWith(name, "/")) { - if (mkdir(name.c_str(), entry.unix_mode) == -1) { - // If the directory already exists, that's fine. - if (errno == EEXIST) { - struct stat sb; - if (stat(name.c_str(), &sb) != -1 && S_ISDIR(sb.st_mode)) return; - } - die(errno, "couldn't extract directory %s", dst.c_str()); - } - return; - } - - // Create the file. - int fd = open(name.c_str(), O_CREAT | O_WRONLY | O_CLOEXEC | O_EXCL, entry.unix_mode); - if (fd == -1 && errno == EEXIST) { - if (overwrite_mode == kNever) return; - if (overwrite_mode == kPrompt && !PromptOverwrite(dst)) return; - // Either overwrite_mode is kAlways or the user consented to this specific case. - fd = open(name.c_str(), O_WRONLY | O_CREAT | O_CLOEXEC | O_TRUNC, entry.unix_mode); - } - if (fd == -1) die(errno, "couldn't create file %s", dst.c_str()); - - // Actually extract into the file. - if (!flag_q) printf(" inflating: %s\n", dst.c_str()); - int err = ExtractEntryToFile(zah, &entry, fd); - if (err < 0) die(0, "failed to extract %s: %s", dst.c_str(), ErrorCodeString(err)); - close(fd); -} - -static void ListOne(const ZipEntry& entry, const std::string& name) { - tm t = entry.GetModificationTime(); - char time[32]; - snprintf(time, sizeof(time), "%04d-%02d-%02d %02d:%02d", t.tm_year + 1900, t.tm_mon + 1, - t.tm_mday, t.tm_hour, t.tm_min); - if (flag_v) { - printf("%8d %s %7d %3.0f%% %s %08x %s\n", entry.uncompressed_length, - (entry.method == kCompressStored) ? "Stored" : "Defl:N", entry.compressed_length, - CompressionRatio(entry.uncompressed_length, entry.compressed_length), time, entry.crc32, - name.c_str()); - } else { - printf("%9d %s %s\n", entry.uncompressed_length, time, name.c_str()); - } -} - -static void InfoOne(const ZipEntry& entry, const std::string& name) { - if (flag_1) { - // "android-ndk-r19b/sources/android/NOTICE" - printf("%s\n", name.c_str()); - return; - } - - int version = entry.version_made_by & 0xff; - int os = (entry.version_made_by >> 8) & 0xff; - - // TODO: Support suid/sgid? Non-Unix/non-FAT host file system attributes? - const char* src_fs = "???"; - char mode[] = "??? "; - if (os == 0) { - src_fs = "fat"; - // https://docs.microsoft.com/en-us/windows/win32/fileio/file-attribute-constants - int attrs = entry.external_file_attributes & 0xff; - mode[0] = (attrs & 0x10) ? 'd' : '-'; - mode[1] = 'r'; - mode[2] = (attrs & 0x01) ? '-' : 'w'; - // The man page also mentions ".btm", but that seems to be obsolete? - mode[3] = EndsWith(name, ".exe") || EndsWith(name, ".com") || EndsWith(name, ".bat") || - EndsWith(name, ".cmd") - ? 'x' - : '-'; - mode[4] = (attrs & 0x20) ? 'a' : '-'; - mode[5] = (attrs & 0x02) ? 'h' : '-'; - mode[6] = (attrs & 0x04) ? 's' : '-'; - } else if (os == 3) { - src_fs = "unx"; - mode[0] = S_ISDIR(entry.unix_mode) ? 'd' : (S_ISREG(entry.unix_mode) ? '-' : '?'); - mode[1] = entry.unix_mode & S_IRUSR ? 'r' : '-'; - mode[2] = entry.unix_mode & S_IWUSR ? 'w' : '-'; - mode[3] = entry.unix_mode & S_IXUSR ? 'x' : '-'; - mode[4] = entry.unix_mode & S_IRGRP ? 'r' : '-'; - mode[5] = entry.unix_mode & S_IWGRP ? 'w' : '-'; - mode[6] = entry.unix_mode & S_IXGRP ? 'x' : '-'; - mode[7] = entry.unix_mode & S_IROTH ? 'r' : '-'; - mode[8] = entry.unix_mode & S_IWOTH ? 'w' : '-'; - mode[9] = entry.unix_mode & S_IXOTH ? 'x' : '-'; - } - - char method[5] = "stor"; - if (entry.method == kCompressDeflated) { - snprintf(method, sizeof(method), "def%c", "NXFS"[(entry.gpbf >> 1) & 0x3]); - } - - // TODO: zipinfo (unlike unzip) sometimes uses time zone? - // TODO: this uses 4-digit years because we're not barbarians unless interoperability forces it. - tm t = entry.GetModificationTime(); - char time[32]; - snprintf(time, sizeof(time), "%04d-%02d-%02d %02d:%02d", t.tm_year + 1900, t.tm_mon + 1, - t.tm_mday, t.tm_hour, t.tm_min); - - // "-rw-r--r-- 3.0 unx 577 t- defX 19-Feb-12 16:09 android-ndk-r19b/sources/android/NOTICE" - printf("%s %2d.%d %s %8d %c%c %s %s %s\n", mode, version / 10, version % 10, src_fs, - entry.uncompressed_length, entry.is_text ? 't' : 'b', - entry.has_data_descriptor ? 'X' : 'x', method, time, name.c_str()); -} - -static void ProcessOne(ZipArchiveHandle zah, ZipEntry& entry, const std::string& name) { - if (role == kUnzip) { - if (flag_l || flag_v) { - // -l or -lv or -lq or -v. - ListOne(entry, name); - } else { - // Actually extract. - if (flag_p) { - ExtractToPipe(zah, entry, name); - } else { - ExtractOne(zah, entry, name); - } - } - } else { - // zipinfo or zipinfo -1. - InfoOne(entry, name); - } - total_uncompressed_length += entry.uncompressed_length; - total_compressed_length += entry.compressed_length; - ++file_count; -} - -static void ProcessAll(ZipArchiveHandle zah) { - MaybeShowHeader(zah); - - // libziparchive iteration order doesn't match the central directory. - // We could sort, but that would cost extra and wouldn't match either. - void* cookie; - int err = StartIteration(zah, &cookie); - if (err != 0) { - die(0, "couldn't iterate %s: %s", archive_name, ErrorCodeString(err)); - } - - ZipEntry entry; - std::string name; - while ((err = Next(cookie, &entry, &name)) >= 0) { - if (ShouldInclude(name)) ProcessOne(zah, entry, name); - } - - if (err < -1) die(0, "failed iterating %s: %s", archive_name, ErrorCodeString(err)); - EndIteration(cookie); - - MaybeShowFooter(); -} - -static void ShowHelp(bool full) { - if (role == kUnzip) { - fprintf(full ? stdout : stderr, "usage: unzip [-d DIR] [-lnopqv] ZIP [FILE...] [-x FILE...]\n"); - if (!full) exit(EXIT_FAILURE); - - printf( - "\n" - "Extract FILEs from ZIP archive. Default is all files. Both the include and\n" - "exclude (-x) lists use shell glob patterns.\n" - "\n" - "-d DIR Extract into DIR\n" - "-l List contents (-lq excludes archive name, -lv is verbose)\n" - "-n Never overwrite files (default: prompt)\n" - "-o Always overwrite files\n" - "-p Pipe to stdout\n" - "-q Quiet\n" - "-v List contents verbosely\n" - "-x FILE Exclude files\n"); - } else { - fprintf(full ? stdout : stderr, "usage: zipinfo [-1] ZIP [FILE...] [-x FILE...]\n"); - if (!full) exit(EXIT_FAILURE); - - printf( - "\n" - "Show information about FILEs from ZIP archive. Default is all files.\n" - "Both the include and exclude (-x) lists use shell glob patterns.\n" - "\n" - "-1 Show filenames only, one per line\n" - "-x FILE Exclude files\n"); - } - exit(EXIT_SUCCESS); -} - -static void HandleCommonOption(int opt) { - switch (opt) { - case 'h': - ShowHelp(true); - break; - case 'x': - flag_x = true; - break; - case 1: - // -x swallows all following arguments, so we use '-' in the getopt - // string and collect files here. - if (!archive_name) { - archive_name = optarg; - } else if (flag_x) { - excludes.insert(optarg); - } else { - includes.insert(optarg); - } - break; - default: - ShowHelp(false); - break; - } -} - -int main(int argc, char* argv[]) { - // Who am I, and what am I doing? - g_progname = basename(argv[0]); - if (!strcmp(g_progname, "ziptool") && argc > 1) return main(argc - 1, argv + 1); - if (!strcmp(g_progname, "unzip")) { - role = kUnzip; - } else if (!strcmp(g_progname, "zipinfo")) { - role = kZipinfo; - } else { - die(0, "run as ziptool with unzip or zipinfo as the first argument, or symlink"); - } - - static const struct option opts[] = { - {"help", no_argument, 0, 'h'}, - {}, - }; - - if (role == kUnzip) { - // `unzip -Z` is "zipinfo mode", so in that case just restart... - if (argc > 1 && !strcmp(argv[1], "-Z")) { - argv[1] = const_cast<char*>("zipinfo"); - return main(argc - 1, argv + 1); - } - - int opt; - while ((opt = getopt_long(argc, argv, "-d:hlnopqvx", opts, nullptr)) != -1) { - switch (opt) { - case 'd': - flag_d = optarg; - if (!EndsWith(flag_d, "/")) flag_d += '/'; - break; - case 'l': - flag_l = true; - break; - case 'n': - overwrite_mode = kNever; - break; - case 'o': - overwrite_mode = kAlways; - break; - case 'p': - flag_p = flag_q = true; - break; - case 'q': - flag_q = true; - break; - case 'v': - flag_v = true; - break; - default: - HandleCommonOption(opt); - break; - } - } - } else { - int opt; - while ((opt = getopt_long(argc, argv, "-1hx", opts, nullptr)) != -1) { - switch (opt) { - case '1': - flag_1 = true; - break; - default: - HandleCommonOption(opt); - break; - } - } - } - - if (!archive_name) die(0, "missing archive filename"); - - // We can't support "-" to unzip from stdin because libziparchive relies on mmap. - ZipArchiveHandle zah; - int32_t err; - if ((err = OpenArchive(archive_name, &zah)) != 0) { - die(0, "couldn't open %s: %s", archive_name, ErrorCodeString(err)); - } - - // Implement -d by changing into that directory. - // We'll create implicit directories based on paths in the zip file, and we'll create - // the -d directory itself, but we require that *parents* of the -d directory already exists. - // This is pretty arbitrary, but it's the behavior of the original unzip. - if (!flag_d.empty()) { - if (mkdir(flag_d.c_str(), 0777) == -1 && errno != EEXIST) { - die(errno, "couldn't created %s", flag_d.c_str()); - } - if (chdir(flag_d.c_str()) == -1) { - die(errno, "couldn't chdir to %s", flag_d.c_str()); - } - } - - ProcessAll(zah); - - CloseArchive(zah); - return 0; -} diff --git a/llkd/README.md b/llkd/README.md index 191f98819..6f92f1474 100644 --- a/llkd/README.md +++ b/llkd/README.md @@ -1,199 +1,237 @@ -Android Live-LocK Daemon -======================== +<!-- +Project: /_project.yaml +Book: /_book.yaml -Introduction ------------- +{% include "_versions.html" %} +--> -Android Live-LocK Daemon (llkd) is used to catch kernel deadlocks and mitigate. +<!-- + Copyright 2020 The Android Open Source Project -Code is structured to allow integration into another service as either as part -of the main loop, or spun off as a thread should that be necessary. A default -standalone implementation is provided by llkd component. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at -The 'C' interface from libllkd component is thus: + http://www.apache.org/licenses/LICENSE-2.0 - #include "llkd.h" - bool llkInit(const char* threadname) /* return true if enabled */ - unsigned llkCheckMillseconds(void) /* ms to sleep for next check */ + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> -If a threadname is provided, a thread will be automatically spawned, otherwise -caller must call llkCheckMilliseconds in its main loop. Function will return -the period of time before the next expected call to this handler. +# Android Live-LocK Daemon (llkd) -Operations ----------- +Android 10 <!-- {{ androidQVersionNumber }} --> includes the Android Live-LocK Daemon +(`llkd`), which is designed to catch and mitigate kernel deadlocks. The `llkd` +component provides a default standalone implementation, but you can +alternatively integrate the `llkd` code into another service, either as part of +the main loop or as a separate thread. -There are two detection scenarios. Persistent D or Z state, and persistent +## Detection scenarios <!-- {:#detection-scenarios} --> + +The `llkd` has two detection scenarios: Persistent D or Z state, and persistent stack signature. -If a thread is in D or Z state with no forward progress for longer than -ro.llk.timeout_ms, or ro.llk.[D|Z].timeout_ms, kill the process or parent -process respectively. If another scan shows the same process continues to -exist, then have a confirmed live-lock condition and need to panic. Panic -the kernel in a manner to provide the greatest bugreporting details as to the -condition. Add a alarm self watchdog should llkd ever get locked up that is -double the expected time to flow through the mainloop. Sampling is every -ro.llk_sample_ms. - -For usedebug releases only, persistent stack signature checking is enabled. -If a thread in any state but Z, has a persistent listed ro.llk.stack kernel -symbol always being reported, even if there is forward scheduling progress, for -longer than ro.llk.timeout_ms, or ro.llk.stack.timeout_ms, then issue a kill -to the process. If another scan shows the same process continues to exist, -then have a confirmed live-lock condition and need to panic. There is no -ABA detection since forward scheduling progress is allowed, thus the condition -for the symbols are: - -- Check is looking for " __symbol__+0x" or " __symbol__.cfi+0x" in - /proc/__pid__/stack. -- The __symbol__ should be rare and short lived enough that on a typical - system the function is seen at most only once in a sample over the timeout - period of ro.llk.stack.timeout_ms, samples occur every ro.llk.check_ms. This - can be the only way to prevent a false trigger as there is no ABA protection. -- Persistent continuously when the live lock condition exists. -- Should be just below the function that is calling the lock that could - contend, because if the lock is below or in the symbol function, the - symbol will show in all affected processes, not just the one that - caused the lockup. - -Default will not monitor init, or [kthreadd] and all that [kthreadd] spawns. -This reduces the effectiveness of llkd by limiting its coverage. If there is -value in covering [kthreadd] spawned threads, the requirement will be that -the drivers not remain in a persistent 'D' state, or that they have mechanisms -to recover the thread should it be killed externally (this is good driver -coding hygiene, a common request to add such to publicly reviewed kernel.org -maintained drivers). For instance use wait_event_interruptible() instead of -wait_event(). The blacklists can be adjusted accordingly if these -conditions are met to cover kernel components. For the stack symbol checking, -there is an additional process blacklist so that we do not incide sepolicy -violations on services that block ptrace operations. - -An accompanying gTest set have been added, and will setup a persistent D or Z -process, with and without forward progress, but not in a live-lock state -because that would require a buggy kernel, or a module or kernel modification -to stimulate. The test will check that llkd will mitigate first by killing -the appropriate process. D state is setup by vfork() waiting for exec() in -child process. Z state is setup by fork() and an un-waited for child process. -Should be noted that both of these conditions should never happen on Android -on purpose, and llkd effectively sweeps up processes that create these -conditions. If the test can, it will reconfigure llkd to expedite the test -duration by adjusting the ro.llk.* Android properties. Tests run the D state -with some scheduling progress to ensure that ABA checking prevents false -triggers. If 100% reliable ABA on platform, then ro.llk.killtest can be -set to false; however this will result in some of the unit tests to panic -kernel instead of deal with more graceful kill operation. - -Android Properties ------------------- - -The following are the Android Properties llkd respond to. -*prop*_ms named properties are in milliseconds. -Properties that use comma (*,*) separator for lists, use a leading separator to -preserve default and add or subtract entries with (*optional*) plus (*+*) and -minus (*-*) prefixes respectively. -For these lists, the string "*false*" is synonymous with an *empty* list, -and *blank* or *missing* resorts to the specified *default* value. - -#### ro.config.low_ram -device is configured with limited memory. - -#### ro.debuggable -device is configured for userdebug or eng build. - -#### ro.llk.sysrq_t -default not ro.config.low_ram, or ro.debuggable if property is "eng". -if true do sysrq t (dump all threads). - -#### ro.llk.enable -default false, allow live-lock daemon to be enabled. - -#### llk.enable -default ro.llk.enable, and evaluated for eng. - -#### ro.khungtask.enable -default false, allow [khungtask] daemon to be enabled. - -#### khungtask.enable -default ro.khungtask.enable and evaluated for eng. - -#### ro.llk.mlockall -default false, enable call to mlockall(). - -#### ro.khungtask.timeout -default value 12 minutes, [khungtask] maximum timelimit. - -#### ro.llk.timeout_ms -default 10 minutes, D or Z maximum timelimit, double this value and it sets -the alarm watchdog for llkd. - -#### ro.llk.D.timeout_ms -default ro.llk.timeout_ms, D maximum timelimit. - -#### ro.llk.Z.timeout_ms -default ro.llk.timeout_ms, Z maximum timelimit. - -#### ro.llk.stack.timeout_ms -default ro.llk.timeout_ms, -checking for persistent stack symbols maximum timelimit. -Only active on userdebug or eng builds. - -#### ro.llk.check_ms -default 2 minutes samples of threads for D or Z. - -#### ro.llk.stack -default cma_alloc,__get_user_pages,bit_wait_io,wait_on_page_bit_killable -comma separated list of kernel symbols. -Look for kernel stack symbols that if ever persistently present can -indicate a subsystem is locked up. -Beware, check does not on purpose do forward scheduling ABA except by polling -every ro.llk_check_ms over the period ro.llk.stack.timeout_ms, so stack symbol -should be exceptionally rare and fleeting. -One must be convinced that it is virtually *impossible* for symbol to show up -persistently in all samples of the stack. -Again, looks for a match for either " **symbol**+0x" or " **symbol**.cfi+0x" -in stack expansion. -Only available on userdebug or eng builds, limited privileges due to security -concerns on user builds prevents this checking. - -#### ro.llk.blacklist.process -default 0,1,2 (kernel, init and [kthreadd]) plus process names -init,[kthreadd],[khungtaskd],lmkd,llkd,watchdogd, -[watchdogd],[watchdogd/0],...,[watchdogd/***get_nprocs**-1*]. -Do not watch these processes. A process can be comm, cmdline or pid reference. -NB: automated default here can be larger than the current maximum property -size of 92. -NB: false is a very very very unlikely process to want to blacklist. - -#### ro.llk.blacklist.parent -default 0,2,adbd&[setsid] (kernel, [kthreadd] and adbd *only for zombie setsid*). -Do not watch processes that have this parent. -An ampersand (*&*) separator is used to specify that the parent is ignored -only in combination with the target child process. -Ampersand was selected because it is never part of a process name, -however a setprop in the shell requires it to be escaped or quoted; -init rc file where this is normally specified does not have this issue. -A parent or target processes can be specified as comm, cmdline or pid reference. - -#### ro.llk.blacklist.uid -default *empty* or false, comma separated list of uid numbers or names. -Do not watch processes that match this uid. - -#### ro.llk.blacklist.process.stack -default process names init,lmkd.llkd,llkd,keystore,ueventd,apexd,logd. -This subset of processes are not monitored for live lock stack signatures. -Also prevents the sepolicy violation associated with processes that block -ptrace, as these can not be checked anyways. -Only active on userdebug and eng builds. - -Architectural Concerns ----------------------- - -- built-in [khungtask] daemon is too generic and trips on driver code that - sits around in D state too much. To switch to S instead makes the task(s) - killable, so the drivers should be able to resurrect them if needed. -- Properties are limited to 92 characters. -- Create kernel module and associated gTest to actually test panic. -- Create gTest to test out blacklist (ro.llk.blacklist.*properties* generally - not be inputs). Could require more test-only interfaces to libllkd. -- Speed up gTest using something else than ro.llk.*properties*, which should - not be inputs as they should be baked into the product. +### Persistent D or Z state <!-- {:#persistent-d-or-z-state} --> + +If a thread is in D (uninterruptible sleep) or Z (zombie) state with no forward +progress for longer than `ro.llk.timeout_ms or ro.llk.[D|Z].timeout_ms`, the +`llkd` kills the process (or parent process). If a subsequent scan shows the +same process continues to exist, the `llkd` confirms a live-lock condition and +panics the kernel in a manner that provides the most detailed bug report for the +condition. + +The `llkd` includes a self watchdog that alarms if `llkd` locks up; watchdog is +double the expected time to flow through the mainloop and sampling is every +`ro.llk_sample_ms`. + +### Persistent stack signature <!-- {:#persistent-stack-signature} --> + +For userdebug releases, the `llkd` can detect kernel live-locks using persistent +stack signature checking. If a thread in any state except Z has a persistent +listed `ro.llk.stack` kernel symbol that is reported for longer than +`ro.llk.timeout_ms` or `ro.llk.stack.timeout_ms`, the `llkd` kills the process +(even if there is forward scheduling progress). If a subsequent scan shows the +same process continues to exist, the `llkd` confirms a live-lock condition and +panics the kernel in a manner that provides the most detailed bug report for the +condition. + +Note: Because forward scheduling progress is allowed, the `llkd` does not +perform [ABA detection](https://en.wikipedia.org/wiki/ABA_problem){:.external}. + +The `lldk` check persists continuously when the live lock condition exists and +looks for the composed strings `" symbol+0x"` or `" symbol.cfi+0x"` in the +`/proc/pid/stack` file on Linux. The list of symbols is in `ro.llk.stack` and +defaults to the comma-separated list of +"`cma_alloc,__get_user_pages,bit_wait_io,wait_on_page_bit_killable`". + +Symbols should be rare and short-lived enough that on a typical system the +function is seen only once in a sample over the timeout period of +`ro.llk.stack.timeout_ms` (samples occur every `ro.llk.check_ms`). Due to lack +of ABA protection, this is the only way to prevent a false trigger. The symbol +function must appear below the function calling the lock that could contend. If +the lock is below or in the symbol function, the symbol appears in all affected +processes, not just the one that caused the lockup. + +## Coverage <!-- {:#coverage} --> + +The default implementation of `llkd` does not monitor `init`, `[kthreadd]`, or +`[kthreadd]` spawns. For the `llkd` to cover `[kthreadd]`-spawned threads: + +* Drivers must not remain in a persistent D state, + +OR + +* Drivers must have mechanisms to recover the thread should it be killed + externally. For example, use `wait_event_interruptible()` instead of + `wait_event()`. + +If one of the above conditions is met, the `llkd` ignorelist can be adjusted to +cover kernel components. Stack symbol checking involves an additional process +ignore list to prevent sepolicy violations on services that block `ptrace` +operations. + +## Android properties <!-- {:#android-properties} --> + +The `llkd` responds to several Android properties (listed below). + +* Properties named `prop_ms` are in milliseconds. +* Properties that use comma (,) separator for lists use a leading separator to + preserve the default entry, then add or subtract entries with optional plus + (+) and minus (-) prefixes respectively. For these lists, the string "false" + is synonymous with an empty list, and blank or missing entries resort to the + specified default value. + +### ro.config.low_ram <!-- {:#ro-config-low-ram} --> + +Device is configured with limited memory. + +### ro.debuggable <!-- {:#ro-debuggable} --> + +Device is configured for userdebug or eng build. + +### ro.llk.sysrq_t <!-- {:#ro-llk-sysrq-t} --> + +If property is "eng", the default is not `ro.config.low_ram` or `ro.debuggable`. +If true, dump all threads (`sysrq t`). + +### ro.llk.enable <!-- {:#ro-llk-enable} --> + +Allow live-lock daemon to be enabled. Default is false. + +### llk.enable <!-- {:#llk-enable} --> + +Evaluated for eng builds. Default is `ro.llk.enable`. + +### ro.khungtask.enable <!-- {:#ro-khungtask-enable} --> + +Allow `[khungtask]` daemon to be enabled. Default is false. + +### khungtask.enable <!-- {:#khungtask-enable} --> + +Evaluated for eng builds. Default is `ro.khungtask.enable`. + +### ro.llk.mlockall <!-- {:#ro-llk-mlockall} --> + +Enable call to `mlockall()`. Default is false. + +### ro.khungtask.timeout <!-- {:#ro-khungtask-timeout} --> + +`[khungtask]` maximum time limit. Default is 12 minutes. + +### ro.llk.timeout_ms <!-- {:#ro-llk-timeout-ms} --> + +D or Z maximum time limit. Default is 10 minutes. Double this value to set the +alarm watchdog for `llkd`. + +### ro.llk.D.timeout_ms <!-- {:#ro-llk-D-timeout-ms} --> + +D maximum time limit. Default is `ro.llk.timeout_ms`. + +### ro.llk.Z.timeout_ms <!-- {:#ro-llk-Z-timeout-ms} --> + +Z maximum time limit. Default is `ro.llk.timeout_ms`. + +### ro.llk.stack.timeout_ms <!-- {:#ro-llk-stack-timeout-ms} --> + +Checks for persistent stack symbols maximum time limit. Default is +`ro.llk.timeout_ms`. **Active only on userdebug or eng builds**. + +### ro.llk.check_ms <!-- {:#ro-llk-check-ms} --> + +Samples of threads for D or Z. Default is two minutes. + +### ro.llk.stack <!-- {:#ro-llk-stack} --> + +Checks for kernel stack symbols that if persistently present can indicate a +subsystem is locked up. Default is +`cma_alloc,__get_user_pages,bit_wait_io,wait_on_page_bit_killable` +comma-separated list of kernel symbols. The check doesn't do forward scheduling +ABA except by polling every `ro.llk_check_ms` over the period +`ro.llk.stack.timeout_ms`, so stack symbols should be exceptionally rare and +fleeting (it is highly unlikely for a symbol to show up persistently in all +samples of the stack). Checks for a match for `" symbol+0x"` or +`" symbol.cfi+0x"` in stack expansion. **Available only on userdebug or eng +builds**; security concerns on user builds result in limited privileges that +prevent this check. + +### ro.llk.ignorelist.process <!-- {:#ro-llk-ignorelist-process} --> + +The `llkd` does not watch the specified processes. Default is `0,1,2` (`kernel`, +`init`, and `[kthreadd]`) plus process names +`init,[kthreadd],[khungtaskd],lmkd,llkd,watchdogd, [watchdogd],[watchdogd/0],...,[watchdogd/get_nprocs-1]`. +A process can be a `comm`, `cmdline`, or `pid` reference. An automated default +can be larger than the current maximum property size of 92. + +Note: `false` is an extremely unlikely process to want to ignore. + +### ro.llk.ignorelist.parent <!-- {:#ro-llk-ignorelist-parent} --> + +The `llkd` does not watch processes that have the specified parent(s). Default +is `0,2,adbd&[setsid]` (`kernel`, `[kthreadd]`, and `adbd` only for zombie +`setsid`). An ampersand (&) separator specifies that the parent is ignored only +in combination with the target child process. Ampersand was selected because it +is never part of a process name; however, a `setprop` in the shell requires the +ampersand to be escaped or quoted, although the `init rc` file where this is +normally specified does not have this issue. A parent or target process can be a +`comm`, `cmdline`, or `pid` reference. + +### ro.llk.ignorelist.uid <!-- {:#ro-llk-ignorelist-uid} --> + +The `llkd` does not watch processes that match the specified uid(s). +Comma-separated list of uid numbers or names. Default is empty or false. + +### ro.llk.ignorelist.process.stack <!-- {:#ro-llk-ignorelist-process-stack} --> + +The `llkd` does not monitor the specified subset of processes for live lock stack +signatures. Default is process names +`init,lmkd.llkd,llkd,keystore,ueventd,apexd,logd`. Prevents the sepolicy +violation associated with processes that block `ptrace` (as these can't be +checked). **Active only on userdebug and eng builds**. For details on build +types, refer to [Building Android](/setup/build/building#choose-a-target). + +## Architectural concerns <!-- {:#architectural-concerns} --> + +* Properties are limited to 92 characters. However, this is not limited for + defaults defined in the `include/llkd.h` file in the sources. +* The built-in `[khungtask]` daemon is too generic and trips on driver code that + sits around in D state too much. Switching drivers to sleep, or S state, + would make task(s) killable, and need to be resurrectable by drivers on an + as-need basis. + +## Library interface (optional) <!-- {:#library-interface-optional} --> + +You can optionally incorporate the `llkd` into another privileged daemon using +the following C interface from the `libllkd` component: + +``` +#include "llkd.h" +bool llkInit(const char* threadname) /* return true if enabled */ +unsigned llkCheckMillseconds(void) /* ms to sleep for next check */ +``` + +If a threadname is provided, a thread automatically spawns, otherwise the caller +must call `llkCheckMilliseconds` in its main loop. The function returns the +period of time before the next expected call to this handler. diff --git a/llkd/include/llkd.h b/llkd/include/llkd.h index 3586ca1b1..4b20a56da 100644 --- a/llkd/include/llkd.h +++ b/llkd/include/llkd.h @@ -30,37 +30,37 @@ bool llkInit(const char* threadname); /* threadname NULL, not spawned */ unsigned llkCheckMilliseconds(void); /* clang-format off */ -#define LLK_ENABLE_WRITEABLE_PROPERTY "llk.enable" -#define LLK_ENABLE_PROPERTY "ro." LLK_ENABLE_WRITEABLE_PROPERTY -#define LLK_ENABLE_DEFAULT false /* "eng" and userdebug true */ -#define KHT_ENABLE_WRITEABLE_PROPERTY "khungtask.enable" -#define KHT_ENABLE_PROPERTY "ro." KHT_ENABLE_WRITEABLE_PROPERTY -#define LLK_ENABLE_SYSRQ_T_PROPERTY "ro.llk.sysrq_t" -#define LLK_ENABLE_SYSRQ_T_DEFAULT true -#define LLK_MLOCKALL_PROPERTY "ro.llk.mlockall" -#define LLK_MLOCKALL_DEFAULT true -#define LLK_KILLTEST_PROPERTY "ro.llk.killtest" -#define LLK_KILLTEST_DEFAULT true -#define LLK_TIMEOUT_MS_PROPERTY "ro.llk.timeout_ms" -#define KHT_TIMEOUT_PROPERTY "ro.khungtask.timeout" -#define LLK_D_TIMEOUT_MS_PROPERTY "ro.llk.D.timeout_ms" -#define LLK_Z_TIMEOUT_MS_PROPERTY "ro.llk.Z.timeout_ms" -#define LLK_STACK_TIMEOUT_MS_PROPERTY "ro.llk.stack.timeout_ms" -#define LLK_CHECK_MS_PROPERTY "ro.llk.check_ms" +#define LLK_ENABLE_WRITEABLE_PROPERTY "llk.enable" +#define LLK_ENABLE_PROPERTY "ro." LLK_ENABLE_WRITEABLE_PROPERTY +#define LLK_ENABLE_DEFAULT false /* "eng" and userdebug true */ +#define KHT_ENABLE_WRITEABLE_PROPERTY "khungtask.enable" +#define KHT_ENABLE_PROPERTY "ro." KHT_ENABLE_WRITEABLE_PROPERTY +#define LLK_ENABLE_SYSRQ_T_PROPERTY "ro.llk.sysrq_t" +#define LLK_ENABLE_SYSRQ_T_DEFAULT true +#define LLK_MLOCKALL_PROPERTY "ro.llk.mlockall" +#define LLK_MLOCKALL_DEFAULT true +#define LLK_KILLTEST_PROPERTY "ro.llk.killtest" +#define LLK_KILLTEST_DEFAULT true +#define LLK_TIMEOUT_MS_PROPERTY "ro.llk.timeout_ms" +#define KHT_TIMEOUT_PROPERTY "ro.khungtask.timeout" +#define LLK_D_TIMEOUT_MS_PROPERTY "ro.llk.D.timeout_ms" +#define LLK_Z_TIMEOUT_MS_PROPERTY "ro.llk.Z.timeout_ms" +#define LLK_STACK_TIMEOUT_MS_PROPERTY "ro.llk.stack.timeout_ms" +#define LLK_CHECK_MS_PROPERTY "ro.llk.check_ms" /* LLK_CHECK_MS_DEFAULT = actual timeout_ms / LLK_CHECKS_PER_TIMEOUT_DEFAULT */ -#define LLK_CHECKS_PER_TIMEOUT_DEFAULT 5 -#define LLK_CHECK_STACK_PROPERTY "ro.llk.stack" -#define LLK_CHECK_STACK_DEFAULT \ +#define LLK_CHECKS_PER_TIMEOUT_DEFAULT 5 +#define LLK_CHECK_STACK_PROPERTY "ro.llk.stack" +#define LLK_CHECK_STACK_DEFAULT \ "cma_alloc,__get_user_pages,bit_wait_io,wait_on_page_bit_killable" -#define LLK_BLACKLIST_PROCESS_PROPERTY "ro.llk.blacklist.process" -#define LLK_BLACKLIST_PROCESS_DEFAULT \ +#define LLK_IGNORELIST_PROCESS_PROPERTY "ro.llk.ignorelist.process" +#define LLK_IGNORELIST_PROCESS_DEFAULT \ "0,1,2,init,[kthreadd],[khungtaskd],lmkd,llkd,watchdogd,[watchdogd],[watchdogd/0]" -#define LLK_BLACKLIST_PARENT_PROPERTY "ro.llk.blacklist.parent" -#define LLK_BLACKLIST_PARENT_DEFAULT "0,2,[kthreadd],adbd&[setsid]" -#define LLK_BLACKLIST_UID_PROPERTY "ro.llk.blacklist.uid" -#define LLK_BLACKLIST_UID_DEFAULT "" -#define LLK_BLACKLIST_STACK_PROPERTY "ro.llk.blacklist.process.stack" -#define LLK_BLACKLIST_STACK_DEFAULT "init,lmkd.llkd,llkd,keystore,ueventd,apexd" +#define LLK_IGNORELIST_PARENT_PROPERTY "ro.llk.ignorelist.parent" +#define LLK_IGNORELIST_PARENT_DEFAULT "0,2,[kthreadd],adbd&[setsid]" +#define LLK_IGNORELIST_UID_PROPERTY "ro.llk.ignorelist.uid" +#define LLK_IGNORELIST_UID_DEFAULT "" +#define LLK_IGNORELIST_STACK_PROPERTY "ro.llk.ignorelist.process.stack" +#define LLK_IGNORELIST_STACK_DEFAULT "init,lmkd.llkd,llkd,keystore,ueventd,apexd" /* clang-format on */ __END_DECLS diff --git a/llkd/libllkd.cpp b/llkd/libllkd.cpp index 8ad9900ec..a24d900dc 100644 --- a/llkd/libllkd.cpp +++ b/llkd/libllkd.cpp @@ -98,26 +98,26 @@ seconds khtTimeout = duration_cast<seconds>(llkTimeoutMs * (1 + LLK_CHECKS_PER_T std::unordered_set<std::string> llkCheckStackSymbols; #endif -// Blacklist variables, initialized with comma separated lists of high false +// Ignorelist variables, initialized with comma separated lists of high false // positive and/or dangerous references, e.g. without self restart, for pid, // ppid, name and uid: // list of pids, or tids or names to skip. kernel pid (0), init pid (1), // [kthreadd] pid (2), ourselves, "init", "[kthreadd]", "lmkd", "llkd" or // combinations of watchdogd in kernel and user space. -std::unordered_set<std::string> llkBlacklistProcess; +std::unordered_set<std::string> llkIgnorelistProcess; // list of parent pids, comm or cmdline names to skip. default: // kernel pid (0), [kthreadd] (2), or ourselves, enforced and implied -std::unordered_set<std::string> llkBlacklistParent; +std::unordered_set<std::string> llkIgnorelistParent; // list of parent and target processes to skip. default: // adbd *and* [setsid] -std::unordered_map<std::string, std::unordered_set<std::string>> llkBlacklistParentAndChild; +std::unordered_map<std::string, std::unordered_set<std::string>> llkIgnorelistParentAndChild; // list of uids, and uid names, to skip, default nothing -std::unordered_set<std::string> llkBlacklistUid; +std::unordered_set<std::string> llkIgnorelistUid; #ifdef __PTRACE_ENABLED__ // list of names to skip stack checking. "init", "lmkd", "llkd", "keystore" or // "logd" (if not userdebug). -std::unordered_set<std::string> llkBlacklistStack; +std::unordered_set<std::string> llkIgnorelistStack; #endif class dir { @@ -626,9 +626,9 @@ std::string llkFormat(bool flag) { return flag ? "true" : "false"; } -std::string llkFormat(const std::unordered_set<std::string>& blacklist) { +std::string llkFormat(const std::unordered_set<std::string>& ignorelist) { std::string ret; - for (const auto& entry : blacklist) { + for (const auto& entry : ignorelist) { if (!ret.empty()) ret += ","; ret += entry; } @@ -636,10 +636,10 @@ std::string llkFormat(const std::unordered_set<std::string>& blacklist) { } std::string llkFormat( - const std::unordered_map<std::string, std::unordered_set<std::string>>& blacklist, + const std::unordered_map<std::string, std::unordered_set<std::string>>& ignorelist, bool leading_comma = false) { std::string ret; - for (const auto& entry : blacklist) { + for (const auto& entry : ignorelist) { for (const auto& target : entry.second) { if (leading_comma || !ret.empty()) ret += ","; ret += entry.first + "&" + target; @@ -699,61 +699,61 @@ std::unordered_set<std::string> llkSplit(const std::string& prop, const std::str } bool llkSkipName(const std::string& name, - const std::unordered_set<std::string>& blacklist = llkBlacklistProcess) { - if (name.empty() || blacklist.empty()) return false; + const std::unordered_set<std::string>& ignorelist = llkIgnorelistProcess) { + if (name.empty() || ignorelist.empty()) return false; - return blacklist.find(name) != blacklist.end(); + return ignorelist.find(name) != ignorelist.end(); } bool llkSkipProc(proc* procp, - const std::unordered_set<std::string>& blacklist = llkBlacklistProcess) { + const std::unordered_set<std::string>& ignorelist = llkIgnorelistProcess) { if (!procp) return false; - if (llkSkipName(std::to_string(procp->pid), blacklist)) return true; - if (llkSkipName(procp->getComm(), blacklist)) return true; - if (llkSkipName(procp->getCmdline(), blacklist)) return true; - if (llkSkipName(android::base::Basename(procp->getCmdline()), blacklist)) return true; + if (llkSkipName(std::to_string(procp->pid), ignorelist)) return true; + if (llkSkipName(procp->getComm(), ignorelist)) return true; + if (llkSkipName(procp->getCmdline(), ignorelist)) return true; + if (llkSkipName(android::base::Basename(procp->getCmdline()), ignorelist)) return true; return false; } const std::unordered_set<std::string>& llkSkipName( const std::string& name, - const std::unordered_map<std::string, std::unordered_set<std::string>>& blacklist) { + const std::unordered_map<std::string, std::unordered_set<std::string>>& ignorelist) { static const std::unordered_set<std::string> empty; - if (name.empty() || blacklist.empty()) return empty; - auto found = blacklist.find(name); - if (found == blacklist.end()) return empty; + if (name.empty() || ignorelist.empty()) return empty; + auto found = ignorelist.find(name); + if (found == ignorelist.end()) return empty; return found->second; } bool llkSkipPproc(proc* pprocp, proc* procp, const std::unordered_map<std::string, std::unordered_set<std::string>>& - blacklist = llkBlacklistParentAndChild) { - if (!pprocp || !procp || blacklist.empty()) return false; - if (llkSkipProc(procp, llkSkipName(std::to_string(pprocp->pid), blacklist))) return true; - if (llkSkipProc(procp, llkSkipName(pprocp->getComm(), blacklist))) return true; - if (llkSkipProc(procp, llkSkipName(pprocp->getCmdline(), blacklist))) return true; + ignorelist = llkIgnorelistParentAndChild) { + if (!pprocp || !procp || ignorelist.empty()) return false; + if (llkSkipProc(procp, llkSkipName(std::to_string(pprocp->pid), ignorelist))) return true; + if (llkSkipProc(procp, llkSkipName(pprocp->getComm(), ignorelist))) return true; + if (llkSkipProc(procp, llkSkipName(pprocp->getCmdline(), ignorelist))) return true; return llkSkipProc(procp, - llkSkipName(android::base::Basename(pprocp->getCmdline()), blacklist)); + llkSkipName(android::base::Basename(pprocp->getCmdline()), ignorelist)); } bool llkSkipPid(pid_t pid) { - return llkSkipName(std::to_string(pid), llkBlacklistProcess); + return llkSkipName(std::to_string(pid), llkIgnorelistProcess); } bool llkSkipPpid(pid_t ppid) { - return llkSkipName(std::to_string(ppid), llkBlacklistParent); + return llkSkipName(std::to_string(ppid), llkIgnorelistParent); } bool llkSkipUid(uid_t uid) { // Match by number? - if (llkSkipName(std::to_string(uid), llkBlacklistUid)) { + if (llkSkipName(std::to_string(uid), llkIgnorelistUid)) { return true; } // Match by name? auto pwd = ::getpwuid(uid); return (pwd != nullptr) && __predict_true(pwd->pw_name != nullptr) && - __predict_true(pwd->pw_name[0] != '\0') && llkSkipName(pwd->pw_name, llkBlacklistUid); + __predict_true(pwd->pw_name[0] != '\0') && llkSkipName(pwd->pw_name, llkIgnorelistUid); } bool getValidTidDir(dirent* dp, std::string* piddir) { @@ -811,7 +811,7 @@ bool llkCheckStack(proc* procp, const std::string& piddir) { } // Don't check process that are known to block ptrace, save sepolicy noise. - if (llkSkipProc(procp, llkBlacklistStack)) return false; + if (llkSkipProc(procp, llkIgnorelistStack)) return false; auto kernel_stack = ReadFile(piddir + "/stack"); if (kernel_stack.empty()) { LOG(VERBOSE) << piddir << "/stack empty comm=" << procp->getComm() @@ -917,12 +917,12 @@ void llkLogConfig(void) { << LLK_CHECK_MS_PROPERTY "=" << llkFormat(llkCheckMs) << "\n" #ifdef __PTRACE_ENABLED__ << LLK_CHECK_STACK_PROPERTY "=" << llkFormat(llkCheckStackSymbols) << "\n" - << LLK_BLACKLIST_STACK_PROPERTY "=" << llkFormat(llkBlacklistStack) << "\n" + << LLK_IGNORELIST_STACK_PROPERTY "=" << llkFormat(llkIgnorelistStack) << "\n" #endif - << LLK_BLACKLIST_PROCESS_PROPERTY "=" << llkFormat(llkBlacklistProcess) << "\n" - << LLK_BLACKLIST_PARENT_PROPERTY "=" << llkFormat(llkBlacklistParent) - << llkFormat(llkBlacklistParentAndChild, true) << "\n" - << LLK_BLACKLIST_UID_PROPERTY "=" << llkFormat(llkBlacklistUid); + << LLK_IGNORELIST_PROCESS_PROPERTY "=" << llkFormat(llkIgnorelistProcess) << "\n" + << LLK_IGNORELIST_PARENT_PROPERTY "=" << llkFormat(llkIgnorelistParent) + << llkFormat(llkIgnorelistParentAndChild, true) << "\n" + << LLK_IGNORELIST_UID_PROPERTY "=" << llkFormat(llkIgnorelistUid); } void* llkThread(void* obj) { @@ -932,14 +932,14 @@ void* llkThread(void* obj) { std::string name = std::to_string(::gettid()); if (!llkSkipName(name)) { - llkBlacklistProcess.emplace(name); + llkIgnorelistProcess.emplace(name); } name = static_cast<const char*>(obj); prctl(PR_SET_NAME, name.c_str()); if (__predict_false(!llkSkipName(name))) { - llkBlacklistProcess.insert(name); + llkIgnorelistProcess.insert(name); } - // No longer modifying llkBlacklistProcess. + // No longer modifying llkIgnorelistProcess. llkRunning = true; llkLogConfig(); while (llkRunning) { @@ -1122,12 +1122,12 @@ milliseconds llkCheck(bool checkRunning) { } if (pprocp) { if (llkSkipPproc(pprocp, procp)) break; - if (llkSkipProc(pprocp, llkBlacklistParent)) break; + if (llkSkipProc(pprocp, llkIgnorelistParent)) break; } else { - if (llkSkipName(std::to_string(ppid), llkBlacklistParent)) break; + if (llkSkipName(std::to_string(ppid), llkIgnorelistParent)) break; } - if ((llkBlacklistUid.size() != 0) && llkSkipUid(procp->getUid())) { + if ((llkIgnorelistUid.size() != 0) && llkSkipUid(procp->getUid())) { continue; } @@ -1320,29 +1320,29 @@ bool llkInit(const char* threadname) { if (debuggable) { llkCheckStackSymbols = llkSplit(LLK_CHECK_STACK_PROPERTY, LLK_CHECK_STACK_DEFAULT); } - std::string defaultBlacklistStack(LLK_BLACKLIST_STACK_DEFAULT); - if (!debuggable) defaultBlacklistStack += ",logd,/system/bin/logd"; - llkBlacklistStack = llkSplit(LLK_BLACKLIST_STACK_PROPERTY, defaultBlacklistStack); + std::string defaultIgnorelistStack(LLK_IGNORELIST_STACK_DEFAULT); + if (!debuggable) defaultIgnorelistStack += ",logd,/system/bin/logd"; + llkIgnorelistStack = llkSplit(LLK_IGNORELIST_STACK_PROPERTY, defaultIgnorelistStack); #endif - std::string defaultBlacklistProcess( - std::to_string(kernelPid) + "," + std::to_string(initPid) + "," + - std::to_string(kthreaddPid) + "," + std::to_string(::getpid()) + "," + - std::to_string(::gettid()) + "," LLK_BLACKLIST_PROCESS_DEFAULT); + std::string defaultIgnorelistProcess( + std::to_string(kernelPid) + "," + std::to_string(initPid) + "," + + std::to_string(kthreaddPid) + "," + std::to_string(::getpid()) + "," + + std::to_string(::gettid()) + "," LLK_IGNORELIST_PROCESS_DEFAULT); if (threadname) { - defaultBlacklistProcess += ","s + threadname; + defaultIgnorelistProcess += ","s + threadname; } for (int cpu = 1; cpu < get_nprocs_conf(); ++cpu) { - defaultBlacklistProcess += ",[watchdog/" + std::to_string(cpu) + "]"; + defaultIgnorelistProcess += ",[watchdog/" + std::to_string(cpu) + "]"; } - llkBlacklistProcess = llkSplit(LLK_BLACKLIST_PROCESS_PROPERTY, defaultBlacklistProcess); + llkIgnorelistProcess = llkSplit(LLK_IGNORELIST_PROCESS_PROPERTY, defaultIgnorelistProcess); if (!llkSkipName("[khungtaskd]")) { // ALWAYS ignore as special - llkBlacklistProcess.emplace("[khungtaskd]"); + llkIgnorelistProcess.emplace("[khungtaskd]"); } - llkBlacklistParent = llkSplit(LLK_BLACKLIST_PARENT_PROPERTY, - std::to_string(kernelPid) + "," + std::to_string(kthreaddPid) + - "," LLK_BLACKLIST_PARENT_DEFAULT); - // derive llkBlacklistParentAndChild by moving entries with '&' from above - for (auto it = llkBlacklistParent.begin(); it != llkBlacklistParent.end();) { + llkIgnorelistParent = llkSplit(LLK_IGNORELIST_PARENT_PROPERTY, + std::to_string(kernelPid) + "," + std::to_string(kthreaddPid) + + "," LLK_IGNORELIST_PARENT_DEFAULT); + // derive llkIgnorelistParentAndChild by moving entries with '&' from above + for (auto it = llkIgnorelistParent.begin(); it != llkIgnorelistParent.end();) { auto pos = it->find('&'); if (pos == std::string::npos) { ++it; @@ -1350,18 +1350,18 @@ bool llkInit(const char* threadname) { } auto parent = it->substr(0, pos); auto child = it->substr(pos + 1); - it = llkBlacklistParent.erase(it); + it = llkIgnorelistParent.erase(it); - auto found = llkBlacklistParentAndChild.find(parent); - if (found == llkBlacklistParentAndChild.end()) { - llkBlacklistParentAndChild.emplace(std::make_pair( + auto found = llkIgnorelistParentAndChild.find(parent); + if (found == llkIgnorelistParentAndChild.end()) { + llkIgnorelistParentAndChild.emplace(std::make_pair( std::move(parent), std::unordered_set<std::string>({std::move(child)}))); } else { found->second.emplace(std::move(child)); } } - llkBlacklistUid = llkSplit(LLK_BLACKLIST_UID_PROPERTY, LLK_BLACKLIST_UID_DEFAULT); + llkIgnorelistUid = llkSplit(LLK_IGNORELIST_UID_PROPERTY, LLK_IGNORELIST_UID_DEFAULT); // internal watchdog ::signal(SIGALRM, llkAlarmHandler); diff --git a/logcat/event.logtags b/logcat/event.logtags index 56a670a17..93c3d6d09 100644 --- a/logcat/event.logtags +++ b/logcat/event.logtags @@ -116,6 +116,9 @@ # audio # 61000 - 61199 reserved for audioserver +# input +# 62000 - 62199 reserved for inputflinger + # com.android.server.policy # 70000 - 70199 reserved for PhoneWindowManager and other policies diff --git a/logcat/logcat.cpp b/logcat/logcat.cpp index b8c143d98..cf98dad7c 100644 --- a/logcat/logcat.cpp +++ b/logcat/logcat.cpp @@ -36,6 +36,7 @@ #include <memory> #include <regex> +#include <set> #include <string> #include <utility> #include <vector> @@ -50,6 +51,7 @@ #include <android/log.h> #include <log/event_tag_map.h> #include <log/log_id.h> +#include <log/log_read.h> #include <log/logprint.h> #include <private/android_logger.h> #include <processgroup/sched_policy.h> @@ -331,14 +333,14 @@ Logd control: This can individually control each buffer's size with -b. -S, --statistics Output statistics. --pid can be used to provide pid specific stats. - -p, --prune Print prune white and ~black list. Service is specified as UID, - UID/PID or /PID. Weighed for quicker pruning if prefix with ~, - otherwise weighed for longevity if unadorned. All other pruning - activity is oldest first. Special case ~! represents an automatic - quicker pruning for the noisiest UID as determined by the current - statistics. - -P, --prune='<list> ...' Set prune white and ~black list, using same format as listed above. - Must be quoted. + -p, --prune Print prune rules. Each rule is specified as UID, UID/PID or /PID. A + '~' prefix indicates that elements matching the rule should be pruned + with higher priority otherwise they're pruned with lower priority. All + other pruning activity is oldest first. Special case ~! represents an + automatic pruning for the noisiest UID as determined by the current + statistics. Special case ~1000/! represents pruning of the worst PID + within AID_SYSTEM when AID_SYSTEM is the noisiest UID. + -P, --prune='<list> ...' Set prune rules, using same format as listed above. Must be quoted. Filtering: -s Set default filter to silent. Equivalent to filterspec '*:S' @@ -357,6 +359,10 @@ Filtering: -T '<time>' Print the lines since specified time (not imply -d). count is pure numerical, time is 'MM-DD hh:mm:ss.mmm...' 'YYYY-MM-DD hh:mm:ss.mmm...' or 'sssss.mmm...' format. + --uid=<uids> Only display log messages from UIDs present in the comma separate list + <uids>. No name look-up is performed, so UIDs must be provided as + numeric values. This option is only useful for the 'root', 'log', and + 'system' users since only those users can view logs from other users. )init"); fprintf(stderr, "\nfilterspecs are a series of \n" @@ -458,7 +464,7 @@ static log_time lastLogTime(const char* outputFileName) { closedir); if (!dir.get()) return retval; - log_time now(android_log_clockid()); + log_time now(CLOCK_REALTIME); size_t len = strlen(file); log_time modulo(0, NS_PER_SEC); @@ -527,13 +533,14 @@ int Logcat::Run(int argc, char** argv) { unsigned long setLogSize = 0; const char* setPruneList = nullptr; const char* setId = nullptr; - int mode = ANDROID_LOG_RDONLY; + int mode = 0; std::string forceFilters; size_t tail_lines = 0; log_time tail_time(log_time::EPOCH); size_t pid = 0; bool got_t = false; unsigned id_mask = 0; + std::set<uid_t> uids; if (argc == 2 && !strcmp(argv[1], "--help")) { show_help(); @@ -553,6 +560,7 @@ int Logcat::Run(int argc, char** argv) { static const char id_str[] = "id"; static const char wrap_str[] = "wrap"; static const char print_str[] = "print"; + static const char uid_str[] = "uid"; // clang-format off static const struct option long_options[] = { { "binary", no_argument, nullptr, 'B' }, @@ -580,6 +588,7 @@ int Logcat::Run(int argc, char** argv) { { "statistics", no_argument, nullptr, 'S' }, // hidden and undocumented reserved alias for -t { "tail", required_argument, nullptr, 't' }, + { uid_str, required_argument, nullptr, 0 }, // support, but ignore and do not document, the optional argument { wrap_str, optional_argument, nullptr, 0 }, { nullptr, 0, nullptr, 0 } @@ -605,8 +614,7 @@ int Logcat::Run(int argc, char** argv) { break; } if (long_options[option_index].name == wrap_str) { - mode |= ANDROID_LOG_WRAP | ANDROID_LOG_RDONLY | - ANDROID_LOG_NONBLOCK; + mode |= ANDROID_LOG_WRAP | ANDROID_LOG_NONBLOCK; // ToDo: implement API that supports setting a wrap timeout size_t dummy = ANDROID_LOG_WRAP_DEFAULT_TIMEOUT; if (optarg && (!ParseUint(optarg, &dummy) || dummy < 1)) { @@ -631,6 +639,17 @@ int Logcat::Run(int argc, char** argv) { if (long_options[option_index].name == id_str) { setId = (optarg && optarg[0]) ? optarg : nullptr; } + if (long_options[option_index].name == uid_str) { + auto uid_strings = Split(optarg, delimiters); + for (const auto& uid_string : uid_strings) { + uid_t uid; + if (!ParseUint(uid_string, &uid)) { + error(EXIT_FAILURE, 0, "Unable to parse UID '%s'", uid_string.c_str()); + } + uids.emplace(uid); + } + break; + } break; case 's': @@ -640,21 +659,19 @@ int Logcat::Run(int argc, char** argv) { case 'c': clearLog = true; - mode |= ANDROID_LOG_WRONLY; break; case 'L': - mode |= ANDROID_LOG_RDONLY | ANDROID_LOG_PSTORE | - ANDROID_LOG_NONBLOCK; + mode |= ANDROID_LOG_PSTORE | ANDROID_LOG_NONBLOCK; break; case 'd': - mode |= ANDROID_LOG_RDONLY | ANDROID_LOG_NONBLOCK; + mode |= ANDROID_LOG_NONBLOCK; break; case 't': got_t = true; - mode |= ANDROID_LOG_RDONLY | ANDROID_LOG_NONBLOCK; + mode |= ANDROID_LOG_NONBLOCK; FALLTHROUGH_INTENDED; case 'T': if (strspn(optarg, "0123456789") != strlen(optarg)) { @@ -1166,6 +1183,10 @@ If you have enabled significant logging, look into using the -G option to increa LOG_ID_MAX); } + if (!uids.empty() && uids.count(log_msg.entry.uid) == 0) { + continue; + } + PrintDividers(log_msg.id(), printDividers); if (print_binary_) { diff --git a/logcat/tests/logcat_test.cpp b/logcat/tests/logcat_test.cpp index b32b43737..61aa9381c 100644 --- a/logcat/tests/logcat_test.cpp +++ b/logcat/tests/logcat_test.cpp @@ -16,6 +16,7 @@ #include <ctype.h> #include <dirent.h> +#include <pwd.h> #include <signal.h> #include <stdint.h> #include <stdio.h> @@ -28,10 +29,12 @@ #include <unistd.h> #include <memory> +#include <regex> #include <string> #include <android-base/file.h> #include <android-base/macros.h> +#include <android-base/parseint.h> #include <android-base/stringprintf.h> #include <android-base/strings.h> #include <gtest/gtest.h> @@ -174,11 +177,6 @@ static size_t inject(ssize_t count) { } TEST(logcat, year) { - if (android_log_clockid() == CLOCK_MONOTONIC) { - fprintf(stderr, "Skipping test, logd is monotonic time\n"); - return; - } - int count; int tries = 3; // in case run too soon after system start or buffer clear @@ -249,11 +247,6 @@ static char* fgetLongTime(char* buffer, size_t buflen, FILE* fp) { } TEST(logcat, tz) { - if (android_log_clockid() == CLOCK_MONOTONIC) { - fprintf(stderr, "Skipping test, logd is monotonic time\n"); - return; - } - int tries = 4; // in case run too soon after system start or buffer clear int count; @@ -485,8 +478,8 @@ TEST(logcat, End_to_End) { continue; } - log_time tx((const char*)&t); - if (ts == tx) { + log_time* tx = reinterpret_cast<log_time*>(&t); + if (ts == *tx) { ++count; } } @@ -531,8 +524,8 @@ TEST(logcat, End_to_End_multitude) { continue; } - log_time tx((const char*)&t); - if (ts == tx) { + log_time* tx = reinterpret_cast<log_time*>(&t); + if (ts == *tx) { ++count; } } @@ -1308,7 +1301,7 @@ TEST(logcat, blocking_clear) { } #endif -static bool get_white_black(char** list) { +static bool get_prune_rules(char** list) { FILE* fp = popen(logcat_executable " -p 2>/dev/null", "r"); if (fp == NULL) { fprintf(stderr, "ERROR: logcat -p 2>/dev/null\n"); @@ -1341,7 +1334,7 @@ static bool get_white_black(char** list) { return *list != NULL; } -static bool set_white_black(const char* list) { +static bool set_prune_rules(const char* list) { char buffer[BIG_BUFFER]; snprintf(buffer, sizeof(buffer), logcat_executable " -P '%s' 2>&1", list ? list : ""); @@ -1370,28 +1363,28 @@ static bool set_white_black(const char* list) { return pclose(fp) == 0; } -TEST(logcat, white_black_adjust) { +TEST(logcat, prune_rules_adjust) { char* list = NULL; char* adjust = NULL; - get_white_black(&list); + get_prune_rules(&list); static const char adjustment[] = "~! 300/20 300/25 2000 ~1000/5 ~1000/30"; - ASSERT_EQ(true, set_white_black(adjustment)); - ASSERT_EQ(true, get_white_black(&adjust)); + ASSERT_EQ(true, set_prune_rules(adjustment)); + ASSERT_EQ(true, get_prune_rules(&adjust)); EXPECT_STREQ(adjustment, adjust); free(adjust); adjust = NULL; static const char adjustment2[] = "300/20 300/21 2000 ~1000"; - ASSERT_EQ(true, set_white_black(adjustment2)); - ASSERT_EQ(true, get_white_black(&adjust)); + ASSERT_EQ(true, set_prune_rules(adjustment2)); + ASSERT_EQ(true, get_prune_rules(&adjust)); EXPECT_STREQ(adjustment2, adjust); free(adjust); adjust = NULL; - ASSERT_EQ(true, set_white_black(list)); - get_white_black(&adjust); + ASSERT_EQ(true, set_prune_rules(list)); + get_prune_rules(&adjust); EXPECT_STREQ(list ? list : "", adjust ? adjust : ""); free(adjust); adjust = NULL; @@ -1758,3 +1751,83 @@ TEST(logcat, invalid_buffer) { ASSERT_TRUE(android::base::StartsWith(output, "unknown buffer foo\n")); } + +static void SniffUid(const std::string& line, uid_t& uid) { + auto uid_regex = std::regex{"\\S+\\s+\\S+\\s+(\\S+).*"}; + + auto trimmed_line = android::base::Trim(line); + + std::smatch match_results; + ASSERT_TRUE(std::regex_match(trimmed_line, match_results, uid_regex)) + << "Unable to find UID in line '" << trimmed_line << "'"; + auto uid_string = match_results[1]; + if (!android::base::ParseUint(uid_string, &uid)) { + auto pwd = getpwnam(uid_string.str().c_str()); + ASSERT_NE(nullptr, pwd) << "uid '" << uid_string << "' in line '" << trimmed_line << "'"; + uid = pwd->pw_uid; + } +} + +static void UidsInLog(std::optional<std::vector<uid_t>> filter_uid, std::map<uid_t, size_t>& uids) { + std::string command; + if (filter_uid) { + std::vector<std::string> uid_strings; + for (const auto& uid : *filter_uid) { + uid_strings.emplace_back(std::to_string(uid)); + } + command = android::base::StringPrintf(logcat_executable + " -v uid -b all -d 2>/dev/null --uid=%s", + android::base::Join(uid_strings, ",").c_str()); + } else { + command = logcat_executable " -v uid -b all -d 2>/dev/null"; + } + auto fp = std::unique_ptr<FILE, decltype(&pclose)>(popen(command.c_str(), "r"), pclose); + ASSERT_NE(nullptr, fp); + + char buffer[BIG_BUFFER]; + while (fgets(buffer, sizeof(buffer), fp.get())) { + // Ignore dividers, e.g. '--------- beginning of radio' + if (android::base::StartsWith(buffer, "---------")) { + continue; + } + uid_t uid; + SniffUid(buffer, uid); + uids[uid]++; + } +} + +static std::vector<uid_t> TopTwoInMap(const std::map<uid_t, size_t>& uids) { + std::pair<uid_t, size_t> top = {0, 0}; + std::pair<uid_t, size_t> second = {0, 0}; + for (const auto& [uid, count] : uids) { + if (count > top.second) { + top = second; + top = {uid, count}; + } else if (count > second.second) { + second = {uid, count}; + } + } + return {top.first, second.first}; +} + +TEST(logcat, uid_filter) { + std::map<uid_t, size_t> uids; + UidsInLog({}, uids); + + ASSERT_GT(uids.size(), 2U); + auto top_uids = TopTwoInMap(uids); + + // Test filtering with --uid=<top uid> + std::map<uid_t, size_t> uids_only_top; + std::vector<uid_t> top_uid = {top_uids[0]}; + UidsInLog(top_uid, uids_only_top); + + EXPECT_EQ(1U, uids_only_top.size()); + + // Test filtering with --uid=<top uid>,<2nd top uid> + std::map<uid_t, size_t> uids_only_top2; + std::vector<uid_t> top2_uids = {top_uids[0], top_uids[1]}; + UidsInLog(top2_uids, uids_only_top2); + + EXPECT_EQ(2U, uids_only_top2.size()); +} diff --git a/logd/Android.bp b/logd/Android.bp index b337b7c2a..036cb7e44 100644 --- a/logd/Android.bp +++ b/logd/Android.bp @@ -28,39 +28,63 @@ event_flag = [ "-DLIBLOG_LOG_TAG=1006", ] +cc_defaults { + name: "logd_defaults", + + shared_libs: [ + "libbase", + "libz", + ], + static_libs: ["libzstd"], + cflags: [ + "-Wextra", + "-Wthread-safety", + ] + event_flag, + + lto: { + thin: true, + }, + cpp_std: "experimental", +} + cc_library_static { name: "liblogd", - + defaults: ["logd_defaults"], + host_supported: true, srcs: [ - "LogCommand.cpp", - "CommandListener.cpp", - "LogListener.cpp", - "LogReader.cpp", - "FlushCommand.cpp", - "LogBuffer.cpp", + "ChattyLogBuffer.cpp", + "CompressionEngine.cpp", + "LogReaderList.cpp", + "LogReaderThread.cpp", "LogBufferElement.cpp", - "LogTimes.cpp", "LogStatistics.cpp", - "LogWhiteBlackList.cpp", - "libaudit.c", - "LogAudit.cpp", - "LogKlog.cpp", "LogTags.cpp", + "PruneList.cpp", + "SerializedFlushToState.cpp", + "SerializedLogBuffer.cpp", + "SerializedLogChunk.cpp", + "SimpleLogBuffer.cpp", ], logtags: ["event.logtags"], - shared_libs: ["libbase"], - export_include_dirs: ["."], - - cflags: ["-Werror"] + event_flag, } cc_binary { name: "logd", + defaults: ["logd_defaults"], init_rc: ["logd.rc"], - srcs: ["main.cpp"], + srcs: [ + "main.cpp", + "LogPermissions.cpp", + "CommandListener.cpp", + "LogListener.cpp", + "LogReader.cpp", + "LogAudit.cpp", + "LogKlog.cpp", + "libaudit.cpp", + ], static_libs: [ "liblog", @@ -70,31 +94,24 @@ cc_binary { shared_libs: [ "libsysutils", "libcutils", - "libbase", "libpackagelistparser", "libprocessgroup", "libcap", ], - - cflags: ["-Werror"], } cc_binary { name: "auditctl", - srcs: ["auditctl.cpp"], - - static_libs: [ - "liblogd", + srcs: [ + "auditctl.cpp", + "libaudit.cpp", ], shared_libs: ["libbase"], cflags: [ - "-Wall", "-Wextra", - "-Werror", - "-Wconversion" ], } @@ -103,3 +120,63 @@ prebuilt_etc { src: "logtagd.rc", sub_dir: "init", } + +// ----------------------------------------------------------------------------- +// Unit tests. +// ----------------------------------------------------------------------------- + +cc_defaults { + name: "logd-unit-test-defaults", + + cflags: [ + "-fstack-protector-all", + "-g", + "-Wall", + "-Wextra", + "-Werror", + "-fno-builtin", + ] + event_flag, + + srcs: [ + "ChattyLogBufferTest.cpp", + "logd_test.cpp", + "LogBufferTest.cpp", + "SerializedLogChunkTest.cpp", + "SerializedFlushToStateTest.cpp", + ], + + static_libs: [ + "libbase", + "libcutils", + "liblog", + "liblogd", + "libselinux", + "libz", + "libzstd", + ], +} + +// Build tests for the logger. Run with: +// adb shell /data/nativetest/logd-unit-tests/logd-unit-tests +cc_test { + name: "logd-unit-tests", + host_supported: true, + defaults: ["logd-unit-test-defaults"], +} + +cc_test { + name: "CtsLogdTestCases", + defaults: ["logd-unit-test-defaults"], + multilib: { + lib32: { + suffix: "32", + }, + lib64: { + suffix: "64", + }, + }, + test_suites: [ + "cts", + "vts10", + ], +} diff --git a/logd/tests/AndroidTest.xml b/logd/AndroidTest.xml index a25dc440f..a25dc440f 100644 --- a/logd/tests/AndroidTest.xml +++ b/logd/AndroidTest.xml diff --git a/logd/ChattyLogBuffer.cpp b/logd/ChattyLogBuffer.cpp new file mode 100644 index 000000000..fd183e4d6 --- /dev/null +++ b/logd/ChattyLogBuffer.cpp @@ -0,0 +1,619 @@ +/* + * Copyright (C) 2012-2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// for manual checking of stale entries during ChattyLogBuffer::erase() +//#define DEBUG_CHECK_FOR_STALE_ENTRIES + +#include "ChattyLogBuffer.h" + +#include <ctype.h> +#include <endian.h> +#include <errno.h> +#include <stdio.h> +#include <string.h> +#include <sys/cdefs.h> +#include <sys/user.h> +#include <time.h> +#include <unistd.h> + +#include <limits> +#include <unordered_map> +#include <utility> + +#include <private/android_logger.h> + +#include "LogUtils.h" + +#ifndef __predict_false +#define __predict_false(exp) __builtin_expect((exp) != 0, 0) +#endif + +ChattyLogBuffer::ChattyLogBuffer(LogReaderList* reader_list, LogTags* tags, PruneList* prune, + LogStatistics* stats) + : SimpleLogBuffer(reader_list, tags, stats), prune_(prune) {} + +ChattyLogBuffer::~ChattyLogBuffer() {} + +enum match_type { DIFFERENT, SAME, SAME_LIBLOG }; + +static enum match_type Identical(const LogBufferElement& elem, const LogBufferElement& last) { + ssize_t lenl = elem.msg_len(); + if (lenl <= 0) return DIFFERENT; // value if this represents a chatty elem + ssize_t lenr = last.msg_len(); + if (lenr <= 0) return DIFFERENT; // value if this represents a chatty elem + if (elem.uid() != last.uid()) return DIFFERENT; + if (elem.pid() != last.pid()) return DIFFERENT; + if (elem.tid() != last.tid()) return DIFFERENT; + + // last is more than a minute old, stop squashing identical messages + if (elem.realtime().nsec() > (last.realtime().nsec() + 60 * NS_PER_SEC)) return DIFFERENT; + + // Identical message + const char* msgl = elem.msg(); + const char* msgr = last.msg(); + if (lenl == lenr) { + if (!fastcmp<memcmp>(msgl, msgr, lenl)) return SAME; + // liblog tagged messages (content gets summed) + if (elem.log_id() == LOG_ID_EVENTS && lenl == sizeof(android_log_event_int_t) && + !fastcmp<memcmp>(msgl, msgr, sizeof(android_log_event_int_t) - sizeof(int32_t)) && + elem.GetTag() == LIBLOG_LOG_TAG) { + return SAME_LIBLOG; + } + } + + // audit message (except sequence number) identical? + if (IsBinary(last.log_id()) && + lenl > static_cast<ssize_t>(sizeof(android_log_event_string_t)) && + lenr > static_cast<ssize_t>(sizeof(android_log_event_string_t))) { + if (fastcmp<memcmp>(msgl, msgr, sizeof(android_log_event_string_t) - sizeof(int32_t))) { + return DIFFERENT; + } + msgl += sizeof(android_log_event_string_t); + lenl -= sizeof(android_log_event_string_t); + msgr += sizeof(android_log_event_string_t); + lenr -= sizeof(android_log_event_string_t); + } + static const char avc[] = "): avc: "; + const char* avcl = android::strnstr(msgl, lenl, avc); + if (!avcl) return DIFFERENT; + lenl -= avcl - msgl; + const char* avcr = android::strnstr(msgr, lenr, avc); + if (!avcr) return DIFFERENT; + lenr -= avcr - msgr; + if (lenl != lenr) return DIFFERENT; + if (fastcmp<memcmp>(avcl + strlen(avc), avcr + strlen(avc), lenl - strlen(avc))) { + return DIFFERENT; + } + return SAME; +} + +void ChattyLogBuffer::LogInternal(LogBufferElement&& elem) { + // b/137093665: don't coalesce security messages. + if (elem.log_id() == LOG_ID_SECURITY) { + SimpleLogBuffer::LogInternal(std::move(elem)); + return; + } + int log_id = elem.log_id(); + + // Initialize last_logged_elements_ to a copy of elem if logging the first element for a log_id. + if (!last_logged_elements_[log_id]) { + last_logged_elements_[log_id].emplace(elem); + SimpleLogBuffer::LogInternal(std::move(elem)); + return; + } + + LogBufferElement& current_last = *last_logged_elements_[log_id]; + enum match_type match = Identical(elem, current_last); + + if (match == DIFFERENT) { + if (duplicate_elements_[log_id]) { + // If we previously had 3+ identical messages, log the chatty message. + if (duplicate_elements_[log_id]->dropped_count() > 0) { + SimpleLogBuffer::LogInternal(std::move(*duplicate_elements_[log_id])); + } + duplicate_elements_[log_id].reset(); + // Log the saved copy of the last identical message seen. + SimpleLogBuffer::LogInternal(std::move(current_last)); + } + last_logged_elements_[log_id].emplace(elem); + SimpleLogBuffer::LogInternal(std::move(elem)); + return; + } + + // 2 identical message: set duplicate_elements_ appropriately. + if (!duplicate_elements_[log_id]) { + duplicate_elements_[log_id].emplace(std::move(current_last)); + last_logged_elements_[log_id].emplace(std::move(elem)); + return; + } + + // 3+ identical LIBLOG event messages: coalesce them into last_logged_elements_. + if (match == SAME_LIBLOG) { + const android_log_event_int_t* current_last_event = + reinterpret_cast<const android_log_event_int_t*>(current_last.msg()); + int64_t current_last_count = current_last_event->payload.data; + android_log_event_int_t* elem_event = + reinterpret_cast<android_log_event_int_t*>(const_cast<char*>(elem.msg())); + int64_t elem_count = elem_event->payload.data; + + int64_t total = current_last_count + elem_count; + if (total > std::numeric_limits<int32_t>::max()) { + SimpleLogBuffer::LogInternal(std::move(current_last)); + last_logged_elements_[log_id].emplace(std::move(elem)); + return; + } + stats()->AddTotal(current_last.log_id(), current_last.msg_len()); + elem_event->payload.data = total; + last_logged_elements_[log_id].emplace(std::move(elem)); + return; + } + + // 3+ identical messages (not LIBLOG) messages: increase the drop count. + uint16_t dropped_count = duplicate_elements_[log_id]->dropped_count(); + if (dropped_count == std::numeric_limits<uint16_t>::max()) { + SimpleLogBuffer::LogInternal(std::move(*duplicate_elements_[log_id])); + dropped_count = 0; + } + // We're dropping the current_last log so add its stats to the total. + stats()->AddTotal(current_last.log_id(), current_last.msg_len()); + // Use current_last for tracking the dropped count to always use the latest timestamp. + current_last.SetDropped(dropped_count + 1); + duplicate_elements_[log_id].emplace(std::move(current_last)); + last_logged_elements_[log_id].emplace(std::move(elem)); +} + +LogBufferElementCollection::iterator ChattyLogBuffer::Erase(LogBufferElementCollection::iterator it, + bool coalesce) { + LogBufferElement& element = *it; + log_id_t id = element.log_id(); + + // Remove iterator references in the various lists that will become stale + // after the element is erased from the main logging list. + + { // start of scope for found iterator + int key = (id == LOG_ID_EVENTS || id == LOG_ID_SECURITY) ? element.GetTag() : element.uid(); + LogBufferIteratorMap::iterator found = mLastWorst[id].find(key); + if ((found != mLastWorst[id].end()) && (it == found->second)) { + mLastWorst[id].erase(found); + } + } + + { // start of scope for pid found iterator + // element->uid() may not be AID_SYSTEM for next-best-watermark. + // will not assume id != LOG_ID_EVENTS or LOG_ID_SECURITY for KISS and + // long term code stability, find() check should be fast for those ids. + LogBufferPidIteratorMap::iterator found = mLastWorstPidOfSystem[id].find(element.pid()); + if (found != mLastWorstPidOfSystem[id].end() && it == found->second) { + mLastWorstPidOfSystem[id].erase(found); + } + } + +#ifdef DEBUG_CHECK_FOR_STALE_ENTRIES + LogBufferElementCollection::iterator bad = it; + int key = (id == LOG_ID_EVENTS || id == LOG_ID_SECURITY) ? element->GetTag() : element->uid(); +#endif + + if (coalesce) { + stats()->Erase(element.ToLogStatisticsElement()); + } else { + stats()->Subtract(element.ToLogStatisticsElement()); + } + + it = SimpleLogBuffer::Erase(it); + +#ifdef DEBUG_CHECK_FOR_STALE_ENTRIES + log_id_for_each(i) { + for (auto b : mLastWorst[i]) { + if (bad == b.second) { + LOG(ERROR) << StringPrintf("stale mLastWorst[%d] key=%d mykey=%d", i, b.first, key); + } + } + for (auto b : mLastWorstPidOfSystem[i]) { + if (bad == b.second) { + LOG(ERROR) << StringPrintf("stale mLastWorstPidOfSystem[%d] pid=%d", i, b.first); + } + } + } +#endif + return it; +} + +// Define a temporary mechanism to report the last LogBufferElement pointer +// for the specified uid, pid and tid. Used below to help merge-sort when +// pruning for worst UID. +class LogBufferElementLast { + typedef std::unordered_map<uint64_t, LogBufferElement*> LogBufferElementMap; + LogBufferElementMap map; + + public: + bool coalesce(LogBufferElement* element, uint16_t dropped) { + uint64_t key = LogBufferElementKey(element->uid(), element->pid(), element->tid()); + LogBufferElementMap::iterator it = map.find(key); + if (it != map.end()) { + LogBufferElement* found = it->second; + uint16_t moreDropped = found->dropped_count(); + if ((dropped + moreDropped) > USHRT_MAX) { + map.erase(it); + } else { + found->SetDropped(dropped + moreDropped); + return true; + } + } + return false; + } + + void add(LogBufferElement* element) { + uint64_t key = LogBufferElementKey(element->uid(), element->pid(), element->tid()); + map[key] = element; + } + + void clear() { map.clear(); } + + void clear(LogBufferElement* element) { + uint64_t current = element->realtime().nsec() - (EXPIRE_RATELIMIT * NS_PER_SEC); + for (LogBufferElementMap::iterator it = map.begin(); it != map.end();) { + LogBufferElement* mapElement = it->second; + if (mapElement->dropped_count() >= EXPIRE_THRESHOLD && + current > mapElement->realtime().nsec()) { + it = map.erase(it); + } else { + ++it; + } + } + } + + private: + uint64_t LogBufferElementKey(uid_t uid, pid_t pid, pid_t tid) { + return uint64_t(uid) << 32 | uint64_t(pid) << 16 | uint64_t(tid); + } +}; + +// prune "pruneRows" of type "id" from the buffer. +// +// This garbage collection task is used to expire log entries. It is called to +// remove all logs (clear), all UID logs (unprivileged clear), or every +// 256 or 10% of the total logs (whichever is less) to prune the logs. +// +// First there is a prep phase where we discover the reader region lock that +// acts as a backstop to any pruning activity to stop there and go no further. +// +// There are three major pruning loops that follow. All expire from the oldest +// entries. Since there are multiple log buffers, the Android logging facility +// will appear to drop entries 'in the middle' when looking at multiple log +// sources and buffers. This effect is slightly more prominent when we prune +// the worst offender by logging source. Thus the logs slowly loose content +// and value as you move back in time. This is preferred since chatty sources +// invariably move the logs value down faster as less chatty sources would be +// expired in the noise. +// +// The first pass prunes elements that match 3 possible rules: +// 1) A high priority prune rule, for example ~100/20, which indicates elements from UID 100 and PID +// 20 should be pruned in this first pass. +// 2) The default chatty pruning rule, ~!. This rule sums the total size spent on log messages for +// each UID this log buffer. If the highest sum consumes more than 12.5% of the log buffer, then +// these elements from that UID are pruned. +// 3) The default AID_SYSTEM pruning rule, ~1000/!. This rule is a special case to 2), if +// AID_SYSTEM is the top consumer of the log buffer, then this rule sums the total size spent on +// log messages for each PID in AID_SYSTEM in this log buffer and prunes elements from the PID +// with the highest sum. +// This pass reevaluates the sums for rules 2) and 3) for every log message pruned. It creates +// 'chatty' entries for the elements that it prunes and merges related chatty entries together. It +// completes when one of three conditions have been met: +// 1) The requested element count has been pruned. +// 2) There are no elements that match any of these rules. +// 3) A reader is referencing the oldest element that would match these rules. +// +// The second pass prunes elements starting from the beginning of the log. It skips elements that +// match any low priority prune rules. It completes when one of three conditions have been met: +// 1) The requested element count has been pruned. +// 2) All elements except those mwatching low priority prune rules have been pruned. +// 3) A reader is referencing the oldest element that would match these rules. +// +// The final pass only happens if there are any low priority prune rules and if the first two passes +// were unable to prune the requested number of elements. It prunes elements all starting from the +// beginning of the log, regardless of if they match any low priority prune rules. +// +// If the requested number of logs was unable to be pruned, KickReader() is called to mitigate the +// situation before the next call to Prune() and the function returns false. Otherwise, if the +// requested number of logs or all logs present in the buffer are pruned, in the case of Clear(), +// it returns true. +bool ChattyLogBuffer::Prune(log_id_t id, unsigned long pruneRows, uid_t caller_uid) { + LogReaderThread* oldest = nullptr; + bool clearAll = pruneRows == ULONG_MAX; + + auto reader_threads_lock = std::lock_guard{reader_list()->reader_threads_lock()}; + + // Region locked? + for (const auto& reader_thread : reader_list()->reader_threads()) { + if (!reader_thread->IsWatching(id)) { + continue; + } + if (!oldest || oldest->start() > reader_thread->start() || + (oldest->start() == reader_thread->start() && + reader_thread->deadline().time_since_epoch().count() != 0)) { + oldest = reader_thread.get(); + } + } + + LogBufferElementCollection::iterator it; + + if (__predict_false(caller_uid != AID_ROOT)) { // unlikely + // Only here if clear all request from non system source, so chatty + // filter logistics is not required. + it = GetOldest(id); + while (it != logs().end()) { + LogBufferElement& element = *it; + + if (element.log_id() != id || element.uid() != caller_uid) { + ++it; + continue; + } + + if (oldest && oldest->start() <= element.sequence()) { + KickReader(oldest, id, pruneRows); + return false; + } + + it = Erase(it); + if (--pruneRows == 0) { + return true; + } + } + return true; + } + + // First prune pass. + bool check_high_priority = id != LOG_ID_SECURITY && prune_->HasHighPriorityPruneRules(); + while (!clearAll && (pruneRows > 0)) { + // recalculate the worst offender on every batched pass + int worst = -1; // not valid for uid() or getKey() + size_t worst_sizes = 0; + size_t second_worst_sizes = 0; + pid_t worstPid = 0; // POSIX guarantees PID != 0 + + if (worstUidEnabledForLogid(id) && prune_->worst_uid_enabled()) { + // Calculate threshold as 12.5% of available storage + size_t threshold = max_size(id) / 8; + + if (id == LOG_ID_EVENTS || id == LOG_ID_SECURITY) { + stats()->WorstTwoTags(threshold, &worst, &worst_sizes, &second_worst_sizes); + // per-pid filter for AID_SYSTEM sources is too complex + } else { + stats()->WorstTwoUids(id, threshold, &worst, &worst_sizes, &second_worst_sizes); + + if (worst == AID_SYSTEM && prune_->worst_pid_of_system_enabled()) { + stats()->WorstTwoSystemPids(id, worst_sizes, &worstPid, &second_worst_sizes); + } + } + } + + // skip if we have neither a worst UID or high priority prune rules + if (worst == -1 && !check_high_priority) { + break; + } + + bool kick = false; + bool leading = true; // true if starting from the oldest log entry, false if starting from + // a specific chatty entry. + // Perform at least one mandatory garbage collection cycle in following + // - clear leading chatty tags + // - coalesce chatty tags + // - check age-out of preserved logs + bool gc = pruneRows <= 1; + if (!gc && (worst != -1)) { + { // begin scope for worst found iterator + LogBufferIteratorMap::iterator found = mLastWorst[id].find(worst); + if (found != mLastWorst[id].end() && found->second != logs().end()) { + leading = false; + it = found->second; + } + } + if (worstPid) { // begin scope for pid worst found iterator + // FYI: worstPid only set if !LOG_ID_EVENTS and + // !LOG_ID_SECURITY, not going to make that assumption ... + LogBufferPidIteratorMap::iterator found = mLastWorstPidOfSystem[id].find(worstPid); + if (found != mLastWorstPidOfSystem[id].end() && found->second != logs().end()) { + leading = false; + it = found->second; + } + } + } + if (leading) { + it = GetOldest(id); + } + static const log_time too_old{EXPIRE_HOUR_THRESHOLD * 60 * 60, 0}; + LogBufferElementCollection::iterator lastt; + lastt = logs().end(); + --lastt; + LogBufferElementLast last; + while (it != logs().end()) { + LogBufferElement& element = *it; + + if (oldest && oldest->start() <= element.sequence()) { + // Do not let chatty eliding trigger any reader mitigation + break; + } + + if (element.log_id() != id) { + ++it; + continue; + } + // below this point element->log_id() == id + + uint16_t dropped = element.dropped_count(); + + // remove any leading drops + if (leading && dropped) { + it = Erase(it); + continue; + } + + if (dropped && last.coalesce(&element, dropped)) { + it = Erase(it, true); + continue; + } + + int key = (id == LOG_ID_EVENTS || id == LOG_ID_SECURITY) ? element.GetTag() + : element.uid(); + + if (check_high_priority && prune_->IsHighPriority(&element)) { + last.clear(&element); + it = Erase(it); + if (dropped) { + continue; + } + + pruneRows--; + if (pruneRows == 0) { + break; + } + + if (key == worst) { + kick = true; + if (worst_sizes < second_worst_sizes) { + break; + } + worst_sizes -= element.msg_len(); + } + continue; + } + + if (element.realtime() < (lastt->realtime() - too_old) || + element.realtime() > lastt->realtime()) { + break; + } + + if (dropped) { + last.add(&element); + if (worstPid && ((!gc && element.pid() == worstPid) || + mLastWorstPidOfSystem[id].find(element.pid()) == + mLastWorstPidOfSystem[id].end())) { + // element->uid() may not be AID_SYSTEM, next best + // watermark if current one empty. id is not LOG_ID_EVENTS + // or LOG_ID_SECURITY because of worstPid check. + mLastWorstPidOfSystem[id][element.pid()] = it; + } + if ((!gc && !worstPid && (key == worst)) || + (mLastWorst[id].find(key) == mLastWorst[id].end())) { + mLastWorst[id][key] = it; + } + ++it; + continue; + } + + if (key != worst || (worstPid && element.pid() != worstPid)) { + leading = false; + last.clear(&element); + ++it; + continue; + } + // key == worst below here + // If worstPid set, then element->pid() == worstPid below here + + pruneRows--; + if (pruneRows == 0) { + break; + } + + kick = true; + + uint16_t len = element.msg_len(); + + // do not create any leading drops + if (leading) { + it = Erase(it); + } else { + stats()->Drop(element.ToLogStatisticsElement()); + element.SetDropped(1); + if (last.coalesce(&element, 1)) { + it = Erase(it, true); + } else { + last.add(&element); + if (worstPid && (!gc || mLastWorstPidOfSystem[id].find(worstPid) == + mLastWorstPidOfSystem[id].end())) { + // element->uid() may not be AID_SYSTEM, next best + // watermark if current one empty. id is not + // LOG_ID_EVENTS or LOG_ID_SECURITY because of worstPid. + mLastWorstPidOfSystem[id][worstPid] = it; + } + if ((!gc && !worstPid) || mLastWorst[id].find(worst) == mLastWorst[id].end()) { + mLastWorst[id][worst] = it; + } + ++it; + } + } + if (worst_sizes < second_worst_sizes) { + break; + } + worst_sizes -= len; + } + last.clear(); + + if (!kick || !prune_->worst_uid_enabled()) { + break; // the following loop will ask bad clients to skip/drop + } + } + + // Second prune pass. + bool skipped_low_priority_prune = false; + bool check_low_priority = + id != LOG_ID_SECURITY && prune_->HasLowPriorityPruneRules() && !clearAll; + it = GetOldest(id); + while (pruneRows > 0 && it != logs().end()) { + LogBufferElement& element = *it; + + if (element.log_id() != id) { + it++; + continue; + } + + if (oldest && oldest->start() <= element.sequence()) { + if (!skipped_low_priority_prune) KickReader(oldest, id, pruneRows); + break; + } + + if (check_low_priority && !element.dropped_count() && prune_->IsLowPriority(&element)) { + skipped_low_priority_prune = true; + it++; + continue; + } + + it = Erase(it); + pruneRows--; + } + + // Third prune pass. + if (skipped_low_priority_prune && pruneRows > 0) { + it = GetOldest(id); + while (it != logs().end() && pruneRows > 0) { + LogBufferElement& element = *it; + + if (element.log_id() != id) { + ++it; + continue; + } + + if (oldest && oldest->start() <= element.sequence()) { + KickReader(oldest, id, pruneRows); + break; + } + + it = Erase(it); + pruneRows--; + } + } + + return pruneRows == 0 || it == logs().end(); +} diff --git a/logd/ChattyLogBuffer.h b/logd/ChattyLogBuffer.h new file mode 100644 index 000000000..ce3dc7bc2 --- /dev/null +++ b/logd/ChattyLogBuffer.h @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2012-2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <sys/types.h> + +#include <list> +#include <optional> +#include <string> + +#include <android-base/thread_annotations.h> +#include <android/log.h> +#include <private/android_filesystem_config.h> +#include <sysutils/SocketClient.h> + +#include "LogBuffer.h" +#include "LogBufferElement.h" +#include "LogReaderList.h" +#include "LogReaderThread.h" +#include "LogStatistics.h" +#include "LogTags.h" +#include "LogWriter.h" +#include "PruneList.h" +#include "SimpleLogBuffer.h" +#include "rwlock.h" + +typedef std::list<LogBufferElement> LogBufferElementCollection; + +class ChattyLogBuffer : public SimpleLogBuffer { + // watermark of any worst/chatty uid processing + typedef std::unordered_map<uid_t, LogBufferElementCollection::iterator> LogBufferIteratorMap; + LogBufferIteratorMap mLastWorst[LOG_ID_MAX] GUARDED_BY(lock_); + // watermark of any worst/chatty pid of system processing + typedef std::unordered_map<pid_t, LogBufferElementCollection::iterator> LogBufferPidIteratorMap; + LogBufferPidIteratorMap mLastWorstPidOfSystem[LOG_ID_MAX] GUARDED_BY(lock_); + + public: + ChattyLogBuffer(LogReaderList* reader_list, LogTags* tags, PruneList* prune, + LogStatistics* stats); + ~ChattyLogBuffer(); + + protected: + bool Prune(log_id_t id, unsigned long pruneRows, uid_t uid) REQUIRES(lock_) override; + void LogInternal(LogBufferElement&& elem) REQUIRES(lock_) override; + + private: + LogBufferElementCollection::iterator Erase(LogBufferElementCollection::iterator it, + bool coalesce = false) REQUIRES(lock_); + + PruneList* prune_; + + // This always contains a copy of the last message logged, for deduplication. + std::optional<LogBufferElement> last_logged_elements_[LOG_ID_MAX] GUARDED_BY(lock_); + // This contains an element if duplicate messages are seen. + // Its `dropped` count is `duplicates seen - 1`. + std::optional<LogBufferElement> duplicate_elements_[LOG_ID_MAX] GUARDED_BY(lock_); +}; diff --git a/logd/ChattyLogBufferTest.cpp b/logd/ChattyLogBufferTest.cpp new file mode 100644 index 000000000..3d9005ab6 --- /dev/null +++ b/logd/ChattyLogBufferTest.cpp @@ -0,0 +1,335 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "LogBufferTest.h" + +class ChattyLogBufferTest : public LogBufferTest {}; + +TEST_P(ChattyLogBufferTest, deduplication_simple) { + auto make_message = [&](uint32_t sec, const char* tag, const char* msg, + bool regex = false) -> LogMessage { + logger_entry entry = { + .pid = 1, .tid = 1, .sec = sec, .nsec = 1, .lid = LOG_ID_MAIN, .uid = 0}; + std::string message; + message.push_back(ANDROID_LOG_INFO); + message.append(tag); + message.push_back('\0'); + message.append(msg); + message.push_back('\0'); + return {entry, message, regex}; + }; + + // clang-format off + std::vector<LogMessage> log_messages = { + make_message(0, "test_tag", "duplicate"), + make_message(1, "test_tag", "duplicate"), + make_message(2, "test_tag", "not_same"), + make_message(3, "test_tag", "duplicate"), + make_message(4, "test_tag", "duplicate"), + make_message(5, "test_tag", "not_same"), + make_message(6, "test_tag", "duplicate"), + make_message(7, "test_tag", "duplicate"), + make_message(8, "test_tag", "duplicate"), + make_message(9, "test_tag", "not_same"), + make_message(10, "test_tag", "duplicate"), + make_message(11, "test_tag", "duplicate"), + make_message(12, "test_tag", "duplicate"), + make_message(13, "test_tag", "duplicate"), + make_message(14, "test_tag", "duplicate"), + make_message(15, "test_tag", "duplicate"), + make_message(16, "test_tag", "not_same"), + make_message(100, "test_tag", "duplicate"), + make_message(200, "test_tag", "duplicate"), + make_message(300, "test_tag", "duplicate"), + }; + // clang-format on + FixupMessages(&log_messages); + LogMessages(log_messages); + + std::vector<LogMessage> read_log_messages; + std::unique_ptr<LogWriter> test_writer(new TestWriter(&read_log_messages, nullptr)); + std::unique_ptr<FlushToState> flush_to_state = log_buffer_->CreateFlushToState(1, kLogMaskAll); + EXPECT_TRUE(log_buffer_->FlushTo(test_writer.get(), *flush_to_state, nullptr)); + + std::vector<LogMessage> expected_log_messages = { + make_message(0, "test_tag", "duplicate"), + make_message(1, "test_tag", "duplicate"), + make_message(2, "test_tag", "not_same"), + make_message(3, "test_tag", "duplicate"), + make_message(4, "test_tag", "duplicate"), + make_message(5, "test_tag", "not_same"), + // 3 duplicate logs together print the first, a 1 count chatty message, then the last. + make_message(6, "test_tag", "duplicate"), + make_message(7, "chatty", "uid=0\\([^\\)]+\\) [^ ]+ identical 1 line", true), + make_message(8, "test_tag", "duplicate"), + make_message(9, "test_tag", "not_same"), + // 6 duplicate logs together print the first, a 4 count chatty message, then the last. + make_message(10, "test_tag", "duplicate"), + make_message(14, "chatty", "uid=0\\([^\\)]+\\) [^ ]+ identical 4 lines", true), + make_message(15, "test_tag", "duplicate"), + make_message(16, "test_tag", "not_same"), + // duplicate logs > 1 minute apart are not deduplicated. + make_message(100, "test_tag", "duplicate"), + make_message(200, "test_tag", "duplicate"), + make_message(300, "test_tag", "duplicate"), + }; + FixupMessages(&expected_log_messages); + CompareLogMessages(expected_log_messages, read_log_messages); +}; + +TEST_P(ChattyLogBufferTest, deduplication_overflow) { + auto make_message = [&](uint32_t sec, const char* tag, const char* msg, + bool regex = false) -> LogMessage { + logger_entry entry = { + .pid = 1, .tid = 1, .sec = sec, .nsec = 1, .lid = LOG_ID_MAIN, .uid = 0}; + std::string message; + message.push_back(ANDROID_LOG_INFO); + message.append(tag); + message.push_back('\0'); + message.append(msg); + message.push_back('\0'); + return {entry, message, regex}; + }; + + uint32_t sec = 0; + std::vector<LogMessage> log_messages = { + make_message(sec++, "test_tag", "normal"), + }; + size_t expired_per_chatty_message = std::numeric_limits<uint16_t>::max(); + for (size_t i = 0; i < expired_per_chatty_message + 3; ++i) { + log_messages.emplace_back(make_message(sec++, "test_tag", "duplicate")); + } + log_messages.emplace_back(make_message(sec++, "test_tag", "normal")); + FixupMessages(&log_messages); + LogMessages(log_messages); + + std::vector<LogMessage> read_log_messages; + std::unique_ptr<LogWriter> test_writer(new TestWriter(&read_log_messages, nullptr)); + std::unique_ptr<FlushToState> flush_to_state = log_buffer_->CreateFlushToState(1, kLogMaskAll); + EXPECT_TRUE(log_buffer_->FlushTo(test_writer.get(), *flush_to_state, nullptr)); + + std::vector<LogMessage> expected_log_messages = { + make_message(0, "test_tag", "normal"), + make_message(1, "test_tag", "duplicate"), + make_message(expired_per_chatty_message + 1, "chatty", + "uid=0\\([^\\)]+\\) [^ ]+ identical 65535 lines", true), + make_message(expired_per_chatty_message + 2, "chatty", + "uid=0\\([^\\)]+\\) [^ ]+ identical 1 line", true), + make_message(expired_per_chatty_message + 3, "test_tag", "duplicate"), + make_message(expired_per_chatty_message + 4, "test_tag", "normal"), + }; + FixupMessages(&expected_log_messages); + CompareLogMessages(expected_log_messages, read_log_messages); +} + +TEST_P(ChattyLogBufferTest, deduplication_liblog) { + auto make_message = [&](uint32_t sec, int32_t tag, int32_t count) -> LogMessage { + logger_entry entry = { + .pid = 1, .tid = 1, .sec = sec, .nsec = 1, .lid = LOG_ID_EVENTS, .uid = 0}; + android_log_event_int_t liblog_event = { + .header.tag = tag, .payload.type = EVENT_TYPE_INT, .payload.data = count}; + return {entry, std::string(reinterpret_cast<char*>(&liblog_event), sizeof(liblog_event)), + false}; + }; + + // LIBLOG_LOG_TAG + std::vector<LogMessage> log_messages = { + make_message(0, 1234, 1), + make_message(1, LIBLOG_LOG_TAG, 3), + make_message(2, 1234, 2), + make_message(3, LIBLOG_LOG_TAG, 3), + make_message(4, LIBLOG_LOG_TAG, 4), + make_message(5, 1234, 223), + make_message(6, LIBLOG_LOG_TAG, 2), + make_message(7, LIBLOG_LOG_TAG, 3), + make_message(8, LIBLOG_LOG_TAG, 4), + make_message(9, 1234, 227), + make_message(10, LIBLOG_LOG_TAG, 1), + make_message(11, LIBLOG_LOG_TAG, 3), + make_message(12, LIBLOG_LOG_TAG, 2), + make_message(13, LIBLOG_LOG_TAG, 3), + make_message(14, LIBLOG_LOG_TAG, 5), + make_message(15, 1234, 227), + make_message(16, LIBLOG_LOG_TAG, 2), + make_message(17, LIBLOG_LOG_TAG, std::numeric_limits<int32_t>::max()), + make_message(18, LIBLOG_LOG_TAG, 3), + make_message(19, LIBLOG_LOG_TAG, 5), + make_message(20, 1234, 227), + }; + FixupMessages(&log_messages); + LogMessages(log_messages); + + std::vector<LogMessage> read_log_messages; + std::unique_ptr<LogWriter> test_writer(new TestWriter(&read_log_messages, nullptr)); + std::unique_ptr<FlushToState> flush_to_state = log_buffer_->CreateFlushToState(1, kLogMaskAll); + EXPECT_TRUE(log_buffer_->FlushTo(test_writer.get(), *flush_to_state, nullptr)); + + std::vector<LogMessage> expected_log_messages = { + make_message(0, 1234, 1), + make_message(1, LIBLOG_LOG_TAG, 3), + make_message(2, 1234, 2), + make_message(3, LIBLOG_LOG_TAG, 3), + make_message(4, LIBLOG_LOG_TAG, 4), + make_message(5, 1234, 223), + // More than 2 liblog events (3 here), sum their value into the third message. + make_message(6, LIBLOG_LOG_TAG, 2), + make_message(8, LIBLOG_LOG_TAG, 7), + make_message(9, 1234, 227), + // More than 2 liblog events (5 here), sum their value into the third message. + make_message(10, LIBLOG_LOG_TAG, 1), + make_message(14, LIBLOG_LOG_TAG, 13), + make_message(15, 1234, 227), + // int32_t max is the max for a chatty message, beyond that we must use new messages. + make_message(16, LIBLOG_LOG_TAG, 2), + make_message(17, LIBLOG_LOG_TAG, std::numeric_limits<int32_t>::max()), + make_message(19, LIBLOG_LOG_TAG, 8), + make_message(20, 1234, 227), + }; + FixupMessages(&expected_log_messages); + CompareLogMessages(expected_log_messages, read_log_messages); +}; + +TEST_P(ChattyLogBufferTest, no_leading_chatty_simple) { + auto make_message = [&](uint32_t sec, int32_t pid, uint32_t uid, uint32_t lid, const char* tag, + const char* msg, bool regex = false) -> LogMessage { + logger_entry entry = {.pid = pid, .tid = 1, .sec = sec, .nsec = 1, .lid = lid, .uid = uid}; + std::string message; + message.push_back(ANDROID_LOG_INFO); + message.append(tag); + message.push_back('\0'); + message.append(msg); + message.push_back('\0'); + return {entry, message, regex}; + }; + + // clang-format off + std::vector<LogMessage> log_messages = { + make_message(1, 1, 1, LOG_ID_MAIN, "test_tag", "duplicate1"), + make_message(2, 2, 2, LOG_ID_SYSTEM, "test_tag", "duplicate2"), + make_message(3, 2, 2, LOG_ID_SYSTEM, "test_tag", "duplicate2"), + make_message(4, 2, 2, LOG_ID_SYSTEM, "test_tag", "duplicate2"), + make_message(6, 2, 2, LOG_ID_SYSTEM, "test_tag", "not duplicate2"), + make_message(7, 1, 1, LOG_ID_MAIN, "test_tag", "duplicate1"), + make_message(8, 1, 1, LOG_ID_MAIN, "test_tag", "duplicate1"), + make_message(9, 1, 1, LOG_ID_MAIN, "test_tag", "duplicate1"), + make_message(10, 1, 1, LOG_ID_MAIN, "test_tag", "not duplicate1"), + }; + // clang-format on + FixupMessages(&log_messages); + LogMessages(log_messages); + + // After logging log_messages, the below is what should be in the buffer: + // PID=1, LOG_ID_MAIN duplicate1 + // [1] PID=2, LOG_ID_SYSTEM duplicate2 + // PID=2, LOG_ID_SYSTEM chatty drop + // PID=2, LOG_ID_SYSTEM duplicate2 + // PID=2, LOG_ID_SYSTEM not duplicate2 + // [2] PID=1, LOG_ID_MAIN chatty drop + // [3] PID=1, LOG_ID_MAIN duplicate1 + // PID=1, LOG_ID_MAIN not duplicate1 + + // We then read from the 2nd sequence number, starting from log message [1], but filtering out + // everything but PID=1, which results in us starting with log message [2], which is a chatty + // drop. Code prior to this test case would erroneously print it. The intended behavior that + // this test checks prints logs starting from log message [3]. + + // clang-format off + std::vector<LogMessage> expected_log_messages = { + make_message(9, 1, 1, LOG_ID_MAIN, "test_tag", "duplicate1"), + make_message(10, 1, 1, LOG_ID_MAIN, "test_tag", "not duplicate1"), + }; + FixupMessages(&expected_log_messages); + // clang-format on + + std::vector<LogMessage> read_log_messages; + bool released = false; + { + auto lock = std::unique_lock{reader_list_.reader_threads_lock()}; + std::unique_ptr<LogWriter> test_writer(new TestWriter(&read_log_messages, &released)); + std::unique_ptr<LogReaderThread> log_reader( + new LogReaderThread(log_buffer_.get(), &reader_list_, std::move(test_writer), true, + 0, ~0, 1, {}, 2, {})); + reader_list_.reader_threads().emplace_back(std::move(log_reader)); + } + + while (!released) { + usleep(5000); + } + + CompareLogMessages(expected_log_messages, read_log_messages); +} + +TEST_P(ChattyLogBufferTest, no_leading_chatty_tail) { + auto make_message = [&](uint32_t sec, const char* tag, const char* msg, + bool regex = false) -> LogMessage { + logger_entry entry = { + .pid = 1, .tid = 1, .sec = sec, .nsec = 1, .lid = LOG_ID_MAIN, .uid = 0}; + std::string message; + message.push_back(ANDROID_LOG_INFO); + message.append(tag); + message.push_back('\0'); + message.append(msg); + message.push_back('\0'); + return {entry, message, regex}; + }; + + // clang-format off + std::vector<LogMessage> log_messages = { + make_message(1, "test_tag", "duplicate"), + make_message(2, "test_tag", "duplicate"), + make_message(3, "test_tag", "duplicate"), + make_message(4, "test_tag", "not_duplicate"), + }; + // clang-format on + FixupMessages(&log_messages); + LogMessages(log_messages); + + // After logging log_messages, the below is what should be in the buffer: + // "duplicate" + // chatty + // "duplicate" + // "not duplicate" + + // We then read the tail 3 messages expecting there to not be a chatty message, meaning that we + // should only see the last two messages. + + // clang-format off + std::vector<LogMessage> expected_log_messages = { + make_message(3, "test_tag", "duplicate"), + make_message(4, "test_tag", "not_duplicate"), + }; + FixupMessages(&expected_log_messages); + // clang-format on + + std::vector<LogMessage> read_log_messages; + bool released = false; + { + auto lock = std::unique_lock{reader_list_.reader_threads_lock()}; + std::unique_ptr<LogWriter> test_writer(new TestWriter(&read_log_messages, &released)); + std::unique_ptr<LogReaderThread> log_reader( + new LogReaderThread(log_buffer_.get(), &reader_list_, std::move(test_writer), true, + 3, ~0, 0, {}, 1, {})); + reader_list_.reader_threads().emplace_back(std::move(log_reader)); + } + + while (!released) { + usleep(5000); + } + + CompareLogMessages(expected_log_messages, read_log_messages); +} + +INSTANTIATE_TEST_CASE_P(ChattyLogBufferTests, ChattyLogBufferTest, testing::Values("chatty")); diff --git a/logd/CommandListener.cpp b/logd/CommandListener.cpp index 694b5fa3c..2eeb0d977 100644 --- a/logd/CommandListener.cpp +++ b/logd/CommandListener.cpp @@ -14,6 +14,8 @@ * limitations under the License. */ +#include "CommandListener.h" + #include <arpa/inet.h> #include <ctype.h> #include <dirent.h> @@ -29,46 +31,30 @@ #include <string> +#include <android-base/logging.h> #include <android-base/stringprintf.h> #include <cutils/sockets.h> +#include <log/log_properties.h> #include <private/android_filesystem_config.h> #include <sysutils/SocketClient.h> -#include "CommandListener.h" -#include "LogCommand.h" -#include "LogUtils.h" - -CommandListener::CommandListener(LogBuffer* buf, LogReader* /*reader*/, - LogListener* /*swl*/) - : FrameworkListener(getLogSocket()) { - // registerCmd(new ShutdownCmd(buf, writer, swl)); - registerCmd(new ClearCmd(buf)); - registerCmd(new GetBufSizeCmd(buf)); - registerCmd(new SetBufSizeCmd(buf)); - registerCmd(new GetBufSizeUsedCmd(buf)); - registerCmd(new GetStatisticsCmd(buf)); - registerCmd(new SetPruneListCmd(buf)); - registerCmd(new GetPruneListCmd(buf)); - registerCmd(new GetEventTagCmd(buf)); - registerCmd(new ReinitCmd()); +#include "LogPermissions.h" + +CommandListener::CommandListener(LogBuffer* buf, LogTags* tags, PruneList* prune, + LogStatistics* stats) + : FrameworkListener(getLogSocket()), buf_(buf), tags_(tags), prune_(prune), stats_(stats) { + registerCmd(new ClearCmd(this)); + registerCmd(new GetBufSizeCmd(this)); + registerCmd(new SetBufSizeCmd(this)); + registerCmd(new GetBufSizeUsedCmd(this)); + registerCmd(new GetStatisticsCmd(this)); + registerCmd(new SetPruneListCmd(this)); + registerCmd(new GetPruneListCmd(this)); + registerCmd(new GetEventTagCmd(this)); + registerCmd(new ReinitCmd(this)); registerCmd(new ExitCmd(this)); } -CommandListener::ShutdownCmd::ShutdownCmd(LogReader* reader, LogListener* swl) - : LogCommand("shutdown"), mReader(*reader), mSwl(*swl) { -} - -int CommandListener::ShutdownCmd::runCommand(SocketClient* /*cli*/, - int /*argc*/, char** /*argv*/) { - mSwl.stopListener(); - mReader.stopListener(); - exit(0); -} - -CommandListener::ClearCmd::ClearCmd(LogBuffer* buf) - : LogCommand("clear"), mBuf(*buf) { -} - static void setname() { static bool name_set; if (!name_set) { @@ -96,14 +82,10 @@ int CommandListener::ClearCmd::runCommand(SocketClient* cli, int argc, return 0; } - cli->sendMsg(mBuf.clear((log_id_t)id, uid) ? "busy" : "success"); + cli->sendMsg(buf()->Clear((log_id_t)id, uid) ? "success" : "busy"); return 0; } -CommandListener::GetBufSizeCmd::GetBufSizeCmd(LogBuffer* buf) - : LogCommand("getLogSize"), mBuf(*buf) { -} - int CommandListener::GetBufSizeCmd::runCommand(SocketClient* cli, int argc, char** argv) { setname(); @@ -118,17 +100,13 @@ int CommandListener::GetBufSizeCmd::runCommand(SocketClient* cli, int argc, return 0; } - unsigned long size = mBuf.getSize((log_id_t)id); + unsigned long size = buf()->GetSize((log_id_t)id); char buf[512]; snprintf(buf, sizeof(buf), "%lu", size); cli->sendMsg(buf); return 0; } -CommandListener::SetBufSizeCmd::SetBufSizeCmd(LogBuffer* buf) - : LogCommand("setLogSize"), mBuf(*buf) { -} - int CommandListener::SetBufSizeCmd::runCommand(SocketClient* cli, int argc, char** argv) { setname(); @@ -149,7 +127,7 @@ int CommandListener::SetBufSizeCmd::runCommand(SocketClient* cli, int argc, } unsigned long size = atol(argv[2]); - if (mBuf.setSize((log_id_t)id, size)) { + if (buf()->SetSize((log_id_t)id, size)) { cli->sendMsg("Range Error"); return 0; } @@ -158,10 +136,6 @@ int CommandListener::SetBufSizeCmd::runCommand(SocketClient* cli, int argc, return 0; } -CommandListener::GetBufSizeUsedCmd::GetBufSizeUsedCmd(LogBuffer* buf) - : LogCommand("getLogSizeUsed"), mBuf(*buf) { -} - int CommandListener::GetBufSizeUsedCmd::runCommand(SocketClient* cli, int argc, char** argv) { setname(); @@ -176,17 +150,13 @@ int CommandListener::GetBufSizeUsedCmd::runCommand(SocketClient* cli, int argc, return 0; } - unsigned long size = mBuf.getSizeUsed((log_id_t)id); + unsigned long size = stats()->Sizes((log_id_t)id); char buf[512]; snprintf(buf, sizeof(buf), "%lu", size); cli->sendMsg(buf); return 0; } -CommandListener::GetStatisticsCmd::GetStatisticsCmd(LogBuffer* buf) - : LogCommand("getStatistics"), mBuf(*buf) { -} - // This returns a string with a length prefix with the format <length>\n<data>\n\f. The length // prefix includes the length of the prefix itself. static std::string PackageString(const std::string& str) { @@ -241,27 +211,17 @@ int CommandListener::GetStatisticsCmd::runCommand(SocketClient* cli, int argc, } } - cli->sendMsg(PackageString(mBuf.formatStatistics(uid, pid, logMask)).c_str()); + cli->sendMsg(PackageString(stats()->Format(uid, pid, logMask)).c_str()); return 0; } -CommandListener::GetPruneListCmd::GetPruneListCmd(LogBuffer* buf) - : LogCommand("getPruneList"), mBuf(*buf) { -} - -int CommandListener::GetPruneListCmd::runCommand(SocketClient* cli, - int /*argc*/, char** /*argv*/) { +int CommandListener::GetPruneListCmd::runCommand(SocketClient* cli, int, char**) { setname(); - cli->sendMsg(PackageString(mBuf.formatPrune()).c_str()); + cli->sendMsg(PackageString(prune()->Format()).c_str()); return 0; } -CommandListener::SetPruneListCmd::SetPruneListCmd(LogBuffer* buf) - : LogCommand("setPruneList"), mBuf(*buf) { -} - -int CommandListener::SetPruneListCmd::runCommand(SocketClient* cli, int argc, - char** argv) { +int CommandListener::SetPruneListCmd::runCommand(SocketClient* cli, int argc, char** argv) { setname(); if (!clientHasLogCredentials(cli)) { cli->sendMsg("Permission Denied"); @@ -276,22 +236,15 @@ int CommandListener::SetPruneListCmd::runCommand(SocketClient* cli, int argc, str += argv[i]; } - int ret = mBuf.initPrune(str.c_str()); - - if (ret) { + if (!prune()->Init(str.c_str())) { cli->sendMsg("Invalid"); return 0; } cli->sendMsg("success"); - return 0; } -CommandListener::GetEventTagCmd::GetEventTagCmd(LogBuffer* buf) - : LogCommand("getEventTag"), mBuf(*buf) { -} - int CommandListener::GetEventTagCmd::runCommand(SocketClient* cli, int argc, char** argv) { setname(); @@ -328,39 +281,45 @@ int CommandListener::GetEventTagCmd::runCommand(SocketClient* cli, int argc, cli->sendMsg("can not mix id= with either format= or name="); return 0; } - cli->sendMsg(PackageString(mBuf.formatEntry(atoi(id), uid)).c_str()); + cli->sendMsg(PackageString(tags()->formatEntry(atoi(id), uid)).c_str()); return 0; } - cli->sendMsg(PackageString(mBuf.formatGetEventTag(uid, name, format)).c_str()); + cli->sendMsg(PackageString(tags()->formatGetEventTag(uid, name, format)).c_str()); return 0; } -CommandListener::ReinitCmd::ReinitCmd() : LogCommand("reinit") { -} - int CommandListener::ReinitCmd::runCommand(SocketClient* cli, int /*argc*/, char** /*argv*/) { setname(); - reinit_signal_handler(SIGHUP); + LOG(INFO) << "logd reinit"; + buf()->Init(); + prune()->Init(nullptr); + + // This only works on userdebug and eng devices to re-read the + // /data/misc/logd/event-log-tags file right after /data is mounted. + // The operation is near to boot and should only happen once. There + // are races associated with its use since it can trigger a Rebuild + // of the file, but that is a can-not-happen since the file was not + // read yet. More dangerous if called later, but if all is well it + // should just skip over everything and not write any new entries. + if (__android_log_is_debuggable()) { + tags()->ReadFileEventLogTags(tags()->debug_event_log_tags); + } cli->sendMsg("success"); return 0; } -CommandListener::ExitCmd::ExitCmd(CommandListener* parent) - : LogCommand("EXIT"), mParent(*parent) { -} - int CommandListener::ExitCmd::runCommand(SocketClient* cli, int /*argc*/, char** /*argv*/) { setname(); cli->sendMsg("success"); - release(cli); + parent_->release(cli); return 0; } diff --git a/logd/CommandListener.h b/logd/CommandListener.h index ed9941938..c3080ab97 100644 --- a/logd/CommandListener.h +++ b/logd/CommandListener.h @@ -14,85 +14,55 @@ * limitations under the License. */ -#ifndef _COMMANDLISTENER_H__ -#define _COMMANDLISTENER_H__ +#pragma once +#include <sysutils/FrameworkCommand.h> #include <sysutils/FrameworkListener.h> + #include "LogBuffer.h" -#include "LogCommand.h" #include "LogListener.h" -#include "LogReader.h" - -// See main.cpp for implementation -void reinit_signal_handler(int /*signal*/); +#include "LogStatistics.h" +#include "LogTags.h" +#include "PruneList.h" class CommandListener : public FrameworkListener { - public: - CommandListener(LogBuffer* buf, LogReader* reader, LogListener* swl); - virtual ~CommandListener() { - } + public: + CommandListener(LogBuffer* buf, LogTags* tags, PruneList* prune, LogStatistics* log_statistics); + virtual ~CommandListener() {} - private: + private: static int getLogSocket(); - class ShutdownCmd : public LogCommand { - LogReader& mReader; - LogListener& mSwl; + LogBuffer* buf_; + LogTags* tags_; + PruneList* prune_; + LogStatistics* stats_; - public: - ShutdownCmd(LogReader* reader, LogListener* swl); - virtual ~ShutdownCmd() { - } - int runCommand(SocketClient* c, int argc, char** argv); - }; - -#define LogBufferCmd(name) \ - class name##Cmd : public LogCommand { \ - LogBuffer& mBuf; \ - \ - public: \ - explicit name##Cmd(LogBuffer* buf); \ - virtual ~name##Cmd() { \ - } \ - int runCommand(SocketClient* c, int argc, char** argv); \ +#define LogCmd(name, command_string) \ + class name##Cmd : public FrameworkCommand { \ + public: \ + explicit name##Cmd(CommandListener* parent) \ + : FrameworkCommand(#command_string), parent_(parent) {} \ + virtual ~name##Cmd() {} \ + int runCommand(SocketClient* c, int argc, char** argv); \ + \ + private: \ + LogBuffer* buf() const { return parent_->buf_; } \ + LogTags* tags() const { return parent_->tags_; } \ + PruneList* prune() const { return parent_->prune_; } \ + LogStatistics* stats() const { return parent_->stats_; } \ + CommandListener* parent_; \ } - LogBufferCmd(Clear); - LogBufferCmd(GetBufSize); - LogBufferCmd(SetBufSize); - LogBufferCmd(GetBufSizeUsed); - LogBufferCmd(GetStatistics); - LogBufferCmd(GetPruneList); - LogBufferCmd(SetPruneList); - LogBufferCmd(GetEventTag); - -#define LogCmd(name) \ - class name##Cmd : public LogCommand { \ - public: \ - name##Cmd(); \ - virtual ~name##Cmd() { \ - } \ - int runCommand(SocketClient* c, int argc, char** argv); \ - } - - LogCmd(Reinit); - -#define LogParentCmd(name) \ - class name##Cmd : public LogCommand { \ - CommandListener& mParent; \ - \ - public: \ - name##Cmd(); \ - explicit name##Cmd(CommandListener* parent); \ - virtual ~name##Cmd() { \ - } \ - int runCommand(SocketClient* c, int argc, char** argv); \ - void release(SocketClient* c) { \ - mParent.release(c); \ - } \ - } - - LogParentCmd(Exit); + LogCmd(Clear, clear); + LogCmd(GetBufSize, getLogSize); + LogCmd(SetBufSize, setLogSize); + LogCmd(GetBufSizeUsed, getLogSizeUsed); + LogCmd(GetStatistics, getStatistics); + LogCmd(GetPruneList, getPruneList); + LogCmd(SetPruneList, setPruneList); + LogCmd(GetEventTag, getEventTag); + LogCmd(Reinit, reinit); + LogCmd(Exit, EXIT); +#undef LogCmd }; - -#endif diff --git a/logd/CompressionEngine.cpp b/logd/CompressionEngine.cpp new file mode 100644 index 000000000..f9c59792f --- /dev/null +++ b/logd/CompressionEngine.cpp @@ -0,0 +1,104 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "CompressionEngine.h" + +#include <limits> + +#include <android-base/logging.h> +#include <zlib.h> +#include <zstd.h> + +CompressionEngine& CompressionEngine::GetInstance() { + static CompressionEngine* engine = new ZstdCompressionEngine(); + return *engine; +} + +bool ZlibCompressionEngine::Compress(std::span<uint8_t> in, std::vector<uint8_t>& out) { + z_stream strm; + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + int ret = deflateInit(&strm, Z_DEFAULT_COMPRESSION); + if (ret != Z_OK) { + LOG(FATAL) << "deflateInit() failed"; + } + + CHECK_LE(in.size(), static_cast<int64_t>(std::numeric_limits<uint32_t>::max())); + uint32_t out_size = deflateBound(&strm, in.size()); + + out.resize(out_size); + strm.avail_in = in.size(); + strm.next_in = const_cast<uint8_t*>(in.data()); + strm.avail_out = out_size; + strm.next_out = out.data(); + ret = deflate(&strm, Z_FINISH); + CHECK_EQ(ret, Z_STREAM_END); + + uint32_t compressed_data_size = strm.total_out; + deflateEnd(&strm); + out.resize(compressed_data_size); + + return true; +} + +bool ZlibCompressionEngine::Decompress(const std::vector<uint8_t>& in, std::vector<uint8_t>& out, + size_t out_size) { + out.resize(out_size); + + z_stream strm; + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.avail_in = in.size(); + strm.next_in = const_cast<uint8_t*>(in.data()); + strm.avail_out = out.size(); + strm.next_out = out.data(); + + inflateInit(&strm); + int ret = inflate(&strm, Z_NO_FLUSH); + + CHECK_EQ(strm.avail_in, 0U); + CHECK_EQ(strm.avail_out, 0U); + CHECK_EQ(ret, Z_STREAM_END); + inflateEnd(&strm); + + return true; +} + +bool ZstdCompressionEngine::Compress(std::span<uint8_t> in, std::vector<uint8_t>& out) { + size_t out_size = ZSTD_compressBound(in.size()); + out.resize(out_size); + + out_size = ZSTD_compress(out.data(), out_size, in.data(), in.size(), 1); + if (ZSTD_isError(out_size)) { + LOG(FATAL) << "ZSTD_compress failed: " << ZSTD_getErrorName(out_size); + } + out.resize(out_size); + + return true; +} + +bool ZstdCompressionEngine::Decompress(const std::vector<uint8_t>& in, std::vector<uint8_t>& out, + size_t out_size) { + out.resize(out_size); + size_t result = ZSTD_decompress(out.data(), out.size(), in.data(), in.size()); + if (ZSTD_isError(result)) { + LOG(FATAL) << "ZSTD_decompress failed: " << ZSTD_getErrorName(result); + } + CHECK_EQ(result, out.size()); + return true; +} diff --git a/logd/CompressionEngine.h b/logd/CompressionEngine.h new file mode 100644 index 000000000..d760ceaa5 --- /dev/null +++ b/logd/CompressionEngine.h @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <span> +#include <vector> + +class CompressionEngine { + public: + static CompressionEngine& GetInstance(); + + virtual ~CompressionEngine(){}; + + virtual bool Compress(std::span<uint8_t> in, std::vector<uint8_t>& out) = 0; + // Decompress the contents of `in` into `out`. `out_size` must be set to the decompressed size + // of the contents. + virtual bool Decompress(const std::vector<uint8_t>& in, std::vector<uint8_t>& out, + size_t out_size) = 0; +}; + +class ZlibCompressionEngine : public CompressionEngine { + public: + bool Compress(std::span<uint8_t> in, std::vector<uint8_t>& out) override; + bool Decompress(const std::vector<uint8_t>& in, std::vector<uint8_t>& out, + size_t out_size) override; +}; + +class ZstdCompressionEngine : public CompressionEngine { + public: + bool Compress(std::span<uint8_t> in, std::vector<uint8_t>& out) override; + bool Decompress(const std::vector<uint8_t>& in, std::vector<uint8_t>& out, + size_t out_size) override; +};
\ No newline at end of file diff --git a/logd/FlushCommand.cpp b/logd/FlushCommand.cpp deleted file mode 100644 index bd1755553..000000000 --- a/logd/FlushCommand.cpp +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (C) 2012-2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <stdlib.h> - -#include <private/android_filesystem_config.h> - -#include "FlushCommand.h" -#include "LogBuffer.h" -#include "LogBufferElement.h" -#include "LogCommand.h" -#include "LogReader.h" -#include "LogTimes.h" -#include "LogUtils.h" - -// runSocketCommand is called once for every open client on the -// log reader socket. Here we manage and associated the reader -// client tracking and log region locks LastLogTimes list of -// LogTimeEntrys, and spawn a transitory per-client thread to -// work at filing data to the socket. -// -// global LogTimeEntry::wrlock() is used to protect access, -// reference counts are used to ensure that individual -// LogTimeEntry lifetime is managed when not protected. -void FlushCommand::runSocketCommand(SocketClient* client) { - LogTimeEntry* entry = nullptr; - LastLogTimes& times = mReader.logbuf().mTimes; - - LogTimeEntry::wrlock(); - LastLogTimes::iterator it = times.begin(); - while (it != times.end()) { - entry = it->get(); - if (entry->mClient == client) { - if (!entry->isWatchingMultiple(mLogMask)) { - LogTimeEntry::unlock(); - return; - } - if (entry->mTimeout.tv_sec || entry->mTimeout.tv_nsec) { - if (mReader.logbuf().isMonotonic()) { - LogTimeEntry::unlock(); - return; - } - // If the user changes the time in a gross manner that - // invalidates the timeout, fall through and trigger. - log_time now(CLOCK_REALTIME); - if (((entry->mEnd + entry->mTimeout) > now) && - (now > entry->mEnd)) { - LogTimeEntry::unlock(); - return; - } - } - entry->triggerReader_Locked(); - LogTimeEntry::unlock(); - return; - } - it++; - } - - LogTimeEntry::unlock(); -} - -bool FlushCommand::hasReadLogs(SocketClient* client) { - return clientHasLogCredentials(client); -} - -static bool clientHasSecurityCredentials(SocketClient* client) { - return (client->getUid() == AID_SYSTEM) || (client->getGid() == AID_SYSTEM); -} - -bool FlushCommand::hasSecurityLogs(SocketClient* client) { - return clientHasSecurityCredentials(client); -} diff --git a/logd/FlushCommand.h b/logd/FlushCommand.h deleted file mode 100644 index ceaf39362..000000000 --- a/logd/FlushCommand.h +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (C) 2012-2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef _FLUSH_COMMAND_H -#define _FLUSH_COMMAND_H - -#include <private/android_logger.h> -#include <sysutils/SocketClientCommand.h> - -class LogBufferElement; - -#include "LogTimes.h" - -class LogReader; - -class FlushCommand : public SocketClientCommand { - LogReader& mReader; - log_mask_t mLogMask; - - public: - explicit FlushCommand(LogReader& reader, log_mask_t logMask) - : mReader(reader), mLogMask(logMask) { - } - - virtual void runSocketCommand(SocketClient* client); - - static bool hasReadLogs(SocketClient* client); - static bool hasSecurityLogs(SocketClient* client); -}; - -#endif diff --git a/logd/LogAudit.cpp b/logd/LogAudit.cpp index d9cc0dba3..0ce9796b0 100644 --- a/logd/LogAudit.cpp +++ b/logd/LogAudit.cpp @@ -14,6 +14,8 @@ * limitations under the License. */ +#include "LogAudit.h" + #include <ctype.h> #include <endian.h> #include <errno.h> @@ -34,10 +36,7 @@ #include <private/android_filesystem_config.h> #include <private/android_logger.h> -#include "LogAudit.h" -#include "LogBuffer.h" #include "LogKlog.h" -#include "LogReader.h" #include "LogUtils.h" #include "libaudit.h" @@ -45,16 +44,14 @@ '<', '0' + LOG_MAKEPRI(LOG_AUTH, LOG_PRI(PRI)) / 10, \ '0' + LOG_MAKEPRI(LOG_AUTH, LOG_PRI(PRI)) % 10, '>' -LogAudit::LogAudit(LogBuffer* buf, LogReader* reader, int fdDmesg) +LogAudit::LogAudit(LogBuffer* buf, int fdDmesg, LogStatistics* stats) : SocketListener(getLogSocket(), false), logbuf(buf), - reader(reader), fdDmesg(fdDmesg), - main(__android_logger_property_get_bool("ro.logd.auditd.main", - BOOL_DEFAULT_TRUE)), - events(__android_logger_property_get_bool("ro.logd.auditd.events", - BOOL_DEFAULT_TRUE)), - initialized(false) { + main(__android_logger_property_get_bool("ro.logd.auditd.main", BOOL_DEFAULT_TRUE)), + events(__android_logger_property_get_bool("ro.logd.auditd.events", BOOL_DEFAULT_TRUE)), + initialized(false), + stats_(stats) { static const char auditd_message[] = { KMSG_PRIORITY(LOG_INFO), 'l', 'o', @@ -211,9 +208,7 @@ int LogAudit::logPrint(const char* fmt, ...) { ++cp; } tid = pid; - logbuf->wrlock(); - uid = logbuf->pidToUid(pid); - logbuf->unlock(); + uid = stats_->PidToUid(pid); memmove(pidptr, cp, strlen(cp) + 1); } @@ -247,22 +242,10 @@ int LogAudit::logPrint(const char* fmt, ...) { static const char audit_str[] = " audit("; char* timeptr = strstr(str, audit_str); - if (timeptr && - ((cp = now.strptime(timeptr + sizeof(audit_str) - 1, "%s.%q"))) && + if (timeptr && ((cp = now.strptime(timeptr + sizeof(audit_str) - 1, "%s.%q"))) && (*cp == ':')) { memcpy(timeptr + sizeof(audit_str) - 1, "0.0", 3); memmove(timeptr + sizeof(audit_str) - 1 + 3, cp, strlen(cp) + 1); - if (!isMonotonic()) { - if (android::isMonotonic(now)) { - LogKlog::convertMonotonicToReal(now); - } - } else { - if (!android::isMonotonic(now)) { - LogKlog::convertRealToMonotonic(now); - } - } - } else if (isMonotonic()) { - now = log_time(CLOCK_MONOTONIC); } else { now = log_time(CLOCK_REALTIME); } @@ -277,7 +260,7 @@ int LogAudit::logPrint(const char* fmt, ...) { : LOGGER_ENTRY_MAX_PAYLOAD; size_t message_len = str_len + sizeof(android_log_event_string_t); - log_mask_t notify = 0; + unsigned int notify = 0; if (events) { // begin scope for event buffer uint32_t buffer[(message_len + sizeof(uint32_t) - 1) / sizeof(uint32_t)]; @@ -291,9 +274,8 @@ int LogAudit::logPrint(const char* fmt, ...) { memcpy(event->data + str_len - denial_metadata.length(), denial_metadata.c_str(), denial_metadata.length()); - rc = logbuf->log( - LOG_ID_EVENTS, now, uid, pid, tid, reinterpret_cast<char*>(event), - (message_len <= UINT16_MAX) ? (uint16_t)message_len : UINT16_MAX); + rc = logbuf->Log(LOG_ID_EVENTS, now, uid, pid, tid, reinterpret_cast<char*>(event), + (message_len <= UINT16_MAX) ? (uint16_t)message_len : UINT16_MAX); if (rc >= 0) { notify |= 1 << LOG_ID_EVENTS; } @@ -313,9 +295,7 @@ int LogAudit::logPrint(const char* fmt, ...) { pid = tid; comm = "auditd"; } else { - logbuf->wrlock(); - comm = commfree = logbuf->pidToName(pid); - logbuf->unlock(); + comm = commfree = stats_->PidToName(pid); if (!comm) { comm = "unknown"; } @@ -347,9 +327,8 @@ int LogAudit::logPrint(const char* fmt, ...) { strncpy(newstr + 1 + str_len + prefix_len + suffix_len, denial_metadata.c_str(), denial_metadata.length()); - rc = logbuf->log( - LOG_ID_MAIN, now, uid, pid, tid, newstr, - (message_len <= UINT16_MAX) ? (uint16_t)message_len : UINT16_MAX); + rc = logbuf->Log(LOG_ID_MAIN, now, uid, pid, tid, newstr, + (message_len <= UINT16_MAX) ? (uint16_t)message_len : UINT16_MAX); if (rc >= 0) { notify |= 1 << LOG_ID_MAIN; @@ -361,7 +340,6 @@ int LogAudit::logPrint(const char* fmt, ...) { free(str); if (notify) { - reader->notifyNewLog(notify); if (rc < 0) { rc = message_len; } diff --git a/logd/LogAudit.h b/logd/LogAudit.h index c3d7a3ede..181920e38 100644 --- a/logd/LogAudit.h +++ b/logd/LogAudit.h @@ -14,36 +14,30 @@ * limitations under the License. */ -#ifndef _LOGD_LOG_AUDIT_H__ -#define _LOGD_LOG_AUDIT_H__ +#pragma once #include <map> #include <sysutils/SocketListener.h> #include "LogBuffer.h" - -class LogReader; +#include "LogStatistics.h" class LogAudit : public SocketListener { LogBuffer* logbuf; - LogReader* reader; int fdDmesg; // fdDmesg >= 0 is functionally bool dmesg bool main; bool events; bool initialized; - public: - LogAudit(LogBuffer* buf, LogReader* reader, int fdDmesg); + public: + LogAudit(LogBuffer* buf, int fdDmesg, LogStatistics* stats); int log(char* buf, size_t len); - bool isMonotonic() { - return logbuf->isMonotonic(); - } - protected: + protected: virtual bool onDataAvailable(SocketClient* cli); - private: + private: static int getLogSocket(); std::map<std::string, std::string> populateDenialMap(); std::string denialParse(const std::string& denial, char terminator, @@ -51,6 +45,6 @@ class LogAudit : public SocketListener { void auditParse(const std::string& string, uid_t uid, std::string* bug_num); int logPrint(const char* fmt, ...) __attribute__((__format__(__printf__, 2, 3))); -}; -#endif + LogStatistics* stats_; +}; diff --git a/logd/LogBuffer.cpp b/logd/LogBuffer.cpp deleted file mode 100644 index 1cf20617f..000000000 --- a/logd/LogBuffer.cpp +++ /dev/null @@ -1,1225 +0,0 @@ -/* - * Copyright (C) 2012-2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -// for manual checking of stale entries during LogBuffer::erase() -//#define DEBUG_CHECK_FOR_STALE_ENTRIES - -#include <ctype.h> -#include <endian.h> -#include <errno.h> -#include <stdio.h> -#include <string.h> -#include <sys/cdefs.h> -#include <sys/user.h> -#include <time.h> -#include <unistd.h> - -#include <unordered_map> - -#include <cutils/properties.h> -#include <private/android_logger.h> - -#include "LogBuffer.h" -#include "LogKlog.h" -#include "LogReader.h" -#include "LogUtils.h" - -#ifndef __predict_false -#define __predict_false(exp) __builtin_expect((exp) != 0, 0) -#endif - -// Default -#define log_buffer_size(id) mMaxSize[id] - -const log_time LogBuffer::pruneMargin(3, 0); - -void LogBuffer::init() { - log_id_for_each(i) { - mLastSet[i] = false; - mLast[i] = mLogElements.begin(); - - if (setSize(i, __android_logger_get_buffer_size(i))) { - setSize(i, LOG_BUFFER_MIN_SIZE); - } - } - bool lastMonotonic = monotonic; - monotonic = android_log_clockid() == CLOCK_MONOTONIC; - if (lastMonotonic != monotonic) { - // - // Fixup all timestamps, may not be 100% accurate, but better than - // throwing what we have away when we get 'surprised' by a change. - // In-place element fixup so no need to check reader-lock. Entries - // should already be in timestamp order, but we could end up with a - // few out-of-order entries if new monotonics come in before we - // are notified of the reinit change in status. A Typical example would - // be: - // --------- beginning of system - // 10.494082 184 201 D Cryptfs : Just triggered post_fs_data - // --------- beginning of kernel - // 0.000000 0 0 I : Initializing cgroup subsys - // as the act of mounting /data would trigger persist.logd.timestamp to - // be corrected. 1/30 corner case YMMV. - // - rdlock(); - LogBufferElementCollection::iterator it = mLogElements.begin(); - while ((it != mLogElements.end())) { - LogBufferElement* e = *it; - if (monotonic) { - if (!android::isMonotonic(e->mRealTime)) { - LogKlog::convertRealToMonotonic(e->mRealTime); - if ((e->mRealTime.tv_nsec % 1000) == 0) { - e->mRealTime.tv_nsec++; - } - } - } else { - if (android::isMonotonic(e->mRealTime)) { - LogKlog::convertMonotonicToReal(e->mRealTime); - if ((e->mRealTime.tv_nsec % 1000) == 0) { - e->mRealTime.tv_nsec++; - } - } - } - ++it; - } - unlock(); - } - - // We may have been triggered by a SIGHUP. Release any sleeping reader - // threads to dump their current content. - // - // NB: this is _not_ performed in the context of a SIGHUP, it is - // performed during startup, and in context of reinit administrative thread - LogTimeEntry::wrlock(); - - LastLogTimes::iterator times = mTimes.begin(); - while (times != mTimes.end()) { - LogTimeEntry* entry = times->get(); - entry->triggerReader_Locked(); - times++; - } - - LogTimeEntry::unlock(); -} - -LogBuffer::LogBuffer(LastLogTimes* times) - : monotonic(android_log_clockid() == CLOCK_MONOTONIC), mTimes(*times) { - pthread_rwlock_init(&mLogElementsLock, nullptr); - - log_id_for_each(i) { - lastLoggedElements[i] = nullptr; - droppedElements[i] = nullptr; - } - - init(); -} - -LogBuffer::~LogBuffer() { - log_id_for_each(i) { - delete lastLoggedElements[i]; - delete droppedElements[i]; - } -} - -enum match_type { DIFFERENT, SAME, SAME_LIBLOG }; - -static enum match_type identical(LogBufferElement* elem, - LogBufferElement* last) { - // is it mostly identical? - // if (!elem) return DIFFERENT; - ssize_t lenl = elem->getMsgLen(); - if (lenl <= 0) return DIFFERENT; // value if this represents a chatty elem - // if (!last) return DIFFERENT; - ssize_t lenr = last->getMsgLen(); - if (lenr <= 0) return DIFFERENT; // value if this represents a chatty elem - // if (elem->getLogId() != last->getLogId()) return DIFFERENT; - if (elem->getUid() != last->getUid()) return DIFFERENT; - if (elem->getPid() != last->getPid()) return DIFFERENT; - if (elem->getTid() != last->getTid()) return DIFFERENT; - - // last is more than a minute old, stop squashing identical messages - if (elem->getRealTime().nsec() > - (last->getRealTime().nsec() + 60 * NS_PER_SEC)) - return DIFFERENT; - - // Identical message - const char* msgl = elem->getMsg(); - const char* msgr = last->getMsg(); - if (lenl == lenr) { - if (!fastcmp<memcmp>(msgl, msgr, lenl)) return SAME; - // liblog tagged messages (content gets summed) - if ((elem->getLogId() == LOG_ID_EVENTS) && - (lenl == sizeof(android_log_event_int_t)) && - !fastcmp<memcmp>(msgl, msgr, sizeof(android_log_event_int_t) - - sizeof(int32_t)) && - (elem->getTag() == LIBLOG_LOG_TAG)) { - return SAME_LIBLOG; - } - } - - // audit message (except sequence number) identical? - if (last->isBinary() && - (lenl > static_cast<ssize_t>(sizeof(android_log_event_string_t))) && - (lenr > static_cast<ssize_t>(sizeof(android_log_event_string_t)))) { - if (fastcmp<memcmp>(msgl, msgr, sizeof(android_log_event_string_t) - - sizeof(int32_t))) { - return DIFFERENT; - } - msgl += sizeof(android_log_event_string_t); - lenl -= sizeof(android_log_event_string_t); - msgr += sizeof(android_log_event_string_t); - lenr -= sizeof(android_log_event_string_t); - } - static const char avc[] = "): avc: "; - const char* avcl = android::strnstr(msgl, lenl, avc); - if (!avcl) return DIFFERENT; - lenl -= avcl - msgl; - const char* avcr = android::strnstr(msgr, lenr, avc); - if (!avcr) return DIFFERENT; - lenr -= avcr - msgr; - if (lenl != lenr) return DIFFERENT; - if (fastcmp<memcmp>(avcl + strlen(avc), avcr + strlen(avc), - lenl - strlen(avc))) { - return DIFFERENT; - } - return SAME; -} - -int LogBuffer::log(log_id_t log_id, log_time realtime, uid_t uid, pid_t pid, - pid_t tid, const char* msg, uint16_t len) { - if (log_id >= LOG_ID_MAX) { - return -EINVAL; - } - - // Slip the time by 1 nsec if the incoming lands on xxxxxx000 ns. - // This prevents any chance that an outside source can request an - // exact entry with time specified in ms or us precision. - if ((realtime.tv_nsec % 1000) == 0) ++realtime.tv_nsec; - - LogBufferElement* elem = new LogBufferElement(log_id, realtime, uid, pid, tid, msg, len); - - // b/137093665: don't coalesce security messages. - if (log_id == LOG_ID_SECURITY) { - wrlock(); - log(elem); - unlock(); - - return len; - } - - int prio = ANDROID_LOG_INFO; - const char* tag = nullptr; - size_t tag_len = 0; - if (log_id == LOG_ID_EVENTS || log_id == LOG_ID_STATS) { - tag = tagToName(elem->getTag()); - if (tag) { - tag_len = strlen(tag); - } - } else { - prio = *msg; - tag = msg + 1; - tag_len = strnlen(tag, len - 1); - } - if (!__android_log_is_loggable_len(prio, tag, tag_len, ANDROID_LOG_VERBOSE)) { - // Log traffic received to total - wrlock(); - stats.addTotal(elem); - unlock(); - delete elem; - return -EACCES; - } - - wrlock(); - LogBufferElement* currentLast = lastLoggedElements[log_id]; - if (currentLast) { - LogBufferElement* dropped = droppedElements[log_id]; - uint16_t count = dropped ? dropped->getDropped() : 0; - // - // State Init - // incoming: - // dropped = nullptr - // currentLast = nullptr; - // elem = incoming message - // outgoing: - // dropped = nullptr -> State 0 - // currentLast = copy of elem - // log elem - // State 0 - // incoming: - // count = 0 - // dropped = nullptr - // currentLast = copy of last message - // elem = incoming message - // outgoing: if match != DIFFERENT - // dropped = copy of first identical message -> State 1 - // currentLast = reference to elem - // break: if match == DIFFERENT - // dropped = nullptr -> State 0 - // delete copy of last message (incoming currentLast) - // currentLast = copy of elem - // log elem - // State 1 - // incoming: - // count = 0 - // dropped = copy of first identical message - // currentLast = reference to last held-back incoming - // message - // elem = incoming message - // outgoing: if match == SAME - // delete copy of first identical message (dropped) - // dropped = reference to last held-back incoming - // message set to chatty count of 1 -> State 2 - // currentLast = reference to elem - // outgoing: if match == SAME_LIBLOG - // dropped = copy of first identical message -> State 1 - // take sum of currentLast and elem - // if sum overflows: - // log currentLast - // currentLast = reference to elem - // else - // delete currentLast - // currentLast = reference to elem, sum liblog. - // break: if match == DIFFERENT - // delete dropped - // dropped = nullptr -> State 0 - // log reference to last held-back (currentLast) - // currentLast = copy of elem - // log elem - // State 2 - // incoming: - // count = chatty count - // dropped = chatty message holding count - // currentLast = reference to last held-back incoming - // message. - // dropped = chatty message holding count - // elem = incoming message - // outgoing: if match != DIFFERENT - // delete chatty message holding count - // dropped = reference to last held-back incoming - // message, set to chatty count + 1 - // currentLast = reference to elem - // break: if match == DIFFERENT - // log dropped (chatty message) - // dropped = nullptr -> State 0 - // log reference to last held-back (currentLast) - // currentLast = copy of elem - // log elem - // - enum match_type match = identical(elem, currentLast); - if (match != DIFFERENT) { - if (dropped) { - // Sum up liblog tag messages? - if ((count == 0) /* at Pass 1 */ && (match == SAME_LIBLOG)) { - android_log_event_int_t* event = - reinterpret_cast<android_log_event_int_t*>( - const_cast<char*>(currentLast->getMsg())); - // - // To unit test, differentiate with something like: - // event->header.tag = htole32(CHATTY_LOG_TAG); - // here, then instead of delete currentLast below, - // log(currentLast) to see the incremental sums form. - // - uint32_t swab = event->payload.data; - unsigned long long total = htole32(swab); - event = reinterpret_cast<android_log_event_int_t*>( - const_cast<char*>(elem->getMsg())); - swab = event->payload.data; - - lastLoggedElements[LOG_ID_EVENTS] = elem; - total += htole32(swab); - // check for overflow - if (total >= UINT32_MAX) { - log(currentLast); - unlock(); - return len; - } - stats.addTotal(currentLast); - delete currentLast; - swab = total; - event->payload.data = htole32(swab); - unlock(); - return len; - } - if (count == USHRT_MAX) { - log(dropped); - count = 1; - } else { - delete dropped; - ++count; - } - } - if (count) { - stats.addTotal(currentLast); - currentLast->setDropped(count); - } - droppedElements[log_id] = currentLast; - lastLoggedElements[log_id] = elem; - unlock(); - return len; - } - if (dropped) { // State 1 or 2 - if (count) { // State 2 - log(dropped); // report chatty - } else { // State 1 - delete dropped; - } - droppedElements[log_id] = nullptr; - log(currentLast); // report last message in the series - } else { // State 0 - delete currentLast; - } - } - lastLoggedElements[log_id] = new LogBufferElement(*elem); - - log(elem); - unlock(); - - return len; -} - -// assumes LogBuffer::wrlock() held, owns elem, look after garbage collection -void LogBuffer::log(LogBufferElement* elem) { - // cap on how far back we will sort in-place, otherwise append - static uint32_t too_far_back = 5; // five seconds - // Insert elements in time sorted order if possible - // NB: if end is region locked, place element at end of list - LogBufferElementCollection::iterator it = mLogElements.end(); - LogBufferElementCollection::iterator last = it; - if (__predict_true(it != mLogElements.begin())) --it; - if (__predict_false(it == mLogElements.begin()) || - __predict_true((*it)->getRealTime() <= elem->getRealTime()) || - __predict_false((((*it)->getRealTime().tv_sec - too_far_back) > - elem->getRealTime().tv_sec) && - (elem->getLogId() != LOG_ID_KERNEL) && - ((*it)->getLogId() != LOG_ID_KERNEL))) { - mLogElements.push_back(elem); - } else { - log_time end(log_time::EPOCH); - bool end_set = false; - bool end_always = false; - - LogTimeEntry::rdlock(); - - LastLogTimes::iterator times = mTimes.begin(); - while (times != mTimes.end()) { - LogTimeEntry* entry = times->get(); - if (!entry->mNonBlock) { - end_always = true; - break; - } - // it passing mEnd is blocked by the following checks. - if (!end_set || (end <= entry->mEnd)) { - end = entry->mEnd; - end_set = true; - } - times++; - } - - if (end_always || (end_set && (end > (*it)->getRealTime()))) { - mLogElements.push_back(elem); - } else { - // should be short as timestamps are localized near end() - do { - last = it; - if (__predict_false(it == mLogElements.begin())) { - break; - } - --it; - } while (((*it)->getRealTime() > elem->getRealTime()) && - (!end_set || (end <= (*it)->getRealTime()))); - mLogElements.insert(last, elem); - } - LogTimeEntry::unlock(); - } - - stats.add(elem); - maybePrune(elem->getLogId()); -} - -// Prune at most 10% of the log entries or maxPrune, whichever is less. -// -// LogBuffer::wrlock() must be held when this function is called. -void LogBuffer::maybePrune(log_id_t id) { - size_t sizes = stats.sizes(id); - unsigned long maxSize = log_buffer_size(id); - if (sizes > maxSize) { - size_t sizeOver = sizes - ((maxSize * 9) / 10); - size_t elements = stats.realElements(id); - size_t minElements = elements / 100; - if (minElements < minPrune) { - minElements = minPrune; - } - unsigned long pruneRows = elements * sizeOver / sizes; - if (pruneRows < minElements) { - pruneRows = minElements; - } - if (pruneRows > maxPrune) { - pruneRows = maxPrune; - } - prune(id, pruneRows); - } -} - -LogBufferElementCollection::iterator LogBuffer::erase( - LogBufferElementCollection::iterator it, bool coalesce) { - LogBufferElement* element = *it; - log_id_t id = element->getLogId(); - - // Remove iterator references in the various lists that will become stale - // after the element is erased from the main logging list. - - { // start of scope for found iterator - int key = ((id == LOG_ID_EVENTS) || (id == LOG_ID_SECURITY)) - ? element->getTag() - : element->getUid(); - LogBufferIteratorMap::iterator found = mLastWorst[id].find(key); - if ((found != mLastWorst[id].end()) && (it == found->second)) { - mLastWorst[id].erase(found); - } - } - - { // start of scope for pid found iterator - // element->getUid() may not be AID_SYSTEM for next-best-watermark. - // will not assume id != LOG_ID_EVENTS or LOG_ID_SECURITY for KISS and - // long term code stability, find() check should be fast for those ids. - LogBufferPidIteratorMap::iterator found = - mLastWorstPidOfSystem[id].find(element->getPid()); - if ((found != mLastWorstPidOfSystem[id].end()) && - (it == found->second)) { - mLastWorstPidOfSystem[id].erase(found); - } - } - - bool setLast[LOG_ID_MAX]; - bool doSetLast = false; - log_id_for_each(i) { - doSetLast |= setLast[i] = mLastSet[i] && (it == mLast[i]); - } -#ifdef DEBUG_CHECK_FOR_STALE_ENTRIES - LogBufferElementCollection::iterator bad = it; - int key = ((id == LOG_ID_EVENTS) || (id == LOG_ID_SECURITY)) - ? element->getTag() - : element->getUid(); -#endif - it = mLogElements.erase(it); - if (doSetLast) { - log_id_for_each(i) { - if (setLast[i]) { - if (__predict_false(it == mLogElements.end())) { // impossible - mLastSet[i] = false; - mLast[i] = mLogElements.begin(); - } else { - mLast[i] = it; // push down the road as next-best-watermark - } - } - } - } -#ifdef DEBUG_CHECK_FOR_STALE_ENTRIES - log_id_for_each(i) { - for (auto b : mLastWorst[i]) { - if (bad == b.second) { - android::prdebug("stale mLastWorst[%d] key=%d mykey=%d\n", i, - b.first, key); - } - } - for (auto b : mLastWorstPidOfSystem[i]) { - if (bad == b.second) { - android::prdebug("stale mLastWorstPidOfSystem[%d] pid=%d\n", i, - b.first); - } - } - if (mLastSet[i] && (bad == mLast[i])) { - android::prdebug("stale mLast[%d]\n", i); - mLastSet[i] = false; - mLast[i] = mLogElements.begin(); - } - } -#endif - if (coalesce) { - stats.erase(element); - } else { - stats.subtract(element); - } - delete element; - - return it; -} - -// Define a temporary mechanism to report the last LogBufferElement pointer -// for the specified uid, pid and tid. Used below to help merge-sort when -// pruning for worst UID. -class LogBufferElementKey { - const union { - struct { - uint32_t uid; - uint16_t pid; - uint16_t tid; - } __packed; - uint64_t value; - } __packed; - - public: - LogBufferElementKey(uid_t uid, pid_t pid, pid_t tid) - : uid(uid), pid(pid), tid(tid) { - } - explicit LogBufferElementKey(uint64_t key) : value(key) { - } - - uint64_t getKey() { - return value; - } -}; - -class LogBufferElementLast { - typedef std::unordered_map<uint64_t, LogBufferElement*> LogBufferElementMap; - LogBufferElementMap map; - - public: - bool coalesce(LogBufferElement* element, uint16_t dropped) { - LogBufferElementKey key(element->getUid(), element->getPid(), - element->getTid()); - LogBufferElementMap::iterator it = map.find(key.getKey()); - if (it != map.end()) { - LogBufferElement* found = it->second; - uint16_t moreDropped = found->getDropped(); - if ((dropped + moreDropped) > USHRT_MAX) { - map.erase(it); - } else { - found->setDropped(dropped + moreDropped); - return true; - } - } - return false; - } - - void add(LogBufferElement* element) { - LogBufferElementKey key(element->getUid(), element->getPid(), - element->getTid()); - map[key.getKey()] = element; - } - - inline void clear() { - map.clear(); - } - - void clear(LogBufferElement* element) { - log_time current = - element->getRealTime() - log_time(EXPIRE_RATELIMIT, 0); - for (LogBufferElementMap::iterator it = map.begin(); it != map.end();) { - LogBufferElement* mapElement = it->second; - if ((mapElement->getDropped() >= EXPIRE_THRESHOLD) && - (current > mapElement->getRealTime())) { - it = map.erase(it); - } else { - ++it; - } - } - } -}; - -// Determine if watermark is within pruneMargin + 1s from the end of the list, -// the caller will use this result to set an internal busy flag indicating -// the prune operation could not be completed because a reader is blocking -// the request. -bool LogBuffer::isBusy(log_time watermark) { - LogBufferElementCollection::iterator ei = mLogElements.end(); - --ei; - return watermark < ((*ei)->getRealTime() - pruneMargin - log_time(1, 0)); -} - -// If the selected reader is blocking our pruning progress, decide on -// what kind of mitigation is necessary to unblock the situation. -void LogBuffer::kickMe(LogTimeEntry* me, log_id_t id, unsigned long pruneRows) { - if (stats.sizes(id) > (2 * log_buffer_size(id))) { // +100% - // A misbehaving or slow reader has its connection - // dropped if we hit too much memory pressure. - android::prdebug("Kicking blocked reader, pid %d, from LogBuffer::kickMe()\n", - me->mClient->getPid()); - me->release_Locked(); - } else if (me->mTimeout.tv_sec || me->mTimeout.tv_nsec) { - // Allow a blocked WRAP timeout reader to - // trigger and start reporting the log data. - me->triggerReader_Locked(); - } else { - // tell slow reader to skip entries to catch up - android::prdebug( - "Skipping %lu entries from slow reader, pid %d, from LogBuffer::kickMe()\n", - pruneRows, me->mClient->getPid()); - me->triggerSkip_Locked(id, pruneRows); - } -} - -// prune "pruneRows" of type "id" from the buffer. -// -// This garbage collection task is used to expire log entries. It is called to -// remove all logs (clear), all UID logs (unprivileged clear), or every -// 256 or 10% of the total logs (whichever is less) to prune the logs. -// -// First there is a prep phase where we discover the reader region lock that -// acts as a backstop to any pruning activity to stop there and go no further. -// -// There are three major pruning loops that follow. All expire from the oldest -// entries. Since there are multiple log buffers, the Android logging facility -// will appear to drop entries 'in the middle' when looking at multiple log -// sources and buffers. This effect is slightly more prominent when we prune -// the worst offender by logging source. Thus the logs slowly loose content -// and value as you move back in time. This is preferred since chatty sources -// invariably move the logs value down faster as less chatty sources would be -// expired in the noise. -// -// The first loop performs blacklisting and worst offender pruning. Falling -// through when there are no notable worst offenders and have not hit the -// region lock preventing further worst offender pruning. This loop also looks -// after managing the chatty log entries and merging to help provide -// statistical basis for blame. The chatty entries are not a notification of -// how much logs you may have, but instead represent how much logs you would -// have had in a virtual log buffer that is extended to cover all the in-memory -// logs without loss. They last much longer than the represented pruned logs -// since they get multiplied by the gains in the non-chatty log sources. -// -// The second loop get complicated because an algorithm of watermarks and -// history is maintained to reduce the order and keep processing time -// down to a minimum at scale. These algorithms can be costly in the face -// of larger log buffers, or severly limited processing time granted to a -// background task at lowest priority. -// -// This second loop does straight-up expiration from the end of the logs -// (again, remember for the specified log buffer id) but does some whitelist -// preservation. Thus whitelist is a Hail Mary low priority, blacklists and -// spam filtration all take priority. This second loop also checks if a region -// lock is causing us to buffer too much in the logs to help the reader(s), -// and will tell the slowest reader thread to skip log entries, and if -// persistent and hits a further threshold, kill the reader thread. -// -// The third thread is optional, and only gets hit if there was a whitelist -// and more needs to be pruned against the backstop of the region lock. -// -// LogBuffer::wrlock() must be held when this function is called. -// -bool LogBuffer::prune(log_id_t id, unsigned long pruneRows, uid_t caller_uid) { - LogTimeEntry* oldest = nullptr; - bool busy = false; - bool clearAll = pruneRows == ULONG_MAX; - - LogTimeEntry::rdlock(); - - // Region locked? - LastLogTimes::iterator times = mTimes.begin(); - while (times != mTimes.end()) { - LogTimeEntry* entry = times->get(); - if (entry->isWatching(id) && - (!oldest || (oldest->mStart > entry->mStart) || - ((oldest->mStart == entry->mStart) && - (entry->mTimeout.tv_sec || entry->mTimeout.tv_nsec)))) { - oldest = entry; - } - times++; - } - log_time watermark(log_time::tv_sec_max, log_time::tv_nsec_max); - if (oldest) watermark = oldest->mStart - pruneMargin; - - LogBufferElementCollection::iterator it; - - if (__predict_false(caller_uid != AID_ROOT)) { // unlikely - // Only here if clear all request from non system source, so chatty - // filter logistics is not required. - it = mLastSet[id] ? mLast[id] : mLogElements.begin(); - while (it != mLogElements.end()) { - LogBufferElement* element = *it; - - if ((element->getLogId() != id) || - (element->getUid() != caller_uid)) { - ++it; - continue; - } - - if (!mLastSet[id] || ((*mLast[id])->getLogId() != id)) { - mLast[id] = it; - mLastSet[id] = true; - } - - if (oldest && (watermark <= element->getRealTime())) { - busy = isBusy(watermark); - if (busy) kickMe(oldest, id, pruneRows); - break; - } - - it = erase(it); - if (--pruneRows == 0) { - break; - } - } - LogTimeEntry::unlock(); - return busy; - } - - // prune by worst offenders; by blacklist, UID, and by PID of system UID - bool hasBlacklist = (id != LOG_ID_SECURITY) && mPrune.naughty(); - while (!clearAll && (pruneRows > 0)) { - // recalculate the worst offender on every batched pass - int worst = -1; // not valid for getUid() or getKey() - size_t worst_sizes = 0; - size_t second_worst_sizes = 0; - pid_t worstPid = 0; // POSIX guarantees PID != 0 - - if (worstUidEnabledForLogid(id) && mPrune.worstUidEnabled()) { - // Calculate threshold as 12.5% of available storage - size_t threshold = log_buffer_size(id) / 8; - - if ((id == LOG_ID_EVENTS) || (id == LOG_ID_SECURITY)) { - stats.sortTags(AID_ROOT, (pid_t)0, 2, id) - .findWorst(worst, worst_sizes, second_worst_sizes, - threshold); - // per-pid filter for AID_SYSTEM sources is too complex - } else { - stats.sort(AID_ROOT, (pid_t)0, 2, id) - .findWorst(worst, worst_sizes, second_worst_sizes, - threshold); - - if ((worst == AID_SYSTEM) && mPrune.worstPidOfSystemEnabled()) { - stats.sortPids(worst, (pid_t)0, 2, id) - .findWorst(worstPid, worst_sizes, second_worst_sizes); - } - } - } - - // skip if we have neither worst nor naughty filters - if ((worst == -1) && !hasBlacklist) { - break; - } - - bool kick = false; - bool leading = true; - it = mLastSet[id] ? mLast[id] : mLogElements.begin(); - // Perform at least one mandatory garbage collection cycle in following - // - clear leading chatty tags - // - coalesce chatty tags - // - check age-out of preserved logs - bool gc = pruneRows <= 1; - if (!gc && (worst != -1)) { - { // begin scope for worst found iterator - LogBufferIteratorMap::iterator found = - mLastWorst[id].find(worst); - if ((found != mLastWorst[id].end()) && - (found->second != mLogElements.end())) { - leading = false; - it = found->second; - } - } - if (worstPid) { // begin scope for pid worst found iterator - // FYI: worstPid only set if !LOG_ID_EVENTS and - // !LOG_ID_SECURITY, not going to make that assumption ... - LogBufferPidIteratorMap::iterator found = - mLastWorstPidOfSystem[id].find(worstPid); - if ((found != mLastWorstPidOfSystem[id].end()) && - (found->second != mLogElements.end())) { - leading = false; - it = found->second; - } - } - } - static const timespec too_old = { EXPIRE_HOUR_THRESHOLD * 60 * 60, 0 }; - LogBufferElementCollection::iterator lastt; - lastt = mLogElements.end(); - --lastt; - LogBufferElementLast last; - while (it != mLogElements.end()) { - LogBufferElement* element = *it; - - if (oldest && (watermark <= element->getRealTime())) { - busy = isBusy(watermark); - // Do not let chatty eliding trigger any reader mitigation - break; - } - - if (element->getLogId() != id) { - ++it; - continue; - } - // below this point element->getLogId() == id - - if (leading && (!mLastSet[id] || ((*mLast[id])->getLogId() != id))) { - mLast[id] = it; - mLastSet[id] = true; - } - - uint16_t dropped = element->getDropped(); - - // remove any leading drops - if (leading && dropped) { - it = erase(it); - continue; - } - - if (dropped && last.coalesce(element, dropped)) { - it = erase(it, true); - continue; - } - - int key = ((id == LOG_ID_EVENTS) || (id == LOG_ID_SECURITY)) - ? element->getTag() - : element->getUid(); - - if (hasBlacklist && mPrune.naughty(element)) { - last.clear(element); - it = erase(it); - if (dropped) { - continue; - } - - pruneRows--; - if (pruneRows == 0) { - break; - } - - if (key == worst) { - kick = true; - if (worst_sizes < second_worst_sizes) { - break; - } - worst_sizes -= element->getMsgLen(); - } - continue; - } - - if ((element->getRealTime() < ((*lastt)->getRealTime() - too_old)) || - (element->getRealTime() > (*lastt)->getRealTime())) { - break; - } - - if (dropped) { - last.add(element); - if (worstPid && - ((!gc && (element->getPid() == worstPid)) || - (mLastWorstPidOfSystem[id].find(element->getPid()) == - mLastWorstPidOfSystem[id].end()))) { - // element->getUid() may not be AID_SYSTEM, next best - // watermark if current one empty. id is not LOG_ID_EVENTS - // or LOG_ID_SECURITY because of worstPid check. - mLastWorstPidOfSystem[id][element->getPid()] = it; - } - if ((!gc && !worstPid && (key == worst)) || - (mLastWorst[id].find(key) == mLastWorst[id].end())) { - mLastWorst[id][key] = it; - } - ++it; - continue; - } - - if ((key != worst) || - (worstPid && (element->getPid() != worstPid))) { - leading = false; - last.clear(element); - ++it; - continue; - } - // key == worst below here - // If worstPid set, then element->getPid() == worstPid below here - - pruneRows--; - if (pruneRows == 0) { - break; - } - - kick = true; - - uint16_t len = element->getMsgLen(); - - // do not create any leading drops - if (leading) { - it = erase(it); - } else { - stats.drop(element); - element->setDropped(1); - if (last.coalesce(element, 1)) { - it = erase(it, true); - } else { - last.add(element); - if (worstPid && - (!gc || (mLastWorstPidOfSystem[id].find(worstPid) == - mLastWorstPidOfSystem[id].end()))) { - // element->getUid() may not be AID_SYSTEM, next best - // watermark if current one empty. id is not - // LOG_ID_EVENTS or LOG_ID_SECURITY because of worstPid. - mLastWorstPidOfSystem[id][worstPid] = it; - } - if ((!gc && !worstPid) || - (mLastWorst[id].find(worst) == mLastWorst[id].end())) { - mLastWorst[id][worst] = it; - } - ++it; - } - } - if (worst_sizes < second_worst_sizes) { - break; - } - worst_sizes -= len; - } - last.clear(); - - if (!kick || !mPrune.worstUidEnabled()) { - break; // the following loop will ask bad clients to skip/drop - } - } - - bool whitelist = false; - bool hasWhitelist = (id != LOG_ID_SECURITY) && mPrune.nice() && !clearAll; - it = mLastSet[id] ? mLast[id] : mLogElements.begin(); - while ((pruneRows > 0) && (it != mLogElements.end())) { - LogBufferElement* element = *it; - - if (element->getLogId() != id) { - it++; - continue; - } - - if (!mLastSet[id] || ((*mLast[id])->getLogId() != id)) { - mLast[id] = it; - mLastSet[id] = true; - } - - if (oldest && (watermark <= element->getRealTime())) { - busy = isBusy(watermark); - if (!whitelist && busy) kickMe(oldest, id, pruneRows); - break; - } - - if (hasWhitelist && !element->getDropped() && mPrune.nice(element)) { - // WhiteListed - whitelist = true; - it++; - continue; - } - - it = erase(it); - pruneRows--; - } - - // Do not save the whitelist if we are reader range limited - if (whitelist && (pruneRows > 0)) { - it = mLastSet[id] ? mLast[id] : mLogElements.begin(); - while ((it != mLogElements.end()) && (pruneRows > 0)) { - LogBufferElement* element = *it; - - if (element->getLogId() != id) { - ++it; - continue; - } - - if (!mLastSet[id] || ((*mLast[id])->getLogId() != id)) { - mLast[id] = it; - mLastSet[id] = true; - } - - if (oldest && (watermark <= element->getRealTime())) { - busy = isBusy(watermark); - if (busy) kickMe(oldest, id, pruneRows); - break; - } - - it = erase(it); - pruneRows--; - } - } - - LogTimeEntry::unlock(); - - return (pruneRows > 0) && busy; -} - -// clear all rows of type "id" from the buffer. -bool LogBuffer::clear(log_id_t id, uid_t uid) { - bool busy = true; - // If it takes more than 4 tries (seconds) to clear, then kill reader(s) - for (int retry = 4;;) { - if (retry == 1) { // last pass - // Check if it is still busy after the sleep, we say prune - // one entry, not another clear run, so we are looking for - // the quick side effect of the return value to tell us if - // we have a _blocked_ reader. - wrlock(); - busy = prune(id, 1, uid); - unlock(); - // It is still busy, blocked reader(s), lets kill them all! - // otherwise, lets be a good citizen and preserve the slow - // readers and let the clear run (below) deal with determining - // if we are still blocked and return an error code to caller. - if (busy) { - LogTimeEntry::wrlock(); - LastLogTimes::iterator times = mTimes.begin(); - while (times != mTimes.end()) { - LogTimeEntry* entry = times->get(); - // Killer punch - if (entry->isWatching(id)) { - android::prdebug( - "Kicking blocked reader, pid %d, from LogBuffer::clear()\n", - entry->mClient->getPid()); - entry->release_Locked(); - } - times++; - } - LogTimeEntry::unlock(); - } - } - wrlock(); - busy = prune(id, ULONG_MAX, uid); - unlock(); - if (!busy || !--retry) { - break; - } - sleep(1); // Let reader(s) catch up after notification - } - return busy; -} - -// get the used space associated with "id". -unsigned long LogBuffer::getSizeUsed(log_id_t id) { - rdlock(); - size_t retval = stats.sizes(id); - unlock(); - return retval; -} - -// set the total space allocated to "id" -int LogBuffer::setSize(log_id_t id, unsigned long size) { - // Reasonable limits ... - if (!__android_logger_valid_buffer_size(size)) { - return -1; - } - wrlock(); - log_buffer_size(id) = size; - unlock(); - return 0; -} - -// get the total space allocated to "id" -unsigned long LogBuffer::getSize(log_id_t id) { - rdlock(); - size_t retval = log_buffer_size(id); - unlock(); - return retval; -} - -log_time LogBuffer::flushTo(SocketClient* reader, const log_time& start, - pid_t* lastTid, bool privileged, bool security, - int (*filter)(const LogBufferElement* element, - void* arg), - void* arg) { - LogBufferElementCollection::iterator it; - uid_t uid = reader->getUid(); - - rdlock(); - - if (start == log_time::EPOCH) { - // client wants to start from the beginning - it = mLogElements.begin(); - } else { - // Cap to 300 iterations we look back for out-of-order entries. - size_t count = 300; - - // Client wants to start from some specified time. Chances are - // we are better off starting from the end of the time sorted list. - LogBufferElementCollection::iterator last; - for (last = it = mLogElements.end(); it != mLogElements.begin(); - /* do nothing */) { - --it; - LogBufferElement* element = *it; - if (element->getRealTime() > start) { - last = it; - } else if (element->getRealTime() == start) { - last = ++it; - break; - } else if (!--count) { - break; - } - } - it = last; - } - - log_time curr = start; - - LogBufferElement* lastElement = nullptr; // iterator corruption paranoia - static const size_t maxSkip = 4194304; // maximum entries to skip - size_t skip = maxSkip; - for (; it != mLogElements.end(); ++it) { - LogBufferElement* element = *it; - - if (!--skip) { - android::prdebug("reader.per: too many elements skipped"); - break; - } - if (element == lastElement) { - android::prdebug("reader.per: identical elements"); - break; - } - lastElement = element; - - if (!privileged && (element->getUid() != uid)) { - continue; - } - - if (!security && (element->getLogId() == LOG_ID_SECURITY)) { - continue; - } - - // NB: calling out to another object with wrlock() held (safe) - if (filter) { - int ret = (*filter)(element, arg); - if (ret == false) { - continue; - } - if (ret != true) { - break; - } - } - - bool sameTid = false; - if (lastTid) { - sameTid = lastTid[element->getLogId()] == element->getTid(); - // Dropped (chatty) immediately following a valid log from the - // same source in the same log buffer indicates we have a - // multiple identical squash. chatty that differs source - // is due to spam filter. chatty to chatty of different - // source is also due to spam filter. - lastTid[element->getLogId()] = - (element->getDropped() && !sameTid) ? 0 : element->getTid(); - } - - unlock(); - - // range locking in LastLogTimes looks after us - curr = element->flushTo(reader, this, sameTid); - - if (curr == element->FLUSH_ERROR) { - return curr; - } - - skip = maxSkip; - rdlock(); - } - unlock(); - - return curr; -} - -std::string LogBuffer::formatStatistics(uid_t uid, pid_t pid, - unsigned int logMask) { - wrlock(); - - std::string ret = stats.format(uid, pid, logMask); - - unlock(); - - return ret; -} diff --git a/logd/LogBuffer.h b/logd/LogBuffer.h index c2d5b9734..c5d333a9e 100644 --- a/logd/LogBuffer.h +++ b/logd/LogBuffer.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2014 The Android Open Source Project + * Copyright (C) 2020 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,176 +14,62 @@ * limitations under the License. */ -#ifndef _LOGD_LOG_BUFFER_H__ -#define _LOGD_LOG_BUFFER_H__ +#pragma once #include <sys/types.h> -#include <list> -#include <string> - -#include <android/log.h> -#include <private/android_filesystem_config.h> -#include <sysutils/SocketClient.h> - -#include "LogBufferElement.h" -#include "LogStatistics.h" -#include "LogTags.h" -#include "LogTimes.h" -#include "LogWhiteBlackList.h" - -// -// We are either in 1970ish (MONOTONIC) or 2016+ish (REALTIME) so to -// differentiate without prejudice, we use 1972 to delineate, earlier -// is likely monotonic, later is real. Otherwise we start using a -// dividing line between monotonic and realtime if more than a minute -// difference between them. -// -namespace android { - -static bool isMonotonic(const log_time& mono) { - static const uint32_t EPOCH_PLUS_2_YEARS = 2 * 24 * 60 * 60 * 1461 / 4; - static const uint32_t EPOCH_PLUS_MINUTE = 60; - - if (mono.tv_sec >= EPOCH_PLUS_2_YEARS) { - return false; - } - - log_time now(CLOCK_REALTIME); - - /* Timezone and ntp time setup? */ - if (now.tv_sec >= EPOCH_PLUS_2_YEARS) { - return true; - } - - /* no way to differentiate realtime from monotonic time */ - if (now.tv_sec < EPOCH_PLUS_MINUTE) { - return false; - } - - log_time cpu(CLOCK_MONOTONIC); - /* too close to call to differentiate monotonic times from realtime */ - if ((cpu.tv_sec + EPOCH_PLUS_MINUTE) >= now.tv_sec) { - return false; - } - - /* dividing line half way between monotonic and realtime */ - return mono.tv_sec < ((cpu.tv_sec + now.tv_sec) / 2); -} -} - -typedef std::list<LogBufferElement*> LogBufferElementCollection; +#include <functional> -class LogBuffer { - LogBufferElementCollection mLogElements; - pthread_rwlock_t mLogElementsLock; - - LogStatistics stats; - - PruneList mPrune; - // watermark for last per log id - LogBufferElementCollection::iterator mLast[LOG_ID_MAX]; - bool mLastSet[LOG_ID_MAX]; - // watermark of any worst/chatty uid processing - typedef std::unordered_map<uid_t, LogBufferElementCollection::iterator> - LogBufferIteratorMap; - LogBufferIteratorMap mLastWorst[LOG_ID_MAX]; - // watermark of any worst/chatty pid of system processing - typedef std::unordered_map<pid_t, LogBufferElementCollection::iterator> - LogBufferPidIteratorMap; - LogBufferPidIteratorMap mLastWorstPidOfSystem[LOG_ID_MAX]; - - unsigned long mMaxSize[LOG_ID_MAX]; - - bool monotonic; - - LogTags tags; - - LogBufferElement* lastLoggedElements[LOG_ID_MAX]; - LogBufferElement* droppedElements[LOG_ID_MAX]; - void log(LogBufferElement* elem); - - public: - LastLogTimes& mTimes; - - explicit LogBuffer(LastLogTimes* times); - ~LogBuffer(); - void init(); - bool isMonotonic() { - return monotonic; - } - - int log(log_id_t log_id, log_time realtime, uid_t uid, pid_t pid, pid_t tid, const char* msg, - uint16_t len); - // lastTid is an optional context to help detect if the last previous - // valid message was from the same source so we can differentiate chatty - // filter types (identical or expired) - log_time flushTo(SocketClient* writer, const log_time& start, - pid_t* lastTid, // &lastTid[LOG_ID_MAX] or nullptr - bool privileged, bool security, - int (*filter)(const LogBufferElement* element, - void* arg) = nullptr, - void* arg = nullptr); - - bool clear(log_id_t id, uid_t uid = AID_ROOT); - unsigned long getSize(log_id_t id); - int setSize(log_id_t id, unsigned long size); - unsigned long getSizeUsed(log_id_t id); - - std::string formatStatistics(uid_t uid, pid_t pid, unsigned int logMask); - - void enableStatistics() { - stats.enableStatistics(); - } - - int initPrune(const char* cp) { - return mPrune.init(cp); - } - std::string formatPrune() { - return mPrune.format(); - } - - std::string formatGetEventTag(uid_t uid, const char* name, - const char* format) { - return tags.formatGetEventTag(uid, name, format); - } - std::string formatEntry(uint32_t tag, uid_t uid) { - return tags.formatEntry(tag, uid); - } - const char* tagToName(uint32_t tag) { - return tags.tagToName(tag); - } - - // helper must be protected directly or implicitly by wrlock()/unlock() - const char* pidToName(pid_t pid) { - return stats.pidToName(pid); - } - uid_t pidToUid(pid_t pid) { return stats.pidToUid(pid); } - const char* uidToName(uid_t uid) { - return stats.uidToName(uid); - } - void wrlock() { - pthread_rwlock_wrlock(&mLogElementsLock); - } - void rdlock() { - pthread_rwlock_rdlock(&mLogElementsLock); - } - void unlock() { - pthread_rwlock_unlock(&mLogElementsLock); - } - - private: - static constexpr size_t minPrune = 4; - static constexpr size_t maxPrune = 256; - static const log_time pruneMargin; - - void maybePrune(log_id_t id); - bool isBusy(log_time watermark); - void kickMe(LogTimeEntry* me, log_id_t id, unsigned long pruneRows); - - bool prune(log_id_t id, unsigned long pruneRows, uid_t uid = AID_ROOT); - LogBufferElementCollection::iterator erase( - LogBufferElementCollection::iterator it, bool coalesce = false); +#include <log/log.h> +#include <log/log_read.h> + +#include "LogWriter.h" + +// A mask to represent which log buffers a reader is watching, values are (1 << LOG_ID_MAIN), etc. +using LogMask = uint32_t; +constexpr uint32_t kLogMaskAll = 0xFFFFFFFF; + +// State that a LogBuffer may want to persist across calls to FlushTo(). +class FlushToState { + public: + FlushToState(uint64_t start, LogMask log_mask) : start_(start), log_mask_(log_mask) {} + virtual ~FlushToState() {} + + uint64_t start() const { return start_; } + void set_start(uint64_t start) { start_ = start; } + + LogMask log_mask() const { return log_mask_; } + + private: + uint64_t start_; + LogMask log_mask_; +}; + +// Enum for the return values of the `filter` function passed to FlushTo(). +enum class FilterResult { + kSkip, + kStop, + kWrite, }; -#endif // _LOGD_LOG_BUFFER_H__ +class LogBuffer { + public: + virtual ~LogBuffer() {} + + virtual void Init() = 0; + + virtual int Log(log_id_t log_id, log_time realtime, uid_t uid, pid_t pid, pid_t tid, + const char* msg, uint16_t len) = 0; + + virtual std::unique_ptr<FlushToState> CreateFlushToState(uint64_t start, LogMask log_mask) = 0; + virtual bool FlushTo( + LogWriter* writer, FlushToState& state, + const std::function<FilterResult(log_id_t log_id, pid_t pid, uint64_t sequence, + log_time realtime)>& filter) = 0; + + virtual bool Clear(log_id_t id, uid_t uid) = 0; + virtual unsigned long GetSize(log_id_t id) = 0; + virtual int SetSize(log_id_t id, unsigned long size) = 0; + + virtual uint64_t sequence() const = 0; +}; diff --git a/logd/LogBufferElement.cpp b/logd/LogBufferElement.cpp index ec8193355..dd779f97b 100644 --- a/logd/LogBufferElement.cpp +++ b/logd/LogBufferElement.cpp @@ -14,6 +14,8 @@ * limitations under the License. */ +#include "LogBufferElement.h" + #include <ctype.h> #include <endian.h> #include <fcntl.h> @@ -22,87 +24,109 @@ #include <time.h> #include <unistd.h> +#include <log/log_read.h> #include <private/android_logger.h> -#include "LogBuffer.h" -#include "LogBufferElement.h" -#include "LogCommand.h" -#include "LogReader.h" +#include "LogStatistics.h" #include "LogUtils.h" -const log_time LogBufferElement::FLUSH_ERROR((uint32_t)-1, (uint32_t)-1); -atomic_int_fast64_t LogBufferElement::sequence(1); - -LogBufferElement::LogBufferElement(log_id_t log_id, log_time realtime, - uid_t uid, pid_t pid, pid_t tid, - const char* msg, uint16_t len) - : mUid(uid), - mPid(pid), - mTid(tid), - mRealTime(realtime), - mMsgLen(len), - mLogId(log_id), - mDropped(false) { - mMsg = new char[len]; - memcpy(mMsg, msg, len); +LogBufferElement::LogBufferElement(log_id_t log_id, log_time realtime, uid_t uid, pid_t pid, + pid_t tid, uint64_t sequence, const char* msg, uint16_t len) + : uid_(uid), + pid_(pid), + tid_(tid), + sequence_(sequence), + realtime_(realtime), + msg_len_(len), + log_id_(log_id), + dropped_(false) { + msg_ = new char[len]; + memcpy(msg_, msg, len); } LogBufferElement::LogBufferElement(const LogBufferElement& elem) - : mUid(elem.mUid), - mPid(elem.mPid), - mTid(elem.mTid), - mRealTime(elem.mRealTime), - mMsgLen(elem.mMsgLen), - mLogId(elem.mLogId), - mDropped(elem.mDropped) { - if (mDropped) { - mTag = elem.getTag(); + : uid_(elem.uid_), + pid_(elem.pid_), + tid_(elem.tid_), + sequence_(elem.sequence_), + realtime_(elem.realtime_), + msg_len_(elem.msg_len_), + log_id_(elem.log_id_), + dropped_(elem.dropped_) { + if (dropped_) { + tag_ = elem.GetTag(); + } else { + msg_ = new char[msg_len_]; + memcpy(msg_, elem.msg_, msg_len_); + } +} + +LogBufferElement::LogBufferElement(LogBufferElement&& elem) noexcept + : uid_(elem.uid_), + pid_(elem.pid_), + tid_(elem.tid_), + sequence_(elem.sequence_), + realtime_(elem.realtime_), + msg_len_(elem.msg_len_), + log_id_(elem.log_id_), + dropped_(elem.dropped_) { + if (dropped_) { + tag_ = elem.GetTag(); } else { - mMsg = new char[mMsgLen]; - memcpy(mMsg, elem.mMsg, mMsgLen); + msg_ = elem.msg_; + elem.msg_ = nullptr; } } LogBufferElement::~LogBufferElement() { - if (!mDropped) { - delete[] mMsg; + if (!dropped_) { + delete[] msg_; } } -uint32_t LogBufferElement::getTag() const { +uint32_t LogBufferElement::GetTag() const { // Binary buffers have no tag. - if (!isBinary()) { + if (!IsBinary(log_id())) { return 0; } - // Dropped messages store the tag in place of mMsg. - if (mDropped) { - return mTag; + // Dropped messages store the tag in place of msg_. + if (dropped_) { + return tag_; } - // For non-dropped messages, we get the tag from the message header itself. - if (mMsgLen < sizeof(android_event_header_t)) { - return 0; - } + return MsgToTag(msg(), msg_len()); +} - return reinterpret_cast<const android_event_header_t*>(mMsg)->tag; +LogStatisticsElement LogBufferElement::ToLogStatisticsElement() const { + return LogStatisticsElement{ + .uid = uid(), + .pid = pid(), + .tid = tid(), + .tag = GetTag(), + .realtime = realtime(), + .msg = msg(), + .msg_len = msg_len(), + .dropped_count = dropped_count(), + .log_id = log_id(), + }; } -uint16_t LogBufferElement::setDropped(uint16_t value) { - if (mDropped) { - return mDroppedCount = value; +uint16_t LogBufferElement::SetDropped(uint16_t value) { + if (dropped_) { + return dropped_count_ = value; } - // The tag information is saved in mMsg data, which is in a union with mTag, used after mDropped - // is set to true. Therefore we save the tag value aside, delete mMsg, then set mTag to the tag + // The tag information is saved in msg_ data, which is in a union with tag_, used after dropped_ + // is set to true. Therefore we save the tag value aside, delete msg_, then set tag_ to the tag // value in its place. - auto old_tag = getTag(); - delete[] mMsg; - mMsg = nullptr; + auto old_tag = GetTag(); + delete[] msg_; + msg_ = nullptr; - mTag = old_tag; - mDropped = true; - return mDroppedCount = value; + tag_ = old_tag; + dropped_ = true; + return dropped_count_ = value; } // caller must own and free character string @@ -110,7 +134,7 @@ char* android::tidToName(pid_t tid) { char* retval = nullptr; char buffer[256]; snprintf(buffer, sizeof(buffer), "/proc/%u/comm", tid); - int fd = open(buffer, O_RDONLY); + int fd = open(buffer, O_RDONLY | O_CLOEXEC); if (fd >= 0) { ssize_t ret = read(fd, buffer, sizeof(buffer)); if (ret >= (ssize_t)sizeof(buffer)) { @@ -150,8 +174,8 @@ char* android::tidToName(pid_t tid) { return retval; } -// assumption: mMsg == NULL -size_t LogBufferElement::populateDroppedMessage(char*& buffer, LogBuffer* parent, +// assumption: msg_ == NULL +size_t LogBufferElement::PopulateDroppedMessage(char*& buffer, LogStatistics* stats, bool lastSame) { static const char tag[] = "chatty"; @@ -161,17 +185,13 @@ size_t LogBufferElement::populateDroppedMessage(char*& buffer, LogBuffer* parent } static const char format_uid[] = "uid=%u%s%s %s %u line%s"; - parent->wrlock(); - const char* name = parent->uidToName(mUid); - parent->unlock(); - const char* commName = android::tidToName(mTid); - if (!commName && (mTid != mPid)) { - commName = android::tidToName(mPid); + const char* name = stats->UidToName(uid_); + const char* commName = android::tidToName(tid_); + if (!commName && (tid_ != pid_)) { + commName = android::tidToName(pid_); } if (!commName) { - parent->wrlock(); - commName = parent->pidToName(mPid); - parent->unlock(); + commName = stats->PidToName(pid_); } if (name && name[0] && commName && (name[0] == commName[0])) { size_t len = strlen(name + 1); @@ -187,28 +207,27 @@ size_t LogBufferElement::populateDroppedMessage(char*& buffer, LogBuffer* parent } if (name) { char* buf = nullptr; - asprintf(&buf, "(%s)", name); - if (buf) { + int result = asprintf(&buf, "(%s)", name); + if (result != -1) { free(const_cast<char*>(name)); name = buf; } } if (commName) { char* buf = nullptr; - asprintf(&buf, " %s", commName); - if (buf) { + int result = asprintf(&buf, " %s", commName); + if (result != -1) { free(const_cast<char*>(commName)); commName = buf; } } // identical to below to calculate the buffer size required const char* type = lastSame ? "identical" : "expire"; - size_t len = snprintf(nullptr, 0, format_uid, mUid, name ? name : "", - commName ? commName : "", type, getDropped(), - (getDropped() > 1) ? "s" : ""); + size_t len = snprintf(nullptr, 0, format_uid, uid_, name ? name : "", commName ? commName : "", + type, dropped_count(), (dropped_count() > 1) ? "s" : ""); size_t hdrLen; - if (isBinary()) { + if (IsBinary(log_id())) { hdrLen = sizeof(android_log_event_string_t); } else { hdrLen = 1 + sizeof(tag); @@ -222,7 +241,7 @@ size_t LogBufferElement::populateDroppedMessage(char*& buffer, LogBuffer* parent } size_t retval = hdrLen + len; - if (isBinary()) { + if (IsBinary(log_id())) { android_log_event_string_t* event = reinterpret_cast<android_log_event_string_t*>(buffer); @@ -235,45 +254,37 @@ size_t LogBufferElement::populateDroppedMessage(char*& buffer, LogBuffer* parent strcpy(buffer + 1, tag); } - snprintf(buffer + hdrLen, len + 1, format_uid, mUid, name ? name : "", - commName ? commName : "", type, getDropped(), - (getDropped() > 1) ? "s" : ""); + snprintf(buffer + hdrLen, len + 1, format_uid, uid_, name ? name : "", commName ? commName : "", + type, dropped_count(), (dropped_count() > 1) ? "s" : ""); free(const_cast<char*>(name)); free(const_cast<char*>(commName)); return retval; } -log_time LogBufferElement::flushTo(SocketClient* reader, LogBuffer* parent, bool lastSame) { +bool LogBufferElement::FlushTo(LogWriter* writer, LogStatistics* stats, bool lastSame) { struct logger_entry entry = {}; entry.hdr_size = sizeof(struct logger_entry); - entry.lid = mLogId; - entry.pid = mPid; - entry.tid = mTid; - entry.uid = mUid; - entry.sec = mRealTime.tv_sec; - entry.nsec = mRealTime.tv_nsec; - - struct iovec iovec[2]; - iovec[0].iov_base = &entry; - iovec[0].iov_len = entry.hdr_size; + entry.lid = log_id_; + entry.pid = pid_; + entry.tid = tid_; + entry.uid = uid_; + entry.sec = realtime_.tv_sec; + entry.nsec = realtime_.tv_nsec; char* buffer = nullptr; - - if (mDropped) { - entry.len = populateDroppedMessage(buffer, parent, lastSame); - if (!entry.len) return mRealTime; - iovec[1].iov_base = buffer; + const char* msg; + if (dropped_) { + entry.len = PopulateDroppedMessage(buffer, stats, lastSame); + if (!entry.len) return true; + msg = buffer; } else { - entry.len = mMsgLen; - iovec[1].iov_base = mMsg; + msg = msg_; + entry.len = msg_len_; } - iovec[1].iov_len = entry.len; - log_time retval = reader->sendDatav(iovec, 1 + (entry.len != 0)) - ? FLUSH_ERROR - : mRealTime; + bool retval = writer->Write(entry, msg); if (buffer) free(buffer); diff --git a/logd/LogBufferElement.h b/logd/LogBufferElement.h index fd790e4be..b263ca537 100644 --- a/logd/LogBufferElement.h +++ b/logd/LogBufferElement.h @@ -16,15 +16,15 @@ #pragma once -#include <stdatomic.h> #include <stdint.h> #include <stdlib.h> #include <sys/types.h> #include <log/log.h> -#include <sysutils/SocketClient.h> -class LogBuffer; +#include "LogWriter.h" + +#include "LogStatistics.h" #define EXPIRE_HOUR_THRESHOLD 24 // Only expire chatty UID logs to preserve // non-chatty UIDs less than this age in hours @@ -33,67 +33,48 @@ class LogBuffer; #define EXPIRE_RATELIMIT 10 // maximum rate in seconds to report expiration class __attribute__((packed)) LogBufferElement { - friend LogBuffer; - - // sized to match reality of incoming log packets - const uint32_t mUid; - const uint32_t mPid; - const uint32_t mTid; - log_time mRealTime; - union { - char* mMsg; // mDropped == false - int32_t mTag; // mDropped == true - }; - union { - const uint16_t mMsgLen; // mDropped == false - uint16_t mDroppedCount; // mDropped == true - }; - const uint8_t mLogId; - bool mDropped; + public: + LogBufferElement(log_id_t log_id, log_time realtime, uid_t uid, pid_t pid, pid_t tid, + uint64_t sequence, const char* msg, uint16_t len); + LogBufferElement(const LogBufferElement& elem); + LogBufferElement(LogBufferElement&& elem) noexcept; + ~LogBufferElement(); - static atomic_int_fast64_t sequence; + uint32_t GetTag() const; + uint16_t SetDropped(uint16_t value); - // assumption: mDropped == true - size_t populateDroppedMessage(char*& buffer, LogBuffer* parent, - bool lastSame); + bool FlushTo(LogWriter* writer, LogStatistics* parent, bool lastSame); - public: - LogBufferElement(log_id_t log_id, log_time realtime, uid_t uid, pid_t pid, - pid_t tid, const char* msg, uint16_t len); - LogBufferElement(const LogBufferElement& elem); - ~LogBufferElement(); + LogStatisticsElement ToLogStatisticsElement() const; - bool isBinary(void) const { - return (mLogId == LOG_ID_EVENTS) || (mLogId == LOG_ID_SECURITY); - } + log_id_t log_id() const { return static_cast<log_id_t>(log_id_); } + uid_t uid() const { return uid_; } + pid_t pid() const { return pid_; } + pid_t tid() const { return tid_; } + uint16_t msg_len() const { return dropped_ ? 0 : msg_len_; } + const char* msg() const { return dropped_ ? nullptr : msg_; } + uint64_t sequence() const { return sequence_; } + log_time realtime() const { return realtime_; } + uint16_t dropped_count() const { return dropped_ ? dropped_count_ : 0; } - log_id_t getLogId() const { - return static_cast<log_id_t>(mLogId); - } - uid_t getUid(void) const { - return mUid; - } - pid_t getPid(void) const { - return mPid; - } - pid_t getTid(void) const { - return mTid; - } - uint32_t getTag() const; - uint16_t getDropped(void) const { - return mDropped ? mDroppedCount : 0; - } - uint16_t setDropped(uint16_t value); - uint16_t getMsgLen() const { - return mDropped ? 0 : mMsgLen; - } - const char* getMsg() const { - return mDropped ? nullptr : mMsg; - } - log_time getRealTime(void) const { - return mRealTime; - } + private: + // assumption: mDropped == true + size_t PopulateDroppedMessage(char*& buffer, LogStatistics* parent, bool lastSame); - static const log_time FLUSH_ERROR; - log_time flushTo(SocketClient* writer, LogBuffer* parent, bool lastSame); + // sized to match reality of incoming log packets + const uint32_t uid_; + const uint32_t pid_; + const uint32_t tid_; + uint64_t sequence_; + log_time realtime_; + union { + char* msg_; // mDropped == false + int32_t tag_; // mDropped == true + }; + union { + const uint16_t msg_len_; // mDropped == false + uint16_t dropped_count_; // mDropped == true + }; + const uint8_t log_id_; + bool dropped_; }; diff --git a/logd/LogBufferTest.cpp b/logd/LogBufferTest.cpp new file mode 100644 index 000000000..47d2a2f92 --- /dev/null +++ b/logd/LogBufferTest.cpp @@ -0,0 +1,459 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "LogBufferTest.h" + +#include <unistd.h> + +#include <limits> +#include <memory> +#include <regex> +#include <vector> + +#include <android-base/stringprintf.h> +#include <android-base/strings.h> + +#include "LogBuffer.h" +#include "LogReaderThread.h" +#include "LogWriter.h" + +using android::base::Join; +using android::base::Split; +using android::base::StringPrintf; + +#ifndef __ANDROID__ +unsigned long __android_logger_get_buffer_size(log_id_t) { + return 1024 * 1024; +} + +bool __android_logger_valid_buffer_size(unsigned long) { + return true; +} +#endif + +char* android::uidToName(uid_t) { + return nullptr; +} + +static std::vector<std::string> CompareLoggerEntries(const logger_entry& expected, + const logger_entry& result, bool ignore_len) { + std::vector<std::string> errors; + if (!ignore_len && expected.len != result.len) { + errors.emplace_back( + StringPrintf("len: expected %" PRIu16 " vs %" PRIu16, expected.len, result.len)); + } + if (expected.hdr_size != result.hdr_size) { + errors.emplace_back(StringPrintf("hdr_size: %" PRIu16 " vs %" PRIu16, expected.hdr_size, + result.hdr_size)); + } + if (expected.pid != result.pid) { + errors.emplace_back( + StringPrintf("pid: expected %" PRIi32 " vs %" PRIi32, expected.pid, result.pid)); + } + if (expected.tid != result.tid) { + errors.emplace_back( + StringPrintf("tid: expected %" PRIu32 " vs %" PRIu32, expected.tid, result.tid)); + } + if (expected.sec != result.sec) { + errors.emplace_back( + StringPrintf("sec: expected %" PRIu32 " vs %" PRIu32, expected.sec, result.sec)); + } + if (expected.nsec != result.nsec) { + errors.emplace_back( + StringPrintf("nsec: expected %" PRIu32 " vs %" PRIu32, expected.nsec, result.nsec)); + } + if (expected.lid != result.lid) { + errors.emplace_back( + StringPrintf("lid: expected %" PRIu32 " vs %" PRIu32, expected.lid, result.lid)); + } + if (expected.uid != result.uid) { + errors.emplace_back( + StringPrintf("uid: expected %" PRIu32 " vs %" PRIu32, expected.uid, result.uid)); + } + return errors; +} + +static std::string MakePrintable(std::string in) { + if (in.size() > 80) { + in = in.substr(0, 80) + "..."; + } + std::string result; + for (const char c : in) { + if (isprint(c)) { + result.push_back(c); + } else { + result.append(StringPrintf("\\%02x", static_cast<int>(c) & 0xFF)); + } + } + return result; +} + +static std::string CompareMessages(const std::string& expected, const std::string& result) { + if (expected == result) { + return {}; + } + size_t diff_index = 0; + for (; diff_index < std::min(expected.size(), result.size()); ++diff_index) { + if (expected[diff_index] != result[diff_index]) { + break; + } + } + + if (diff_index < 80) { + auto expected_short = MakePrintable(expected); + auto result_short = MakePrintable(result); + return StringPrintf("msg: expected '%s' vs '%s'", expected_short.c_str(), + result_short.c_str()); + } + + auto expected_short = MakePrintable(expected.substr(diff_index)); + auto result_short = MakePrintable(result.substr(diff_index)); + return StringPrintf("msg: index %zu: expected '%s' vs '%s'", diff_index, expected_short.c_str(), + result_short.c_str()); +} + +static std::string CompareRegexMessages(const std::string& expected, const std::string& result) { + auto expected_pieces = Split(expected, std::string("\0", 1)); + auto result_pieces = Split(result, std::string("\0", 1)); + + if (expected_pieces.size() != 3 || result_pieces.size() != 3) { + return StringPrintf( + "msg: should have 3 null delimited strings found %d in expected, %d in result: " + "'%s' vs '%s'", + static_cast<int>(expected_pieces.size()), static_cast<int>(result_pieces.size()), + MakePrintable(expected).c_str(), MakePrintable(result).c_str()); + } + if (expected_pieces[0] != result_pieces[0]) { + return StringPrintf("msg: tag/priority mismatch expected '%s' vs '%s'", + MakePrintable(expected_pieces[0]).c_str(), + MakePrintable(result_pieces[0]).c_str()); + } + std::regex expected_tag_regex(expected_pieces[1]); + if (!std::regex_search(result_pieces[1], expected_tag_regex)) { + return StringPrintf("msg: message regex mismatch expected '%s' vs '%s'", + MakePrintable(expected_pieces[1]).c_str(), + MakePrintable(result_pieces[1]).c_str()); + } + if (expected_pieces[2] != result_pieces[2]) { + return StringPrintf("msg: nothing expected after final null character '%s' vs '%s'", + MakePrintable(expected_pieces[2]).c_str(), + MakePrintable(result_pieces[2]).c_str()); + } + return {}; +} + +void CompareLogMessages(const std::vector<LogMessage>& expected, + const std::vector<LogMessage>& result) { + EXPECT_EQ(expected.size(), result.size()); + size_t end = std::min(expected.size(), result.size()); + size_t num_errors = 0; + for (size_t i = 0; i < end; ++i) { + auto errors = + CompareLoggerEntries(expected[i].entry, result[i].entry, expected[i].regex_compare); + auto msg_error = expected[i].regex_compare + ? CompareRegexMessages(expected[i].message, result[i].message) + : CompareMessages(expected[i].message, result[i].message); + if (!msg_error.empty()) { + errors.emplace_back(msg_error); + } + if (!errors.empty()) { + GTEST_LOG_(ERROR) << "Mismatch log message " << i << "\n" << Join(errors, "\n"); + ++num_errors; + } + } + EXPECT_EQ(0U, num_errors); +} + +void FixupMessages(std::vector<LogMessage>* messages) { + for (auto& [entry, message, _] : *messages) { + entry.hdr_size = sizeof(logger_entry); + entry.len = message.size(); + } +} + +TEST_P(LogBufferTest, smoke) { + std::vector<LogMessage> log_messages = { + {{ + .pid = 1, + .tid = 1, + .sec = 1234, + .nsec = 323001, + .lid = LOG_ID_MAIN, + .uid = 0, + }, + "smoke test"}, + }; + FixupMessages(&log_messages); + LogMessages(log_messages); + + std::vector<LogMessage> read_log_messages; + std::unique_ptr<LogWriter> test_writer(new TestWriter(&read_log_messages, nullptr)); + std::unique_ptr<FlushToState> flush_to_state = log_buffer_->CreateFlushToState(1, kLogMaskAll); + EXPECT_TRUE(log_buffer_->FlushTo(test_writer.get(), *flush_to_state, nullptr)); + EXPECT_EQ(2ULL, flush_to_state->start()); + CompareLogMessages(log_messages, read_log_messages); +} + +TEST_P(LogBufferTest, smoke_with_reader_thread) { + std::vector<LogMessage> log_messages = { + {{.pid = 1, .tid = 2, .sec = 10000, .nsec = 20001, .lid = LOG_ID_MAIN, .uid = 0}, + "first"}, + {{.pid = 10, .tid = 2, .sec = 10000, .nsec = 20002, .lid = LOG_ID_MAIN, .uid = 0}, + "second"}, + {{.pid = 100, .tid = 2, .sec = 10000, .nsec = 20003, .lid = LOG_ID_KERNEL, .uid = 0}, + "third"}, + {{.pid = 10, .tid = 2, .sec = 10000, .nsec = 20004, .lid = LOG_ID_MAIN, .uid = 0}, + "fourth"}, + {{.pid = 1, .tid = 2, .sec = 10000, .nsec = 20005, .lid = LOG_ID_RADIO, .uid = 0}, + "fifth"}, + {{.pid = 2, .tid = 2, .sec = 10000, .nsec = 20006, .lid = LOG_ID_RADIO, .uid = 0}, + "sixth"}, + {{.pid = 3, .tid = 2, .sec = 10000, .nsec = 20007, .lid = LOG_ID_RADIO, .uid = 0}, + "seventh"}, + {{.pid = 4, .tid = 2, .sec = 10000, .nsec = 20008, .lid = LOG_ID_MAIN, .uid = 0}, + "eighth"}, + {{.pid = 5, .tid = 2, .sec = 10000, .nsec = 20009, .lid = LOG_ID_CRASH, .uid = 0}, + "nineth"}, + {{.pid = 6, .tid = 2, .sec = 10000, .nsec = 20011, .lid = LOG_ID_MAIN, .uid = 0}, + "tenth"}, + }; + FixupMessages(&log_messages); + LogMessages(log_messages); + + std::vector<LogMessage> read_log_messages; + bool released = false; + + { + auto lock = std::unique_lock{reader_list_.reader_threads_lock()}; + std::unique_ptr<LogWriter> test_writer(new TestWriter(&read_log_messages, &released)); + std::unique_ptr<LogReaderThread> log_reader( + new LogReaderThread(log_buffer_.get(), &reader_list_, std::move(test_writer), true, + 0, kLogMaskAll, 0, {}, 1, {})); + reader_list_.reader_threads().emplace_back(std::move(log_reader)); + } + + while (!released) { + usleep(5000); + } + { + auto lock = std::unique_lock{reader_list_.reader_threads_lock()}; + EXPECT_EQ(0U, reader_list_.reader_threads().size()); + } + CompareLogMessages(log_messages, read_log_messages); +} + +// Generate random messages, set the 'sec' parameter explicit though, to be able to track the +// expected order of messages. +LogMessage GenerateRandomLogMessage(uint32_t sec) { + auto rand_uint32 = [](int max) -> uint32_t { return rand() % max; }; + logger_entry entry = { + .hdr_size = sizeof(logger_entry), + .pid = rand() % 5000, + .tid = rand_uint32(5000), + .sec = sec, + .nsec = rand_uint32(NS_PER_SEC), + .lid = rand_uint32(LOG_ID_STATS), + .uid = rand_uint32(100000), + }; + + // See comment in ChattyLogBuffer::Log() for why this is disallowed. + if (entry.nsec % 1000 == 0) { + ++entry.nsec; + } + + if (entry.lid == LOG_ID_EVENTS) { + entry.lid = LOG_ID_KERNEL; + } + + std::string message; + char priority = ANDROID_LOG_INFO + rand() % 2; + message.push_back(priority); + + int tag_length = 2 + rand() % 10; + for (int i = 0; i < tag_length; ++i) { + message.push_back('a' + rand() % 26); + } + message.push_back('\0'); + + int msg_length = 2 + rand() % 1000; + for (int i = 0; i < msg_length; ++i) { + message.push_back('a' + rand() % 26); + } + message.push_back('\0'); + + entry.len = message.size(); + + return {entry, message}; +} + +TEST_P(LogBufferTest, random_messages) { + srand(1); + std::vector<LogMessage> log_messages; + for (size_t i = 0; i < 1000; ++i) { + log_messages.emplace_back(GenerateRandomLogMessage(i)); + } + LogMessages(log_messages); + + std::vector<LogMessage> read_log_messages; + bool released = false; + + { + auto lock = std::unique_lock{reader_list_.reader_threads_lock()}; + std::unique_ptr<LogWriter> test_writer(new TestWriter(&read_log_messages, &released)); + std::unique_ptr<LogReaderThread> log_reader( + new LogReaderThread(log_buffer_.get(), &reader_list_, std::move(test_writer), true, + 0, kLogMaskAll, 0, {}, 1, {})); + reader_list_.reader_threads().emplace_back(std::move(log_reader)); + } + + while (!released) { + usleep(5000); + } + { + auto lock = std::unique_lock{reader_list_.reader_threads_lock()}; + EXPECT_EQ(0U, reader_list_.reader_threads().size()); + } + CompareLogMessages(log_messages, read_log_messages); +} + +TEST_P(LogBufferTest, read_last_sequence) { + std::vector<LogMessage> log_messages = { + {{.pid = 1, .tid = 2, .sec = 10000, .nsec = 20001, .lid = LOG_ID_MAIN, .uid = 0}, + "first"}, + {{.pid = 10, .tid = 2, .sec = 10000, .nsec = 20002, .lid = LOG_ID_MAIN, .uid = 0}, + "second"}, + {{.pid = 100, .tid = 2, .sec = 10000, .nsec = 20003, .lid = LOG_ID_MAIN, .uid = 0}, + "third"}, + }; + FixupMessages(&log_messages); + LogMessages(log_messages); + + std::vector<LogMessage> read_log_messages; + bool released = false; + + { + auto lock = std::unique_lock{reader_list_.reader_threads_lock()}; + std::unique_ptr<LogWriter> test_writer(new TestWriter(&read_log_messages, &released)); + std::unique_ptr<LogReaderThread> log_reader( + new LogReaderThread(log_buffer_.get(), &reader_list_, std::move(test_writer), true, + 0, kLogMaskAll, 0, {}, 3, {})); + reader_list_.reader_threads().emplace_back(std::move(log_reader)); + } + + while (!released) { + usleep(5000); + } + { + auto lock = std::unique_lock{reader_list_.reader_threads_lock()}; + EXPECT_EQ(0U, reader_list_.reader_threads().size()); + } + std::vector<LogMessage> expected_log_messages = {log_messages.back()}; + CompareLogMessages(expected_log_messages, read_log_messages); +} + +TEST_P(LogBufferTest, clear_logs) { + // Log 3 initial logs. + std::vector<LogMessage> log_messages = { + {{.pid = 1, .tid = 2, .sec = 10000, .nsec = 20001, .lid = LOG_ID_MAIN, .uid = 0}, + "first"}, + {{.pid = 10, .tid = 2, .sec = 10000, .nsec = 20002, .lid = LOG_ID_MAIN, .uid = 0}, + "second"}, + {{.pid = 100, .tid = 2, .sec = 10000, .nsec = 20003, .lid = LOG_ID_MAIN, .uid = 0}, + "third"}, + }; + FixupMessages(&log_messages); + LogMessages(log_messages); + + std::vector<LogMessage> read_log_messages; + bool released = false; + + // Connect a blocking reader. + { + auto lock = std::unique_lock{reader_list_.reader_threads_lock()}; + std::unique_ptr<LogWriter> test_writer(new TestWriter(&read_log_messages, &released)); + std::unique_ptr<LogReaderThread> log_reader( + new LogReaderThread(log_buffer_.get(), &reader_list_, std::move(test_writer), false, + 0, kLogMaskAll, 0, {}, 1, {})); + reader_list_.reader_threads().emplace_back(std::move(log_reader)); + } + + // Wait up to 250ms for the reader to read the first 3 logs. + constexpr int kMaxRetryCount = 50; + int count = 0; + for (; count < kMaxRetryCount; ++count) { + usleep(5000); + auto lock = std::unique_lock{reader_list_.reader_threads_lock()}; + if (reader_list_.reader_threads().back()->start() == 4) { + break; + } + } + ASSERT_LT(count, kMaxRetryCount); + + // Clear the log buffer. + log_buffer_->Clear(LOG_ID_MAIN, 0); + + // Log 3 more logs. + std::vector<LogMessage> after_clear_messages = { + {{.pid = 1, .tid = 2, .sec = 10000, .nsec = 20001, .lid = LOG_ID_MAIN, .uid = 0}, + "4th"}, + {{.pid = 10, .tid = 2, .sec = 10000, .nsec = 20002, .lid = LOG_ID_MAIN, .uid = 0}, + "5th"}, + {{.pid = 100, .tid = 2, .sec = 10000, .nsec = 20003, .lid = LOG_ID_MAIN, .uid = 0}, + "6th"}, + }; + FixupMessages(&after_clear_messages); + LogMessages(after_clear_messages); + + // Wait up to 250ms for the reader to read the 3 additional logs. + for (count = 0; count < kMaxRetryCount; ++count) { + usleep(5000); + auto lock = std::unique_lock{reader_list_.reader_threads_lock()}; + if (reader_list_.reader_threads().back()->start() == 7) { + break; + } + } + ASSERT_LT(count, kMaxRetryCount); + + // Release the reader, wait for it to get the signal then check that it has been deleted. + { + auto lock = std::unique_lock{reader_list_.reader_threads_lock()}; + reader_list_.reader_threads().back()->release_Locked(); + } + while (!released) { + usleep(5000); + } + { + auto lock = std::unique_lock{reader_list_.reader_threads_lock()}; + EXPECT_EQ(0U, reader_list_.reader_threads().size()); + } + + // Check that we have read all 6 messages. + std::vector<LogMessage> expected_log_messages = log_messages; + expected_log_messages.insert(expected_log_messages.end(), after_clear_messages.begin(), + after_clear_messages.end()); + CompareLogMessages(expected_log_messages, read_log_messages); + + // Finally, call FlushTo and ensure that only the 3 logs after the clear remain in the buffer. + std::vector<LogMessage> read_log_messages_after_clear; + std::unique_ptr<LogWriter> test_writer(new TestWriter(&read_log_messages_after_clear, nullptr)); + std::unique_ptr<FlushToState> flush_to_state = log_buffer_->CreateFlushToState(1, kLogMaskAll); + EXPECT_TRUE(log_buffer_->FlushTo(test_writer.get(), *flush_to_state, nullptr)); + EXPECT_EQ(7ULL, flush_to_state->start()); + CompareLogMessages(after_clear_messages, read_log_messages_after_clear); +} + +INSTANTIATE_TEST_CASE_P(LogBufferTests, LogBufferTest, + testing::Values("chatty", "serialized", "simple")); diff --git a/logd/LogBufferTest.h b/logd/LogBufferTest.h new file mode 100644 index 000000000..5d57ad1cf --- /dev/null +++ b/logd/LogBufferTest.h @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <string> +#include <vector> + +#include <gtest/gtest.h> + +#include "ChattyLogBuffer.h" +#include "LogReaderList.h" +#include "LogStatistics.h" +#include "LogTags.h" +#include "PruneList.h" +#include "SerializedLogBuffer.h" +#include "SimpleLogBuffer.h" + +struct LogMessage { + logger_entry entry; + std::string message; + bool regex_compare = false; // Only set for expected messages, when true 'message' should be + // interpretted as a regex. +}; + +// Compares the ordered list of expected and result, causing a test failure with appropriate +// information on failure. +void CompareLogMessages(const std::vector<LogMessage>& expected, + const std::vector<LogMessage>& result); +// Sets hdr_size and len parameters appropriately. +void FixupMessages(std::vector<LogMessage>* messages); + +class TestWriter : public LogWriter { + public: + TestWriter(std::vector<LogMessage>* msgs, bool* released) + : LogWriter(0, true), msgs_(msgs), released_(released) {} + bool Write(const logger_entry& entry, const char* message) override { + msgs_->emplace_back(LogMessage{entry, std::string(message, entry.len), false}); + return true; + } + + void Release() { + if (released_) *released_ = true; + } + + std::string name() const override { return "test_writer"; } + + private: + std::vector<LogMessage>* msgs_; + bool* released_; +}; + +class LogBufferTest : public testing::TestWithParam<std::string> { + protected: + void SetUp() override { + if (GetParam() == "chatty") { + log_buffer_.reset(new ChattyLogBuffer(&reader_list_, &tags_, &prune_, &stats_)); + } else if (GetParam() == "serialized") { + log_buffer_.reset(new SerializedLogBuffer(&reader_list_, &tags_, &stats_)); + } else if (GetParam() == "simple") { + log_buffer_.reset(new SimpleLogBuffer(&reader_list_, &tags_, &stats_)); + } else { + FAIL() << "Unknown buffer type selected for test"; + } + } + + void LogMessages(const std::vector<LogMessage>& messages) { + for (auto& [entry, message, _] : messages) { + log_buffer_->Log(static_cast<log_id_t>(entry.lid), log_time(entry.sec, entry.nsec), + entry.uid, entry.pid, entry.tid, message.c_str(), message.size()); + } + } + + LogReaderList reader_list_; + LogTags tags_; + PruneList prune_; + LogStatistics stats_{false}; + std::unique_ptr<LogBuffer> log_buffer_; +}; diff --git a/logd/LogKlog.cpp b/logd/LogKlog.cpp index edd326aec..dbdf7fdd0 100644 --- a/logd/LogKlog.cpp +++ b/logd/LogKlog.cpp @@ -14,6 +14,8 @@ * limitations under the License. */ +#include "LogKlog.h" + #include <ctype.h> #include <errno.h> #include <inttypes.h> @@ -29,8 +31,6 @@ #include <private/android_logger.h> #include "LogBuffer.h" -#include "LogKlog.h" -#include "LogReader.h" #define KMSG_PRIORITY(PRI) \ '<', '0' + (LOG_SYSLOG | (PRI)) / 10, '0' + (LOG_SYSLOG | (PRI)) % 10, '>' @@ -201,15 +201,14 @@ log_time LogKlog::correction = (log_time(CLOCK_REALTIME) < log_time(CLOCK_MONOTO ? log_time(log_time::EPOCH) : (log_time(CLOCK_REALTIME) - log_time(CLOCK_MONOTONIC)); -LogKlog::LogKlog(LogBuffer* buf, LogReader* reader, int fdWrite, int fdRead, - bool auditd) +LogKlog::LogKlog(LogBuffer* buf, int fdWrite, int fdRead, bool auditd, LogStatistics* stats) : SocketListener(fdRead, false), logbuf(buf), - reader(reader), signature(CLOCK_MONOTONIC), initialized(false), enableLogging(true), - auditd(auditd) { + auditd(auditd), + stats_(stats) { static const char klogd_message[] = "%s%s%" PRIu64 "\n"; char buffer[strlen(priority_message) + strlen(klogdStr) + strlen(klogd_message) + 20]; @@ -309,8 +308,6 @@ log_time LogKlog::sniffTime(const char*& buf, ssize_t len, bool reverse) { } buf = cp; - if (isMonotonic()) return now; - const char* b; if (((b = android::strnstr(cp, len, suspendStr))) && (((b += strlen(suspendStr)) - cp) < len)) { @@ -356,11 +353,7 @@ log_time LogKlog::sniffTime(const char*& buf, ssize_t len, bool reverse) { convertMonotonicToReal(now); } else { - if (isMonotonic()) { - now = log_time(CLOCK_MONOTONIC); - } else { - now = log_time(CLOCK_REALTIME); - } + now = log_time(CLOCK_REALTIME); } return now; } @@ -431,45 +424,6 @@ static int parseKernelPrio(const char*& buf, ssize_t len) { return pri; } -// Passed the entire SYSLOG_ACTION_READ_ALL buffer and interpret a -// compensated start time. -void LogKlog::synchronize(const char* buf, ssize_t len) { - const char* cp = android::strnstr(buf, len, suspendStr); - if (!cp) { - cp = android::strnstr(buf, len, resumeStr); - if (!cp) return; - } else { - const char* rp = android::strnstr(buf, len, resumeStr); - if (rp && (rp < cp)) cp = rp; - } - - do { - --cp; - } while ((cp > buf) && (*cp != '\n')); - if (*cp == '\n') { - ++cp; - } - parseKernelPrio(cp, len - (cp - buf)); - - log_time now = sniffTime(cp, len - (cp - buf), true); - - const char* suspended = android::strnstr(buf, len, suspendedStr); - if (!suspended || (suspended > cp)) { - return; - } - cp = suspended; - - do { - --cp; - } while ((cp > buf) && (*cp != '\n')); - if (*cp == '\n') { - ++cp; - } - parseKernelPrio(cp, len - (cp - buf)); - - sniffTime(cp, len - (cp - buf), true); -} - // Convert kernel log priority number into an Android Logger priority number static int convertKernelPrioToAndroidPrio(int pri) { switch (pri & LOG_PRIMASK) { @@ -573,9 +527,7 @@ int LogKlog::log(const char* buf, ssize_t len) { const pid_t tid = pid; uid_t uid = AID_ROOT; if (pid) { - logbuf->wrlock(); - uid = logbuf->pidToUid(pid); - logbuf->unlock(); + uid = stats_->PidToUid(pid); } // Parse (rules at top) to pull out a tag from the incoming kernel message. @@ -713,10 +665,9 @@ int LogKlog::log(const char* buf, ssize_t len) { ((size == 2) && (isdigit(tag[0]) || isdigit(tag[1]))) || // register names like x18 but not driver names like en0 ((size == 3) && (isdigit(tag[1]) && isdigit(tag[2]))) || - // blacklist + // ignore ((size == cpuLen) && !fastcmp<strncmp>(tag, cpu, cpuLen)) || - ((size == warningLen) && - !fastcmp<strncasecmp>(tag, warning, warningLen)) || + ((size == warningLen) && !fastcmp<strncasecmp>(tag, warning, warningLen)) || ((size == errorLen) && !fastcmp<strncasecmp>(tag, error, errorLen)) || ((size == infoLen) && !fastcmp<strncasecmp>(tag, info, infoLen))) { p = start; @@ -789,7 +740,7 @@ int LogKlog::log(const char* buf, ssize_t len) { memcpy(np, p, b); np[b] = '\0'; - if (!isMonotonic()) { + { // Watch out for singular race conditions with timezone causing near // integer quarter-hour jumps in the time and compensate accordingly. // Entries will be temporal within near_seconds * 2. b/21868540 @@ -815,12 +766,7 @@ int LogKlog::log(const char* buf, ssize_t len) { } // Log message - int rc = logbuf->log(LOG_ID_KERNEL, now, uid, pid, tid, newstr, (uint16_t)n); - - // notify readers - if (rc > 0) { - reader->notifyNewLog(static_cast<log_mask_t>(1 << LOG_ID_KERNEL)); - } + int rc = logbuf->Log(LOG_ID_KERNEL, now, uid, pid, tid, newstr, (uint16_t)n); return rc; } diff --git a/logd/LogKlog.h b/logd/LogKlog.h index 6bfd6a841..56e045252 100644 --- a/logd/LogKlog.h +++ b/logd/LogKlog.h @@ -14,18 +14,16 @@ * limitations under the License. */ -#ifndef _LOGD_LOG_KLOG_H__ -#define _LOGD_LOG_KLOG_H__ +#pragma once #include <private/android_logger.h> #include <sysutils/SocketListener.h> -class LogBuffer; -class LogReader; +#include "LogBuffer.h" +#include "LogStatistics.h" class LogKlog : public SocketListener { LogBuffer* logbuf; - LogReader* reader; const log_time signature; // Set once thread is started, separates KLOG_ACTION_READ_ALL // and KLOG_ACTION_READ phases. @@ -38,27 +36,18 @@ class LogKlog : public SocketListener { static log_time correction; - public: - LogKlog(LogBuffer* buf, LogReader* reader, int fdWrite, int fdRead, - bool auditd); + public: + LogKlog(LogBuffer* buf, int fdWrite, int fdRead, bool auditd, LogStatistics* stats); int log(const char* buf, ssize_t len); - void synchronize(const char* buf, ssize_t len); - bool isMonotonic() { - return logbuf->isMonotonic(); - } - static void convertMonotonicToReal(log_time& real) { - real += correction; - } - static void convertRealToMonotonic(log_time& real) { - real -= correction; - } + static void convertMonotonicToReal(log_time& real) { real += correction; } - protected: - log_time sniffTime(const char*& buf, ssize_t len, bool reverse); - pid_t sniffPid(const char*& buf, ssize_t len); - void calculateCorrection(const log_time& monotonic, const char* real_string, ssize_t len); - virtual bool onDataAvailable(SocketClient* cli); -}; + protected: + log_time sniffTime(const char*& buf, ssize_t len, bool reverse); + pid_t sniffPid(const char*& buf, ssize_t len); + void calculateCorrection(const log_time& monotonic, const char* real_string, ssize_t len); + virtual bool onDataAvailable(SocketClient* cli); -#endif + private: + LogStatistics* stats_; +}; diff --git a/logd/LogListener.cpp b/logd/LogListener.cpp index ba610424f..a6ab50b8a 100644 --- a/logd/LogListener.cpp +++ b/logd/LogListener.cpp @@ -22,42 +22,53 @@ #include <sys/un.h> #include <unistd.h> +#include <thread> + #include <cutils/sockets.h> #include <private/android_filesystem_config.h> #include <private/android_logger.h> #include "LogBuffer.h" #include "LogListener.h" -#include "LogUtils.h" +#include "LogPermissions.h" -LogListener::LogListener(LogBuffer* buf, LogReader* reader) - : SocketListener(getLogSocket(), false), logbuf(buf), reader(reader) {} +LogListener::LogListener(LogBuffer* buf) : socket_(GetLogSocket()), logbuf_(buf) {} -bool LogListener::onDataAvailable(SocketClient* cli) { - static bool name_set; - if (!name_set) { - prctl(PR_SET_NAME, "logd.writer"); - name_set = true; +bool LogListener::StartListener() { + if (socket_ <= 0) { + return false; } + auto thread = std::thread(&LogListener::ThreadFunction, this); + thread.detach(); + return true; +} + +void LogListener::ThreadFunction() { + prctl(PR_SET_NAME, "logd.writer"); + while (true) { + HandleData(); + } +} + +void LogListener::HandleData() { // + 1 to ensure null terminator if MAX_PAYLOAD buffer is received - char buffer[sizeof(android_log_header_t) + LOGGER_ENTRY_MAX_PAYLOAD + 1]; - struct iovec iov = { buffer, sizeof(buffer) - 1 }; + __attribute__((uninitialized)) char + buffer[sizeof(android_log_header_t) + LOGGER_ENTRY_MAX_PAYLOAD + 1]; + struct iovec iov = {buffer, sizeof(buffer) - 1}; alignas(4) char control[CMSG_SPACE(sizeof(struct ucred))]; struct msghdr hdr = { nullptr, 0, &iov, 1, control, sizeof(control), 0, }; - int socket = cli->getSocket(); - // To clear the entire buffer is secure/safe, but this contributes to 1.68% // overhead under logging load. We are safe because we check counts, but // still need to clear null terminator // memset(buffer, 0, sizeof(buffer)); - ssize_t n = recvmsg(socket, &hdr, 0); + ssize_t n = recvmsg(socket_, &hdr, 0); if (n <= (ssize_t)(sizeof(android_log_header_t))) { - return false; + return; } buffer[n] = 0; @@ -75,14 +86,14 @@ bool LogListener::onDataAvailable(SocketClient* cli) { } if (cred == nullptr) { - return false; + return; } if (cred->uid == AID_LOGD) { // ignore log messages we send to ourself. // Such log messages are often generated by libraries we depend on // which use standard Android logging. - return false; + return; } android_log_header_t* header = @@ -90,13 +101,13 @@ bool LogListener::onDataAvailable(SocketClient* cli) { log_id_t logId = static_cast<log_id_t>(header->id); if (/* logId < LOG_ID_MIN || */ logId >= LOG_ID_MAX || logId == LOG_ID_KERNEL) { - return false; + return; } if ((logId == LOG_ID_SECURITY) && (!__android_log_security() || !clientHasLogCredentials(cred->uid, cred->gid, cred->pid))) { - return false; + return; } char* msg = ((char*)buffer) + sizeof(android_log_header_t); @@ -105,16 +116,11 @@ bool LogListener::onDataAvailable(SocketClient* cli) { // NB: hdr.msg_flags & MSG_TRUNC is not tested, silently passing a // truncated message to the logs. - int res = logbuf->log(logId, header->realtime, cred->uid, cred->pid, header->tid, msg, - ((size_t)n <= UINT16_MAX) ? (uint16_t)n : UINT16_MAX); - if (res > 0) { - reader->notifyNewLog(static_cast<log_mask_t>(1 << logId)); - } - - return true; + logbuf_->Log(logId, header->realtime, cred->uid, cred->pid, header->tid, msg, + ((size_t)n <= UINT16_MAX) ? (uint16_t)n : UINT16_MAX); } -int LogListener::getLogSocket() { +int LogListener::GetLogSocket() { static const char socketName[] = "logdw"; int sock = android_get_control_socket(socketName); diff --git a/logd/LogListener.h b/logd/LogListener.h index 8fe3da46d..566af5b14 100644 --- a/logd/LogListener.h +++ b/logd/LogListener.h @@ -14,24 +14,20 @@ * limitations under the License. */ -#ifndef _LOGD_LOG_LISTENER_H__ -#define _LOGD_LOG_LISTENER_H__ +#pragma once -#include <sysutils/SocketListener.h> -#include "LogReader.h" +#include "LogBuffer.h" -class LogListener : public SocketListener { - LogBuffer* logbuf; - LogReader* reader; +class LogListener { + public: + explicit LogListener(LogBuffer* buf); + bool StartListener(); - public: - LogListener(LogBuffer* buf, LogReader* reader); + private: + void ThreadFunction(); + void HandleData(); + static int GetLogSocket(); - protected: - virtual bool onDataAvailable(SocketClient* cli); - - private: - static int getLogSocket(); + int socket_; + LogBuffer* logbuf_; }; - -#endif diff --git a/logd/LogCommand.cpp b/logd/LogPermissions.cpp index 8bff9da46..3fd0ea1ae 100644 --- a/logd/LogCommand.cpp +++ b/logd/LogPermissions.cpp @@ -14,6 +14,8 @@ * limitations under the License. */ +#include "LogPermissions.h" + #include <errno.h> #include <stdio.h> #include <stdlib.h> @@ -21,12 +23,6 @@ #include <private/android_filesystem_config.h> -#include "LogCommand.h" -#include "LogUtils.h" - -LogCommand::LogCommand(const char* cmd) : FrameworkCommand(cmd) { -} - // gets a list of supplementary group IDs associated with // the socket peer. This is implemented by opening // /proc/PID/status and look for the "Group:" line. @@ -93,7 +89,7 @@ bool clientHasLogCredentials(uid_t uid, gid_t gid, pid_t pid) { // for (int retry = 3; !(ret = foundGid && foundUid && foundLog) && retry; --retry) { - FILE* file = fopen(filename, "r"); + FILE* file = fopen(filename, "re"); if (!file) { continue; } diff --git a/logd/LogCommand.h b/logd/LogPermissions.h index e10ffa0e6..3130db5f3 100644 --- a/logd/LogCommand.h +++ b/logd/LogPermissions.h @@ -14,17 +14,11 @@ * limitations under the License. */ -#ifndef _LOGD_COMMAND_H -#define _LOGD_COMMAND_H +#pragma once -#include <sysutils/FrameworkCommand.h> -#include <sysutils/SocketClient.h> +#include <sys/types.h> -class LogCommand : public FrameworkCommand { - public: - explicit LogCommand(const char* cmd); - virtual ~LogCommand() { - } -}; +#include <sysutils/SocketClient.h> -#endif +bool clientHasLogCredentials(uid_t uid, gid_t gid, pid_t pid); +bool clientHasLogCredentials(SocketClient* cli); diff --git a/logd/LogReader.cpp b/logd/LogReader.cpp index 9db8c0047..6c976931f 100644 --- a/logd/LogReader.cpp +++ b/logd/LogReader.cpp @@ -21,26 +21,61 @@ #include <sys/socket.h> #include <sys/types.h> +#include <chrono> + +#include <android-base/logging.h> +#include <android-base/stringprintf.h> #include <cutils/sockets.h> +#include <private/android_filesystem_config.h> #include <private/android_logger.h> -#include "FlushCommand.h" #include "LogBuffer.h" #include "LogBufferElement.h" +#include "LogPermissions.h" #include "LogReader.h" #include "LogUtils.h" +#include "LogWriter.h" -LogReader::LogReader(LogBuffer* logbuf) - : SocketListener(getLogSocket(), true), mLogbuf(*logbuf) { +static bool CanReadSecurityLogs(SocketClient* client) { + return client->getUid() == AID_SYSTEM || client->getGid() == AID_SYSTEM; } -// When we are notified a new log entry is available, inform -// listening sockets who are watching this entry's log id. -void LogReader::notifyNewLog(log_mask_t logMask) { - FlushCommand command(*this, logMask); - runOnEachSocket(&command); +static std::string SocketClientToName(SocketClient* client) { + return android::base::StringPrintf("pid %d, fd %d", client->getPid(), client->getSocket()); } +class SocketLogWriter : public LogWriter { + public: + SocketLogWriter(LogReader* reader, SocketClient* client, bool privileged) + : LogWriter(client->getUid(), privileged), reader_(reader), client_(client) {} + + bool Write(const logger_entry& entry, const char* msg) override { + struct iovec iovec[2]; + iovec[0].iov_base = const_cast<logger_entry*>(&entry); + iovec[0].iov_len = entry.hdr_size; + iovec[1].iov_base = const_cast<char*>(msg); + iovec[1].iov_len = entry.len; + + return client_->sendDatav(iovec, 1 + (entry.len != 0)) == 0; + } + + void Release() override { + reader_->release(client_); + client_->decRef(); + } + + void Shutdown() override { shutdown(client_->getSocket(), SHUT_RDWR); } + + std::string name() const override { return SocketClientToName(client_); } + + private: + LogReader* reader_; + SocketClient* client_; +}; + +LogReader::LogReader(LogBuffer* logbuf, LogReaderList* reader_list) + : SocketListener(getLogSocket(), true), log_buffer_(logbuf), reader_list_(reader_list) {} + // Note returning false will release the SocketClient instance. bool LogReader::onDataAvailable(SocketClient* cli) { static bool name_set; @@ -53,22 +88,15 @@ bool LogReader::onDataAvailable(SocketClient* cli) { int len = read(cli->getSocket(), buffer, sizeof(buffer) - 1); if (len <= 0) { - doSocketDelete(cli); + DoSocketDelete(cli); return false; } buffer[len] = '\0'; - // Clients are only allowed to send one command, disconnect them if they - // send another. - LogTimeEntry::wrlock(); - for (const auto& entry : mLogbuf.mTimes) { - if (entry->mClient == cli) { - entry->release_Locked(); - LogTimeEntry::unlock(); - return false; - } + // Clients are only allowed to send one command, disconnect them if they send another. + if (DoSocketDelete(cli)) { + return false; } - LogTimeEntry::unlock(); unsigned long tail = 0; static const char _tail[] = " tail="; @@ -85,12 +113,12 @@ bool LogReader::onDataAvailable(SocketClient* cli) { start.strptime(cp + sizeof(_start) - 1, "%s.%q"); } - uint64_t timeout = 0; + std::chrono::steady_clock::time_point deadline = {}; static const char _timeout[] = " timeout="; cp = strstr(buffer, _timeout); if (cp) { - timeout = atol(cp + sizeof(_timeout) - 1) * NS_PER_SEC + - log_time(CLOCK_REALTIME).nsec(); + long timeout_seconds = atol(cp + sizeof(_timeout) - 1); + deadline = std::chrono::steady_clock::now() + std::chrono::seconds(timeout_seconds); } unsigned int logMask = -1; @@ -99,7 +127,7 @@ bool LogReader::onDataAvailable(SocketClient* cli) { if (cp) { logMask = 0; cp += sizeof(_logIds) - 1; - while (*cp && *cp != '\0') { + while (*cp != '\0') { int val = 0; while (isdigit(*cp)) { val = val * 10 + *cp - '0'; @@ -124,133 +152,92 @@ bool LogReader::onDataAvailable(SocketClient* cli) { if (!fastcmp<strncmp>(buffer, "dumpAndClose", 12)) { // Allow writer to get some cycles, and wait for pending notifications sched_yield(); - LogTimeEntry::wrlock(); - LogTimeEntry::unlock(); + reader_list_->reader_threads_lock().lock(); + reader_list_->reader_threads_lock().unlock(); sched_yield(); nonBlock = true; } - log_time sequence = start; - // - // This somewhat expensive data validation operation is required - // for non-blocking, with timeout. The incoming timestamp must be - // in range of the list, if not, return immediately. This is - // used to prevent us from from getting stuck in timeout processing - // with an invalid time. - // - // Find if time is really present in the logs, monotonic or real, implicit - // conversion from monotonic or real as necessary to perform the check. - // Exit in the check loop ASAP as you find a transition from older to - // newer, but use the last entry found to ensure overlap. - // - if (nonBlock && (sequence != log_time::EPOCH) && timeout) { - class LogFindStart { // A lambda by another name - private: - const pid_t mPid; - const unsigned mLogMask; - bool mStartTimeSet; - log_time mStart; - log_time& mSequence; - log_time mLast; - bool mIsMonotonic; - - public: - LogFindStart(pid_t pid, unsigned logMask, log_time& sequence, - bool isMonotonic) - : mPid(pid), - mLogMask(logMask), - mStartTimeSet(false), - mStart(sequence), - mSequence(sequence), - mLast(sequence), - mIsMonotonic(isMonotonic) { - } + bool privileged = clientHasLogCredentials(cli); + bool can_read_security = CanReadSecurityLogs(cli); + if (!can_read_security) { + logMask &= ~(1 << LOG_ID_SECURITY); + } - static int callback(const LogBufferElement* element, void* obj) { - LogFindStart* me = reinterpret_cast<LogFindStart*>(obj); - if ((!me->mPid || (me->mPid == element->getPid())) && - (me->mLogMask & (1 << element->getLogId()))) { - log_time real = element->getRealTime(); - if (me->mStart == real) { - me->mSequence = real; - me->mStartTimeSet = true; - return -1; - } else if (!me->mIsMonotonic || android::isMonotonic(real)) { - if (me->mStart < real) { - me->mSequence = me->mLast; - me->mStartTimeSet = true; - return -1; - } - me->mLast = real; - } else { - me->mLast = real; - } + std::unique_ptr<LogWriter> socket_log_writer(new SocketLogWriter(this, cli, privileged)); + + uint64_t sequence = 1; + // Convert realtime to sequence number + if (start != log_time::EPOCH) { + bool start_time_set = false; + uint64_t last = sequence; + auto log_find_start = [pid, start, &sequence, &start_time_set, &last]( + log_id_t, pid_t element_pid, uint64_t element_sequence, + log_time element_realtime) -> FilterResult { + if (pid && pid != element_pid) { + return FilterResult::kSkip; + } + if (start == element_realtime) { + sequence = element_sequence; + start_time_set = true; + return FilterResult::kStop; + } else { + if (start < element_realtime) { + sequence = last; + start_time_set = true; + return FilterResult::kStop; } - return false; + last = element_sequence; } + return FilterResult::kSkip; + }; + auto flush_to_state = log_buffer_->CreateFlushToState(sequence, logMask); + log_buffer_->FlushTo(socket_log_writer.get(), *flush_to_state, log_find_start); - bool found() { - return mStartTimeSet; + if (!start_time_set) { + if (nonBlock) { + return false; } - - } logFindStart(pid, logMask, sequence, - logbuf().isMonotonic() && android::isMonotonic(start)); - - logbuf().flushTo(cli, sequence, nullptr, FlushCommand::hasReadLogs(cli), - FlushCommand::hasSecurityLogs(cli), - logFindStart.callback, &logFindStart); - - if (!logFindStart.found()) { - doSocketDelete(cli); - return false; + sequence = log_buffer_->sequence(); } } - android::prdebug( - "logdr: UID=%d GID=%d PID=%d %c tail=%lu logMask=%x pid=%d " - "start=%" PRIu64 "ns timeout=%" PRIu64 "ns\n", - cli->getUid(), cli->getGid(), cli->getPid(), nonBlock ? 'n' : 'b', tail, - logMask, (int)pid, sequence.nsec(), timeout); - - if (sequence == log_time::EPOCH) { - timeout = 0; - } + LOG(INFO) << android::base::StringPrintf( + "logdr: UID=%d GID=%d PID=%d %c tail=%lu logMask=%x pid=%d " + "start=%" PRIu64 "ns deadline=%" PRIi64 "ns", + cli->getUid(), cli->getGid(), cli->getPid(), nonBlock ? 'n' : 'b', tail, logMask, + (int)pid, start.nsec(), static_cast<int64_t>(deadline.time_since_epoch().count())); - LogTimeEntry::wrlock(); - auto entry = std::make_unique<LogTimeEntry>( - *this, cli, nonBlock, tail, logMask, pid, sequence, timeout); - if (!entry->startReader_Locked()) { - LogTimeEntry::unlock(); - return false; + if (start == log_time::EPOCH) { + deadline = {}; } + auto lock = std::lock_guard{reader_list_->reader_threads_lock()}; + auto entry = std::make_unique<LogReaderThread>(log_buffer_, reader_list_, + std::move(socket_log_writer), nonBlock, tail, + logMask, pid, start, sequence, deadline); // release client and entry reference counts once done cli->incRef(); - mLogbuf.mTimes.emplace_front(std::move(entry)); + reader_list_->reader_threads().emplace_front(std::move(entry)); // Set acceptable upper limit to wait for slow reader processing b/27242723 struct timeval t = { LOGD_SNDTIMEO, 0 }; setsockopt(cli->getSocket(), SOL_SOCKET, SO_SNDTIMEO, (const char*)&t, sizeof(t)); - LogTimeEntry::unlock(); - return true; } -void LogReader::doSocketDelete(SocketClient* cli) { - LastLogTimes& times = mLogbuf.mTimes; - LogTimeEntry::wrlock(); - LastLogTimes::iterator it = times.begin(); - while (it != times.end()) { - LogTimeEntry* entry = it->get(); - if (entry->mClient == cli) { - entry->release_Locked(); - break; +bool LogReader::DoSocketDelete(SocketClient* cli) { + auto cli_name = SocketClientToName(cli); + auto lock = std::lock_guard{reader_list_->reader_threads_lock()}; + for (const auto& reader : reader_list_->reader_threads()) { + if (reader->name() == cli_name) { + reader->release_Locked(); + return true; } - it++; } - LogTimeEntry::unlock(); + return false; } int LogReader::getLogSocket() { diff --git a/logd/LogReader.h b/logd/LogReader.h index b5312b60d..b85a58478 100644 --- a/logd/LogReader.h +++ b/logd/LogReader.h @@ -14,35 +14,28 @@ * limitations under the License. */ -#ifndef _LOGD_LOG_WRITER_H__ -#define _LOGD_LOG_WRITER_H__ +#pragma once #include <sysutils/SocketListener.h> -#include "LogTimes.h" +#include "LogBuffer.h" +#include "LogReaderList.h" +#include "LogReaderThread.h" #define LOGD_SNDTIMEO 32 -class LogBuffer; - class LogReader : public SocketListener { - LogBuffer& mLogbuf; - - public: - explicit LogReader(LogBuffer* logbuf); - void notifyNewLog(log_mask_t logMask); - - LogBuffer& logbuf(void) const { - return mLogbuf; - } + public: + explicit LogReader(LogBuffer* logbuf, LogReaderList* reader_list); - protected: + protected: virtual bool onDataAvailable(SocketClient* cli); - private: + private: static int getLogSocket(); - void doSocketDelete(SocketClient* cli); -}; + bool DoSocketDelete(SocketClient* cli); -#endif + LogBuffer* log_buffer_; + LogReaderList* reader_list_; +}; diff --git a/logd/LogReaderList.cpp b/logd/LogReaderList.cpp new file mode 100644 index 000000000..32ba2910d --- /dev/null +++ b/logd/LogReaderList.cpp @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "LogReaderList.h" + +// When we are notified a new log entry is available, inform +// listening sockets who are watching this entry's log id. +void LogReaderList::NotifyNewLog(LogMask log_mask) const { + auto lock = std::lock_guard{reader_threads_lock_}; + + for (const auto& entry : reader_threads_) { + if (!entry->IsWatchingMultiple(log_mask)) { + continue; + } + if (entry->deadline().time_since_epoch().count() != 0) { + continue; + } + entry->triggerReader_Locked(); + } +} diff --git a/base/include/android-base/off64_t.h b/logd/LogReaderList.h index e6b71b81e..594716a5b 100644 --- a/base/include/android-base/off64_t.h +++ b/logd/LogReaderList.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018 The Android Open Source Project + * Copyright (C) 2020 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,21 @@ #pragma once -#if defined(__APPLE__) -/** Mac OS has always had a 64-bit off_t, so it doesn't have off64_t. */ -typedef off_t off64_t; -#endif +#include <list> +#include <memory> +#include <mutex> + +#include "LogBuffer.h" +#include "LogReaderThread.h" + +class LogReaderList { + public: + void NotifyNewLog(LogMask log_mask) const; + + std::list<std::unique_ptr<LogReaderThread>>& reader_threads() { return reader_threads_; } + std::mutex& reader_threads_lock() { return reader_threads_lock_; } + + private: + std::list<std::unique_ptr<LogReaderThread>> reader_threads_; + mutable std::mutex reader_threads_lock_; +};
\ No newline at end of file diff --git a/logd/LogReaderThread.cpp b/logd/LogReaderThread.cpp new file mode 100644 index 000000000..4a8be01fa --- /dev/null +++ b/logd/LogReaderThread.cpp @@ -0,0 +1,180 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "LogReaderThread.h" + +#include <errno.h> +#include <string.h> +#include <sys/prctl.h> + +#include <thread> + +#include "LogBuffer.h" +#include "LogReaderList.h" + +LogReaderThread::LogReaderThread(LogBuffer* log_buffer, LogReaderList* reader_list, + std::unique_ptr<LogWriter> writer, bool non_block, + unsigned long tail, LogMask log_mask, pid_t pid, + log_time start_time, uint64_t start, + std::chrono::steady_clock::time_point deadline) + : log_buffer_(log_buffer), + reader_list_(reader_list), + writer_(std::move(writer)), + pid_(pid), + tail_(tail), + count_(0), + index_(0), + start_time_(start_time), + deadline_(deadline), + non_block_(non_block) { + cleanSkip_Locked(); + flush_to_state_ = log_buffer_->CreateFlushToState(start, log_mask); + auto thread = std::thread{&LogReaderThread::ThreadFunction, this}; + thread.detach(); +} + +void LogReaderThread::ThreadFunction() { + prctl(PR_SET_NAME, "logd.reader.per"); + + auto lock = std::unique_lock{reader_list_->reader_threads_lock()}; + + while (!release_) { + if (deadline_.time_since_epoch().count() != 0) { + if (thread_triggered_condition_.wait_until(lock, deadline_) == + std::cv_status::timeout) { + deadline_ = {}; + } + if (release_) { + break; + } + } + + lock.unlock(); + + if (tail_) { + auto first_pass_state = log_buffer_->CreateFlushToState(flush_to_state_->start(), + flush_to_state_->log_mask()); + log_buffer_->FlushTo( + writer_.get(), *first_pass_state, + [this](log_id_t log_id, pid_t pid, uint64_t sequence, log_time realtime) { + return FilterFirstPass(log_id, pid, sequence, realtime); + }); + } + bool flush_success = log_buffer_->FlushTo( + writer_.get(), *flush_to_state_, + [this](log_id_t log_id, pid_t pid, uint64_t sequence, log_time realtime) { + return FilterSecondPass(log_id, pid, sequence, realtime); + }); + + // We only ignore entries before the original start time for the first flushTo(), if we + // get entries after this first flush before the original start time, then the client + // wouldn't have seen them. + // Note: this is still racy and may skip out of order events that came in since the last + // time the client disconnected and then reconnected with the new start time. The long term + // solution here is that clients must request events since a specific sequence number. + start_time_.tv_sec = 0; + start_time_.tv_nsec = 0; + + lock.lock(); + + if (!flush_success) { + break; + } + + if (non_block_ || release_) { + break; + } + + cleanSkip_Locked(); + + if (deadline_.time_since_epoch().count() == 0) { + thread_triggered_condition_.wait(lock); + } + } + + writer_->Release(); + + auto& log_reader_threads = reader_list_->reader_threads(); + auto it = std::find_if(log_reader_threads.begin(), log_reader_threads.end(), + [this](const auto& other) { return other.get() == this; }); + + if (it != log_reader_threads.end()) { + log_reader_threads.erase(it); + } +} + +// A first pass to count the number of elements +FilterResult LogReaderThread::FilterFirstPass(log_id_t, pid_t pid, uint64_t, log_time realtime) { + auto lock = std::lock_guard{reader_list_->reader_threads_lock()}; + + if ((!pid_ || pid_ == pid) && (start_time_ == log_time::EPOCH || start_time_ <= realtime)) { + ++count_; + } + + return FilterResult::kSkip; +} + +// A second pass to send the selected elements +FilterResult LogReaderThread::FilterSecondPass(log_id_t log_id, pid_t pid, uint64_t, + log_time realtime) { + auto lock = std::lock_guard{reader_list_->reader_threads_lock()}; + + if (skip_ahead_[log_id]) { + skip_ahead_[log_id]--; + return FilterResult::kSkip; + } + + // Truncate to close race between first and second pass + if (non_block_ && tail_ && index_ >= count_) { + return FilterResult::kStop; + } + + if (pid_ && pid_ != pid) { + return FilterResult::kSkip; + } + + if (start_time_ != log_time::EPOCH && realtime <= start_time_) { + return FilterResult::kSkip; + } + + if (release_) { + return FilterResult::kStop; + } + + if (!tail_) { + goto ok; + } + + ++index_; + + if (count_ > tail_ && index_ <= (count_ - tail_)) { + return FilterResult::kSkip; + } + + if (!non_block_) { + tail_ = 0; + } + +ok: + if (!skip_ahead_[log_id]) { + return FilterResult::kWrite; + } + return FilterResult::kSkip; +} + +void LogReaderThread::cleanSkip_Locked(void) { + memset(skip_ahead_, 0, sizeof(skip_ahead_)); +} diff --git a/logd/LogReaderThread.h b/logd/LogReaderThread.h new file mode 100644 index 000000000..1855c0e26 --- /dev/null +++ b/logd/LogReaderThread.h @@ -0,0 +1,107 @@ +/* + * Copyright (C) 2012-2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <pthread.h> +#include <sys/socket.h> +#include <sys/types.h> +#include <time.h> + +#include <chrono> +#include <condition_variable> +#include <list> +#include <memory> + +#include <log/log.h> +#include <sysutils/SocketClient.h> + +#include "LogBuffer.h" +#include "LogWriter.h" + +class LogReaderList; + +class LogReaderThread { + public: + LogReaderThread(LogBuffer* log_buffer, LogReaderList* reader_list, + std::unique_ptr<LogWriter> writer, bool non_block, unsigned long tail, + LogMask log_mask, pid_t pid, log_time start_time, uint64_t sequence, + std::chrono::steady_clock::time_point deadline); + void triggerReader_Locked() { thread_triggered_condition_.notify_all(); } + + void triggerSkip_Locked(log_id_t id, unsigned int skip) { skip_ahead_[id] = skip; } + void cleanSkip_Locked(); + + void release_Locked() { + // gracefully shut down the socket. + writer_->Shutdown(); + release_ = true; + thread_triggered_condition_.notify_all(); + } + + bool IsWatching(log_id_t id) const { return flush_to_state_->log_mask() & (1 << id); } + bool IsWatchingMultiple(LogMask log_mask) const { + return flush_to_state_->log_mask() & log_mask; + } + + std::string name() const { return writer_->name(); } + uint64_t start() const { return flush_to_state_->start(); } + std::chrono::steady_clock::time_point deadline() const { return deadline_; } + FlushToState& flush_to_state() { return *flush_to_state_; } + + private: + void ThreadFunction(); + // flushTo filter callbacks + FilterResult FilterFirstPass(log_id_t log_id, pid_t pid, uint64_t sequence, log_time realtime); + FilterResult FilterSecondPass(log_id_t log_id, pid_t pid, uint64_t sequence, log_time realtime); + + std::condition_variable thread_triggered_condition_; + LogBuffer* log_buffer_; + LogReaderList* reader_list_; + std::unique_ptr<LogWriter> writer_; + + // Set to true to cause the thread to end and the LogReaderThread to delete itself. + bool release_ = false; + + // If set to non-zero, only pids equal to this are read by the reader. + const pid_t pid_; + // When a reader is referencing (via start_) old elements in the log buffer, and the log + // buffer's size grows past its memory limit, the log buffer may request the reader to skip + // ahead a specified number of logs. + unsigned int skip_ahead_[LOG_ID_MAX]; + // LogBuffer::FlushTo() needs to store state across subsequent calls. + std::unique_ptr<FlushToState> flush_to_state_; + + // These next three variables are used for reading only the most recent lines aka `adb logcat + // -t` / `adb logcat -T`. + // tail_ is the number of most recent lines to print. + unsigned long tail_; + // count_ is the result of a first pass through the log buffer to determine how many total + // messages there are. + unsigned long count_; + // index_ is used along with count_ to only start sending lines once index_ > (count_ - tail_) + // and to disconnect the reader (if it is dumpAndClose, `adb logcat -t`), when index_ >= count_. + unsigned long index_; + + // When a reader requests logs starting from a given timestamp, its stored here for the first + // pass, such that logs before this time stamp that are accumulated in the buffer are ignored. + log_time start_time_; + // CLOCK_MONOTONIC based deadline used for log wrapping. If this deadline expires before logs + // wrap, then wake up and send the logs to the reader anyway. + std::chrono::steady_clock::time_point deadline_; + // If this reader is 'dumpAndClose' and will disconnect once it has read its intended logs. + const bool non_block_; +}; diff --git a/logd/LogStatistics.cpp b/logd/LogStatistics.cpp index 431b7784b..3cd8fde5b 100644 --- a/logd/LogStatistics.cpp +++ b/logd/LogStatistics.cpp @@ -14,6 +14,8 @@ * limitations under the License. */ +#include "LogStatistics.h" + #include <ctype.h> #include <fcntl.h> #include <inttypes.h> @@ -27,14 +29,38 @@ #include <private/android_logger.h> -#include "LogStatistics.h" +#include "LogBufferElement.h" static const uint64_t hourSec = 60 * 60; static const uint64_t monthSec = 31 * 24 * hourSec; -size_t LogStatistics::SizesTotal; +std::atomic<size_t> LogStatistics::SizesTotal; -LogStatistics::LogStatistics() : enable(false) { +static std::string TagNameKey(const LogStatisticsElement& element) { + if (IsBinary(element.log_id)) { + uint32_t tag = element.tag; + if (tag) { + const char* cp = android::tagToName(tag); + if (cp) { + return std::string(cp); + } + } + return android::base::StringPrintf("[%" PRIu32 "]", tag); + } + const char* msg = element.msg; + if (!msg) { + return "chatty"; + } + ++msg; + uint16_t len = element.msg_len; + len = (len <= 1) ? 0 : strnlen(msg, len - 1); + if (!len) { + return "<NULL>"; + } + return std::string(msg, len); +} + +LogStatistics::LogStatistics(bool enable_statistics) : enable(enable_statistics) { log_time now(CLOCK_REALTIME); log_id_for_each(id) { mSizes[id] = 0; @@ -62,7 +88,7 @@ char* pidToName(pid_t pid) { } else { char buffer[512]; snprintf(buffer, sizeof(buffer), "/proc/%u/cmdline", pid); - int fd = open(buffer, O_RDONLY); + int fd = open(buffer, O_RDONLY | O_CLOEXEC); if (fd >= 0) { ssize_t ret = read(fd, buffer, sizeof(buffer)); if (ret > 0) { @@ -79,19 +105,18 @@ char* pidToName(pid_t pid) { } } -void LogStatistics::addTotal(LogBufferElement* element) { - if (element->getDropped()) return; +void LogStatistics::AddTotal(log_id_t log_id, uint16_t size) { + auto lock = std::lock_guard{lock_}; - log_id_t log_id = element->getLogId(); - uint16_t size = element->getMsgLen(); mSizesTotal[log_id] += size; SizesTotal += size; ++mElementsTotal[log_id]; } -void LogStatistics::add(LogBufferElement* element) { - log_id_t log_id = element->getLogId(); - uint16_t size = element->getMsgLen(); +void LogStatistics::Add(const LogStatisticsElement& element) { + auto lock = std::lock_guard{lock_}; + log_id_t log_id = element.log_id; + uint16_t size = element.msg_len; mSizes[log_id] += size; ++mElements[log_id]; @@ -100,7 +125,7 @@ void LogStatistics::add(LogBufferElement* element) { // evaluated and trimmed, thus recording size and number of // elements, but we must recognize the manufactured dropped // entry as not contributing to the lifetime totals. - if (element->getDropped()) { + if (element.dropped_count) { ++mDroppedElements[log_id]; } else { mSizesTotal[log_id] += size; @@ -108,7 +133,7 @@ void LogStatistics::add(LogBufferElement* element) { ++mElementsTotal[log_id]; } - log_time stamp(element->getRealTime()); + log_time stamp(element.realtime); if (mNewest[log_id] < stamp) { // A major time update invalidates the statistics :-( log_time diff = stamp - mNewest[log_id]; @@ -133,114 +158,120 @@ void LogStatistics::add(LogBufferElement* element) { return; } - uidTable[log_id].add(element->getUid(), element); - if (element->getUid() == AID_SYSTEM) { - pidSystemTable[log_id].add(element->getPid(), element); + uidTable[log_id].Add(element.uid, element); + if (element.uid == AID_SYSTEM) { + pidSystemTable[log_id].Add(element.pid, element); } if (!enable) { return; } - pidTable.add(element->getPid(), element); - tidTable.add(element->getTid(), element); + pidTable.Add(element.pid, element); + tidTable.Add(element.tid, element); - uint32_t tag = element->getTag(); + uint32_t tag = element.tag; if (tag) { if (log_id == LOG_ID_SECURITY) { - securityTagTable.add(tag, element); + securityTagTable.Add(tag, element); } else { - tagTable.add(tag, element); + tagTable.Add(tag, element); } } - if (!element->getDropped()) { - tagNameTable.add(TagNameKey(element), element); + if (!element.dropped_count) { + tagNameTable.Add(TagNameKey(element), element); } } -void LogStatistics::subtract(LogBufferElement* element) { - log_id_t log_id = element->getLogId(); - uint16_t size = element->getMsgLen(); +void LogStatistics::Subtract(const LogStatisticsElement& element) { + auto lock = std::lock_guard{lock_}; + log_id_t log_id = element.log_id; + uint16_t size = element.msg_len; mSizes[log_id] -= size; --mElements[log_id]; - if (element->getDropped()) { + if (element.dropped_count) { --mDroppedElements[log_id]; } - if (mOldest[log_id] < element->getRealTime()) { - mOldest[log_id] = element->getRealTime(); + if (mOldest[log_id] < element.realtime) { + mOldest[log_id] = element.realtime; } if (log_id == LOG_ID_KERNEL) { return; } - uidTable[log_id].subtract(element->getUid(), element); - if (element->getUid() == AID_SYSTEM) { - pidSystemTable[log_id].subtract(element->getPid(), element); + uidTable[log_id].Subtract(element.uid, element); + if (element.uid == AID_SYSTEM) { + pidSystemTable[log_id].Subtract(element.pid, element); } if (!enable) { return; } - pidTable.subtract(element->getPid(), element); - tidTable.subtract(element->getTid(), element); + pidTable.Subtract(element.pid, element); + tidTable.Subtract(element.tid, element); - uint32_t tag = element->getTag(); + uint32_t tag = element.tag; if (tag) { if (log_id == LOG_ID_SECURITY) { - securityTagTable.subtract(tag, element); + securityTagTable.Subtract(tag, element); } else { - tagTable.subtract(tag, element); + tagTable.Subtract(tag, element); } } - if (!element->getDropped()) { - tagNameTable.subtract(TagNameKey(element), element); + if (!element.dropped_count) { + tagNameTable.Subtract(TagNameKey(element), element); } } // Atomically set an entry to drop // entry->setDropped(1) must follow this call, caller should do this explicitly. -void LogStatistics::drop(LogBufferElement* element) { - log_id_t log_id = element->getLogId(); - uint16_t size = element->getMsgLen(); +void LogStatistics::Drop(const LogStatisticsElement& element) { + auto lock = std::lock_guard{lock_}; + log_id_t log_id = element.log_id; + uint16_t size = element.msg_len; mSizes[log_id] -= size; ++mDroppedElements[log_id]; - if (mNewestDropped[log_id] < element->getRealTime()) { - mNewestDropped[log_id] = element->getRealTime(); + if (mNewestDropped[log_id] < element.realtime) { + mNewestDropped[log_id] = element.realtime; } - uidTable[log_id].drop(element->getUid(), element); - if (element->getUid() == AID_SYSTEM) { - pidSystemTable[log_id].drop(element->getPid(), element); + uidTable[log_id].Drop(element.uid, element); + if (element.uid == AID_SYSTEM) { + pidSystemTable[log_id].Drop(element.pid, element); } if (!enable) { return; } - pidTable.drop(element->getPid(), element); - tidTable.drop(element->getTid(), element); + pidTable.Drop(element.pid, element); + tidTable.Drop(element.tid, element); - uint32_t tag = element->getTag(); + uint32_t tag = element.tag; if (tag) { if (log_id == LOG_ID_SECURITY) { - securityTagTable.drop(tag, element); + securityTagTable.Drop(tag, element); } else { - tagTable.drop(tag, element); + tagTable.Drop(tag, element); } } - tagNameTable.subtract(TagNameKey(element), element); + tagNameTable.Subtract(TagNameKey(element), element); +} + +const char* LogStatistics::UidToName(uid_t uid) const { + auto lock = std::lock_guard{lock_}; + return UidToNameLocked(uid); } // caller must own and free character string -// Requires parent LogBuffer::wrlock() to be held -const char* LogStatistics::uidToName(uid_t uid) const { +const char* LogStatistics::UidToNameLocked(uid_t uid) const { // Local hard coded favourites if (uid == AID_LOGD) { return strdup("auditd"); @@ -278,8 +309,8 @@ const char* LogStatistics::uidToName(uid_t uid) const { ++it) { const PidEntry& entry = it->second; - if (entry.getUid() == uid) { - const char* nameTmp = entry.getName(); + if (entry.uid() == uid) { + const char* nameTmp = entry.name(); if (nameTmp) { if (!name) { @@ -297,6 +328,82 @@ const char* LogStatistics::uidToName(uid_t uid) const { return name; } +template <typename TKey, typename TEntry> +void LogStatistics::WorstTwoWithThreshold(const LogHashtable<TKey, TEntry>& table, size_t threshold, + int* worst, size_t* worst_sizes, + size_t* second_worst_sizes) const { + std::array<const TKey*, 2> max_keys; + std::array<const TEntry*, 2> max_entries; + table.MaxEntries(AID_ROOT, 0, max_keys, max_entries); + if (max_entries[0] == nullptr || max_entries[1] == nullptr) { + return; + } + *worst_sizes = max_entries[0]->getSizes(); + // b/24782000: Allow time horizon to extend roughly tenfold, assume average entry length is + // 100 characters. + if (*worst_sizes > threshold && *worst_sizes > (10 * max_entries[0]->dropped_count())) { + *worst = *max_keys[0]; + *second_worst_sizes = max_entries[1]->getSizes(); + if (*second_worst_sizes < threshold) { + *second_worst_sizes = threshold; + } + } +} + +void LogStatistics::WorstTwoUids(log_id id, size_t threshold, int* worst, size_t* worst_sizes, + size_t* second_worst_sizes) const { + auto lock = std::lock_guard{lock_}; + WorstTwoWithThreshold(uidTable[id], threshold, worst, worst_sizes, second_worst_sizes); +} + +void LogStatistics::WorstTwoTags(size_t threshold, int* worst, size_t* worst_sizes, + size_t* second_worst_sizes) const { + auto lock = std::lock_guard{lock_}; + WorstTwoWithThreshold(tagTable, threshold, worst, worst_sizes, second_worst_sizes); +} + +void LogStatistics::WorstTwoSystemPids(log_id id, size_t worst_uid_sizes, int* worst, + size_t* second_worst_sizes) const { + auto lock = std::lock_guard{lock_}; + std::array<const pid_t*, 2> max_keys; + std::array<const PidEntry*, 2> max_entries; + pidSystemTable[id].MaxEntries(AID_SYSTEM, 0, max_keys, max_entries); + if (max_entries[0] == nullptr || max_entries[1] == nullptr) { + return; + } + + *worst = *max_keys[0]; + *second_worst_sizes = worst_uid_sizes - max_entries[0]->getSizes() + max_entries[1]->getSizes(); +} + +// Prune at most 10% of the log entries or maxPrune, whichever is less. +bool LogStatistics::ShouldPrune(log_id id, unsigned long max_size, + unsigned long* prune_rows) const { + static constexpr size_t kMinPrune = 4; + static constexpr size_t kMaxPrune = 256; + + auto lock = std::lock_guard{lock_}; + size_t sizes = mSizes[id]; + if (sizes <= max_size) { + return false; + } + size_t size_over = sizes - ((max_size * 9) / 10); + size_t elements = mElements[id] - mDroppedElements[id]; + size_t min_elements = elements / 100; + if (min_elements < kMinPrune) { + min_elements = kMinPrune; + } + *prune_rows = elements * size_over / sizes; + if (*prune_rows < min_elements) { + *prune_rows = min_elements; + } + if (*prune_rows > kMaxPrune) { + *prune_rows = kMaxPrune; + } + + return true; +} + std::string UidEntry::formatHeader(const std::string& name, log_id_t id) const { bool isprune = worstUidEnabledForLogid(id); return formatLine(android::base::StringPrintf(name.c_str(), @@ -308,15 +415,14 @@ std::string UidEntry::formatHeader(const std::string& name, log_id_t id) const { } // Helper to truncate name, if too long, and add name dressings -static void formatTmp(const LogStatistics& stat, const char* nameTmp, uid_t uid, - std::string& name, std::string& size, size_t nameLen) { +void LogStatistics::FormatTmp(const char* nameTmp, uid_t uid, std::string& name, std::string& size, + size_t nameLen) const { const char* allocNameTmp = nullptr; - if (!nameTmp) nameTmp = allocNameTmp = stat.uidToName(uid); + if (!nameTmp) nameTmp = allocNameTmp = UidToNameLocked(uid); if (nameTmp) { size_t lenSpace = std::max(nameLen - name.length(), (size_t)1); - size_t len = EntryBaseConstants::total_len - - EntryBaseConstants::pruned_len - size.length() - - name.length() - lenSpace - 2; + size_t len = EntryBase::TOTAL_LEN - EntryBase::PRUNED_LEN - size.length() - name.length() - + lenSpace - 2; size_t lenNameTmp = strlen(nameTmp); while ((len < lenNameTmp) && (lenSpace > 1)) { ++len; @@ -332,12 +438,12 @@ static void formatTmp(const LogStatistics& stat, const char* nameTmp, uid_t uid, } } -std::string UidEntry::format(const LogStatistics& stat, log_id_t id) const { - uid_t uid = getUid(); +std::string UidEntry::format(const LogStatistics& stat, log_id_t id, uid_t uid) const + REQUIRES(stat.lock_) { std::string name = android::base::StringPrintf("%u", uid); std::string size = android::base::StringPrintf("%zu", getSizes()); - formatTmp(stat, nullptr, uid, name, size, 6); + stat.FormatTmp(nullptr, uid, name, size, 6); std::string pruned = ""; if (worstUidEnabledForLogid(id)) { @@ -345,17 +451,17 @@ std::string UidEntry::format(const LogStatistics& stat, log_id_t id) const { for (LogStatistics::uidTable_t::const_iterator it = stat.uidTable[id].begin(); it != stat.uidTable[id].end(); ++it) { - totalDropped += it->second.getDropped(); + totalDropped += it->second.dropped_count(); } - size_t sizes = stat.sizes(id); - size_t totalSize = stat.sizesTotal(id); - size_t totalElements = stat.elementsTotal(id); + size_t sizes = stat.mSizes[id]; + size_t totalSize = stat.mSizesTotal[id]; + size_t totalElements = stat.mElementsTotal[id]; float totalVirtualSize = (float)sizes + (float)totalDropped * totalSize / totalElements; size_t entrySize = getSizes(); float virtualEntrySize = entrySize; int realPermille = virtualEntrySize * 1000.0 / sizes; - size_t dropped = getDropped(); + size_t dropped = dropped_count(); if (dropped) { pruned = android::base::StringPrintf("%zu", dropped); virtualEntrySize += (float)dropped * totalSize / totalElements; @@ -382,8 +488,7 @@ std::string UidEntry::format(const LogStatistics& stat, log_id_t id) const { change = android::base::StringPrintf( "%s%d%s", prefix, (permille + 5) / 10, units); } - ssize_t spaces = EntryBaseConstants::pruned_len - 2 - - pruned.length() - change.length(); + ssize_t spaces = EntryBase::PRUNED_LEN - 2 - pruned.length() - change.length(); if ((spaces <= 0) && pruned.length()) { spaces = 1; } @@ -401,27 +506,25 @@ std::string UidEntry::format(const LogStatistics& stat, log_id_t id) const { } static const size_t maximum_sorted_entries = 32; - std::unique_ptr<const PidEntry* []> sorted = - stat.pidSystemTable[id].sort(uid, (pid_t)0, maximum_sorted_entries); + std::array<const pid_t*, maximum_sorted_entries> sorted_pids; + std::array<const PidEntry*, maximum_sorted_entries> sorted_entries; + stat.pidSystemTable[id].MaxEntries(uid, 0, sorted_pids, sorted_entries); - if (!sorted.get()) { - return output; - } std::string byPid; size_t index; bool hasDropped = false; for (index = 0; index < maximum_sorted_entries; ++index) { - const PidEntry* entry = sorted[index]; + const PidEntry* entry = sorted_entries[index]; if (!entry) { break; } if (entry->getSizes() <= (getSizes() / 100)) { break; } - if (entry->getDropped()) { + if (entry->dropped_count()) { hasDropped = true; } - byPid += entry->format(stat, id); + byPid += entry->format(stat, id, *sorted_pids[index]); } if (index > 1) { // print this only if interesting std::string ditto("\" "); @@ -440,17 +543,15 @@ std::string PidEntry::formatHeader(const std::string& name, std::string("BYTES"), std::string("NUM")); } -std::string PidEntry::format(const LogStatistics& stat, - log_id_t /* id */) const { - uid_t uid = getUid(); - pid_t pid = getPid(); - std::string name = android::base::StringPrintf("%5u/%u", pid, uid); +std::string PidEntry::format(const LogStatistics& stat, log_id_t, pid_t pid) const + REQUIRES(stat.lock_) { + std::string name = android::base::StringPrintf("%5u/%u", pid, uid_); std::string size = android::base::StringPrintf("%zu", getSizes()); - formatTmp(stat, getName(), uid, name, size, 12); + stat.FormatTmp(name_, uid_, name, size, 12); std::string pruned = ""; - size_t dropped = getDropped(); + size_t dropped = dropped_count(); if (dropped) { pruned = android::base::StringPrintf("%zu", dropped); } @@ -465,16 +566,15 @@ std::string TidEntry::formatHeader(const std::string& name, std::string("NUM")); } -std::string TidEntry::format(const LogStatistics& stat, - log_id_t /* id */) const { - uid_t uid = getUid(); - std::string name = android::base::StringPrintf("%5u/%u", getTid(), uid); +std::string TidEntry::format(const LogStatistics& stat, log_id_t, pid_t tid) const + REQUIRES(stat.lock_) { + std::string name = android::base::StringPrintf("%5u/%u", tid, uid_); std::string size = android::base::StringPrintf("%zu", getSizes()); - formatTmp(stat, getName(), uid, name, size, 12); + stat.FormatTmp(name_, uid_, name, size, 12); std::string pruned = ""; - size_t dropped = getDropped(); + size_t dropped = dropped_count(); if (dropped) { pruned = android::base::StringPrintf("%zu", dropped); } @@ -490,16 +590,14 @@ std::string TagEntry::formatHeader(const std::string& name, log_id_t id) const { std::string("BYTES"), std::string(isprune ? "NUM" : "")); } -std::string TagEntry::format(const LogStatistics& /* stat */, - log_id_t /* id */) const { +std::string TagEntry::format(const LogStatistics&, log_id_t, uint32_t) const { std::string name; - uid_t uid = getUid(); - if (uid == (uid_t)-1) { - name = android::base::StringPrintf("%7u", getKey()); + if (uid_ == (uid_t)-1) { + name = android::base::StringPrintf("%7u", key()); } else { - name = android::base::StringPrintf("%7u/%u", getKey(), uid); + name = android::base::StringPrintf("%7u/%u", key(), uid_); } - const char* nameTmp = getName(); + const char* nameTmp = this->name(); if (nameTmp) { name += android::base::StringPrintf( "%*s%s", (int)std::max(14 - name.length(), (size_t)1), "", nameTmp); @@ -508,7 +606,7 @@ std::string TagEntry::format(const LogStatistics& /* stat */, std::string size = android::base::StringPrintf("%zu", getSizes()); std::string pruned = ""; - size_t dropped = getDropped(); + size_t dropped = dropped_count(); if (dropped) { pruned = android::base::StringPrintf("%zu", dropped); } @@ -523,37 +621,33 @@ std::string TagNameEntry::formatHeader(const std::string& name, std::string("BYTES"), std::string("")); } -std::string TagNameEntry::format(const LogStatistics& /* stat */, - log_id_t /* id */) const { +std::string TagNameEntry::format(const LogStatistics&, log_id_t, + const std::string& key_name) const { std::string name; - pid_t tid = getTid(); - pid_t pid = getPid(); std::string pidstr; - if (pid != (pid_t)-1) { - pidstr = android::base::StringPrintf("%u", pid); - if ((tid != (pid_t)-1) && (tid != pid)) pidstr = "/" + pidstr; + if (pid_ != (pid_t)-1) { + pidstr = android::base::StringPrintf("%u", pid_); + if (tid_ != (pid_t)-1 && tid_ != pid_) pidstr = "/" + pidstr; } int len = 9 - pidstr.length(); if (len < 0) len = 0; - if ((tid == (pid_t)-1) || (tid == pid)) { + if (tid_ == (pid_t)-1 || tid_ == pid_) { name = android::base::StringPrintf("%*s", len, ""); } else { - name = android::base::StringPrintf("%*u", len, tid); + name = android::base::StringPrintf("%*u", len, tid_); } name += pidstr; - uid_t uid = getUid(); - if (uid != (uid_t)-1) { - name += android::base::StringPrintf("/%u", uid); + if (uid_ != (uid_t)-1) { + name += android::base::StringPrintf("/%u", uid_); } std::string size = android::base::StringPrintf("%zu", getSizes()); - const char* nameTmp = getName(); + const char* nameTmp = key_name.data(); if (nameTmp) { size_t lenSpace = std::max(16 - name.length(), (size_t)1); - size_t len = EntryBaseConstants::total_len - - EntryBaseConstants::pruned_len - size.length() - - name.length() - lenSpace - 2; + size_t len = EntryBase::TOTAL_LEN - EntryBase::PRUNED_LEN - size.length() - name.length() - + lenSpace - 2; size_t lenNameTmp = strlen(nameTmp); while ((len < lenNameTmp) && (lenSpace > 1)) { ++len; @@ -611,8 +705,37 @@ static std::string formatMsec(uint64_t val) { return output; } -std::string LogStatistics::format(uid_t uid, pid_t pid, - unsigned int logMask) const { +template <typename TKey, typename TEntry> +std::string LogStatistics::FormatTable(const LogHashtable<TKey, TEntry>& table, uid_t uid, + pid_t pid, const std::string& name, log_id_t id) const + REQUIRES(lock_) { + static const size_t maximum_sorted_entries = 32; + std::string output; + std::array<const TKey*, maximum_sorted_entries> sorted_keys; + std::array<const TEntry*, maximum_sorted_entries> sorted_entries; + table.MaxEntries(uid, pid, sorted_keys, sorted_entries); + bool header_printed = false; + for (size_t index = 0; index < maximum_sorted_entries; ++index) { + const TEntry* entry = sorted_entries[index]; + if (!entry) { + break; + } + if (entry->getSizes() <= (sorted_entries[0]->getSizes() / 100)) { + break; + } + if (!header_printed) { + output += "\n\n"; + output += entry->formatHeader(name, id); + header_printed = true; + } + output += entry->format(*this, id, *sorted_keys[index]); + } + return output; +} + +std::string LogStatistics::Format(uid_t uid, pid_t pid, unsigned int logMask) const { + auto lock = std::lock_guard{lock_}; + static const uint16_t spaces_total = 19; // Report on total logging, current and for all time @@ -642,9 +765,9 @@ std::string LogStatistics::format(uid_t uid, pid_t pid, if (!(logMask & (1 << id))) continue; oldLength = output.length(); if (spaces < 0) spaces = 0; - size_t szs = sizesTotal(id); + size_t szs = mSizesTotal[id]; totalSize += szs; - size_t els = elementsTotal(id); + size_t els = mElementsTotal[id]; totalEls += els; output += android::base::StringPrintf("%*s%zu/%zu", spaces, "", szs, els); @@ -663,11 +786,11 @@ std::string LogStatistics::format(uid_t uid, pid_t pid, log_id_for_each(id) { if (!(logMask & (1 << id))) continue; - size_t els = elements(id); + size_t els = mElements[id]; if (els) { oldLength = output.length(); if (spaces < 0) spaces = 0; - size_t szs = sizes(id); + size_t szs = mSizes[id]; totalSize += szs; totalEls += els; output += @@ -749,7 +872,7 @@ std::string LogStatistics::format(uid_t uid, pid_t pid, log_id_for_each(id) { if (!(logMask & (1 << id))) continue; - size_t els = elements(id); + size_t els = mElements[id]; if (els) { oldLength = output.length(); if (spaces < 0) spaces = 0; @@ -758,7 +881,7 @@ std::string LogStatistics::format(uid_t uid, pid_t pid, ((sizeof(LogBufferElement) + sizeof(uint64_t) - 1) & -sizeof(uint64_t)) + sizeof(std::list<LogBufferElement*>); - size_t szs = sizes(id) + els * overhead; + size_t szs = mSizes[id] + els * overhead; totalSize += szs; output += android::base::StringPrintf("%*s%zu", spaces, "", szs); spaces -= output.length() - oldLength; @@ -779,39 +902,38 @@ std::string LogStatistics::format(uid_t uid, pid_t pid, name = (uid == AID_ROOT) ? "Chattiest UIDs in %s log buffer:" : "Logging for your UID in %s log buffer:"; - output += uidTable[id].format(*this, uid, pid, name, id); + output += FormatTable(uidTable[id], uid, pid, name, id); } if (enable) { name = ((uid == AID_ROOT) && !pid) ? "Chattiest PIDs:" : "Logging for this PID:"; - output += pidTable.format(*this, uid, pid, name); + output += FormatTable(pidTable, uid, pid, name); name = "Chattiest TIDs"; if (pid) name += android::base::StringPrintf(" for PID %d", pid); name += ":"; - output += tidTable.format(*this, uid, pid, name); + output += FormatTable(tidTable, uid, pid, name); } if (enable && (logMask & (1 << LOG_ID_EVENTS))) { name = "Chattiest events log buffer TAGs"; if (pid) name += android::base::StringPrintf(" for PID %d", pid); name += ":"; - output += tagTable.format(*this, uid, pid, name, LOG_ID_EVENTS); + output += FormatTable(tagTable, uid, pid, name, LOG_ID_EVENTS); } if (enable && (logMask & (1 << LOG_ID_SECURITY))) { name = "Chattiest security log buffer TAGs"; if (pid) name += android::base::StringPrintf(" for PID %d", pid); name += ":"; - output += - securityTagTable.format(*this, uid, pid, name, LOG_ID_SECURITY); + output += FormatTable(securityTagTable, uid, pid, name, LOG_ID_SECURITY); } if (enable) { name = "Chattiest TAGs"; if (pid) name += android::base::StringPrintf(" for PID %d", pid); name += ":"; - output += tagNameTable.format(*this, uid, pid, name); + output += FormatTable(tagNameTable, uid, pid, name); } return output; @@ -822,7 +944,7 @@ namespace android { uid_t pidToUid(pid_t pid) { char buffer[512]; snprintf(buffer, sizeof(buffer), "/proc/%u/status", pid); - FILE* fp = fopen(buffer, "r"); + FILE* fp = fopen(buffer, "re"); if (fp) { while (fgets(buffer, sizeof(buffer), fp)) { int uid = AID_LOGD; @@ -839,15 +961,17 @@ uid_t pidToUid(pid_t pid) { } } -uid_t LogStatistics::pidToUid(pid_t pid) { - return pidTable.add(pid)->second.getUid(); +uid_t LogStatistics::PidToUid(pid_t pid) { + auto lock = std::lock_guard{lock_}; + return pidTable.Add(pid)->second.uid(); } // caller must free character string -const char* LogStatistics::pidToName(pid_t pid) const { +const char* LogStatistics::PidToName(pid_t pid) const { + auto lock = std::lock_guard{lock_}; // An inconvenient truth ... getName() can alter the object pidTable_t& writablePidTable = const_cast<pidTable_t&>(pidTable); - const char* name = writablePidTable.add(pid)->second.getName(); + const char* name = writablePidTable.Add(pid)->second.name(); if (!name) { return nullptr; } diff --git a/logd/LogStatistics.h b/logd/LogStatistics.h index 0782de3d1..5f558026c 100644 --- a/logd/LogStatistics.h +++ b/logd/LogStatistics.h @@ -14,8 +14,7 @@ * limitations under the License. */ -#ifndef _LOGD_LOG_STATISTICS_H__ -#define _LOGD_LOG_STATISTICS_H__ +#pragma once #include <ctype.h> #include <inttypes.h> @@ -25,24 +24,40 @@ #include <sys/types.h> #include <algorithm> // std::max +#include <array> #include <memory> +#include <mutex> #include <string> #include <string_view> #include <unordered_map> #include <android-base/stringprintf.h> +#include <android-base/thread_annotations.h> #include <android/log.h> #include <log/log_time.h> #include <private/android_filesystem_config.h> #include <utils/FastStrcmp.h> -#include "LogBufferElement.h" #include "LogUtils.h" #define log_id_for_each(i) \ for (log_id_t i = LOG_ID_MIN; (i) < LOG_ID_MAX; (i) = (log_id_t)((i) + 1)) class LogStatistics; +class UidEntry; +class PidEntry; + +struct LogStatisticsElement { + uid_t uid; + pid_t pid; + pid_t tid; + uint32_t tag; + log_time realtime; + const char* msg; + uint16_t msg_len; + uint16_t dropped_count; + log_id_t log_id; +}; template <typename TKey, typename TEntry> class LogHashtable { @@ -63,7 +78,7 @@ class LogHashtable { static const size_t unordered_map_per_entry_overhead = sizeof(void*); static const size_t unordered_map_bucket_overhead = sizeof(void*); - public: + public: size_t size() const { return map.size(); } @@ -79,162 +94,111 @@ class LogHashtable { typedef typename std::unordered_map<TKey, TEntry>::const_iterator const_iterator; - std::unique_ptr<const TEntry* []> sort(uid_t uid, pid_t pid, - size_t len) const { - if (!len) { - std::unique_ptr<const TEntry* []> sorted(nullptr); - return sorted; - } - - const TEntry** retval = new const TEntry*[len]; - memset(retval, 0, sizeof(*retval) * len); - - for (const_iterator it = map.begin(); it != map.end(); ++it) { - const TEntry& entry = it->second; - - if ((uid != AID_ROOT) && (uid != entry.getUid())) { + // Returns a sorted array of up to len highest entries sorted by size. If fewer than len + // entries are found, their positions are set to nullptr. + template <size_t len> + void MaxEntries(uid_t uid, pid_t pid, std::array<const TKey*, len>& out_keys, + std::array<const TEntry*, len>& out_entries) const { + out_keys.fill(nullptr); + out_entries.fill(nullptr); + for (const auto& [key, entry] : map) { + uid_t entry_uid = 0; + if constexpr (std::is_same_v<TEntry, UidEntry>) { + entry_uid = key; + } else { + entry_uid = entry.uid(); + } + if (uid != AID_ROOT && uid != entry_uid) { continue; } - if (pid && entry.getPid() && (pid != entry.getPid())) { + pid_t entry_pid = 0; + if constexpr (std::is_same_v<TEntry, PidEntry>) { + entry_pid = key; + } else { + entry_pid = entry.pid(); + } + if (pid && entry_pid && pid != entry_pid) { continue; } size_t sizes = entry.getSizes(); ssize_t index = len - 1; - while ((!retval[index] || (sizes > retval[index]->getSizes())) && - (--index >= 0)) + while ((!out_entries[index] || sizes > out_entries[index]->getSizes()) && --index >= 0) ; if (++index < (ssize_t)len) { size_t num = len - index - 1; if (num) { - memmove(&retval[index + 1], &retval[index], - num * sizeof(retval[0])); + memmove(&out_keys[index + 1], &out_keys[index], num * sizeof(const TKey*)); + memmove(&out_entries[index + 1], &out_entries[index], + num * sizeof(const TEntry*)); } - retval[index] = &entry; + out_keys[index] = &key; + out_entries[index] = &entry; } } - std::unique_ptr<const TEntry* []> sorted(retval); - return sorted; } - inline iterator add(const TKey& key, const LogBufferElement* element) { + iterator Add(const TKey& key, const LogStatisticsElement& element) { iterator it = map.find(key); if (it == map.end()) { it = map.insert(std::make_pair(key, TEntry(element))).first; } else { - it->second.add(element); + it->second.Add(element); } return it; } - inline iterator add(TKey key) { + iterator Add(const TKey& key) { iterator it = map.find(key); if (it == map.end()) { it = map.insert(std::make_pair(key, TEntry(key))).first; } else { - it->second.add(key); + it->second.Add(key); } return it; } - void subtract(TKey&& key, const LogBufferElement* element) { - iterator it = map.find(std::move(key)); - if ((it != map.end()) && it->second.subtract(element)) { - map.erase(it); - } - } - - void subtract(const TKey& key, const LogBufferElement* element) { + void Subtract(const TKey& key, const LogStatisticsElement& element) { iterator it = map.find(key); - if ((it != map.end()) && it->second.subtract(element)) { + if (it != map.end() && it->second.Subtract(element)) { map.erase(it); } } - inline void drop(TKey key, const LogBufferElement* element) { + void Drop(const TKey& key, const LogStatisticsElement& element) { iterator it = map.find(key); if (it != map.end()) { - it->second.drop(element); + it->second.Drop(element); } } - inline iterator begin() { - return map.begin(); - } - inline const_iterator begin() const { - return map.begin(); - } - inline iterator end() { - return map.end(); - } - inline const_iterator end() const { - return map.end(); - } - - std::string format(const LogStatistics& stat, uid_t uid, pid_t pid, - const std::string& name = std::string(""), - log_id_t id = LOG_ID_MAX) const { - static const size_t maximum_sorted_entries = 32; - std::string output; - std::unique_ptr<const TEntry* []> sorted = - sort(uid, pid, maximum_sorted_entries); - if (!sorted.get()) { - return output; - } - bool headerPrinted = false; - for (size_t index = 0; index < maximum_sorted_entries; ++index) { - const TEntry* entry = sorted[index]; - if (!entry) { - break; - } - if (entry->getSizes() <= (sorted[0]->getSizes() / 100)) { - break; - } - if (!headerPrinted) { - output += "\n\n"; - output += entry->formatHeader(name, id); - headerPrinted = true; - } - output += entry->format(stat, id); - } - return output; - } + iterator begin() { return map.begin(); } + const_iterator begin() const { return map.begin(); } + iterator end() { return map.end(); } + const_iterator end() const { return map.end(); } }; -namespace EntryBaseConstants { -static constexpr size_t pruned_len = 14; -static constexpr size_t total_len = 80; -} - -struct EntryBase { - size_t size; +class EntryBase { + public: + EntryBase() : size_(0) {} + explicit EntryBase(const LogStatisticsElement& element) : size_(element.msg_len) {} - EntryBase() : size(0) { - } - explicit EntryBase(const LogBufferElement* element) - : size(element->getMsgLen()) { - } + size_t getSizes() const { return size_; } - size_t getSizes() const { - return size; + void Add(const LogStatisticsElement& element) { size_ += element.msg_len; } + bool Subtract(const LogStatisticsElement& element) { + size_ -= element.msg_len; + return !size_; } - inline void add(const LogBufferElement* element) { - size += element->getMsgLen(); - } - inline bool subtract(const LogBufferElement* element) { - size -= element->getMsgLen(); - return !size; - } + static constexpr size_t PRUNED_LEN = 14; + static constexpr size_t TOTAL_LEN = 80; static std::string formatLine(const std::string& name, const std::string& size, const std::string& pruned) { - ssize_t drop_len = - std::max(pruned.length() + 1, EntryBaseConstants::pruned_len); - ssize_t size_len = - std::max(size.length() + 1, EntryBaseConstants::total_len - - name.length() - drop_len - 1); + ssize_t drop_len = std::max(pruned.length() + 1, PRUNED_LEN); + ssize_t size_len = std::max(size.length() + 1, TOTAL_LEN - name.length() - drop_len - 1); std::string ret = android::base::StringPrintf( "%s%*s%*s", name.c_str(), (int)size_len, size.c_str(), @@ -246,545 +210,353 @@ struct EntryBase { if (len) ret.erase(pos + 1, len); return ret + "\n"; } -}; -struct EntryBaseDropped : public EntryBase { - size_t dropped; + private: + size_t size_; +}; - EntryBaseDropped() : dropped(0) { - } - explicit EntryBaseDropped(const LogBufferElement* element) - : EntryBase(element), dropped(element->getDropped()) { - } +class EntryBaseDropped : public EntryBase { + public: + EntryBaseDropped() : dropped_(0) {} + explicit EntryBaseDropped(const LogStatisticsElement& element) + : EntryBase(element), dropped_(element.dropped_count) {} - size_t getDropped() const { - return dropped; - } + size_t dropped_count() const { return dropped_; } - inline void add(const LogBufferElement* element) { - dropped += element->getDropped(); - EntryBase::add(element); + void Add(const LogStatisticsElement& element) { + dropped_ += element.dropped_count; + EntryBase::Add(element); } - inline bool subtract(const LogBufferElement* element) { - dropped -= element->getDropped(); - return EntryBase::subtract(element) && !dropped; + bool Subtract(const LogStatisticsElement& element) { + dropped_ -= element.dropped_count; + return EntryBase::Subtract(element) && !dropped_; } - inline void drop(const LogBufferElement* element) { - dropped += 1; - EntryBase::subtract(element); + void Drop(const LogStatisticsElement& element) { + dropped_ += 1; + EntryBase::Subtract(element); } -}; -struct UidEntry : public EntryBaseDropped { - const uid_t uid; - pid_t pid; + private: + size_t dropped_; +}; - explicit UidEntry(const LogBufferElement* element) - : EntryBaseDropped(element), - uid(element->getUid()), - pid(element->getPid()) { - } +class UidEntry : public EntryBaseDropped { + public: + explicit UidEntry(const LogStatisticsElement& element) + : EntryBaseDropped(element), pid_(element.pid) {} - inline const uid_t& getKey() const { - return uid; - } - inline const uid_t& getUid() const { - return getKey(); - } - inline const pid_t& getPid() const { - return pid; - } + pid_t pid() const { return pid_; } - inline void add(const LogBufferElement* element) { - if (pid != element->getPid()) { - pid = -1; + void Add(const LogStatisticsElement& element) { + if (pid_ != element.pid) { + pid_ = -1; } - EntryBaseDropped::add(element); + EntryBaseDropped::Add(element); } std::string formatHeader(const std::string& name, log_id_t id) const; - std::string format(const LogStatistics& stat, log_id_t id) const; + std::string format(const LogStatistics& stat, log_id_t id, uid_t uid) const; + + private: + pid_t pid_; }; namespace android { uid_t pidToUid(pid_t pid); } -struct PidEntry : public EntryBaseDropped { - const pid_t pid; - uid_t uid; - char* name; - +class PidEntry : public EntryBaseDropped { + public: explicit PidEntry(pid_t pid) : EntryBaseDropped(), - pid(pid), - uid(android::pidToUid(pid)), - name(android::pidToName(pid)) { - } - explicit PidEntry(const LogBufferElement* element) - : EntryBaseDropped(element), - pid(element->getPid()), - uid(element->getUid()), - name(android::pidToName(pid)) { - } + uid_(android::pidToUid(pid)), + name_(android::pidToName(pid)) {} + explicit PidEntry(const LogStatisticsElement& element) + : EntryBaseDropped(element), uid_(element.uid), name_(android::pidToName(element.pid)) {} PidEntry(const PidEntry& element) : EntryBaseDropped(element), - pid(element.pid), - uid(element.uid), - name(element.name ? strdup(element.name) : nullptr) { - } - ~PidEntry() { - free(name); - } + uid_(element.uid_), + name_(element.name_ ? strdup(element.name_) : nullptr) {} + ~PidEntry() { free(name_); } - const pid_t& getKey() const { - return pid; - } - const pid_t& getPid() const { - return getKey(); - } - const uid_t& getUid() const { - return uid; - } - const char* getName() const { - return name; - } + uid_t uid() const { return uid_; } + const char* name() const { return name_; } - inline void add(pid_t newPid) { - if (name && !fastcmp<strncmp>(name, "zygote", 6)) { - free(name); - name = nullptr; + void Add(pid_t new_pid) { + if (name_ && !fastcmp<strncmp>(name_, "zygote", 6)) { + free(name_); + name_ = nullptr; } - if (!name) { - name = android::pidToName(newPid); + if (!name_) { + name_ = android::pidToName(new_pid); } } - inline void add(const LogBufferElement* element) { - uid_t incomingUid = element->getUid(); - if (getUid() != incomingUid) { - uid = incomingUid; - free(name); - name = android::pidToName(element->getPid()); + void Add(const LogStatisticsElement& element) { + uid_t incoming_uid = element.uid; + if (uid() != incoming_uid) { + uid_ = incoming_uid; + free(name_); + name_ = android::pidToName(element.pid); } else { - add(element->getPid()); + Add(element.pid); } - EntryBaseDropped::add(element); + EntryBaseDropped::Add(element); } std::string formatHeader(const std::string& name, log_id_t id) const; - std::string format(const LogStatistics& stat, log_id_t id) const; -}; + std::string format(const LogStatistics& stat, log_id_t id, pid_t pid) const; -struct TidEntry : public EntryBaseDropped { - const pid_t tid; - pid_t pid; - uid_t uid; - char* name; + private: + uid_t uid_; + char* name_; +}; +class TidEntry : public EntryBaseDropped { + public: TidEntry(pid_t tid, pid_t pid) : EntryBaseDropped(), - tid(tid), - pid(pid), - uid(android::pidToUid(tid)), - name(android::tidToName(tid)) { - } - explicit TidEntry(const LogBufferElement* element) + pid_(pid), + uid_(android::pidToUid(tid)), + name_(android::tidToName(tid)) {} + explicit TidEntry(const LogStatisticsElement& element) : EntryBaseDropped(element), - tid(element->getTid()), - pid(element->getPid()), - uid(element->getUid()), - name(android::tidToName(tid)) { - } + pid_(element.pid), + uid_(element.uid), + name_(android::tidToName(element.tid)) {} TidEntry(const TidEntry& element) : EntryBaseDropped(element), - tid(element.tid), - pid(element.pid), - uid(element.uid), - name(element.name ? strdup(element.name) : nullptr) { - } - ~TidEntry() { - free(name); - } - - const pid_t& getKey() const { - return tid; - } - const pid_t& getTid() const { - return getKey(); - } - const pid_t& getPid() const { - return pid; - } - const uid_t& getUid() const { - return uid; - } - const char* getName() const { - return name; - } - - inline void add(pid_t incomingTid) { - if (name && !fastcmp<strncmp>(name, "zygote", 6)) { - free(name); - name = nullptr; + pid_(element.pid_), + uid_(element.uid_), + name_(element.name_ ? strdup(element.name_) : nullptr) {} + ~TidEntry() { free(name_); } + + pid_t pid() const { return pid_; } + uid_t uid() const { return uid_; } + const char* name() const { return name_; } + + void Add(pid_t incomingTid) { + if (name_ && !fastcmp<strncmp>(name_, "zygote", 6)) { + free(name_); + name_ = nullptr; } - if (!name) { - name = android::tidToName(incomingTid); + if (!name_) { + name_ = android::tidToName(incomingTid); } } - inline void add(const LogBufferElement* element) { - uid_t incomingUid = element->getUid(); - pid_t incomingPid = element->getPid(); - if ((getUid() != incomingUid) || (getPid() != incomingPid)) { - uid = incomingUid; - pid = incomingPid; - free(name); - name = android::tidToName(element->getTid()); + void Add(const LogStatisticsElement& element) { + uid_t incoming_uid = element.uid; + pid_t incoming_pid = element.pid; + if (uid() != incoming_uid || pid() != incoming_pid) { + uid_ = incoming_uid; + pid_ = incoming_pid; + free(name_); + name_ = android::tidToName(element.tid); } else { - add(element->getTid()); + Add(element.tid); } - EntryBaseDropped::add(element); + EntryBaseDropped::Add(element); } std::string formatHeader(const std::string& name, log_id_t id) const; - std::string format(const LogStatistics& stat, log_id_t id) const; -}; + std::string format(const LogStatistics& stat, log_id_t id, pid_t pid) const; -struct TagEntry : public EntryBaseDropped { - const uint32_t tag; - pid_t pid; - uid_t uid; + private: + pid_t pid_; + uid_t uid_; + char* name_; +}; - explicit TagEntry(const LogBufferElement* element) - : EntryBaseDropped(element), - tag(element->getTag()), - pid(element->getPid()), - uid(element->getUid()) { - } +class TagEntry : public EntryBaseDropped { + public: + explicit TagEntry(const LogStatisticsElement& element) + : EntryBaseDropped(element), tag_(element.tag), pid_(element.pid), uid_(element.uid) {} - const uint32_t& getKey() const { - return tag; - } - const pid_t& getPid() const { - return pid; - } - const uid_t& getUid() const { - return uid; - } - const char* getName() const { - return android::tagToName(tag); - } + uint32_t key() const { return tag_; } + pid_t pid() const { return pid_; } + uid_t uid() const { return uid_; } + const char* name() const { return android::tagToName(tag_); } - inline void add(const LogBufferElement* element) { - if (uid != element->getUid()) { - uid = -1; + void Add(const LogStatisticsElement& element) { + if (uid_ != element.uid) { + uid_ = -1; } - if (pid != element->getPid()) { - pid = -1; + if (pid_ != element.pid) { + pid_ = -1; } - EntryBaseDropped::add(element); + EntryBaseDropped::Add(element); } std::string formatHeader(const std::string& name, log_id_t id) const; - std::string format(const LogStatistics& stat, log_id_t id) const; -}; - -struct TagNameKey { - std::string* alloc; - std::string_view name; // Saves space if const char* - - explicit TagNameKey(const LogBufferElement* element) - : alloc(nullptr), name("", strlen("")) { - if (element->isBinary()) { - uint32_t tag = element->getTag(); - if (tag) { - const char* cp = android::tagToName(tag); - if (cp) { - name = std::string_view(cp, strlen(cp)); - return; - } - } - alloc = new std::string( - android::base::StringPrintf("[%" PRIu32 "]", tag)); - if (!alloc) return; - name = std::string_view(alloc->c_str(), alloc->size()); - return; - } - const char* msg = element->getMsg(); - if (!msg) { - name = std::string_view("chatty", strlen("chatty")); - return; - } - ++msg; - uint16_t len = element->getMsgLen(); - len = (len <= 1) ? 0 : strnlen(msg, len - 1); - if (!len) { - name = std::string_view("<NULL>", strlen("<NULL>")); - return; - } - alloc = new std::string(msg, len); - if (!alloc) return; - name = std::string_view(alloc->c_str(), alloc->size()); - } - - explicit TagNameKey(TagNameKey&& rval) noexcept - : alloc(rval.alloc), name(rval.name.data(), rval.name.length()) { - rval.alloc = nullptr; - } + std::string format(const LogStatistics& stat, log_id_t id, uint32_t) const; - explicit TagNameKey(const TagNameKey& rval) - : alloc(rval.alloc ? new std::string(*rval.alloc) : nullptr), - name(alloc ? alloc->data() : rval.name.data(), rval.name.length()) { - } - - ~TagNameKey() { - if (alloc) delete alloc; - } - - operator const std::string_view() const { - return name; - } - - const char* data() const { - return name.data(); - } - size_t length() const { - return name.length(); - } - - bool operator==(const TagNameKey& rval) const { - if (length() != rval.length()) return false; - if (length() == 0) return true; - return fastcmp<strncmp>(data(), rval.data(), length()) == 0; - } - bool operator!=(const TagNameKey& rval) const { - return !(*this == rval); - } - - size_t getAllocLength() const { - return alloc ? alloc->length() + 1 + sizeof(std::string) : 0; - } + private: + const uint32_t tag_; + pid_t pid_; + uid_t uid_; }; -// Hash for TagNameKey -template <> -struct std::hash<TagNameKey> - : public std::unary_function<const TagNameKey&, size_t> { - size_t operator()(const TagNameKey& __t) const noexcept { - if (!__t.length()) return 0; - return std::hash<std::string_view>()(std::string_view(__t)); - } -}; +class TagNameEntry : public EntryBase { + public: + explicit TagNameEntry(const LogStatisticsElement& element) + : EntryBase(element), tid_(element.tid), pid_(element.pid), uid_(element.uid) {} -struct TagNameEntry : public EntryBase { - pid_t tid; - pid_t pid; - uid_t uid; - TagNameKey name; - - explicit TagNameEntry(const LogBufferElement* element) - : EntryBase(element), - tid(element->getTid()), - pid(element->getPid()), - uid(element->getUid()), - name(element) { - } + pid_t tid() const { return tid_; } + pid_t pid() const { return pid_; } + uid_t uid() const { return uid_; } - const TagNameKey& getKey() const { - return name; - } - const pid_t& getTid() const { - return tid; - } - const pid_t& getPid() const { - return pid; - } - const uid_t& getUid() const { - return uid; - } - const char* getName() const { - return name.data(); - } - size_t getNameAllocLength() const { - return name.getAllocLength(); - } - - inline void add(const LogBufferElement* element) { - if (uid != element->getUid()) { - uid = -1; + void Add(const LogStatisticsElement& element) { + if (uid_ != element.uid) { + uid_ = -1; } - if (pid != element->getPid()) { - pid = -1; + if (pid_ != element.pid) { + pid_ = -1; } - if (tid != element->getTid()) { - tid = -1; + if (tid_ != element.tid) { + tid_ = -1; } - EntryBase::add(element); + EntryBase::Add(element); } std::string formatHeader(const std::string& name, log_id_t id) const; - std::string format(const LogStatistics& stat, log_id_t id) const; -}; + std::string format(const LogStatistics& stat, log_id_t id, const std::string& key_name) const; -template <typename TEntry> -class LogFindWorst { - std::unique_ptr<const TEntry* []> sorted; - - public: - explicit LogFindWorst(std::unique_ptr<const TEntry* []>&& sorted) - : sorted(std::move(sorted)) { - } - - void findWorst(int& worst, size_t& worst_sizes, size_t& second_worst_sizes, - size_t threshold) { - if (sorted.get() && sorted[0] && sorted[1]) { - worst_sizes = sorted[0]->getSizes(); - if ((worst_sizes > threshold) - // Allow time horizon to extend roughly tenfold, assume - // average entry length is 100 characters. - && (worst_sizes > (10 * sorted[0]->getDropped()))) { - worst = sorted[0]->getKey(); - second_worst_sizes = sorted[1]->getSizes(); - if (second_worst_sizes < threshold) { - second_worst_sizes = threshold; - } - } - } - } - - void findWorst(int& worst, size_t worst_sizes, size_t& second_worst_sizes) { - if (sorted.get() && sorted[0] && sorted[1]) { - worst = sorted[0]->getKey(); - second_worst_sizes = - worst_sizes - sorted[0]->getSizes() + sorted[1]->getSizes(); - } - } + private: + pid_t tid_; + pid_t pid_; + uid_t uid_; }; -// Log Statistics class LogStatistics { friend UidEntry; - - size_t mSizes[LOG_ID_MAX]; - size_t mElements[LOG_ID_MAX]; - size_t mDroppedElements[LOG_ID_MAX]; - size_t mSizesTotal[LOG_ID_MAX]; - size_t mElementsTotal[LOG_ID_MAX]; - log_time mOldest[LOG_ID_MAX]; - log_time mNewest[LOG_ID_MAX]; - log_time mNewestDropped[LOG_ID_MAX]; - static size_t SizesTotal; + friend PidEntry; + friend TidEntry; + + size_t mSizes[LOG_ID_MAX] GUARDED_BY(lock_); + size_t mElements[LOG_ID_MAX] GUARDED_BY(lock_); + size_t mDroppedElements[LOG_ID_MAX] GUARDED_BY(lock_); + size_t mSizesTotal[LOG_ID_MAX] GUARDED_BY(lock_); + size_t mElementsTotal[LOG_ID_MAX] GUARDED_BY(lock_); + log_time mOldest[LOG_ID_MAX] GUARDED_BY(lock_); + log_time mNewest[LOG_ID_MAX] GUARDED_BY(lock_); + log_time mNewestDropped[LOG_ID_MAX] GUARDED_BY(lock_); + static std::atomic<size_t> SizesTotal; bool enable; // uid to size list typedef LogHashtable<uid_t, UidEntry> uidTable_t; - uidTable_t uidTable[LOG_ID_MAX]; + uidTable_t uidTable[LOG_ID_MAX] GUARDED_BY(lock_); // pid of system to size list typedef LogHashtable<pid_t, PidEntry> pidSystemTable_t; - pidSystemTable_t pidSystemTable[LOG_ID_MAX]; + pidSystemTable_t pidSystemTable[LOG_ID_MAX] GUARDED_BY(lock_); // pid to uid list typedef LogHashtable<pid_t, PidEntry> pidTable_t; - pidTable_t pidTable; + pidTable_t pidTable GUARDED_BY(lock_); // tid to uid list typedef LogHashtable<pid_t, TidEntry> tidTable_t; - tidTable_t tidTable; + tidTable_t tidTable GUARDED_BY(lock_); // tag list typedef LogHashtable<uint32_t, TagEntry> tagTable_t; - tagTable_t tagTable; + tagTable_t tagTable GUARDED_BY(lock_); // security tag list - tagTable_t securityTagTable; + tagTable_t securityTagTable GUARDED_BY(lock_); // global tag list - typedef LogHashtable<TagNameKey, TagNameEntry> tagNameTable_t; + typedef LogHashtable<std::string, TagNameEntry> tagNameTable_t; tagNameTable_t tagNameTable; - size_t sizeOf() const { + size_t sizeOf() const REQUIRES(lock_) { size_t size = sizeof(*this) + pidTable.sizeOf() + tidTable.sizeOf() + tagTable.sizeOf() + securityTagTable.sizeOf() + tagNameTable.sizeOf() + (pidTable.size() * sizeof(pidTable_t::iterator)) + (tagTable.size() * sizeof(tagTable_t::iterator)); - for (auto it : pidTable) { - const char* name = it.second.getName(); + for (const auto& it : pidTable) { + const char* name = it.second.name(); if (name) size += strlen(name) + 1; } - for (auto it : tidTable) { - const char* name = it.second.getName(); + for (const auto& it : tidTable) { + const char* name = it.second.name(); if (name) size += strlen(name) + 1; } - for (auto it : tagNameTable) size += it.second.getNameAllocLength(); + for (const auto& it : tagNameTable) { + size += sizeof(std::string); + size_t len = it.first.size(); + // Account for short string optimization: if the string's length is <= 22 bytes for 64 + // bit or <= 10 bytes for 32 bit, then there is no additional allocation. + if ((sizeof(std::string) == 24 && len > 22) || + (sizeof(std::string) != 24 && len > 10)) { + size += len; + } + } log_id_for_each(id) { size += uidTable[id].sizeOf(); size += uidTable[id].size() * sizeof(uidTable_t::iterator); size += pidSystemTable[id].sizeOf(); - size += - pidSystemTable[id].size() * sizeof(pidSystemTable_t::iterator); + size += pidSystemTable[id].size() * sizeof(pidSystemTable_t::iterator); } return size; } - public: - LogStatistics(); + public: + explicit LogStatistics(bool enable_statistics); - void enableStatistics() { - enable = true; - } - - void addTotal(LogBufferElement* entry); - void add(LogBufferElement* entry); - void subtract(LogBufferElement* entry); + void AddTotal(log_id_t log_id, uint16_t size) EXCLUDES(lock_); + void Add(const LogStatisticsElement& entry) EXCLUDES(lock_); + void Subtract(const LogStatisticsElement& entry) EXCLUDES(lock_); // entry->setDropped(1) must follow this call - void drop(LogBufferElement* entry); + void Drop(const LogStatisticsElement& entry) EXCLUDES(lock_); // Correct for coalescing two entries referencing dropped content - void erase(LogBufferElement* element) { - log_id_t log_id = element->getLogId(); + void Erase(const LogStatisticsElement& element) EXCLUDES(lock_) { + auto lock = std::lock_guard{lock_}; + log_id_t log_id = element.log_id; --mElements[log_id]; --mDroppedElements[log_id]; } - LogFindWorst<UidEntry> sort(uid_t uid, pid_t pid, size_t len, log_id id) { - return LogFindWorst<UidEntry>(uidTable[id].sort(uid, pid, len)); - } - LogFindWorst<PidEntry> sortPids(uid_t uid, pid_t pid, size_t len, - log_id id) { - return LogFindWorst<PidEntry>(pidSystemTable[id].sort(uid, pid, len)); - } - LogFindWorst<TagEntry> sortTags(uid_t uid, pid_t pid, size_t len, log_id) { - return LogFindWorst<TagEntry>(tagTable.sort(uid, pid, len)); - } + void WorstTwoUids(log_id id, size_t threshold, int* worst, size_t* worst_sizes, + size_t* second_worst_sizes) const EXCLUDES(lock_); + void WorstTwoTags(size_t threshold, int* worst, size_t* worst_sizes, + size_t* second_worst_sizes) const EXCLUDES(lock_); + void WorstTwoSystemPids(log_id id, size_t worst_uid_sizes, int* worst, + size_t* second_worst_sizes) const EXCLUDES(lock_); + + bool ShouldPrune(log_id id, unsigned long max_size, unsigned long* prune_rows) const + EXCLUDES(lock_); - // fast track current value by id only - size_t sizes(log_id_t id) const { + // Snapshot of the sizes for a given log buffer. + size_t Sizes(log_id_t id) const EXCLUDES(lock_) { + auto lock = std::lock_guard{lock_}; return mSizes[id]; } - size_t elements(log_id_t id) const { - return mElements[id]; - } - size_t realElements(log_id_t id) const { - return mElements[id] - mDroppedElements[id]; - } - size_t sizesTotal(log_id_t id) const { - return mSizesTotal[id]; - } - size_t elementsTotal(log_id_t id) const { - return mElementsTotal[id]; - } + // TODO: Get rid of this entirely. static size_t sizesTotal() { return SizesTotal; } - std::string format(uid_t uid, pid_t pid, unsigned int logMask) const; + std::string Format(uid_t uid, pid_t pid, unsigned int logMask) const EXCLUDES(lock_); - // helper (must be locked directly or implicitly by mLogElementsLock) - const char* pidToName(pid_t pid) const; - uid_t pidToUid(pid_t pid); - const char* uidToName(uid_t uid) const; -}; + const char* PidToName(pid_t pid) const EXCLUDES(lock_); + uid_t PidToUid(pid_t pid) EXCLUDES(lock_); + const char* UidToName(uid_t uid) const EXCLUDES(lock_); -#endif // _LOGD_LOG_STATISTICS_H__ + private: + template <typename TKey, typename TEntry> + void WorstTwoWithThreshold(const LogHashtable<TKey, TEntry>& table, size_t threshold, + int* worst, size_t* worst_sizes, size_t* second_worst_sizes) const; + template <typename TKey, typename TEntry> + std::string FormatTable(const LogHashtable<TKey, TEntry>& table, uid_t uid, pid_t pid, + const std::string& name = std::string(""), + log_id_t id = LOG_ID_MAX) const REQUIRES(lock_); + void FormatTmp(const char* nameTmp, uid_t uid, std::string& name, std::string& size, + size_t nameLen) const REQUIRES(lock_); + const char* UidToNameLocked(uid_t uid) const REQUIRES(lock_); + + mutable std::mutex lock_; +}; diff --git a/logd/LogTags.cpp b/logd/LogTags.cpp index 0cc7886ea..8e18f9d46 100644 --- a/logd/LogTags.cpp +++ b/logd/LogTags.cpp @@ -29,13 +29,17 @@ #include <string> #include <android-base/file.h> +#include <android-base/logging.h> #include <android-base/macros.h> #include <android-base/scopeguard.h> #include <android-base/stringprintf.h> +#include <android-base/threads.h> #include <log/log_event_list.h> #include <log/log_properties.h> +#include <log/log_read.h> #include <private/android_filesystem_config.h> +#include "LogStatistics.h" #include "LogTags.h" #include "LogUtils.h" @@ -98,8 +102,7 @@ bool LogTags::RebuildFileEventLogTags(const char* filename, bool warn) { struct tm tm; localtime_r(&now, &tm); char timebuf[20]; - size_t len = - strftime(timebuf, sizeof(timebuf), "%Y-%m-%d %H:%M:%S", &tm); + strftime(timebuf, sizeof(timebuf), "%Y-%m-%d %H:%M:%S", &tm); android::base::WriteStringToFd( android::base::StringPrintf( "# Rebuilt %.20s, content owned by logd\n", timebuf), @@ -113,10 +116,11 @@ bool LogTags::RebuildFileEventLogTags(const char* filename, bool warn) { } if (warn) { - android::prdebug( - ((fd < 0) ? "%s failed to rebuild" - : "%s missing, damaged or truncated; rebuilt"), - filename); + if (fd < 0) { + LOG(ERROR) << filename << " failed to rebuild"; + } else { + LOG(ERROR) << filename << " missing, damaged or truncated; rebuilt"; + } } if (fd >= 0) { @@ -180,15 +184,13 @@ void LogTags::AddEventLogTags(uint32_t tag, uid_t uid, const std::string& Name, WritePersistEventLogTags(tag, uid, source); } else if (warn && !newOne && source) { // For the files, we want to report dupes. - android::prdebug("Multiple tag %" PRIu32 " %s %s %s", tag, Name.c_str(), - Format.c_str(), source); + LOG(DEBUG) << "Multiple tag " << tag << " " << Name << " " << Format << " " << source; } } // Read the event log tags file, and build up our internal database void LogTags::ReadFileEventLogTags(const char* filename, bool warn) { bool etc = !strcmp(filename, system_event_log_tags); - bool debug = !etc && !strcmp(filename, debug_event_log_tags); if (!etc) { RebuildFileEventLogTags(filename, warn); @@ -215,7 +217,7 @@ void LogTags::ReadFileEventLogTags(const char* filename, bool warn) { } else if (isdigit(*cp)) { unsigned long Tag = strtoul(cp, &cp, 10); if (warn && (Tag > emptyTag)) { - android::prdebug("tag too large %lu", Tag); + LOG(WARNING) << "tag too large " << Tag; } while ((cp < endp) && (*cp != '\n') && isspace(*cp)) ++cp; if (cp >= endp) break; @@ -230,9 +232,8 @@ void LogTags::ReadFileEventLogTags(const char* filename, bool warn) { std::string Name(name, cp - name); #ifdef ALLOW_NOISY_LOGGING_OF_PROBLEM_WITH_LOTS_OF_TECHNICAL_DEBT static const size_t maximum_official_tag_name_size = 24; - if (warn && - (Name.length() > maximum_official_tag_name_size)) { - android::prdebug("tag name too long %s", Name.c_str()); + if (warn && (Name.length() > maximum_official_tag_name_size)) { + LOG(WARNING) << "tag name too long " << Name; } #endif if (hasAlpha && @@ -263,8 +264,8 @@ void LogTags::ReadFileEventLogTags(const char* filename, bool warn) { filename, warn); } else { if (warn) { - android::prdebug("tag name invalid %.*s", - (int)(cp - name + 1), name); + LOG(ERROR) << android::base::StringPrintf("tag name invalid %.*s", + (int)(cp - name + 1), name); } lineStart = nullptr; } @@ -275,7 +276,7 @@ void LogTags::ReadFileEventLogTags(const char* filename, bool warn) { cp++; } } else if (warn) { - android::prdebug("Cannot read %s", filename); + LOG(ERROR) << "Cannot read " << filename; } } @@ -289,9 +290,8 @@ static inline uint32_t get4LE(const char* msg) { // special pmsg event for log tags, and build up our internal // database with any found. void LogTags::ReadPersistEventLogTags() { - struct logger_list* logger_list = android_logger_list_alloc( - ANDROID_LOG_RDONLY | ANDROID_LOG_PSTORE | ANDROID_LOG_NONBLOCK, 0, - (pid_t)0); + struct logger_list* logger_list = + android_logger_list_alloc(ANDROID_LOG_PSTORE | ANDROID_LOG_NONBLOCK, 0, (pid_t)0); if (!logger_list) return; struct logger* e = android_logger_open(logger_list, LOG_ID_EVENTS); @@ -391,23 +391,6 @@ const char* android::tagToName(uint32_t tag) { return me->tagToName(tag); } -// Prototype in LogUtils.h allowing external access to our database. -// -// This only works on userdebug and eng devices to re-read the -// /data/misc/logd/event-log-tags file right after /data is mounted. -// The operation is near to boot and should only happen once. There -// are races associated with its use since it can trigger a Rebuild -// of the file, but that is a can-not-happen since the file was not -// read yet. More dangerous if called later, but if all is well it -// should just skip over everything and not write any new entries. -void android::ReReadEventLogTags() { - LogTags* me = logtags; - - if (me && __android_log_is_debuggable()) { - me->ReadFileEventLogTags(me->debug_event_log_tags); - } -} - // converts an event tag into a format const char* LogTags::tagToFormat(uint32_t tag) const { tag2format_const_iterator iform; @@ -496,8 +479,8 @@ uint32_t LogTags::nameToTag_locked(const std::string& name, const char* format, static int openFile(const char* name, int mode, bool warning) { int fd = TEMP_FAILURE_RETRY(open(name, mode)); - if ((fd < 0) && warning) { - android::prdebug("Failed open %s (%d)", name, errno); + if (fd < 0 && warning) { + PLOG(ERROR) << "Failed to open " << name; } return fd; } @@ -512,7 +495,7 @@ void LogTags::WritePmsgEventLogTags(uint32_t tag, uid_t uid) { // Every 16K (half the smallest configurable pmsg buffer size) record static const size_t rate_to_pmsg = 16 * 1024; - if (lastTotal && ((android::sizesTotal() - lastTotal) < rate_to_pmsg)) { + if (lastTotal && (LogStatistics::sizesTotal() - lastTotal) < rate_to_pmsg) { return; } @@ -565,13 +548,13 @@ void LogTags::WritePmsgEventLogTags(uint32_t tag, uid_t uid) { */ struct timespec ts; - clock_gettime(android_log_clockid(), &ts); + clock_gettime(CLOCK_REALTIME, &ts); android_log_header_t header = { - .id = LOG_ID_EVENTS, - .tid = (uint16_t)gettid(), - .realtime.tv_sec = (uint32_t)ts.tv_sec, - .realtime.tv_nsec = (uint32_t)ts.tv_nsec, + .id = LOG_ID_EVENTS, + .tid = static_cast<uint16_t>(android::base::GetThreadId()), + .realtime.tv_sec = static_cast<uint32_t>(ts.tv_sec), + .realtime.tv_nsec = static_cast<uint32_t>(ts.tv_nsec), }; uint32_t outTag = TAG_DEF_LOG_TAG; @@ -682,7 +665,7 @@ void LogTags::WritePersistEventLogTags(uint32_t tag, uid_t uid, } } - lastTotal = android::sizesTotal(); + lastTotal = LogStatistics::sizesTotal(); if (!lastTotal) ++lastTotal; // record totals for next watermark. diff --git a/logd/LogTags.h b/logd/LogTags.h index e4d165a7c..cce700c26 100644 --- a/logd/LogTags.h +++ b/logd/LogTags.h @@ -14,13 +14,13 @@ * limitations under the License. */ -#ifndef _LOGD_LOG_TAGS_H__ -#define _LOGD_LOG_TAGS_H__ +#pragma once #include <string> #include <unordered_map> #include <unordered_set> +#include <private/android_filesystem_config.h> #include <utils/RWLock.h> class LogTags { @@ -120,5 +120,3 @@ class LogTags { std::string formatGetEventTag(uid_t uid, const char* name, const char* format); }; - -#endif // _LOGD_LOG_TAGS_H__ diff --git a/logd/LogTimes.cpp b/logd/LogTimes.cpp deleted file mode 100644 index 208d67f49..000000000 --- a/logd/LogTimes.cpp +++ /dev/null @@ -1,243 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <errno.h> -#include <string.h> -#include <sys/prctl.h> - -#include <private/android_logger.h> - -#include "FlushCommand.h" -#include "LogBuffer.h" -#include "LogReader.h" -#include "LogTimes.h" - -pthread_mutex_t LogTimeEntry::timesLock = PTHREAD_MUTEX_INITIALIZER; - -LogTimeEntry::LogTimeEntry(LogReader& reader, SocketClient* client, - bool nonBlock, unsigned long tail, log_mask_t logMask, - pid_t pid, log_time start, uint64_t timeout) - : leadingDropped(false), - mReader(reader), - mLogMask(logMask), - mPid(pid), - mCount(0), - mTail(tail), - mIndex(0), - mClient(client), - mStart(start), - mNonBlock(nonBlock), - mEnd(log_time(android_log_clockid())) { - mTimeout.tv_sec = timeout / NS_PER_SEC; - mTimeout.tv_nsec = timeout % NS_PER_SEC; - memset(mLastTid, 0, sizeof(mLastTid)); - pthread_cond_init(&threadTriggeredCondition, nullptr); - cleanSkip_Locked(); -} - -bool LogTimeEntry::startReader_Locked() { - pthread_attr_t attr; - - if (!pthread_attr_init(&attr)) { - if (!pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED)) { - if (!pthread_create(&mThread, &attr, LogTimeEntry::threadStart, - this)) { - pthread_attr_destroy(&attr); - return true; - } - } - pthread_attr_destroy(&attr); - } - - return false; -} - -void* LogTimeEntry::threadStart(void* obj) { - prctl(PR_SET_NAME, "logd.reader.per"); - - LogTimeEntry* me = reinterpret_cast<LogTimeEntry*>(obj); - - SocketClient* client = me->mClient; - - LogBuffer& logbuf = me->mReader.logbuf(); - - bool privileged = FlushCommand::hasReadLogs(client); - bool security = FlushCommand::hasSecurityLogs(client); - - me->leadingDropped = true; - - wrlock(); - - log_time start = me->mStart; - - while (!me->mRelease) { - if (me->mTimeout.tv_sec || me->mTimeout.tv_nsec) { - if (pthread_cond_timedwait(&me->threadTriggeredCondition, - ×Lock, &me->mTimeout) == ETIMEDOUT) { - me->mTimeout.tv_sec = 0; - me->mTimeout.tv_nsec = 0; - } - if (me->mRelease) { - break; - } - } - - unlock(); - - if (me->mTail) { - logbuf.flushTo(client, start, nullptr, privileged, security, - FilterFirstPass, me); - me->leadingDropped = true; - } - start = logbuf.flushTo(client, start, me->mLastTid, privileged, - security, FilterSecondPass, me); - - wrlock(); - - if (start == LogBufferElement::FLUSH_ERROR) { - break; - } - - me->mStart = start + log_time(0, 1); - - if (me->mNonBlock || me->mRelease) { - break; - } - - me->cleanSkip_Locked(); - - if (!me->mTimeout.tv_sec && !me->mTimeout.tv_nsec) { - pthread_cond_wait(&me->threadTriggeredCondition, ×Lock); - } - } - - LogReader& reader = me->mReader; - reader.release(client); - - client->decRef(); - - LastLogTimes& times = reader.logbuf().mTimes; - auto it = - std::find_if(times.begin(), times.end(), - [&me](const auto& other) { return other.get() == me; }); - - if (it != times.end()) { - times.erase(it); - } - - unlock(); - - return nullptr; -} - -// A first pass to count the number of elements -int LogTimeEntry::FilterFirstPass(const LogBufferElement* element, void* obj) { - LogTimeEntry* me = reinterpret_cast<LogTimeEntry*>(obj); - - LogTimeEntry::wrlock(); - - if (me->leadingDropped) { - if (element->getDropped()) { - LogTimeEntry::unlock(); - return false; - } - me->leadingDropped = false; - } - - if (me->mCount == 0) { - me->mStart = element->getRealTime(); - } - - if ((!me->mPid || (me->mPid == element->getPid())) && - (me->isWatching(element->getLogId()))) { - ++me->mCount; - } - - LogTimeEntry::unlock(); - - return false; -} - -// A second pass to send the selected elements -int LogTimeEntry::FilterSecondPass(const LogBufferElement* element, void* obj) { - LogTimeEntry* me = reinterpret_cast<LogTimeEntry*>(obj); - - LogTimeEntry::wrlock(); - - me->mStart = element->getRealTime(); - - if (me->skipAhead[element->getLogId()]) { - me->skipAhead[element->getLogId()]--; - goto skip; - } - - if (me->leadingDropped) { - if (element->getDropped()) { - goto skip; - } - me->leadingDropped = false; - } - - // Truncate to close race between first and second pass - if (me->mNonBlock && me->mTail && (me->mIndex >= me->mCount)) { - goto stop; - } - - if (!me->isWatching(element->getLogId())) { - goto skip; - } - - if (me->mPid && (me->mPid != element->getPid())) { - goto skip; - } - - if (me->mRelease) { - goto stop; - } - - if (!me->mTail) { - goto ok; - } - - ++me->mIndex; - - if ((me->mCount > me->mTail) && (me->mIndex <= (me->mCount - me->mTail))) { - goto skip; - } - - if (!me->mNonBlock) { - me->mTail = 0; - } - -ok: - if (!me->skipAhead[element->getLogId()]) { - LogTimeEntry::unlock(); - return true; - } -// FALLTHRU - -skip: - LogTimeEntry::unlock(); - return false; - -stop: - LogTimeEntry::unlock(); - return -1; -} - -void LogTimeEntry::cleanSkip_Locked(void) { - memset(skipAhead, 0, sizeof(skipAhead)); -} diff --git a/logd/LogTimes.h b/logd/LogTimes.h deleted file mode 100644 index 9f6f20335..000000000 --- a/logd/LogTimes.h +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright (C) 2012-2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _LOGD_LOG_TIMES_H__ -#define _LOGD_LOG_TIMES_H__ - -#include <pthread.h> -#include <sys/socket.h> -#include <sys/types.h> -#include <time.h> - -#include <list> -#include <memory> - -#include <log/log.h> -#include <sysutils/SocketClient.h> - -typedef unsigned int log_mask_t; - -class LogReader; -class LogBufferElement; - -class LogTimeEntry { - static pthread_mutex_t timesLock; - bool mRelease = false; - bool leadingDropped; - pthread_cond_t threadTriggeredCondition; - pthread_t mThread; - LogReader& mReader; - static void* threadStart(void* me); - const log_mask_t mLogMask; - const pid_t mPid; - unsigned int skipAhead[LOG_ID_MAX]; - pid_t mLastTid[LOG_ID_MAX]; - unsigned long mCount; - unsigned long mTail; - unsigned long mIndex; - - public: - LogTimeEntry(LogReader& reader, SocketClient* client, bool nonBlock, - unsigned long tail, log_mask_t logMask, pid_t pid, - log_time start, uint64_t timeout); - - SocketClient* mClient; - log_time mStart; - struct timespec mTimeout; - const bool mNonBlock; - const log_time mEnd; // only relevant if mNonBlock - - // Protect List manipulations - static void wrlock(void) { - pthread_mutex_lock(×Lock); - } - static void rdlock(void) { - pthread_mutex_lock(×Lock); - } - static void unlock(void) { - pthread_mutex_unlock(×Lock); - } - - bool startReader_Locked(); - - void triggerReader_Locked(void) { - pthread_cond_signal(&threadTriggeredCondition); - } - - void triggerSkip_Locked(log_id_t id, unsigned int skip) { - skipAhead[id] = skip; - } - void cleanSkip_Locked(void); - - void release_Locked(void) { - // gracefully shut down the socket. - shutdown(mClient->getSocket(), SHUT_RDWR); - mRelease = true; - pthread_cond_signal(&threadTriggeredCondition); - } - - bool isWatching(log_id_t id) const { - return mLogMask & (1 << id); - } - bool isWatchingMultiple(log_mask_t logMask) const { - return mLogMask & logMask; - } - // flushTo filter callbacks - static int FilterFirstPass(const LogBufferElement* element, void* me); - static int FilterSecondPass(const LogBufferElement* element, void* me); -}; - -typedef std::list<std::unique_ptr<LogTimeEntry>> LastLogTimes; - -#endif // _LOGD_LOG_TIMES_H__ diff --git a/logd/LogUtils.h b/logd/LogUtils.h index fa9f39895..df78a508a 100644 --- a/logd/LogUtils.h +++ b/logd/LogUtils.h @@ -14,8 +14,7 @@ * limitations under the License. */ -#ifndef _LOGD_LOG_UTILS_H__ -#define _LOGD_LOG_UTILS_H__ +#pragma once #include <sys/cdefs.h> #include <sys/types.h> @@ -31,17 +30,13 @@ namespace android { // Furnished in main.cpp. Caller must own and free returned value char* uidToName(uid_t uid); -void prdebug(const char* fmt, ...) __printflike(1, 2); -// Furnished in LogStatistics.cpp. -size_t sizesTotal(); // Caller must own and free returned value char* pidToName(pid_t pid); char* tidToName(pid_t tid); // Furnished in LogTags.cpp. Thread safe. const char* tagToName(uint32_t tag); -void ReReadEventLogTags(); // Furnished by LogKlog.cpp char* log_strntok_r(char* s, ssize_t& len, char*& saveptr, ssize_t& sublen); @@ -64,13 +59,21 @@ static inline const char* strnstr(const char* s, ssize_t len, } } -// Furnished in LogCommand.cpp -bool clientHasLogCredentials(uid_t uid, gid_t gid, pid_t pid); -bool clientHasLogCredentials(SocketClient* cli); +// Returns true if the log buffer is meant for binary logs. +static inline bool IsBinary(log_id_t log_id) { + return log_id == LOG_ID_EVENTS || log_id == LOG_ID_STATS || log_id == LOG_ID_SECURITY; +} + +// Returns the numeric log tag for binary log messages. +static inline uint32_t MsgToTag(const char* msg, uint16_t msg_len) { + if (msg_len < sizeof(android_event_header_t)) { + return 0; + } + + return reinterpret_cast<const android_event_header_t*>(msg)->tag; +} static inline bool worstUidEnabledForLogid(log_id_t id) { return (id == LOG_ID_MAIN) || (id == LOG_ID_SYSTEM) || (id == LOG_ID_RADIO) || (id == LOG_ID_EVENTS); } - -#endif // _LOGD_LOG_UTILS_H__ diff --git a/logd/LogWhiteBlackList.cpp b/logd/LogWhiteBlackList.cpp deleted file mode 100644 index 9d762dca4..000000000 --- a/logd/LogWhiteBlackList.cpp +++ /dev/null @@ -1,271 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <ctype.h> - -#include <android-base/stringprintf.h> -#include <cutils/properties.h> - -#include "LogWhiteBlackList.h" - -// White and Black list - -Prune::Prune(uid_t uid, pid_t pid) : mUid(uid), mPid(pid) { -} - -int Prune::cmp(uid_t uid, pid_t pid) const { - if ((mUid == uid_all) || (mUid == uid)) { - if (mPid == pid_all) { - return 0; - } - return pid - mPid; - } - return uid - mUid; -} - -std::string Prune::format() { - if (mUid != uid_all) { - if (mPid != pid_all) { - return android::base::StringPrintf("%u/%u", mUid, mPid); - } - return android::base::StringPrintf("%u", mUid); - } - if (mPid != pid_all) { - return android::base::StringPrintf("/%u", mPid); - } - // NB: mPid == pid_all can not happen if mUid == uid_all - return std::string("/"); -} - -PruneList::PruneList() { - init(nullptr); -} - -PruneList::~PruneList() { - PruneCollection::iterator it; - for (it = mNice.begin(); it != mNice.end();) { - it = mNice.erase(it); - } - for (it = mNaughty.begin(); it != mNaughty.end();) { - it = mNaughty.erase(it); - } -} - -int PruneList::init(const char* str) { - mWorstUidEnabled = true; - mWorstPidOfSystemEnabled = true; - PruneCollection::iterator it; - for (it = mNice.begin(); it != mNice.end();) { - it = mNice.erase(it); - } - for (it = mNaughty.begin(); it != mNaughty.end();) { - it = mNaughty.erase(it); - } - - static const char _default[] = "default"; - // default here means take ro.logd.filter, persist.logd.filter then - // internal default in that order. - if (str && !strcmp(str, _default)) { - str = nullptr; - } - static const char _disable[] = "disable"; - if (str && !strcmp(str, _disable)) { - str = ""; - } - - std::string filter; - - if (str) { - filter = str; - } else { - char property[PROPERTY_VALUE_MAX]; - property_get("ro.logd.filter", property, _default); - filter = property; - property_get("persist.logd.filter", property, filter.c_str()); - // default here means take ro.logd.filter - if (strcmp(property, _default)) { - filter = property; - } - } - - // default here means take internal default. - if (filter == _default) { - // See README.property for description of filter format - filter = "~! ~1000/!"; - } - if (filter == _disable) { - filter = ""; - } - - mWorstUidEnabled = false; - mWorstPidOfSystemEnabled = false; - - for (str = filter.c_str(); *str; ++str) { - if (isspace(*str)) { - continue; - } - - PruneCollection* list; - if ((*str == '~') || (*str == '!')) { // ~ supported, ! undocumented - ++str; - // special case, translates to worst UID at priority in blacklist - if (*str == '!') { - mWorstUidEnabled = true; - ++str; - if (!*str) { - break; - } - if (!isspace(*str)) { - return 1; - } - continue; - } - // special case, translated to worst PID of System at priority - static const char worstPid[] = "1000/!"; - if (!strncmp(str, worstPid, sizeof(worstPid) - 1)) { - mWorstPidOfSystemEnabled = true; - str += sizeof(worstPid) - 1; - if (!*str) { - break; - } - if (!isspace(*str)) { - return 1; - } - continue; - } - if (!*str) { - return 1; - } - list = &mNaughty; - } else { - list = &mNice; - } - - uid_t uid = Prune::uid_all; - if (isdigit(*str)) { - uid = 0; - do { - uid = uid * 10 + *str++ - '0'; - } while (isdigit(*str)); - } - - pid_t pid = Prune::pid_all; - if (*str == '/') { - ++str; - if (isdigit(*str)) { - pid = 0; - do { - pid = pid * 10 + *str++ - '0'; - } while (isdigit(*str)); - } - } - - if ((uid == Prune::uid_all) && (pid == Prune::pid_all)) { - return 1; - } - - if (*str && !isspace(*str)) { - return 1; - } - - // insert sequentially into list - PruneCollection::iterator it = list->begin(); - while (it != list->end()) { - Prune& p = *it; - int m = uid - p.mUid; - if (m == 0) { - if (p.mPid == p.pid_all) { - break; - } - if ((pid == p.pid_all) && (p.mPid != p.pid_all)) { - it = list->erase(it); - continue; - } - m = pid - p.mPid; - } - if (m <= 0) { - if (m < 0) { - list->insert(it, Prune(uid, pid)); - } - break; - } - ++it; - } - if (it == list->end()) { - list->push_back(Prune(uid, pid)); - } - if (!*str) { - break; - } - } - - return 0; -} - -std::string PruneList::format() { - static const char nice_format[] = " %s"; - const char* fmt = nice_format + 1; - - std::string string; - - if (mWorstUidEnabled) { - string = "~!"; - fmt = nice_format; - if (mWorstPidOfSystemEnabled) { - string += " ~1000/!"; - } - } - - PruneCollection::iterator it; - - for (it = mNice.begin(); it != mNice.end(); ++it) { - string += android::base::StringPrintf(fmt, (*it).format().c_str()); - fmt = nice_format; - } - - static const char naughty_format[] = " ~%s"; - fmt = naughty_format + (*fmt != ' '); - for (it = mNaughty.begin(); it != mNaughty.end(); ++it) { - string += android::base::StringPrintf(fmt, (*it).format().c_str()); - fmt = naughty_format; - } - - return string; -} - -// ToDo: Lists are in sorted order, Prune->cmp() returns + or - -// If there is scaling issues, resort to a better algorithm than linear -// based on these assumptions. - -bool PruneList::naughty(LogBufferElement* element) { - PruneCollection::iterator it; - for (it = mNaughty.begin(); it != mNaughty.end(); ++it) { - if (!(*it).cmp(element)) { - return true; - } - } - return false; -} - -bool PruneList::nice(LogBufferElement* element) { - PruneCollection::iterator it; - for (it = mNice.begin(); it != mNice.end(); ++it) { - if (!(*it).cmp(element)) { - return true; - } - } - return false; -} diff --git a/logd/LogWhiteBlackList.h b/logd/LogWhiteBlackList.h deleted file mode 100644 index 6e9893b6d..000000000 --- a/logd/LogWhiteBlackList.h +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _LOGD_LOG_WHITE_BLACK_LIST_H__ -#define _LOGD_LOG_WHITE_BLACK_LIST_H__ - -#include <sys/types.h> - -#include <string.h> -#include <list> - -#include "LogBufferElement.h" - -// White and Blacklist - -class Prune { - friend class PruneList; - - const uid_t mUid; - const pid_t mPid; - int cmp(uid_t uid, pid_t pid) const; - - public: - static const uid_t uid_all = (uid_t)-1; - static const pid_t pid_all = (pid_t)-1; - - Prune(uid_t uid, pid_t pid); - - uid_t getUid() const { - return mUid; - } - pid_t getPid() const { - return mPid; - } - - int cmp(LogBufferElement* e) const { - return cmp(e->getUid(), e->getPid()); - } - - std::string format(); -}; - -typedef std::list<Prune> PruneCollection; - -class PruneList { - PruneCollection mNaughty; - PruneCollection mNice; - bool mWorstUidEnabled; - bool mWorstPidOfSystemEnabled; - - public: - PruneList(); - ~PruneList(); - - int init(const char* str); - - bool naughty(LogBufferElement* element); - bool naughty(void) { - return !mNaughty.empty(); - } - bool nice(LogBufferElement* element); - bool nice(void) { - return !mNice.empty(); - } - bool worstUidEnabled() const { - return mWorstUidEnabled; - } - bool worstPidOfSystemEnabled() const { - return mWorstPidOfSystemEnabled; - } - - std::string format(); -}; - -#endif // _LOGD_LOG_WHITE_BLACK_LIST_H__ diff --git a/logd/LogWriter.h b/logd/LogWriter.h new file mode 100644 index 000000000..d43c604ed --- /dev/null +++ b/logd/LogWriter.h @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <string> + +#include <log/log_read.h> + +// An interface for writing logs to a reader. +class LogWriter { + public: + LogWriter(uid_t uid, bool privileged) : uid_(uid), privileged_(privileged) {} + virtual ~LogWriter() {} + + virtual bool Write(const logger_entry& entry, const char* msg) = 0; + virtual void Shutdown() {} + virtual void Release() {} + + virtual std::string name() const = 0; + uid_t uid() const { return uid_; } + + bool privileged() const { return privileged_; } + + private: + uid_t uid_; + + // If this writer sees logs from all UIDs or only its own UID. See clientHasLogCredentials(). + bool privileged_; +};
\ No newline at end of file diff --git a/logd/PruneList.cpp b/logd/PruneList.cpp new file mode 100644 index 000000000..c3859f39f --- /dev/null +++ b/logd/PruneList.cpp @@ -0,0 +1,201 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "PruneList.h" + +#include <ctype.h> + +#include <android-base/logging.h> +#include <android-base/properties.h> +#include <android-base/stringprintf.h> +#include <android-base/strings.h> + +bool Prune::Matches(LogBufferElement* element) const { + return (uid_ == UID_ALL || uid_ == element->uid()) && + (pid_ == PID_ALL || pid_ == element->pid()); +} + +std::string Prune::Format() const { + if (uid_ != UID_ALL) { + if (pid_ != PID_ALL) { + return android::base::StringPrintf("%u/%u", uid_, pid_); + } + return android::base::StringPrintf("%u", uid_); + } + if (pid_ != PID_ALL) { + return android::base::StringPrintf("/%u", pid_); + } + // NB: pid_ == PID_ALL can not happen if uid_ == UID_ALL + return std::string("/"); +} + +PruneList::PruneList() { + Init(nullptr); +} + +bool PruneList::Init(const char* str) { + high_priority_prune_.clear(); + low_priority_prune_.clear(); + + // default here means take ro.logd.filter, persist.logd.filter then internal default in order. + if (str && !strcmp(str, "default")) { + str = nullptr; + } + if (str && !strcmp(str, "disable")) { + str = ""; + } + + std::string filter; + + if (str) { + filter = str; + } else { + filter = android::base::GetProperty("ro.logd.filter", "default"); + auto persist_filter = android::base::GetProperty("persist.logd.filter", "default"); + // default here means take ro.logd.filter + if (persist_filter != "default") { + filter = persist_filter; + } + } + + // default here means take internal default. + if (filter == "default") { + filter = "~! ~1000/!"; + } + if (filter == "disable") { + filter = ""; + } + + worst_uid_enabled_ = false; + worst_pid_of_system_enabled_ = false; + + for (str = filter.c_str(); *str; ++str) { + if (isspace(*str)) { + continue; + } + + std::list<Prune>* list; + if (*str == '~' || *str == '!') { // ~ supported, ! undocumented + ++str; + // special case, prune the worst UID of those using at least 1/8th of the buffer. + if (*str == '!') { + worst_uid_enabled_ = true; + ++str; + if (!*str) { + break; + } + if (!isspace(*str)) { + LOG(ERROR) << "Nothing expected after '~!', but found '" << str << "'"; + return false; + } + continue; + } + // special case, translated to worst PID of System at priority + static const char WORST_SYSTEM_PID[] = "1000/!"; + if (!strncmp(str, WORST_SYSTEM_PID, sizeof(WORST_SYSTEM_PID) - 1)) { + worst_pid_of_system_enabled_ = true; + str += sizeof(WORST_SYSTEM_PID) - 1; + if (!*str) { + break; + } + if (!isspace(*str)) { + LOG(ERROR) << "Nothing expected after '~1000/!', but found '" << str << "'"; + return false; + } + continue; + } + if (!*str) { + LOG(ERROR) << "Expected UID or PID after '~', but found nothing"; + return false; + } + list = &high_priority_prune_; + } else { + list = &low_priority_prune_; + } + + uid_t uid = Prune::UID_ALL; + if (isdigit(*str)) { + uid = 0; + do { + uid = uid * 10 + *str++ - '0'; + } while (isdigit(*str)); + } + + pid_t pid = Prune::PID_ALL; + if (*str == '/') { + ++str; + if (isdigit(*str)) { + pid = 0; + do { + pid = pid * 10 + *str++ - '0'; + } while (isdigit(*str)); + } + } + + if (uid == Prune::UID_ALL && pid == Prune::PID_ALL) { + LOG(ERROR) << "Expected UID/PID combination, but found none"; + return false; + } + + if (*str && !isspace(*str)) { + LOG(ERROR) << "Nothing expected after UID/PID combination, but found '" << str << "'"; + return false; + } + + list->emplace_back(uid, pid); + if (!*str) { + break; + } + } + + return true; +} + +std::string PruneList::Format() const { + std::vector<std::string> prune_rules; + + if (worst_uid_enabled_) { + prune_rules.emplace_back("~!"); + } + if (worst_pid_of_system_enabled_) { + prune_rules.emplace_back("~1000/!"); + } + for (const auto& rule : low_priority_prune_) { + prune_rules.emplace_back(rule.Format()); + } + for (const auto& rule : high_priority_prune_) { + prune_rules.emplace_back("~" + rule.Format()); + } + return android::base::Join(prune_rules, " "); +} + +bool PruneList::IsHighPriority(LogBufferElement* element) const { + for (const auto& rule : high_priority_prune_) { + if (rule.Matches(element)) { + return true; + } + } + return false; +} + +bool PruneList::IsLowPriority(LogBufferElement* element) const { + for (const auto& rule : low_priority_prune_) { + if (rule.Matches(element)) { + return true; + } + } + return false; +} diff --git a/logd/PruneList.h b/logd/PruneList.h new file mode 100644 index 000000000..94de5c520 --- /dev/null +++ b/logd/PruneList.h @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <sys/types.h> + +#include <string.h> +#include <list> + +#include "LogBufferElement.h" + +class Prune { + public: + static const uid_t UID_ALL = (uid_t)-1; + static const pid_t PID_ALL = (pid_t)-1; + + Prune(uid_t uid, pid_t pid) : uid_(uid), pid_(pid) {} + + bool Matches(LogBufferElement* element) const; + std::string Format() const; + + uid_t uid() const { return uid_; } + pid_t pid() const { return pid_; } + + private: + const uid_t uid_; + const pid_t pid_; +}; + +class PruneList { + public: + PruneList(); + + bool Init(const char* str); + std::string Format() const; + + bool IsHighPriority(LogBufferElement* element) const; + bool IsLowPriority(LogBufferElement* element) const; + + bool HasHighPriorityPruneRules() const { return !high_priority_prune_.empty(); } + bool HasLowPriorityPruneRules() const { return !low_priority_prune_.empty(); } + + bool worst_uid_enabled() const { return worst_uid_enabled_; } + bool worst_pid_of_system_enabled() const { return worst_pid_of_system_enabled_; } + + private: + std::list<Prune> high_priority_prune_; + std::list<Prune> low_priority_prune_; + + bool worst_uid_enabled_; + bool worst_pid_of_system_enabled_; +}; diff --git a/logd/README.property b/logd/README.property index 1b7e1652c..ab9c4d435 100644 --- a/logd/README.property +++ b/logd/README.property @@ -44,10 +44,6 @@ ro.logd.filter string "~! ~1000/!" default for persist.logd.filter. oldest entries of chattiest UID, and the chattiest PID of system (1000, or AID_SYSTEM). -persist.logd.timestamp string ro The recording timestamp source. - "m[onotonic]" is the only supported - key character, otherwise realtime. -ro.logd.timestamp string realtime default for persist.logd.timestamp log.tag string persist The global logging level, VERBOSE, DEBUG, INFO, WARN, ERROR, ASSERT or SILENT. Only the first character is @@ -69,8 +65,8 @@ NB: - number - support multipliers (K or M) for convenience. Range is limited to between 64K and 256M for log buffer sizes. Individual log buffer ids such as main, system, ... override global default. -- Pruning filter is of form of a space-separated list of [~][UID][/PID] - references, where '~' prefix means to blacklist otherwise whitelist. For - blacklisting, UID or PID may be a '!' to instead reference the chattiest - client, with the restriction that the PID must be in the UID group 1000 - (system or AID_SYSTEM). +- Pruning filter rules are specified as UID, UID/PID or /PID. A '~' prefix indicates that elements + matching the rule should be pruned with higher priority otherwise they're pruned with lower + priority. All other pruning activity is oldest first. Special case ~! represents an automatic + pruning for the noisiest UID as determined by the current statistics. Special case ~1000/! + represents pruning of the worst PID within AID_SYSTEM when AID_SYSTEM is the noisiest UID. diff --git a/logd/SerializedFlushToState.cpp b/logd/SerializedFlushToState.cpp new file mode 100644 index 000000000..6e2e8b053 --- /dev/null +++ b/logd/SerializedFlushToState.cpp @@ -0,0 +1,158 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "SerializedFlushToState.h" + +#include <android-base/logging.h> + +SerializedFlushToState::SerializedFlushToState(uint64_t start, LogMask log_mask) + : FlushToState(start, log_mask) { + log_id_for_each(i) { + if (((1 << i) & log_mask) == 0) { + continue; + } + logs_needed_from_next_position_[i] = true; + } +} + +SerializedFlushToState::~SerializedFlushToState() { + log_id_for_each(i) { + if (log_positions_[i]) { + log_positions_[i]->buffer_it->DecReaderRefCount(true); + } + } +} + +void SerializedFlushToState::CreateLogPosition(log_id_t log_id) { + CHECK(!logs_[log_id].empty()); + LogPosition log_position; + auto it = logs_[log_id].begin(); + while (it != logs_[log_id].end() && start() > it->highest_sequence_number()) { + ++it; + } + if (it == logs_[log_id].end()) { + --it; + } + it->IncReaderRefCount(); + log_position.buffer_it = it; + + // Find the offset of the first log with sequence number >= start(). + int read_offset = 0; + while (read_offset < it->write_offset()) { + const auto* entry = it->log_entry(read_offset); + if (entry->sequence() >= start()) { + break; + } + read_offset += entry->total_len(); + } + log_position.read_offset = read_offset; + + log_positions_[log_id].emplace(log_position); +} + +void SerializedFlushToState::AddMinHeapEntry(log_id_t log_id) { + auto& buffer_it = log_positions_[log_id]->buffer_it; + auto read_offset = log_positions_[log_id]->read_offset; + + // If there is another log to read in this buffer, add it to the min heap. + if (read_offset < buffer_it->write_offset()) { + auto* entry = buffer_it->log_entry(read_offset); + min_heap_.emplace(log_id, entry); + } else if (read_offset == buffer_it->write_offset()) { + // If there are no more logs to read in this buffer and it's the last buffer, then + // set logs_needed_from_next_position_ to wait until more logs get logged. + if (buffer_it == std::prev(logs_[log_id].end())) { + logs_needed_from_next_position_[log_id] = true; + } else { + // Otherwise, if there is another buffer piece, move to that and do the same check. + buffer_it->DecReaderRefCount(true); + ++buffer_it; + buffer_it->IncReaderRefCount(); + log_positions_[log_id]->read_offset = 0; + if (buffer_it->write_offset() == 0) { + logs_needed_from_next_position_[log_id] = true; + } else { + auto* entry = buffer_it->log_entry(0); + min_heap_.emplace(log_id, entry); + } + } + } else { + // read_offset > buffer_it->write_offset() should never happen. + CHECK(false); + } +} + +void SerializedFlushToState::CheckForNewLogs() { + log_id_for_each(i) { + if (!logs_needed_from_next_position_[i]) { + continue; + } + if (!log_positions_[i]) { + if (logs_[i].empty()) { + continue; + } + CreateLogPosition(i); + } + logs_needed_from_next_position_[i] = false; + // If it wasn't possible to insert, logs_needed_from_next_position will be set back to true. + AddMinHeapEntry(i); + } +} + +MinHeapElement SerializedFlushToState::PopNextUnreadLog() { + auto top = min_heap_.top(); + min_heap_.pop(); + + auto* entry = top.entry; + auto log_id = top.log_id; + + log_positions_[log_id]->read_offset += entry->total_len(); + + AddMinHeapEntry(log_id); + + return top; +} + +void SerializedFlushToState::Prune(log_id_t log_id, + const std::list<SerializedLogChunk>::iterator& buffer_it) { + // If we don't have a position for this log or if we're not referencing buffer_it, ignore. + if (!log_positions_[log_id].has_value() || log_positions_[log_id]->buffer_it != buffer_it) { + return; + } + + // // Decrease the ref count since we're deleting our reference. + buffer_it->DecReaderRefCount(false); + + // Delete in the reference. + log_positions_[log_id].reset(); + + // Remove the MinHeapElement referencing log_id, if it exists, but retain the others. + std::vector<MinHeapElement> old_elements; + while (!min_heap_.empty()) { + auto& element = min_heap_.top(); + if (element.log_id != log_id) { + old_elements.emplace_back(element); + } + min_heap_.pop(); + } + for (auto&& element : old_elements) { + min_heap_.emplace(element); + } + + // Finally set logs_needed_from_next_position_, so CheckForNewLogs() will re-create the + // log_position_ object during the next read. + logs_needed_from_next_position_[log_id] = true; +} diff --git a/logd/SerializedFlushToState.h b/logd/SerializedFlushToState.h new file mode 100644 index 000000000..74b3de55c --- /dev/null +++ b/logd/SerializedFlushToState.h @@ -0,0 +1,101 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <bitset> +#include <list> +#include <queue> + +#include "LogBuffer.h" +#include "SerializedLogChunk.h" +#include "SerializedLogEntry.h" + +struct LogPosition { + std::list<SerializedLogChunk>::iterator buffer_it; + int read_offset; +}; + +struct MinHeapElement { + MinHeapElement(log_id_t log_id, const SerializedLogEntry* entry) + : log_id(log_id), entry(entry) {} + log_id_t log_id; + const SerializedLogEntry* entry; + // The change of comparison operators is intentional, std::priority_queue uses operator<() to + // compare but creates a max heap. Since we want a min heap, we return the opposite result. + bool operator<(const MinHeapElement& rhs) const { + return entry->sequence() > rhs.entry->sequence(); + } +}; + +// This class tracks the specific point where a FlushTo client has read through the logs. It +// directly references the std::list<> iterators from the parent SerializedLogBuffer and the offset +// into each log chunk where it has last read. All interactions with this class, except for its +// construction, must be done with SerializedLogBuffer::lock_ held. No log chunks that it +// references may be pruned, which is handled by ensuring prune does not touch any log chunk with +// highest sequence number greater or equal to start(). +class SerializedFlushToState : public FlushToState { + public: + // Initializes this state object. For each log buffer set in log_mask, this sets + // logs_needed_from_next_position_. + SerializedFlushToState(uint64_t start, LogMask log_mask); + + // Decrease the reference of all referenced logs. This happens when a reader is disconnected. + ~SerializedFlushToState() override; + + // We can't hold SerializedLogBuffer::lock_ in the constructor, so we must initialize logs here. + void InitializeLogs(std::list<SerializedLogChunk>* logs) { + if (logs_ == nullptr) logs_ = logs; + } + + // Checks to see if any log buffers set in logs_needed_from_next_position_ have new logs and + // calls AddMinHeapEntry() if so. + void CheckForNewLogs(); + + bool HasUnreadLogs() { return !min_heap_.empty(); } + + // Pops the next unread log from the min heap. Add the next log for that log_id to the min heap + // if one is available otherwise set logs_needed_from_next_position_ to indicate that we're + // waiting for more logs. + MinHeapElement PopNextUnreadLog(); + + // If the parent log buffer prunes logs, the reference that this class contains may become + // invalid, so this must be called first to drop the reference to buffer_it, if any. + void Prune(log_id_t log_id, const std::list<SerializedLogChunk>::iterator& buffer_it); + + private: + // If there is a log in the serialized log buffer for `log_id` at the read_offset, add it to the + // min heap for reading, otherwise set logs_needed_from_next_position_ to indicate that we're + // waiting for the next log. + void AddMinHeapEntry(log_id_t log_id); + + // Create a LogPosition object for the given log_id by searching through the log chunks for the + // first chunk and then first log entry within that chunk that is greater or equal to start(). + void CreateLogPosition(log_id_t log_id); + + std::list<SerializedLogChunk>* logs_ = nullptr; + // An optional structure that contains an iterator to the serialized log buffer and offset into + // it that this logger should handle next. + std::optional<LogPosition> log_positions_[LOG_ID_MAX]; + // A bit for each log that is set if a given log_id has no logs or if this client has read all + // of its logs. In order words: `logs_[i].empty() || (buffer_it == std::prev(logs_.end) && + // next_log_position == logs_write_position_)`. These will be re-checked in each + // loop in case new logs came in. + std::bitset<LOG_ID_MAX> logs_needed_from_next_position_ = {}; + // A min heap that has up to one entry per log buffer, sorted by sequence number, of the next + // element that this reader should read. + std::priority_queue<MinHeapElement> min_heap_; +}; diff --git a/logd/SerializedFlushToStateTest.cpp b/logd/SerializedFlushToStateTest.cpp new file mode 100644 index 000000000..a1d21ac13 --- /dev/null +++ b/logd/SerializedFlushToStateTest.cpp @@ -0,0 +1,254 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "SerializedFlushToState.h" + +#include <map> + +#include <android-base/logging.h> +#include <android-base/stringprintf.h> +#include <android-base/strings.h> +#include <gtest/gtest.h> + +using android::base::Join; +using android::base::StringPrintf; + +constexpr size_t kChunkSize = 3 * 4096; + +class SerializedFlushToStateTest : public testing::Test { + protected: + void SetUp() override { + // This test spams many unneeded INFO logs, so we suppress them. + old_log_severity_ = android::base::SetMinimumLogSeverity(android::base::WARNING); + } + void TearDown() override { android::base::SetMinimumLogSeverity(old_log_severity_); } + + std::string TestReport(const std::vector<uint64_t>& expected, + const std::vector<uint64_t>& read) { + auto sequence_to_log_id = [&](uint64_t sequence) -> int { + for (const auto& [log_id, sequences] : sequence_numbers_per_buffer_) { + if (std::find(sequences.begin(), sequences.end(), sequence) != sequences.end()) { + return log_id; + } + } + return -1; + }; + + std::map<int, std::vector<uint64_t>> missing_sequences; + std::vector<uint64_t> missing_expected; + std::set_difference(expected.begin(), expected.end(), read.begin(), read.end(), + std::back_inserter(missing_expected)); + for (uint64_t sequence : missing_expected) { + int log_id = sequence_to_log_id(sequence); + missing_sequences[log_id].emplace_back(sequence); + } + + std::map<int, std::vector<uint64_t>> extra_sequences; + std::vector<uint64_t> extra_read; + std::set_difference(read.begin(), read.end(), expected.begin(), expected.end(), + std::back_inserter(extra_read)); + for (uint64_t sequence : extra_read) { + int log_id = sequence_to_log_id(sequence); + extra_sequences[log_id].emplace_back(sequence); + } + + std::vector<std::string> errors; + for (const auto& [log_id, sequences] : missing_sequences) { + errors.emplace_back( + StringPrintf("Log id %d missing %zu sequences", log_id, sequences.size())); + } + + for (const auto& [log_id, sequences] : extra_sequences) { + errors.emplace_back( + StringPrintf("Log id %d has extra %zu sequences", log_id, sequences.size())); + } + + return Join(errors, ", "); + } + + // Read sequence numbers in order from SerializedFlushToState for every mask combination and all + // sequence numbers from 0 through the highest logged sequence number + 1. + // This assumes that all of the logs have already been written. + void TestAllReading() { + uint64_t max_sequence = sequence_ + 1; + uint32_t max_mask = (1 << LOG_ID_MAX) - 1; + for (uint64_t sequence = 0; sequence < max_sequence; ++sequence) { + for (uint32_t mask = 0; mask < max_mask; ++mask) { + auto state = SerializedFlushToState{sequence, mask}; + state.InitializeLogs(log_chunks_); + state.CheckForNewLogs(); + TestReading(sequence, mask, state); + } + } + } + + // Similar to TestAllReading() except that it doesn't assume any logs are in the buffer, instead + // it calls write_logs() in a loop for sequence/mask combination. It clears log_chunks_ and + // sequence_numbers_per_buffer_ between calls, such that only the sequence numbers written in + // the previous call to write_logs() are expected. + void TestAllReadingWithFutureMessages(const std::function<bool(int)>& write_logs) { + uint64_t max_sequence = sequence_ + 1; + uint32_t max_mask = (1 << LOG_ID_MAX) - 1; + for (uint64_t sequence = 1; sequence < max_sequence; ++sequence) { + for (uint32_t mask = 1; mask < max_mask; ++mask) { + log_id_for_each(i) { log_chunks_[i].clear(); } + auto state = SerializedFlushToState{sequence, mask}; + state.InitializeLogs(log_chunks_); + int loop_count = 0; + while (write_logs(loop_count++)) { + state.CheckForNewLogs(); + TestReading(sequence, mask, state); + sequence_numbers_per_buffer_.clear(); + } + } + } + } + + void TestReading(uint64_t start, LogMask log_mask, SerializedFlushToState& state) { + std::vector<uint64_t> expected_sequence; + log_id_for_each(i) { + if (((1 << i) & log_mask) == 0) { + continue; + } + for (const auto& sequence : sequence_numbers_per_buffer_[i]) { + if (sequence >= start) { + expected_sequence.emplace_back(sequence); + } + } + } + std::sort(expected_sequence.begin(), expected_sequence.end()); + + std::vector<uint64_t> read_sequence; + + while (state.HasUnreadLogs()) { + auto top = state.PopNextUnreadLog(); + read_sequence.emplace_back(top.entry->sequence()); + } + + EXPECT_TRUE(std::is_sorted(read_sequence.begin(), read_sequence.end())); + + EXPECT_EQ(expected_sequence.size(), read_sequence.size()); + + EXPECT_EQ(expected_sequence, read_sequence) + << "start: " << start << " log_mask: " << log_mask << " " + << TestReport(expected_sequence, read_sequence); + } + + // Add a chunk with the given messages to the a given log buffer. Keep track of the sequence + // numbers for future validation. Optionally mark the block as having finished writing. + void AddChunkWithMessages(int buffer, bool finish_writing, + const std::vector<std::string>& messages) { + auto chunk = SerializedLogChunk{kChunkSize}; + for (const auto& message : messages) { + auto sequence = sequence_++; + sequence_numbers_per_buffer_[buffer].emplace_back(sequence); + ASSERT_TRUE(chunk.CanLog(message.size() + 1)); + chunk.Log(sequence, log_time(), 0, 1, 1, message.c_str(), message.size() + 1); + } + if (finish_writing) { + chunk.FinishWriting(); + } + log_chunks_[buffer].emplace_back(std::move(chunk)); + } + + android::base::LogSeverity old_log_severity_; + std::map<int, std::vector<uint64_t>> sequence_numbers_per_buffer_; + std::list<SerializedLogChunk> log_chunks_[LOG_ID_MAX]; + uint64_t sequence_ = 1; +}; + +// 0: multiple chunks, with variable number of entries, with/without finishing writing +// 1: 1 chunk with 1 log and finished writing +// 2: 1 chunk with 1 log and not finished writing +// 3: 1 chunk with 0 logs and not finished writing +// 4: 1 chunk with 0 logs and finished writing (impossible, but SerializedFlushToState handles it) +// 5-7: 0 chunks +TEST_F(SerializedFlushToStateTest, smoke) { + AddChunkWithMessages(true, 0, {"1st", "2nd"}); + AddChunkWithMessages(true, 1, {"3rd"}); + AddChunkWithMessages(false, 0, {"4th"}); + AddChunkWithMessages(true, 0, {"4th", "5th", "more", "even", "more", "go", "here"}); + AddChunkWithMessages(false, 2, {"6th"}); + AddChunkWithMessages(true, 0, {"7th"}); + AddChunkWithMessages(false, 3, {}); + AddChunkWithMessages(true, 4, {}); + + TestAllReading(); +} + +TEST_F(SerializedFlushToStateTest, random) { + srand(1); + for (int count = 0; count < 20; ++count) { + unsigned int num_messages = 1 + rand() % 15; + auto messages = std::vector<std::string>{num_messages, "same message"}; + + bool compress = rand() % 2; + int buf = rand() % LOG_ID_MAX; + + AddChunkWithMessages(compress, buf, messages); + } + + TestAllReading(); +} + +// Same start as smoke, but we selectively write logs to the buffers and ensure they're read. +TEST_F(SerializedFlushToStateTest, future_writes) { + auto write_logs = [&](int loop_count) { + switch (loop_count) { + case 0: + // Initial writes. + AddChunkWithMessages(true, 0, {"1st", "2nd"}); + AddChunkWithMessages(true, 1, {"3rd"}); + AddChunkWithMessages(false, 0, {"4th"}); + AddChunkWithMessages(true, 0, {"4th", "5th", "more", "even", "more", "go", "here"}); + AddChunkWithMessages(false, 2, {"6th"}); + AddChunkWithMessages(true, 0, {"7th"}); + AddChunkWithMessages(false, 3, {}); + AddChunkWithMessages(true, 4, {}); + break; + case 1: + // Smoke test, add a simple chunk. + AddChunkWithMessages(true, 0, {"1st", "2nd"}); + break; + case 2: + // Add chunks to all but one of the logs. + AddChunkWithMessages(true, 0, {"1st", "2nd"}); + AddChunkWithMessages(true, 1, {"1st", "2nd"}); + AddChunkWithMessages(true, 2, {"1st", "2nd"}); + AddChunkWithMessages(true, 3, {"1st", "2nd"}); + AddChunkWithMessages(true, 4, {"1st", "2nd"}); + AddChunkWithMessages(true, 5, {"1st", "2nd"}); + AddChunkWithMessages(true, 6, {"1st", "2nd"}); + break; + case 3: + // Finally add chunks to all logs. + AddChunkWithMessages(true, 0, {"1st", "2nd"}); + AddChunkWithMessages(true, 1, {"1st", "2nd"}); + AddChunkWithMessages(true, 2, {"1st", "2nd"}); + AddChunkWithMessages(true, 3, {"1st", "2nd"}); + AddChunkWithMessages(true, 4, {"1st", "2nd"}); + AddChunkWithMessages(true, 5, {"1st", "2nd"}); + AddChunkWithMessages(true, 6, {"1st", "2nd"}); + AddChunkWithMessages(true, 7, {"1st", "2nd"}); + break; + default: + return false; + } + return true; + }; + + TestAllReadingWithFutureMessages(write_logs); +} diff --git a/logd/SerializedLogBuffer.cpp b/logd/SerializedLogBuffer.cpp new file mode 100644 index 000000000..70b800f9c --- /dev/null +++ b/logd/SerializedLogBuffer.cpp @@ -0,0 +1,352 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "SerializedLogBuffer.h" + +#include <limits> +#include <thread> + +#include <android-base/logging.h> +#include <android-base/scopeguard.h> + +#include "LogStatistics.h" +#include "SerializedFlushToState.h" + +SerializedLogBuffer::SerializedLogBuffer(LogReaderList* reader_list, LogTags* tags, + LogStatistics* stats) + : reader_list_(reader_list), tags_(tags), stats_(stats) { + Init(); +} + +SerializedLogBuffer::~SerializedLogBuffer() {} + +void SerializedLogBuffer::Init() { + log_id_for_each(i) { + if (SetSize(i, __android_logger_get_buffer_size(i))) { + SetSize(i, LOG_BUFFER_MIN_SIZE); + } + } + + // Release any sleeping reader threads to dump their current content. + auto reader_threads_lock = std::lock_guard{reader_list_->reader_threads_lock()}; + for (const auto& reader_thread : reader_list_->reader_threads()) { + reader_thread->triggerReader_Locked(); + } +} + +bool SerializedLogBuffer::ShouldLog(log_id_t log_id, const char* msg, uint16_t len) { + if (log_id == LOG_ID_SECURITY) { + return true; + } + + int prio = ANDROID_LOG_INFO; + const char* tag = nullptr; + size_t tag_len = 0; + if (IsBinary(log_id)) { + int32_t tag_int = MsgToTag(msg, len); + tag = tags_->tagToName(tag_int); + if (tag) { + tag_len = strlen(tag); + } + } else { + prio = *msg; + tag = msg + 1; + tag_len = strnlen(tag, len - 1); + } + return __android_log_is_loggable_len(prio, tag, tag_len, ANDROID_LOG_VERBOSE); +} + +int SerializedLogBuffer::Log(log_id_t log_id, log_time realtime, uid_t uid, pid_t pid, pid_t tid, + const char* msg, uint16_t len) { + if (log_id >= LOG_ID_MAX || len == 0) { + return -EINVAL; + } + + if (!ShouldLog(log_id, msg, len)) { + stats_->AddTotal(log_id, len); + return -EACCES; + } + + auto sequence = sequence_.fetch_add(1, std::memory_order_relaxed); + + auto lock = std::lock_guard{lock_}; + + if (logs_[log_id].empty()) { + logs_[log_id].push_back(SerializedLogChunk(max_size_[log_id] / 4)); + } + + auto total_len = sizeof(SerializedLogEntry) + len; + if (!logs_[log_id].back().CanLog(total_len)) { + logs_[log_id].back().FinishWriting(); + logs_[log_id].push_back(SerializedLogChunk(max_size_[log_id] / 4)); + } + + auto entry = logs_[log_id].back().Log(sequence, realtime, uid, pid, tid, msg, len); + stats_->Add(entry->ToLogStatisticsElement(log_id)); + + MaybePrune(log_id); + + reader_list_->NotifyNewLog(1 << log_id); + return len; +} + +void SerializedLogBuffer::MaybePrune(log_id_t log_id) { + auto get_total_size = [](const auto& buffer) { + size_t total_size = 0; + for (const auto& chunk : buffer) { + total_size += chunk.PruneSize(); + } + return total_size; + }; + size_t total_size = get_total_size(logs_[log_id]); + if (total_size > max_size_[log_id]) { + Prune(log_id, total_size - max_size_[log_id], 0); + LOG(INFO) << "Pruned Logs from log_id: " << log_id << ", previous size: " << total_size + << " after size: " << get_total_size(logs_[log_id]); + } +} + +// Decompresses the chunks, call LogStatistics::Subtract() on each entry, then delete the chunks and +// the list. Note that the SerializedLogChunk objects have been removed from logs_ and their +// references have been deleted from any SerializedFlushToState objects, so this can be safely done +// without holding lock_. It is done in a separate thread to avoid delaying the writer thread. The +// lambda for the thread takes ownership of the 'chunks' list and thus when the thread ends and the +// lambda is deleted, the objects are deleted. +void SerializedLogBuffer::DeleteLogChunks(std::list<SerializedLogChunk>&& chunks, log_id_t log_id) { + auto delete_thread = std::thread{[chunks = std::move(chunks), log_id, this]() mutable { + for (auto& chunk : chunks) { + chunk.IncReaderRefCount(); + int read_offset = 0; + while (read_offset < chunk.write_offset()) { + auto* entry = chunk.log_entry(read_offset); + stats_->Subtract(entry->ToLogStatisticsElement(log_id)); + read_offset += entry->total_len(); + } + chunk.DecReaderRefCount(false); + } + }}; + delete_thread.detach(); +} + +void SerializedLogBuffer::NotifyReadersOfPrune( + log_id_t log_id, const std::list<SerializedLogChunk>::iterator& chunk) { + for (const auto& reader_thread : reader_list_->reader_threads()) { + auto& state = reinterpret_cast<SerializedFlushToState&>(reader_thread->flush_to_state()); + state.Prune(log_id, chunk); + } +} + +bool SerializedLogBuffer::Prune(log_id_t log_id, size_t bytes_to_free, uid_t uid) { + // Don't prune logs that are newer than the point at which any reader threads are reading from. + LogReaderThread* oldest = nullptr; + auto reader_threads_lock = std::lock_guard{reader_list_->reader_threads_lock()}; + for (const auto& reader_thread : reader_list_->reader_threads()) { + if (!reader_thread->IsWatching(log_id)) { + continue; + } + if (!oldest || oldest->start() > reader_thread->start() || + (oldest->start() == reader_thread->start() && + reader_thread->deadline().time_since_epoch().count() != 0)) { + oldest = reader_thread.get(); + } + } + + auto& log_buffer = logs_[log_id]; + + std::list<SerializedLogChunk> chunks_to_prune; + auto prune_chunks = android::base::make_scope_guard([&chunks_to_prune, log_id, this] { + DeleteLogChunks(std::move(chunks_to_prune), log_id); + }); + + auto it = log_buffer.begin(); + while (it != log_buffer.end()) { + if (oldest != nullptr && it->highest_sequence_number() >= oldest->start()) { + break; + } + + // Increment ahead of time since we're going to splice this iterator from the list. + auto it_to_prune = it++; + + // The sequence number check ensures that all readers have read all logs in this chunk, but + // they may still hold a reference to the chunk to track their last read log_position. + // Notify them to delete the reference. + NotifyReadersOfPrune(log_id, it_to_prune); + + if (uid != 0) { + // Reorder the log buffer to remove logs from the given UID. If there are no logs left + // in the buffer after the removal, delete it. + if (it_to_prune->ClearUidLogs(uid, log_id, stats_)) { + log_buffer.erase(it_to_prune); + } + } else { + size_t buffer_size = it_to_prune->PruneSize(); + chunks_to_prune.splice(chunks_to_prune.end(), log_buffer, it_to_prune); + if (buffer_size >= bytes_to_free) { + return true; + } + bytes_to_free -= buffer_size; + } + } + + // If we've deleted all buffers without bytes_to_free hitting 0, then we're called by Clear() + // and should return true. + if (it == log_buffer.end()) { + return true; + } + + // Otherwise we are stuck due to a reader, so mitigate it. + CHECK(oldest != nullptr); + KickReader(oldest, log_id, bytes_to_free); + return false; +} + +// If the selected reader is blocking our pruning progress, decide on +// what kind of mitigation is necessary to unblock the situation. +void SerializedLogBuffer::KickReader(LogReaderThread* reader, log_id_t id, size_t bytes_to_free) { + if (bytes_to_free >= max_size_[id]) { // +100% + // A misbehaving or slow reader is dropped if we hit too much memory pressure. + LOG(WARNING) << "Kicking blocked reader, " << reader->name() + << ", from LogBuffer::kickMe()"; + reader->release_Locked(); + } else if (reader->deadline().time_since_epoch().count() != 0) { + // Allow a blocked WRAP deadline reader to trigger and start reporting the log data. + reader->triggerReader_Locked(); + } else { + // Tell slow reader to skip entries to catch up. + unsigned long prune_rows = bytes_to_free / 300; + LOG(WARNING) << "Skipping " << prune_rows << " entries from slow reader, " << reader->name() + << ", from LogBuffer::kickMe()"; + reader->triggerSkip_Locked(id, prune_rows); + } +} + +std::unique_ptr<FlushToState> SerializedLogBuffer::CreateFlushToState(uint64_t start, + LogMask log_mask) { + return std::make_unique<SerializedFlushToState>(start, log_mask); +} + +bool SerializedLogBuffer::FlushTo( + LogWriter* writer, FlushToState& abstract_state, + const std::function<FilterResult(log_id_t log_id, pid_t pid, uint64_t sequence, + log_time realtime)>& filter) { + auto lock = std::unique_lock{lock_}; + + auto& state = reinterpret_cast<SerializedFlushToState&>(abstract_state); + state.InitializeLogs(logs_); + state.CheckForNewLogs(); + + while (state.HasUnreadLogs()) { + MinHeapElement top = state.PopNextUnreadLog(); + auto* entry = top.entry; + auto log_id = top.log_id; + + if (entry->sequence() < state.start()) { + continue; + } + state.set_start(entry->sequence()); + + if (!writer->privileged() && entry->uid() != writer->uid()) { + continue; + } + + if (filter) { + auto ret = filter(log_id, entry->pid(), entry->sequence(), entry->realtime()); + if (ret == FilterResult::kSkip) { + continue; + } + if (ret == FilterResult::kStop) { + break; + } + } + + lock.unlock(); + // We never prune logs equal to or newer than any LogReaderThreads' `start` value, so the + // `entry` pointer is safe here without the lock + if (!entry->Flush(writer, log_id)) { + return false; + } + lock.lock(); + + // Since we released the log above, buffers that aren't in our min heap may now have had + // logs added, so re-check them. + state.CheckForNewLogs(); + } + + state.set_start(state.start() + 1); + return true; +} + +bool SerializedLogBuffer::Clear(log_id_t id, uid_t uid) { + // Try three times to clear, then disconnect the readers and try one final time. + for (int retry = 0; retry < 3; ++retry) { + { + auto lock = std::lock_guard{lock_}; + bool prune_success = Prune(id, ULONG_MAX, uid); + if (prune_success) { + return true; + } + } + sleep(1); + } + // Check if it is still busy after the sleep, we try to prune one entry, not another clear run, + // so we are looking for the quick side effect of the return value to tell us if we have a + // _blocked_ reader. + bool busy = false; + { + auto lock = std::lock_guard{lock_}; + busy = !Prune(id, 1, uid); + } + // It is still busy, disconnect all readers. + if (busy) { + auto reader_threads_lock = std::lock_guard{reader_list_->reader_threads_lock()}; + for (const auto& reader_thread : reader_list_->reader_threads()) { + if (reader_thread->IsWatching(id)) { + LOG(WARNING) << "Kicking blocked reader, " << reader_thread->name() + << ", from LogBuffer::clear()"; + reader_thread->release_Locked(); + } + } + } + auto lock = std::lock_guard{lock_}; + return Prune(id, ULONG_MAX, uid); +} + +unsigned long SerializedLogBuffer::GetSize(log_id_t id) { + auto lock = std::lock_guard{lock_}; + size_t retval = 2 * max_size_[id] / 3; // See the comment in SetSize(). + return retval; +} + +// New SerializedLogChunk objects will be allocated according to the new size, but older one are +// unchanged. MaybePrune() is called on the log buffer to reduce it to an appropriate size if the +// new size is lower. +// ChattyLogBuffer/SimpleLogBuffer don't consider the 'Overhead' of LogBufferElement or the +// std::list<> overhead as part of the log size. SerializedLogBuffer does by its very nature, so +// the 'size' metric is not compatible between them. Their actual memory usage is between 1.5x and +// 2x of what they claim to use, so we conservatively set our internal size as size + size / 2. +int SerializedLogBuffer::SetSize(log_id_t id, unsigned long size) { + // Reasonable limits ... + if (!__android_logger_valid_buffer_size(size)) { + return -1; + } + + auto lock = std::lock_guard{lock_}; + max_size_[id] = size + size / 2; + + MaybePrune(id); + + return 0; +} diff --git a/logd/SerializedLogBuffer.h b/logd/SerializedLogBuffer.h new file mode 100644 index 000000000..346f51f68 --- /dev/null +++ b/logd/SerializedLogBuffer.h @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <atomic> +#include <bitset> +#include <list> +#include <mutex> +#include <queue> +#include <vector> + +#include <android-base/thread_annotations.h> + +#include "LogBuffer.h" +#include "LogReaderList.h" +#include "LogStatistics.h" +#include "LogTags.h" +#include "SerializedLogChunk.h" +#include "SerializedLogEntry.h" +#include "rwlock.h" + +class SerializedLogBuffer final : public LogBuffer { + public: + SerializedLogBuffer(LogReaderList* reader_list, LogTags* tags, LogStatistics* stats); + ~SerializedLogBuffer(); + void Init() override; + + int Log(log_id_t log_id, log_time realtime, uid_t uid, pid_t pid, pid_t tid, const char* msg, + uint16_t len) override; + std::unique_ptr<FlushToState> CreateFlushToState(uint64_t start, LogMask log_mask) override; + bool FlushTo(LogWriter* writer, FlushToState& state, + const std::function<FilterResult(log_id_t log_id, pid_t pid, uint64_t sequence, + log_time realtime)>& filter) override; + + bool Clear(log_id_t id, uid_t uid) override; + unsigned long GetSize(log_id_t id) override; + int SetSize(log_id_t id, unsigned long size) override; + + uint64_t sequence() const override { return sequence_.load(std::memory_order_relaxed); } + + private: + bool ShouldLog(log_id_t log_id, const char* msg, uint16_t len); + void MaybePrune(log_id_t log_id) REQUIRES(lock_); + bool Prune(log_id_t log_id, size_t bytes_to_free, uid_t uid) REQUIRES(lock_); + void KickReader(LogReaderThread* reader, log_id_t id, size_t bytes_to_free) + REQUIRES_SHARED(lock_); + void NotifyReadersOfPrune(log_id_t log_id, const std::list<SerializedLogChunk>::iterator& chunk) + REQUIRES(reader_list_->reader_threads_lock()); + void DeleteLogChunks(std::list<SerializedLogChunk>&& chunks, log_id_t log_id); + + LogReaderList* reader_list_; + LogTags* tags_; + LogStatistics* stats_; + + unsigned long max_size_[LOG_ID_MAX] GUARDED_BY(lock_) = {}; + std::list<SerializedLogChunk> logs_[LOG_ID_MAX] GUARDED_BY(lock_); + RwLock lock_; + + std::atomic<uint64_t> sequence_ = 1; +}; diff --git a/logd/SerializedLogChunk.cpp b/logd/SerializedLogChunk.cpp new file mode 100644 index 000000000..2516003c1 --- /dev/null +++ b/logd/SerializedLogChunk.cpp @@ -0,0 +1,114 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "SerializedLogChunk.h" + +#include <android-base/logging.h> + +#include "CompressionEngine.h" + +SerializedLogChunk::~SerializedLogChunk() { + CHECK_EQ(reader_ref_count_, 0U); +} + +void SerializedLogChunk::Compress() { + if (compressed_log_.empty()) { + CompressionEngine::GetInstance().Compress({contents_.data(), write_offset_}, + compressed_log_); + LOG(INFO) << "Compressed Log, buffer max size: " << contents_.size() + << " size used: " << write_offset_ + << " compressed size: " << compressed_log_.size(); + } + contents_.resize(0); +} + +// TODO: Develop a better reference counting strategy to guard against the case where the writer is +// much faster than the reader, and we needlessly compess / decompress the logs. +void SerializedLogChunk::IncReaderRefCount() { + if (++reader_ref_count_ != 1 || writer_active_) { + return; + } + CompressionEngine::GetInstance().Decompress(compressed_log_, contents_, write_offset_); +} + +void SerializedLogChunk::DecReaderRefCount(bool compress) { + CHECK_NE(reader_ref_count_, 0U); + if (--reader_ref_count_ != 0) { + return; + } + if (compress && !writer_active_) { + Compress(); + } +} + +bool SerializedLogChunk::ClearUidLogs(uid_t uid, log_id_t log_id, LogStatistics* stats) { + CHECK_EQ(reader_ref_count_, 0U); + if (write_offset_ == 0) { + return true; + } + + IncReaderRefCount(); + + int read_offset = 0; + int new_write_offset = 0; + while (read_offset < write_offset_) { + const auto* entry = log_entry(read_offset); + if (entry->uid() == uid) { + read_offset += entry->total_len(); + if (stats != nullptr) { + stats->Subtract(entry->ToLogStatisticsElement(log_id)); + } + continue; + } + size_t entry_total_len = entry->total_len(); + if (read_offset != new_write_offset) { + memmove(contents_.data() + new_write_offset, contents_.data() + read_offset, + entry_total_len); + } + read_offset += entry_total_len; + new_write_offset += entry_total_len; + } + + if (new_write_offset == 0) { + DecReaderRefCount(false); + return true; + } + + // Clear the old compressed logs and set write_offset_ appropriately for DecReaderRefCount() + // to compress the new partially cleared log. + if (new_write_offset != write_offset_) { + compressed_log_.clear(); + write_offset_ = new_write_offset; + } + + DecReaderRefCount(true); + + return false; +} + +bool SerializedLogChunk::CanLog(size_t len) { + return write_offset_ + len <= contents_.size(); +} + +SerializedLogEntry* SerializedLogChunk::Log(uint64_t sequence, log_time realtime, uid_t uid, + pid_t pid, pid_t tid, const char* msg, uint16_t len) { + auto new_log_address = contents_.data() + write_offset_; + auto* entry = new (new_log_address) SerializedLogEntry(uid, pid, tid, sequence, realtime, len); + memcpy(entry->msg(), msg, len); + write_offset_ += entry->total_len(); + highest_sequence_number_ = sequence; + return entry; +}
\ No newline at end of file diff --git a/logd/SerializedLogChunk.h b/logd/SerializedLogChunk.h new file mode 100644 index 000000000..a8ac8cd28 --- /dev/null +++ b/logd/SerializedLogChunk.h @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <sys/types.h> + +#include <vector> + +#include "LogWriter.h" +#include "SerializedLogEntry.h" + +class SerializedLogChunk { + public: + explicit SerializedLogChunk(size_t size) : contents_(size) {} + ~SerializedLogChunk(); + + void Compress(); + void IncReaderRefCount(); + // Decrease the reader ref count and compress the log if appropriate. `compress` should only be + // set to false in the case that the log buffer will be deleted afterwards. + void DecReaderRefCount(bool compress); + + // Must have no readers referencing this. Return true if there are no logs left in this chunk. + bool ClearUidLogs(uid_t uid, log_id_t log_id, LogStatistics* stats); + + bool CanLog(size_t len); + SerializedLogEntry* Log(uint64_t sequence, log_time realtime, uid_t uid, pid_t pid, pid_t tid, + const char* msg, uint16_t len); + + // If this buffer has been compressed, we only consider its compressed size when accounting for + // memory consumption for pruning. This is since the uncompressed log is only by used by + // readers, and thus not a representation of how much these logs cost to keep in memory. + size_t PruneSize() const { return compressed_log_.size() ?: contents_.size(); } + + void FinishWriting() { + writer_active_ = false; + if (reader_ref_count_ == 0) { + Compress(); + } + } + + const SerializedLogEntry* log_entry(int offset) const { + return reinterpret_cast<const SerializedLogEntry*>(data() + offset); + } + const uint8_t* data() const { return contents_.data(); } + int write_offset() const { return write_offset_; } + uint64_t highest_sequence_number() const { return highest_sequence_number_; } + + private: + // The decompressed contents of this log buffer. Deallocated when the ref_count reaches 0 and + // writer_active_ is false. + std::vector<uint8_t> contents_; + int write_offset_ = 0; + uint32_t reader_ref_count_ = 0; + bool writer_active_ = true; + uint64_t highest_sequence_number_ = 1; + std::vector<uint8_t> compressed_log_; +}; diff --git a/logd/SerializedLogChunkTest.cpp b/logd/SerializedLogChunkTest.cpp new file mode 100644 index 000000000..657250335 --- /dev/null +++ b/logd/SerializedLogChunkTest.cpp @@ -0,0 +1,284 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "SerializedLogChunk.h" + +#include <limits> + +#include <android-base/stringprintf.h> +#include <android/log.h> +#include <gtest/gtest.h> + +using android::base::StringPrintf; + +TEST(SerializedLogChunk, smoke) { + size_t chunk_size = 10 * 4096; + auto chunk = SerializedLogChunk{chunk_size}; + EXPECT_EQ(chunk_size, chunk.PruneSize()); + + static const char log_message[] = "log message"; + size_t expected_total_len = sizeof(SerializedLogEntry) + sizeof(log_message); + ASSERT_TRUE(chunk.CanLog(expected_total_len)); + EXPECT_TRUE(chunk.CanLog(chunk_size)); + EXPECT_FALSE(chunk.CanLog(chunk_size + 1)); + + log_time time(CLOCK_REALTIME); + auto* entry = chunk.Log(1234, time, 0, 1, 2, log_message, sizeof(log_message)); + ASSERT_NE(nullptr, entry); + + EXPECT_EQ(1234U, entry->sequence()); + EXPECT_EQ(time, entry->realtime()); + EXPECT_EQ(0U, entry->uid()); + EXPECT_EQ(1, entry->pid()); + EXPECT_EQ(2, entry->tid()); + EXPECT_EQ(sizeof(log_message), entry->msg_len()); + EXPECT_STREQ(log_message, entry->msg()); + EXPECT_EQ(expected_total_len, entry->total_len()); + + EXPECT_FALSE(chunk.CanLog(chunk_size)); + EXPECT_EQ(static_cast<int>(expected_total_len), chunk.write_offset()); + EXPECT_EQ(1234U, chunk.highest_sequence_number()); +} + +TEST(SerializedLogChunk, fill_log_exactly) { + static const char log_message[] = "this is a log message"; + size_t individual_message_size = sizeof(SerializedLogEntry) + sizeof(log_message); + size_t chunk_size = individual_message_size * 3; + auto chunk = SerializedLogChunk{chunk_size}; + EXPECT_EQ(chunk_size, chunk.PruneSize()); + + ASSERT_TRUE(chunk.CanLog(individual_message_size)); + EXPECT_NE(nullptr, chunk.Log(1, log_time(), 1000, 1, 1, log_message, sizeof(log_message))); + + ASSERT_TRUE(chunk.CanLog(individual_message_size)); + EXPECT_NE(nullptr, chunk.Log(2, log_time(), 1000, 2, 1, log_message, sizeof(log_message))); + + ASSERT_TRUE(chunk.CanLog(individual_message_size)); + EXPECT_NE(nullptr, chunk.Log(3, log_time(), 1000, 3, 1, log_message, sizeof(log_message))); + + EXPECT_FALSE(chunk.CanLog(1)); +} + +TEST(SerializedLogChunk, three_logs) { + size_t chunk_size = 10 * 4096; + auto chunk = SerializedLogChunk{chunk_size}; + + chunk.Log(2, log_time(0x1234, 0x5678), 0x111, 0x222, 0x333, "initial message", + strlen("initial message")); + chunk.Log(3, log_time(0x2345, 0x6789), 0x444, 0x555, 0x666, "second message", + strlen("second message")); + auto uint64_t_max = std::numeric_limits<uint64_t>::max(); + auto uint32_t_max = std::numeric_limits<uint32_t>::max(); + chunk.Log(uint64_t_max, log_time(uint32_t_max, uint32_t_max), uint32_t_max, uint32_t_max, + uint32_t_max, "last message", strlen("last message")); + + static const char expected_buffer_data[] = + "\x11\x01\x00\x00\x22\x02\x00\x00\x33\x03\x00\x00" // UID PID TID + "\x02\x00\x00\x00\x00\x00\x00\x00" // Sequence + "\x34\x12\x00\x00\x78\x56\x00\x00" // Timestamp + "\x0F\x00initial message" // msg_len + message + "\x44\x04\x00\x00\x55\x05\x00\x00\x66\x06\x00\x00" // UID PID TID + "\x03\x00\x00\x00\x00\x00\x00\x00" // Sequence + "\x45\x23\x00\x00\x89\x67\x00\x00" // Timestamp + "\x0E\x00second message" // msg_len + message + "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" // UID PID TID + "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" // Sequence + "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" // Timestamp + "\x0C\x00last message"; // msg_len + message + + for (size_t i = 0; i < chunk_size; ++i) { + if (i < sizeof(expected_buffer_data)) { + EXPECT_EQ(static_cast<uint8_t>(expected_buffer_data[i]), chunk.data()[i]) + << "position: " << i; + } else { + EXPECT_EQ(0, chunk.data()[i]) << "position: " << i; + } + } +} + +// Check that the CHECK() in DecReaderRefCount() if the ref count goes bad is caught. +TEST(SerializedLogChunk, catch_DecCompressedRef_CHECK) { + size_t chunk_size = 10 * 4096; + auto chunk = SerializedLogChunk{chunk_size}; + EXPECT_DEATH({ chunk.DecReaderRefCount(true); }, ""); + EXPECT_DEATH({ chunk.DecReaderRefCount(false); }, ""); +} + +// Check that the CHECK() in ClearUidLogs() if the ref count is greater than 0 is caught. +TEST(SerializedLogChunk, catch_ClearUidLogs_CHECK) { + size_t chunk_size = 10 * 4096; + auto chunk = SerializedLogChunk{chunk_size}; + chunk.IncReaderRefCount(); + EXPECT_DEATH({ chunk.ClearUidLogs(1000, LOG_ID_MAIN, nullptr); }, ""); + chunk.DecReaderRefCount(false); +} + +class UidClearTest : public testing::TestWithParam<bool> { + protected: + template <typename Write, typename Check> + void Test(const Write& write, const Check& check, bool expected_result) { + write(chunk_); + + bool finish_writing = GetParam(); + if (finish_writing) { + chunk_.FinishWriting(); + } + EXPECT_EQ(expected_result, chunk_.ClearUidLogs(kUidToClear, LOG_ID_MAIN, nullptr)); + if (finish_writing) { + chunk_.IncReaderRefCount(); + } + + check(chunk_); + + if (finish_writing) { + chunk_.DecReaderRefCount(false); + } + } + + static constexpr size_t kChunkSize = 10 * 4096; + static constexpr uid_t kUidToClear = 1000; + static constexpr uid_t kOtherUid = 1234; + + SerializedLogChunk chunk_{kChunkSize}; +}; + +// Test that ClearUidLogs() is a no-op if there are no logs of that UID in the buffer. +TEST_P(UidClearTest, no_logs_in_chunk) { + auto write = [](SerializedLogChunk&) {}; + auto check = [](SerializedLogChunk&) {}; + + Test(write, check, true); +} + +// Test that ClearUidLogs() is a no-op if there are no logs of that UID in the buffer. +TEST_P(UidClearTest, no_logs_from_uid) { + static const char msg[] = "this is a log message"; + auto write = [](SerializedLogChunk& chunk) { + chunk.Log(1, log_time(), kOtherUid, 1, 2, msg, sizeof(msg)); + }; + + auto check = [](SerializedLogChunk& chunk) { + auto* entry = chunk.log_entry(0); + EXPECT_STREQ(msg, entry->msg()); + }; + + Test(write, check, false); +} + +// Test that ClearUidLogs() returns true if all logs in a given buffer correspond to the given UID. +TEST_P(UidClearTest, all_single) { + static const char msg[] = "this is a log message"; + auto write = [](SerializedLogChunk& chunk) { + chunk.Log(1, log_time(), kUidToClear, 1, 2, msg, sizeof(msg)); + }; + auto check = [](SerializedLogChunk&) {}; + + Test(write, check, true); +} + +// Test that ClearUidLogs() returns true if all logs in a given buffer correspond to the given UID. +TEST_P(UidClearTest, all_multiple) { + static const char msg[] = "this is a log message"; + auto write = [](SerializedLogChunk& chunk) { + chunk.Log(2, log_time(), kUidToClear, 1, 2, msg, sizeof(msg)); + chunk.Log(3, log_time(), kUidToClear, 1, 2, msg, sizeof(msg)); + chunk.Log(4, log_time(), kUidToClear, 1, 2, msg, sizeof(msg)); + }; + auto check = [](SerializedLogChunk&) {}; + + Test(write, check, true); +} + +static std::string MakePrintable(const uint8_t* in, size_t length) { + std::string result; + for (size_t i = 0; i < length; ++i) { + uint8_t c = in[i]; + if (isprint(c)) { + result.push_back(c); + } else { + result.append(StringPrintf("\\%02x", static_cast<int>(c) & 0xFF)); + } + } + return result; +} + +// This test clears UID logs at the beginning and end of the buffer, as well as two back to back +// logs in the interior. +TEST_P(UidClearTest, clear_beginning_and_end) { + static const char msg1[] = "this is a log message"; + static const char msg2[] = "non-cleared message"; + static const char msg3[] = "back to back cleared messages"; + static const char msg4[] = "second in a row gone"; + static const char msg5[] = "but we save this one"; + static const char msg6[] = "and this 1!"; + static const char msg7[] = "the last one goes too"; + auto write = [](SerializedLogChunk& chunk) { + ASSERT_NE(nullptr, chunk.Log(1, log_time(), kUidToClear, 1, 2, msg1, sizeof(msg1))); + ASSERT_NE(nullptr, chunk.Log(2, log_time(), kOtherUid, 1, 2, msg2, sizeof(msg2))); + ASSERT_NE(nullptr, chunk.Log(3, log_time(), kUidToClear, 1, 2, msg3, sizeof(msg3))); + ASSERT_NE(nullptr, chunk.Log(4, log_time(), kUidToClear, 1, 2, msg4, sizeof(msg4))); + ASSERT_NE(nullptr, chunk.Log(5, log_time(), kOtherUid, 1, 2, msg5, sizeof(msg5))); + ASSERT_NE(nullptr, chunk.Log(6, log_time(), kOtherUid, 1, 2, msg6, sizeof(msg6))); + ASSERT_NE(nullptr, chunk.Log(7, log_time(), kUidToClear, 1, 2, msg7, sizeof(msg7))); + }; + + auto check = [](SerializedLogChunk& chunk) { + size_t read_offset = 0; + auto* entry = chunk.log_entry(read_offset); + EXPECT_STREQ(msg2, entry->msg()); + read_offset += entry->total_len(); + + entry = chunk.log_entry(read_offset); + EXPECT_STREQ(msg5, entry->msg()); + read_offset += entry->total_len(); + + entry = chunk.log_entry(read_offset); + EXPECT_STREQ(msg6, entry->msg()) << MakePrintable(chunk.data(), chunk.write_offset()); + read_offset += entry->total_len(); + + EXPECT_EQ(static_cast<int>(read_offset), chunk.write_offset()); + }; + Test(write, check, false); +} + +// This tests the opposite case of beginning_and_end, in which we don't clear the beginning or end +// logs. There is a single log pruned in the middle instead of back to back logs. +TEST_P(UidClearTest, save_beginning_and_end) { + static const char msg1[] = "saved first message"; + static const char msg2[] = "cleared interior message"; + static const char msg3[] = "last message stays"; + auto write = [](SerializedLogChunk& chunk) { + ASSERT_NE(nullptr, chunk.Log(1, log_time(), kOtherUid, 1, 2, msg1, sizeof(msg1))); + ASSERT_NE(nullptr, chunk.Log(2, log_time(), kUidToClear, 1, 2, msg2, sizeof(msg2))); + ASSERT_NE(nullptr, chunk.Log(3, log_time(), kOtherUid, 1, 2, msg3, sizeof(msg3))); + }; + + auto check = [](SerializedLogChunk& chunk) { + size_t read_offset = 0; + auto* entry = chunk.log_entry(read_offset); + EXPECT_STREQ(msg1, entry->msg()); + read_offset += entry->total_len(); + + entry = chunk.log_entry(read_offset); + EXPECT_STREQ(msg3, entry->msg()); + read_offset += entry->total_len(); + + EXPECT_EQ(static_cast<int>(read_offset), chunk.write_offset()); + }; + Test(write, check, false); +} + +INSTANTIATE_TEST_CASE_P(UidClearTests, UidClearTest, testing::Values(true, false)); diff --git a/logd/SerializedLogEntry.h b/logd/SerializedLogEntry.h new file mode 100644 index 000000000..f599abeaa --- /dev/null +++ b/logd/SerializedLogEntry.h @@ -0,0 +1,98 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <stdint.h> +#include <stdlib.h> +#include <sys/types.h> + +#include <log/log.h> +#include <log/log_read.h> + +#include "LogStatistics.h" +#include "LogWriter.h" + +// These structs are packed into a single chunk of memory for each log type within a +// SerializedLogChunk object. Their message is contained immediately at the end of the struct. The +// address of the next log in the buffer is *this + sizeof(SerializedLogEntry) + msg_len_. If that +// value would overflow the chunk of memory associated with the SerializedLogChunk object, then a +// new SerializedLogChunk must be allocated to contain the next SerializedLogEntry. +class __attribute__((packed)) SerializedLogEntry { + public: + SerializedLogEntry(uid_t uid, pid_t pid, pid_t tid, uint64_t sequence, log_time realtime, + uint16_t len) + : uid_(uid), + pid_(pid), + tid_(tid), + sequence_(sequence), + realtime_(realtime), + msg_len_(len) {} + SerializedLogEntry(const SerializedLogEntry& elem) = delete; + SerializedLogEntry& operator=(const SerializedLogEntry& elem) = delete; + ~SerializedLogEntry() { + // Never place anything in this destructor. This class is in place constructed and never + // destructed. + } + + LogStatisticsElement ToLogStatisticsElement(log_id_t log_id) const { + return LogStatisticsElement{ + .uid = uid(), + .pid = pid(), + .tid = tid(), + .tag = IsBinary(log_id) ? MsgToTag(msg(), msg_len()) : 0, + .realtime = realtime(), + .msg = msg(), + .msg_len = msg_len(), + .dropped_count = 0, + .log_id = log_id, + }; + } + + bool Flush(LogWriter* writer, log_id_t log_id) const { + struct logger_entry entry = {}; + + entry.hdr_size = sizeof(struct logger_entry); + entry.lid = log_id; + entry.pid = pid(); + entry.tid = tid(); + entry.uid = uid(); + entry.sec = realtime().tv_sec; + entry.nsec = realtime().tv_nsec; + entry.len = msg_len(); + + return writer->Write(entry, msg()); + } + + uid_t uid() const { return uid_; } + pid_t pid() const { return pid_; } + pid_t tid() const { return tid_; } + uint16_t msg_len() const { return msg_len_; } + uint64_t sequence() const { return sequence_; } + log_time realtime() const { return realtime_; } + + char* msg() { return reinterpret_cast<char*>(this) + sizeof(*this); } + const char* msg() const { return reinterpret_cast<const char*>(this) + sizeof(*this); } + uint16_t total_len() const { return sizeof(*this) + msg_len_; } + + private: + const uint32_t uid_; + const uint32_t pid_; + const uint32_t tid_; + const uint64_t sequence_; + const log_time realtime_; + const uint16_t msg_len_; +}; diff --git a/logd/SimpleLogBuffer.cpp b/logd/SimpleLogBuffer.cpp new file mode 100644 index 000000000..ec08d54d4 --- /dev/null +++ b/logd/SimpleLogBuffer.cpp @@ -0,0 +1,359 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "SimpleLogBuffer.h" + +#include <android-base/logging.h> + +#include "LogBufferElement.h" + +SimpleLogBuffer::SimpleLogBuffer(LogReaderList* reader_list, LogTags* tags, LogStatistics* stats) + : reader_list_(reader_list), tags_(tags), stats_(stats) { + Init(); +} + +SimpleLogBuffer::~SimpleLogBuffer() {} + +void SimpleLogBuffer::Init() { + log_id_for_each(i) { + if (SetSize(i, __android_logger_get_buffer_size(i))) { + SetSize(i, LOG_BUFFER_MIN_SIZE); + } + } + + // Release any sleeping reader threads to dump their current content. + auto reader_threads_lock = std::lock_guard{reader_list_->reader_threads_lock()}; + for (const auto& reader_thread : reader_list_->reader_threads()) { + reader_thread->triggerReader_Locked(); + } +} + +std::list<LogBufferElement>::iterator SimpleLogBuffer::GetOldest(log_id_t log_id) { + auto it = logs().begin(); + if (oldest_[log_id]) { + it = *oldest_[log_id]; + } + while (it != logs().end() && it->log_id() != log_id) { + it++; + } + if (it != logs().end()) { + oldest_[log_id] = it; + } + return it; +} + +bool SimpleLogBuffer::ShouldLog(log_id_t log_id, const char* msg, uint16_t len) { + if (log_id == LOG_ID_SECURITY) { + return true; + } + + int prio = ANDROID_LOG_INFO; + const char* tag = nullptr; + size_t tag_len = 0; + if (IsBinary(log_id)) { + int32_t numeric_tag = MsgToTag(msg, len); + tag = tags_->tagToName(numeric_tag); + if (tag) { + tag_len = strlen(tag); + } + } else { + prio = *msg; + tag = msg + 1; + tag_len = strnlen(tag, len - 1); + } + return __android_log_is_loggable_len(prio, tag, tag_len, ANDROID_LOG_VERBOSE); +} + +int SimpleLogBuffer::Log(log_id_t log_id, log_time realtime, uid_t uid, pid_t pid, pid_t tid, + const char* msg, uint16_t len) { + if (log_id >= LOG_ID_MAX) { + return -EINVAL; + } + + if (!ShouldLog(log_id, msg, len)) { + // Log traffic received to total + stats_->AddTotal(log_id, len); + return -EACCES; + } + + // Slip the time by 1 nsec if the incoming lands on xxxxxx000 ns. + // This prevents any chance that an outside source can request an + // exact entry with time specified in ms or us precision. + if ((realtime.tv_nsec % 1000) == 0) ++realtime.tv_nsec; + + auto lock = std::lock_guard{lock_}; + auto sequence = sequence_.fetch_add(1, std::memory_order_relaxed); + LogInternal(LogBufferElement(log_id, realtime, uid, pid, tid, sequence, msg, len)); + return len; +} + +void SimpleLogBuffer::LogInternal(LogBufferElement&& elem) { + log_id_t log_id = elem.log_id(); + + logs_.emplace_back(std::move(elem)); + stats_->Add(logs_.back().ToLogStatisticsElement()); + MaybePrune(log_id); + reader_list_->NotifyNewLog(1 << log_id); +} + +// These extra parameters are only required for chatty, but since they're a no-op for +// SimpleLogBuffer, it's easier to include them here, then to duplicate FlushTo() for +// ChattyLogBuffer. +class ChattyFlushToState : public FlushToState { + public: + ChattyFlushToState(uint64_t start, LogMask log_mask) : FlushToState(start, log_mask) {} + + pid_t* last_tid() { return last_tid_; } + + bool drop_chatty_messages() const { return drop_chatty_messages_; } + void set_drop_chatty_messages(bool value) { drop_chatty_messages_ = value; } + + private: + pid_t last_tid_[LOG_ID_MAX] = {}; + bool drop_chatty_messages_ = true; +}; + +std::unique_ptr<FlushToState> SimpleLogBuffer::CreateFlushToState(uint64_t start, + LogMask log_mask) { + return std::make_unique<ChattyFlushToState>(start, log_mask); +} + +bool SimpleLogBuffer::FlushTo( + LogWriter* writer, FlushToState& abstract_state, + const std::function<FilterResult(log_id_t log_id, pid_t pid, uint64_t sequence, + log_time realtime)>& filter) { + auto shared_lock = SharedLock{lock_}; + + auto& state = reinterpret_cast<ChattyFlushToState&>(abstract_state); + + std::list<LogBufferElement>::iterator it; + if (state.start() <= 1) { + // client wants to start from the beginning + it = logs_.begin(); + } else { + // Client wants to start from some specified time. Chances are + // we are better off starting from the end of the time sorted list. + for (it = logs_.end(); it != logs_.begin(); + /* do nothing */) { + --it; + if (it->sequence() == state.start()) { + break; + } else if (it->sequence() < state.start()) { + it++; + break; + } + } + } + + for (; it != logs_.end(); ++it) { + LogBufferElement& element = *it; + + state.set_start(element.sequence()); + + if (!writer->privileged() && element.uid() != writer->uid()) { + continue; + } + + if (((1 << element.log_id()) & state.log_mask()) == 0) { + continue; + } + + if (filter) { + FilterResult ret = + filter(element.log_id(), element.pid(), element.sequence(), element.realtime()); + if (ret == FilterResult::kSkip) { + continue; + } + if (ret == FilterResult::kStop) { + break; + } + } + + // drop_chatty_messages is initialized to true, so if the first message that we attempt to + // flush is a chatty message, we drop it. Once we see a non-chatty message it gets set to + // false to let further chatty messages be printed. + if (state.drop_chatty_messages()) { + if (element.dropped_count() != 0) { + continue; + } + state.set_drop_chatty_messages(false); + } + + bool same_tid = state.last_tid()[element.log_id()] == element.tid(); + // Dropped (chatty) immediately following a valid log from the same source in the same log + // buffer indicates we have a multiple identical squash. chatty that differs source is due + // to spam filter. chatty to chatty of different source is also due to spam filter. + state.last_tid()[element.log_id()] = + (element.dropped_count() && !same_tid) ? 0 : element.tid(); + + shared_lock.unlock(); + // We never prune logs equal to or newer than any LogReaderThreads' `start` value, so the + // `element` pointer is safe here without the lock + if (!element.FlushTo(writer, stats_, same_tid)) { + return false; + } + shared_lock.lock_shared(); + } + + state.set_start(state.start() + 1); + return true; +} + +bool SimpleLogBuffer::Clear(log_id_t id, uid_t uid) { + // Try three times to clear, then disconnect the readers and try one final time. + for (int retry = 0; retry < 3; ++retry) { + { + auto lock = std::lock_guard{lock_}; + if (Prune(id, ULONG_MAX, uid)) { + return true; + } + } + sleep(1); + } + // Check if it is still busy after the sleep, we try to prune one entry, not another clear run, + // so we are looking for the quick side effect of the return value to tell us if we have a + // _blocked_ reader. + bool busy = false; + { + auto lock = std::lock_guard{lock_}; + busy = !Prune(id, 1, uid); + } + // It is still busy, disconnect all readers. + if (busy) { + auto reader_threads_lock = std::lock_guard{reader_list_->reader_threads_lock()}; + for (const auto& reader_thread : reader_list_->reader_threads()) { + if (reader_thread->IsWatching(id)) { + LOG(WARNING) << "Kicking blocked reader, " << reader_thread->name() + << ", from LogBuffer::clear()"; + reader_thread->release_Locked(); + } + } + } + auto lock = std::lock_guard{lock_}; + return Prune(id, ULONG_MAX, uid); +} + +// get the total space allocated to "id" +unsigned long SimpleLogBuffer::GetSize(log_id_t id) { + auto lock = SharedLock{lock_}; + size_t retval = max_size_[id]; + return retval; +} + +// set the total space allocated to "id" +int SimpleLogBuffer::SetSize(log_id_t id, unsigned long size) { + // Reasonable limits ... + if (!__android_logger_valid_buffer_size(size)) { + return -1; + } + + auto lock = std::lock_guard{lock_}; + max_size_[id] = size; + return 0; +} + +void SimpleLogBuffer::MaybePrune(log_id_t id) { + unsigned long prune_rows; + if (stats_->ShouldPrune(id, max_size_[id], &prune_rows)) { + Prune(id, prune_rows, 0); + } +} + +bool SimpleLogBuffer::Prune(log_id_t id, unsigned long prune_rows, uid_t caller_uid) { + auto reader_threads_lock = std::lock_guard{reader_list_->reader_threads_lock()}; + + // Don't prune logs that are newer than the point at which any reader threads are reading from. + LogReaderThread* oldest = nullptr; + for (const auto& reader_thread : reader_list_->reader_threads()) { + if (!reader_thread->IsWatching(id)) { + continue; + } + if (!oldest || oldest->start() > reader_thread->start() || + (oldest->start() == reader_thread->start() && + reader_thread->deadline().time_since_epoch().count() != 0)) { + oldest = reader_thread.get(); + } + } + + auto it = GetOldest(id); + + while (it != logs_.end()) { + LogBufferElement& element = *it; + + if (element.log_id() != id) { + ++it; + continue; + } + + if (caller_uid != 0 && element.uid() != caller_uid) { + ++it; + continue; + } + + if (oldest && oldest->start() <= element.sequence()) { + KickReader(oldest, id, prune_rows); + return false; + } + + stats_->Subtract(element.ToLogStatisticsElement()); + it = Erase(it); + if (--prune_rows == 0) { + return true; + } + } + return true; +} + +std::list<LogBufferElement>::iterator SimpleLogBuffer::Erase( + std::list<LogBufferElement>::iterator it) { + bool oldest_is_it[LOG_ID_MAX]; + log_id_for_each(i) { oldest_is_it[i] = oldest_[i] && it == *oldest_[i]; } + + it = logs_.erase(it); + + log_id_for_each(i) { + if (oldest_is_it[i]) { + if (__predict_false(it == logs().end())) { + oldest_[i] = std::nullopt; + } else { + oldest_[i] = it; // Store the next iterator even if it does not correspond to + // the same log_id, as a starting point for GetOldest(). + } + } + } + + return it; +} + +// If the selected reader is blocking our pruning progress, decide on +// what kind of mitigation is necessary to unblock the situation. +void SimpleLogBuffer::KickReader(LogReaderThread* reader, log_id_t id, unsigned long prune_rows) { + if (stats_->Sizes(id) > (2 * max_size_[id])) { // +100% + // A misbehaving or slow reader has its connection + // dropped if we hit too much memory pressure. + LOG(WARNING) << "Kicking blocked reader, " << reader->name() + << ", from LogBuffer::kickMe()"; + reader->release_Locked(); + } else if (reader->deadline().time_since_epoch().count() != 0) { + // Allow a blocked WRAP deadline reader to trigger and start reporting the log data. + reader->triggerReader_Locked(); + } else { + // tell slow reader to skip entries to catch up + LOG(WARNING) << "Skipping " << prune_rows << " entries from slow reader, " << reader->name() + << ", from LogBuffer::kickMe()"; + reader->triggerSkip_Locked(id, prune_rows); + } +} diff --git a/logd/SimpleLogBuffer.h b/logd/SimpleLogBuffer.h new file mode 100644 index 000000000..9f7d69903 --- /dev/null +++ b/logd/SimpleLogBuffer.h @@ -0,0 +1,83 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <atomic> +#include <list> +#include <mutex> + +#include "LogBuffer.h" +#include "LogBufferElement.h" +#include "LogReaderList.h" +#include "LogStatistics.h" +#include "LogTags.h" +#include "rwlock.h" + +class SimpleLogBuffer : public LogBuffer { + public: + SimpleLogBuffer(LogReaderList* reader_list, LogTags* tags, LogStatistics* stats); + ~SimpleLogBuffer(); + void Init() override final; + + int Log(log_id_t log_id, log_time realtime, uid_t uid, pid_t pid, pid_t tid, const char* msg, + uint16_t len) override; + std::unique_ptr<FlushToState> CreateFlushToState(uint64_t start, LogMask log_mask) override; + bool FlushTo(LogWriter* writer, FlushToState& state, + const std::function<FilterResult(log_id_t log_id, pid_t pid, uint64_t sequence, + log_time realtime)>& filter) override; + + bool Clear(log_id_t id, uid_t uid) override; + unsigned long GetSize(log_id_t id) override; + int SetSize(log_id_t id, unsigned long size) override final; + + uint64_t sequence() const override { return sequence_.load(std::memory_order_relaxed); } + + protected: + virtual bool Prune(log_id_t id, unsigned long prune_rows, uid_t uid) REQUIRES(lock_); + virtual void LogInternal(LogBufferElement&& elem) REQUIRES(lock_); + + // Returns an iterator to the oldest element for a given log type, or logs_.end() if + // there are no logs for the given log type. Requires logs_lock_ to be held. + std::list<LogBufferElement>::iterator GetOldest(log_id_t log_id) REQUIRES(lock_); + std::list<LogBufferElement>::iterator Erase(std::list<LogBufferElement>::iterator it) + REQUIRES(lock_); + void KickReader(LogReaderThread* reader, log_id_t id, unsigned long prune_rows) + REQUIRES_SHARED(lock_); + + LogStatistics* stats() { return stats_; } + LogReaderList* reader_list() { return reader_list_; } + unsigned long max_size(log_id_t id) REQUIRES_SHARED(lock_) { return max_size_[id]; } + std::list<LogBufferElement>& logs() { return logs_; } + + RwLock lock_; + + private: + bool ShouldLog(log_id_t log_id, const char* msg, uint16_t len); + void MaybePrune(log_id_t id) REQUIRES(lock_); + + LogReaderList* reader_list_; + LogTags* tags_; + LogStatistics* stats_; + + std::atomic<uint64_t> sequence_ = 1; + + unsigned long max_size_[LOG_ID_MAX] GUARDED_BY(lock_); + std::list<LogBufferElement> logs_ GUARDED_BY(lock_); + // Keeps track of the iterator to the oldest log message of a given log type, as an + // optimization when pruning logs. Use GetOldest() to retrieve. + std::optional<std::list<LogBufferElement>::iterator> oldest_[LOG_ID_MAX] GUARDED_BY(lock_); +}; diff --git a/logd/fuzz/Android.bp b/logd/fuzz/Android.bp index 299242db6..9834ff058 100644 --- a/logd/fuzz/Android.bp +++ b/logd/fuzz/Android.bp @@ -25,6 +25,9 @@ cc_fuzz { "liblog", "liblogd", "libcutils", + "libsysutils", + "libz", + "libzstd", ], cflags: ["-Werror"], } diff --git a/logd/fuzz/log_buffer_log_fuzzer.cpp b/logd/fuzz/log_buffer_log_fuzzer.cpp index 4d1589b50..a7a17921a 100644 --- a/logd/fuzz/log_buffer_log_fuzzer.cpp +++ b/logd/fuzz/log_buffer_log_fuzzer.cpp @@ -15,8 +15,10 @@ */ #include <string> -#include "../LogBuffer.h" -#include "../LogTimes.h" +#include "../ChattyLogBuffer.h" +#include "../LogReaderList.h" +#include "../LogReaderThread.h" +#include "../LogStatistics.h" // We don't want to waste a lot of entropy on messages #define MAX_MSG_LENGTH 5 @@ -36,7 +38,8 @@ struct LogInput { unsigned int log_mask; }; -int write_log_messages(const uint8_t** pdata, size_t* data_left, LogBuffer* log_buffer) { +int write_log_messages(const uint8_t** pdata, size_t* data_left, LogBuffer* log_buffer, + LogStatistics* stats) { const uint8_t* data = *pdata; const LogInput* logInput = reinterpret_cast<const LogInput*>(data); data += sizeof(LogInput); @@ -69,20 +72,13 @@ int write_log_messages(const uint8_t** pdata, size_t* data_left, LogBuffer* log_ // Other elements not in enum. log_id_t log_id = static_cast<log_id_t>(unsigned(logInput->log_id) % (LOG_ID_MAX + 1)); - log_buffer->log(log_id, logInput->realtime, logInput->uid, logInput->pid, logInput->tid, msg, + log_buffer->Log(log_id, logInput->realtime, logInput->uid, logInput->pid, logInput->tid, msg, sizeof(uint32_t) + msg_length + 1); - log_buffer->formatStatistics(logInput->uid, logInput->pid, logInput->log_mask); + stats->Format(logInput->uid, logInput->pid, logInput->log_mask); *pdata = data; return 1; } -// Because system/core/logd/main.cpp redefines these. -void prdebug(char const* fmt, ...) { - va_list ap; - va_start(ap, fmt); - vfprintf(stderr, fmt, ap); - va_end(ap); -} char* uidToName(uid_t) { return strdup("fake"); } @@ -93,23 +89,25 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { return 0; } - LastLogTimes times; - LogBuffer log_buffer(×); + LogReaderList reader_list; + LogTags tags; + PruneList prune_list; + LogStatistics stats(true); + LogBuffer* log_buffer = new ChattyLogBuffer(&reader_list, &tags, &prune_list, &stats); size_t data_left = size; const uint8_t** pdata = &data; - log_buffer.enableStatistics(); - log_buffer.initPrune(nullptr); + prune_list.Init(nullptr); // We want to get pruning code to get called. - log_id_for_each(i) { log_buffer.setSize(i, 10000); } + log_id_for_each(i) { log_buffer->SetSize(i, 10000); } while (data_left >= sizeof(LogInput) + 2 * sizeof(uint8_t)) { - if (!write_log_messages(pdata, &data_left, &log_buffer)) { + if (!write_log_messages(pdata, &data_left, log_buffer, &stats)) { return 0; } } - log_id_for_each(i) { log_buffer.clear(i); } + log_id_for_each(i) { log_buffer->Clear(i, 0); } return 0; } } // namespace android diff --git a/logd/libaudit.c b/logd/libaudit.cpp index f452c71ab..ccea0a2b7 100644 --- a/logd/libaudit.c +++ b/logd/libaudit.cpp @@ -18,11 +18,13 @@ * */ +#include "libaudit.h" + #include <errno.h> #include <string.h> #include <unistd.h> -#include "libaudit.h" +#include <limits> /** * Waits for an ack from the kernel @@ -32,22 +34,15 @@ * This function returns 0 on success, else -errno. */ static int get_ack(int fd) { - int rc; - struct audit_message rep; - - /* Sanity check, this is an internal interface this shouldn't happen */ - if (fd < 0) { - return -EINVAL; - } - - rc = audit_get_reply(fd, &rep, GET_REPLY_BLOCKING, MSG_PEEK); + struct audit_message rep = {}; + int rc = audit_get_reply(fd, &rep, GET_REPLY_BLOCKING, MSG_PEEK); if (rc < 0) { return rc; } if (rep.nlh.nlmsg_type == NLMSG_ERROR) { audit_get_reply(fd, &rep, GET_REPLY_BLOCKING, 0); - rc = ((struct nlmsgerr*)rep.data)->error; + rc = reinterpret_cast<struct nlmsgerr*>(rep.data)->error; if (rc) { return -rc; } @@ -70,19 +65,11 @@ static int get_ack(int fd) { * This function returns a positive sequence number on success, else -errno. */ static int audit_send(int fd, int type, const void* data, size_t size) { - int rc; - static int16_t sequence = 0; - struct audit_message req; - struct sockaddr_nl addr; - - memset(&req, 0, sizeof(req)); - memset(&addr, 0, sizeof(addr)); - - /* We always send netlink messaged */ - addr.nl_family = AF_NETLINK; + struct sockaddr_nl addr = {.nl_family = AF_NETLINK}; /* Set up the netlink headers */ - req.nlh.nlmsg_type = type; + struct audit_message req = {}; + req.nlh.nlmsg_type = static_cast<uint16_t>(type); req.nlh.nlmsg_len = NLMSG_SPACE(size); req.nlh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK; @@ -107,29 +94,23 @@ static int audit_send(int fd, int type, const void* data, size_t size) { /* * Only increment the sequence number on a guarantee * you will send it to the kernel. - * - * Also, the sequence is defined as a u32 in the kernel - * struct. Using an int here might not work on 32/64 bit splits. A - * signed 64 bit value can overflow a u32..but a u32 - * might not fit in the response, so we need to use s32. - * Which is still kind of hackish since int could be 16 bits - * in size. The only safe type to use here is a signed 16 - * bit value. */ - req.nlh.nlmsg_seq = ++sequence; - - /* While failing and its due to interrupts */ + static uint32_t sequence = 0; + if (sequence == std::numeric_limits<uint32_t>::max()) { + sequence = 1; + } else { + sequence++; + } + req.nlh.nlmsg_seq = sequence; - rc = TEMP_FAILURE_RETRY(sendto(fd, &req, req.nlh.nlmsg_len, 0, - (struct sockaddr*)&addr, sizeof(addr))); + ssize_t rc = TEMP_FAILURE_RETRY( + sendto(fd, &req, req.nlh.nlmsg_len, 0, (struct sockaddr*)&addr, sizeof(addr))); /* Not all the bytes were sent */ if (rc < 0) { - rc = -errno; - goto out; + return -errno; } else if ((uint32_t)rc != req.nlh.nlmsg_len) { - rc = -EPROTO; - goto out; + return -EPROTO; } /* We sent all the bytes, get the ack */ @@ -138,32 +119,22 @@ static int audit_send(int fd, int type, const void* data, size_t size) { /* If the ack failed, return the error, else return the sequence number */ rc = (rc == 0) ? (int)sequence : rc; -out: - /* Don't let sequence roll to negative */ - if (sequence < 0) { - sequence = 0; - } - return rc; } int audit_setup(int fd, pid_t pid) { - int rc; - struct audit_message rep; - struct audit_status status; - - memset(&status, 0, sizeof(status)); - /* * In order to set the auditd PID we send an audit message over the netlink * socket with the pid field of the status struct set to our current pid, * and the the mask set to AUDIT_STATUS_PID */ - status.pid = pid; - status.mask = AUDIT_STATUS_PID; + struct audit_status status = { + .mask = AUDIT_STATUS_PID, + .pid = static_cast<uint32_t>(pid), + }; /* Let the kernel know this pid will be registering for audit events */ - rc = audit_send(fd, AUDIT_SET, &status, sizeof(status)); + int rc = audit_send(fd, AUDIT_SET, &status, sizeof(status)); if (rc < 0) { return rc; } @@ -178,6 +149,7 @@ int audit_setup(int fd, pid_t pid) { * so I went to non-blocking and it seemed to fix the bug. * Need to investigate further. */ + struct audit_message rep = {}; audit_get_reply(fd, &rep, GET_REPLY_NONBLOCKING, 0); return 0; @@ -188,27 +160,18 @@ int audit_open() { } int audit_rate_limit(int fd, uint32_t limit) { - struct audit_status status; - memset(&status, 0, sizeof(status)); - status.mask = AUDIT_STATUS_RATE_LIMIT; - status.rate_limit = limit; /* audit entries per second */ + struct audit_status status = { + .mask = AUDIT_STATUS_RATE_LIMIT, .rate_limit = limit, /* audit entries per second */ + }; return audit_send(fd, AUDIT_SET, &status, sizeof(status)); } int audit_get_reply(int fd, struct audit_message* rep, reply_t block, int peek) { - ssize_t len; - int flags; - int rc = 0; - - struct sockaddr_nl nladdr; - socklen_t nladdrlen = sizeof(nladdr); - if (fd < 0) { return -EBADF; } - /* Set up the flags for recv from */ - flags = (block == GET_REPLY_NONBLOCKING) ? MSG_DONTWAIT : 0; + int flags = (block == GET_REPLY_NONBLOCKING) ? MSG_DONTWAIT : 0; flags |= peek; /* @@ -216,19 +179,20 @@ int audit_get_reply(int fd, struct audit_message* rep, reply_t block, int peek) * the interface shows that EINTR can never be returned, other errors, * however, can be returned. */ - len = TEMP_FAILURE_RETRY(recvfrom(fd, rep, sizeof(*rep), flags, - (struct sockaddr*)&nladdr, &nladdrlen)); + struct sockaddr_nl nladdr; + socklen_t nladdrlen = sizeof(nladdr); + ssize_t len = TEMP_FAILURE_RETRY( + recvfrom(fd, rep, sizeof(*rep), flags, (struct sockaddr*)&nladdr, &nladdrlen)); /* * EAGAIN should be re-tried until success or another error manifests. */ if (len < 0) { - rc = -errno; - if (block == GET_REPLY_NONBLOCKING && rc == -EAGAIN) { + if (block == GET_REPLY_NONBLOCKING && errno == EAGAIN) { /* If request is non blocking and errno is EAGAIN, just return 0 */ return 0; } - return rc; + return -errno; } if (nladdrlen != sizeof(nladdr)) { @@ -242,10 +206,10 @@ int audit_get_reply(int fd, struct audit_message* rep, reply_t block, int peek) /* Check if the reply from the kernel was ok */ if (!NLMSG_OK(&rep->nlh, (size_t)len)) { - rc = (len == sizeof(*rep)) ? -EFBIG : -EBADE; + return len == sizeof(*rep) ? -EFBIG : -EBADE; } - return rc; + return 0; } void audit_close(int fd) { diff --git a/logd/libaudit.h b/logd/libaudit.h index b4a92a8a3..27b086693 100644 --- a/logd/libaudit.h +++ b/logd/libaudit.h @@ -17,8 +17,7 @@ * Written by William Roberts <w.roberts@sta.samsung.com> */ -#ifndef _LIBAUDIT_H_ -#define _LIBAUDIT_H_ +#pragma once #include <stdint.h> #include <sys/cdefs.h> @@ -102,5 +101,3 @@ extern int audit_setup(int fd, pid_t pid); extern int audit_rate_limit(int fd, uint32_t limit); __END_DECLS - -#endif diff --git a/logd/tests/logd_test.cpp b/logd/logd_test.cpp index 10bac62f3..ed34ea413 100644 --- a/logd/tests/logd_test.cpp +++ b/logd/logd_test.cpp @@ -32,13 +32,14 @@ #include <android-base/stringprintf.h> #include <cutils/sockets.h> #include <gtest/gtest.h> +#include <log/log_read.h> #include <private/android_filesystem_config.h> #include <private/android_logger.h> #ifdef __ANDROID__ #include <selinux/selinux.h> #endif -#include "../LogReader.h" // pickup LOGD_SNDTIMEO +#include "LogReader.h" // pickup LOGD_SNDTIMEO #ifdef __ANDROID__ static void send_to_control(char* buf, size_t len) { @@ -605,7 +606,7 @@ TEST(logd, timeout) { // A few tries to get it right just in case wrap kicks in due to // content providers being active during the test. int i = 5; - log_time start(android_log_clockid()); + log_time start(CLOCK_REALTIME); start.tv_sec -= 30; // reach back a moderate period of time while (--i) { @@ -681,7 +682,7 @@ TEST(logd, timeout) { if (msg > start) { start = msg; start.tv_sec += 30; - log_time now = log_time(android_log_clockid()); + log_time now = log_time(CLOCK_REALTIME); if (start > now) { start = now; --start.tv_sec; @@ -872,10 +873,8 @@ void __android_log_btwrite_multiple__helper(int count) { ASSERT_EQ(0, info.si_status); struct logger_list* logger_list; - ASSERT_TRUE(nullptr != - (logger_list = android_logger_list_open( - LOG_ID_EVENTS, ANDROID_LOG_RDONLY | ANDROID_LOG_NONBLOCK, - 0, pid))); + ASSERT_TRUE(nullptr != (logger_list = android_logger_list_open(LOG_ID_EVENTS, + ANDROID_LOG_NONBLOCK, 0, pid))); int expected_count = (count < 2) ? count : 2; int expected_chatty_count = (count <= 2) ? 0 : 1; @@ -905,10 +904,10 @@ void __android_log_btwrite_multiple__helper(int count) { (log_msg.entry.len == (4 + 1 + 8))) { if (tag != 0) continue; - log_time tx(eventData + 4 + 1); - if (ts == tx) { + log_time* tx = reinterpret_cast<log_time*>(eventData + 4 + 1); + if (ts == *tx) { ++count; - } else if (ts1 == tx) { + } else if (ts1 == *tx) { ++second_count; } } else if (eventData[4] == EVENT_TYPE_STRING) { diff --git a/logd/main.cpp b/logd/main.cpp index 23bbf864d..46b65679a 100644 --- a/logd/main.cpp +++ b/logd/main.cpp @@ -36,9 +36,10 @@ #include <memory> +#include <android-base/logging.h> #include <android-base/macros.h> +#include <android-base/stringprintf.h> #include <cutils/android_get_control_file.h> -#include <cutils/properties.h> #include <cutils/sockets.h> #include <log/event_tag_map.h> #include <packagelistparser/packagelistparser.h> @@ -47,12 +48,18 @@ #include <processgroup/sched_policy.h> #include <utils/threads.h> +#include "ChattyLogBuffer.h" #include "CommandListener.h" #include "LogAudit.h" #include "LogBuffer.h" #include "LogKlog.h" #include "LogListener.h" +#include "LogReader.h" +#include "LogStatistics.h" +#include "LogTags.h" #include "LogUtils.h" +#include "SerializedLogBuffer.h" +#include "SimpleLogBuffer.h" #define KMSG_PRIORITY(PRI) \ '<', '0' + LOG_MAKEPRI(LOG_DAEMON, LOG_PRI(PRI)) / 10, \ @@ -66,19 +73,18 @@ static int drop_privs(bool klogd, bool auditd) { sched_param param = {}; if (set_sched_policy(0, SP_BACKGROUND) < 0) { - android::prdebug("failed to set background scheduling policy"); + PLOG(ERROR) << "failed to set background scheduling policy"; return -1; } if (sched_setscheduler((pid_t)0, SCHED_BATCH, ¶m) < 0) { - android::prdebug("failed to set batch scheduler"); + PLOG(ERROR) << "failed to set batch scheduler"; return -1; } - if (!__android_logger_property_get_bool("ro.debuggable", - BOOL_DEFAULT_FALSE) && + if (!__android_logger_property_get_bool("ro.debuggable", BOOL_DEFAULT_FALSE) && prctl(PR_SET_DUMPABLE, 0) == -1) { - android::prdebug("failed to clear PR_SET_DUMPABLE"); + PLOG(ERROR) << "failed to clear PR_SET_DUMPABLE"; return -1; } @@ -101,99 +107,13 @@ static int drop_privs(bool klogd, bool auditd) { return -1; } if (cap_set_proc(caps.get()) < 0) { - android::prdebug("failed to set CAP_SYSLOG or CAP_AUDIT_CONTROL (%d)", errno); + PLOG(ERROR) << "failed to set CAP_SYSLOG or CAP_AUDIT_CONTROL"; return -1; } return 0; } -// Property helper -static bool check_flag(const char* prop, const char* flag) { - const char* cp = strcasestr(prop, flag); - if (!cp) { - return false; - } - // We only will document comma (,) - static const char sep[] = ",:;|+ \t\f"; - if ((cp != prop) && !strchr(sep, cp[-1])) { - return false; - } - cp += strlen(flag); - return !*cp || !!strchr(sep, *cp); -} - -static int fdDmesg = -1; -void android::prdebug(const char* fmt, ...) { - if (fdDmesg < 0) { - return; - } - - static const char message[] = { - KMSG_PRIORITY(LOG_DEBUG), 'l', 'o', 'g', 'd', ':', ' ' - }; - char buffer[256]; - memcpy(buffer, message, sizeof(message)); - - va_list ap; - va_start(ap, fmt); - int n = vsnprintf(buffer + sizeof(message), - sizeof(buffer) - sizeof(message), fmt, ap); - va_end(ap); - if (n > 0) { - buffer[sizeof(buffer) - 1] = '\0'; - if (!strchr(buffer, '\n')) { - buffer[sizeof(buffer) - 2] = '\0'; - strlcat(buffer, "\n", sizeof(buffer)); - } - write(fdDmesg, buffer, strlen(buffer)); - } -} - -static sem_t reinit; -static bool reinit_running = false; -static LogBuffer* logBuf = nullptr; - -static void* reinit_thread_start(void* /*obj*/) { - prctl(PR_SET_NAME, "logd.daemon"); - - while (reinit_running && !sem_wait(&reinit) && reinit_running) { - if (fdDmesg >= 0) { - static const char reinit_message[] = { KMSG_PRIORITY(LOG_INFO), - 'l', - 'o', - 'g', - 'd', - '.', - 'd', - 'a', - 'e', - 'm', - 'o', - 'n', - ':', - ' ', - 'r', - 'e', - 'i', - 'n', - 'i', - 't', - '\n' }; - write(fdDmesg, reinit_message, sizeof(reinit_message)); - } - - // Anything that reads persist.<property> - if (logBuf) { - logBuf->init(); - logBuf->initPrune(nullptr); - } - android::ReReadEventLogTags(); - } - - return nullptr; -} - char* android::uidToName(uid_t u) { struct Userdata { uid_t uid; @@ -220,12 +140,6 @@ char* android::uidToName(uid_t u) { return userdata.name; } -// Serves as a global method to trigger reinitialization -// and as a function that can be provided to signal(). -void reinit_signal_handler(int /*signal*/) { - sem_post(&reinit); -} - static void readDmesg(LogAudit* al, LogKlog* kl) { if (!al && !kl) { return; @@ -250,10 +164,6 @@ static void readDmesg(LogAudit* al, LogKlog* kl) { } buf[--len] = '\0'; - if (kl && kl->isMonotonic()) { - kl->synchronize(buf.get(), len); - } - ssize_t sublen; for (char *ptr = nullptr, *tok = buf.get(); (rc >= 0) && !!(tok = android::log_strntok_r(tok, len, ptr, sublen)); @@ -311,8 +221,20 @@ int main(int argc, char* argv[]) { return issueReinit(); } + android::base::InitLogging( + argv, [](android::base::LogId log_id, android::base::LogSeverity severity, + const char* tag, const char* file, unsigned int line, const char* message) { + if (tag && strcmp(tag, "logd") != 0) { + auto prefixed_message = android::base::StringPrintf("%s: %s", tag, message); + android::base::KernelLogger(log_id, severity, "logd", file, line, + prefixed_message.c_str()); + } else { + android::base::KernelLogger(log_id, severity, "logd", file, line, message); + } + }); + static const char dev_kmsg[] = "/dev/kmsg"; - fdDmesg = android_get_control_file(dev_kmsg); + int fdDmesg = android_get_control_file(dev_kmsg); if (fdDmesg < 0) { fdDmesg = TEMP_FAILURE_RETRY(open(dev_kmsg, O_WRONLY | O_CLOEXEC)); } @@ -328,7 +250,7 @@ int main(int argc, char* argv[]) { fdPmesg = TEMP_FAILURE_RETRY( open(proc_kmsg, O_RDONLY | O_NDELAY | O_CLOEXEC)); } - if (fdPmesg < 0) android::prdebug("Failed to open %s\n", proc_kmsg); + if (fdPmesg < 0) PLOG(ERROR) << "Failed to open " << proc_kmsg; } bool auditd = __android_logger_property_get_bool("ro.logd.auditd", BOOL_DEFAULT_TRUE); @@ -336,49 +258,34 @@ int main(int argc, char* argv[]) { return EXIT_FAILURE; } - // Reinit Thread - sem_init(&reinit, 0, 0); - pthread_attr_t attr; - if (!pthread_attr_init(&attr)) { - struct sched_param param; - - memset(¶m, 0, sizeof(param)); - pthread_attr_setschedparam(&attr, ¶m); - pthread_attr_setschedpolicy(&attr, SCHED_BATCH); - if (!pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED)) { - pthread_t thread; - reinit_running = true; - if (pthread_create(&thread, &attr, reinit_thread_start, nullptr)) { - reinit_running = false; - } - } - pthread_attr_destroy(&attr); - } + // A cache of event log tags + LogTags log_tags; + + // Pruning configuration. + PruneList prune_list; + + // Partial (required for chatty) or full logging statistics. + bool enable_full_log_statistics = __android_logger_property_get_bool( + "logd.statistics", BOOL_DEFAULT_TRUE | BOOL_DEFAULT_FLAG_PERSIST | + BOOL_DEFAULT_FLAG_ENG | BOOL_DEFAULT_FLAG_SVELTE); + LogStatistics log_statistics(enable_full_log_statistics); // Serves the purpose of managing the last logs times read on a // socket connection, and as a reader lock on a range of log // entries. + LogReaderList reader_list; - LastLogTimes* times = new LastLogTimes(); - - // LogBuffer is the object which is responsible for holding all - // log entries. - - logBuf = new LogBuffer(times); - - signal(SIGHUP, reinit_signal_handler); - - if (__android_logger_property_get_bool( - "logd.statistics", BOOL_DEFAULT_TRUE | BOOL_DEFAULT_FLAG_PERSIST | - BOOL_DEFAULT_FLAG_ENG | - BOOL_DEFAULT_FLAG_SVELTE)) { - logBuf->enableStatistics(); + // LogBuffer is the object which is responsible for holding all log entries. + LogBuffer* logBuf; + if (true) { + logBuf = new ChattyLogBuffer(&reader_list, &log_tags, &prune_list, &log_statistics); + } else { + logBuf = new SimpleLogBuffer(&reader_list, &log_tags, &log_statistics); } // LogReader listens on /dev/socket/logdr. When a client // connects, log entries in the LogBuffer are written to the client. - - LogReader* reader = new LogReader(logBuf); + LogReader* reader = new LogReader(logBuf, &reader_list); if (reader->startListener()) { return EXIT_FAILURE; } @@ -386,17 +293,14 @@ int main(int argc, char* argv[]) { // LogListener listens on /dev/socket/logdw for client // initiated log messages. New log entries are added to LogBuffer // and LogReader is notified to send updates to connected clients. - - LogListener* swl = new LogListener(logBuf, reader); - // Backlog and /proc/sys/net/unix/max_dgram_qlen set to large value - if (swl->startListener(600)) { + LogListener* swl = new LogListener(logBuf); + if (!swl->StartListener()) { return EXIT_FAILURE; } // Command listener listens on /dev/socket/logd for incoming logd // administrative commands. - - CommandListener* cl = new CommandListener(logBuf, reader, swl); + CommandListener* cl = new CommandListener(logBuf, &log_tags, &prune_list, &log_statistics); if (cl->startListener()) { return EXIT_FAILURE; } @@ -404,25 +308,22 @@ int main(int argc, char* argv[]) { // LogAudit listens on NETLINK_AUDIT socket for selinux // initiated log messages. New log entries are added to LogBuffer // and LogReader is notified to send updates to connected clients. - LogAudit* al = nullptr; if (auditd) { - al = new LogAudit(logBuf, reader, - __android_logger_property_get_bool( - "ro.logd.auditd.dmesg", BOOL_DEFAULT_TRUE) - ? fdDmesg - : -1); + int dmesg_fd = __android_logger_property_get_bool("ro.logd.auditd.dmesg", BOOL_DEFAULT_TRUE) + ? fdDmesg + : -1; + al = new LogAudit(logBuf, dmesg_fd, &log_statistics); } LogKlog* kl = nullptr; if (klogd) { - kl = new LogKlog(logBuf, reader, fdDmesg, fdPmesg, al != nullptr); + kl = new LogKlog(logBuf, fdDmesg, fdPmesg, al != nullptr, &log_statistics); } readDmesg(al, kl); // failure is an option ... messages are in dmesg (required by standard) - if (kl && kl->startListener()) { delete kl; } diff --git a/logd/rwlock.h b/logd/rwlock.h new file mode 100644 index 000000000..c37721ee3 --- /dev/null +++ b/logd/rwlock.h @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <pthread.h> + +#include <android-base/macros.h> +#include <android-base/thread_annotations.h> + +// As of the end of May 2020, std::shared_mutex is *not* simply a pthread_rwlock, but rather a +// combination of std::mutex and std::condition variable, which is obviously less efficient. This +// immitates what std::shared_mutex should be doing and is compatible with RAII thread wrappers. + +class SHARED_CAPABILITY("mutex") RwLock { + public: + RwLock() {} + ~RwLock() {} + + void lock() ACQUIRE() { pthread_rwlock_wrlock(&rwlock_); } + void lock_shared() ACQUIRE_SHARED() { pthread_rwlock_rdlock(&rwlock_); } + + void unlock() RELEASE() { pthread_rwlock_unlock(&rwlock_); } + + private: + pthread_rwlock_t rwlock_ = PTHREAD_RWLOCK_INITIALIZER; +}; + +// std::shared_lock does not have thread annotations, so we need our own. + +class SCOPED_CAPABILITY SharedLock { + public: + explicit SharedLock(RwLock& lock) ACQUIRE_SHARED(lock) : lock_(lock) { lock_.lock_shared(); } + ~SharedLock() RELEASE() { lock_.unlock(); } + + void lock_shared() ACQUIRE_SHARED() { lock_.lock_shared(); } + void unlock() RELEASE() { lock_.unlock(); } + + DISALLOW_IMPLICIT_CONSTRUCTORS(SharedLock); + + private: + RwLock& lock_; +}; diff --git a/logd/tests/Android.bp b/logd/tests/Android.bp deleted file mode 100644 index 9a5defa17..000000000 --- a/logd/tests/Android.bp +++ /dev/null @@ -1,68 +0,0 @@ -// -// Copyright (C) 2014 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -// ----------------------------------------------------------------------------- -// Unit tests. -// ----------------------------------------------------------------------------- - -cc_defaults { - name: "logd-unit-test-defaults", - - cflags: [ - "-fstack-protector-all", - "-g", - "-Wall", - "-Wextra", - "-Werror", - "-fno-builtin", - - "-DAUDITD_LOG_TAG=1003", - "-DCHATTY_LOG_TAG=1004", - ], - - srcs: ["logd_test.cpp"], - - static_libs: [ - "libbase", - "libcutils", - "libselinux", - "liblog", - ], -} - -// Build tests for the logger. Run with: -// adb shell /data/nativetest/logd-unit-tests/logd-unit-tests -cc_test { - name: "logd-unit-tests", - defaults: ["logd-unit-test-defaults"], -} - -cc_test { - name: "CtsLogdTestCases", - defaults: ["logd-unit-test-defaults"], - multilib: { - lib32: { - suffix: "32", - }, - lib64: { - suffix: "64", - }, - }, - test_suites: [ - "cts", - "vts10", - ], -} diff --git a/rootdir/etc/public.libraries.android.txt b/rootdir/etc/public.libraries.android.txt index 405f5a9fc..5de422f62 100644 --- a/rootdir/etc/public.libraries.android.txt +++ b/rootdir/etc/public.libraries.android.txt @@ -16,6 +16,7 @@ libjnigraphics.so liblog.so libmediandk.so libm.so +libnativehelper.so libnativewindow.so libneuralnetworks.so nopreload libOpenMAXAL.so diff --git a/rootdir/etc/public.libraries.iot.txt b/rootdir/etc/public.libraries.iot.txt index b56534070..77f8bb802 100644 --- a/rootdir/etc/public.libraries.iot.txt +++ b/rootdir/etc/public.libraries.iot.txt @@ -17,6 +17,7 @@ libjnigraphics.so liblog.so libmediandk.so libm.so +libnativehelper.so libnativewindow.so libneuralnetworks.so libOpenMAXAL.so diff --git a/rootdir/etc/public.libraries.wear.txt b/rootdir/etc/public.libraries.wear.txt index 7cbda0812..82196e4e8 100644 --- a/rootdir/etc/public.libraries.wear.txt +++ b/rootdir/etc/public.libraries.wear.txt @@ -16,6 +16,7 @@ libjnigraphics.so liblog.so libmediandk.so libm.so +libnativehelper.so libnativewindow.so libneuralnetworks.so libOpenMAXAL.so diff --git a/rootdir/init.rc b/rootdir/init.rc index 73ac7fd0d..7b57a888d 100644 --- a/rootdir/init.rc +++ b/rootdir/init.rc @@ -493,6 +493,9 @@ on post-fs chown root log /proc/slabinfo chmod 0440 /proc/slabinfo + chown root log /proc/pagetypeinfo + chmod 0440 /proc/pagetypeinfo + #change permissions on kmsg & sysrq-trigger so bugreports can grab kthread stacks chown root system /proc/kmsg chmod 0440 /proc/kmsg @@ -833,6 +836,11 @@ on zygote-start && property:ro.crypto.state=encrypted && property:ro.crypto.type start zygote start zygote_secondary +on boot && property:ro.config.low_ram=true + # Tweak background writeout + write /proc/sys/vm/dirty_expire_centisecs 200 + write /proc/sys/vm/dirty_background_ratio 5 + on boot # basic network init ifup lo @@ -854,11 +862,7 @@ on boot chown root system /sys/block/zram0/writeback chmod 0664 /sys/block/zram0/writeback - # Tweak background writeout - write /proc/sys/vm/dirty_expire_centisecs 200 - write /proc/sys/vm/dirty_background_ratio 5 - - # F2FS tuning. Set cp_interval larger than dirty_expire_centisecs + # F2FS tuning. Set cp_interval larger than dirty_expire_centisecs, 30 secs, # to avoid power consumption when system becomes mostly idle. Be careful # to make it too large, since it may bring userdata loss, if they # are not aware of using fsync()/sync() to prepare sudden power-cut. diff --git a/rootdir/init.zygote32_64.rc b/rootdir/init.zygote32_64.rc deleted file mode 100644 index fe6dcfa47..000000000 --- a/rootdir/init.zygote32_64.rc +++ /dev/null @@ -1,25 +0,0 @@ -service zygote /system/bin/app_process32 -Xzygote /system/bin --zygote --start-system-server --socket-name=zygote - class main - priority -20 - user root - group root readproc reserved_disk - socket zygote stream 660 root system - socket usap_pool_primary stream 660 root system - onrestart exec_background - system system -- /system/bin/vdc volume abort_fuse - onrestart write /sys/power/state on - onrestart restart audioserver - onrestart restart cameraserver - onrestart restart media - onrestart restart netd - onrestart restart wificond - writepid /dev/cpuset/foreground/tasks - -service zygote_secondary /system/bin/app_process64 -Xzygote /system/bin --zygote --socket-name=zygote_secondary - class main - priority -20 - user root - group root readproc reserved_disk - socket zygote_secondary stream 660 root system - socket usap_pool_secondary stream 660 root system - onrestart restart zygote - writepid /dev/cpuset/foreground/tasks diff --git a/run-as/run-as.cpp b/run-as/run-as.cpp index 432c434b4..cc92c68d0 100644 --- a/run-as/run-as.cpp +++ b/run-as/run-as.cpp @@ -247,12 +247,12 @@ int main(int argc, char* argv[]) { std::string seinfo = std::string(info.seinfo) + ":fromRunAs"; if (selinux_android_setcontext(uid, 0, seinfo.c_str(), pkgname) < 0) { - error(1, errno, "couldn't set SELinux security context"); + error(1, errno, "couldn't set SELinux security context '%s'", seinfo.c_str()); } // cd into the data directory, and set $HOME correspondingly. if (TEMP_FAILURE_RETRY(chdir(info.data_dir)) == -1) { - error(1, errno, "couldn't chdir to package's data directory"); + error(1, errno, "couldn't chdir to package's data directory '%s'", info.data_dir); } setenv("HOME", info.data_dir, 1); diff --git a/shell_and_utilities/Android.bp b/shell_and_utilities/Android.bp index b5a5fb6ce..f83c43ea3 100644 --- a/shell_and_utilities/Android.bp +++ b/shell_and_utilities/Android.bp @@ -36,7 +36,7 @@ phony { "sh.recovery", "toolbox.recovery", "toybox.recovery", - "unzip.recovery", + "ziptool.recovery", ], } diff --git a/trusty/keymaster/4.0/android.hardware.keymaster@4.0-service.trusty.xml b/trusty/keymaster/4.0/android.hardware.keymaster@4.0-service.trusty.xml index aa30707ca..65eb8549a 100644 --- a/trusty/keymaster/4.0/android.hardware.keymaster@4.0-service.trusty.xml +++ b/trusty/keymaster/4.0/android.hardware.keymaster@4.0-service.trusty.xml @@ -2,10 +2,6 @@ <hal format="hidl"> <name>android.hardware.keymaster</name> <transport>hwbinder</transport> - <version>4.0</version> - <interface> - <name>IKeymasterDevice</name> - <instance>default</instance> - </interface> + <fqname>@4.0::IKeymasterDevice/default</fqname> </hal> </manifest> diff --git a/trusty/keymaster/Android.bp b/trusty/keymaster/Android.bp index f32a69ea8..6840baa54 100644 --- a/trusty/keymaster/Android.bp +++ b/trusty/keymaster/Android.bp @@ -14,70 +14,6 @@ // limitations under the License. // -// WARNING: Everything listed here will be built on ALL platforms, -// including x86, the emulator, and the SDK. Modules must be uniquely -// named (liblights.panda), and must build everywhere, or limit themselves -// to only building on ARM if they include assembly. Individual makefiles -// are responsible for having their own logic, for fine-grained control. - -// trusty_keymaster is a binary used only for on-device testing. It -// runs Trusty Keymaster through a basic set of operations with RSA -// and ECDSA keys. -cc_binary { - name: "trusty_keymaster_tipc", - vendor: true, - srcs: [ - "ipc/trusty_keymaster_ipc.cpp", - "legacy/trusty_keymaster_device.cpp", - "legacy/trusty_keymaster_main.cpp", - ], - cflags: [ - "-Wall", - "-Werror", - ], - - local_include_dirs: ["include"], - - shared_libs: [ - "libcrypto", - "libcutils", - "libkeymaster_portable", - "libtrusty", - "libkeymaster_messages", - "libsoftkeymasterdevice", - "liblog", - ], -} - -// keystore.trusty is the HAL used by keystore on Trusty devices. -cc_library_shared { - name: "keystore.trusty", - vendor: true, - relative_install_path: "hw", - srcs: [ - "ipc/trusty_keymaster_ipc.cpp", - "legacy/module.cpp", - "legacy/trusty_keymaster_device.cpp", - ], - - cflags: [ - "-fvisibility=hidden", - "-Wall", - "-Werror", - ], - - local_include_dirs: ["include"], - - shared_libs: [ - "libcrypto", - "libkeymaster_messages", - "libtrusty", - "liblog", - "libcutils", - ], - header_libs: ["libhardware_headers"], -} - cc_binary { name: "android.hardware.keymaster@3.0-service.trusty", defaults: ["hidl_defaults"], diff --git a/trusty/keymaster/legacy/Makefile b/trusty/keymaster/legacy/Makefile deleted file mode 100644 index f57538189..000000000 --- a/trusty/keymaster/legacy/Makefile +++ /dev/null @@ -1,199 +0,0 @@ -##### -# Local unit test Makefile -# -# This makefile builds and runs the trusty_keymaster unit tests locally on the development -# machine, not on an Android device. -# -# To build and run these tests, one pre-requisite must be manually installed: BoringSSL. -# This Makefile expects to find BoringSSL in a directory adjacent to $ANDROID_BUILD_TOP. -# To get and build it, first install the Ninja build tool (e.g. apt-get install -# ninja-build), then do: -# -# cd $ANDROID_BUILD_TOP/.. -# git clone https://boringssl.googlesource.com/boringssl -# cd boringssl -# mdkir build -# cd build -# cmake -GNinja .. -# ninja -# -# Then return to $ANDROID_BUILD_TOP/system/keymaster and run "make". -##### - -BASE=../../../.. -SUBS=system/core \ - system/keymaster \ - hardware/libhardware \ - external/gtest -GTEST=$(BASE)/external/gtest -KM=$(BASE)/system/keymaster - -INCLUDES=$(foreach dir,$(SUBS),-I $(BASE)/$(dir)/include) \ - -I $(BASE)/libnativehelper/include/nativehelper \ - -I ../tipc/include \ - -I $(BASE)/system/keymaster \ - -I $(GTEST) \ - -I$(BASE)/../boringssl/include - -ifdef USE_CLANG -CC=/usr/bin/clang -CXX=/usr/bin/clang -CLANG_TEST_DEFINE=-DKEYMASTER_CLANG_TEST_BUILD -COMPILER_SPECIFIC_ARGS=-std=c++11 $(CLANG_TEST_DEFINE) -else -COMPILER_SPECIFIC_ARGS=-std=c++0x -fprofile-arcs -endif - -CPPFLAGS=$(INCLUDES) -g -O0 -MD -CXXFLAGS=-Wall -Werror -Wno-unused -Winit-self -Wpointer-arith -Wunused-parameter \ - -Wmissing-declarations -ftest-coverage \ - -Wno-deprecated-declarations -fno-exceptions -DKEYMASTER_NAME_TAGS \ - $(COMPILER_SPECIFIC_ARGS) -LDLIBS=-L$(BASE)/../boringssl/build/crypto -lcrypto -lpthread -lstdc++ - -CPPSRCS=\ - $(KM)/aead_mode_operation.cpp \ - $(KM)/aes_key.cpp \ - $(KM)/aes_operation.cpp \ - $(KM)/android_keymaster.cpp \ - $(KM)/android_keymaster_messages.cpp \ - $(KM)/android_keymaster_messages_test.cpp \ - $(KM)/android_keymaster_test.cpp \ - $(KM)/android_keymaster_test_utils.cpp \ - $(KM)/android_keymaster_utils.cpp \ - $(KM)/asymmetric_key.cpp \ - $(KM)/auth_encrypted_key_blob.cpp \ - $(KM)/auth_encrypted_key_blob.cpp \ - $(KM)/authorization_set.cpp \ - $(KM)/authorization_set_test.cpp \ - $(KM)/ec_key.cpp \ - $(KM)/ec_keymaster0_key.cpp \ - $(KM)/ecdsa_operation.cpp \ - $(KM)/hmac_key.cpp \ - $(KM)/hmac_operation.cpp \ - $(KM)/integrity_assured_key_blob.cpp \ - $(KM)/key.cpp \ - $(KM)/key_blob_test.cpp \ - $(KM)/keymaster0_engine.cpp \ - $(KM)/logger.cpp \ - $(KM)/ocb_utils.cpp \ - $(KM)/openssl_err.cpp \ - $(KM)/openssl_utils.cpp \ - $(KM)/operation.cpp \ - $(KM)/operation_table.cpp \ - $(KM)/rsa_key.cpp \ - $(KM)/rsa_keymaster0_key.cpp \ - $(KM)/rsa_operation.cpp \ - $(KM)/serializable.cpp \ - $(KM)/soft_keymaster_context.cpp \ - $(KM)/symmetric_key.cpp \ - $(KM)/unencrypted_key_blob.cpp \ - trusty_keymaster_device.cpp \ - trusty_keymaster_device_test.cpp -CCSRCS=$(GTEST)/src/gtest-all.cc -CSRCS=ocb.c - -OBJS=$(CPPSRCS:.cpp=.o) $(CCSRCS:.cc=.o) $(CSRCS:.c=.o) -DEPS=$(CPPSRCS:.cpp=.d) $(CCSRCS:.cc=.d) $(CSRCS:.c=.d) -GCDA=$(CPPSRCS:.cpp=.gcda) $(CCSRCS:.cc=.gcda) $(CSRCS:.c=.gcda) -GCNO=$(CPPSRCS:.cpp=.gcno) $(CCSRCS:.cc=.gcno) $(CSRCS:.c=.gcno) - -LINK.o=$(LINK.cc) - -BINARIES=trusty_keymaster_device_test - -ifdef TRUSTY -BINARIES += trusty_keymaster_device_test -endif # TRUSTY - -.PHONY: coverage memcheck massif clean run - -%.run: % - ./$< - touch $@ - -run: $(BINARIES:=.run) - -coverage: coverage.info - genhtml coverage.info --output-directory coverage - -coverage.info: run - lcov --capture --directory=. --output-file coverage.info - -%.coverage : % - $(MAKE) clean && $(MAKE) $< - ./$< - lcov --capture --directory=. --output-file coverage.info - genhtml coverage.info --output-directory coverage - -#UNINIT_OPTS=--track-origins=yes -UNINIT_OPTS=--undef-value-errors=no - -MEMCHECK_OPTS=--leak-check=full \ - --show-reachable=yes \ - --vgdb=full \ - $(UNINIT_OPTS) \ - --error-exitcode=1 - -MASSIF_OPTS=--tool=massif \ - --stacks=yes - -%.memcheck : % - valgrind $(MEMCHECK_OPTS) ./$< && \ - touch $@ - -%.massif : % - valgrind $(MASSIF_OPTS) --massif-out-file=$@ ./$< - -memcheck: $(BINARIES:=.memcheck) - -massif: $(BINARIES:=.massif) - -trusty_keymaster_device_test: trusty_keymaster_device_test.o \ - trusty_keymaster_device.o \ - $(KM)/aead_mode_operation.o \ - $(KM)/aes_key.o \ - $(KM)/aes_operation.o \ - $(KM)/android_keymaster.o \ - $(KM)/android_keymaster_messages.o \ - $(KM)/android_keymaster_test_utils.o \ - $(KM)/android_keymaster_utils.o \ - $(KM)/asymmetric_key.o \ - $(KM)/auth_encrypted_key_blob.o \ - $(KM)/auth_encrypted_key_blob.o \ - $(KM)/authorization_set.o \ - $(KM)/ec_key.o \ - $(KM)/ec_keymaster0_key.cpp \ - $(KM)/ecdsa_operation.o \ - $(KM)/hmac_key.o \ - $(KM)/hmac_operation.o \ - $(KM)/integrity_assured_key_blob.o \ - $(KM)/key.o \ - $(KM)/keymaster0_engine.o \ - $(KM)/logger.o \ - $(KM)/ocb.o \ - $(KM)/ocb_utils.o \ - $(KM)/openssl_err.o \ - $(KM)/openssl_utils.o \ - $(KM)/operation.o \ - $(KM)/operation_table.o \ - $(KM)/rsa_key.o \ - $(KM)/rsa_keymaster0_key.o \ - $(KM)/rsa_operation.o \ - $(KM)/serializable.o \ - $(KM)/soft_keymaster_context.o \ - $(KM)/symmetric_key.o \ - $(GTEST)/src/gtest-all.o - -$(GTEST)/src/gtest-all.o: CXXFLAGS:=$(subst -Wmissing-declarations,,$(CXXFLAGS)) -ocb.o: CFLAGS=$(CLANG_TEST_DEFINE) - -clean: - rm -f $(OBJS) $(DEPS) $(GCDA) $(GCNO) $(BINARIES) \ - $(BINARIES:=.run) $(BINARIES:=.memcheck) $(BINARIES:=.massif) \ - coverage.info - rm -rf coverage - --include $(CPPSRCS:.cpp=.d) --include $(CCSRCS:.cc=.d) - diff --git a/trusty/keymaster/legacy/module.cpp b/trusty/keymaster/legacy/module.cpp deleted file mode 100644 index 7aa1a4eaa..000000000 --- a/trusty/keymaster/legacy/module.cpp +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include <errno.h> -#include <string.h> - -#include <hardware/hardware.h> -#include <hardware/keymaster0.h> - -#include <trusty_keymaster/legacy/trusty_keymaster_device.h> - -using keymaster::TrustyKeymasterDevice; - -/* - * Generic device handling - */ -static int trusty_keymaster_open(const hw_module_t* module, const char* name, - hw_device_t** device) { - if (strcmp(name, KEYSTORE_KEYMASTER) != 0) { - return -EINVAL; - } - - TrustyKeymasterDevice* dev = new TrustyKeymasterDevice(module); - if (dev == NULL) { - return -ENOMEM; - } - *device = dev->hw_device(); - // Do not delete dev; it will get cleaned up when the caller calls device->close(), and must - // exist until then. - return 0; -} - -static struct hw_module_methods_t keystore_module_methods = { - .open = trusty_keymaster_open, -}; - -struct keystore_module HAL_MODULE_INFO_SYM __attribute__((visibility("default"))) = { - .common = - { - .tag = HARDWARE_MODULE_TAG, - .module_api_version = KEYMASTER_MODULE_API_VERSION_2_0, - .hal_api_version = HARDWARE_HAL_API_VERSION, - .id = KEYSTORE_HARDWARE_MODULE_ID, - .name = "Trusty Keymaster HAL", - .author = "The Android Open Source Project", - .methods = &keystore_module_methods, - .dso = 0, - .reserved = {}, - }, -}; diff --git a/trusty/keymaster/legacy/trusty_keymaster_device.cpp b/trusty/keymaster/legacy/trusty_keymaster_device.cpp deleted file mode 100644 index 88c3e7bbd..000000000 --- a/trusty/keymaster/legacy/trusty_keymaster_device.cpp +++ /dev/null @@ -1,761 +0,0 @@ -/* - * Copyright 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define LOG_TAG "TrustyKeymaster" - -#include <assert.h> -#include <errno.h> -#include <openssl/evp.h> -#include <openssl/x509.h> -#include <stddef.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <time.h> - -#include <algorithm> -#include <type_traits> - -#include <hardware/keymaster2.h> -#include <keymaster/authorization_set.h> -#include <log/log.h> - -#include <trusty_keymaster/ipc/keymaster_ipc.h> -#include <trusty_keymaster/ipc/trusty_keymaster_ipc.h> -#include <trusty_keymaster/legacy/trusty_keymaster_device.h> - -const size_t kMaximumAttestationChallengeLength = 128; -const size_t kMaximumFinishInputLength = 2048; - -namespace keymaster { - -TrustyKeymasterDevice::TrustyKeymasterDevice(const hw_module_t* module) { - static_assert(std::is_standard_layout<TrustyKeymasterDevice>::value, - "TrustyKeymasterDevice must be standard layout"); - static_assert(offsetof(TrustyKeymasterDevice, device_) == 0, - "device_ must be the first member of TrustyKeymasterDevice"); - static_assert(offsetof(TrustyKeymasterDevice, device_.common) == 0, - "common must be the first member of keymaster2_device"); - - ALOGI("Creating device"); - ALOGD("Device address: %p", this); - - device_ = {}; - - device_.common.tag = HARDWARE_DEVICE_TAG; - device_.common.version = 1; - device_.common.module = const_cast<hw_module_t*>(module); - device_.common.close = close_device; - - device_.flags = KEYMASTER_SUPPORTS_EC; - - device_.configure = configure; - device_.add_rng_entropy = add_rng_entropy; - device_.generate_key = generate_key; - device_.get_key_characteristics = get_key_characteristics; - device_.import_key = import_key; - device_.export_key = export_key; - device_.attest_key = attest_key; - device_.upgrade_key = upgrade_key; - device_.delete_key = delete_key; - device_.delete_all_keys = delete_all_keys; - device_.begin = begin; - device_.update = update; - device_.finish = finish; - device_.abort = abort; - - int rc = trusty_keymaster_connect(); - error_ = translate_error(rc); - if (rc < 0) { - ALOGE("failed to connect to keymaster (%d)", rc); - return; - } - - GetVersionRequest version_request; - GetVersionResponse version_response; - error_ = trusty_keymaster_send(KM_GET_VERSION, version_request, &version_response); - if (error_ == KM_ERROR_INVALID_ARGUMENT || error_ == KM_ERROR_UNIMPLEMENTED) { - ALOGE("\"Bad parameters\" error on GetVersion call. Version 0 is not supported."); - error_ = KM_ERROR_VERSION_MISMATCH; - return; - } - message_version_ = MessageVersion(version_response.major_ver, version_response.minor_ver, - version_response.subminor_ver); - if (message_version_ < 0) { - // Can't translate version? Keymaster implementation must be newer. - ALOGE("Keymaster version %d.%d.%d not supported.", version_response.major_ver, - version_response.minor_ver, version_response.subminor_ver); - error_ = KM_ERROR_VERSION_MISMATCH; - } -} - -TrustyKeymasterDevice::~TrustyKeymasterDevice() { - trusty_keymaster_disconnect(); -} - -namespace { - -// Allocates a new buffer with malloc and copies the contents of |buffer| to it. Caller takes -// ownership of the returned buffer. -uint8_t* DuplicateBuffer(const uint8_t* buffer, size_t size) { - uint8_t* tmp = reinterpret_cast<uint8_t*>(malloc(size)); - if (tmp) { - memcpy(tmp, buffer, size); - } - return tmp; -} - -template <typename RequestType> -void AddClientAndAppData(const keymaster_blob_t* client_id, const keymaster_blob_t* app_data, - RequestType* request) { - request->additional_params.Clear(); - if (client_id && client_id->data_length > 0) { - request->additional_params.push_back(TAG_APPLICATION_ID, *client_id); - } - if (app_data && app_data->data_length > 0) { - request->additional_params.push_back(TAG_APPLICATION_DATA, *app_data); - } -} - -} // unnamed namespace - -keymaster_error_t TrustyKeymasterDevice::configure(const keymaster_key_param_set_t* params) { - ALOGD("Device received configure\n"); - - if (error_ != KM_ERROR_OK) { - return error_; - } - if (!params) { - return KM_ERROR_UNEXPECTED_NULL_POINTER; - } - - AuthorizationSet params_copy(*params); - ConfigureRequest request(message_version_); - if (!params_copy.GetTagValue(TAG_OS_VERSION, &request.os_version) || - !params_copy.GetTagValue(TAG_OS_PATCHLEVEL, &request.os_patchlevel)) { - ALOGD("Configuration parameters must contain OS version and patch level"); - return KM_ERROR_INVALID_ARGUMENT; - } - - ConfigureResponse response(message_version_); - keymaster_error_t err = trusty_keymaster_send(KM_CONFIGURE, request, &response); - if (err != KM_ERROR_OK) { - return err; - } - - return KM_ERROR_OK; -} - -keymaster_error_t TrustyKeymasterDevice::add_rng_entropy(const uint8_t* data, size_t data_length) { - ALOGD("Device received add_rng_entropy"); - - if (error_ != KM_ERROR_OK) { - return error_; - } - - AddEntropyRequest request(message_version_); - request.random_data.Reinitialize(data, data_length); - AddEntropyResponse response(message_version_); - return trusty_keymaster_send(KM_ADD_RNG_ENTROPY, request, &response); -} - -keymaster_error_t TrustyKeymasterDevice::generate_key( - const keymaster_key_param_set_t* params, keymaster_key_blob_t* key_blob, - keymaster_key_characteristics_t* characteristics) { - ALOGD("Device received generate_key"); - - if (error_ != KM_ERROR_OK) { - return error_; - } - if (!params) { - return KM_ERROR_UNEXPECTED_NULL_POINTER; - } - if (!key_blob) { - return KM_ERROR_OUTPUT_PARAMETER_NULL; - } - - GenerateKeyRequest request(message_version_); - request.key_description.Reinitialize(*params); - request.key_description.push_back(TAG_CREATION_DATETIME, java_time(time(NULL))); - - GenerateKeyResponse response(message_version_); - keymaster_error_t err = trusty_keymaster_send(KM_GENERATE_KEY, request, &response); - if (err != KM_ERROR_OK) { - return err; - } - - key_blob->key_material_size = response.key_blob.key_material_size; - key_blob->key_material = - DuplicateBuffer(response.key_blob.key_material, response.key_blob.key_material_size); - if (!key_blob->key_material) { - return KM_ERROR_MEMORY_ALLOCATION_FAILED; - } - - if (characteristics) { - response.enforced.CopyToParamSet(&characteristics->hw_enforced); - response.unenforced.CopyToParamSet(&characteristics->sw_enforced); - } - - return KM_ERROR_OK; -} - -keymaster_error_t TrustyKeymasterDevice::get_key_characteristics( - const keymaster_key_blob_t* key_blob, const keymaster_blob_t* client_id, - const keymaster_blob_t* app_data, keymaster_key_characteristics_t* characteristics) { - ALOGD("Device received get_key_characteristics"); - - if (error_ != KM_ERROR_OK) { - return error_; - } - if (!key_blob || !key_blob->key_material) { - return KM_ERROR_UNEXPECTED_NULL_POINTER; - } - if (!characteristics) { - return KM_ERROR_OUTPUT_PARAMETER_NULL; - } - - GetKeyCharacteristicsRequest request(message_version_); - request.SetKeyMaterial(*key_blob); - AddClientAndAppData(client_id, app_data, &request); - - GetKeyCharacteristicsResponse response(message_version_); - keymaster_error_t err = trusty_keymaster_send(KM_GET_KEY_CHARACTERISTICS, request, &response); - if (err != KM_ERROR_OK) { - return err; - } - - response.enforced.CopyToParamSet(&characteristics->hw_enforced); - response.unenforced.CopyToParamSet(&characteristics->sw_enforced); - - return KM_ERROR_OK; -} - -keymaster_error_t TrustyKeymasterDevice::import_key( - const keymaster_key_param_set_t* params, keymaster_key_format_t key_format, - const keymaster_blob_t* key_data, keymaster_key_blob_t* key_blob, - keymaster_key_characteristics_t* characteristics) { - ALOGD("Device received import_key"); - - if (error_ != KM_ERROR_OK) { - return error_; - } - if (!params || !key_data) { - return KM_ERROR_UNEXPECTED_NULL_POINTER; - } - if (!key_blob) { - return KM_ERROR_OUTPUT_PARAMETER_NULL; - } - - ImportKeyRequest request(message_version_); - request.key_description.Reinitialize(*params); - request.key_description.push_back(TAG_CREATION_DATETIME, java_time(time(NULL))); - - request.key_format = key_format; - request.SetKeyMaterial(key_data->data, key_data->data_length); - - ImportKeyResponse response(message_version_); - keymaster_error_t err = trusty_keymaster_send(KM_IMPORT_KEY, request, &response); - if (err != KM_ERROR_OK) { - return err; - } - - key_blob->key_material_size = response.key_blob.key_material_size; - key_blob->key_material = - DuplicateBuffer(response.key_blob.key_material, response.key_blob.key_material_size); - if (!key_blob->key_material) { - return KM_ERROR_MEMORY_ALLOCATION_FAILED; - } - - if (characteristics) { - response.enforced.CopyToParamSet(&characteristics->hw_enforced); - response.unenforced.CopyToParamSet(&characteristics->sw_enforced); - } - - return KM_ERROR_OK; -} - -keymaster_error_t TrustyKeymasterDevice::export_key(keymaster_key_format_t export_format, - const keymaster_key_blob_t* key_to_export, - const keymaster_blob_t* client_id, - const keymaster_blob_t* app_data, - keymaster_blob_t* export_data) { - ALOGD("Device received export_key"); - - if (error_ != KM_ERROR_OK) { - return error_; - } - if (!key_to_export || !key_to_export->key_material) { - return KM_ERROR_UNEXPECTED_NULL_POINTER; - } - if (!export_data) { - return KM_ERROR_OUTPUT_PARAMETER_NULL; - } - - export_data->data = nullptr; - export_data->data_length = 0; - - ExportKeyRequest request(message_version_); - request.key_format = export_format; - request.SetKeyMaterial(*key_to_export); - AddClientAndAppData(client_id, app_data, &request); - - ExportKeyResponse response(message_version_); - keymaster_error_t err = trusty_keymaster_send(KM_EXPORT_KEY, request, &response); - if (err != KM_ERROR_OK) { - return err; - } - - export_data->data_length = response.key_data_length; - export_data->data = DuplicateBuffer(response.key_data, response.key_data_length); - if (!export_data->data) { - return KM_ERROR_MEMORY_ALLOCATION_FAILED; - } - - return KM_ERROR_OK; -} - -keymaster_error_t TrustyKeymasterDevice::attest_key(const keymaster_key_blob_t* key_to_attest, - const keymaster_key_param_set_t* attest_params, - keymaster_cert_chain_t* cert_chain) { - ALOGD("Device received attest_key"); - - if (error_ != KM_ERROR_OK) { - return error_; - } - if (!key_to_attest || !attest_params) { - return KM_ERROR_UNEXPECTED_NULL_POINTER; - } - if (!cert_chain) { - return KM_ERROR_OUTPUT_PARAMETER_NULL; - } - - cert_chain->entry_count = 0; - cert_chain->entries = nullptr; - - AttestKeyRequest request(message_version_); - request.SetKeyMaterial(*key_to_attest); - request.attest_params.Reinitialize(*attest_params); - - keymaster_blob_t attestation_challenge = {}; - request.attest_params.GetTagValue(TAG_ATTESTATION_CHALLENGE, &attestation_challenge); - if (attestation_challenge.data_length > kMaximumAttestationChallengeLength) { - ALOGE("%zu-byte attestation challenge; only %zu bytes allowed", - attestation_challenge.data_length, kMaximumAttestationChallengeLength); - return KM_ERROR_INVALID_INPUT_LENGTH; - } - - AttestKeyResponse response(message_version_); - keymaster_error_t err = trusty_keymaster_send(KM_ATTEST_KEY, request, &response); - if (err != KM_ERROR_OK) { - return err; - } - - // Allocate and clear storage for cert_chain. - keymaster_cert_chain_t& rsp_chain = response.certificate_chain; - cert_chain->entries = reinterpret_cast<keymaster_blob_t*>( - malloc(rsp_chain.entry_count * sizeof(*cert_chain->entries))); - if (!cert_chain->entries) { - return KM_ERROR_MEMORY_ALLOCATION_FAILED; - } - cert_chain->entry_count = rsp_chain.entry_count; - for (keymaster_blob_t& entry : array_range(cert_chain->entries, cert_chain->entry_count)) { - entry = {}; - } - - // Copy cert_chain contents - size_t i = 0; - for (keymaster_blob_t& entry : array_range(rsp_chain.entries, rsp_chain.entry_count)) { - cert_chain->entries[i].data = DuplicateBuffer(entry.data, entry.data_length); - if (!cert_chain->entries[i].data) { - keymaster_free_cert_chain(cert_chain); - return KM_ERROR_MEMORY_ALLOCATION_FAILED; - } - cert_chain->entries[i].data_length = entry.data_length; - ++i; - } - - return KM_ERROR_OK; -} - -keymaster_error_t TrustyKeymasterDevice::upgrade_key( - const keymaster_key_blob_t* key_to_upgrade, const keymaster_key_param_set_t* upgrade_params, - keymaster_key_blob_t* upgraded_key) { - ALOGD("Device received upgrade_key"); - - if (error_ != KM_ERROR_OK) { - return error_; - } - if (!key_to_upgrade || !upgrade_params) { - return KM_ERROR_UNEXPECTED_NULL_POINTER; - } - if (!upgraded_key) { - return KM_ERROR_OUTPUT_PARAMETER_NULL; - } - - UpgradeKeyRequest request(message_version_); - request.SetKeyMaterial(*key_to_upgrade); - request.upgrade_params.Reinitialize(*upgrade_params); - - UpgradeKeyResponse response(message_version_); - keymaster_error_t err = trusty_keymaster_send(KM_UPGRADE_KEY, request, &response); - if (err != KM_ERROR_OK) { - return err; - } - - upgraded_key->key_material_size = response.upgraded_key.key_material_size; - upgraded_key->key_material = DuplicateBuffer(response.upgraded_key.key_material, - response.upgraded_key.key_material_size); - if (!upgraded_key->key_material) { - return KM_ERROR_MEMORY_ALLOCATION_FAILED; - } - - return KM_ERROR_OK; -} - -keymaster_error_t TrustyKeymasterDevice::begin(keymaster_purpose_t purpose, - const keymaster_key_blob_t* key, - const keymaster_key_param_set_t* in_params, - keymaster_key_param_set_t* out_params, - keymaster_operation_handle_t* operation_handle) { - ALOGD("Device received begin"); - - if (error_ != KM_ERROR_OK) { - return error_; - } - if (!key || !key->key_material) { - return KM_ERROR_UNEXPECTED_NULL_POINTER; - } - if (!operation_handle) { - return KM_ERROR_OUTPUT_PARAMETER_NULL; - } - - if (out_params) { - *out_params = {}; - } - - BeginOperationRequest request(message_version_); - request.purpose = purpose; - request.SetKeyMaterial(*key); - request.additional_params.Reinitialize(*in_params); - - BeginOperationResponse response(message_version_); - keymaster_error_t err = trusty_keymaster_send(KM_BEGIN_OPERATION, request, &response); - if (err != KM_ERROR_OK) { - return err; - } - - if (response.output_params.size() > 0) { - if (out_params) { - response.output_params.CopyToParamSet(out_params); - } else { - return KM_ERROR_OUTPUT_PARAMETER_NULL; - } - } - *operation_handle = response.op_handle; - - return KM_ERROR_OK; -} - -keymaster_error_t TrustyKeymasterDevice::update(keymaster_operation_handle_t operation_handle, - const keymaster_key_param_set_t* in_params, - const keymaster_blob_t* input, - size_t* input_consumed, - keymaster_key_param_set_t* out_params, - keymaster_blob_t* output) { - ALOGD("Device received update"); - - if (error_ != KM_ERROR_OK) { - return error_; - } - if (!input) { - return KM_ERROR_UNEXPECTED_NULL_POINTER; - } - if (!input_consumed) { - return KM_ERROR_OUTPUT_PARAMETER_NULL; - } - - if (out_params) { - *out_params = {}; - } - if (output) { - *output = {}; - } - - UpdateOperationRequest request(message_version_); - request.op_handle = operation_handle; - if (in_params) { - request.additional_params.Reinitialize(*in_params); - } - if (input && input->data_length > 0) { - size_t max_input_size = TRUSTY_KEYMASTER_SEND_BUF_SIZE - request.SerializedSize(); - request.input.Reinitialize(input->data, std::min(input->data_length, max_input_size)); - } - - UpdateOperationResponse response(message_version_); - keymaster_error_t err = trusty_keymaster_send(KM_UPDATE_OPERATION, request, &response); - if (err != KM_ERROR_OK) { - return err; - } - - if (response.output_params.size() > 0) { - if (out_params) { - response.output_params.CopyToParamSet(out_params); - } else { - return KM_ERROR_OUTPUT_PARAMETER_NULL; - } - } - *input_consumed = response.input_consumed; - if (output) { - output->data_length = response.output.available_read(); - output->data = DuplicateBuffer(response.output.peek_read(), output->data_length); - if (!output->data) { - return KM_ERROR_MEMORY_ALLOCATION_FAILED; - } - } else if (response.output.available_read() > 0) { - return KM_ERROR_OUTPUT_PARAMETER_NULL; - } - - return KM_ERROR_OK; -} - -keymaster_error_t TrustyKeymasterDevice::finish(keymaster_operation_handle_t operation_handle, - const keymaster_key_param_set_t* in_params, - const keymaster_blob_t* input, - const keymaster_blob_t* signature, - keymaster_key_param_set_t* out_params, - keymaster_blob_t* output) { - ALOGD("Device received finish"); - - if (error_ != KM_ERROR_OK) { - return error_; - } - if (input && input->data_length > kMaximumFinishInputLength) { - ALOGE("%zu-byte input to finish; only %zu bytes allowed", input->data_length, - kMaximumFinishInputLength); - return KM_ERROR_INVALID_INPUT_LENGTH; - } - - if (out_params) { - *out_params = {}; - } - if (output) { - *output = {}; - } - - FinishOperationRequest request(message_version_); - request.op_handle = operation_handle; - if (signature && signature->data && signature->data_length > 0) { - request.signature.Reinitialize(signature->data, signature->data_length); - } - if (input && input->data && input->data_length) { - request.input.Reinitialize(input->data, input->data_length); - } - if (in_params) { - request.additional_params.Reinitialize(*in_params); - } - - FinishOperationResponse response(message_version_); - keymaster_error_t err = trusty_keymaster_send(KM_FINISH_OPERATION, request, &response); - if (err != KM_ERROR_OK) { - return err; - } - - if (response.output_params.size() > 0) { - if (out_params) { - response.output_params.CopyToParamSet(out_params); - } else { - return KM_ERROR_OUTPUT_PARAMETER_NULL; - } - } - if (output) { - output->data_length = response.output.available_read(); - output->data = DuplicateBuffer(response.output.peek_read(), output->data_length); - if (!output->data) { - return KM_ERROR_MEMORY_ALLOCATION_FAILED; - } - } else if (response.output.available_read() > 0) { - return KM_ERROR_OUTPUT_PARAMETER_NULL; - } - - return KM_ERROR_OK; -} - -keymaster_error_t TrustyKeymasterDevice::abort(keymaster_operation_handle_t operation_handle) { - ALOGD("Device received abort"); - - if (error_ != KM_ERROR_OK) { - return error_; - } - - AbortOperationRequest request(message_version_); - request.op_handle = operation_handle; - AbortOperationResponse response(message_version_); - return trusty_keymaster_send(KM_ABORT_OPERATION, request, &response); -} - -keymaster_error_t TrustyKeymasterDevice::delete_key(const keymaster_key_blob_t* key) { - ALOGD("Device received delete_key"); - - if (error_ != KM_ERROR_OK) { - return error_; - } - - if (!key || !key->key_material) - return KM_ERROR_UNEXPECTED_NULL_POINTER; - - DeleteKeyRequest request(message_version_); - request.SetKeyMaterial(*key); - DeleteKeyResponse response(message_version_); - return trusty_keymaster_send(KM_DELETE_KEY, request, &response); -} - -keymaster_error_t TrustyKeymasterDevice::delete_all_keys() { - ALOGD("Device received delete_all_key"); - - if (error_ != KM_ERROR_OK) { - return error_; - } - - DeleteAllKeysRequest request(message_version_); - DeleteAllKeysResponse response(message_version_); - return trusty_keymaster_send(KM_DELETE_ALL_KEYS, request, &response); -} - -hw_device_t* TrustyKeymasterDevice::hw_device() { - return &device_.common; -} - -static inline TrustyKeymasterDevice* convert_device(const keymaster2_device_t* dev) { - return reinterpret_cast<TrustyKeymasterDevice*>(const_cast<keymaster2_device_t*>(dev)); -} - -/* static */ -int TrustyKeymasterDevice::close_device(hw_device_t* dev) { - delete reinterpret_cast<TrustyKeymasterDevice*>(dev); - return 0; -} - -/* static */ -keymaster_error_t TrustyKeymasterDevice::configure(const keymaster2_device_t* dev, - const keymaster_key_param_set_t* params) { - return convert_device(dev)->configure(params); -} - -/* static */ -keymaster_error_t TrustyKeymasterDevice::add_rng_entropy(const keymaster2_device_t* dev, - const uint8_t* data, size_t data_length) { - return convert_device(dev)->add_rng_entropy(data, data_length); -} - -/* static */ -keymaster_error_t TrustyKeymasterDevice::generate_key( - const keymaster2_device_t* dev, const keymaster_key_param_set_t* params, - keymaster_key_blob_t* key_blob, keymaster_key_characteristics_t* characteristics) { - return convert_device(dev)->generate_key(params, key_blob, characteristics); -} - -/* static */ -keymaster_error_t TrustyKeymasterDevice::get_key_characteristics( - const keymaster2_device_t* dev, const keymaster_key_blob_t* key_blob, - const keymaster_blob_t* client_id, const keymaster_blob_t* app_data, - keymaster_key_characteristics_t* characteristics) { - return convert_device(dev)->get_key_characteristics(key_blob, client_id, app_data, - characteristics); -} - -/* static */ -keymaster_error_t TrustyKeymasterDevice::import_key( - const keymaster2_device_t* dev, const keymaster_key_param_set_t* params, - keymaster_key_format_t key_format, const keymaster_blob_t* key_data, - keymaster_key_blob_t* key_blob, keymaster_key_characteristics_t* characteristics) { - return convert_device(dev)->import_key(params, key_format, key_data, key_blob, characteristics); -} - -/* static */ -keymaster_error_t TrustyKeymasterDevice::export_key(const keymaster2_device_t* dev, - keymaster_key_format_t export_format, - const keymaster_key_blob_t* key_to_export, - const keymaster_blob_t* client_id, - const keymaster_blob_t* app_data, - keymaster_blob_t* export_data) { - return convert_device(dev)->export_key(export_format, key_to_export, client_id, app_data, - export_data); -} - -/* static */ -keymaster_error_t TrustyKeymasterDevice::attest_key(const keymaster2_device_t* dev, - const keymaster_key_blob_t* key_to_attest, - const keymaster_key_param_set_t* attest_params, - keymaster_cert_chain_t* cert_chain) { - return convert_device(dev)->attest_key(key_to_attest, attest_params, cert_chain); -} - -/* static */ -keymaster_error_t TrustyKeymasterDevice::upgrade_key( - const keymaster2_device_t* dev, const keymaster_key_blob_t* key_to_upgrade, - const keymaster_key_param_set_t* upgrade_params, keymaster_key_blob_t* upgraded_key) { - return convert_device(dev)->upgrade_key(key_to_upgrade, upgrade_params, upgraded_key); -} - -/* static */ -keymaster_error_t TrustyKeymasterDevice::begin(const keymaster2_device_t* dev, - keymaster_purpose_t purpose, - const keymaster_key_blob_t* key, - const keymaster_key_param_set_t* in_params, - keymaster_key_param_set_t* out_params, - keymaster_operation_handle_t* operation_handle) { - return convert_device(dev)->begin(purpose, key, in_params, out_params, operation_handle); -} - -/* static */ -keymaster_error_t TrustyKeymasterDevice::update( - const keymaster2_device_t* dev, keymaster_operation_handle_t operation_handle, - const keymaster_key_param_set_t* in_params, const keymaster_blob_t* input, - size_t* input_consumed, keymaster_key_param_set_t* out_params, keymaster_blob_t* output) { - return convert_device(dev)->update(operation_handle, in_params, input, input_consumed, - out_params, output); -} - -/* static */ -keymaster_error_t TrustyKeymasterDevice::finish(const keymaster2_device_t* dev, - keymaster_operation_handle_t operation_handle, - const keymaster_key_param_set_t* in_params, - const keymaster_blob_t* input, - const keymaster_blob_t* signature, - keymaster_key_param_set_t* out_params, - keymaster_blob_t* output) { - return convert_device(dev)->finish(operation_handle, in_params, input, signature, out_params, - output); -} - -/* static */ -keymaster_error_t TrustyKeymasterDevice::abort(const keymaster2_device_t* dev, - keymaster_operation_handle_t operation_handle) { - return convert_device(dev)->abort(operation_handle); -} - -/* static */ -keymaster_error_t TrustyKeymasterDevice::delete_key(const keymaster2_device_t* dev, - const keymaster_key_blob_t* key) { - return convert_device(dev)->delete_key(key); -} - -/* static */ -keymaster_error_t TrustyKeymasterDevice::delete_all_keys(const keymaster2_device_t* dev) { - return convert_device(dev)->delete_all_keys(); -} - -} // namespace keymaster diff --git a/trusty/keymaster/legacy/trusty_keymaster_device_test.cpp b/trusty/keymaster/legacy/trusty_keymaster_device_test.cpp deleted file mode 100644 index 68def5872..000000000 --- a/trusty/keymaster/legacy/trusty_keymaster_device_test.cpp +++ /dev/null @@ -1,561 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include <algorithm> -#include <fstream> -#include <memory> - -#include <gtest/gtest.h> -#include <openssl/engine.h> - -#include <hardware/keymaster0.h> - -#include <keymaster/android_keymaster.h> -#include <keymaster/android_keymaster_messages.h> -#include <keymaster/android_keymaster_utils.h> -#include <keymaster/keymaster_tags.h> -#include <keymaster/soft_keymaster_context.h> - -#include <trusty_keymaster/legacy/trusty_keymaster_device.h> -#include "android_keymaster_test_utils.h" -#include "openssl_utils.h" - -using std::ifstream; -using std::istreambuf_iterator; -using std::string; - -static keymaster::AndroidKeymaster* impl_ = nullptr; - -extern "C" { -int __android_log_print(); -} - -int __android_log_print() { - return 0; -} - -int main(int argc, char** argv) { - ::testing::InitGoogleTest(&argc, argv); - int result = RUN_ALL_TESTS(); - // Clean up stuff OpenSSL leaves around, so Valgrind doesn't complain. - CRYPTO_cleanup_all_ex_data(); - ERR_free_strings(); - return result; -} - -int trusty_keymaster_connect() { - impl_ = new keymaster::AndroidKeymaster(new keymaster::SoftKeymasterContext(nullptr), 16); -} - -void trusty_keymaster_disconnect() { - delete static_cast<keymaster::AndroidKeymaster*>(priv_); -} - -template <typename Req, typename Rsp> -static int fake_call(keymaster::AndroidKeymaster* device, - void (keymaster::AndroidKeymaster::*method)(const Req&, Rsp*), void* in_buf, - uint32_t in_size, void* out_buf, uint32_t* out_size) { - Req req; - const uint8_t* in = static_cast<uint8_t*>(in_buf); - req.Deserialize(&in, in + in_size); - Rsp rsp; - (device->*method)(req, &rsp); - - *out_size = rsp.SerializedSize(); - uint8_t* out = static_cast<uint8_t*>(out_buf); - rsp.Serialize(out, out + *out_size); - return 0; -} - -int trusty_keymaster_call(uint32_t cmd, void* in_buf, uint32_t in_size, void* out_buf, - uint32_t* out_size) { - switch (cmd) { - case KM_GENERATE_KEY: - return fake_call(impl_, &keymaster::AndroidKeymaster::GenerateKey, in_buf, in_size, - out_buf, out_size); - case KM_BEGIN_OPERATION: - return fake_call(impl_, &keymaster::AndroidKeymaster::BeginOperation, in_buf, in_size, - out_buf, out_size); - case KM_UPDATE_OPERATION: - return fake_call(impl_, &keymaster::AndroidKeymaster::UpdateOperation, in_buf, in_size, - out_buf, out_size); - case KM_FINISH_OPERATION: - return fake_call(impl_, &keymaster::AndroidKeymaster::FinishOperation, in_buf, in_size, - out_buf, out_size); - case KM_IMPORT_KEY: - return fake_call(impl_, &keymaster::AndroidKeymaster::ImportKey, in_buf, in_size, - out_buf, out_size); - case KM_EXPORT_KEY: - return fake_call(impl_, &keymaster::AndroidKeymaster::ExportKey, in_buf, in_size, - out_buf, out_size); - } - return -EINVAL; -} - -namespace keymaster { -namespace test { - -class TrustyKeymasterTest : public testing::Test { - protected: - TrustyKeymasterTest() : device(NULL) {} - - keymaster_rsa_keygen_params_t build_rsa_params() { - keymaster_rsa_keygen_params_t rsa_params; - rsa_params.public_exponent = 65537; - rsa_params.modulus_size = 2048; - return rsa_params; - } - - uint8_t* build_message(size_t length) { - uint8_t* msg = new uint8_t[length]; - memset(msg, 'a', length); - return msg; - } - - size_t dsa_message_len(const keymaster_dsa_keygen_params_t& params) { - switch (params.key_size) { - case 256: - case 1024: - return 48; - case 2048: - case 4096: - return 72; - default: - // Oops. - return 0; - } - } - - TrustyKeymasterDevice device; -}; - -class Malloc_Delete { - public: - Malloc_Delete(void* p) : p_(p) {} - ~Malloc_Delete() { free(p_); } - - private: - void* p_; -}; - -typedef TrustyKeymasterTest KeyGenTest; -TEST_F(KeyGenTest, RsaSuccess) { - keymaster_rsa_keygen_params_t params = build_rsa_params(); - uint8_t* ptr = NULL; - size_t size; - ASSERT_EQ(0, device.generate_keypair(TYPE_RSA, ¶ms, &ptr, &size)); - EXPECT_GT(size, 0U); - Malloc_Delete key_deleter(ptr); -} - -TEST_F(KeyGenTest, EcdsaSuccess) { - keymaster_ec_keygen_params_t ec_params = {256}; - uint8_t* ptr = NULL; - size_t size; - ASSERT_EQ(0, device.generate_keypair(TYPE_EC, &ec_params, &ptr, &size)); - EXPECT_GT(size, 0U); - Malloc_Delete key_deleter(ptr); -} - -typedef TrustyKeymasterTest SigningTest; -TEST_F(SigningTest, RsaSuccess) { - keymaster_rsa_keygen_params_t params = build_rsa_params(); - uint8_t* ptr = NULL; - size_t size; - ASSERT_EQ(0, device.generate_keypair(TYPE_RSA, ¶ms, &ptr, &size)); - EXPECT_GT(size, 0U); - Malloc_Delete key_deleter(ptr); - - keymaster_rsa_sign_params_t sig_params = {DIGEST_NONE, PADDING_NONE}; - size_t message_len = params.modulus_size / 8; - std::unique_ptr<uint8_t[]> message(build_message(message_len)); - uint8_t* signature; - size_t siglen; - EXPECT_EQ(KM_ERROR_OK, device.sign_data(&sig_params, ptr, size, message.get(), message_len, - &signature, &siglen)); - Malloc_Delete sig_deleter(signature); - EXPECT_EQ(message_len, siglen); -} - -TEST_F(SigningTest, RsaShortMessage) { - keymaster_rsa_keygen_params_t params = build_rsa_params(); - uint8_t* ptr = NULL; - size_t size; - ASSERT_EQ(0, device.generate_keypair(TYPE_RSA, ¶ms, &ptr, &size)); - EXPECT_GT(size, 0U); - Malloc_Delete key_deleter(ptr); - - keymaster_rsa_sign_params_t sig_params = {DIGEST_NONE, PADDING_NONE}; - size_t message_len = params.modulus_size / 8 - 1; - std::unique_ptr<uint8_t[]> message(build_message(message_len)); - uint8_t* signature; - size_t siglen; - EXPECT_EQ(KM_ERROR_UNKNOWN_ERROR, device.sign_data(&sig_params, ptr, size, message.get(), - message_len, &signature, &siglen)); -} - -TEST_F(SigningTest, RsaLongMessage) { - keymaster_rsa_keygen_params_t params = build_rsa_params(); - uint8_t* ptr = NULL; - size_t size; - ASSERT_EQ(0, device.generate_keypair(TYPE_RSA, ¶ms, &ptr, &size)); - EXPECT_GT(size, 0U); - Malloc_Delete key_deleter(ptr); - - keymaster_rsa_sign_params_t sig_params = {DIGEST_NONE, PADDING_NONE}; - size_t message_len = params.modulus_size / 8 + 1; - std::unique_ptr<uint8_t[]> message(build_message(message_len)); - uint8_t* signature; - size_t siglen; - EXPECT_EQ(KM_ERROR_UNKNOWN_ERROR, device.sign_data(&sig_params, ptr, size, message.get(), - message_len, &signature, &siglen)); -} - -TEST_F(SigningTest, EcdsaSuccess) { - keymaster_ec_keygen_params_t params = {256}; - uint8_t* ptr = NULL; - size_t size; - ASSERT_EQ(0, device.generate_keypair(TYPE_EC, ¶ms, &ptr, &size)); - EXPECT_GT(size, 0U); - Malloc_Delete key_deleter(ptr); - - keymaster_ec_sign_params_t sig_params = {DIGEST_NONE}; - uint8_t message[] = "12345678901234567890123456789012"; - uint8_t* signature; - size_t siglen; - ASSERT_EQ(KM_ERROR_OK, device.sign_data(&sig_params, ptr, size, message, - array_size(message) - 1, &signature, &siglen)); - Malloc_Delete sig_deleter(signature); - EXPECT_GT(siglen, 69U); - EXPECT_LT(siglen, 73U); -} - -TEST_F(SigningTest, EcdsaEmptyMessageSuccess) { - keymaster_ec_keygen_params_t params = {256}; - uint8_t* ptr = NULL; - size_t size; - ASSERT_EQ(0, device.generate_keypair(TYPE_EC, ¶ms, &ptr, &size)); - EXPECT_GT(size, 0U); - Malloc_Delete key_deleter(ptr); - - keymaster_ec_sign_params_t sig_params = {DIGEST_NONE}; - uint8_t message[] = ""; - uint8_t* signature; - size_t siglen; - ASSERT_EQ(KM_ERROR_OK, device.sign_data(&sig_params, ptr, size, message, - array_size(message) - 1, &signature, &siglen)); - Malloc_Delete sig_deleter(signature); - EXPECT_GT(siglen, 69U); - EXPECT_LT(siglen, 73U); -} - -TEST_F(SigningTest, EcdsaLargeMessageSuccess) { - keymaster_ec_keygen_params_t params = {256}; - uint8_t* ptr = NULL; - size_t size; - ASSERT_EQ(0, device.generate_keypair(TYPE_EC, ¶ms, &ptr, &size)); - EXPECT_GT(size, 0U); - Malloc_Delete key_deleter(ptr); - - keymaster_ec_sign_params_t sig_params = {DIGEST_NONE}; - size_t message_len = 1024 * 7; - std::unique_ptr<uint8_t[]> message(new uint8_t[message_len]); - // contents of message don't matter. - uint8_t* signature; - size_t siglen; - ASSERT_EQ(KM_ERROR_OK, device.sign_data(&sig_params, ptr, size, message.get(), message_len, - &signature, &siglen)); - Malloc_Delete sig_deleter(signature); - EXPECT_GT(siglen, 69U); - EXPECT_LT(siglen, 73U); -} - -typedef TrustyKeymasterTest VerificationTest; -TEST_F(VerificationTest, RsaSuccess) { - keymaster_rsa_keygen_params_t params = build_rsa_params(); - uint8_t* ptr = NULL; - size_t size; - ASSERT_EQ(0, device.generate_keypair(TYPE_RSA, ¶ms, &ptr, &size)); - EXPECT_GT(size, 0U); - Malloc_Delete key_deleter(ptr); - - keymaster_rsa_sign_params_t sig_params = {DIGEST_NONE, PADDING_NONE}; - size_t message_len = params.modulus_size / 8; - std::unique_ptr<uint8_t[]> message(build_message(message_len)); - uint8_t* signature; - size_t siglen; - EXPECT_EQ(KM_ERROR_OK, device.sign_data(&sig_params, ptr, size, message.get(), message_len, - &signature, &siglen)); - Malloc_Delete sig_deleter(signature); - - EXPECT_EQ(KM_ERROR_OK, device.verify_data(&sig_params, ptr, size, message.get(), message_len, - signature, siglen)); -} - -TEST_F(VerificationTest, RsaBadSignature) { - keymaster_rsa_keygen_params_t params = build_rsa_params(); - uint8_t* ptr = NULL; - size_t size; - ASSERT_EQ(0, device.generate_keypair(TYPE_RSA, ¶ms, &ptr, &size)); - EXPECT_GT(size, 0U); - Malloc_Delete key_deleter(ptr); - - keymaster_rsa_sign_params_t sig_params = {DIGEST_NONE, PADDING_NONE}; - size_t message_len = params.modulus_size / 8; - std::unique_ptr<uint8_t[]> message(build_message(message_len)); - uint8_t* signature; - size_t siglen; - EXPECT_EQ(KM_ERROR_OK, device.sign_data(&sig_params, ptr, size, message.get(), message_len, - &signature, &siglen)); - - Malloc_Delete sig_deleter(signature); - signature[siglen / 2]++; - EXPECT_EQ(KM_ERROR_VERIFICATION_FAILED, - device.verify_data(&sig_params, ptr, size, message.get(), message_len, signature, - siglen)); -} - -TEST_F(VerificationTest, RsaBadMessage) { - keymaster_rsa_keygen_params_t params = build_rsa_params(); - uint8_t* ptr = NULL; - size_t size; - ASSERT_EQ(0, device.generate_keypair(TYPE_RSA, ¶ms, &ptr, &size)); - EXPECT_GT(size, 0U); - Malloc_Delete key_deleter(ptr); - - keymaster_rsa_sign_params_t sig_params = {DIGEST_NONE, PADDING_NONE}; - size_t message_len = params.modulus_size / 8; - std::unique_ptr<uint8_t[]> message(build_message(message_len)); - uint8_t* signature; - size_t siglen; - EXPECT_EQ(KM_ERROR_OK, device.sign_data(&sig_params, ptr, size, message.get(), message_len, - &signature, &siglen)); - Malloc_Delete sig_deleter(signature); - message[0]++; - EXPECT_EQ(KM_ERROR_VERIFICATION_FAILED, - device.verify_data(&sig_params, ptr, size, message.get(), message_len, signature, - siglen)); -} - -TEST_F(VerificationTest, RsaShortMessage) { - keymaster_rsa_keygen_params_t params = build_rsa_params(); - uint8_t* ptr = NULL; - size_t size; - ASSERT_EQ(0, device.generate_keypair(TYPE_RSA, ¶ms, &ptr, &size)); - EXPECT_GT(size, 0U); - Malloc_Delete key_deleter(ptr); - - keymaster_rsa_sign_params_t sig_params = {DIGEST_NONE, PADDING_NONE}; - size_t message_len = params.modulus_size / 8; - std::unique_ptr<uint8_t[]> message(build_message(message_len)); - uint8_t* signature; - size_t siglen; - EXPECT_EQ(KM_ERROR_OK, device.sign_data(&sig_params, ptr, size, message.get(), message_len, - &signature, &siglen)); - - Malloc_Delete sig_deleter(signature); - EXPECT_EQ(KM_ERROR_INVALID_INPUT_LENGTH, - device.verify_data(&sig_params, ptr, size, message.get(), message_len - 1, signature, - siglen)); -} - -TEST_F(VerificationTest, RsaLongMessage) { - keymaster_rsa_keygen_params_t params = build_rsa_params(); - uint8_t* ptr = NULL; - size_t size; - ASSERT_EQ(0, device.generate_keypair(TYPE_RSA, ¶ms, &ptr, &size)); - EXPECT_GT(size, 0U); - Malloc_Delete key_deleter(ptr); - - keymaster_rsa_sign_params_t sig_params = {DIGEST_NONE, PADDING_NONE}; - size_t message_len = params.modulus_size / 8; - std::unique_ptr<uint8_t[]> message(build_message(message_len + 1)); - uint8_t* signature; - size_t siglen; - EXPECT_EQ(KM_ERROR_OK, device.sign_data(&sig_params, ptr, size, message.get(), message_len, - &signature, &siglen)); - Malloc_Delete sig_deleter(signature); - EXPECT_EQ(KM_ERROR_INVALID_INPUT_LENGTH, - device.verify_data(&sig_params, ptr, size, message.get(), message_len + 1, signature, - siglen)); -} - -TEST_F(VerificationTest, EcdsaSuccess) { - keymaster_ec_keygen_params_t params = {256}; - uint8_t* ptr = NULL; - size_t size; - ASSERT_EQ(0, device.generate_keypair(TYPE_EC, ¶ms, &ptr, &size)); - EXPECT_GT(size, 0U); - Malloc_Delete key_deleter(ptr); - - keymaster_ec_sign_params_t sig_params = {DIGEST_NONE}; - uint8_t message[] = "12345678901234567890123456789012"; - uint8_t* signature; - size_t siglen; - ASSERT_EQ(KM_ERROR_OK, device.sign_data(&sig_params, ptr, size, message, - array_size(message) - 1, &signature, &siglen)); - Malloc_Delete sig_deleter(signature); - EXPECT_EQ(KM_ERROR_OK, device.verify_data(&sig_params, ptr, size, message, - array_size(message) - 1, signature, siglen)); -} - -TEST_F(VerificationTest, EcdsaLargeMessageSuccess) { - keymaster_ec_keygen_params_t params = {256}; - uint8_t* ptr = NULL; - size_t size; - ASSERT_EQ(0, device.generate_keypair(TYPE_EC, ¶ms, &ptr, &size)); - EXPECT_GT(size, 0U); - Malloc_Delete key_deleter(ptr); - - keymaster_ec_sign_params_t sig_params = {DIGEST_NONE}; - size_t message_len = 1024 * 7; - std::unique_ptr<uint8_t[]> message(new uint8_t[message_len]); - // contents of message don't matter. - uint8_t* signature; - size_t siglen; - ASSERT_EQ(KM_ERROR_OK, device.sign_data(&sig_params, ptr, size, message.get(), message_len, - &signature, &siglen)); - Malloc_Delete sig_deleter(signature); - EXPECT_EQ(KM_ERROR_OK, device.verify_data(&sig_params, ptr, size, message.get(), message_len, - signature, siglen)); -} - -static string read_file(const string& file_name) { - ifstream file_stream(file_name, std::ios::binary); - istreambuf_iterator<char> file_begin(file_stream); - istreambuf_iterator<char> file_end; - return string(file_begin, file_end); -} - -typedef TrustyKeymasterTest ImportKeyTest; -TEST_F(ImportKeyTest, RsaSuccess) { - string pk8_key = read_file("../../../../system/keymaster/rsa_privkey_pk8.der"); - ASSERT_EQ(633U, pk8_key.size()); - - uint8_t* key = NULL; - size_t size; - ASSERT_EQ(KM_ERROR_OK, device.import_keypair(reinterpret_cast<const uint8_t*>(pk8_key.data()), - pk8_key.size(), &key, &size)); - Malloc_Delete key_deleter(key); - - keymaster_rsa_sign_params_t sig_params = {DIGEST_NONE, PADDING_NONE}; - size_t message_size = 1024 /* key size */ / 8; - std::unique_ptr<uint8_t[]> message(new uint8_t[message_size]); - memset(message.get(), 'a', message_size); - uint8_t* signature; - size_t siglen; - ASSERT_EQ(KM_ERROR_OK, device.sign_data(&sig_params, key, size, message.get(), message_size, - &signature, &siglen)); - Malloc_Delete sig_deleter(signature); - EXPECT_EQ(KM_ERROR_OK, device.verify_data(&sig_params, key, size, message.get(), message_size, - signature, siglen)); -} - -TEST_F(ImportKeyTest, EcdsaSuccess) { - string pk8_key = read_file("../../../../system/keymaster/ec_privkey_pk8.der"); - ASSERT_EQ(138U, pk8_key.size()); - - uint8_t* key = NULL; - size_t size; - ASSERT_EQ(KM_ERROR_OK, device.import_keypair(reinterpret_cast<const uint8_t*>(pk8_key.data()), - pk8_key.size(), &key, &size)); - Malloc_Delete key_deleter(key); - - keymaster_ec_sign_params_t sig_params = {DIGEST_NONE}; - uint8_t message[] = "12345678901234567890123456789012"; - uint8_t* signature; - size_t siglen; - ASSERT_EQ(KM_ERROR_OK, device.sign_data(&sig_params, key, size, message, - array_size(message) - 1, &signature, &siglen)); - Malloc_Delete sig_deleter(signature); - EXPECT_EQ(KM_ERROR_OK, device.verify_data(&sig_params, key, size, message, - array_size(message) - 1, signature, siglen)); -} - -struct EVP_PKEY_CTX_Delete { - void operator()(EVP_PKEY_CTX* p) { EVP_PKEY_CTX_free(p); } -}; - -static void VerifySignature(const uint8_t* key, size_t key_len, const uint8_t* signature, - size_t signature_len, const uint8_t* message, size_t message_len) { - std::unique_ptr<EVP_PKEY, EVP_PKEY_Delete> pkey(d2i_PUBKEY(NULL, &key, key_len)); - ASSERT_TRUE(pkey.get() != NULL); - std::unique_ptr<EVP_PKEY_CTX, EVP_PKEY_CTX_Delete> ctx(EVP_PKEY_CTX_new(pkey.get(), NULL)); - ASSERT_TRUE(ctx.get() != NULL); - ASSERT_EQ(1, EVP_PKEY_verify_init(ctx.get())); - if (EVP_PKEY_type(pkey->type) == EVP_PKEY_RSA) - ASSERT_EQ(1, EVP_PKEY_CTX_set_rsa_padding(ctx.get(), RSA_NO_PADDING)); - EXPECT_EQ(1, EVP_PKEY_verify(ctx.get(), signature, signature_len, message, message_len)); -} - -typedef TrustyKeymasterTest ExportKeyTest; -TEST_F(ExportKeyTest, RsaSuccess) { - keymaster_rsa_keygen_params_t params = build_rsa_params(); - uint8_t* ptr = NULL; - size_t size; - ASSERT_EQ(0, device.generate_keypair(TYPE_RSA, ¶ms, &ptr, &size)); - EXPECT_GT(size, 0U); - Malloc_Delete key_deleter(ptr); - - uint8_t* exported; - size_t exported_size; - EXPECT_EQ(KM_ERROR_OK, device.get_keypair_public(ptr, size, &exported, &exported_size)); - Malloc_Delete exported_deleter(exported); - - // Sign a message so we can verify it with the exported pubkey. - keymaster_rsa_sign_params_t sig_params = {DIGEST_NONE, PADDING_NONE}; - size_t message_len = params.modulus_size / 8; - std::unique_ptr<uint8_t[]> message(build_message(message_len)); - uint8_t* signature; - size_t siglen; - EXPECT_EQ(KM_ERROR_OK, device.sign_data(&sig_params, ptr, size, message.get(), message_len, - &signature, &siglen)); - Malloc_Delete sig_deleter(signature); - EXPECT_EQ(message_len, siglen); - const uint8_t* tmp = exported; - - VerifySignature(exported, exported_size, signature, siglen, message.get(), message_len); -} - -typedef TrustyKeymasterTest ExportKeyTest; -TEST_F(ExportKeyTest, EcdsaSuccess) { - keymaster_ec_keygen_params_t params = {256}; - uint8_t* key = NULL; - size_t size; - ASSERT_EQ(0, device.generate_keypair(TYPE_EC, ¶ms, &key, &size)); - EXPECT_GT(size, 0U); - Malloc_Delete key_deleter(key); - - uint8_t* exported; - size_t exported_size; - EXPECT_EQ(KM_ERROR_OK, device.get_keypair_public(key, size, &exported, &exported_size)); - Malloc_Delete exported_deleter(exported); - - // Sign a message so we can verify it with the exported pubkey. - keymaster_ec_sign_params_t sig_params = {DIGEST_NONE}; - uint8_t message[] = "12345678901234567890123456789012"; - uint8_t* signature; - size_t siglen; - ASSERT_EQ(KM_ERROR_OK, device.sign_data(&sig_params, key, size, message, - array_size(message) - 1, &signature, &siglen)); - Malloc_Delete sig_deleter(signature); - EXPECT_EQ(KM_ERROR_OK, device.verify_data(&sig_params, key, size, message, - array_size(message) - 1, signature, siglen)); - - VerifySignature(exported, exported_size, signature, siglen, message, array_size(message) - 1); -} - -} // namespace test -} // namespace keymaster diff --git a/trusty/keymaster/legacy/trusty_keymaster_main.cpp b/trusty/keymaster/legacy/trusty_keymaster_main.cpp deleted file mode 100644 index e3e70e66e..000000000 --- a/trusty/keymaster/legacy/trusty_keymaster_main.cpp +++ /dev/null @@ -1,408 +0,0 @@ -/* - * Copyright 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <keymaster/keymaster_configuration.h> - -#include <stdio.h> -#include <memory> - -#include <openssl/evp.h> -#include <openssl/x509.h> - -#include <trusty_keymaster/legacy/trusty_keymaster_device.h> - -using keymaster::TrustyKeymasterDevice; - -unsigned char rsa_privkey_pk8_der[] = { - 0x30, 0x82, 0x02, 0x75, 0x02, 0x01, 0x00, 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, - 0xf7, 0x0d, 0x01, 0x01, 0x01, 0x05, 0x00, 0x04, 0x82, 0x02, 0x5f, 0x30, 0x82, 0x02, 0x5b, - 0x02, 0x01, 0x00, 0x02, 0x81, 0x81, 0x00, 0xc6, 0x09, 0x54, 0x09, 0x04, 0x7d, 0x86, 0x34, - 0x81, 0x2d, 0x5a, 0x21, 0x81, 0x76, 0xe4, 0x5c, 0x41, 0xd6, 0x0a, 0x75, 0xb1, 0x39, 0x01, - 0xf2, 0x34, 0x22, 0x6c, 0xff, 0xe7, 0x76, 0x52, 0x1c, 0x5a, 0x77, 0xb9, 0xe3, 0x89, 0x41, - 0x7b, 0x71, 0xc0, 0xb6, 0xa4, 0x4d, 0x13, 0xaf, 0xe4, 0xe4, 0xa2, 0x80, 0x5d, 0x46, 0xc9, - 0xda, 0x29, 0x35, 0xad, 0xb1, 0xff, 0x0c, 0x1f, 0x24, 0xea, 0x06, 0xe6, 0x2b, 0x20, 0xd7, - 0x76, 0x43, 0x0a, 0x4d, 0x43, 0x51, 0x57, 0x23, 0x3c, 0x6f, 0x91, 0x67, 0x83, 0xc3, 0x0e, - 0x31, 0x0f, 0xcb, 0xd8, 0x9b, 0x85, 0xc2, 0xd5, 0x67, 0x71, 0x16, 0x97, 0x85, 0xac, 0x12, - 0xbc, 0xa2, 0x44, 0xab, 0xda, 0x72, 0xbf, 0xb1, 0x9f, 0xc4, 0x4d, 0x27, 0xc8, 0x1e, 0x1d, - 0x92, 0xde, 0x28, 0x4f, 0x40, 0x61, 0xed, 0xfd, 0x99, 0x28, 0x07, 0x45, 0xea, 0x6d, 0x25, - 0x02, 0x03, 0x01, 0x00, 0x01, 0x02, 0x81, 0x80, 0x1b, 0xe0, 0xf0, 0x4d, 0x9c, 0xae, 0x37, - 0x18, 0x69, 0x1f, 0x03, 0x53, 0x38, 0x30, 0x8e, 0x91, 0x56, 0x4b, 0x55, 0x89, 0x9f, 0xfb, - 0x50, 0x84, 0xd2, 0x46, 0x0e, 0x66, 0x30, 0x25, 0x7e, 0x05, 0xb3, 0xce, 0xab, 0x02, 0x97, - 0x2d, 0xfa, 0xbc, 0xd6, 0xce, 0x5f, 0x6e, 0xe2, 0x58, 0x9e, 0xb6, 0x79, 0x11, 0xed, 0x0f, - 0xac, 0x16, 0xe4, 0x3a, 0x44, 0x4b, 0x8c, 0x86, 0x1e, 0x54, 0x4a, 0x05, 0x93, 0x36, 0x57, - 0x72, 0xf8, 0xba, 0xf6, 0xb2, 0x2f, 0xc9, 0xe3, 0xc5, 0xf1, 0x02, 0x4b, 0x06, 0x3a, 0xc0, - 0x80, 0xa7, 0xb2, 0x23, 0x4c, 0xf8, 0xae, 0xe8, 0xf6, 0xc4, 0x7b, 0xbf, 0x4f, 0xd3, 0xac, - 0xe7, 0x24, 0x02, 0x90, 0xbe, 0xf1, 0x6c, 0x0b, 0x3f, 0x7f, 0x3c, 0xdd, 0x64, 0xce, 0x3a, - 0xb5, 0x91, 0x2c, 0xf6, 0xe3, 0x2f, 0x39, 0xab, 0x18, 0x83, 0x58, 0xaf, 0xcc, 0xcd, 0x80, - 0x81, 0x02, 0x41, 0x00, 0xe4, 0xb4, 0x9e, 0xf5, 0x0f, 0x76, 0x5d, 0x3b, 0x24, 0xdd, 0xe0, - 0x1a, 0xce, 0xaa, 0xf1, 0x30, 0xf2, 0xc7, 0x66, 0x70, 0xa9, 0x1a, 0x61, 0xae, 0x08, 0xaf, - 0x49, 0x7b, 0x4a, 0x82, 0xbe, 0x6d, 0xee, 0x8f, 0xcd, 0xd5, 0xe3, 0xf7, 0xba, 0x1c, 0xfb, - 0x1f, 0x0c, 0x92, 0x6b, 0x88, 0xf8, 0x8c, 0x92, 0xbf, 0xab, 0x13, 0x7f, 0xba, 0x22, 0x85, - 0x22, 0x7b, 0x83, 0xc3, 0x42, 0xff, 0x7c, 0x55, 0x02, 0x41, 0x00, 0xdd, 0xab, 0xb5, 0x83, - 0x9c, 0x4c, 0x7f, 0x6b, 0xf3, 0xd4, 0x18, 0x32, 0x31, 0xf0, 0x05, 0xb3, 0x1a, 0xa5, 0x8a, - 0xff, 0xdd, 0xa5, 0xc7, 0x9e, 0x4c, 0xce, 0x21, 0x7f, 0x6b, 0xc9, 0x30, 0xdb, 0xe5, 0x63, - 0xd4, 0x80, 0x70, 0x6c, 0x24, 0xe9, 0xeb, 0xfc, 0xab, 0x28, 0xa6, 0xcd, 0xef, 0xd3, 0x24, - 0xb7, 0x7e, 0x1b, 0xf7, 0x25, 0x1b, 0x70, 0x90, 0x92, 0xc2, 0x4f, 0xf5, 0x01, 0xfd, 0x91, - 0x02, 0x40, 0x23, 0xd4, 0x34, 0x0e, 0xda, 0x34, 0x45, 0xd8, 0xcd, 0x26, 0xc1, 0x44, 0x11, - 0xda, 0x6f, 0xdc, 0xa6, 0x3c, 0x1c, 0xcd, 0x4b, 0x80, 0xa9, 0x8a, 0xd5, 0x2b, 0x78, 0xcc, - 0x8a, 0xd8, 0xbe, 0xb2, 0x84, 0x2c, 0x1d, 0x28, 0x04, 0x05, 0xbc, 0x2f, 0x6c, 0x1b, 0xea, - 0x21, 0x4a, 0x1d, 0x74, 0x2a, 0xb9, 0x96, 0xb3, 0x5b, 0x63, 0xa8, 0x2a, 0x5e, 0x47, 0x0f, - 0xa8, 0x8d, 0xbf, 0x82, 0x3c, 0xdd, 0x02, 0x40, 0x1b, 0x7b, 0x57, 0x44, 0x9a, 0xd3, 0x0d, - 0x15, 0x18, 0x24, 0x9a, 0x5f, 0x56, 0xbb, 0x98, 0x29, 0x4d, 0x4b, 0x6a, 0xc1, 0x2f, 0xfc, - 0x86, 0x94, 0x04, 0x97, 0xa5, 0xa5, 0x83, 0x7a, 0x6c, 0xf9, 0x46, 0x26, 0x2b, 0x49, 0x45, - 0x26, 0xd3, 0x28, 0xc1, 0x1e, 0x11, 0x26, 0x38, 0x0f, 0xde, 0x04, 0xc2, 0x4f, 0x91, 0x6d, - 0xec, 0x25, 0x08, 0x92, 0xdb, 0x09, 0xa6, 0xd7, 0x7c, 0xdb, 0xa3, 0x51, 0x02, 0x40, 0x77, - 0x62, 0xcd, 0x8f, 0x4d, 0x05, 0x0d, 0xa5, 0x6b, 0xd5, 0x91, 0xad, 0xb5, 0x15, 0xd2, 0x4d, - 0x7c, 0xcd, 0x32, 0xcc, 0xa0, 0xd0, 0x5f, 0x86, 0x6d, 0x58, 0x35, 0x14, 0xbd, 0x73, 0x24, - 0xd5, 0xf3, 0x36, 0x45, 0xe8, 0xed, 0x8b, 0x4a, 0x1c, 0xb3, 0xcc, 0x4a, 0x1d, 0x67, 0x98, - 0x73, 0x99, 0xf2, 0xa0, 0x9f, 0x5b, 0x3f, 0xb6, 0x8c, 0x88, 0xd5, 0xe5, 0xd9, 0x0a, 0xc3, - 0x34, 0x92, 0xd6}; -unsigned int rsa_privkey_pk8_der_len = 633; - -unsigned char dsa_privkey_pk8_der[] = { - 0x30, 0x82, 0x01, 0x4b, 0x02, 0x01, 0x00, 0x30, 0x82, 0x01, 0x2b, 0x06, 0x07, 0x2a, 0x86, - 0x48, 0xce, 0x38, 0x04, 0x01, 0x30, 0x82, 0x01, 0x1e, 0x02, 0x81, 0x81, 0x00, 0xa3, 0xf3, - 0xe9, 0xb6, 0x7e, 0x7d, 0x88, 0xf6, 0xb7, 0xe5, 0xf5, 0x1f, 0x3b, 0xee, 0xac, 0xd7, 0xad, - 0xbc, 0xc9, 0xd1, 0x5a, 0xf8, 0x88, 0xc4, 0xef, 0x6e, 0x3d, 0x74, 0x19, 0x74, 0xe7, 0xd8, - 0xe0, 0x26, 0x44, 0x19, 0x86, 0xaf, 0x19, 0xdb, 0x05, 0xe9, 0x3b, 0x8b, 0x58, 0x58, 0xde, - 0xe5, 0x4f, 0x48, 0x15, 0x01, 0xea, 0xe6, 0x83, 0x52, 0xd7, 0xc1, 0x21, 0xdf, 0xb9, 0xb8, - 0x07, 0x66, 0x50, 0xfb, 0x3a, 0x0c, 0xb3, 0x85, 0xee, 0xbb, 0x04, 0x5f, 0xc2, 0x6d, 0x6d, - 0x95, 0xfa, 0x11, 0x93, 0x1e, 0x59, 0x5b, 0xb1, 0x45, 0x8d, 0xe0, 0x3d, 0x73, 0xaa, 0xf2, - 0x41, 0x14, 0x51, 0x07, 0x72, 0x3d, 0xa2, 0xf7, 0x58, 0xcd, 0x11, 0xa1, 0x32, 0xcf, 0xda, - 0x42, 0xb7, 0xcc, 0x32, 0x80, 0xdb, 0x87, 0x82, 0xec, 0x42, 0xdb, 0x5a, 0x55, 0x24, 0x24, - 0xa2, 0xd1, 0x55, 0x29, 0xad, 0xeb, 0x02, 0x15, 0x00, 0xeb, 0xea, 0x17, 0xd2, 0x09, 0xb3, - 0xd7, 0x21, 0x9a, 0x21, 0x07, 0x82, 0x8f, 0xab, 0xfe, 0x88, 0x71, 0x68, 0xf7, 0xe3, 0x02, - 0x81, 0x80, 0x19, 0x1c, 0x71, 0xfd, 0xe0, 0x03, 0x0c, 0x43, 0xd9, 0x0b, 0xf6, 0xcd, 0xd6, - 0xa9, 0x70, 0xe7, 0x37, 0x86, 0x3a, 0x78, 0xe9, 0xa7, 0x47, 0xa7, 0x47, 0x06, 0x88, 0xb1, - 0xaf, 0xd7, 0xf3, 0xf1, 0xa1, 0xd7, 0x00, 0x61, 0x28, 0x88, 0x31, 0x48, 0x60, 0xd8, 0x11, - 0xef, 0xa5, 0x24, 0x1a, 0x81, 0xc4, 0x2a, 0xe2, 0xea, 0x0e, 0x36, 0xd2, 0xd2, 0x05, 0x84, - 0x37, 0xcf, 0x32, 0x7d, 0x09, 0xe6, 0x0f, 0x8b, 0x0c, 0xc8, 0xc2, 0xa4, 0xb1, 0xdc, 0x80, - 0xca, 0x68, 0xdf, 0xaf, 0xd2, 0x90, 0xc0, 0x37, 0x58, 0x54, 0x36, 0x8f, 0x49, 0xb8, 0x62, - 0x75, 0x8b, 0x48, 0x47, 0xc0, 0xbe, 0xf7, 0x9a, 0x92, 0xa6, 0x68, 0x05, 0xda, 0x9d, 0xaf, - 0x72, 0x9a, 0x67, 0xb3, 0xb4, 0x14, 0x03, 0xae, 0x4f, 0x4c, 0x76, 0xb9, 0xd8, 0x64, 0x0a, - 0xba, 0x3b, 0xa8, 0x00, 0x60, 0x4d, 0xae, 0x81, 0xc3, 0xc5, 0x04, 0x17, 0x02, 0x15, 0x00, - 0x81, 0x9d, 0xfd, 0x53, 0x0c, 0xc1, 0x8f, 0xbe, 0x8b, 0xea, 0x00, 0x26, 0x19, 0x29, 0x33, - 0x91, 0x84, 0xbe, 0xad, 0x81}; -unsigned int dsa_privkey_pk8_der_len = 335; - -unsigned char ec_privkey_pk8_der[] = { - 0x30, 0x81, 0x87, 0x02, 0x01, 0x00, 0x30, 0x13, 0x06, 0x07, 0x2a, 0x86, 0x48, 0xce, - 0x3d, 0x02, 0x01, 0x06, 0x08, 0x2a, 0x86, 0x48, 0xce, 0x3d, 0x03, 0x01, 0x07, 0x04, - 0x6d, 0x30, 0x6b, 0x02, 0x01, 0x01, 0x04, 0x20, 0x73, 0x7c, 0x2e, 0xcd, 0x7b, 0x8d, - 0x19, 0x40, 0xbf, 0x29, 0x30, 0xaa, 0x9b, 0x4e, 0xd3, 0xff, 0x94, 0x1e, 0xed, 0x09, - 0x36, 0x6b, 0xc0, 0x32, 0x99, 0x98, 0x64, 0x81, 0xf3, 0xa4, 0xd8, 0x59, 0xa1, 0x44, - 0x03, 0x42, 0x00, 0x04, 0xbf, 0x85, 0xd7, 0x72, 0x0d, 0x07, 0xc2, 0x54, 0x61, 0x68, - 0x3b, 0xc6, 0x48, 0xb4, 0x77, 0x8a, 0x9a, 0x14, 0xdd, 0x8a, 0x02, 0x4e, 0x3b, 0xdd, - 0x8c, 0x7d, 0xdd, 0x9a, 0xb2, 0xb5, 0x28, 0xbb, 0xc7, 0xaa, 0x1b, 0x51, 0xf1, 0x4e, - 0xbb, 0xbb, 0x0b, 0xd0, 0xce, 0x21, 0xbc, 0xc4, 0x1c, 0x6e, 0xb0, 0x00, 0x83, 0xcf, - 0x33, 0x76, 0xd1, 0x1f, 0xd4, 0x49, 0x49, 0xe0, 0xb2, 0x18, 0x3b, 0xfe}; -unsigned int ec_privkey_pk8_der_len = 138; - -keymaster_key_param_t ec_params[] = { - keymaster_param_enum(KM_TAG_ALGORITHM, KM_ALGORITHM_EC), - keymaster_param_long(KM_TAG_EC_CURVE, KM_EC_CURVE_P_521), - keymaster_param_enum(KM_TAG_PURPOSE, KM_PURPOSE_SIGN), - keymaster_param_enum(KM_TAG_PURPOSE, KM_PURPOSE_VERIFY), - keymaster_param_enum(KM_TAG_DIGEST, KM_DIGEST_NONE), - keymaster_param_bool(KM_TAG_NO_AUTH_REQUIRED), -}; -keymaster_key_param_set_t ec_param_set = {ec_params, sizeof(ec_params) / sizeof(*ec_params)}; - -keymaster_key_param_t rsa_params[] = { - keymaster_param_enum(KM_TAG_ALGORITHM, KM_ALGORITHM_RSA), - keymaster_param_int(KM_TAG_KEY_SIZE, 1024), - keymaster_param_long(KM_TAG_RSA_PUBLIC_EXPONENT, 65537), - keymaster_param_enum(KM_TAG_PURPOSE, KM_PURPOSE_SIGN), - keymaster_param_enum(KM_TAG_PURPOSE, KM_PURPOSE_VERIFY), - keymaster_param_enum(KM_TAG_PADDING, KM_PAD_NONE), - keymaster_param_enum(KM_TAG_DIGEST, KM_DIGEST_NONE), - keymaster_param_bool(KM_TAG_NO_AUTH_REQUIRED), -}; -keymaster_key_param_set_t rsa_param_set = {rsa_params, sizeof(rsa_params) / sizeof(*rsa_params)}; - -struct EVP_PKEY_Delete { - void operator()(EVP_PKEY* p) const { EVP_PKEY_free(p); } -}; - -struct EVP_PKEY_CTX_Delete { - void operator()(EVP_PKEY_CTX* p) { EVP_PKEY_CTX_free(p); } -}; - -static bool do_operation(TrustyKeymasterDevice* device, keymaster_purpose_t purpose, - keymaster_key_blob_t* key, keymaster_blob_t* input, - keymaster_blob_t* signature, keymaster_blob_t* output) { - keymaster_key_param_t params[] = { - keymaster_param_enum(KM_TAG_PADDING, KM_PAD_NONE), - keymaster_param_enum(KM_TAG_DIGEST, KM_DIGEST_NONE), - }; - keymaster_key_param_set_t param_set = {params, sizeof(params) / sizeof(*params)}; - keymaster_operation_handle_t op_handle; - keymaster_error_t error = device->begin(purpose, key, ¶m_set, nullptr, &op_handle); - if (error != KM_ERROR_OK) { - printf("Keymaster begin() failed: %d\n", error); - return false; - } - size_t input_consumed; - error = device->update(op_handle, nullptr, input, &input_consumed, nullptr, nullptr); - if (error != KM_ERROR_OK) { - printf("Keymaster update() failed: %d\n", error); - return false; - } - if (input_consumed != input->data_length) { - // This should never happen. If it does, it's a bug in the keymaster implementation. - printf("Keymaster update() did not consume all data.\n"); - device->abort(op_handle); - return false; - } - error = device->finish(op_handle, nullptr, nullptr, signature, nullptr, output); - if (error != KM_ERROR_OK) { - printf("Keymaster finish() failed: %d\n", error); - return false; - } - return true; -} - -static bool test_import_rsa(TrustyKeymasterDevice* device) { - printf("===================\n"); - printf("= RSA Import Test =\n"); - printf("===================\n\n"); - - printf("=== Importing RSA keypair === \n"); - keymaster_key_blob_t key; - keymaster_blob_t private_key = {rsa_privkey_pk8_der, rsa_privkey_pk8_der_len}; - int error = - device->import_key(&rsa_param_set, KM_KEY_FORMAT_PKCS8, &private_key, &key, nullptr); - if (error != KM_ERROR_OK) { - printf("Error importing RSA key: %d\n\n", error); - return false; - } - std::unique_ptr<const uint8_t[]> key_deleter(key.key_material); - - printf("=== Signing with imported RSA key ===\n"); - size_t message_len = 1024 / 8; - std::unique_ptr<uint8_t[]> message(new uint8_t[message_len]); - memset(message.get(), 'a', message_len); - keymaster_blob_t input = {message.get(), message_len}, signature; - - if (!do_operation(device, KM_PURPOSE_SIGN, &key, &input, nullptr, &signature)) { - printf("Error signing data with imported RSA key\n\n"); - return false; - } - std::unique_ptr<const uint8_t[]> signature_deleter(signature.data); - - printf("=== Verifying with imported RSA key === \n"); - if (!do_operation(device, KM_PURPOSE_VERIFY, &key, &input, &signature, nullptr)) { - printf("Error verifying data with imported RSA key\n\n"); - return false; - } - - printf("\n"); - return true; -} - -static bool test_rsa(TrustyKeymasterDevice* device) { - printf("============\n"); - printf("= RSA Test =\n"); - printf("============\n\n"); - - printf("=== Generating RSA key pair ===\n"); - keymaster_key_blob_t key; - int error = device->generate_key(&rsa_param_set, &key, nullptr); - if (error != KM_ERROR_OK) { - printf("Error generating RSA key pair: %d\n\n", error); - return false; - } - std::unique_ptr<const uint8_t[]> key_deleter(key.key_material); - - printf("=== Signing with RSA key === \n"); - size_t message_len = 1024 / 8; - std::unique_ptr<uint8_t[]> message(new uint8_t[message_len]); - memset(message.get(), 'a', message_len); - keymaster_blob_t input = {message.get(), message_len}, signature; - - if (!do_operation(device, KM_PURPOSE_SIGN, &key, &input, nullptr, &signature)) { - printf("Error signing data with RSA key\n\n"); - return false; - } - std::unique_ptr<const uint8_t[]> signature_deleter(signature.data); - - printf("=== Verifying with RSA key === \n"); - if (!do_operation(device, KM_PURPOSE_VERIFY, &key, &input, &signature, nullptr)) { - printf("Error verifying data with RSA key\n\n"); - return false; - } - - printf("=== Exporting RSA public key ===\n"); - keymaster_blob_t exported_key; - error = device->export_key(KM_KEY_FORMAT_X509, &key, nullptr, nullptr, &exported_key); - if (error != KM_ERROR_OK) { - printf("Error exporting RSA public key: %d\n\n", error); - return false; - } - - printf("=== Verifying with exported key ===\n"); - const uint8_t* tmp = exported_key.data; - std::unique_ptr<EVP_PKEY, EVP_PKEY_Delete> pkey( - d2i_PUBKEY(NULL, &tmp, exported_key.data_length)); - std::unique_ptr<EVP_PKEY_CTX, EVP_PKEY_CTX_Delete> ctx(EVP_PKEY_CTX_new(pkey.get(), NULL)); - if (EVP_PKEY_verify_init(ctx.get()) != 1) { - printf("Error initializing openss EVP context\n\n"); - return false; - } - if (EVP_PKEY_type(pkey->type) != EVP_PKEY_RSA) { - printf("Exported key was the wrong type?!?\n\n"); - return false; - } - - EVP_PKEY_CTX_set_rsa_padding(ctx.get(), RSA_NO_PADDING); - if (EVP_PKEY_verify(ctx.get(), signature.data, signature.data_length, message.get(), - message_len) != 1) { - printf("Verification with exported pubkey failed.\n\n"); - return false; - } else { - printf("Verification succeeded\n"); - } - - printf("\n"); - return true; -} - -static bool test_import_ecdsa(TrustyKeymasterDevice* device) { - printf("=====================\n"); - printf("= ECDSA Import Test =\n"); - printf("=====================\n\n"); - - printf("=== Importing ECDSA keypair === \n"); - keymaster_key_blob_t key; - keymaster_blob_t private_key = {ec_privkey_pk8_der, ec_privkey_pk8_der_len}; - int error = device->import_key(&ec_param_set, KM_KEY_FORMAT_PKCS8, &private_key, &key, nullptr); - if (error != KM_ERROR_OK) { - printf("Error importing ECDSA key: %d\n\n", error); - return false; - } - std::unique_ptr<const uint8_t[]> deleter(key.key_material); - - printf("=== Signing with imported ECDSA key ===\n"); - size_t message_len = 30 /* arbitrary */; - std::unique_ptr<uint8_t[]> message(new uint8_t[message_len]); - memset(message.get(), 'a', message_len); - keymaster_blob_t input = {message.get(), message_len}, signature; - - if (!do_operation(device, KM_PURPOSE_SIGN, &key, &input, nullptr, &signature)) { - printf("Error signing data with imported ECDSA key\n\n"); - return false; - } - std::unique_ptr<const uint8_t[]> signature_deleter(signature.data); - - printf("=== Verifying with imported ECDSA key === \n"); - if (!do_operation(device, KM_PURPOSE_VERIFY, &key, &input, &signature, nullptr)) { - printf("Error verifying data with imported ECDSA key\n\n"); - return false; - } - - printf("\n"); - return true; -} - -static bool test_ecdsa(TrustyKeymasterDevice* device) { - printf("==============\n"); - printf("= ECDSA Test =\n"); - printf("==============\n\n"); - - printf("=== Generating ECDSA key pair ===\n"); - keymaster_key_blob_t key; - int error = device->generate_key(&ec_param_set, &key, nullptr); - if (error != KM_ERROR_OK) { - printf("Error generating ECDSA key pair: %d\n\n", error); - return false; - } - std::unique_ptr<const uint8_t[]> key_deleter(key.key_material); - - printf("=== Signing with ECDSA key === \n"); - size_t message_len = 30 /* arbitrary */; - std::unique_ptr<uint8_t[]> message(new uint8_t[message_len]); - memset(message.get(), 'a', message_len); - keymaster_blob_t input = {message.get(), message_len}, signature; - - if (!do_operation(device, KM_PURPOSE_SIGN, &key, &input, nullptr, &signature)) { - printf("Error signing data with ECDSA key\n\n"); - return false; - } - std::unique_ptr<const uint8_t[]> signature_deleter(signature.data); - - printf("=== Verifying with ECDSA key === \n"); - if (!do_operation(device, KM_PURPOSE_VERIFY, &key, &input, &signature, nullptr)) { - printf("Error verifying data with ECDSA key\n\n"); - return false; - } - - printf("=== Exporting ECDSA public key ===\n"); - keymaster_blob_t exported_key; - error = device->export_key(KM_KEY_FORMAT_X509, &key, nullptr, nullptr, &exported_key); - if (error != KM_ERROR_OK) { - printf("Error exporting ECDSA public key: %d\n\n", error); - return false; - } - - printf("=== Verifying with exported key ===\n"); - const uint8_t* tmp = exported_key.data; - std::unique_ptr<EVP_PKEY, EVP_PKEY_Delete> pkey( - d2i_PUBKEY(NULL, &tmp, exported_key.data_length)); - std::unique_ptr<EVP_PKEY_CTX, EVP_PKEY_CTX_Delete> ctx(EVP_PKEY_CTX_new(pkey.get(), NULL)); - if (EVP_PKEY_verify_init(ctx.get()) != 1) { - printf("Error initializing openssl EVP context\n\n"); - return false; - } - if (EVP_PKEY_type(pkey->type) != EVP_PKEY_EC) { - printf("Exported key was the wrong type?!?\n\n"); - return false; - } - - if (EVP_PKEY_verify(ctx.get(), signature.data, signature.data_length, message.get(), - message_len) != 1) { - printf("Verification with exported pubkey failed.\n\n"); - return false; - } else { - printf("Verification succeeded\n"); - } - - printf("\n"); - return true; -} - -int main(void) { - TrustyKeymasterDevice device(NULL); - keymaster::ConfigureDevice(reinterpret_cast<keymaster2_device_t*>(&device)); - if (device.session_error() != KM_ERROR_OK) { - printf("Failed to initialize Trusty session: %d\n", device.session_error()); - return 1; - } - printf("Trusty session initialized\n"); - - bool success = true; - success &= test_rsa(&device); - success &= test_import_rsa(&device); - success &= test_ecdsa(&device); - success &= test_import_ecdsa(&device); - - if (success) { - printf("\nTESTS PASSED!\n"); - } else { - printf("\n!!!!TESTS FAILED!!!\n"); - } - - return success ? 0 : 1; -} diff --git a/trusty/storage/proxy/rpmb.c b/trusty/storage/proxy/rpmb.c index 7dfd0d01b..d1ed6493f 100644 --- a/trusty/storage/proxy/rpmb.c +++ b/trusty/storage/proxy/rpmb.c @@ -231,7 +231,7 @@ static int send_ufs_rpmb_req(int sg_fd, const struct storage_rpmb_send_req* req) if (req->read_size) { /* Prepare SECURITY PROTOCOL IN command. */ - out_cdb.length = __builtin_bswap32(req->read_size); + in_cdb.length = __builtin_bswap32(req->read_size); sg_io_hdr_t io_hdr; set_sg_io_hdr(&io_hdr, SG_DXFER_FROM_DEV, sizeof(in_cdb), sizeof(sense_buffer), req->read_size, read_buf, (unsigned char*)&in_cdb, sense_buffer); |