diff options
| -rw-r--r-- | libc/Android.bp | 6 | ||||
| -rw-r--r-- | libc/bionic/malloc_common.cpp | 703 | ||||
| -rw-r--r-- | libc/bionic/malloc_common.h | 62 | ||||
| -rw-r--r-- | libc/bionic/malloc_common_dynamic.cpp | 420 | ||||
| -rw-r--r-- | libc/bionic/malloc_common_dynamic.h | 42 | ||||
| -rw-r--r-- | libc/bionic/malloc_heapprofd.cpp | 314 | ||||
| -rw-r--r-- | libc/bionic/malloc_heapprofd.h | 41 | ||||
| -rw-r--r-- | libc/symbol_ordering | 6 | ||||
| -rw-r--r-- | tests/malloc_test.cpp | 39 |
9 files changed, 958 insertions, 675 deletions
diff --git a/libc/Android.bp b/libc/Android.bp index dc437d803..40f0aaecd 100644 --- a/libc/Android.bp +++ b/libc/Android.bp @@ -1300,7 +1300,9 @@ cc_library_static { cc_library_static { name: "libc_ndk", defaults: ["libc_defaults"], - srcs: libc_common_src_files + ["bionic/malloc_common.cpp"], + srcs: libc_common_src_files + [ + "bionic/malloc_common.cpp", + ], multilib: { lib32: { srcs: libc_common_src_files_32, @@ -1493,6 +1495,8 @@ filegroup { "arch-common/bionic/crtbrand.S", "bionic/icu.cpp", "bionic/malloc_common.cpp", + "bionic/malloc_common_dynamic.cpp", + "bionic/malloc_heapprofd.cpp", "bionic/NetdClient.cpp", "arch-common/bionic/crtend_so.S", ], diff --git a/libc/bionic/malloc_common.cpp b/libc/bionic/malloc_common.cpp index b35aa2711..80e82f77c 100644 --- a/libc/bionic/malloc_common.cpp +++ b/libc/bionic/malloc_common.cpp @@ -41,69 +41,17 @@ // get_malloc_leak_info. // write_malloc_leak_info: Writes the leak info data to a file. -#include <pthread.h> -#include <stdatomic.h> +#include <stdint.h> -#include <private/bionic_defs.h> #include <private/bionic_config.h> -#include <private/bionic_globals.h> -#include <private/bionic_malloc.h> -#include <private/bionic_malloc_dispatch.h> -#if __has_feature(hwaddress_sanitizer) -// FIXME: implement these in HWASan allocator. -extern "C" int __sanitizer_iterate(uintptr_t base __unused, size_t size __unused, - void (*callback)(uintptr_t base, size_t size, void* arg) __unused, - void* arg __unused) { - return 0; -} - -extern "C" void __sanitizer_malloc_disable() { -} - -extern "C" void __sanitizer_malloc_enable() { -} -#include <sanitizer/hwasan_interface.h> -#define Malloc(function) __sanitizer_ ## function - -#else // __has_feature(hwaddress_sanitizer) -#include "jemalloc.h" -#define Malloc(function) je_ ## function -#endif - -template <typename T> -static T* RemoveConst(const T* x) { - return const_cast<T*>(x); -} - -// RemoveConst is a workaround for bug in current libcxx. Fix in -// https://reviews.llvm.org/D47613 -#define atomic_load_explicit_const(obj, order) atomic_load_explicit(RemoveConst(obj), order) +#include "malloc_common.h" -static constexpr MallocDispatch __libc_malloc_default_dispatch - __attribute__((unused)) = { - Malloc(calloc), - Malloc(free), - Malloc(mallinfo), - Malloc(malloc), - Malloc(malloc_usable_size), - Malloc(memalign), - Malloc(posix_memalign), -#if defined(HAVE_DEPRECATED_MALLOC_FUNCS) - Malloc(pvalloc), -#endif - Malloc(realloc), -#if defined(HAVE_DEPRECATED_MALLOC_FUNCS) - Malloc(valloc), -#endif - Malloc(iterate), - Malloc(malloc_disable), - Malloc(malloc_enable), - Malloc(mallopt), - Malloc(aligned_alloc), - }; +// ============================================================================= +// Global variables instantations. +// ============================================================================= -// Malloc hooks. +// Malloc hooks globals. void* (*volatile __malloc_hook)(size_t, const void*); void* (*volatile __realloc_hook)(void*, size_t, const void*); void (*volatile __free_hook)(void*, const void*); @@ -111,15 +59,11 @@ void* (*volatile __memalign_hook)(size_t, size_t, const void*); // In a VM process, this is set to 1 after fork()ing out of zygote. int gMallocLeakZygoteChild = 0; +// ============================================================================= // ============================================================================= // Allocation functions // ============================================================================= -static inline const MallocDispatch* GetDispatchTable() { - return atomic_load_explicit_const(&__libc_globals->current_dispatch_table, - memory_order_acquire); -} - extern "C" void* calloc(size_t n_elements, size_t elem_size) { auto dispatch_table = GetDispatchTable(); if (__predict_false(dispatch_table != nullptr)) { @@ -227,602 +171,7 @@ extern "C" void* valloc(size_t bytes) { return Malloc(valloc)(bytes); } #endif - -// We implement malloc debugging only in libc.so, so the code below -// must be excluded if we compile this file for static libc.a -#if !defined(LIBC_STATIC) - -#include <dlfcn.h> -#include <fcntl.h> -#include <stdio.h> -#include <stdlib.h> -#include <unistd.h> - -#include <async_safe/log.h> -#include <sys/system_properties.h> - -extern "C" int __cxa_atexit(void (*func)(void *), void *arg, void *dso); - -static const char* HOOKS_SHARED_LIB = "libc_malloc_hooks.so"; -static const char* HOOKS_PROPERTY_ENABLE = "libc.debug.hooks.enable"; -static const char* HOOKS_ENV_ENABLE = "LIBC_HOOKS_ENABLE"; - -static const char* DEBUG_SHARED_LIB = "libc_malloc_debug.so"; -static const char* DEBUG_PROPERTY_OPTIONS = "libc.debug.malloc.options"; -static const char* DEBUG_PROPERTY_PROGRAM = "libc.debug.malloc.program"; -static const char* DEBUG_ENV_OPTIONS = "LIBC_DEBUG_MALLOC_OPTIONS"; - -static const char* HEAPPROFD_SHARED_LIB = "heapprofd_client.so"; -static const char* HEAPPROFD_PREFIX = "heapprofd"; -static const char* HEAPPROFD_PROPERTY_ENABLE = "heapprofd.enable"; -static const int HEAPPROFD_SIGNAL = __SIGRTMIN + 4; - -enum FunctionEnum : uint8_t { - FUNC_INITIALIZE, - FUNC_FINALIZE, - FUNC_GET_MALLOC_LEAK_INFO, - FUNC_FREE_MALLOC_LEAK_INFO, - FUNC_MALLOC_BACKTRACE, - FUNC_WRITE_LEAK_INFO, - FUNC_LAST, -}; -static void* g_functions[FUNC_LAST]; - -typedef void (*finalize_func_t)(); -typedef bool (*init_func_t)(const MallocDispatch*, int*, const char*); -typedef void (*get_malloc_leak_info_func_t)(uint8_t**, size_t*, size_t*, size_t*, size_t*); -typedef void (*free_malloc_leak_info_func_t)(uint8_t*); -typedef bool (*write_malloc_leak_info_func_t)(FILE*); -typedef ssize_t (*malloc_backtrace_func_t)(void*, uintptr_t*, size_t); - -// ============================================================================= -// Log functions -// ============================================================================= -#define error_log(format, ...) \ - async_safe_format_log(ANDROID_LOG_ERROR, "libc", (format), ##__VA_ARGS__ ) -#define info_log(format, ...) \ - async_safe_format_log(ANDROID_LOG_INFO, "libc", (format), ##__VA_ARGS__ ) -// ============================================================================= - -// In a Zygote child process, this is set to true if profiling of this process -// is allowed. Note that this set at a later time than the above -// gMallocLeakZygoteChild. The latter is set during the fork (while still in -// zygote's SELinux domain). While this bit is set after the child is -// specialized (and has transferred SELinux domains if applicable). -static _Atomic bool gMallocZygoteChildProfileable = false; - // ============================================================================= -// Exported for use by ddms. -// ============================================================================= - -// Retrieve native heap information. -// -// "*info" is set to a buffer we allocate -// "*overall_size" is set to the size of the "info" buffer -// "*info_size" is set to the size of a single entry -// "*total_memory" is set to the sum of all allocations we're tracking; does -// not include heap overhead -// "*backtrace_size" is set to the maximum number of entries in the back trace -extern "C" void get_malloc_leak_info(uint8_t** info, size_t* overall_size, - size_t* info_size, size_t* total_memory, size_t* backtrace_size) { - void* func = g_functions[FUNC_GET_MALLOC_LEAK_INFO]; - if (func == nullptr) { - return; - } - reinterpret_cast<get_malloc_leak_info_func_t>(func)(info, overall_size, info_size, total_memory, - backtrace_size); -} - -extern "C" void free_malloc_leak_info(uint8_t* info) { - void* func = g_functions[FUNC_FREE_MALLOC_LEAK_INFO]; - if (func == nullptr) { - return; - } - reinterpret_cast<free_malloc_leak_info_func_t>(func)(info); -} - -extern "C" void write_malloc_leak_info(FILE* fp) { - if (fp == nullptr) { - error_log("write_malloc_leak_info called with a nullptr"); - return; - } - - void* func = g_functions[FUNC_WRITE_LEAK_INFO]; - bool written = false; - if (func != nullptr) { - written = reinterpret_cast<write_malloc_leak_info_func_t>(func)(fp); - } - - if (!written) { - fprintf(fp, "Native heap dump not available. To enable, run these commands (requires root):\n"); - fprintf(fp, "# adb shell stop\n"); - fprintf(fp, "# adb shell setprop libc.debug.malloc.options backtrace\n"); - fprintf(fp, "# adb shell start\n"); - } -} - -// ============================================================================= - -template<typename FunctionType> -static bool InitMallocFunction(void* malloc_impl_handler, FunctionType* func, const char* prefix, const char* suffix) { - char symbol[128]; - snprintf(symbol, sizeof(symbol), "%s_%s", prefix, suffix); - *func = reinterpret_cast<FunctionType>(dlsym(malloc_impl_handler, symbol)); - if (*func == nullptr) { - error_log("%s: dlsym(\"%s\") failed", getprogname(), symbol); - return false; - } - return true; -} - -static bool InitMallocFunctions(void* impl_handler, MallocDispatch* table, const char* prefix) { - if (!InitMallocFunction<MallocFree>(impl_handler, &table->free, prefix, "free")) { - return false; - } - if (!InitMallocFunction<MallocCalloc>(impl_handler, &table->calloc, prefix, "calloc")) { - return false; - } - if (!InitMallocFunction<MallocMallinfo>(impl_handler, &table->mallinfo, prefix, "mallinfo")) { - return false; - } - if (!InitMallocFunction<MallocMallopt>(impl_handler, &table->mallopt, prefix, "mallopt")) { - return false; - } - if (!InitMallocFunction<MallocMalloc>(impl_handler, &table->malloc, prefix, "malloc")) { - return false; - } - if (!InitMallocFunction<MallocMallocUsableSize>(impl_handler, &table->malloc_usable_size, prefix, - "malloc_usable_size")) { - return false; - } - if (!InitMallocFunction<MallocMemalign>(impl_handler, &table->memalign, prefix, "memalign")) { - return false; - } - if (!InitMallocFunction<MallocPosixMemalign>(impl_handler, &table->posix_memalign, prefix, - "posix_memalign")) { - return false; - } - if (!InitMallocFunction<MallocAlignedAlloc>(impl_handler, &table->aligned_alloc, - prefix, "aligned_alloc")) { - return false; - } - if (!InitMallocFunction<MallocRealloc>(impl_handler, &table->realloc, prefix, "realloc")) { - return false; - } - if (!InitMallocFunction<MallocIterate>(impl_handler, &table->iterate, prefix, "iterate")) { - return false; - } - if (!InitMallocFunction<MallocMallocDisable>(impl_handler, &table->malloc_disable, prefix, - "malloc_disable")) { - return false; - } - if (!InitMallocFunction<MallocMallocEnable>(impl_handler, &table->malloc_enable, prefix, - "malloc_enable")) { - return false; - } -#if defined(HAVE_DEPRECATED_MALLOC_FUNCS) - if (!InitMallocFunction<MallocPvalloc>(impl_handler, &table->pvalloc, prefix, "pvalloc")) { - return false; - } - if (!InitMallocFunction<MallocValloc>(impl_handler, &table->valloc, prefix, "valloc")) { - return false; - } -#endif - - return true; -} - -static void MallocFiniImpl(void*) { - // Our BSD stdio implementation doesn't close the standard streams, - // it only flushes them. Other unclosed FILE*s will show up as - // malloc leaks, but to avoid the standard streams showing up in - // leak reports, close them here. - fclose(stdin); - fclose(stdout); - fclose(stderr); - - reinterpret_cast<finalize_func_t>(g_functions[FUNC_FINALIZE])(); -} - -static bool CheckLoadMallocHooks(char** options) { - char* env = getenv(HOOKS_ENV_ENABLE); - if ((env == nullptr || env[0] == '\0' || env[0] == '0') && - (__system_property_get(HOOKS_PROPERTY_ENABLE, *options) == 0 || *options[0] == '\0' || *options[0] == '0')) { - return false; - } - *options = nullptr; - return true; -} - -static bool CheckLoadMallocDebug(char** options) { - // If DEBUG_MALLOC_ENV_OPTIONS is set then it overrides the system properties. - char* env = getenv(DEBUG_ENV_OPTIONS); - if (env == nullptr || env[0] == '\0') { - if (__system_property_get(DEBUG_PROPERTY_OPTIONS, *options) == 0 || *options[0] == '\0') { - return false; - } - - // Check to see if only a specific program should have debug malloc enabled. - char program[PROP_VALUE_MAX]; - if (__system_property_get(DEBUG_PROPERTY_PROGRAM, program) != 0 && - strstr(getprogname(), program) == nullptr) { - return false; - } - } else { - *options = env; - } - return true; -} - -static bool GetHeapprofdProgramProperty(char* data, size_t size) { - constexpr char prefix[] = "heapprofd.enable."; - // - 1 to skip nullbyte, which we will write later. - constexpr size_t prefix_size = sizeof(prefix) - 1; - if (size < prefix_size) { - error_log("%s: Overflow constructing heapprofd property", getprogname()); - return false; - } - memcpy(data, prefix, prefix_size); - - int fd = open("/proc/self/cmdline", O_RDONLY | O_CLOEXEC); - if (fd == -1) { - error_log("%s: Failed to open /proc/self/cmdline", getprogname()); - return false; - } - char cmdline[128]; - ssize_t rd = read(fd, cmdline, sizeof(cmdline) - 1); - close(fd); - if (rd == -1) { - error_log("%s: Failed to read /proc/self/cmdline", getprogname()); - return false; - } - cmdline[rd] = '\0'; - char* first_arg = static_cast<char*>(memchr(cmdline, '\0', rd)); - if (first_arg == nullptr || first_arg == cmdline + size - 1) { - error_log("%s: Overflow reading cmdline", getprogname()); - return false; - } - // For consistency with what we do with Java app cmdlines, trim everything - // after the @ sign of the first arg. - char* first_at = static_cast<char*>(memchr(cmdline, '@', rd)); - if (first_at != nullptr && first_at < first_arg) { - *first_at = '\0'; - first_arg = first_at; - } - - char* start = static_cast<char*>(memrchr(cmdline, '/', first_arg - cmdline)); - if (start == first_arg) { - // The first argument ended in a slash. - error_log("%s: cmdline ends in /", getprogname()); - return false; - } else if (start == nullptr) { - start = cmdline; - } else { - // Skip the /. - start++; - } - - size_t name_size = static_cast<size_t>(first_arg - start); - if (name_size >= size - prefix_size) { - error_log("%s: overflow constructing heapprofd property.", getprogname()); - return false; - } - // + 1 to also copy the trailing null byte. - memcpy(data + prefix_size, start, name_size + 1); - return true; -} - -static bool CheckLoadHeapprofd() { - // First check for heapprofd.enable. If it is set to "all", enable - // heapprofd for all processes. Otherwise, check heapprofd.enable.${prog}, - // if it is set and not 0, enable heap profiling for this process. - char property_value[PROP_VALUE_MAX]; - if (__system_property_get(HEAPPROFD_PROPERTY_ENABLE, property_value) == 0) { - return false; - } - if (strcmp(property_value, "all") == 0) { - return true; - } - - char program_property[128]; - if (!GetHeapprofdProgramProperty(program_property, - sizeof(program_property))) { - return false; - } - if (__system_property_get(program_property, property_value) == 0) { - return false; - } - return program_property[0] != '\0'; -} - -static void ClearGlobalFunctions() { - for (size_t i = 0; i < FUNC_LAST; i++) { - g_functions[i] = nullptr; - } -} - -static bool InitSharedLibrary(void* impl_handle, const char* shared_lib, const char* prefix, MallocDispatch* dispatch_table) { - static constexpr const char* names[] = { - "initialize", - "finalize", - "get_malloc_leak_info", - "free_malloc_leak_info", - "malloc_backtrace", - "write_malloc_leak_info", - }; - for (size_t i = 0; i < FUNC_LAST; i++) { - char symbol[128]; - snprintf(symbol, sizeof(symbol), "%s_%s", prefix, names[i]); - g_functions[i] = dlsym(impl_handle, symbol); - if (g_functions[i] == nullptr) { - error_log("%s: %s routine not found in %s", getprogname(), symbol, shared_lib); - ClearGlobalFunctions(); - return false; - } - } - - if (!InitMallocFunctions(impl_handle, dispatch_table, prefix)) { - ClearGlobalFunctions(); - return false; - } - return true; -} - -static void* LoadSharedLibrary(const char* shared_lib, const char* prefix, MallocDispatch* dispatch_table) { - void* impl_handle = dlopen(shared_lib, RTLD_NOW | RTLD_LOCAL); - if (impl_handle == nullptr) { - error_log("%s: Unable to open shared library %s: %s", getprogname(), shared_lib, dlerror()); - return nullptr; - } - - if (!InitSharedLibrary(impl_handle, shared_lib, prefix, dispatch_table)) { - dlclose(impl_handle); - impl_handle = nullptr; - } - - return impl_handle; -} - -// The handle returned by dlopen when previously loading the heapprofd -// hooks. nullptr if they had not been loaded before. -static _Atomic (void*) g_heapprofd_handle = nullptr; - -static void InstallHooks(libc_globals* globals, const char* options, - const char* prefix, const char* shared_lib) { - void* impl_handle = atomic_load(&g_heapprofd_handle); - bool reusing_handle = impl_handle != nullptr; - if (reusing_handle) { - if (!InitSharedLibrary(impl_handle, shared_lib, prefix, &globals->malloc_dispatch_table)) { - return; - } - } else { - impl_handle = LoadSharedLibrary(shared_lib, prefix, &globals->malloc_dispatch_table); - if (impl_handle == nullptr) { - return; - } - } - init_func_t init_func = reinterpret_cast<init_func_t>(g_functions[FUNC_INITIALIZE]); - if (!init_func(&__libc_malloc_default_dispatch, &gMallocLeakZygoteChild, options)) { - error_log("%s: failed to enable malloc %s", getprogname(), prefix); - if (!reusing_handle) { - // We should not close if we are re-using an old handle, as we cannot be - // sure other threads are not currently in the hooks. - dlclose(impl_handle); - } - ClearGlobalFunctions(); - return; - } - - // Do a pointer swap so that all of the functions become valid at once to - // avoid any initialization order problems. - atomic_store(&globals->current_dispatch_table, &globals->malloc_dispatch_table); - - atomic_store(&g_heapprofd_handle, impl_handle); - - info_log("%s: malloc %s enabled", getprogname(), prefix); - - // Use atexit to trigger the cleanup function. This avoids a problem - // where another atexit function is used to cleanup allocated memory, - // but the finalize function was already called. This particular error - // seems to be triggered by a zygote spawned process calling exit. - int ret_value = __cxa_atexit(MallocFiniImpl, nullptr, nullptr); - if (ret_value != 0) { - error_log("failed to set atexit cleanup function: %d", ret_value); - } -} - -// The logic for triggering heapprofd (at runtime) is as follows: -// 1. HEAPPROFD_SIGNAL is received by the process, entering the -// MaybeInstallInitHeapprofdHook signal handler. -// 2. If the initialization is not already in flight -// (g_heapprofd_init_in_progress is false), the malloc hook is set to -// point at InitHeapprofdHook, and g_heapprofd_init_in_progress is set to -// true. -// 3. The next malloc call enters InitHeapprofdHook, which removes the malloc -// hook, and spawns a detached pthread to run the InitHeapprofd task. -// (g_heapprofd_init_hook_installed atomic is used to perform this once.) -// 4. InitHeapprofd, on a dedicated pthread, loads the heapprofd client library, -// installs the full set of heapprofd hooks, and invokes the client's -// initializer. The dedicated pthread then terminates. -// 5. g_heapprofd_init_in_progress and g_heapprofd_init_hook_installed are -// reset to false such that heapprofd can be reinitialized. Reinitialization -// means that a new profiling session is started, and any still active is -// torn down. -// -// The incremental hooking and a dedicated task thread are used since we cannot -// do heavy work within a signal handler, or when blocking a malloc invocation. - -static _Atomic bool g_heapprofd_init_in_progress = false; -static _Atomic bool g_heapprofd_init_hook_installed = false; - -extern "C" void MaybeInstallInitHeapprofdHook(int); - -// Initializes memory allocation framework once per process. -static void MallocInitImpl(libc_globals* globals) { - struct sigaction action = {}; - action.sa_handler = MaybeInstallInitHeapprofdHook; - sigaction(HEAPPROFD_SIGNAL, &action, nullptr); - - const char* prefix; - const char* shared_lib; - char prop[PROP_VALUE_MAX]; - char* options = prop; - // Prefer malloc debug since it existed first and is a more complete - // malloc interceptor than the hooks. - if (CheckLoadMallocDebug(&options)) { - prefix = "debug"; - shared_lib = DEBUG_SHARED_LIB; - } else if (CheckLoadMallocHooks(&options)) { - prefix = "hooks"; - shared_lib = HOOKS_SHARED_LIB; - } else if (CheckLoadHeapprofd()) { - prefix = "heapprofd"; - shared_lib = HEAPPROFD_SHARED_LIB; - } else { - return; - } - if (!atomic_exchange(&g_heapprofd_init_in_progress, true)) { - InstallHooks(globals, options, prefix, shared_lib); - atomic_store(&g_heapprofd_init_in_progress, false); - } -} - -// Initializes memory allocation framework. -// This routine is called from __libc_init routines in libc_init_dynamic.cpp. -__BIONIC_WEAK_FOR_NATIVE_BRIDGE -__LIBC_HIDDEN__ void __libc_init_malloc(libc_globals* globals) { - MallocInitImpl(globals); -} - -static void* InitHeapprofd(void*) { - __libc_globals.mutate([](libc_globals* globals) { - InstallHooks(globals, nullptr, HEAPPROFD_PREFIX, HEAPPROFD_SHARED_LIB); - }); - atomic_store(&g_heapprofd_init_in_progress, false); - // Allow to install hook again to re-initialize heap profiling after the - // current session finished. - atomic_store(&g_heapprofd_init_hook_installed, false); - return nullptr; -} - -static void* InitHeapprofdHook(size_t bytes) { - if (!atomic_exchange(&g_heapprofd_init_hook_installed, true)) { - __libc_globals.mutate([](libc_globals* globals) { - atomic_store(&globals->current_dispatch_table, nullptr); - }); - - pthread_t thread_id; - if (pthread_create(&thread_id, nullptr, InitHeapprofd, nullptr) == -1) - error_log("%s: heapprofd: failed to pthread_create.", getprogname()); - else if (pthread_detach(thread_id) == -1) - error_log("%s: heapprofd: failed to pthread_detach", getprogname()); - if (pthread_setname_np(thread_id, "heapprofdinit") == -1) - error_log("%s: heapprod: failed to pthread_setname_np", getprogname()); - } - return Malloc(malloc)(bytes); -} - -static constexpr MallocDispatch __heapprofd_dispatch - __attribute__((unused)) = { - Malloc(calloc), - Malloc(free), - Malloc(mallinfo), - InitHeapprofdHook, - Malloc(malloc_usable_size), - Malloc(memalign), - Malloc(posix_memalign), -#if defined(HAVE_DEPRECATED_MALLOC_FUNCS) - Malloc(pvalloc), -#endif - Malloc(realloc), -#if defined(HAVE_DEPRECATED_MALLOC_FUNCS) - Malloc(valloc), -#endif - Malloc(iterate), - Malloc(malloc_disable), - Malloc(malloc_enable), - Malloc(mallopt), - Malloc(aligned_alloc), - }; - -extern "C" void MaybeInstallInitHeapprofdHook(int) { - // Zygote child processes must be marked profileable. - if (gMallocLeakZygoteChild && - !atomic_load_explicit_const(&gMallocZygoteChildProfileable, memory_order_acquire)) { - return; - } - - if (!atomic_exchange(&g_heapprofd_init_in_progress, true)) { - __libc_globals.mutate([](libc_globals* globals) { - atomic_store(&globals->current_dispatch_table, &__heapprofd_dispatch); - }); - } -} - -#endif // !LIBC_STATIC - -// ============================================================================= -// Platform-internal mallopt variant. -// ============================================================================= - -#if !defined(LIBC_STATIC) -bool MallocDispatchReset() { - if (!atomic_exchange(&g_heapprofd_init_in_progress, true)) { - __libc_globals.mutate([](libc_globals* globals) { - atomic_store(&globals->current_dispatch_table, nullptr); - }); - atomic_store(&g_heapprofd_init_in_progress, false); - return true; - } - errno = EAGAIN; - return false; -} - -// Marks this process as a profileable zygote child. -bool HandleInitZygoteChildProfiling() { - atomic_store_explicit(&gMallocZygoteChildProfileable, true, - memory_order_release); - - // Conditionally start "from startup" profiling. - if (CheckLoadHeapprofd()) { - // Directly call the signal handler (will correctly guard against - // concurrent signal delivery). - MaybeInstallInitHeapprofdHook(HEAPPROFD_SIGNAL); - } - return true; -} - -#else - -bool MallocDispatchReset() { - return true; -} - -bool HandleInitZygoteChildProfiling() { - return true; -} - -#endif // !defined(LIBC_STATIC) - -bool android_mallopt(int opcode, void* arg, size_t arg_size) { - if (opcode == M_INIT_ZYGOTE_CHILD_PROFILING) { - if (arg != nullptr || arg_size != 0) { - errno = EINVAL; - return false; - } - return HandleInitZygoteChildProfiling(); - } - if (opcode == M_RESET_HOOKS) { - if (arg != nullptr || arg_size != 0) { - errno = EINVAL; - return false; - } - return MallocDispatchReset(); - } - - errno = ENOTSUP; - return false; -} // ============================================================================= // Exported for use by libmemunreachable. @@ -858,16 +207,36 @@ extern "C" void malloc_enable() { return Malloc(malloc_enable)(); } -#ifndef LIBC_STATIC -extern "C" ssize_t malloc_backtrace(void* pointer, uintptr_t* frames, size_t frame_count) { - void* func = g_functions[FUNC_MALLOC_BACKTRACE]; - if (func == nullptr) { - return 0; - } - return reinterpret_cast<malloc_backtrace_func_t>(func)(pointer, frames, frame_count); -} -#else +#if defined(LIBC_STATIC) extern "C" ssize_t malloc_backtrace(void*, uintptr_t*, size_t) { return 0; } #endif + +#if __has_feature(hwaddress_sanitizer) +// FIXME: implement these in HWASan allocator. +extern "C" int __sanitizer_iterate(uintptr_t base __unused, size_t size __unused, + void (*callback)(uintptr_t base, size_t size, void* arg) __unused, + void* arg __unused) { + return 0; +} + +extern "C" void __sanitizer_malloc_disable() { +} + +extern "C" void __sanitizer_malloc_enable() { +} +#endif +// ============================================================================= + +// ============================================================================= +// Platform-internal mallopt variant. +// ============================================================================= +#if defined(LIBC_STATIC) +extern "C" bool android_mallopt(int, void*, size_t) { + // There are no options supported on static executables. + errno = ENOTSUP; + return false; +} +#endif +// ============================================================================= diff --git a/libc/bionic/malloc_common.h b/libc/bionic/malloc_common.h new file mode 100644 index 000000000..94a6df4bf --- /dev/null +++ b/libc/bionic/malloc_common.h @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#pragma once + +#include <stdatomic.h> + +#include <async_safe/log.h> +#include <private/bionic_globals.h> +#include <private/bionic_malloc_dispatch.h> + +#if __has_feature(hwaddress_sanitizer) + +#include <sanitizer/hwasan_interface.h> +#define Malloc(function) __sanitizer_ ## function + +#else // __has_feature(hwaddress_sanitizer) + +#include "jemalloc.h" +#define Malloc(function) je_ ## function + +#endif + +extern int gMallocLeakZygoteChild; + +static inline const MallocDispatch* GetDispatchTable() { + return atomic_load_explicit(&__libc_globals->current_dispatch_table, memory_order_acquire); +} + +// ============================================================================= +// Log functions +// ============================================================================= +#define error_log(format, ...) \ + async_safe_format_log(ANDROID_LOG_ERROR, "libc", (format), ##__VA_ARGS__ ) +#define info_log(format, ...) \ + async_safe_format_log(ANDROID_LOG_INFO, "libc", (format), ##__VA_ARGS__ ) +// ============================================================================= diff --git a/libc/bionic/malloc_common_dynamic.cpp b/libc/bionic/malloc_common_dynamic.cpp new file mode 100644 index 000000000..1c3f53fd6 --- /dev/null +++ b/libc/bionic/malloc_common_dynamic.cpp @@ -0,0 +1,420 @@ +/* + * Copyright (C) 2009 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#if defined(LIBC_STATIC) +#error This file should not be compiled for static targets. +#endif + +// Contains a thin layer that calls whatever real native allocator +// has been defined. For the libc shared library, this allows the +// implementation of a debug malloc that can intercept all of the allocation +// calls and add special debugging code to attempt to catch allocation +// errors. All of the debugging code is implemented in a separate shared +// library that is only loaded when the property "libc.debug.malloc.options" +// is set to a non-zero value. There are three functions exported to +// allow ddms, or other external users to get information from the debug +// allocation. +// get_malloc_leak_info: Returns information about all of the known native +// allocations that are currently in use. +// free_malloc_leak_info: Frees the data allocated by the call to +// get_malloc_leak_info. +// write_malloc_leak_info: Writes the leak info data to a file. + +#include <dlfcn.h> +#include <fcntl.h> +#include <pthread.h> +#include <stdatomic.h> +#include <stdbool.h> +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> + +#include <private/bionic_config.h> +#include <private/bionic_defs.h> +#include <private/bionic_malloc_dispatch.h> + +#include <sys/system_properties.h> + +#include "malloc_common.h" +#include "malloc_common_dynamic.h" +#include "malloc_heapprofd.h" + +static constexpr MallocDispatch __libc_malloc_default_dispatch + __attribute__((unused)) = { + Malloc(calloc), + Malloc(free), + Malloc(mallinfo), + Malloc(malloc), + Malloc(malloc_usable_size), + Malloc(memalign), + Malloc(posix_memalign), +#if defined(HAVE_DEPRECATED_MALLOC_FUNCS) + Malloc(pvalloc), +#endif + Malloc(realloc), +#if defined(HAVE_DEPRECATED_MALLOC_FUNCS) + Malloc(valloc), +#endif + Malloc(iterate), + Malloc(malloc_disable), + Malloc(malloc_enable), + Malloc(mallopt), + Malloc(aligned_alloc), + }; + +static constexpr char kHooksSharedLib[] = "libc_malloc_hooks.so"; +static constexpr char kHooksPrefix[] = "hooks"; +static constexpr char kHooksPropertyEnable[] = "libc.debug.hooks.enable"; +static constexpr char kHooksEnvEnable[] = "LIBC_HOOKS_ENABLE"; + +static constexpr char kDebugSharedLib[] = "libc_malloc_debug.so"; +static constexpr char kDebugPrefix[] = "debug"; +static constexpr char kDebugPropertyOptions[] = "libc.debug.malloc.options"; +static constexpr char kDebugPropertyProgram[] = "libc.debug.malloc.program"; +static constexpr char kDebugEnvOptions[] = "LIBC_DEBUG_MALLOC_OPTIONS"; + +typedef void (*finalize_func_t)(); +typedef bool (*init_func_t)(const MallocDispatch*, int*, const char*); +typedef void (*get_malloc_leak_info_func_t)(uint8_t**, size_t*, size_t*, size_t*, size_t*); +typedef void (*free_malloc_leak_info_func_t)(uint8_t*); +typedef bool (*write_malloc_leak_info_func_t)(FILE*); +typedef ssize_t (*malloc_backtrace_func_t)(void*, uintptr_t*, size_t); + +enum FunctionEnum : uint8_t { + FUNC_INITIALIZE, + FUNC_FINALIZE, + FUNC_GET_MALLOC_LEAK_INFO, + FUNC_FREE_MALLOC_LEAK_INFO, + FUNC_MALLOC_BACKTRACE, + FUNC_WRITE_LEAK_INFO, + FUNC_LAST, +}; +static void* gFunctions[FUNC_LAST]; + +extern "C" int __cxa_atexit(void (*func)(void *), void *arg, void *dso); + +template<typename FunctionType> +static bool InitMallocFunction(void* malloc_impl_handler, FunctionType* func, const char* prefix, const char* suffix) { + char symbol[128]; + snprintf(symbol, sizeof(symbol), "%s_%s", prefix, suffix); + *func = reinterpret_cast<FunctionType>(dlsym(malloc_impl_handler, symbol)); + if (*func == nullptr) { + error_log("%s: dlsym(\"%s\") failed", getprogname(), symbol); + return false; + } + return true; +} + +static bool InitMallocFunctions(void* impl_handler, MallocDispatch* table, const char* prefix) { + if (!InitMallocFunction<MallocFree>(impl_handler, &table->free, prefix, "free")) { + return false; + } + if (!InitMallocFunction<MallocCalloc>(impl_handler, &table->calloc, prefix, "calloc")) { + return false; + } + if (!InitMallocFunction<MallocMallinfo>(impl_handler, &table->mallinfo, prefix, "mallinfo")) { + return false; + } + if (!InitMallocFunction<MallocMallopt>(impl_handler, &table->mallopt, prefix, "mallopt")) { + return false; + } + if (!InitMallocFunction<MallocMalloc>(impl_handler, &table->malloc, prefix, "malloc")) { + return false; + } + if (!InitMallocFunction<MallocMallocUsableSize>(impl_handler, &table->malloc_usable_size, prefix, + "malloc_usable_size")) { + return false; + } + if (!InitMallocFunction<MallocMemalign>(impl_handler, &table->memalign, prefix, "memalign")) { + return false; + } + if (!InitMallocFunction<MallocPosixMemalign>(impl_handler, &table->posix_memalign, prefix, + "posix_memalign")) { + return false; + } + if (!InitMallocFunction<MallocAlignedAlloc>(impl_handler, &table->aligned_alloc, + prefix, "aligned_alloc")) { + return false; + } + if (!InitMallocFunction<MallocRealloc>(impl_handler, &table->realloc, prefix, "realloc")) { + return false; + } + if (!InitMallocFunction<MallocIterate>(impl_handler, &table->iterate, prefix, "iterate")) { + return false; + } + if (!InitMallocFunction<MallocMallocDisable>(impl_handler, &table->malloc_disable, prefix, + "malloc_disable")) { + return false; + } + if (!InitMallocFunction<MallocMallocEnable>(impl_handler, &table->malloc_enable, prefix, + "malloc_enable")) { + return false; + } +#if defined(HAVE_DEPRECATED_MALLOC_FUNCS) + if (!InitMallocFunction<MallocPvalloc>(impl_handler, &table->pvalloc, prefix, "pvalloc")) { + return false; + } + if (!InitMallocFunction<MallocValloc>(impl_handler, &table->valloc, prefix, "valloc")) { + return false; + } +#endif + + return true; +} + +static void MallocFiniImpl(void*) { + // Our BSD stdio implementation doesn't close the standard streams, + // it only flushes them. Other unclosed FILE*s will show up as + // malloc leaks, but to avoid the standard streams showing up in + // leak reports, close them here. + fclose(stdin); + fclose(stdout); + fclose(stderr); + + reinterpret_cast<finalize_func_t>(gFunctions[FUNC_FINALIZE])(); +} + +static bool CheckLoadMallocHooks(char** options) { + char* env = getenv(kHooksEnvEnable); + if ((env == nullptr || env[0] == '\0' || env[0] == '0') && + (__system_property_get(kHooksPropertyEnable, *options) == 0 || *options[0] == '\0' || *options[0] == '0')) { + return false; + } + *options = nullptr; + return true; +} + +static bool CheckLoadMallocDebug(char** options) { + // If kDebugMallocEnvOptions is set then it overrides the system properties. + char* env = getenv(kDebugEnvOptions); + if (env == nullptr || env[0] == '\0') { + if (__system_property_get(kDebugPropertyOptions, *options) == 0 || *options[0] == '\0') { + return false; + } + + // Check to see if only a specific program should have debug malloc enabled. + char program[PROP_VALUE_MAX]; + if (__system_property_get(kDebugPropertyProgram, program) != 0 && + strstr(getprogname(), program) == nullptr) { + return false; + } + } else { + *options = env; + } + return true; +} + +static void ClearGlobalFunctions() { + for (size_t i = 0; i < FUNC_LAST; i++) { + gFunctions[i] = nullptr; + } +} + +bool InitSharedLibrary(void* impl_handle, const char* shared_lib, const char* prefix, MallocDispatch* dispatch_table) { + static constexpr const char* names[] = { + "initialize", + "finalize", + "get_malloc_leak_info", + "free_malloc_leak_info", + "malloc_backtrace", + "write_malloc_leak_info", + }; + for (size_t i = 0; i < FUNC_LAST; i++) { + char symbol[128]; + snprintf(symbol, sizeof(symbol), "%s_%s", prefix, names[i]); + gFunctions[i] = dlsym(impl_handle, symbol); + if (gFunctions[i] == nullptr) { + error_log("%s: %s routine not found in %s", getprogname(), symbol, shared_lib); + ClearGlobalFunctions(); + return false; + } + } + + if (!InitMallocFunctions(impl_handle, dispatch_table, prefix)) { + ClearGlobalFunctions(); + return false; + } + return true; +} + +void* LoadSharedLibrary(const char* shared_lib, const char* prefix, MallocDispatch* dispatch_table) { + void* impl_handle = dlopen(shared_lib, RTLD_NOW | RTLD_LOCAL); + if (impl_handle == nullptr) { + error_log("%s: Unable to open shared library %s: %s", getprogname(), shared_lib, dlerror()); + return nullptr; + } + + if (!InitSharedLibrary(impl_handle, shared_lib, prefix, dispatch_table)) { + dlclose(impl_handle); + impl_handle = nullptr; + } + + return impl_handle; +} + +bool FinishInstallHooks(libc_globals* globals, const char* options, const char* prefix) { + init_func_t init_func = reinterpret_cast<init_func_t>(gFunctions[FUNC_INITIALIZE]); + if (!init_func(&__libc_malloc_default_dispatch, &gMallocLeakZygoteChild, options)) { + error_log("%s: failed to enable malloc %s", getprogname(), prefix); + ClearGlobalFunctions(); + return false; + } + + // Do a pointer swap so that all of the functions become valid at once to + // avoid any initialization order problems. + atomic_store(&globals->current_dispatch_table, &globals->malloc_dispatch_table); + + info_log("%s: malloc %s enabled", getprogname(), prefix); + + // Use atexit to trigger the cleanup function. This avoids a problem + // where another atexit function is used to cleanup allocated memory, + // but the finalize function was already called. This particular error + // seems to be triggered by a zygote spawned process calling exit. + int ret_value = __cxa_atexit(MallocFiniImpl, nullptr, nullptr); + if (ret_value != 0) { + // We don't consider this a fatal error. + info_log("failed to set atexit cleanup function: %d", ret_value); + } + return true; +} + +static void InstallHooks(libc_globals* globals, const char* options, const char* prefix, + const char* shared_lib) { + void* impl_handle = LoadSharedLibrary(shared_lib, prefix, &globals->malloc_dispatch_table); + if (impl_handle == nullptr) { + return; + } + + init_func_t init_func = reinterpret_cast<init_func_t>(gFunctions[FUNC_INITIALIZE]); + if (!init_func(&__libc_malloc_default_dispatch, &gMallocLeakZygoteChild, options)) { + error_log("%s: failed to enable malloc %s", getprogname(), prefix); + ClearGlobalFunctions(); + return; + } + + if (!FinishInstallHooks(globals, options, prefix)) { + dlclose(impl_handle); + } +} + +// Initializes memory allocation framework once per process. +static void MallocInitImpl(libc_globals* globals) { + char prop[PROP_VALUE_MAX]; + char* options = prop; + + // Prefer malloc debug since it existed first and is a more complete + // malloc interceptor than the hooks. + if (CheckLoadMallocDebug(&options)) { + InstallHooks(globals, options, kDebugPrefix, kDebugSharedLib); + } else if (CheckLoadMallocHooks(&options)) { + InstallHooks(globals, options, kHooksPrefix, kHooksSharedLib); + } else if (HeapprofdShouldLoad()) { + HeapprofdInstallHooksAtInit(globals); + } + + // Install this last to avoid as many race conditions as possible. + HeapprofdInstallSignalHandler(); +} + +// Initializes memory allocation framework. +// This routine is called from __libc_init routines in libc_init_dynamic.cpp. +__BIONIC_WEAK_FOR_NATIVE_BRIDGE +__LIBC_HIDDEN__ void __libc_init_malloc(libc_globals* globals) { + MallocInitImpl(globals); +} + +// ============================================================================= +// Functions to support dumping of native heap allocations using malloc debug. +// ============================================================================= + +// Retrieve native heap information. +// +// "*info" is set to a buffer we allocate +// "*overall_size" is set to the size of the "info" buffer +// "*info_size" is set to the size of a single entry +// "*total_memory" is set to the sum of all allocations we're tracking; does +// not include heap overhead +// "*backtrace_size" is set to the maximum number of entries in the back trace +extern "C" void get_malloc_leak_info(uint8_t** info, size_t* overall_size, + size_t* info_size, size_t* total_memory, size_t* backtrace_size) { + void* func = gFunctions[FUNC_GET_MALLOC_LEAK_INFO]; + if (func == nullptr) { + return; + } + reinterpret_cast<get_malloc_leak_info_func_t>(func)(info, overall_size, info_size, total_memory, + backtrace_size); +} + +extern "C" void free_malloc_leak_info(uint8_t* info) { + void* func = gFunctions[FUNC_FREE_MALLOC_LEAK_INFO]; + if (func == nullptr) { + return; + } + reinterpret_cast<free_malloc_leak_info_func_t>(func)(info); +} + +extern "C" void write_malloc_leak_info(FILE* fp) { + if (fp == nullptr) { + error_log("write_malloc_leak_info called with a nullptr"); + return; + } + + void* func = gFunctions[FUNC_WRITE_LEAK_INFO]; + bool written = false; + if (func != nullptr) { + written = reinterpret_cast<write_malloc_leak_info_func_t>(func)(fp); + } + + if (!written) { + fprintf(fp, "Native heap dump not available. To enable, run these commands (requires root):\n"); + fprintf(fp, "# adb shell stop\n"); + fprintf(fp, "# adb shell setprop libc.debug.malloc.options backtrace\n"); + fprintf(fp, "# adb shell start\n"); + } +} +// ============================================================================= + +// ============================================================================= +// Exported for use by libmemunreachable. +// ============================================================================= +extern "C" ssize_t malloc_backtrace(void* pointer, uintptr_t* frames, size_t frame_count) { + void* func = gFunctions[FUNC_MALLOC_BACKTRACE]; + if (func == nullptr) { + return 0; + } + return reinterpret_cast<malloc_backtrace_func_t>(func)(pointer, frames, frame_count); +} +// ============================================================================= + +// ============================================================================= +// Platform-internal mallopt variant. +// ============================================================================= +extern "C" bool android_mallopt(int opcode, void* arg, size_t arg_size) { + return HeapprofdMallopt(opcode, arg, arg_size); +} +// ============================================================================= diff --git a/libc/bionic/malloc_common_dynamic.h b/libc/bionic/malloc_common_dynamic.h new file mode 100644 index 000000000..8794ed044 --- /dev/null +++ b/libc/bionic/malloc_common_dynamic.h @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#pragma once + +#include <stdbool.h> + +#include <private/bionic_globals.h> +#include <private/bionic_malloc_dispatch.h> + +// Function prototypes. +bool InitSharedLibrary(void* impl_handle, const char* shared_lib, const char* prefix, + MallocDispatch* dispatch_table); + +void* LoadSharedLibrary(const char* shared_lib, const char* prefix, MallocDispatch* dispatch_table); + +bool FinishInstallHooks(libc_globals* globals, const char* options, const char* prefix); diff --git a/libc/bionic/malloc_heapprofd.cpp b/libc/bionic/malloc_heapprofd.cpp new file mode 100644 index 000000000..fb7266a02 --- /dev/null +++ b/libc/bionic/malloc_heapprofd.cpp @@ -0,0 +1,314 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#if defined(LIBC_STATIC) +#error This file should not be compiled for static targets. +#endif + +#include <dlfcn.h> +#include <fcntl.h> +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> + +#include <private/bionic_config.h> +#include <private/bionic_malloc.h> +#include <private/bionic_malloc_dispatch.h> +#include <sys/system_properties.h> + +#include "malloc_common.h" +#include "malloc_common_dynamic.h" +#include "malloc_heapprofd.h" + +static constexpr char kHeapprofdSharedLib[] = "heapprofd_client.so"; +static constexpr char kHeapprofdPrefix[] = "heapprofd"; +static constexpr char kHeapprofdPropertyEnable[] = "heapprofd.enable"; +static constexpr int kHeapprofdSignal = __SIGRTMIN + 4; + +// The logic for triggering heapprofd (at runtime) is as follows: +// 1. HEAPPROFD_SIGNAL is received by the process, entering the +// MaybeInstallInitHeapprofdHook signal handler. +// 2. If the initialization is not already in flight +// (gHeapprofdInitInProgress is false), the malloc hook is set to +// point at InitHeapprofdHook, and gHeapprofdInitInProgress is set to +// true. +// 3. The next malloc call enters InitHeapprofdHook, which removes the malloc +// hook, and spawns a detached pthread to run the InitHeapprofd task. +// (gHeapprofdInitHook_installed atomic is used to perform this once.) +// 4. InitHeapprofd, on a dedicated pthread, loads the heapprofd client library, +// installs the full set of heapprofd hooks, and invokes the client's +// initializer. The dedicated pthread then terminates. +// 5. gHeapprofdInitInProgress and gHeapprofdInitHookInstalled are +// reset to false such that heapprofd can be reinitialized. Reinitialization +// means that a new profiling session is started, and any still active is +// torn down. +// +// The incremental hooking and a dedicated task thread are used since we cannot +// do heavy work within a signal handler, or when blocking a malloc invocation. + +// The handle returned by dlopen when previously loading the heapprofd +// hooks. nullptr if shared library has not been already been loaded. +static _Atomic (void*) gHeapprofdHandle = nullptr; + +static _Atomic bool gHeapprofdInitInProgress = false; +static _Atomic bool gHeapprofdInitHookInstalled = false; + +// In a Zygote child process, this is set to true if profiling of this process +// is allowed. Note that this is set at a later time than the global +// gMallocLeakZygoteChild. The latter is set during the fork (while still in +// zygote's SELinux domain). While this bit is set after the child is +// specialized (and has transferred SELinux domains if applicable). +static _Atomic bool gMallocZygoteChildProfileable = false; + +extern "C" void* MallocInitHeapprofdHook(size_t); + +static constexpr MallocDispatch __heapprofd_init_dispatch + __attribute__((unused)) = { + Malloc(calloc), + Malloc(free), + Malloc(mallinfo), + MallocInitHeapprofdHook, + Malloc(malloc_usable_size), + Malloc(memalign), + Malloc(posix_memalign), +#if defined(HAVE_DEPRECATED_MALLOC_FUNCS) + Malloc(pvalloc), +#endif + Malloc(realloc), +#if defined(HAVE_DEPRECATED_MALLOC_FUNCS) + Malloc(valloc), +#endif + Malloc(iterate), + Malloc(malloc_disable), + Malloc(malloc_enable), + Malloc(mallopt), + Malloc(aligned_alloc), + }; + +static void MaybeInstallInitHeapprofdHook(int) { + // Zygote child processes must be marked profileable. + if (gMallocLeakZygoteChild && + !atomic_load_explicit(&gMallocZygoteChildProfileable, memory_order_acquire)) { + return; + } + + if (!atomic_exchange(&gHeapprofdInitInProgress, true)) { + __libc_globals.mutate([](libc_globals* globals) { + atomic_store(&globals->current_dispatch_table, &__heapprofd_init_dispatch); + }); + } +} + +static bool GetHeapprofdProgramProperty(char* data, size_t size) { + constexpr char prefix[] = "heapprofd.enable."; + // - 1 to skip nullbyte, which we will write later. + constexpr size_t prefix_size = sizeof(prefix) - 1; + if (size < prefix_size) { + error_log("%s: Overflow constructing heapprofd property", getprogname()); + return false; + } + memcpy(data, prefix, prefix_size); + + int fd = open("/proc/self/cmdline", O_RDONLY | O_CLOEXEC); + if (fd == -1) { + error_log("%s: Failed to open /proc/self/cmdline", getprogname()); + return false; + } + char cmdline[128]; + ssize_t rd = read(fd, cmdline, sizeof(cmdline) - 1); + close(fd); + if (rd == -1) { + error_log("%s: Failed to read /proc/self/cmdline", getprogname()); + return false; + } + cmdline[rd] = '\0'; + char* first_arg = static_cast<char*>(memchr(cmdline, '\0', rd)); + if (first_arg == nullptr || first_arg == cmdline + size - 1) { + error_log("%s: Overflow reading cmdline", getprogname()); + return false; + } + // For consistency with what we do with Java app cmdlines, trim everything + // after the @ sign of the first arg. + char* first_at = static_cast<char*>(memchr(cmdline, '@', rd)); + if (first_at != nullptr && first_at < first_arg) { + *first_at = '\0'; + first_arg = first_at; + } + + char* start = static_cast<char*>(memrchr(cmdline, '/', first_arg - cmdline)); + if (start == first_arg) { + // The first argument ended in a slash. + error_log("%s: cmdline ends in /", getprogname()); + return false; + } else if (start == nullptr) { + start = cmdline; + } else { + // Skip the /. + start++; + } + + size_t name_size = static_cast<size_t>(first_arg - start); + if (name_size >= size - prefix_size) { + error_log("%s: overflow constructing heapprofd property.", getprogname()); + return false; + } + // + 1 to also copy the trailing null byte. + memcpy(data + prefix_size, start, name_size + 1); + return true; +} + +bool HeapprofdShouldLoad() { + // First check for heapprofd.enable. If it is set to "all", enable + // heapprofd for all processes. Otherwise, check heapprofd.enable.${prog}, + // if it is set and not 0, enable heap profiling for this process. + char property_value[PROP_VALUE_MAX]; + if (__system_property_get(kHeapprofdPropertyEnable, property_value) == 0) { + return false; + } + if (strcmp(property_value, "all") == 0) { + return true; + } + + char program_property[128]; + if (!GetHeapprofdProgramProperty(program_property, + sizeof(program_property))) { + return false; + } + if (__system_property_get(program_property, property_value) == 0) { + return false; + } + return program_property[0] != '\0'; +} + +void HeapprofdInstallSignalHandler() { + struct sigaction action = {}; + action.sa_handler = MaybeInstallInitHeapprofdHook; + sigaction(kHeapprofdSignal, &action, nullptr); +} + +static void CommonInstallHooks(libc_globals* globals) { + void* impl_handle = atomic_load(&gHeapprofdHandle); + bool reusing_handle = impl_handle != nullptr; + if (!reusing_handle) { + impl_handle = LoadSharedLibrary(kHeapprofdSharedLib, kHeapprofdPrefix, &globals->malloc_dispatch_table); + if (impl_handle == nullptr) { + return; + } + } else if (!InitSharedLibrary(impl_handle, kHeapprofdSharedLib, kHeapprofdPrefix, &globals->malloc_dispatch_table)) { + return; + } + + if (FinishInstallHooks(globals, nullptr, kHeapprofdPrefix)) { + atomic_store(&gHeapprofdHandle, impl_handle); + } else if (!reusing_handle) { + dlclose(impl_handle); + } + + atomic_store(&gHeapprofdInitInProgress, false); +} + +void HeapprofdInstallHooksAtInit(libc_globals* globals) { + if (atomic_exchange(&gHeapprofdInitInProgress, true)) { + return; + } + CommonInstallHooks(globals); +} + +static void* InitHeapprofd(void*) { + __libc_globals.mutate([](libc_globals* globals) { + CommonInstallHooks(globals); + }); + + // Allow to install hook again to re-initialize heap profiling after the + // current session finished. + atomic_store(&gHeapprofdInitHookInstalled, false); + return nullptr; +} + +extern "C" void* MallocInitHeapprofdHook(size_t bytes) { + if (!atomic_exchange(&gHeapprofdInitHookInstalled, true)) { + __libc_globals.mutate([](libc_globals* globals) { + atomic_store(&globals->current_dispatch_table, nullptr); + }); + + pthread_t thread_id; + if (pthread_create(&thread_id, nullptr, InitHeapprofd, nullptr) != 0) { + error_log("%s: heapprofd: failed to pthread_create.", getprogname()); + } else if (pthread_detach(thread_id) != 0) { + error_log("%s: heapprofd: failed to pthread_detach", getprogname()); + } + if (pthread_setname_np(thread_id, "heapprofdinit") != 0) { + error_log("%s: heapprod: failed to pthread_setname_np", getprogname()); + } + } + return Malloc(malloc)(bytes); +} + +// Marks this process as a profileable zygote child. +static bool HandleInitZygoteChildProfiling() { + atomic_store_explicit(&gMallocZygoteChildProfileable, true, memory_order_release); + + // Conditionally start "from startup" profiling. + if (HeapprofdShouldLoad()) { + // Directly call the signal handler (will correctly guard against + // concurrent signal delivery). + MaybeInstallInitHeapprofdHook(kHeapprofdSignal); + } + return true; +} + +static bool DispatchReset() { + if (!atomic_exchange(&gHeapprofdInitInProgress, true)) { + __libc_globals.mutate([](libc_globals* globals) { + atomic_store(&globals->current_dispatch_table, nullptr); + }); + atomic_store(&gHeapprofdInitInProgress, false); + return true; + } + errno = EAGAIN; + return false; +} + +bool HeapprofdMallopt(int opcode, void* arg, size_t arg_size) { + if (opcode == M_INIT_ZYGOTE_CHILD_PROFILING) { + if (arg != nullptr || arg_size != 0) { + errno = EINVAL; + return false; + } + return HandleInitZygoteChildProfiling(); + } + if (opcode == M_RESET_HOOKS) { + if (arg != nullptr || arg_size != 0) { + errno = EINVAL; + return false; + } + return DispatchReset(); + } + errno = ENOTSUP; + return false; +} diff --git a/libc/bionic/malloc_heapprofd.h b/libc/bionic/malloc_heapprofd.h new file mode 100644 index 000000000..91188b920 --- /dev/null +++ b/libc/bionic/malloc_heapprofd.h @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#pragma once + +#include <stdint.h> + +#include <private/bionic_globals.h> + +bool HeapprofdShouldLoad(); + +void HeapprofdInstallHooksAtInit(libc_globals* globals); + +void HeapprofdInstallSignalHandler(); + +bool HeapprofdMallopt(int optcode, void* arg, size_t arg_size); diff --git a/libc/symbol_ordering b/libc/symbol_ordering index 8b2d153a5..c39fac55c 100644 --- a/libc/symbol_ordering +++ b/libc/symbol_ordering @@ -6,8 +6,8 @@ je_background_thread_enabled_state je_can_enable_background_thread _ZZ17__find_icu_symbolPKcE9found_icu -_ZL28g_heapprofd_init_in_progress -_ZL31g_heapprofd_init_hook_installed +_ZL24gHeapprofdInitInProgress +_ZL27gHeapprofdInitHookInstalled je_opt_abort je_opt_abort_conf je_opt_junk_alloc @@ -174,7 +174,7 @@ random_mutex __res_randomid.__libc_mutex_random locallock g_atexit_lock -_ZL11g_functions +_ZL10gFunctions _ZL13vendor_passwd _ZL12vendor_group tm diff --git a/tests/malloc_test.cpp b/tests/malloc_test.cpp index 1431cc134..658f8bdc9 100644 --- a/tests/malloc_test.cpp +++ b/tests/malloc_test.cpp @@ -16,6 +16,7 @@ #include <gtest/gtest.h> +#include <elf.h> #include <limits.h> #include <stdint.h> #include <stdlib.h> @@ -24,6 +25,8 @@ #include <tinyxml2.h> +#include <android-base/file.h> + #include "private/bionic_config.h" #include "private/bionic_malloc.h" #include "utils.h" @@ -620,20 +623,48 @@ TEST(android_mallopt, error_on_unexpected_option) { #endif } +bool IsDynamic() { +#if defined(__LP64__) + Elf64_Ehdr ehdr; +#else + Elf32_Ehdr ehdr; +#endif + std::string path(android::base::GetExecutablePath()); + + int fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (fd == -1) { + // Assume dynamic on error. + return true; + } + bool read_completed = android::base::ReadFully(fd, &ehdr, sizeof(ehdr)); + close(fd); + // Assume dynamic in error cases. + return !read_completed || ehdr.e_type == ET_DYN; +} + TEST(android_mallopt, init_zygote_child_profiling) { #if defined(__BIONIC__) // Successful call. errno = 0; - EXPECT_EQ(true, android_mallopt(M_INIT_ZYGOTE_CHILD_PROFILING, nullptr, 0)); - EXPECT_EQ(0, errno); + if (IsDynamic()) { + EXPECT_EQ(true, android_mallopt(M_INIT_ZYGOTE_CHILD_PROFILING, nullptr, 0)); + EXPECT_EQ(0, errno); + } else { + // Not supported in static executables. + EXPECT_EQ(false, android_mallopt(M_INIT_ZYGOTE_CHILD_PROFILING, nullptr, 0)); + EXPECT_EQ(ENOTSUP, errno); + } // Unexpected arguments rejected. errno = 0; char unexpected = 0; EXPECT_EQ(false, android_mallopt(M_INIT_ZYGOTE_CHILD_PROFILING, &unexpected, 1)); - EXPECT_EQ(EINVAL, errno); + if (IsDynamic()) { + EXPECT_EQ(EINVAL, errno); + } else { + EXPECT_EQ(ENOTSUP, errno); + } #else GTEST_LOG_(INFO) << "This tests a bionic implementation detail.\n"; #endif } - |