Add a platform API for setting an allocation limit.

Introduce an M_SET_ALLOCATION_LIMIT enumerator for android_mallopt(),
which can be used to set an upper bound on the total size of all
allocations made using the memory allocation APIs.

This is useful for programs such as audioextractor and mediaserver
which need to set such a limit as a security mitigation. Currently
these programs are using setrlimit(RLIMIT_AS) which isn't exactly
what these programs want to control. RLIMIT_AS is also problematic
under sanitizers which allocate large amounts of address space as
shadow memory, and is especially problematic under shadow call stack,
which requires 16MB of address space per thread.

Add new unit tests for bionic.

Add new unit tests for malloc debug that verify that when the limit
is enabled, malloc debug still functions for nearly every allocation
function.

Bug: 118642754
Test: Ran bionic-unit-tests/bionic-unit-tests-static.
Test: Ran malloc debug tests and perfetto integration tests.
Change-Id: I735403c4d2c87f00fb2cdef81d00af0af446b2bb

1 files changed, 213 insertions, 8 deletions

diff --git a/libc/malloc_debug/tests/malloc_debug_system_tests.cpp b/libc/malloc_debug/tests/malloc_debug_system_tests.cpp
index ccefb25d2..4fcd04c81 100644
--- a/libc/malloc_debug/tests/malloc_debug_system_tests.cpp
+++ b/libc/malloc_debug/tests/malloc_debug_system_tests.cpp
@@ -37,13 +37,15 @@
 #include <time.h>
 #include <unistd.h>
 
+#include <android-base/stringprintf.h>
 #include <gtest/gtest.h>
-
 #include <log/log.h>
 
 #include <string>
 #include <vector>
 
+#include "private/bionic_malloc.h"
+
 static constexpr time_t kTimeoutSeconds = 5;
 
 static void Exec(const char* test_name, const char* debug_options, pid_t* pid) {
@@ -60,13 +62,15 @@ static void Exec(const char* test_name, const char* debug_options, pid_t* pid) {
     ASSERT_NE(0, dup2(fds[1], STDERR_FILENO));
 
     std::vector<const char*> args;
-    args.push_back(testing::internal::GetArgvs()[0].c_str());
+    // Get a copy of this argument so it doesn't disappear on us.
+    std::string exec(testing::internal::GetArgvs()[0]);
+    args.push_back(exec.c_str());
     args.push_back("--gtest_also_run_disabled_tests");
     std::string filter_arg = std::string("--gtest_filter=") + test_name;
     args.push_back(filter_arg.c_str());
     args.push_back(nullptr);
     execv(args[0], reinterpret_cast<char* const*>(const_cast<char**>(args.data())));
-    exit(1);
+    exit(20);
   }
   ASSERT_NE(-1, *pid);
   close(fds[1]);
@@ -196,16 +200,217 @@ TEST(MallocDebugSystemTest, smoke) {
   ASSERT_NO_FATAL_FAILURE(FindStrings(pid, std::vector<const char*>{"malloc debug enabled"}));
 }
 
-TEST(MallocTests, DISABLED_leak_memory) {
+static void SetAllocationLimit() {
+  // Set to a large value, this is only to enable the limit code and
+  // verify that malloc debug is still called properly.
+  size_t limit = 500 * 1024 * 1024;
+  ASSERT_TRUE(android_mallopt(M_SET_ALLOCATION_LIMIT_BYTES, &limit, sizeof(limit)));
+}
+
+static void AlignedAlloc() {
+  void* ptr = aligned_alloc(64, 1152);
+  ASSERT_TRUE(ptr != nullptr);
+  memset(ptr, 0, 1152);
+}
+
+TEST(MallocTests, DISABLED_leak_memory_aligned_alloc) {
+  AlignedAlloc();
+}
+
+TEST(MallocTests, DISABLED_leak_memory_limit_aligned_alloc) {
+  SetAllocationLimit();
+  AlignedAlloc();
+}
+
+static void Calloc() {
+  void* ptr = calloc(1, 1123);
+  ASSERT_TRUE(ptr != nullptr);
+  memset(ptr, 1, 1123);
+}
+
+TEST(MallocTests, DISABLED_leak_memory_calloc) {
+  Calloc();
+}
+
+TEST(MallocTests, DISABLED_leak_memory_limit_calloc) {
+  SetAllocationLimit();
+  Calloc();
+}
+
+static void Malloc() {
   void* ptr = malloc(1123);
   ASSERT_TRUE(ptr != nullptr);
   memset(ptr, 0, 1123);
 }
 
+TEST(MallocTests, DISABLED_leak_memory_malloc) {
+  Malloc();
+}
+
+TEST(MallocTests, DISABLED_leak_memory_limit_malloc) {
+  SetAllocationLimit();
+  Malloc();
+}
+
+static void Memalign() {
+  void* ptr = memalign(64, 1123);
+  ASSERT_TRUE(ptr != nullptr);
+  memset(ptr, 0, 1123);
+}
+
+TEST(MallocTests, DISABLED_leak_memory_memalign) {
+  Memalign();
+}
+
+TEST(MallocTests, DISABLED_leak_memory_limit_memalign) {
+  SetAllocationLimit();
+  Memalign();
+}
+
+static void PosixMemalign() {
+  void* ptr;
+  ASSERT_EQ(0, posix_memalign(&ptr, 64, 1123));
+  ASSERT_TRUE(ptr != nullptr);
+  memset(ptr, 0, 1123);
+}
+
+TEST(MallocTests, DISABLED_leak_memory_posix_memalign) {
+  PosixMemalign();
+}
+
+TEST(MallocTests, DISABLED_leak_memory_limit_posix_memalign) {
+  SetAllocationLimit();
+  PosixMemalign();
+}
+
+static void Reallocarray() {
+  void* ptr = reallocarray(nullptr, 1, 1123);
+  ASSERT_TRUE(ptr != nullptr);
+  memset(ptr, 0, 1123);
+}
+
+TEST(MallocTests, DISABLED_leak_memory_reallocarray) {
+  Reallocarray();
+}
+
+TEST(MallocTests, DISABLED_leak_memory_limit_reallocarray) {
+  SetAllocationLimit();
+  Reallocarray();
+}
+
+static void Realloc() {
+  void* ptr = realloc(nullptr, 1123);
+  ASSERT_TRUE(ptr != nullptr);
+  memset(ptr, 0, 1123);
+}
+
+TEST(MallocTests, DISABLED_leak_memory_realloc) {
+  Realloc();
+}
+
+TEST(MallocTests, DISABLED_leak_memory_limit_realloc) {
+  SetAllocationLimit();
+  Realloc();
+}
+
+#if !defined(__LP64__)
+extern "C" void* pvalloc(size_t);
+
+static void Pvalloc() {
+  void* ptr = pvalloc(1123);
+  ASSERT_TRUE(ptr != nullptr);
+  memset(ptr, 0, 1123);
+}
+
+TEST(MallocTests, DISABLED_leak_memory_pvalloc) {
+  Pvalloc();
+}
+
+TEST(MallocTests, DISABLED_leak_memory_limit_pvalloc) {
+  SetAllocationLimit();
+  Pvalloc();
+}
+
+extern "C" void* valloc(size_t);
+
+static void Valloc() {
+  void* ptr = valloc(1123);
+  ASSERT_TRUE(ptr != nullptr);
+  memset(ptr, 0, 1123);
+}
+
+TEST(MallocTests, DISABLED_leak_memory_valloc) {
+  Valloc();
+}
+
+TEST(MallocTests, DISABLED_leak_memory_limit_valloc) {
+  SetAllocationLimit();
+  Valloc();
+}
+#endif
+
+static void VerifyLeak(const char* test_prefix) {
+  struct FunctionInfo {
+    const char* name;
+    size_t size;
+  };
+  static FunctionInfo functions[] = {
+    {
+      "aligned_alloc",
+      1152,
+    },
+    {
+      "calloc",
+      1123,
+    },
+    {
+      "malloc",
+      1123,
+    },
+    {
+      "memalign",
+      1123,
+    },
+    {
+      "posix_memalign",
+      1123,
+    },
+    {
+      "reallocarray",
+      1123,
+    },
+    {
+      "realloc",
+      1123,
+    },
+#if !defined(__LP64__)
+    {
+      "pvalloc",
+      4096,
+    },
+    {
+      "valloc",
+      1123,
+    }
+#endif
+  };
+
+  for (size_t i = 0; i < sizeof(functions) / sizeof(FunctionInfo); i++) {
+    pid_t pid;
+    SCOPED_TRACE(testing::Message() << functions[i].name << " expected size " << functions[i].size);
+    std::string test = std::string("MallocTests.DISABLED_") + test_prefix + functions[i].name;
+    EXPECT_NO_FATAL_FAILURE(Exec(test.c_str(), "backtrace leak_track", &pid));
+
+    std::string expected_leak = android::base::StringPrintf("leaked block of size %zu at", functions[i].size);
+    EXPECT_NO_FATAL_FAILURE(FindStrings(
+        pid, std::vector<const char*>{"malloc debug enabled", expected_leak.c_str()}));
+  }
+}
+
 TEST(MallocDebugSystemTest, verify_leak) {
-  pid_t pid;
-  ASSERT_NO_FATAL_FAILURE(Exec("MallocTests.DISABLED_leak_memory", "backtrace leak_track", &pid));
+  VerifyLeak("leak_memory_");
+}
 
-  ASSERT_NO_FATAL_FAILURE(FindStrings(
-      pid, std::vector<const char*>{"malloc debug enabled", "leaked block of size 1123 at"}));
+TEST(MallocDebugSystemTest, verify_leak_allocation_limit) {
+  VerifyLeak("leak_memory_limit_");
 }