debuggerd_fallback: don't recursively abort.

Calls to abort() will always result in our signal handler being called,
because abort will manually unblock SIGABRT before raising it. This
can lead to deadlock when handling address space exhaustion in the
fallback handler. To fix this, switch our mutex to a recursive mutex,
and manually keep track of our lock count.

Bug: http://b/72929749
Test: debuggerd_test --gtest_filter="CrasherTest.seccomp_crash_oom"
Change-Id: I609f263ce93550350b17757189326b627129d4a7

1 files changed, 50 insertions, 1 deletions

diff --git a/debuggerd/debuggerd_test.cpp b/debuggerd/debuggerd_test.cpp
index f8b4bad6e..397ff2f11 100644
--- a/debuggerd/debuggerd_test.cpp
+++ b/debuggerd/debuggerd_test.cpp
@@ -20,6 +20,7 @@
 #include <sys/capability.h>
 #include <sys/prctl.h>
 #include <sys/ptrace.h>
+#include <sys/resource.h>
 #include <sys/syscall.h>
 #include <sys/types.h>
 #include <unistd.h>
@@ -570,7 +571,7 @@ TEST_F(CrasherTest, fake_pid) {
 static const char* const kDebuggerdSeccompPolicy =
     "/system/etc/seccomp_policy/crash_dump." ABI_STRING ".policy";
 
-pid_t seccomp_fork() {
+static pid_t seccomp_fork_impl(void (*prejail)()) {
   unique_fd policy_fd(open(kDebuggerdSeccompPolicy, O_RDONLY | O_CLOEXEC));
   if (policy_fd == -1) {
     LOG(FATAL) << "failed to open policy " << kDebuggerdSeccompPolicy;
@@ -607,10 +608,18 @@ pid_t seccomp_fork() {
     continue;
   }
 
+  if (prejail) {
+    prejail();
+  }
+
   minijail_enter(jail.get());
   return result;
 }
 
+static pid_t seccomp_fork() {
+  return seccomp_fork_impl(nullptr);
+}
+
 TEST_F(CrasherTest, seccomp_crash) {
   int intercept_result;
   unique_fd output_fd;
@@ -628,6 +637,46 @@ TEST_F(CrasherTest, seccomp_crash) {
   ASSERT_BACKTRACE_FRAME(result, "abort");
 }
 
+static pid_t seccomp_fork_rlimit() {
+  return seccomp_fork_impl([]() {
+    struct rlimit rlim = {
+        .rlim_cur = 512 * 1024 * 1024,
+        .rlim_max = 512 * 1024 * 1024,
+    };
+
+    if (setrlimit(RLIMIT_AS, &rlim) != 0) {
+      raise(SIGINT);
+    }
+  });
+}
+
+TEST_F(CrasherTest, seccomp_crash_oom) {
+  int intercept_result;
+  unique_fd output_fd;
+
+  StartProcess(
+      []() {
+        std::vector<void*> vec;
+        for (int i = 0; i < 512; ++i) {
+          char* buf = static_cast<char*>(malloc(1024 * 1024));
+          if (!buf) {
+            abort();
+          }
+          memset(buf, 0xff, 1024 * 1024);
+          vec.push_back(buf);
+        }
+      },
+      &seccomp_fork_rlimit);
+
+  StartIntercept(&output_fd);
+  FinishCrasher();
+  AssertDeath(SIGABRT);
+  FinishIntercept(&intercept_result);
+  ASSERT_EQ(1, intercept_result) << "tombstoned reported failure";
+
+  // We can't actually generate a backtrace, just make sure that the process terminates.
+}
+
 __attribute__((noinline)) extern "C" bool raise_debugger_signal(DebuggerdDumpType dump_type) {
   siginfo_t siginfo;
   siginfo.si_code = SI_QUEUE;