Merge "Add zygote native fork loop"

5 files changed, 791 insertions, 173 deletions

diff --git a/core/jni/Android.bp b/core/jni/Android.bp
index 0d05a6b82d36..e58ad79de89e 100644
--- a/core/jni/Android.bp
+++ b/core/jni/Android.bp
@@ -206,6 +206,7 @@ cc_library_shared {
                 "com_android_internal_os_KernelCpuUidBpfMapReader.cpp",
                 "com_android_internal_os_KernelSingleUidTimeReader.cpp",
                 "com_android_internal_os_Zygote.cpp",
+                "com_android_internal_os_ZygoteCommandBuffer.cpp",
                 "com_android_internal_os_ZygoteInit.cpp",
                 "hwbinder/EphemeralStorage.cpp",
                 "fd_utils.cpp",
diff --git a/core/jni/AndroidRuntime.cpp b/core/jni/AndroidRuntime.cpp
index 3198cb1b8140..c01f741862b1 100644
--- a/core/jni/AndroidRuntime.cpp
+++ b/core/jni/AndroidRuntime.cpp
@@ -193,6 +193,7 @@ extern int register_com_android_internal_os_FuseAppLoop(JNIEnv* env);
 extern int register_com_android_internal_os_KernelCpuUidBpfMapReader(JNIEnv *env);
 extern int register_com_android_internal_os_KernelSingleUidTimeReader(JNIEnv *env);
 extern int register_com_android_internal_os_Zygote(JNIEnv *env);
+extern int register_com_android_internal_os_ZygoteCommandBuffer(JNIEnv *env);
 extern int register_com_android_internal_os_ZygoteInit(JNIEnv *env);
 extern int register_com_android_internal_util_VirtualRefBasePtr(JNIEnv *env);
 
@@ -1524,6 +1525,7 @@ static const RegJNIRec gRegJNI[] = {
         REG_JNI(register_com_android_internal_net_NetworkUtilsInternal),
         REG_JNI(register_com_android_internal_os_ClassLoaderFactory),
         REG_JNI(register_com_android_internal_os_Zygote),
+        REG_JNI(register_com_android_internal_os_ZygoteCommandBuffer),
         REG_JNI(register_com_android_internal_os_ZygoteInit),
         REG_JNI(register_com_android_internal_util_VirtualRefBasePtr),
         REG_JNI(register_android_hardware_Camera),
diff --git a/core/jni/com_android_internal_os_Zygote.cpp b/core/jni/com_android_internal_os_Zygote.cpp
index 903ecaef4938..4cef2b099589 100644
--- a/core/jni/com_android_internal_os_Zygote.cpp
+++ b/core/jni/com_android_internal_os_Zygote.cpp
@@ -17,6 +17,8 @@
 #define LOG_TAG "Zygote"
 #define ATRACE_TAG ATRACE_TAG_DALVIK
 
+#include "com_android_internal_os_Zygote.h"
+
 #include <async_safe/log.h>
 
 // sys/mount.h has to come before linux/fs.h due to redefinition of MS_RDONLY, MS_BIND, etc
@@ -91,19 +93,6 @@
 
 #include "nativebridge/native_bridge.h"
 
-/* Functions in the callchain during the fork shall not be protected with
-   Armv8.3-A Pointer Authentication, otherwise child will not be able to return. */
-#ifdef __ARM_FEATURE_PAC_DEFAULT
-#ifdef __ARM_FEATURE_BTI_DEFAULT
-#define NO_PAC_FUNC __attribute__((target("branch-protection=bti")))
-#else
-#define NO_PAC_FUNC __attribute__((target("branch-protection=none")))
-#endif /* __ARM_FEATURE_BTI_DEFAULT */
-#else /* !__ARM_FEATURE_PAC_DEFAULT */
-#define NO_PAC_FUNC
-#endif /* __ARM_FEATURE_PAC_DEFAULT */
-
-
 namespace {
 
 // TODO (chriswailes): Add a function to initialize native Zygote data.
@@ -118,8 +107,7 @@ using android::base::StringPrintf;
 using android::base::WriteStringToFile;
 using android::base::GetBoolProperty;
 
-#define CREATE_ERROR(...) StringPrintf("%s:%d: ", __FILE__, __LINE__). \
-                              append(StringPrintf(__VA_ARGS__))
+using android::zygote::ZygoteFailure;
 
 // This type is duplicated in fd_utils.h
 typedef const std::function<void(std::string)>& fail_fn_t;
@@ -215,7 +203,7 @@ class UsapTableEntry {
   static constexpr EntryStorage INVALID_ENTRY_VALUE = {-1, -1};
 
   std::atomic<EntryStorage> mStorage;
-  static_assert(decltype(mStorage)::is_always_lock_free);
+  static_assert(decltype(mStorage)::is_always_lock_free);  // Accessed from signal handler.
 
  public:
   constexpr UsapTableEntry() : mStorage(INVALID_ENTRY_VALUE) {}
@@ -942,36 +930,6 @@ void SetThreadName(const std::string& thread_name) {
 }
 
 /**
- * A failure function used to report fatal errors to the managed runtime.  This
- * function is often curried with the process name information and then passed
- * to called functions.
- *
- * @param env  Managed runtime environment
- * @param process_name  A native representation of the process name
- * @param managed_process_name  A managed representation of the process name
- * @param msg  The error message to be reported
- */
-[[noreturn]]
-static void ZygoteFailure(JNIEnv* env,
-                          const char* process_name,
-                          jstring managed_process_name,
-                          const std::string& msg) {
-  std::unique_ptr<ScopedUtfChars> scoped_managed_process_name_ptr = nullptr;
-  if (managed_process_name != nullptr) {
-    scoped_managed_process_name_ptr.reset(new ScopedUtfChars(env, managed_process_name));
-    if (scoped_managed_process_name_ptr->c_str() != nullptr) {
-      process_name = scoped_managed_process_name_ptr->c_str();
-    }
-  }
-
-  const std::string& error_msg =
-      (process_name == nullptr) ? msg : StringPrintf("(%s) %s", process_name, msg.c_str());
-
-  env->FatalError(error_msg.c_str());
-  __builtin_unreachable();
-}
-
-/**
  * A helper method for converting managed strings to native strings.  A fatal
  * error is generated if a problem is encountered in extracting a non-null
  * string.
@@ -1098,86 +1056,6 @@ static void PAuthKeyChange(JNIEnv* env) {
 #endif
 }
 
-// Utility routine to fork a process from the zygote.
-NO_PAC_FUNC
-static pid_t ForkCommon(JNIEnv* env, bool is_system_server,
-                        const std::vector<int>& fds_to_close,
-                        const std::vector<int>& fds_to_ignore,
-                        bool is_priority_fork) {
-  SetSignalHandlers();
-
-  // Curry a failure function.
-  auto fail_fn = std::bind(ZygoteFailure, env, is_system_server ? "system_server" : "zygote",
-                           nullptr, _1);
-
-  // Temporarily block SIGCHLD during forks. The SIGCHLD handler might
-  // log, which would result in the logging FDs we close being reopened.
-  // This would cause failures because the FDs are not allowlisted.
-  //
-  // Note that the zygote process is single threaded at this point.
-  BlockSignal(SIGCHLD, fail_fn);
-
-  // Close any logging related FDs before we start evaluating the list of
-  // file descriptors.
-  __android_log_close();
-  AStatsSocket_close();
-
-  // If this is the first fork for this zygote, create the open FD table.  If
-  // it isn't, we just need to check whether the list of open files has changed
-  // (and it shouldn't in the normal case).
-  if (gOpenFdTable == nullptr) {
-    gOpenFdTable = FileDescriptorTable::Create(fds_to_ignore, fail_fn);
-  } else {
-    gOpenFdTable->Restat(fds_to_ignore, fail_fn);
-  }
-
-  android_fdsan_error_level fdsan_error_level = android_fdsan_get_error_level();
-
-  // Purge unused native memory in an attempt to reduce the amount of false
-  // sharing with the child process.  By reducing the size of the libc_malloc
-  // region shared with the child process we reduce the number of pages that
-  // transition to the private-dirty state when malloc adjusts the meta-data
-  // on each of the pages it is managing after the fork.
-  mallopt(M_PURGE, 0);
-
-  pid_t pid = fork();
-
-  if (pid == 0) {
-    if (is_priority_fork) {
-      setpriority(PRIO_PROCESS, 0, PROCESS_PRIORITY_MAX);
-    } else {
-      setpriority(PRIO_PROCESS, 0, PROCESS_PRIORITY_MIN);
-    }
-
-    // The child process.
-    PAuthKeyChange(env);
-    PreApplicationInit();
-
-    // Clean up any descriptors which must be closed immediately
-    DetachDescriptors(env, fds_to_close, fail_fn);
-
-    // Invalidate the entries in the USAP table.
-    ClearUsapTable();
-
-    // Re-open all remaining open file descriptors so that they aren't shared
-    // with the zygote across a fork.
-    gOpenFdTable->ReopenOrDetach(fail_fn);
-
-    // Turn fdsan back on.
-    android_fdsan_set_error_level(fdsan_error_level);
-
-    // Reset the fd to the unsolicited zygote socket
-    gSystemServerSocketFd = -1;
-  } else {
-    ALOGD("Forked child process %d", pid);
-  }
-
-  // We blocked SIGCHLD prior to a fork, we unblock it here.
-  UnblockSignal(SIGCHLD, fail_fn);
-
-  return pid;
-}
-
 // Create an app data directory over tmpfs overlayed CE / DE storage, and bind mount it
 // from the actual app data directory in data mirror.
 static bool createAndMountAppData(std::string_view package_name,
@@ -1993,9 +1871,10 @@ static void AddUsapTableEntry(pid_t usap_pid, int read_pipe_fd) {
   static int sUsapTableInsertIndex = 0;
 
   int search_index = sUsapTableInsertIndex;
-
   do {
     if (gUsapTable[search_index].SetIfInvalid(usap_pid, read_pipe_fd)) {
+      ++gUsapPoolCount;
+
       // Start our next search right after where we finished this one.
       sUsapTableInsertIndex = (search_index + 1) % gUsapTable.size();
 
@@ -2013,7 +1892,7 @@ static void AddUsapTableEntry(pid_t usap_pid, int read_pipe_fd) {
 /**
  * Invalidates the entry in the USAPTable corresponding to the provided
  * process ID if it is present.  If an entry was removed the USAP pool
- * count is decremented.
+ * count is decremented. May be called from signal handler.
  *
  * @param usap_pid  Process ID of the USAP entry to invalidate
  * @return True if an entry was invalidated; false otherwise
@@ -2089,6 +1968,121 @@ static void UnmountStorageOnInit(JNIEnv* env) {
 
 namespace android {
 
+/**
+ * A failure function used to report fatal errors to the managed runtime.  This
+ * function is often curried with the process name information and then passed
+ * to called functions.
+ *
+ * @param env  Managed runtime environment
+ * @param process_name  A native representation of the process name
+ * @param managed_process_name  A managed representation of the process name
+ * @param msg  The error message to be reported
+ */
+[[noreturn]]
+void zygote::ZygoteFailure(JNIEnv* env,
+                           const char* process_name,
+                           jstring managed_process_name,
+                           const std::string& msg) {
+  std::unique_ptr<ScopedUtfChars> scoped_managed_process_name_ptr = nullptr;
+  if (managed_process_name != nullptr) {
+    scoped_managed_process_name_ptr.reset(new ScopedUtfChars(env, managed_process_name));
+    if (scoped_managed_process_name_ptr->c_str() != nullptr) {
+      process_name = scoped_managed_process_name_ptr->c_str();
+    }
+  }
+
+  const std::string& error_msg =
+      (process_name == nullptr || process_name[0] == '\0') ?
+      msg : StringPrintf("(%s) %s", process_name, msg.c_str());
+
+  env->FatalError(error_msg.c_str());
+  __builtin_unreachable();
+}
+
+// Utility routine to fork a process from the zygote.
+NO_PAC_FUNC
+pid_t zygote::ForkCommon(JNIEnv* env, bool is_system_server,
+                         const std::vector<int>& fds_to_close,
+                         const std::vector<int>& fds_to_ignore,
+                         bool is_priority_fork,
+                         bool purge) {
+  SetSignalHandlers();
+
+  // Curry a failure function.
+  auto fail_fn = std::bind(zygote::ZygoteFailure, env,
+                           is_system_server ? "system_server" : "zygote",
+                           nullptr, _1);
+
+  // Temporarily block SIGCHLD during forks. The SIGCHLD handler might
+  // log, which would result in the logging FDs we close being reopened.
+  // This would cause failures because the FDs are not allowlisted.
+  //
+  // Note that the zygote process is single threaded at this point.
+  BlockSignal(SIGCHLD, fail_fn);
+
+  // Close any logging related FDs before we start evaluating the list of
+  // file descriptors.
+  __android_log_close();
+  AStatsSocket_close();
+
+  // If this is the first fork for this zygote, create the open FD table.  If
+  // it isn't, we just need to check whether the list of open files has changed
+  // (and it shouldn't in the normal case).
+  if (gOpenFdTable == nullptr) {
+    gOpenFdTable = FileDescriptorTable::Create(fds_to_ignore, fail_fn);
+  } else {
+    gOpenFdTable->Restat(fds_to_ignore, fail_fn);
+  }
+
+  android_fdsan_error_level fdsan_error_level = android_fdsan_get_error_level();
+
+  if (purge) {
+    // Purge unused native memory in an attempt to reduce the amount of false
+    // sharing with the child process.  By reducing the size of the libc_malloc
+    // region shared with the child process we reduce the number of pages that
+    // transition to the private-dirty state when malloc adjusts the meta-data
+    // on each of the pages it is managing after the fork.
+    mallopt(M_PURGE, 0);
+  }
+
+  pid_t pid = fork();
+
+  if (pid == 0) {
+    if (is_priority_fork) {
+      setpriority(PRIO_PROCESS, 0, PROCESS_PRIORITY_MAX);
+    } else {
+      setpriority(PRIO_PROCESS, 0, PROCESS_PRIORITY_MIN);
+    }
+
+    // The child process.
+    PAuthKeyChange(env);
+    PreApplicationInit();
+
+    // Clean up any descriptors which must be closed immediately
+    DetachDescriptors(env, fds_to_close, fail_fn);
+
+    // Invalidate the entries in the USAP table.
+    ClearUsapTable();
+
+    // Re-open all remaining open file descriptors so that they aren't shared
+    // with the zygote across a fork.
+    gOpenFdTable->ReopenOrDetach(fail_fn);
+
+    // Turn fdsan back on.
+    android_fdsan_set_error_level(fdsan_error_level);
+
+    // Reset the fd to the unsolicited zygote socket
+    gSystemServerSocketFd = -1;
+  } else {
+    ALOGD("Forked child process %d", pid);
+  }
+
+  // We blocked SIGCHLD prior to a fork, we unblock it here.
+  UnblockSignal(SIGCHLD, fail_fn);
+
+  return pid;
+}
+
 static void com_android_internal_os_Zygote_nativePreApplicationInit(JNIEnv*, jclass) {
   PreApplicationInit();
 }
@@ -2105,7 +2099,8 @@ static jint com_android_internal_os_Zygote_nativeForkAndSpecialize(
     jlong capabilities = CalculateCapabilities(env, uid, gid, gids, is_child_zygote);
 
     if (UNLIKELY(managed_fds_to_close == nullptr)) {
-      ZygoteFailure(env, "zygote", nice_name, "Zygote received a null fds_to_close vector.");
+      zygote::ZygoteFailure(env, "zygote", nice_name,
+                            "Zygote received a null fds_to_close vector.");
     }
 
     std::vector<int> fds_to_close =
@@ -2131,7 +2126,7 @@ static jint com_android_internal_os_Zygote_nativeForkAndSpecialize(
         fds_to_ignore.push_back(gSystemServerSocketFd);
     }
 
-    pid_t pid = ForkCommon(env, false, fds_to_close, fds_to_ignore, true);
+    pid_t pid = zygote::ForkCommon(env, false, fds_to_close, fds_to_ignore, true);
 
     if (pid == 0) {
       SpecializeCommon(env, uid, gid, gids, runtime_flags, rlimits,
@@ -2166,10 +2161,10 @@ static jint com_android_internal_os_Zygote_nativeForkSystemServer(
       fds_to_ignore.push_back(gSystemServerSocketFd);
   }
 
-  pid_t pid = ForkCommon(env, true,
-                         fds_to_close,
-                         fds_to_ignore,
-                         true);
+  pid_t pid = zygote::ForkCommon(env, true,
+                                 fds_to_close,
+                                 fds_to_ignore,
+                                 true);
   if (pid == 0) {
       // System server prcoess does not need data isolation so no need to
       // know pkg_data_info_list.
@@ -2209,58 +2204,74 @@ static jint com_android_internal_os_Zygote_nativeForkSystemServer(
  * ensuring proper file descriptor hygiene.
  *
  * @param env  Managed runtime environment
- * @param read_pipe_fd  The read FD for the USAP reporting pipe.  Manually closed by blastlas
- * in managed code.
+ * @param read_pipe_fd  The read FD for the USAP reporting pipe.  Manually closed by the child
+ * in managed code. -1 indicates none.
  * @param write_pipe_fd  The write FD for the USAP reporting pipe.  Manually closed by the
- * zygote in managed code.
+ * zygote in managed code. -1 indicates none.
  * @param managed_session_socket_fds  A list of anonymous session sockets that must be ignored by
  * the FD hygiene code and automatically "closed" in the new USAP.
+ * @param args_known Arguments for specialization are available; no need to read from a socket
  * @param is_priority_fork  Controls the nice level assigned to the newly created process
- * @return
+ * @return child pid in the parent, 0 in the child
  */
 NO_PAC_FUNC
-static jint com_android_internal_os_Zygote_nativeForkUsap(JNIEnv* env,
-                                                          jclass,
-                                                          jint read_pipe_fd,
-                                                          jint write_pipe_fd,
-                                                          jintArray managed_session_socket_fds,
-                                                          jboolean is_priority_fork) {
-  std::vector<int> fds_to_close(MakeUsapPipeReadFDVector()),
-                   fds_to_ignore(fds_to_close);
-
+static jint com_android_internal_os_Zygote_nativeForkApp(JNIEnv* env,
+                                                         jclass,
+                                                         jint read_pipe_fd,
+                                                         jint write_pipe_fd,
+                                                         jintArray managed_session_socket_fds,
+                                                         jboolean args_known,
+                                                         jboolean is_priority_fork) {
   std::vector<int> session_socket_fds =
       ExtractJIntArray(env, "USAP", nullptr, managed_session_socket_fds)
           .value_or(std::vector<int>());
+  return zygote::forkApp(env, read_pipe_fd, write_pipe_fd, session_socket_fds,
+                            args_known == JNI_TRUE, is_priority_fork == JNI_TRUE, true);
+}
 
-  // The USAP Pool Event FD is created during the initialization of the
-  // USAP pool and should always be valid here.
+NO_PAC_FUNC
+int zygote::forkApp(JNIEnv* env,
+                    int read_pipe_fd,
+                    int write_pipe_fd,
+                    const std::vector<int>& session_socket_fds,
+                    bool args_known,
+                    bool is_priority_fork,
+                    bool purge) {
+
+  std::vector<int> fds_to_close(MakeUsapPipeReadFDVector()),
+                   fds_to_ignore(fds_to_close);
 
   fds_to_close.push_back(gZygoteSocketFD);
-  fds_to_close.push_back(gUsapPoolEventFD);
-  fds_to_close.insert(fds_to_close.end(), session_socket_fds.begin(), session_socket_fds.end());
   if (gSystemServerSocketFd != -1) {
       fds_to_close.push_back(gSystemServerSocketFd);
   }
+  if (args_known) {
+      fds_to_close.push_back(gUsapPoolSocketFD);
+  }
+  fds_to_close.insert(fds_to_close.end(), session_socket_fds.begin(), session_socket_fds.end());
 
-  fds_to_ignore.push_back(gZygoteSocketFD);
   fds_to_ignore.push_back(gUsapPoolSocketFD);
-  fds_to_ignore.push_back(gUsapPoolEventFD);
-  fds_to_ignore.push_back(read_pipe_fd);
-  fds_to_ignore.push_back(write_pipe_fd);
+  fds_to_ignore.push_back(gZygoteSocketFD);
+  if (read_pipe_fd != -1) {
+      fds_to_ignore.push_back(read_pipe_fd);
+  }
+  if (write_pipe_fd != -1) {
+      fds_to_ignore.push_back(write_pipe_fd);
+  }
   fds_to_ignore.insert(fds_to_ignore.end(), session_socket_fds.begin(), session_socket_fds.end());
+
+  if (gUsapPoolEventFD != -1) {
+      fds_to_close.push_back(gUsapPoolEventFD);
+      fds_to_ignore.push_back(gUsapPoolEventFD);
+  }
   if (gSystemServerSocketFd != -1) {
+      if (args_known) {
+          fds_to_close.push_back(gSystemServerSocketFd);
+      }
       fds_to_ignore.push_back(gSystemServerSocketFd);
   }
-
-  pid_t usap_pid = ForkCommon(env, /* is_system_server= */ false, fds_to_close, fds_to_ignore,
-                              is_priority_fork == JNI_TRUE);
-
-  if (usap_pid != 0) {
-    ++gUsapPoolCount;
-    AddUsapTableEntry(usap_pid, read_pipe_fd);
-  }
-
-  return usap_pid;
+  return zygote::ForkCommon(env, /* is_system_server= */ false, fds_to_close,
+                            fds_to_ignore, is_priority_fork == JNI_TRUE, purge);
 }
 
 static void com_android_internal_os_Zygote_nativeAllowFileAcrossFork(
@@ -2374,7 +2385,7 @@ static void com_android_internal_os_Zygote_nativeInitNativeState(JNIEnv* env, jc
    */
 
   if (!SetTaskProfiles(0, {})) {
-    ZygoteFailure(env, "zygote", nullptr, "Zygote SetTaskProfiles failed");
+    zygote::ZygoteFailure(env, "zygote", nullptr, "Zygote SetTaskProfiles failed");
   }
 }
 
@@ -2392,15 +2403,21 @@ static jintArray com_android_internal_os_Zygote_nativeGetUsapPipeFDs(JNIEnv* env
   return managed_usap_fds;
 }
 
+/*
+ * Add the given pid and file descriptor to the Usap table. CriticalNative method.
+ */
+static void com_android_internal_os_Zygote_nativeAddUsapTableEntry(jint pid, jint read_pipe_fd) {
+  AddUsapTableEntry(pid, read_pipe_fd);
+}
+
 /**
- * A JNI wrapper around RemoveUsapTableEntry.
+ * A JNI wrapper around RemoveUsapTableEntry. CriticalNative method.
  *
  * @param env  Managed runtime environment
  * @param usap_pid  Process ID of the USAP entry to invalidate
  * @return  True if an entry was invalidated; false otherwise.
  */
-static jboolean com_android_internal_os_Zygote_nativeRemoveUsapTableEntry(JNIEnv* env, jclass,
-                                                                          jint usap_pid) {
+static jboolean com_android_internal_os_Zygote_nativeRemoveUsapTableEntry(jint usap_pid) {
   return RemoveUsapTableEntry(usap_pid);
 }
 
@@ -2415,7 +2432,8 @@ static jboolean com_android_internal_os_Zygote_nativeRemoveUsapTableEntry(JNIEnv
 static jint com_android_internal_os_Zygote_nativeGetUsapPoolEventFD(JNIEnv* env, jclass) {
   if (gUsapPoolEventFD == -1) {
     if ((gUsapPoolEventFD = eventfd(0, 0)) == -1) {
-      ZygoteFailure(env, "zygote", nullptr, StringPrintf("Unable to create eventfd: %s", strerror(errno)));
+      zygote::ZygoteFailure(env, "zygote", nullptr,
+                            StringPrintf("Unable to create eventfd: %s", strerror(errno)));
     }
   }
 
@@ -2458,12 +2476,12 @@ static void com_android_internal_os_Zygote_nativeEmptyUsapPool(JNIEnv* env, jcla
 }
 
 static void com_android_internal_os_Zygote_nativeBlockSigTerm(JNIEnv* env, jclass) {
-  auto fail_fn = std::bind(ZygoteFailure, env, "usap", nullptr, _1);
+  auto fail_fn = std::bind(zygote::ZygoteFailure, env, "usap", nullptr, _1);
   BlockSignal(SIGTERM, fail_fn);
 }
 
 static void com_android_internal_os_Zygote_nativeUnblockSigTerm(JNIEnv* env, jclass) {
-  auto fail_fn = std::bind(ZygoteFailure, env, "usap", nullptr, _1);
+  auto fail_fn = std::bind(zygote::ZygoteFailure, env, "usap", nullptr, _1);
   UnblockSignal(SIGTERM, fail_fn);
 }
 
@@ -2569,7 +2587,10 @@ static const JNINativeMethod gMethods[] = {
          (void*)com_android_internal_os_Zygote_nativePreApplicationInit},
         {"nativeInstallSeccompUidGidFilter", "(II)V",
          (void*)com_android_internal_os_Zygote_nativeInstallSeccompUidGidFilter},
-        {"nativeForkUsap", "(II[IZ)I", (void*)com_android_internal_os_Zygote_nativeForkUsap},
+        {"nativeForkApp", "(II[IZZ)I", (void*)com_android_internal_os_Zygote_nativeForkApp},
+        // @CriticalNative
+        {"nativeAddUsapTableEntry", "(II)V",
+         (void*)com_android_internal_os_Zygote_nativeAddUsapTableEntry},
         {"nativeSpecializeAppProcess",
          "(II[II[[IILjava/lang/String;Ljava/lang/String;ZLjava/lang/String;Ljava/lang/"
          "String;Z[Ljava/lang/String;[Ljava/lang/String;ZZ)V",
@@ -2578,6 +2599,10 @@ static const JNINativeMethod gMethods[] = {
          (void*)com_android_internal_os_Zygote_nativeInitNativeState},
         {"nativeGetUsapPipeFDs", "()[I",
          (void*)com_android_internal_os_Zygote_nativeGetUsapPipeFDs},
+        // @CriticalNative
+        {"nativeAddUsapTableEntry", "(II)V",
+         (void*)com_android_internal_os_Zygote_nativeAddUsapTableEntry},
+        // @CriticalNative
         {"nativeRemoveUsapTableEntry", "(I)Z",
          (void*)com_android_internal_os_Zygote_nativeRemoveUsapTableEntry},
         {"nativeGetUsapPoolEventFD", "()I",
diff --git a/core/jni/com_android_internal_os_Zygote.h b/core/jni/com_android_internal_os_Zygote.h
new file mode 100644
index 000000000000..d2da91476bc7
--- /dev/null
+++ b/core/jni/com_android_internal_os_Zygote.h
@@ -0,0 +1,78 @@
+/*
+ * Copyright (C) 2008 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _COM_ANDROID_INTERNAL_OS_ZYGOTE_H
+#define _COM_ANDROID_INTERNAL_OS_ZYGOTE_H
+
+#define LOG_TAG "Zygote"
+#define ATRACE_TAG ATRACE_TAG_DALVIK
+
+/* Functions in the callchain during the fork shall not be protected with
+   Armv8.3-A Pointer Authentication, otherwise child will not be able to return. */
+#ifdef __ARM_FEATURE_PAC_DEFAULT
+#ifdef __ARM_FEATURE_BTI_DEFAULT
+#define NO_PAC_FUNC __attribute__((target("branch-protection=bti")))
+#else
+#define NO_PAC_FUNC __attribute__((target("branch-protection=none")))
+#endif /* __ARM_FEATURE_BTI_DEFAULT */
+#else /* !__ARM_FEATURE_PAC_DEFAULT */
+#define NO_PAC_FUNC
+#endif /* __ARM_FEATURE_PAC_DEFAULT */
+
+#include <jni.h>
+#include <vector>
+#include <android-base/stringprintf.h>
+
+#define CREATE_ERROR(...) StringPrintf("%s:%d: ", __FILE__, __LINE__). \
+                              append(StringPrintf(__VA_ARGS__))
+
+namespace android {
+namespace zygote {
+
+NO_PAC_FUNC
+pid_t ForkCommon(JNIEnv* env,bool is_system_server,
+                 const std::vector<int>& fds_to_close,
+                 const std::vector<int>& fds_to_ignore,
+                 bool is_priority_fork,
+                 bool purge = true);
+
+/**
+ * Fork a process. The pipe fds are used for usap communication, or -1 in
+ * other cases. Session_socket_fds are FDs used for zygote communication that must be dealt
+ * with hygienically, but are not otherwise used here. Args_known indicates that the process
+ * will be immediately specialized with arguments that are already known, so no usap
+ * communication is required. Is_priority_fork should be true if this is on the app startup
+ * critical path. Purge specifies that unused pages should be purged before the fork.
+ */
+NO_PAC_FUNC
+int forkApp(JNIEnv* env,
+            int read_pipe_fd,
+            int write_pipe_fd,
+            const std::vector<int>& session_socket_fds,
+            bool args_known,
+            bool is_priority_fork,
+            bool purge);
+
+[[noreturn]]
+void ZygoteFailure(JNIEnv* env,
+                   const char* process_name,
+                   jstring managed_process_name,
+                   const std::string& msg);
+
+}  // namespace zygote
+}  // namespace android
+
+#endif // _COM_ANDROID_INTERNAL_OS_ZYGOTE_
diff --git a/core/jni/com_android_internal_os_ZygoteCommandBuffer.cpp b/core/jni/com_android_internal_os_ZygoteCommandBuffer.cpp
new file mode 100644
index 000000000000..011e8f8f1b8c
--- /dev/null
+++ b/core/jni/com_android_internal_os_ZygoteCommandBuffer.cpp
@@ -0,0 +1,512 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "com_android_internal_os_Zygote.h"
+
+#include <algorithm>
+#include <android-base/logging.h>
+#include <async_safe/log.h>
+#include <cctype>
+#include <chrono>
+#include <core_jni_helpers.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <jni.h>
+#include <nativehelper/JNIHelp.h>
+#include <optional>
+#include <poll.h>
+#include <unistd.h>
+#include <utility>
+#include <utils/misc.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <vector>
+
+namespace android {
+
+using namespace std::placeholders;
+using android::base::StringPrintf;
+using android::zygote::ZygoteFailure;
+
+// WARNING: Knows a little about the wire protocol used to communicate with Zygote.
+// TODO: Fix error handling.
+
+constexpr size_t MAX_COMMAND_BYTES = 12200;
+constexpr size_t NICE_NAME_BYTES = 50;
+
+// A buffer optionally bundled with a file descriptor from which we can fill it.
+// Does not own the file descriptor; destroying a NativeCommandBuffer does not
+// close the descriptor.
+class NativeCommandBuffer {
+ public:
+  NativeCommandBuffer(int sourceFd): mEnd(0), mNext(0), mLinesLeft(0), mFd(sourceFd) {}
+
+  // Read mNext line from mFd, filling mBuffer from file descriptor, as needed.
+  // Return a pair of pointers pointing to the first character, and one past the
+  // mEnd of the line, i.e. at the newline. Returns nothing on failure.
+  template<class FailFn>
+  std::optional<std::pair<char*, char*>> readLine(FailFn fail_fn) {
+    char* result = mBuffer + mNext;
+    while (true) {
+      if (mNext == mEnd) {
+        if (mEnd == MAX_COMMAND_BYTES) {
+          return {};
+        }
+        if (mFd == -1) {
+          fail_fn("ZygoteCommandBuffer.readLine attempted to read from mFd -1");
+        }
+        ssize_t nread = TEMP_FAILURE_RETRY(read(mFd, mBuffer + mEnd, MAX_COMMAND_BYTES - mEnd));
+        if (nread <= 0) {
+          if (nread == 0) {
+            return {};
+          }
+          fail_fn(CREATE_ERROR("session socket read failed: %s", strerror(errno)));
+        } else if (nread == MAX_COMMAND_BYTES - mEnd) {
+          // This is pessimistic by one character, but close enough.
+          fail_fn("ZygoteCommandBuffer overflowed: command too long");
+        }
+        mEnd += nread;
+      }
+      // UTF-8 does not allow newline to occur as part of a multibyte character.
+      char* nl = static_cast<char *>(memchr(mBuffer + mNext, '\n', mEnd - mNext));
+      if (nl == nullptr) {
+        mNext = mEnd;
+      } else {
+        mNext = nl - mBuffer + 1;
+        if (--mLinesLeft < 0) {
+          fail_fn("ZygoteCommandBuffer.readLine attempted to read past mEnd of command");
+        }
+        return std::make_pair(result, nl);
+      }
+    }
+  }
+
+  void reset() {
+    mNext = 0;
+  }
+
+  // Make sure the current command is fully buffered, without reading past the current command.
+  template<class FailFn>
+  void readAllLines(FailFn fail_fn) {
+     while (mLinesLeft > 0) {
+       readLine(fail_fn);
+    }
+  }
+
+  void clear() {
+    // Don't bother to actually clear the buffer; it'll be unmapped in the child anyway.
+    reset();
+    mNiceName[0] = '\0';
+    mEnd = 0;
+  }
+
+  // Insert line into the mBuffer. Checks that the mBuffer is not associated with an mFd.
+  // Implicitly adds newline separators. Allows mBuffer contents to be explicitly set.
+  void insert(const char* line, size_t lineLen) {
+    DCHECK(mFd == -1);
+    CHECK(mEnd + lineLen < MAX_COMMAND_BYTES);
+    strncpy(mBuffer + mEnd, line, lineLen);
+    mBuffer[mEnd + lineLen] = '\n';
+    mEnd += lineLen + 1;
+  }
+
+  // Clear mBuffer, start reading new command, return the number of arguments, leaving mBuffer
+  // positioned at the beginning of first argument. Return 0 on EOF.
+  template<class FailFn>
+  int getCount(FailFn fail_fn) {
+    mLinesLeft = 1;
+    auto line = readLine(fail_fn);
+    if (!line.has_value()) {
+      return 0;
+    }
+    char* countString = line.value().first;  // Newline terminated.
+    long nArgs = atol(countString);
+    if (nArgs <= 0 || nArgs >= MAX_COMMAND_BYTES / 2) {
+      fail_fn(CREATE_ERROR("Unreasonable argument count %ld", nArgs));
+    }
+    mLinesLeft = nArgs;
+    return static_cast<int>(nArgs);
+  }
+
+  // Is the mBuffer a simple fork command?
+  // We disallow request to wrap the child process, child zygotes, anything that
+  // mentions capabilities or requests uid < minUid.
+  // We insist that --setuid and --setgid arguments are explicitly included and that the
+  // command starts with --runtime-args.
+  // Assumes we are positioned at the beginning of the command after the argument count,
+  // and leaves the position at some indeterminate position in the buffer.
+  // As a side effect, this sets mNiceName to a non-empty string, if possible.
+  template<class FailFn>
+  bool isSimpleForkCommand(int minUid, FailFn fail_fn) {
+    if (mLinesLeft <= 0 || mLinesLeft  >= MAX_COMMAND_BYTES / 2) {
+      return false;
+    }
+    static const char* RUNTIME_ARGS = "--runtime-args";
+    static const char* INVOKE_WITH = "--invoke-with";
+    static const char* CHILD_ZYGOTE = "--start-child-zygote";
+    static const char* SETUID = "--setuid=";
+    static const char* SETGID = "--setgid=";
+    static const char* CAPABILITIES = "--capabilities";
+    static const char* NICE_NAME = "--nice-name=";
+    static const size_t RA_LENGTH = strlen(RUNTIME_ARGS);
+    static const size_t IW_LENGTH = strlen(INVOKE_WITH);
+    static const size_t CZ_LENGTH = strlen(CHILD_ZYGOTE);
+    static const size_t SU_LENGTH = strlen(SETUID);
+    static const size_t SG_LENGTH = strlen(SETGID);
+    static const size_t CA_LENGTH = strlen(CAPABILITIES);
+    static const size_t NN_LENGTH = strlen(NICE_NAME);
+
+    bool saw_setuid = false, saw_setgid = false;
+    bool saw_runtime_args = false;
+
+    while (mLinesLeft > 0) {
+      auto read_result = readLine(fail_fn);
+      if (!read_result.has_value()) {
+        return false;
+      }
+      auto [arg_start, arg_end] = read_result.value();
+      if (arg_end - arg_start == RA_LENGTH
+          && strncmp(arg_start, RUNTIME_ARGS, RA_LENGTH) == 0) {
+        saw_runtime_args = true;
+        continue;
+      }
+      if (arg_end - arg_start >= NN_LENGTH
+          && strncmp(arg_start, NICE_NAME, NN_LENGTH) == 0) {
+        size_t name_len = arg_end - (arg_start + NN_LENGTH);
+        size_t copy_len = std::min(name_len, NICE_NAME_BYTES - 1);
+        memcpy(mNiceName, arg_start + NN_LENGTH, copy_len);
+        mNiceName[copy_len] = '\0';
+        continue;
+      }
+      if (arg_end - arg_start == IW_LENGTH
+          && strncmp(arg_start, INVOKE_WITH, IW_LENGTH) == 0) {
+        // This also removes the need for invoke-with security checks here.
+        return false;
+      }
+      if (arg_end - arg_start == CZ_LENGTH
+          && strncmp(arg_start, CHILD_ZYGOTE, CZ_LENGTH) == 0) {
+        return false;
+      }
+      if (arg_end - arg_start >= CA_LENGTH
+          && strncmp(arg_start, CAPABILITIES, CA_LENGTH) == 0) {
+        return false;
+      }
+      if (arg_end - arg_start >= SU_LENGTH
+          && strncmp(arg_start, SETUID, SU_LENGTH) == 0) {
+        int uid = digitsVal(arg_start + SU_LENGTH, arg_end);
+        if (uid < minUid) {
+          return false;
+        }
+        saw_setuid = true;
+        continue;
+      }
+      if (arg_end - arg_start >= SG_LENGTH
+          && strncmp(arg_start, SETGID, SG_LENGTH) == 0) {
+        int gid = digitsVal(arg_start + SG_LENGTH, arg_end);
+        if (gid == -1) {
+          return false;
+        }
+        saw_setgid = true;
+      }
+    }
+    return saw_runtime_args && saw_setuid && saw_setgid;
+  }
+
+  void setFd(int new_fd) {
+    mFd = new_fd;
+  }
+
+  int getFd() const {
+    return mFd;
+  }
+
+  const char* niceNameAddr() const {
+    return mNiceName;
+  }
+
+  // Debug only:
+  void logState() const {
+    ALOGD("mbuffer starts with %c%c, nice name is %s, "
+          "mEnd = %u, mNext = %u, mLinesLeft = %d, mFd = %d",
+          mBuffer[0], (mBuffer[1] == '\n' ? ' ' : mBuffer[1]),
+          niceNameAddr(),
+          static_cast<unsigned>(mEnd), static_cast<unsigned>(mNext),
+          static_cast<int>(mLinesLeft), mFd);
+  }
+
+ private:
+  // Picky version of atoi(). No sign or unexpected characters allowed. Return -1 on failure.
+  static int digitsVal(char* start, char* end) {
+    int result = 0;
+    if (end - start > 6) {
+      return -1;
+    }
+    for (char* dp = start; dp < end; ++dp) {
+      if (*dp < '0' || *dp > '9') {
+        ALOGW("Argument failed integer format check");
+        return -1;
+      }
+      result = 10 * result + (*dp - '0');
+    }
+    return result;
+  }
+
+  uint32_t mEnd;  // Index of first empty byte in the mBuffer.
+  uint32_t mNext;  // Index of first character past last line returned by readLine.
+  int32_t mLinesLeft;  // Lines in current command that haven't yet been read.
+  int mFd;  // Open file descriptor from which we can read more. -1 if none.
+  char mNiceName[NICE_NAME_BYTES];
+  char mBuffer[MAX_COMMAND_BYTES];
+};
+
+static_assert(sizeof(NativeCommandBuffer) < 3 * 4096);
+
+static int buffersAllocd(0);
+
+// Get a new NativeCommandBuffer. Can only be called once between freeNativeBuffer calls,
+// so that only one buffer exists at a time.
+jlong com_android_internal_os_ZygoteCommandBuffer_getNativeBuffer(JNIEnv* env, jclass, jint fd) {
+  CHECK(buffersAllocd == 0);
+  ++buffersAllocd;
+  // MMap explicitly to get it page aligned.
+  void *bufferMem = mmap(NULL, sizeof(NativeCommandBuffer), PROT_READ | PROT_WRITE,
+                         MAP_ANONYMOUS | MAP_PRIVATE | MAP_POPULATE, -1, 0);
+  // Currently we mmap and unmap one for every request handled by the Java code.
+  // That could be improved, but unclear it matters.
+  if (bufferMem == MAP_FAILED) {
+    ZygoteFailure(env, nullptr, nullptr, "Failed to map argument buffer");
+  }
+  return (jlong) new(bufferMem) NativeCommandBuffer(fd);
+}
+
+// Delete native command buffer.
+void com_android_internal_os_ZygoteCommandBuffer_freeNativeBuffer(JNIEnv* env, jclass,
+                                                                  jlong j_buffer) {
+  CHECK(buffersAllocd == 1);
+  NativeCommandBuffer* n_buffer = reinterpret_cast<NativeCommandBuffer*>(j_buffer);
+  n_buffer->~NativeCommandBuffer();
+  if (munmap(n_buffer, sizeof(NativeCommandBuffer)) != 0) {
+    ZygoteFailure(env, nullptr, nullptr, "Failed to unmap argument buffer");
+  }
+  --buffersAllocd;
+}
+
+// Clear the buffer, read the line containing the count, and return the count.
+jint com_android_internal_os_ZygoteCommandBuffer_nativeGetCount(JNIEnv* env, jclass,
+                                                                jlong j_buffer) {
+  NativeCommandBuffer* n_buffer = reinterpret_cast<NativeCommandBuffer*>(j_buffer);
+  auto fail_fn = std::bind(ZygoteFailure, env, nullptr, nullptr, _1);
+  return n_buffer->getCount(fail_fn);
+}
+
+// Explicitly insert a string as the last line (argument) of the buffer.
+void com_android_internal_os_ZygoteCommandBuffer_insert(JNIEnv* env, jclass, jlong j_buffer,
+                                                        jstring line) {
+  NativeCommandBuffer* n_buffer = reinterpret_cast<NativeCommandBuffer*>(j_buffer);
+  size_t lineLen = static_cast<size_t>(env->GetStringUTFLength(line));
+  const char* cstring = env->GetStringUTFChars(line, NULL);
+  n_buffer->insert(cstring, lineLen);
+  env->ReleaseStringUTFChars(line, cstring);
+}
+
+// Read a line from the buffer, refilling as necessary.
+jstring com_android_internal_os_ZygoteCommandBuffer_nativeNextArg(JNIEnv* env, jclass,
+                                                                  jlong j_buffer) {
+  NativeCommandBuffer* n_buffer = reinterpret_cast<NativeCommandBuffer*>(j_buffer);
+  auto fail_fn = std::bind(ZygoteFailure, env, n_buffer->niceNameAddr(), nullptr, _1);
+  auto line = n_buffer->readLine(fail_fn);
+  if (!line.has_value()) {
+    fail_fn("Incomplete zygote command");
+  }
+  auto [cresult, endp] = line.value();
+  // OK to temporarily clobber the buffer, since this is not thread safe, and we're modifying
+  // the buffer anyway.
+  *endp = '\0';
+  jstring result = env->NewStringUTF(cresult);
+  *endp = '\n';
+  return result;
+}
+
+// Read all lines from the current command into the buffer, and then reset the buffer, so
+// we will start reading again at the beginning of the command, starting with the argument
+// count. And we don't need access to the fd to do so.
+void com_android_internal_os_ZygoteCommandBuffer_nativeReadFullyAndReset(JNIEnv* env, jclass,
+                                                                         jlong j_buffer) {
+  NativeCommandBuffer* n_buffer = reinterpret_cast<NativeCommandBuffer*>(j_buffer);
+  auto fail_fn = std::bind(ZygoteFailure, env, n_buffer->niceNameAddr(), nullptr, _1);
+  n_buffer->readAllLines(fail_fn);
+  n_buffer->reset();
+}
+
+// Fork a child as specified by the current command buffer, and refill the command
+// buffer from the given socket. So long as the result is another simple fork command,
+// repeat this process.
+// It must contain a fork command, which is currently restricted not to fork another
+// zygote or involve a wrapper process.
+// The initial buffer should be partially or entirely read; we read it fully and reset it.
+// When we return, the buffer contains the command we couldn't handle, and has been reset().
+// We return false in the parent when we see a command we didn't understand, and thus the
+// command in the buffer still needs to be executed.
+// We return true in each child.
+// We only process fork commands if the peer uid matches expected_uid.
+// For every fork command after the first, we check that the requested uid is at
+// least minUid.
+NO_PAC_FUNC
+jboolean com_android_internal_os_ZygoteCommandBuffer_nativeForkRepeatedly(
+            JNIEnv* env,
+            jclass,
+            jlong j_buffer,
+            jint zygote_socket_fd,
+            jint expected_uid,
+            jint minUid,
+            jstring managed_nice_name) {
+
+  NativeCommandBuffer* n_buffer = reinterpret_cast<NativeCommandBuffer*>(j_buffer);
+  int session_socket = n_buffer->getFd();
+  std::vector<int> session_socket_fds {session_socket};
+  auto fail_fn_1 = std::bind(ZygoteFailure, env, static_cast<const char*>(nullptr),
+                             static_cast<jstring>(managed_nice_name), _1);
+  // This binds to the nice name address; the actual names are updated by isSimpleForkCommand:
+  auto fail_fn_n = std::bind(ZygoteFailure, env, n_buffer->niceNameAddr(),
+                             static_cast<jstring>(nullptr), _1);
+  auto fail_fn_z = std::bind(ZygoteFailure, env, "zygote", nullptr, _1);
+
+  struct pollfd fd_structs[2];
+  static const int ZYGOTE_IDX = 0;
+  static const int SESSION_IDX = 1;
+  fd_structs[ZYGOTE_IDX].fd = zygote_socket_fd;
+  fd_structs[ZYGOTE_IDX].events = POLLIN;
+  fd_structs[SESSION_IDX].fd = session_socket;
+  fd_structs[SESSION_IDX].events = POLLIN;
+
+  struct timeval timeout;
+  socklen_t timeout_size = sizeof timeout;
+  if (getsockopt(session_socket, SOL_SOCKET, SO_RCVTIMEO, &timeout, &timeout_size) != 0) {
+    fail_fn_z("Failed to retrieve session socket timeout");
+  }
+
+  struct ucred credentials;
+  socklen_t cred_size = sizeof credentials;
+  if (getsockopt(n_buffer->getFd(), SOL_SOCKET, SO_PEERCRED, &credentials, &cred_size) == -1
+      || cred_size != sizeof credentials) {
+    fail_fn_1("ForkMany failed to get initial credentials, %s", strerror(errno));
+  }
+
+  bool first_time = true;
+  do {
+    if (credentials.uid != expected_uid) {
+      return JNI_FALSE;
+    }
+    n_buffer->readAllLines(first_time ? fail_fn_1 : fail_fn_n);
+    n_buffer->reset();
+    int pid = zygote::forkApp(env, /* no pipe FDs */ -1, -1, session_socket_fds,
+                              /*args_known=*/ true, /*is_priority_fork=*/ true,
+                              /*purge=*/ first_time);
+    if (pid == 0) {
+      return JNI_TRUE;
+    }
+    // We're in the parent. Write big-endian pid, followed by a boolean.
+    char pid_buf[5];
+    int tmp_pid = pid;
+    for (int i = 3; i >= 0; --i) {
+      pid_buf[i] = tmp_pid & 0xff;
+      tmp_pid >>= 8;
+    }
+    pid_buf[4] = 0;  // Process is not wrapped.
+    int res = write(session_socket, pid_buf, 5);
+    if (res != 5) {
+      if (res == -1) {
+        (first_time ? fail_fn_1 : fail_fn_n)
+            (CREATE_ERROR("Pid write error %d: %s", errno, strerror(errno)));
+      } else {
+        (first_time ? fail_fn_1 : fail_fn_n)
+            (CREATE_ERROR("Write unexpectedly returned short: %d < 5", res));
+      }
+    }
+    // Clear buffer and get count from next command.
+    n_buffer->clear();
+    for (;;) {
+      // Poll isn't strictly necessary for now. But without it, disconnect is hard to detect.
+      int poll_res = TEMP_FAILURE_RETRY(poll(fd_structs, 2, -1 /* infinite timeout */));
+      if ((fd_structs[SESSION_IDX].revents & POLLIN) != 0) {
+        if (n_buffer->getCount(fail_fn_z) != 0) {
+          break;
+        }  // else disconnected;
+      } else if (poll_res == 0 || (fd_structs[ZYGOTE_IDX].revents & POLLIN) == 0) {
+        fail_fn_z(
+            CREATE_ERROR("Poll returned with no descriptors ready! Poll returned %d", poll_res));
+      }
+      // We've now seen either a disconnect or connect request.
+      close(session_socket);
+      int new_fd = accept(zygote_socket_fd, nullptr, nullptr);
+      if (new_fd == -1) {
+        fail_fn_z("Accept(%d) failed: %s", zygote_socket_fd, strerror(errno));
+      }
+      if (new_fd != session_socket) {
+          // Move new_fd back to the old value, so that we don't have to change Java-level data
+          // structures to reflect a change. This implicitly closes the old one.
+          if (dup2(new_fd, session_socket) != session_socket) {
+            fail_fn_z(CREATE_ERROR("Failed to move fd %d to %d: %s",
+                                   new_fd, session_socket, strerror(errno)));
+          }
+          close(new_fd);
+      }
+      // If we ever return, we effectively reuse the old Java ZygoteConnection.
+      // None of its state needs to change.
+      if (setsockopt(session_socket, SOL_SOCKET, SO_RCVTIMEO, &timeout, timeout_size) != 0) {
+        fail_fn_z(CREATE_ERROR("Failed to set receive timeout for socket %d: %s",
+                               session_socket, strerror(errno)));
+      }
+      if (setsockopt(session_socket, SOL_SOCKET, SO_SNDTIMEO, &timeout, timeout_size) != 0) {
+        fail_fn_z(CREATE_ERROR("Failed to set send timeout for socket %d: %s",
+                               session_socket, strerror(errno)));
+      }
+      if (getsockopt(session_socket, SOL_SOCKET, SO_PEERCRED, &credentials, &cred_size) == -1) {
+        fail_fn_z(CREATE_ERROR("ForkMany failed to get credentials: %s", strerror(errno)));
+      }
+      if (cred_size != sizeof credentials) {
+        fail_fn_z(CREATE_ERROR("ForkMany credential size = %d, should be %d",
+                               cred_size, static_cast<int>(sizeof credentials)));
+      }
+    }
+    first_time = false;
+  } while (n_buffer->isSimpleForkCommand(minUid, fail_fn_n));
+  ALOGW("forkRepeatedly terminated due to non-simple command");
+  n_buffer->logState();
+  n_buffer->reset();
+  return JNI_FALSE;
+}
+
+#define METHOD_NAME(m) com_android_internal_os_ZygoteCommandBuffer_ ## m
+
+static const JNINativeMethod gMethods[] = {
+        {"getNativeBuffer", "(I)J", (void *) METHOD_NAME(getNativeBuffer)},
+        {"freeNativeBuffer", "(J)V", (void *) METHOD_NAME(freeNativeBuffer)},
+        {"insert", "(JLjava/lang/String;)V", (void *) METHOD_NAME(insert)},
+        {"nativeNextArg", "(J)Ljava/lang/String;", (void *) METHOD_NAME(nativeNextArg)},
+        {"nativeReadFullyAndReset", "(J)V", (void *) METHOD_NAME(nativeReadFullyAndReset)},
+        {"nativeGetCount", "(J)I", (void *) METHOD_NAME(nativeGetCount)},
+        {"nativeForkRepeatedly", "(JIIILjava/lang/String;)Z",
+          (void *) METHOD_NAME(nativeForkRepeatedly)},
+};
+
+int register_com_android_internal_os_ZygoteCommandBuffer(JNIEnv* env) {
+  return RegisterMethodsOrDie(env, "com/android/internal/os/ZygoteCommandBuffer", gMethods,
+                              NELEM(gMethods));
+}
+
+}  // namespace android