Add zygote native fork loop

Do not return to Java mode between consecutive fork operations.
This greatly reduces the Zygote overhead, since we no longer need to
stop and restart Java daemons.

By not switching back to Java mode, and being careful about what memory
we touch between forks, we also keep the Zygote heaps much more stable,
facilitating page sharing between the zygote and all its children.
Under normal operation we should no longer allocate any memory in the
zygote between forks. That applies to both the Java and C++ heap.
This makes the zygote behave much more like the mental model many
of us had assumed: It has nearly constant memory contents, which are
copy-on-right cloned at each fork. This does not apply to the initial
system server and webzygote forks, that are currently still handled
differently.

This includes

1. Add ZygoteCommandBuffer, and switch the argument parsing code to use it.
This slightly reduces allocation and enables (3).

2. Support process specialization in the child, even when the arguments
are already know, Leverages existing Usap code.

3. Add support for forking multiple child processes directly to the
ZygoteCommandBuffer data structure. This directly uses the buffer
internals, and avoids returning to Java so long as it can handle the
zygote commands it sees.

FUNCTIONALITY CHANGE:

We now limit the total size of the zygote command, rather than the
number of arguments.

Initial performance observations:

[ These are not perfect, since I'm comparing to numbers before I
started. There may have been other moving parts, but they should be
minor. ]

System-server-observed launch latency:

[Not the best metric, but easy to measure. In particular, this
does not represent a significant reduction in application launch
time.]

Based on measuring the last 10 launches in a lightly used cf AOSP
instance, the system server latency from requesting an app launch to
response with the pid (which does not require the child to execute
anything) went from an average of about 10.7(25) msecs to 6.8(9) and
7.9(16) in two tries with the CL. (The parenthetical numbers are
maxima from among the 10; the variance appears to have decreased
appreciably.)

Dirty pages:

The number of private dirty pages in the zygote itself appears to have
decreased from about 4000 to about 2200. The number of dalvik-main
private dirty pages went from about 1500 to nearly zero.

Initially ART benchmarking service claim -1.88% in PSS. But this is not
consistently repeatable.

Drive-by fix:

Call setAllowNetworkingForProcess on usap / native loop path.

Bug: 159631815
Bug: 174211442
Test: Boots AOSP
Change-Id: I90d2e381bada1b6c9857666d5e87372b6a4c1a70

5 files changed, 791 insertions, 173 deletions

diff --git a/core/jni/Android.bp b/core/jni/Android.bp
index 877349227406..787a0384950a 100644
--- a/core/jni/Android.bp
+++ b/core/jni/Android.bp
@@ -205,6 +205,7 @@ cc_library_shared {
                 "com_android_internal_os_KernelCpuUidBpfMapReader.cpp",
                 "com_android_internal_os_KernelSingleUidTimeReader.cpp",
                 "com_android_internal_os_Zygote.cpp",
+                "com_android_internal_os_ZygoteCommandBuffer.cpp",
                 "com_android_internal_os_ZygoteInit.cpp",
                 "hwbinder/EphemeralStorage.cpp",
                 "fd_utils.cpp",
diff --git a/core/jni/AndroidRuntime.cpp b/core/jni/AndroidRuntime.cpp
index 3198cb1b8140..c01f741862b1 100644
--- a/core/jni/AndroidRuntime.cpp
+++ b/core/jni/AndroidRuntime.cpp
@@ -193,6 +193,7 @@ extern int register_com_android_internal_os_FuseAppLoop(JNIEnv* env);
 extern int register_com_android_internal_os_KernelCpuUidBpfMapReader(JNIEnv *env);
 extern int register_com_android_internal_os_KernelSingleUidTimeReader(JNIEnv *env);
 extern int register_com_android_internal_os_Zygote(JNIEnv *env);
+extern int register_com_android_internal_os_ZygoteCommandBuffer(JNIEnv *env);
 extern int register_com_android_internal_os_ZygoteInit(JNIEnv *env);
 extern int register_com_android_internal_util_VirtualRefBasePtr(JNIEnv *env);
 
@@ -1524,6 +1525,7 @@ static const RegJNIRec gRegJNI[] = {
         REG_JNI(register_com_android_internal_net_NetworkUtilsInternal),
         REG_JNI(register_com_android_internal_os_ClassLoaderFactory),
         REG_JNI(register_com_android_internal_os_Zygote),
+        REG_JNI(register_com_android_internal_os_ZygoteCommandBuffer),
         REG_JNI(register_com_android_internal_os_ZygoteInit),
         REG_JNI(register_com_android_internal_util_VirtualRefBasePtr),
         REG_JNI(register_android_hardware_Camera),
diff --git a/core/jni/com_android_internal_os_Zygote.cpp b/core/jni/com_android_internal_os_Zygote.cpp
index 903ecaef4938..4cef2b099589 100644
--- a/core/jni/com_android_internal_os_Zygote.cpp
+++ b/core/jni/com_android_internal_os_Zygote.cpp
@@ -17,6 +17,8 @@
 #define LOG_TAG "Zygote"
 #define ATRACE_TAG ATRACE_TAG_DALVIK
 
+#include "com_android_internal_os_Zygote.h"
+
 #include <async_safe/log.h>
 
 // sys/mount.h has to come before linux/fs.h due to redefinition of MS_RDONLY, MS_BIND, etc
@@ -91,19 +93,6 @@
 
 #include "nativebridge/native_bridge.h"
 
-/* Functions in the callchain during the fork shall not be protected with
-   Armv8.3-A Pointer Authentication, otherwise child will not be able to return. */
-#ifdef __ARM_FEATURE_PAC_DEFAULT
-#ifdef __ARM_FEATURE_BTI_DEFAULT
-#define NO_PAC_FUNC __attribute__((target("branch-protection=bti")))
-#else
-#define NO_PAC_FUNC __attribute__((target("branch-protection=none")))
-#endif /* __ARM_FEATURE_BTI_DEFAULT */
-#else /* !__ARM_FEATURE_PAC_DEFAULT */
-#define NO_PAC_FUNC
-#endif /* __ARM_FEATURE_PAC_DEFAULT */
-
-
 namespace {
 
 // TODO (chriswailes): Add a function to initialize native Zygote data.
@@ -118,8 +107,7 @@ using android::base::StringPrintf;
 using android::base::WriteStringToFile;
 using android::base::GetBoolProperty;
 
-#define CREATE_ERROR(...) StringPrintf("%s:%d: ", __FILE__, __LINE__). \
-                              append(StringPrintf(__VA_ARGS__))
+using android::zygote::ZygoteFailure;
 
 // This type is duplicated in fd_utils.h
 typedef const std::function<void(std::string)>& fail_fn_t;
@@ -215,7 +203,7 @@ class UsapTableEntry {
   static constexpr EntryStorage INVALID_ENTRY_VALUE = {-1, -1};
 
   std::atomic<EntryStorage> mStorage;
-  static_assert(decltype(mStorage)::is_always_lock_free);
+  static_assert(decltype(mStorage)::is_always_lock_free);  // Accessed from signal handler.
 
  public:
   constexpr UsapTableEntry() : mStorage(INVALID_ENTRY_VALUE) {}
@@ -942,36 +930,6 @@ void SetThreadName(const std::string& thread_name) {
 }
 
 /**
- * A failure function used to report fatal errors to the managed runtime.  This
- * function is often curried with the process name information and then passed
- * to called functions.
- *
- * @param env  Managed runtime environment
- * @param process_name  A native representation of the process name
- * @param managed_process_name  A managed representation of the process name
- * @param msg  The error message to be reported
- */
-[[noreturn]]
-static void ZygoteFailure(JNIEnv* env,
-                          const char* process_name,
-                          jstring managed_process_name,
-                          const std::string& msg) {
-  std::unique_ptr<ScopedUtfChars> scoped_managed_process_name_ptr = nullptr;
-  if (managed_process_name != nullptr) {
-    scoped_managed_process_name_ptr.reset(new ScopedUtfChars(env, managed_process_name));
-    if (scoped_managed_process_name_ptr->c_str() != nullptr) {
-      process_name = scoped_managed_process_name_ptr->c_str();
-    }
-  }
-
-  const std::string& error_msg =
-      (process_name == nullptr) ? msg : StringPrintf("(%s) %s", process_name, msg.c_str());
-
-  env->FatalError(error_msg.c_str());
-  __builtin_unreachable();
-}
-
-/**
  * A helper method for converting managed strings to native strings.  A fatal
  * error is generated if a problem is encountered in extracting a non-null
  * string.
@@ -1098,86 +1056,6 @@ static void PAuthKeyChange(JNIEnv* env) {
 #endif
 }
 
-// Utility routine to fork a process from the zygote.
-NO_PAC_FUNC
-static pid_t ForkCommon(JNIEnv* env, bool is_system_server,
-                        const std::vector<int>& fds_to_close,
-                        const std::vector<int>& fds_to_ignore,
-                        bool is_priority_fork) {
-  SetSignalHandlers();
-
-  // Curry a failure function.
-  auto fail_fn = std::bind(ZygoteFailure, env, is_system_server ? "system_server" : "zygote",
-                           nullptr, _1);
-
-  // Temporarily block SIGCHLD during forks. The SIGCHLD handler might
-  // log, which would result in the logging FDs we close being reopened.
-  // This would cause failures because the FDs are not allowlisted.
-  //
-  // Note that the zygote process is single threaded at this point.
-  BlockSignal(SIGCHLD, fail_fn);
-
-  // Close any logging related FDs before we start evaluating the list of
-  // file descriptors.
-  __android_log_close();
-  AStatsSocket_close();
-
-  // If this is the first fork for this zygote, create the open FD table.  If
-  // it isn't, we just need to check whether the list of open files has changed
-  // (and it shouldn't in the normal case).
-  if (gOpenFdTable == nullptr) {
-    gOpenFdTable = FileDescriptorTable::Create(fds_to_ignore, fail_fn);
-  } else {
-    gOpenFdTable->Restat(fds_to_ignore, fail_fn);
-  }
-
-  android_fdsan_error_level fdsan_error_level = android_fdsan_get_error_level();
-
-  // Purge unused native memory in an attempt to reduce the amount of false
-  // sharing with the child process.  By reducing the size of the libc_malloc
-  // region shared with the child process we reduce the number of pages that
-  // transition to the private-dirty state when malloc adjusts the meta-data
-  // on each of the pages it is managing after the fork.
-  mallopt(M_PURGE, 0);
-
-  pid_t pid = fork();
-
-  if (pid == 0) {
-    if (is_priority_fork) {
-      setpriority(PRIO_PROCESS, 0, PROCESS_PRIORITY_MAX);
-    } else {
-      setpriority(PRIO_PROCESS, 0, PROCESS_PRIORITY_MIN);
-    }
-
-    // The child process.
-    PAuthKeyChange(env);
-    PreApplicationInit();
-
-    // Clean up any descriptors which must be closed immediately
-    DetachDescriptors(env, fds_to_close, fail_fn);
-
-    // Invalidate the entries in the USAP table.
-    ClearUsapTable();
-
-    // Re-open all remaining open file descriptors so that they aren't shared
-    // with the zygote across a fork.
-    gOpenFdTable->ReopenOrDetach(fail_fn);
-
-    // Turn fdsan back on.
-    android_fdsan_set_error_level(fdsan_error_level);
-
-    // Reset the fd to the unsolicited zygote socket
-    gSystemServerSocketFd = -1;
-  } else {
-    ALOGD("Forked child process %d", pid);
-  }
-
-  // We blocked SIGCHLD prior to a fork, we unblock it here.
-  UnblockSignal(SIGCHLD, fail_fn);
-
-  return pid;
-}
-
 // Create an app data directory over tmpfs overlayed CE / DE storage, and bind mount it
 // from the actual app data directory in data mirror.
 static bool createAndMountAppData(std::string_view package_name,
@@ -1993,9 +1871,10 @@ static void AddUsapTableEntry(pid_t usap_pid, int read_pipe_fd) {
   static int sUsapTableInsertIndex = 0;
 
   int search_index = sUsapTableInsertIndex;
-
   do {
     if (gUsapTable[search_index].SetIfInvalid(usap_pid, read_pipe_fd)) {
+      ++gUsapPoolCount;
+
       // Start our next search right after where we finished this one.
       sUsapTableInsertIndex = (search_index + 1) % gUsapTable.size();
 
@@ -2013,7 +1892,7 @@ static void AddUsapTableEntry(pid_t usap_pid, int read_pipe_fd) {
 /**
  * Invalidates the entry in the USAPTable corresponding to the provided
  * process ID if it is present.  If an entry was removed the USAP pool
- * count is decremented.
+ * count is decremented. May be called from signal handler.
  *
  * @param usap_pid  Process ID of the USAP entry to invalidate
  * @return True if an entry was invalidated; false otherwise
@@ -2089,6 +1968,121 @@ static void UnmountStorageOnInit(JNIEnv* env) {
 
 namespace android {
 
+/**
+ * A failure function used to report fatal errors to the managed runtime.  This
+ * function is often curried with the process name information and then passed
+ * to called functions.
+ *
+ * @param env  Managed runtime environment
+ * @param process_name  A native representation of the process name
+ * @param managed_process_name  A managed representation of the process name
+ * @param msg  The error message to be reported
+ */
+[[noreturn]]
+void zygote::ZygoteFailure(JNIEnv* env,
+                           const char* process_name,
+                           jstring managed_process_name,
+                           const std::string& msg) {
+  std::unique_ptr<ScopedUtfChars> scoped_managed_process_name_ptr = nullptr;
+  if (managed_process_name != nullptr) {
+    scoped_managed_process_name_ptr.reset(new ScopedUtfChars(env, managed_process_name));
+    if (scoped_managed_process_name_ptr->c_str() != nullptr) {
+      process_name = scoped_managed_process_name_ptr->c_str();
+    }
+  }
+
+  const std::string& error_msg =
+      (process_name == nullptr || process_name[0] == '\0') ?
+      msg : StringPrintf("(%s) %s", process_name, msg.c_str());
+
+  env->FatalError(error_msg.c_str());
+  __builtin_unreachable();
+}
+
+// Utility routine to fork a process from the zygote.
+NO_PAC_FUNC
+pid_t zygote::ForkCommon(JNIEnv* env, bool is_system_server,
+                         const std::vector<int>& fds_to_close,
+                         const std::vector<int>& fds_to_ignore,
+                         bool is_priority_fork,
+                         bool purge) {
+  SetSignalHandlers();
+
+  // Curry a failure function.
+  auto fail_fn = std::bind(zygote::ZygoteFailure, env,
+                           is_system_server ? "system_server" : "zygote",
+                           nullptr, _1);
+
+  // Temporarily block SIGCHLD during forks. The SIGCHLD handler might
+  // log, which would result in the logging FDs we close being reopened.
+  // This would cause failures because the FDs are not allowlisted.
+  //
+  // Note that the zygote process is single threaded at this point.
+  BlockSignal(SIGCHLD, fail_fn);
+
+  // Close any logging related FDs before we start evaluating the list of
+  // file descriptors.
+  __android_log_close();
+  AStatsSocket_close();
+
+  // If this is the first fork for this zygote, create the open FD table.  If
+  // it isn't, we just need to check whether the list of open files has changed
+  // (and it shouldn't in the normal case).
+  if (gOpenFdTable == nullptr) {
+    gOpenFdTable = FileDescriptorTable::Create(fds_to_ignore, fail_fn);
+  } else {
+    gOpenFdTable->Restat(fds_to_ignore, fail_fn);
+  }
+
+  android_fdsan_error_level fdsan_error_level = android_fdsan_get_error_level();
+
+  if (purge) {
+    // Purge unused native memory in an attempt to reduce the amount of false
+    // sharing with the child process.  By reducing the size of the libc_malloc
+    // region shared with the child process we reduce the number of pages that
+    // transition to the private-dirty state when malloc adjusts the meta-data
+    // on each of the pages it is managing after the fork.
+    mallopt(M_PURGE, 0);
+  }
+
+  pid_t pid = fork();
+
+  if (pid == 0) {
+    if (is_priority_fork) {
+      setpriority(PRIO_PROCESS, 0, PROCESS_PRIORITY_MAX);
+    } else {
+      setpriority(PRIO_PROCESS, 0, PROCESS_PRIORITY_MIN);
+    }
+
+    // The child process.
+    PAuthKeyChange(env);
+    PreApplicationInit();
+
+    // Clean up any descriptors which must be closed immediately
+    DetachDescriptors(env, fds_to_close, fail_fn);
+
+    // Invalidate the entries in the USAP table.
+    ClearUsapTable();
+
+    // Re-open all remaining open file descriptors so that they aren't shared
+    // with the zygote across a fork.
+    gOpenFdTable->ReopenOrDetach(fail_fn);
+
+    // Turn fdsan back on.
+    android_fdsan_set_error_level(fdsan_error_level);
+
+    // Reset the fd to the unsolicited zygote socket
+    gSystemServerSocketFd = -1;
+  } else {
+    ALOGD("Forked child process %d", pid);
+  }
+
+  // We blocked SIGCHLD prior to a fork, we unblock it here.
+  UnblockSignal(SIGCHLD, fail_fn);
+
+  return pid;
+}
+
 static void com_android_internal_os_Zygote_nativePreApplicationInit(JNIEnv*, jclass) {
   PreApplicationInit();
 }
@@ -2105,7 +2099,8 @@ static jint com_android_internal_os_Zygote_nativeForkAndSpecialize(
     jlong capabilities = CalculateCapabilities(env, uid, gid, gids, is_child_zygote);
 
     if (UNLIKELY(managed_fds_to_close == nullptr)) {
-      ZygoteFailure(env, "zygote", nice_name, "Zygote received a null fds_to_close vector.");
+      zygote::ZygoteFailure(env, "zygote", nice_name,
+                            "Zygote received a null fds_to_close vector.");
     }
 
     std::vector<int> fds_to_close =
@@ -2131,7 +2126,7 @@ static jint com_android_internal_os_Zygote_nativeForkAndSpecialize(
         fds_to_ignore.push_back(gSystemServerSocketFd);
     }
 
-    pid_t pid = ForkCommon(env, false, fds_to_close, fds_to_ignore, true);
+    pid_t pid = zygote::ForkCommon(env, false, fds_to_close, fds_to_ignore, true);
 
     if (pid == 0) {
       SpecializeCommon(env, uid, gid, gids, runtime_flags, rlimits,
@@ -2166,10 +2161,10 @@ static jint com_android_internal_os_Zygote_nativeForkSystemServer(
       fds_to_ignore.push_back(gSystemServerSocketFd);
   }
 
-  pid_t pid = ForkCommon(env, true,
-                         fds_to_close,
-                         fds_to_ignore,
-                         true);
+  pid_t pid = zygote::ForkCommon(env, true,
+                                 fds_to_close,
+                                 fds_to_ignore,
+                                 true);
   if (pid == 0) {
       // System server prcoess does not need data isolation so no need to
       // know pkg_data_info_list.
@@ -2209,58 +2204,74 @@ static jint com_android_internal_os_Zygote_nativeForkSystemServer(
  * ensuring proper file descriptor hygiene.
  *
  * @param env  Managed runtime environment
- * @param read_pipe_fd  The read FD for the USAP reporting pipe.  Manually closed by blastlas
- * in managed code.
+ * @param read_pipe_fd  The read FD for the USAP reporting pipe.  Manually closed by the child
+ * in managed code. -1 indicates none.
  * @param write_pipe_fd  The write FD for the USAP reporting pipe.  Manually closed by the
- * zygote in managed code.
+ * zygote in managed code. -1 indicates none.
  * @param managed_session_socket_fds  A list of anonymous session sockets that must be ignored by
  * the FD hygiene code and automatically "closed" in the new USAP.
+ * @param args_known Arguments for specialization are available; no need to read from a socket
  * @param is_priority_fork  Controls the nice level assigned to the newly created process
- * @return
+ * @return child pid in the parent, 0 in the child
  */
 NO_PAC_FUNC
-static jint com_android_internal_os_Zygote_nativeForkUsap(JNIEnv* env,
-                                                          jclass,
-                                                          jint read_pipe_fd,
-                                                          jint write_pipe_fd,
-                                                          jintArray managed_session_socket_fds,
-                                                          jboolean is_priority_fork) {
-  std::vector<int> fds_to_close(MakeUsapPipeReadFDVector()),
-                   fds_to_ignore(fds_to_close);
-
+static jint com_android_internal_os_Zygote_nativeForkApp(JNIEnv* env,
+                                                         jclass,
+                                                         jint read_pipe_fd,
+                                                         jint write_pipe_fd,
+                                                         jintArray managed_session_socket_fds,
+                                                         jboolean args_known,
+                                                         jboolean is_priority_fork) {
   std::vector<int> session_socket_fds =
       ExtractJIntArray(env, "USAP", nullptr, managed_session_socket_fds)
           .value_or(std::vector<int>());
+  return zygote::forkApp(env, read_pipe_fd, write_pipe_fd, session_socket_fds,
+                            args_known == JNI_TRUE, is_priority_fork == JNI_TRUE, true);
+}
 
-  // The USAP Pool Event FD is created during the initialization of the
-  // USAP pool and should always be valid here.
+NO_PAC_FUNC
+int zygote::forkApp(JNIEnv* env,
+                    int read_pipe_fd,
+                    int write_pipe_fd,
+                    const std::vector<int>& session_socket_fds,
+                    bool args_known,
+                    bool is_priority_fork,
+                    bool purge) {
+
+  std::vector<int> fds_to_close(MakeUsapPipeReadFDVector()),
+                   fds_to_ignore(fds_to_close);
 
   fds_to_close.push_back(gZygoteSocketFD);
-  fds_to_close.push_back(gUsapPoolEventFD);
-  fds_to_close.insert(fds_to_close.end(), session_socket_fds.begin(), session_socket_fds.end());
   if (gSystemServerSocketFd != -1) {
       fds_to_close.push_back(gSystemServerSocketFd);
   }
+  if (args_known) {
+      fds_to_close.push_back(gUsapPoolSocketFD);
+  }
+  fds_to_close.insert(fds_to_close.end(), session_socket_fds.begin(), session_socket_fds.end());
 
-  fds_to_ignore.push_back(gZygoteSocketFD);
   fds_to_ignore.push_back(gUsapPoolSocketFD);
-  fds_to_ignore.push_back(gUsapPoolEventFD);
-  fds_to_ignore.push_back(read_pipe_fd);
-  fds_to_ignore.push_back(write_pipe_fd);
+  fds_to_ignore.push_back(gZygoteSocketFD);
+  if (read_pipe_fd != -1) {
+      fds_to_ignore.push_back(read_pipe_fd);
+  }
+  if (write_pipe_fd != -1) {
+      fds_to_ignore.push_back(write_pipe_fd);
+  }
   fds_to_ignore.insert(fds_to_ignore.end(), session_socket_fds.begin(), session_socket_fds.end());
+
+  if (gUsapPoolEventFD != -1) {
+      fds_to_close.push_back(gUsapPoolEventFD);
+      fds_to_ignore.push_back(gUsapPoolEventFD);
+  }
   if (gSystemServerSocketFd != -1) {
+      if (args_known) {
+          fds_to_close.push_back(gSystemServerSocketFd);
+      }
       fds_to_ignore.push_back(gSystemServerSocketFd);
   }
-
-  pid_t usap_pid = ForkCommon(env, /* is_system_server= */ false, fds_to_close, fds_to_ignore,
-                              is_priority_fork == JNI_TRUE);
-
-  if (usap_pid != 0) {
-    ++gUsapPoolCount;
-    AddUsapTableEntry(usap_pid, read_pipe_fd);
-  }
-
-  return usap_pid;
+  return zygote::ForkCommon(env, /* is_system_server= */ false, fds_to_close,
+                            fds_to_ignore, is_priority_fork == JNI_TRUE, purge);
 }
 
 static void com_android_internal_os_Zygote_nativeAllowFileAcrossFork(
@@ -2374,7 +2385,7 @@ static void com_android_internal_os_Zygote_nativeInitNativeState(JNIEnv* env, jc
    */
 
   if (!SetTaskProfiles(0, {})) {
-    ZygoteFailure(env, "zygote", nullptr, "Zygote SetTaskProfiles failed");
+    zygote::ZygoteFailure(env, "zygote", nullptr, "Zygote SetTaskProfiles failed");
   }
 }
 
@@ -2392,15 +2403,21 @@ static jintArray com_android_internal_os_Zygote_nativeGetUsapPipeFDs(JNIEnv* env
   return managed_usap_fds;
 }
 
+/*
+ * Add the given pid and file descriptor to the Usap table. CriticalNative method.
+ */
+static void com_android_internal_os_Zygote_nativeAddUsapTableEntry(jint pid, jint read_pipe_fd) {
+  AddUsapTableEntry(pid, read_pipe_fd);
+}
+
 /**
- * A JNI wrapper around RemoveUsapTableEntry.
+ * A JNI wrapper around RemoveUsapTableEntry. CriticalNative method.
  *
  * @param env  Managed runtime environment
  * @param usap_pid  Process ID of the USAP entry to invalidate
  * @return  True if an entry was invalidated; false otherwise.
  */
-static jboolean com_android_internal_os_Zygote_nativeRemoveUsapTableEntry(JNIEnv* env, jclass,
-                                                                          jint usap_pid) {
+static jboolean com_android_internal_os_Zygote_nativeRemoveUsapTableEntry(jint usap_pid) {
   return RemoveUsapTableEntry(usap_pid);
 }
 
@@ -2415,7 +2432,8 @@ static jboolean com_android_internal_os_Zygote_nativeRemoveUsapTableEntry(JNIEnv
 static jint com_android_internal_os_Zygote_nativeGetUsapPoolEventFD(JNIEnv* env, jclass) {
   if (gUsapPoolEventFD == -1) {
     if ((gUsapPoolEventFD = eventfd(0, 0)) == -1) {
-      ZygoteFailure(env, "zygote", nullptr, StringPrintf("Unable to create eventfd: %s", strerror(errno)));
+      zygote::ZygoteFailure(env, "zygote", nullptr,
+                            StringPrintf("Unable to create eventfd: %s", strerror(errno)));
     }
   }
 
@@ -2458,12 +2476,12 @@ static void com_android_internal_os_Zygote_nativeEmptyUsapPool(JNIEnv* env, jcla
 }
 
 static void com_android_internal_os_Zygote_nativeBlockSigTerm(JNIEnv* env, jclass) {
-  auto fail_fn = std::bind(ZygoteFailure, env, "usap", nullptr, _1);
+  auto fail_fn = std::bind(zygote::ZygoteFailure, env, "usap", nullptr, _1);
   BlockSignal(SIGTERM, fail_fn);
 }
 
 static void com_android_internal_os_Zygote_nativeUnblockSigTerm(JNIEnv* env, jclass) {
-  auto fail_fn = std::bind(ZygoteFailure, env, "usap", nullptr, _1);
+  auto fail_fn = std::bind(zygote::ZygoteFailure, env, "usap", nullptr, _1);
   UnblockSignal(SIGTERM, fail_fn);
 }
 
@@ -2569,7 +2587,10 @@ static const JNINativeMethod gMethods[] = {
          (void*)com_android_internal_os_Zygote_nativePreApplicationInit},
         {"nativeInstallSeccompUidGidFilter", "(II)V",
          (void*)com_android_internal_os_Zygote_nativeInstallSeccompUidGidFilter},
-        {"nativeForkUsap", "(II[IZ)I", (void*)com_android_internal_os_Zygote_nativeForkUsap},
+        {"nativeForkApp", "(II[IZZ)I", (void*)com_android_internal_os_Zygote_nativeForkApp},
+        // @CriticalNative
+        {"nativeAddUsapTableEntry", "(II)V",
+         (void*)com_android_internal_os_Zygote_nativeAddUsapTableEntry},
         {"nativeSpecializeAppProcess",
          "(II[II[[IILjava/lang/String;Ljava/lang/String;ZLjava/lang/String;Ljava/lang/"
          "String;Z[Ljava/lang/String;[Ljava/lang/String;ZZ)V",
@@ -2578,6 +2599,10 @@ static const JNINativeMethod gMethods[] = {
          (void*)com_android_internal_os_Zygote_nativeInitNativeState},
         {"nativeGetUsapPipeFDs", "()[I",
          (void*)com_android_internal_os_Zygote_nativeGetUsapPipeFDs},
+        // @CriticalNative
+        {"nativeAddUsapTableEntry", "(II)V",
+         (void*)com_android_internal_os_Zygote_nativeAddUsapTableEntry},
+        // @CriticalNative
         {"nativeRemoveUsapTableEntry", "(I)Z",
          (void*)com_android_internal_os_Zygote_nativeRemoveUsapTableEntry},
         {"nativeGetUsapPoolEventFD", "()I",
diff --git a/core/jni/com_android_internal_os_Zygote.h b/core/jni/com_android_internal_os_Zygote.h
new file mode 100644
index 000000000000..d2da91476bc7
--- /dev/null
+++ b/core/jni/com_android_internal_os_Zygote.h
@@ -0,0 +1,78 @@
+/*
+ * Copyright (C) 2008 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _COM_ANDROID_INTERNAL_OS_ZYGOTE_H
+#define _COM_ANDROID_INTERNAL_OS_ZYGOTE_H
+
+#define LOG_TAG "Zygote"
+#define ATRACE_TAG ATRACE_TAG_DALVIK
+
+/* Functions in the callchain during the fork shall not be protected with
+   Armv8.3-A Pointer Authentication, otherwise child will not be able to return. */
+#ifdef __ARM_FEATURE_PAC_DEFAULT
+#ifdef __ARM_FEATURE_BTI_DEFAULT
+#define NO_PAC_FUNC __attribute__((target("branch-protection=bti")))
+#else
+#define NO_PAC_FUNC __attribute__((target("branch-protection=none")))
+#endif /* __ARM_FEATURE_BTI_DEFAULT */
+#else /* !__ARM_FEATURE_PAC_DEFAULT */
+#define NO_PAC_FUNC
+#endif /* __ARM_FEATURE_PAC_DEFAULT */
+
+#include <jni.h>
+#include <vector>
+#include <android-base/stringprintf.h>
+
+#define CREATE_ERROR(...) StringPrintf("%s:%d: ", __FILE__, __LINE__). \
+                              append(StringPrintf(__VA_ARGS__))
+
+namespace android {
+namespace zygote {
+
+NO_PAC_FUNC
+pid_t ForkCommon(JNIEnv* env,bool is_system_server,
+                 const std::vector<int>& fds_to_close,
+                 const std::vector<int>& fds_to_ignore,
+                 bool is_priority_fork,
+                 bool purge = true);
+
+/**
+ * Fork a process. The pipe fds are used for usap communication, or -1 in
+ * other cases. Session_socket_fds are FDs used for zygote communication that must be dealt
+ * with hygienically, but are not otherwise used here. Args_known indicates that the process
+ * will be immediately specialized with arguments that are already known, so no usap
+ * communication is required. Is_priority_fork should be true if this is on the app startup
+ * critical path. Purge specifies that unused pages should be purged before the fork.
+ */
+NO_PAC_FUNC
+int forkApp(JNIEnv* env,
+            int read_pipe_fd,
+            int write_pipe_fd,
+            const std::vector<int>& session_socket_fds,
+            bool args_known,
+            bool is_priority_fork,
+            bool purge);
+
+[[noreturn]]
+void ZygoteFailure(JNIEnv* env,
+                   const char* process_name,
+                   jstring managed_process_name,
+                   const std::string& msg);
+
+}  // namespace zygote
+}  // namespace android
+
+#endif // _COM_ANDROID_INTERNAL_OS_ZYGOTE_
diff --git a/core/jni/com_android_internal_os_ZygoteCommandBuffer.cpp b/core/jni/com_android_internal_os_ZygoteCommandBuffer.cpp
new file mode 100644
index 000000000000..011e8f8f1b8c
--- /dev/null
+++ b/core/jni/com_android_internal_os_ZygoteCommandBuffer.cpp
@@ -0,0 +1,512 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "com_android_internal_os_Zygote.h"
+
+#include <algorithm>
+#include <android-base/logging.h>
+#include <async_safe/log.h>
+#include <cctype>
+#include <chrono>
+#include <core_jni_helpers.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <jni.h>
+#include <nativehelper/JNIHelp.h>
+#include <optional>
+#include <poll.h>
+#include <unistd.h>
+#include <utility>
+#include <utils/misc.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <vector>
+
+namespace android {
+
+using namespace std::placeholders;
+using android::base::StringPrintf;
+using android::zygote::ZygoteFailure;
+
+// WARNING: Knows a little about the wire protocol used to communicate with Zygote.
+// TODO: Fix error handling.
+
+constexpr size_t MAX_COMMAND_BYTES = 12200;
+constexpr size_t NICE_NAME_BYTES = 50;
+
+// A buffer optionally bundled with a file descriptor from which we can fill it.
+// Does not own the file descriptor; destroying a NativeCommandBuffer does not
+// close the descriptor.
+class NativeCommandBuffer {
+ public:
+  NativeCommandBuffer(int sourceFd): mEnd(0), mNext(0), mLinesLeft(0), mFd(sourceFd) {}
+
+  // Read mNext line from mFd, filling mBuffer from file descriptor, as needed.
+  // Return a pair of pointers pointing to the first character, and one past the
+  // mEnd of the line, i.e. at the newline. Returns nothing on failure.
+  template<class FailFn>
+  std::optional<std::pair<char*, char*>> readLine(FailFn fail_fn) {
+    char* result = mBuffer + mNext;
+    while (true) {
+      if (mNext == mEnd) {
+        if (mEnd == MAX_COMMAND_BYTES) {
+          return {};
+        }
+        if (mFd == -1) {
+          fail_fn("ZygoteCommandBuffer.readLine attempted to read from mFd -1");
+        }
+        ssize_t nread = TEMP_FAILURE_RETRY(read(mFd, mBuffer + mEnd, MAX_COMMAND_BYTES - mEnd));
+        if (nread <= 0) {
+          if (nread == 0) {
+            return {};
+          }
+          fail_fn(CREATE_ERROR("session socket read failed: %s", strerror(errno)));
+        } else if (nread == MAX_COMMAND_BYTES - mEnd) {
+          // This is pessimistic by one character, but close enough.
+          fail_fn("ZygoteCommandBuffer overflowed: command too long");
+        }
+        mEnd += nread;
+      }
+      // UTF-8 does not allow newline to occur as part of a multibyte character.
+      char* nl = static_cast<char *>(memchr(mBuffer + mNext, '\n', mEnd - mNext));
+      if (nl == nullptr) {
+        mNext = mEnd;
+      } else {
+        mNext = nl - mBuffer + 1;
+        if (--mLinesLeft < 0) {
+          fail_fn("ZygoteCommandBuffer.readLine attempted to read past mEnd of command");
+        }
+        return std::make_pair(result, nl);
+      }
+    }
+  }
+
+  void reset() {
+    mNext = 0;
+  }
+
+  // Make sure the current command is fully buffered, without reading past the current command.
+  template<class FailFn>
+  void readAllLines(FailFn fail_fn) {
+     while (mLinesLeft > 0) {
+       readLine(fail_fn);
+    }
+  }
+
+  void clear() {
+    // Don't bother to actually clear the buffer; it'll be unmapped in the child anyway.
+    reset();
+    mNiceName[0] = '\0';
+    mEnd = 0;
+  }
+
+  // Insert line into the mBuffer. Checks that the mBuffer is not associated with an mFd.
+  // Implicitly adds newline separators. Allows mBuffer contents to be explicitly set.
+  void insert(const char* line, size_t lineLen) {
+    DCHECK(mFd == -1);
+    CHECK(mEnd + lineLen < MAX_COMMAND_BYTES);
+    strncpy(mBuffer + mEnd, line, lineLen);
+    mBuffer[mEnd + lineLen] = '\n';
+    mEnd += lineLen + 1;
+  }
+
+  // Clear mBuffer, start reading new command, return the number of arguments, leaving mBuffer
+  // positioned at the beginning of first argument. Return 0 on EOF.
+  template<class FailFn>
+  int getCount(FailFn fail_fn) {
+    mLinesLeft = 1;
+    auto line = readLine(fail_fn);
+    if (!line.has_value()) {
+      return 0;
+    }
+    char* countString = line.value().first;  // Newline terminated.
+    long nArgs = atol(countString);
+    if (nArgs <= 0 || nArgs >= MAX_COMMAND_BYTES / 2) {
+      fail_fn(CREATE_ERROR("Unreasonable argument count %ld", nArgs));
+    }
+    mLinesLeft = nArgs;
+    return static_cast<int>(nArgs);
+  }
+
+  // Is the mBuffer a simple fork command?
+  // We disallow request to wrap the child process, child zygotes, anything that
+  // mentions capabilities or requests uid < minUid.
+  // We insist that --setuid and --setgid arguments are explicitly included and that the
+  // command starts with --runtime-args.
+  // Assumes we are positioned at the beginning of the command after the argument count,
+  // and leaves the position at some indeterminate position in the buffer.
+  // As a side effect, this sets mNiceName to a non-empty string, if possible.
+  template<class FailFn>
+  bool isSimpleForkCommand(int minUid, FailFn fail_fn) {
+    if (mLinesLeft <= 0 || mLinesLeft  >= MAX_COMMAND_BYTES / 2) {
+      return false;
+    }
+    static const char* RUNTIME_ARGS = "--runtime-args";
+    static const char* INVOKE_WITH = "--invoke-with";
+    static const char* CHILD_ZYGOTE = "--start-child-zygote";
+    static const char* SETUID = "--setuid=";
+    static const char* SETGID = "--setgid=";
+    static const char* CAPABILITIES = "--capabilities";
+    static const char* NICE_NAME = "--nice-name=";
+    static const size_t RA_LENGTH = strlen(RUNTIME_ARGS);
+    static const size_t IW_LENGTH = strlen(INVOKE_WITH);
+    static const size_t CZ_LENGTH = strlen(CHILD_ZYGOTE);
+    static const size_t SU_LENGTH = strlen(SETUID);
+    static const size_t SG_LENGTH = strlen(SETGID);
+    static const size_t CA_LENGTH = strlen(CAPABILITIES);
+    static const size_t NN_LENGTH = strlen(NICE_NAME);
+
+    bool saw_setuid = false, saw_setgid = false;
+    bool saw_runtime_args = false;
+
+    while (mLinesLeft > 0) {
+      auto read_result = readLine(fail_fn);
+      if (!read_result.has_value()) {
+        return false;
+      }
+      auto [arg_start, arg_end] = read_result.value();
+      if (arg_end - arg_start == RA_LENGTH
+          && strncmp(arg_start, RUNTIME_ARGS, RA_LENGTH) == 0) {
+        saw_runtime_args = true;
+        continue;
+      }
+      if (arg_end - arg_start >= NN_LENGTH
+          && strncmp(arg_start, NICE_NAME, NN_LENGTH) == 0) {
+        size_t name_len = arg_end - (arg_start + NN_LENGTH);
+        size_t copy_len = std::min(name_len, NICE_NAME_BYTES - 1);
+        memcpy(mNiceName, arg_start + NN_LENGTH, copy_len);
+        mNiceName[copy_len] = '\0';
+        continue;
+      }
+      if (arg_end - arg_start == IW_LENGTH
+          && strncmp(arg_start, INVOKE_WITH, IW_LENGTH) == 0) {
+        // This also removes the need for invoke-with security checks here.
+        return false;
+      }
+      if (arg_end - arg_start == CZ_LENGTH
+          && strncmp(arg_start, CHILD_ZYGOTE, CZ_LENGTH) == 0) {
+        return false;
+      }
+      if (arg_end - arg_start >= CA_LENGTH
+          && strncmp(arg_start, CAPABILITIES, CA_LENGTH) == 0) {
+        return false;
+      }
+      if (arg_end - arg_start >= SU_LENGTH
+          && strncmp(arg_start, SETUID, SU_LENGTH) == 0) {
+        int uid = digitsVal(arg_start + SU_LENGTH, arg_end);
+        if (uid < minUid) {
+          return false;
+        }
+        saw_setuid = true;
+        continue;
+      }
+      if (arg_end - arg_start >= SG_LENGTH
+          && strncmp(arg_start, SETGID, SG_LENGTH) == 0) {
+        int gid = digitsVal(arg_start + SG_LENGTH, arg_end);
+        if (gid == -1) {
+          return false;
+        }
+        saw_setgid = true;
+      }
+    }
+    return saw_runtime_args && saw_setuid && saw_setgid;
+  }
+
+  void setFd(int new_fd) {
+    mFd = new_fd;
+  }
+
+  int getFd() const {
+    return mFd;
+  }
+
+  const char* niceNameAddr() const {
+    return mNiceName;
+  }
+
+  // Debug only:
+  void logState() const {
+    ALOGD("mbuffer starts with %c%c, nice name is %s, "
+          "mEnd = %u, mNext = %u, mLinesLeft = %d, mFd = %d",
+          mBuffer[0], (mBuffer[1] == '\n' ? ' ' : mBuffer[1]),
+          niceNameAddr(),
+          static_cast<unsigned>(mEnd), static_cast<unsigned>(mNext),
+          static_cast<int>(mLinesLeft), mFd);
+  }
+
+ private:
+  // Picky version of atoi(). No sign or unexpected characters allowed. Return -1 on failure.
+  static int digitsVal(char* start, char* end) {
+    int result = 0;
+    if (end - start > 6) {
+      return -1;
+    }
+    for (char* dp = start; dp < end; ++dp) {
+      if (*dp < '0' || *dp > '9') {
+        ALOGW("Argument failed integer format check");
+        return -1;
+      }
+      result = 10 * result + (*dp - '0');
+    }
+    return result;
+  }
+
+  uint32_t mEnd;  // Index of first empty byte in the mBuffer.
+  uint32_t mNext;  // Index of first character past last line returned by readLine.
+  int32_t mLinesLeft;  // Lines in current command that haven't yet been read.
+  int mFd;  // Open file descriptor from which we can read more. -1 if none.
+  char mNiceName[NICE_NAME_BYTES];
+  char mBuffer[MAX_COMMAND_BYTES];
+};
+
+static_assert(sizeof(NativeCommandBuffer) < 3 * 4096);
+
+static int buffersAllocd(0);
+
+// Get a new NativeCommandBuffer. Can only be called once between freeNativeBuffer calls,
+// so that only one buffer exists at a time.
+jlong com_android_internal_os_ZygoteCommandBuffer_getNativeBuffer(JNIEnv* env, jclass, jint fd) {
+  CHECK(buffersAllocd == 0);
+  ++buffersAllocd;
+  // MMap explicitly to get it page aligned.
+  void *bufferMem = mmap(NULL, sizeof(NativeCommandBuffer), PROT_READ | PROT_WRITE,
+                         MAP_ANONYMOUS | MAP_PRIVATE | MAP_POPULATE, -1, 0);
+  // Currently we mmap and unmap one for every request handled by the Java code.
+  // That could be improved, but unclear it matters.
+  if (bufferMem == MAP_FAILED) {
+    ZygoteFailure(env, nullptr, nullptr, "Failed to map argument buffer");
+  }
+  return (jlong) new(bufferMem) NativeCommandBuffer(fd);
+}
+
+// Delete native command buffer.
+void com_android_internal_os_ZygoteCommandBuffer_freeNativeBuffer(JNIEnv* env, jclass,
+                                                                  jlong j_buffer) {
+  CHECK(buffersAllocd == 1);
+  NativeCommandBuffer* n_buffer = reinterpret_cast<NativeCommandBuffer*>(j_buffer);
+  n_buffer->~NativeCommandBuffer();
+  if (munmap(n_buffer, sizeof(NativeCommandBuffer)) != 0) {
+    ZygoteFailure(env, nullptr, nullptr, "Failed to unmap argument buffer");
+  }
+  --buffersAllocd;
+}
+
+// Clear the buffer, read the line containing the count, and return the count.
+jint com_android_internal_os_ZygoteCommandBuffer_nativeGetCount(JNIEnv* env, jclass,
+                                                                jlong j_buffer) {
+  NativeCommandBuffer* n_buffer = reinterpret_cast<NativeCommandBuffer*>(j_buffer);
+  auto fail_fn = std::bind(ZygoteFailure, env, nullptr, nullptr, _1);
+  return n_buffer->getCount(fail_fn);
+}
+
+// Explicitly insert a string as the last line (argument) of the buffer.
+void com_android_internal_os_ZygoteCommandBuffer_insert(JNIEnv* env, jclass, jlong j_buffer,
+                                                        jstring line) {
+  NativeCommandBuffer* n_buffer = reinterpret_cast<NativeCommandBuffer*>(j_buffer);
+  size_t lineLen = static_cast<size_t>(env->GetStringUTFLength(line));
+  const char* cstring = env->GetStringUTFChars(line, NULL);
+  n_buffer->insert(cstring, lineLen);
+  env->ReleaseStringUTFChars(line, cstring);
+}
+
+// Read a line from the buffer, refilling as necessary.
+jstring com_android_internal_os_ZygoteCommandBuffer_nativeNextArg(JNIEnv* env, jclass,
+                                                                  jlong j_buffer) {
+  NativeCommandBuffer* n_buffer = reinterpret_cast<NativeCommandBuffer*>(j_buffer);
+  auto fail_fn = std::bind(ZygoteFailure, env, n_buffer->niceNameAddr(), nullptr, _1);
+  auto line = n_buffer->readLine(fail_fn);
+  if (!line.has_value()) {
+    fail_fn("Incomplete zygote command");
+  }
+  auto [cresult, endp] = line.value();
+  // OK to temporarily clobber the buffer, since this is not thread safe, and we're modifying
+  // the buffer anyway.
+  *endp = '\0';
+  jstring result = env->NewStringUTF(cresult);
+  *endp = '\n';
+  return result;
+}
+
+// Read all lines from the current command into the buffer, and then reset the buffer, so
+// we will start reading again at the beginning of the command, starting with the argument
+// count. And we don't need access to the fd to do so.
+void com_android_internal_os_ZygoteCommandBuffer_nativeReadFullyAndReset(JNIEnv* env, jclass,
+                                                                         jlong j_buffer) {
+  NativeCommandBuffer* n_buffer = reinterpret_cast<NativeCommandBuffer*>(j_buffer);
+  auto fail_fn = std::bind(ZygoteFailure, env, n_buffer->niceNameAddr(), nullptr, _1);
+  n_buffer->readAllLines(fail_fn);
+  n_buffer->reset();
+}
+
+// Fork a child as specified by the current command buffer, and refill the command
+// buffer from the given socket. So long as the result is another simple fork command,
+// repeat this process.
+// It must contain a fork command, which is currently restricted not to fork another
+// zygote or involve a wrapper process.
+// The initial buffer should be partially or entirely read; we read it fully and reset it.
+// When we return, the buffer contains the command we couldn't handle, and has been reset().
+// We return false in the parent when we see a command we didn't understand, and thus the
+// command in the buffer still needs to be executed.
+// We return true in each child.
+// We only process fork commands if the peer uid matches expected_uid.
+// For every fork command after the first, we check that the requested uid is at
+// least minUid.
+NO_PAC_FUNC
+jboolean com_android_internal_os_ZygoteCommandBuffer_nativeForkRepeatedly(
+            JNIEnv* env,
+            jclass,
+            jlong j_buffer,
+            jint zygote_socket_fd,
+            jint expected_uid,
+            jint minUid,
+            jstring managed_nice_name) {
+
+  NativeCommandBuffer* n_buffer = reinterpret_cast<NativeCommandBuffer*>(j_buffer);
+  int session_socket = n_buffer->getFd();
+  std::vector<int> session_socket_fds {session_socket};
+  auto fail_fn_1 = std::bind(ZygoteFailure, env, static_cast<const char*>(nullptr),
+                             static_cast<jstring>(managed_nice_name), _1);
+  // This binds to the nice name address; the actual names are updated by isSimpleForkCommand:
+  auto fail_fn_n = std::bind(ZygoteFailure, env, n_buffer->niceNameAddr(),
+                             static_cast<jstring>(nullptr), _1);
+  auto fail_fn_z = std::bind(ZygoteFailure, env, "zygote", nullptr, _1);
+
+  struct pollfd fd_structs[2];
+  static const int ZYGOTE_IDX = 0;
+  static const int SESSION_IDX = 1;
+  fd_structs[ZYGOTE_IDX].fd = zygote_socket_fd;
+  fd_structs[ZYGOTE_IDX].events = POLLIN;
+  fd_structs[SESSION_IDX].fd = session_socket;
+  fd_structs[SESSION_IDX].events = POLLIN;
+
+  struct timeval timeout;
+  socklen_t timeout_size = sizeof timeout;
+  if (getsockopt(session_socket, SOL_SOCKET, SO_RCVTIMEO, &timeout, &timeout_size) != 0) {
+    fail_fn_z("Failed to retrieve session socket timeout");
+  }
+
+  struct ucred credentials;
+  socklen_t cred_size = sizeof credentials;
+  if (getsockopt(n_buffer->getFd(), SOL_SOCKET, SO_PEERCRED, &credentials, &cred_size) == -1
+      || cred_size != sizeof credentials) {
+    fail_fn_1("ForkMany failed to get initial credentials, %s", strerror(errno));
+  }
+
+  bool first_time = true;
+  do {
+    if (credentials.uid != expected_uid) {
+      return JNI_FALSE;
+    }
+    n_buffer->readAllLines(first_time ? fail_fn_1 : fail_fn_n);
+    n_buffer->reset();
+    int pid = zygote::forkApp(env, /* no pipe FDs */ -1, -1, session_socket_fds,
+                              /*args_known=*/ true, /*is_priority_fork=*/ true,
+                              /*purge=*/ first_time);
+    if (pid == 0) {
+      return JNI_TRUE;
+    }
+    // We're in the parent. Write big-endian pid, followed by a boolean.
+    char pid_buf[5];
+    int tmp_pid = pid;
+    for (int i = 3; i >= 0; --i) {
+      pid_buf[i] = tmp_pid & 0xff;
+      tmp_pid >>= 8;
+    }
+    pid_buf[4] = 0;  // Process is not wrapped.
+    int res = write(session_socket, pid_buf, 5);
+    if (res != 5) {
+      if (res == -1) {
+        (first_time ? fail_fn_1 : fail_fn_n)
+            (CREATE_ERROR("Pid write error %d: %s", errno, strerror(errno)));
+      } else {
+        (first_time ? fail_fn_1 : fail_fn_n)
+            (CREATE_ERROR("Write unexpectedly returned short: %d < 5", res));
+      }
+    }
+    // Clear buffer and get count from next command.
+    n_buffer->clear();
+    for (;;) {
+      // Poll isn't strictly necessary for now. But without it, disconnect is hard to detect.
+      int poll_res = TEMP_FAILURE_RETRY(poll(fd_structs, 2, -1 /* infinite timeout */));
+      if ((fd_structs[SESSION_IDX].revents & POLLIN) != 0) {
+        if (n_buffer->getCount(fail_fn_z) != 0) {
+          break;
+        }  // else disconnected;
+      } else if (poll_res == 0 || (fd_structs[ZYGOTE_IDX].revents & POLLIN) == 0) {
+        fail_fn_z(
+            CREATE_ERROR("Poll returned with no descriptors ready! Poll returned %d", poll_res));
+      }
+      // We've now seen either a disconnect or connect request.
+      close(session_socket);
+      int new_fd = accept(zygote_socket_fd, nullptr, nullptr);
+      if (new_fd == -1) {
+        fail_fn_z("Accept(%d) failed: %s", zygote_socket_fd, strerror(errno));
+      }
+      if (new_fd != session_socket) {
+          // Move new_fd back to the old value, so that we don't have to change Java-level data
+          // structures to reflect a change. This implicitly closes the old one.
+          if (dup2(new_fd, session_socket) != session_socket) {
+            fail_fn_z(CREATE_ERROR("Failed to move fd %d to %d: %s",
+                                   new_fd, session_socket, strerror(errno)));
+          }
+          close(new_fd);
+      }
+      // If we ever return, we effectively reuse the old Java ZygoteConnection.
+      // None of its state needs to change.
+      if (setsockopt(session_socket, SOL_SOCKET, SO_RCVTIMEO, &timeout, timeout_size) != 0) {
+        fail_fn_z(CREATE_ERROR("Failed to set receive timeout for socket %d: %s",
+                               session_socket, strerror(errno)));
+      }
+      if (setsockopt(session_socket, SOL_SOCKET, SO_SNDTIMEO, &timeout, timeout_size) != 0) {
+        fail_fn_z(CREATE_ERROR("Failed to set send timeout for socket %d: %s",
+                               session_socket, strerror(errno)));
+      }
+      if (getsockopt(session_socket, SOL_SOCKET, SO_PEERCRED, &credentials, &cred_size) == -1) {
+        fail_fn_z(CREATE_ERROR("ForkMany failed to get credentials: %s", strerror(errno)));
+      }
+      if (cred_size != sizeof credentials) {
+        fail_fn_z(CREATE_ERROR("ForkMany credential size = %d, should be %d",
+                               cred_size, static_cast<int>(sizeof credentials)));
+      }
+    }
+    first_time = false;
+  } while (n_buffer->isSimpleForkCommand(minUid, fail_fn_n));
+  ALOGW("forkRepeatedly terminated due to non-simple command");
+  n_buffer->logState();
+  n_buffer->reset();
+  return JNI_FALSE;
+}
+
+#define METHOD_NAME(m) com_android_internal_os_ZygoteCommandBuffer_ ## m
+
+static const JNINativeMethod gMethods[] = {
+        {"getNativeBuffer", "(I)J", (void *) METHOD_NAME(getNativeBuffer)},
+        {"freeNativeBuffer", "(J)V", (void *) METHOD_NAME(freeNativeBuffer)},
+        {"insert", "(JLjava/lang/String;)V", (void *) METHOD_NAME(insert)},
+        {"nativeNextArg", "(J)Ljava/lang/String;", (void *) METHOD_NAME(nativeNextArg)},
+        {"nativeReadFullyAndReset", "(J)V", (void *) METHOD_NAME(nativeReadFullyAndReset)},
+        {"nativeGetCount", "(J)I", (void *) METHOD_NAME(nativeGetCount)},
+        {"nativeForkRepeatedly", "(JIIILjava/lang/String;)Z",
+          (void *) METHOD_NAME(nativeForkRepeatedly)},
+};
+
+int register_com_android_internal_os_ZygoteCommandBuffer(JNIEnv* env) {
+  return RegisterMethodsOrDie(env, "com/android/internal/os/ZygoteCommandBuffer", gMethods,
+                              NELEM(gMethods));
+}
+
+}  // namespace android