ART: Fix OOME case in proxy dispatch

Even just creating the java.lang.reflect.Method to pass to the
handler may throw an exception.

Add coverage to run-test 044.

Bug: 68817306
Test: m test-art-host
Change-Id: Iacf8fd679b9e8a81ff7bf7d5f6227e875ab10518

4 files changed, 94 insertions, 0 deletions

diff --git a/test/044-proxy/expected.txt b/test/044-proxy/expected.txt
index 63a46200d9..eed7b7ea6d 100644
--- a/test/044-proxy/expected.txt
+++ b/test/044-proxy/expected.txt
@@ -97,3 +97,4 @@ JNI_OnLoad called
 callback
 Found constructor.
 Found constructors with 0 exceptions
+Received OOME
diff --git a/test/044-proxy/run b/test/044-proxy/run
new file mode 100644
index 0000000000..4a322f3323
--- /dev/null
+++ b/test/044-proxy/run
@@ -0,0 +1,18 @@
+#!/bin/bash
+#
+# Copyright (C) 2017 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Use a smaller heap so it's easier to fill up.
+exec ${RUN} $@ --runtime-option -Xmx4m
diff --git a/test/044-proxy/src/Main.java b/test/044-proxy/src/Main.java
index 9dadb7c6ea..e44c122e3d 100644
--- a/test/044-proxy/src/Main.java
+++ b/test/044-proxy/src/Main.java
@@ -32,6 +32,7 @@ public class Main {
         FloatSelect.main(null);
         NativeProxy.main(args);
         ConstructorProxy.main();
+        OOMEOnDispatch.main(args);
     }
 
     // The following code maps from the actual proxy class names (eg $Proxy2) to their test output
diff --git a/test/044-proxy/src/OOMEOnDispatch.java b/test/044-proxy/src/OOMEOnDispatch.java
new file mode 100644
index 0000000000..94f267980d
--- /dev/null
+++ b/test/044-proxy/src/OOMEOnDispatch.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.util.ArrayList;
+
+/**
+ * Ensure that one can dispatch without aborting when the heap is full.
+ */
+public class OOMEOnDispatch implements InvocationHandler {
+
+    static ArrayList<Object> storage = new ArrayList<>(100000);
+
+    public static void main(String[] args) {
+        InvocationHandler handler = new OOMEOnDispatch();
+        OOMEInterface inf = (OOMEInterface)Proxy.newProxyInstance(
+                OOMEInterface.class.getClassLoader(), new Class[] { OOMEInterface.class },
+                handler);
+
+        int l = 1024 * 1024;
+        while (l > 8) {
+          try {
+            storage.add(new byte[l]);
+          } catch (OutOfMemoryError e) {
+            l = l/2;
+          }
+        }
+        // Have an extra run with the exact size of Method objects. The above loop should have
+        // filled with enough large objects for simplicity and speed, but ensure exact allocation
+        // size.
+        final int methodAsByteArrayLength = 40 - 12;  // Method size - byte array overhead.
+        for (;;) {
+          try {
+            storage.add(new byte[methodAsByteArrayLength]);
+          } catch (OutOfMemoryError e) {
+            break;
+          }
+        }
+
+        try {
+            inf.foo();
+            storage.clear();
+            System.out.println("Did not receive OOME!");
+        } catch (OutOfMemoryError oome) {
+            storage.clear();
+            System.out.println("Received OOME");
+        }
+    }
+
+    public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+        storage.clear();
+        System.out.println("Should not have reached OOMEOnDispatch.invoke!");
+        return null;
+    }
+}
+
+interface OOMEInterface {
+    public void foo();
+}