Add NetworkStack utilities for reading text

The utilities will be useful for:
 - Implementing the captive portal API
 - Implementing generic probes based on regular expressions

Test: atest NetworkStackTests
Bug: 139269711
Change-Id: I17a9564033f985af9061534f5cffcc8a0e70f9ed

2 files changed, 111 insertions, 10 deletions

diff --git a/src/com/android/server/connectivity/NetworkMonitor.java b/src/com/android/server/connectivity/NetworkMonitor.java
index 6fc146d..9ee4cd9 100644
--- a/src/com/android/server/connectivity/NetworkMonitor.java
+++ b/src/com/android/server/connectivity/NetworkMonitor.java
@@ -153,11 +153,15 @@ import com.android.networkstack.netlink.TcpSocketTracker;
 import com.android.networkstack.util.DnsUtils;
 
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.net.HttpURLConnection;
 import java.net.InetAddress;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.UnknownHostException;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -171,6 +175,8 @@ import java.util.UUID;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /**
  * {@hide}
@@ -1810,15 +1816,10 @@ public class NetworkMonitor extends StateMachine {
         final int oldTag = TrafficStats.getAndSetThreadStatsTag(
                 TrafficStatsConstants.TAG_SYSTEM_PROBE);
         try {
-            urlConnection = (HttpURLConnection) mCleartextDnsNetwork.openConnection(url);
-            urlConnection.setInstanceFollowRedirects(probeType == ValidationProbeEvent.PROBE_PAC);
-            urlConnection.setConnectTimeout(SOCKET_TIMEOUT_MS);
-            urlConnection.setReadTimeout(SOCKET_TIMEOUT_MS);
-            urlConnection.setRequestProperty("Connection", "close");
-            urlConnection.setUseCaches(false);
-            if (mCaptivePortalUserAgent != null) {
-                urlConnection.setRequestProperty("User-Agent", mCaptivePortalUserAgent);
-            }
+            // Follow redirects for PAC probes as such probes verify connectivity by fetching the
+            // PAC proxy file, which may be configured behind a redirect.
+            final boolean followRedirect = probeType == ValidationProbeEvent.PROBE_PAC;
+            urlConnection = makeProbeConnection(url, followRedirect);
             // cannot read request header after connection
             String requestHeader = urlConnection.getRequestProperties().toString();
 
@@ -1886,6 +1887,61 @@ public class NetworkMonitor extends StateMachine {
         }
     }
 
+    private HttpURLConnection makeProbeConnection(URL url, boolean followRedirects)
+            throws IOException {
+        final HttpURLConnection conn = (HttpURLConnection) mCleartextDnsNetwork.openConnection(url);
+        conn.setInstanceFollowRedirects(followRedirects);
+        conn.setConnectTimeout(SOCKET_TIMEOUT_MS);
+        conn.setReadTimeout(SOCKET_TIMEOUT_MS);
+        conn.setRequestProperty("Connection", "close");
+        conn.setUseCaches(false);
+        if (mCaptivePortalUserAgent != null) {
+            conn.setRequestProperty("User-Agent", mCaptivePortalUserAgent);
+        }
+        return conn;
+    }
+
+    @VisibleForTesting
+    @NonNull
+    protected static String readAsString(InputStream is, int maxLength, Charset charset)
+            throws IOException {
+        final InputStreamReader reader = new InputStreamReader(is, charset);
+        final char[] buffer = new char[1000];
+        final StringBuilder builder = new StringBuilder();
+        int totalReadLength = 0;
+        while (totalReadLength < maxLength) {
+            final int availableLength = Math.min(maxLength - totalReadLength, buffer.length);
+            final int currentLength = reader.read(buffer, 0, availableLength);
+            if (currentLength < 0) break; // EOF
+
+            totalReadLength += currentLength;
+            builder.append(buffer, 0, currentLength);
+        }
+        return builder.toString();
+    }
+
+    /**
+     * Attempt to extract the {@link Charset} of the response from its Content-Type header.
+     *
+     * <p>If the {@link Charset} cannot be extracted, UTF-8 is returned by default.
+     */
+    @VisibleForTesting
+    @NonNull
+    protected static Charset extractCharset(@Nullable String contentTypeHeader) {
+        if (contentTypeHeader == null) return StandardCharsets.UTF_8;
+        // See format in https://tools.ietf.org/html/rfc7231#section-3.1.1.1
+        final Pattern charsetPattern = Pattern.compile("; *charset=\"?([^ ;\"]+)\"?",
+                Pattern.CASE_INSENSITIVE);
+        final Matcher matcher = charsetPattern.matcher(contentTypeHeader);
+        if (!matcher.find()) return StandardCharsets.UTF_8;
+
+        try {
+            return Charset.forName(matcher.group(1));
+        } catch (IllegalArgumentException e) {
+            return StandardCharsets.UTF_8;
+        }
+    }
+
     private CaptivePortalProbeResult sendParallelHttpProbes(
             ProxyInfo proxy, URL httpsUrl, URL httpUrl) {
         // Number of probes to wait for. If a probe completes with a conclusive answer
diff --git a/tests/unit/src/com/android/server/connectivity/NetworkMonitorTest.java b/tests/unit/src/com/android/server/connectivity/NetworkMonitorTest.java
index b3f7934..e2c0b04 100644
--- a/tests/unit/src/com/android/server/connectivity/NetworkMonitorTest.java
+++ b/tests/unit/src/com/android/server/connectivity/NetworkMonitorTest.java
@@ -47,6 +47,7 @@ import static com.android.networkstack.apishim.ConstantsShim.KEY_NETWORK_VALIDAT
 import static com.android.networkstack.apishim.ConstantsShim.KEY_TCP_METRICS_COLLECTION_PERIOD_MILLIS;
 import static com.android.networkstack.apishim.ConstantsShim.KEY_TCP_PACKET_FAIL_RATE;
 import static com.android.networkstack.util.DnsUtils.PRIVATE_DNS_PROBE_HOST_SUFFIX;
+import static com.android.server.connectivity.NetworkMonitor.extractCharset;
 
 import static junit.framework.Assert.assertEquals;
 import static junit.framework.Assert.assertFalse;
@@ -138,11 +139,14 @@ import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.lang.reflect.Constructor;
 import java.net.HttpURLConnection;
 import java.net.InetAddress;
 import java.net.URL;
 import java.net.UnknownHostException;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -448,7 +452,6 @@ public class NetworkMonitorTest {
     @After
     public void tearDown() {
         mFakeDns.clearAll();
-        assertTrue(mCreatedNetworkMonitors.size() > 0);
         // Make a local copy of mCreatedNetworkMonitors because during the iteration below,
         // WrappedNetworkMonitor#onQuitting will delete elements from it on the handler threads.
         WrappedNetworkMonitor[] networkMonitors = mCreatedNetworkMonitors.toArray(
@@ -1326,6 +1329,48 @@ public class NetworkMonitorTest {
                 eq(TEST_REDIRECT_URL), anyLong(), argThat(getNotifyNetworkTestedBundleMatcher()));
     }
 
+    @Test
+    public void testExtractCharset() {
+        assertEquals(StandardCharsets.UTF_8, extractCharset(null));
+        assertEquals(StandardCharsets.UTF_8, extractCharset("text/html;charset=utf-8"));
+        assertEquals(StandardCharsets.UTF_8, extractCharset("text/html;charset=UtF-8"));
+        assertEquals(StandardCharsets.UTF_8, extractCharset("text/html; Charset=\"utf-8\""));
+        assertEquals(StandardCharsets.UTF_8, extractCharset("image/png"));
+        assertEquals(StandardCharsets.UTF_8, extractCharset("Text/HTML;"));
+        assertEquals(StandardCharsets.UTF_8, extractCharset("multipart/form-data; boundary=-aa*-"));
+        assertEquals(StandardCharsets.UTF_8, extractCharset("text/plain;something=else"));
+        assertEquals(StandardCharsets.UTF_8, extractCharset("text/plain;charset=ImNotACharset"));
+
+        assertEquals(StandardCharsets.ISO_8859_1, extractCharset("text/plain; CharSeT=ISO-8859-1"));
+        assertEquals(Charset.forName("Shift_JIS"), extractCharset("text/plain;charset=Shift_JIS"));
+        assertEquals(Charset.forName("Windows-1251"), extractCharset(
+                "text/plain;charset=Windows-1251 ; somethingelse"));
+    }
+
+    @Test
+    public void testReadAsString() throws IOException {
+        final String repeatedString = "1aテスト-?";
+        // Infinite stream repeating characters
+        class TestInputStream extends InputStream {
+            private final byte[] mBytes = repeatedString.getBytes(StandardCharsets.UTF_8);
+            private int mPosition = -1;
+
+            @Override
+            public int read() {
+                mPosition = (mPosition + 1) % mBytes.length;
+                return mBytes[mPosition];
+            }
+        }
+
+        final String readString = NetworkMonitor.readAsString(new TestInputStream(),
+                1500 /* maxLength */, StandardCharsets.UTF_8);
+
+        assertEquals(1500, readString.length());
+        for (int i = 0; i < readString.length(); i++) {
+            assertEquals(repeatedString.charAt(i % repeatedString.length()), readString.charAt(i));
+        }
+    }
+
     private void makeDnsTimeoutEvent(WrappedNetworkMonitor wrappedMonitor, int count) {
         for (int i = 0; i < count; i++) {
             wrappedMonitor.getDnsStallDetector().accumulateConsecutiveDnsTimeoutCount(