Matches the URL content by regular expression

This patch provides a way to configure the regular expression
which is used for matching the URL content. Once the result is
matching, then NetworkMonitor will treat the validation result
as fail or success.

Bug: 141406258
Test: 1. Build pass
      2. atest NetworkStackTests

Merged-In: I77747b34fad895565d42ea4c017759c256d61489
Change-Id: I77747b34fad895565d42ea4c017759c256d61489

4 files changed, 247 insertions, 4 deletions

diff --git a/res/values/config.xml b/res/values/config.xml
index 8bd8eff..fdbc9db 100644
--- a/res/values/config.xml
+++ b/res/values/config.xml
@@ -67,4 +67,22 @@
          could result in valid captive portals being incorrectly classified as having no
          connectivity.-->
     <bool name="config_force_dns_probe_private_ip_no_internet">false</bool>
+
+    <!-- Define the min and max range of the content-length that should be in the HTTP response
+         header of probe responses for the validation success/failed regexp to be used. The RegExp
+         will be used to match the probe response content when the content-length is inside this
+         interval(Strictly greater than the config_min_matches_http_content_length and strictly
+         smaller than the config_max_matches_http_content_length). When the content-length is out of
+         this interval, the RegExp will not be used. -->
+    <integer name="config_min_matches_http_content_length">0</integer>
+    <integer name="config_max_matches_http_content_length">0</integer>
+    <!-- A regular expression to match the content of a network validation probe.
+         Treat the network validation as failed when the content matches the
+         config_network_validation_failed_content_regexp and treat the network validation as success
+         when the content matches the config_network_validation_success_content_regexp. If the
+         content matches both of the config_network_validation_failed_content_regexp and
+         the config_network_validation_success_content_regexp, the result will be considered as
+         failed. -->
+    <string name="config_network_validation_failed_content_regexp" translatable="false"></string>
+    <string name="config_network_validation_success_content_regexp" translatable="false"></string>
 </resources>
diff --git a/res/values/overlayable.xml b/res/values/overlayable.xml
index ed86814..7dcd663 100644
--- a/res/values/overlayable.xml
+++ b/res/values/overlayable.xml
@@ -18,8 +18,27 @@
         <policy type="product|system|vendor">
             <!-- Configuration values for NetworkMonitor -->
             <item type="integer" name="config_captive_portal_dns_probe_timeout"/>
+            <!-- Define the min and max range of the content-length that should be in the HTTP
+                 response header of probe responses for the validation success/failed regexp to be
+                 used. The RegExp will be used to match the probe response content when the
+                 content-length is inside this interval(Strictly greater than the
+                 config_min_matches_http_content_length and strictly smaller than the
+                 config_max_matches_http_content_length). When the content-length is out of this
+                 interval, the RegExp will not be used. -->
+            <item type="integer" name="config_min_matches_http_content_length"/>
+            <item type="integer" name="config_max_matches_http_content_length"/>
             <item type="string" name="config_captive_portal_http_url"/>
             <item type="string" name="config_captive_portal_https_url"/>
+            <!-- A regular expression to match the content of a network validation probe.
+                 Treat the network validation as failed when the content matches the
+                 config_network_validation_failed_content_regexp and treat the network validation
+                 as success when the content matches the
+                 config_network_validation_success_content_regexp. If the content matches both of
+                 the config_network_validation_failed_content_regexp and the
+                 config_network_validation_success_content_regexp, the result will be considered as
+                 failed. -->
+            <item type="string" name="config_network_validation_failed_content_regexp"/>
+            <item type="string" name="config_network_validation_success_content_regexp"/>
             <item type="array" name="config_captive_portal_http_urls"/>
             <item type="array" name="config_captive_portal_https_urls"/>
             <item type="array" name="config_captive_portal_fallback_urls"/>
diff --git a/src/com/android/server/connectivity/NetworkMonitor.java b/src/com/android/server/connectivity/NetworkMonitor.java
index 0540258..4aba4f9 100755
--- a/src/com/android/server/connectivity/NetworkMonitor.java
+++ b/src/com/android/server/connectivity/NetworkMonitor.java
@@ -142,6 +142,7 @@ import android.util.Pair;
 
 import androidx.annotation.ArrayRes;
 import androidx.annotation.BoolRes;
+import androidx.annotation.IntegerRes;
 import androidx.annotation.NonNull;
 import androidx.annotation.Nullable;
 import androidx.annotation.StringRes;
@@ -167,6 +168,7 @@ import com.android.server.NetworkStackService.NetworkStackServiceManager;
 import org.json.JSONException;
 import org.json.JSONObject;
 
+import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
@@ -193,6 +195,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
 
 /**
  * {@hide}
@@ -1504,7 +1507,7 @@ public class NetworkMonitor extends StateMachine {
     @VisibleForTesting
     protected Context getContextByMccIfNoSimCardOrDefault() {
         final boolean useNeighborResource =
-                getResBooleanConfig(mContext, R.bool.config_no_sim_card_uses_neighbor_mcc);
+                getResBooleanConfig(mContext, R.bool.config_no_sim_card_uses_neighbor_mcc, false);
         if (!useNeighborResource
                 || TelephonyManager.SIM_STATE_READY == mTelephonyManager.getSimState()) {
             return mContext;
@@ -1552,13 +1555,41 @@ public class NetworkMonitor extends StateMachine {
     }
 
     @VisibleForTesting
-    protected boolean getResBooleanConfig(@NonNull final Context context,
-            @BoolRes int configResource) {
+    boolean getResBooleanConfig(@NonNull final Context context,
+            @BoolRes int configResource, final boolean defaultValue) {
         final Resources res = context.getResources();
         try {
             return res.getBoolean(configResource);
         } catch (Resources.NotFoundException e) {
-            return false;
+            return defaultValue;
+        }
+    }
+
+    /**
+     * Gets integer config from resources.
+     */
+    @VisibleForTesting
+    int getResIntConfig(@NonNull final Context context,
+            @IntegerRes final int configResource, final int defaultValue) {
+        final Resources res = context.getResources();
+        try {
+            return res.getInteger(configResource);
+        } catch (Resources.NotFoundException e) {
+            return defaultValue;
+        }
+    }
+
+    /**
+     * Gets string config from resources.
+     */
+    @VisibleForTesting
+    String getResStringConfig(@NonNull final Context context,
+            @StringRes final int configResource, @Nullable final String defaultValue) {
+        final Resources res = context.getResources();
+        try {
+            return res.getString(configResource);
+        } catch (Resources.NotFoundException e) {
+            return defaultValue;
         }
     }
 
@@ -1999,6 +2030,24 @@ public class NetworkMonitor extends StateMachine {
                                 "Empty 200 response interpreted as failed response.");
                         httpResponseCode = CaptivePortalProbeResult.FAILED_CODE;
                     }
+                } else if (matchesHttpContentLength(contentLength)) {
+                    final InputStream is = new BufferedInputStream(urlConnection.getInputStream());
+                    final String content = readAsString(is, (int) contentLength,
+                            extractCharset(urlConnection.getContentType()));
+                    if (matchesHttpContent(content,
+                            R.string.config_network_validation_failed_content_regexp)) {
+                        httpResponseCode = CaptivePortalProbeResult.FAILED_CODE;
+                    } else if (matchesHttpContent(content,
+                            R.string.config_network_validation_success_content_regexp)) {
+                        httpResponseCode = CaptivePortalProbeResult.SUCCESS_CODE;
+                    }
+
+                    if (httpResponseCode != 200) {
+                        validationLog(probeType, url, "200 response with Content-length ="
+                                + contentLength + ", content matches custom regexp, interpreted"
+                                + " as " + httpResponseCode
+                                + " response.");
+                    }
                 } else if (contentLength <= 4) {
                     // Consider 200 response with "Content-length <= 4" to not be a captive
                     // portal. There's no point in considering this a captive portal as the
@@ -2029,6 +2078,34 @@ public class NetworkMonitor extends StateMachine {
         }
     }
 
+    @VisibleForTesting
+    boolean matchesHttpContent(final String content, @StringRes final int configResource) {
+        final String resString = getResStringConfig(mContext, configResource, "");
+        try {
+            return content.matches(resString);
+        } catch (PatternSyntaxException e) {
+            Log.e(TAG, "Pattern syntax exception occurs when matching the resource=" + resString,
+                    e);
+            return false;
+        }
+    }
+
+    @VisibleForTesting
+    boolean matchesHttpContentLength(final long contentLength) {
+        // Consider that the Resources#getInteger() is returning an integer, so if the contentLength
+        // is lower or equal to 0 or higher than Integer.MAX_VALUE, then it's an invalid value.
+        if (contentLength <= 0) return false;
+        if (contentLength > Integer.MAX_VALUE) {
+            logw("matchesHttpContentLength : Get invalid contentLength = " + contentLength);
+            return false;
+        }
+        return (contentLength > getResIntConfig(mContext,
+                R.integer.config_min_matches_http_content_length, Integer.MAX_VALUE)
+                &&
+                contentLength < getResIntConfig(mContext,
+                R.integer.config_max_matches_http_content_length, 0));
+    }
+
     private HttpURLConnection makeProbeConnection(URL url, boolean followRedirects)
             throws IOException {
         final HttpURLConnection conn = (HttpURLConnection) mCleartextDnsNetwork.openConnection(url);
diff --git a/tests/unit/src/com/android/server/connectivity/NetworkMonitorTest.java b/tests/unit/src/com/android/server/connectivity/NetworkMonitorTest.java
index b0efa33..6a0aa8a 100644
--- a/tests/unit/src/com/android/server/connectivity/NetworkMonitorTest.java
+++ b/tests/unit/src/com/android/server/connectivity/NetworkMonitorTest.java
@@ -592,6 +592,89 @@ public class NetworkMonitorTest {
     }
 
     @Test
+    public void testMatchesHttpContent() throws Exception {
+        final WrappedNetworkMonitor wnm = makeNotMeteredNetworkMonitor();
+        doReturn("[\\s\\S]*line2[\\s\\S]*").when(mResources).getString(
+                R.string.config_network_validation_failed_content_regexp);
+        assertTrue(wnm.matchesHttpContent("This is line1\nThis is line2\nThis is line3",
+                R.string.config_network_validation_failed_content_regexp));
+        assertFalse(wnm.matchesHttpContent("hello",
+                R.string.config_network_validation_failed_content_regexp));
+        // Set an invalid regex and expect to get the false even though the regex is the same as the
+        // content.
+        doReturn("[").when(mResources).getString(
+                R.string.config_network_validation_failed_content_regexp);
+        assertFalse(wnm.matchesHttpContent("[",
+                R.string.config_network_validation_failed_content_regexp));
+    }
+
+    @Test
+    public void testMatchesHttpContentLength() throws Exception {
+        final WrappedNetworkMonitor wnm = makeNotMeteredNetworkMonitor();
+        // Set the range of content length.
+        doReturn(100).when(mResources).getInteger(R.integer.config_min_matches_http_content_length);
+        doReturn(1000).when(mResources).getInteger(
+                R.integer.config_max_matches_http_content_length);
+        assertFalse(wnm.matchesHttpContentLength(100));
+        assertFalse(wnm.matchesHttpContentLength(1000));
+        assertTrue(wnm.matchesHttpContentLength(500));
+
+        // Test the invalid value.
+        assertFalse(wnm.matchesHttpContentLength(-1));
+        assertFalse(wnm.matchesHttpContentLength(0));
+        assertFalse(wnm.matchesHttpContentLength(Integer.MAX_VALUE + 1L));
+
+        // Set the wrong value for min and max config to make sure the function is working even
+        // though the config is wrong.
+        doReturn(1000).when(mResources).getInteger(
+                R.integer.config_min_matches_http_content_length);
+        doReturn(100).when(mResources).getInteger(
+                R.integer.config_max_matches_http_content_length);
+        assertFalse(wnm.matchesHttpContentLength(100));
+        assertFalse(wnm.matchesHttpContentLength(1000));
+        assertFalse(wnm.matchesHttpContentLength(500));
+    }
+
+    @Test
+    public void testGetResStringConfig() throws Exception {
+        final WrappedNetworkMonitor wnm = makeNotMeteredNetworkMonitor();
+        // Set the config and expect to get the customized value.
+        final String regExp = ".*HTTP.*200.*not a captive portal.*";
+        doReturn(regExp).when(mResources).getString(
+                R.string.config_network_validation_failed_content_regexp);
+        assertEquals(regExp, wnm.getResStringConfig(mContext,
+                R.string.config_network_validation_failed_content_regexp, null));
+        doThrow(new Resources.NotFoundException()).when(mResources).getString(eq(
+                R.string.config_network_validation_failed_content_regexp));
+        // If the config is not found, then expect to get the default value - null.
+        assertNull(wnm.getResStringConfig(mContext,
+                R.string.config_network_validation_failed_content_regexp, null));
+    }
+
+    @Test
+    public void testGetResIntConfig() throws Exception {
+        final WrappedNetworkMonitor wnm = makeNotMeteredNetworkMonitor();
+        // Set the config and expect to get the customized value.
+        doReturn(100).when(mResources).getInteger(R.integer.config_min_matches_http_content_length);
+        doReturn(1000).when(mResources).getInteger(
+                R.integer.config_max_matches_http_content_length);
+        assertEquals(100, wnm.getResIntConfig(mContext,
+                R.integer.config_min_matches_http_content_length, Integer.MAX_VALUE));
+        assertEquals(1000, wnm.getResIntConfig(mContext,
+                R.integer.config_max_matches_http_content_length, 0));
+        doThrow(new Resources.NotFoundException())
+                .when(mResources).getInteger(
+                        eq(R.integer.config_min_matches_http_content_length));
+        doThrow(new Resources.NotFoundException())
+                .when(mResources).getInteger(eq(R.integer.config_max_matches_http_content_length));
+        // If the config is not found, then expect to get the default value.
+        assertEquals(Integer.MAX_VALUE, wnm.getResIntConfig(mContext,
+                R.integer.config_min_matches_http_content_length, Integer.MAX_VALUE));
+        assertEquals(0, wnm.getResIntConfig(mContext,
+                R.integer.config_max_matches_http_content_length, 0));
+    }
+
+    @Test
     public void testGetLocationMcc() throws Exception {
         final WrappedNetworkMonitor wnm = makeNotMeteredNetworkMonitor();
         doReturn(PackageManager.PERMISSION_DENIED).when(mContext).checkPermission(
@@ -975,6 +1058,38 @@ public class NetworkMonitorTest {
         verify(mHttpConnection).getResponseCode();
     }
 
+    @Test
+    public void testIsCaptivePortal_HttpsProbeMatchesFailRegex() throws Exception {
+        setStatus(mHttpsConnection, 200);
+        setStatus(mHttpConnection, 500);
+        final String content = "test";
+        doReturn(new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)))
+                .when(mHttpsConnection).getInputStream();
+        doReturn(Long.valueOf(content.length())).when(mHttpsConnection).getContentLengthLong();
+        doReturn(1).when(mResources).getInteger(R.integer.config_min_matches_http_content_length);
+        doReturn(10).when(mResources).getInteger(
+                R.integer.config_max_matches_http_content_length);
+        doReturn("te.t").when(mResources).getString(
+                R.string.config_network_validation_failed_content_regexp);
+        runFailedNetworkTest();
+    }
+
+    @Test
+    public void testIsCaptivePortal_HttpProbeMatchesSuccessRegex() throws Exception {
+        setStatus(mHttpsConnection, 500);
+        setStatus(mHttpConnection, 200);
+        final String content = "test";
+        doReturn(new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)))
+                .when(mHttpConnection).getInputStream();
+        doReturn(Long.valueOf(content.length())).when(mHttpConnection).getContentLengthLong();
+        doReturn(1).when(mResources).getInteger(R.integer.config_min_matches_http_content_length);
+        doReturn(10).when(mResources).getInteger(
+                R.integer.config_max_matches_http_content_length);
+        doReturn("te.t").when(mResources).getString(
+                R.string.config_network_validation_success_content_regexp);
+        runPartialConnectivityNetworkTest(VALIDATION_RESULT_PARTIAL);
+    }
+
     private void setupFallbackSpec() throws IOException {
         setFallbackSpecs("http://example.com@@/@@204@@/@@"
                 + "@@,@@"
@@ -1698,6 +1813,20 @@ public class NetworkMonitorTest {
         }
     }
 
+    @Test
+    public void testReadAsString_StreamShorterThanLimit() throws Exception {
+        final WrappedNetworkMonitor wnm = makeNotMeteredNetworkMonitor();
+        final String content = "The HTTP response code is 200 but it is not a captive portal.";
+        ByteArrayInputStream inputStream = new ByteArrayInputStream(
+                content.getBytes(StandardCharsets.UTF_8));
+        assertEquals(content, wnm.readAsString(inputStream, content.length(),
+                StandardCharsets.UTF_8));
+        // Reset the inputStream and test the case that the stream ends earlier than the limit.
+        inputStream = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8));
+        assertEquals(content, wnm.readAsString(inputStream, content.length() + 10,
+                StandardCharsets.UTF_8));
+    }
+
     private void makeDnsTimeoutEvent(WrappedNetworkMonitor wrappedMonitor, int count) {
         for (int i = 0; i < count; i++) {
             wrappedMonitor.getDnsStallDetector().accumulateConsecutiveDnsTimeoutCount(