Automatically greylist code in 3P packages

generate_hidden_api_lists now receives a file containing package names
which need to be greylisted (although it could be made to work with any
api list required).
Also took the opportunity to clean up the tests to reflect the more
strict code.

Bug: 129387816
Test: m appcompat
Test: frameworks/base/tools/hiddenapi/generate_hiddenapi_lists_test.py
Change-Id: I619f8581d166aa48eda572bc0053d8739d6420eb

2 files changed, 61 insertions, 38 deletions

diff --git a/tools/hiddenapi/generate_hiddenapi_lists.py b/tools/hiddenapi/generate_hiddenapi_lists.py
index 6781eba05534..c856cc36d6f6 100755
--- a/tools/hiddenapi/generate_hiddenapi_lists.py
+++ b/tools/hiddenapi/generate_hiddenapi_lists.py
@@ -21,6 +21,7 @@ from collections import defaultdict
 import os
 import sys
 import re
+import functools
 
 # Names of flags recognized by the `hiddenapi` tool.
 FLAG_WHITELIST = "whitelist"
@@ -58,6 +59,10 @@ ALL_FLAGS_SET = set(ALL_FLAGS)
 # script to skip any entries which do not exist any more.
 FLAG_IGNORE_CONFLICTS_SUFFIX = "-ignore-conflicts"
 
+# Suffix used in command line args to express that all apis within a given set
+# of packages should be assign the given flag.
+FLAG_PACKAGES_SUFFIX = "-packages"
+
 # Regex patterns of fields/methods used in serialization. These are
 # considered public API despite being hidden.
 SERIALIZATION_PATTERNS = [
@@ -91,12 +96,16 @@ def get_args():
 
     for flag in ALL_FLAGS:
         ignore_conflicts_flag = flag + FLAG_IGNORE_CONFLICTS_SUFFIX
+        packages_flag = flag + FLAG_PACKAGES_SUFFIX
         parser.add_argument('--' + flag, dest=flag, nargs='*', default=[], metavar='TXT_FILE',
             help='lists of entries with flag "' + flag + '"')
         parser.add_argument('--' + ignore_conflicts_flag, dest=ignore_conflicts_flag, nargs='*',
             default=[], metavar='TXT_FILE',
             help='lists of entries with flag "' + flag +
                  '". skip entry if missing or flag conflict.')
+        parser.add_argument('--' + packages_flag, dest=packages_flag, nargs='*',
+            default=[], metavar='TXT_FILE',
+            help='lists of packages to be added to ' + flag + ' list')
 
     return parser.parse_args()
 
@@ -128,6 +137,19 @@ def write_lines(filename, lines):
     with open(filename, 'w') as f:
         f.writelines(lines)
 
+def extract_package(signature):
+    """Extracts the package from a signature.
+
+    Args:
+        signature (string): JNI signature of a method or field.
+
+    Returns:
+        The package name of the class containing the field/method.
+    """
+    full_class_name = signature.split(";->")[0]
+    package_name = full_class_name[1:full_class_name.rindex("/")]
+    return package_name.replace('/', '.')
+
 class FlagsDict:
     def __init__(self):
         self._dict_keyset = set()
@@ -206,7 +228,10 @@ class FlagsDict:
         self._dict_keyset.update([ csv[0] for csv in csv_values ])
 
         # Check that all flags are known.
-        csv_flags = set(reduce(lambda x, y: set(x).union(y), [ csv[1:] for csv in csv_values ], []))
+        csv_flags = set(functools.reduce(
+            lambda x, y: set(x).union(y),
+            [ csv[1:] for csv in csv_values ],
+            []))
         self._check_flags_set(csv_flags, source)
 
         # Iterate over all CSV lines, find entry in dict and append flags to it.
@@ -273,6 +298,15 @@ def main(argv):
             valid_entries = flags.get_valid_subset_of_unassigned_apis(read_lines(filename))
             flags.assign_flag(flag, valid_entries, filename)
 
+    # All members in the specified packages will be assigned the appropriate flag.
+    for flag in ALL_FLAGS:
+        for filename in args[flag + FLAG_PACKAGES_SUFFIX]:
+            packages_needing_list = set(read_lines(filename))
+            should_add_signature_to_list = lambda sig,lists: extract_package(
+                sig) in packages_needing_list and not lists
+            valid_entries = flags.filter_apis(should_add_signature_to_list)
+            flags.assign_flag(flag, valid_entries)
+
     # Assign all remaining entries to the blacklist.
     flags.assign_flag(FLAG_BLACKLIST, flags.filter_apis(HAS_NO_API_LIST_ASSIGNED))
 
diff --git a/tools/hiddenapi/generate_hiddenapi_lists_test.py b/tools/hiddenapi/generate_hiddenapi_lists_test.py
index 249f37db5a82..4dc880b107d3 100755
--- a/tools/hiddenapi/generate_hiddenapi_lists_test.py
+++ b/tools/hiddenapi/generate_hiddenapi_lists_test.py
@@ -18,33 +18,23 @@ import unittest
 from generate_hiddenapi_lists import *
 
 class TestHiddenapiListGeneration(unittest.TestCase):
-    def test_init(self):
-        # Check empty lists
-        flags = FlagsDict([], [])
-        self.assertEquals(flags.generate_csv(), [])
-
-        # Check valid input - two public and two private API signatures.
-        flags = FlagsDict(['A', 'B'], ['C', 'D'])
-        self.assertEquals(flags.generate_csv(),
-                          [ 'A,' + FLAG_WHITELIST, 'B,' + FLAG_WHITELIST, 'C', 'D' ])
-
-        # Check invalid input - overlapping public/private API signatures.
-        with self.assertRaises(AssertionError):
-            flags = FlagsDict(['A', 'B'], ['B', 'C', 'D'])
 
     def test_filter_apis(self):
         # Initialize flags so that A and B are put on the whitelist and
         # C, D, E are left unassigned. Try filtering for the unassigned ones.
-        flags = FlagsDict(['A', 'B'], ['C', 'D', 'E'])
+        flags = FlagsDict()
+        flags.parse_and_merge_csv(['A,' + FLAG_WHITELIST, 'B,' + FLAG_WHITELIST,
+                        'C', 'D', 'E'])
         filter_set = flags.filter_apis(lambda api, flags: not flags)
         self.assertTrue(isinstance(filter_set, set))
         self.assertEqual(filter_set, set([ 'C', 'D', 'E' ]))
 
     def test_get_valid_subset_of_unassigned_keys(self):
         # Create flags where only A is unassigned.
-        flags = FlagsDict(['A'], ['B', 'C'])
+        flags = FlagsDict()
+        flags.parse_and_merge_csv(['A,' + FLAG_WHITELIST, 'B', 'C'])
         flags.assign_flag(FLAG_GREYLIST, set(['C']))
-        self.assertEquals(flags.generate_csv(),
+        self.assertEqual(flags.generate_csv(),
             [ 'A,' + FLAG_WHITELIST, 'B', 'C,' + FLAG_GREYLIST ])
 
         # Check three things:
@@ -55,44 +45,30 @@ class TestHiddenapiListGeneration(unittest.TestCase):
             flags.get_valid_subset_of_unassigned_apis(set(['A', 'B', 'D'])), set([ 'B' ]))
 
     def test_parse_and_merge_csv(self):
-        flags = FlagsDict(['A'], ['B'])
-        self.assertEquals(flags.generate_csv(), [ 'A,' + FLAG_WHITELIST, 'B' ])
+        flags = FlagsDict()
 
         # Test empty CSV entry.
-        flags.parse_and_merge_csv(['B'])
-        self.assertEquals(flags.generate_csv(), [ 'A,' + FLAG_WHITELIST, 'B' ])
-
-        # Test assigning an already assigned flag.
-        flags.parse_and_merge_csv(['A,' + FLAG_WHITELIST])
-        self.assertEquals(flags.generate_csv(), [ 'A,' + FLAG_WHITELIST, 'B' ])
+        self.assertEqual(flags.generate_csv(), [])
 
         # Test new additions.
         flags.parse_and_merge_csv([
             'A,' + FLAG_GREYLIST,
             'B,' + FLAG_BLACKLIST + ',' + FLAG_GREYLIST_MAX_O ])
         self.assertEqual(flags.generate_csv(),
-            [ 'A,' + FLAG_GREYLIST + "," + FLAG_WHITELIST,
+            [ 'A,' + FLAG_GREYLIST,
               'B,' + FLAG_BLACKLIST + "," + FLAG_GREYLIST_MAX_O ])
 
-        # Test unknown API signature.
-        with self.assertRaises(AssertionError):
-            flags.parse_and_merge_csv([ 'C' ])
-
         # Test unknown flag.
         with self.assertRaises(AssertionError):
-            flags.parse_and_merge_csv([ 'A,foo' ])
+            flags.parse_and_merge_csv([ 'C,foo' ])
 
     def test_assign_flag(self):
-        flags = FlagsDict(['A'], ['B'])
-        self.assertEquals(flags.generate_csv(), [ 'A,' + FLAG_WHITELIST, 'B' ])
-
-        # Test assigning an already assigned flag.
-        flags.assign_flag(FLAG_WHITELIST, set([ 'A' ]))
-        self.assertEquals(flags.generate_csv(), [ 'A,' + FLAG_WHITELIST, 'B' ])
+        flags = FlagsDict()
+        flags.parse_and_merge_csv(['A,' + FLAG_WHITELIST, 'B'])
 
         # Test new additions.
         flags.assign_flag(FLAG_GREYLIST, set([ 'A', 'B' ]))
-        self.assertEquals(flags.generate_csv(),
+        self.assertEqual(flags.generate_csv(),
             [ 'A,' + FLAG_GREYLIST + "," + FLAG_WHITELIST, 'B,' + FLAG_GREYLIST ])
 
         # Test invalid API signature.
@@ -103,5 +79,18 @@ class TestHiddenapiListGeneration(unittest.TestCase):
         with self.assertRaises(AssertionError):
             flags.assign_flag('foo', set([ 'A' ]))
 
+    def test_extract_package(self):
+        signature = 'Lcom/foo/bar/Baz;->method1()Lcom/bar/Baz;'
+        expected_package = 'com.foo.bar'
+        self.assertEqual(extract_package(signature), expected_package)
+
+        signature = 'Lcom/foo1/bar/MyClass;->method2()V'
+        expected_package = 'com.foo1.bar'
+        self.assertEqual(extract_package(signature), expected_package)
+
+        signature = 'Lcom/foo_bar/baz/MyClass;->method3()V'
+        expected_package = 'com.foo_bar.baz'
+        self.assertEqual(extract_package(signature), expected_package)
+
 if __name__ == '__main__':
     unittest.main()