Rewrite hidden API list generation in Python

Generating hidden API lists has grown in complexity and the original
Makefile rule has become a bottleneck for build times. Rewrite the
logic in Python.

Bug: 113278235
Bug: 73736106
Test: frameworks/base/tools/hiddenapi/generate_hiddenapi_lists_test.py
Change-Id: I63f03133d70260d06c55f482b844a4980dc6f734

2 files changed, 330 insertions, 0 deletions

diff --git a/tools/hiddenapi/generate_hiddenapi_lists.py b/tools/hiddenapi/generate_hiddenapi_lists.py
new file mode 100755
index 000000000000..4a0931a149af
--- /dev/null
+++ b/tools/hiddenapi/generate_hiddenapi_lists.py
@@ -0,0 +1,241 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Generate API lists for non-SDK API enforcement.
+
+usage: generate-hiddenapi-lists.py [-h]
+                                   --input-public INPUT_PUBLIC
+                                   --input-private INPUT_PRIVATE
+                                   [--input-whitelists [INPUT_WHITELISTS [INPUT_WHITELISTS ...]]]
+                                   [--input-greylists [INPUT_GREYLISTS [INPUT_GREYLISTS ...]]]
+                                   [--input-blacklists [INPUT_BLACKLISTS [INPUT_BLACKLISTS ...]]]
+                                   --output-whitelist OUTPUT_WHITELIST
+                                   --output-light-greylist OUTPUT_LIGHT_GREYLIST
+                                   --output-dark-greylist OUTPUT_DARK_GREYLIST
+                                   --output-blacklist OUTPUT_BLACKLIST
+"""
+import argparse
+import os
+import sys
+import re
+
+def get_args():
+    """Parses command line arguments.
+
+    Returns:
+        Namespace: dictionary of parsed arguments
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--input-public', required=True, help='List of all public members')
+    parser.add_argument('--input-private', required=True, help='List of all private members')
+    parser.add_argument(
+        '--input-whitelists', nargs='*',
+        help='Lists of members to force on whitelist')
+    parser.add_argument(
+        '--input-greylists', nargs='*',
+        help='Lists of members to force on light greylist')
+    parser.add_argument(
+        '--input-blacklists', nargs='*',
+        help='Lists of members to force on blacklist')
+    parser.add_argument('--output-whitelist', required=True)
+    parser.add_argument('--output-light-greylist', required=True)
+    parser.add_argument('--output-dark-greylist', required=True)
+    parser.add_argument('--output-blacklist', required=True)
+    return parser.parse_args()
+
+def read_lines(filename):
+    """Reads entire file and return it as a list of lines.
+
+    Args:
+        filename (string): Path to the file to read from.
+
+    Returns:
+        list: Lines of the loaded file as a list of strings.
+    """
+    with open(filename, 'r') as f:
+        return f.readlines()
+
+def write_lines(filename, lines):
+    """Writes list of lines into a file, overwriting the file it it exists.
+
+    Args:
+        filename (string): Path to the file to be writting into.
+        lines (list): List of strings to write into the file.
+    """
+    with open(filename, 'w') as f:
+        f.writelines(lines)
+
+def move_between_sets(subset, src, dst, source = "<unknown>"):
+    """Removes a subset of elements from one set and add it to another.
+
+    Args:
+        subset (set): The subset of `src` to be moved from `src` to `dst`.
+        src (set): Source set. Must be a superset of `subset`.
+        dst (set): Destination set. Must be disjoint with `subset`.
+    """
+    assert src.issuperset(subset), (
+        "Error processing: {}\n"
+        "The following entries were not found:\n"
+        "{}"
+        "Please visit go/hiddenapi for more information.").format(
+            source, "".join(map(lambda x: "  " + str(x), subset.difference(src))))
+    assert dst.isdisjoint(subset)
+    # Order matters if `src` and `subset` are the same object.
+    dst.update(subset)
+    src.difference_update(subset)
+
+def get_package_name(signature):
+    """Returns the package name prefix of a class member signature.
+
+    Example: "Ljava/lang/String;->hashCode()J" --> "Ljava/lang/"
+
+    Args:
+        signature (string): Member signature
+
+    Returns
+        string: Package name of the given member
+    """
+    class_name_end = signature.find("->")
+    assert class_name_end != -1, "Invalid signature: {}".format(signature)
+    package_name_end = signature.rfind("/", 0, class_name_end)
+    assert package_name_end != -1, "Invalid signature: {}".format(signature)
+    return signature[:package_name_end + 1]
+
+def all_package_names(*args):
+    """Returns a set of packages names in given lists of member signatures.
+
+    Example: args = [ set([ "Lpkg1/ClassA;->foo()V", "Lpkg2/ClassB;->bar()J" ]),
+                      set([ "Lpkg1/ClassC;->baz()Z" ]) ]
+             return value = set([ "Lpkg1/", "Lpkg2" ])
+
+    Args:
+        *args (list): List of sets to iterate over and extract the package names
+                      of its elements (member signatures)
+
+    Returns:
+        set: All package names extracted from the given lists of signatures.
+    """
+    packages = set()
+    for arg in args:
+        packages = packages.union(map(get_package_name, arg))
+    return packages
+
+def move_all(src, dst):
+    """Moves all elements of one set to another.
+
+    Args:
+        src (set): Source set. Will become empty.
+        dst (set): Destination set. Will contain all elements of `src`.
+    """
+    move_between_sets(src, src, dst)
+
+def move_from_files(filenames, src, dst):
+    """Loads member signatures from a list of files and moves them to a given set.
+
+    Opens files in `filenames`, reads all their lines and moves those from `src`
+    set to `dst` set.
+
+    Args:
+        filenames (list): List of paths to files to be loaded.
+        src (set): Set that loaded lines should be moved from.
+        dst (set): Set that loaded lines should be moved to.
+    """
+    if filenames:
+        for filename in filenames:
+            move_between_sets(set(read_lines(filename)), src, dst, filename)
+
+def move_serialization(src, dst):
+    """Moves all members matching serialization API signatures between given sets.
+
+    Args:
+        src (set): Set that will be searched for serialization API and that API
+                   will be removed from it.
+        dst (set): Set that serialization API will be moved to.
+    """
+    serialization_patterns = [
+        r'readObject\(Ljava/io/ObjectInputStream;\)V',
+        r'readObjectNoData\(\)V',
+        r'readResolve\(\)Ljava/lang/Object;',
+        r'serialVersionUID:J',
+        r'serialPersistentFields:\[Ljava/io/ObjectStreamField;',
+        r'writeObject\(Ljava/io/ObjectOutputStream;\)V',
+        r'writeReplace\(\)Ljava/lang/Object;',
+    ]
+    regex = re.compile(r'.*->(' + '|'.join(serialization_patterns) + r')$')
+    move_between_sets(filter(lambda api: regex.match(api), src), src, dst)
+
+def move_from_packages(packages, src, dst):
+    """Moves all members of given package names from one set to another.
+
+    Args:
+        packages (list): List of string package names.
+        src (set): Set that will be searched for API matching one of the given
+                   package names. Surch API will be removed from the set.
+        dst (set): Set that matching API will be moved to.
+    """
+    move_between_sets(filter(lambda api: get_package_name(api) in packages, src), src, dst)
+
+def main(argv):
+    args = get_args()
+
+    # Initialize API sets by loading lists of public and private API. Public API
+    # are all members resolvable from SDK API stubs, other members are private.
+    # As an optimization, skip the step of moving public API from a full set of
+    # members and start with a populated whitelist.
+    whitelist = set(read_lines(args.input_public))
+    uncategorized = set(read_lines(args.input_private))
+    light_greylist = set()
+    dark_greylist = set()
+    blacklist = set()
+
+    # Assert that there is no overlap between public and private API.
+    assert whitelist.isdisjoint(uncategorized)
+    num_all_api = len(whitelist) + len(uncategorized)
+
+    # Read all files which manually assign members to specific lists.
+    move_from_files(args.input_whitelists, uncategorized, whitelist)
+    move_from_files(args.input_greylists, uncategorized, light_greylist)
+    move_from_files(args.input_blacklists, uncategorized, blacklist)
+
+    # Iterate over all uncategorized members and move serialization API to light greylist.
+    move_serialization(uncategorized, light_greylist)
+
+    # Extract package names of members from whitelist and light greylist, which
+    # are assumed to have been finalized at this point. Assign all uncategorized
+    # members from the same packages to the dark greylist.
+    dark_greylist_packages = all_package_names(whitelist, light_greylist)
+    move_from_packages(dark_greylist_packages, uncategorized, dark_greylist)
+
+    # Assign all uncategorized members to the blacklist.
+    move_all(uncategorized, blacklist)
+
+    # Assert we have not missed anything.
+    assert whitelist.isdisjoint(light_greylist)
+    assert whitelist.isdisjoint(dark_greylist)
+    assert whitelist.isdisjoint(blacklist)
+    assert light_greylist.isdisjoint(dark_greylist)
+    assert light_greylist.isdisjoint(blacklist)
+    assert dark_greylist.isdisjoint(blacklist)
+    assert num_all_api == len(whitelist) + len(light_greylist) + len(dark_greylist) + len(blacklist)
+
+    # Write final lists to disk.
+    write_lines(args.output_whitelist, whitelist)
+    write_lines(args.output_light_greylist, light_greylist)
+    write_lines(args.output_dark_greylist, dark_greylist)
+    write_lines(args.output_blacklist, blacklist)
+
+if __name__ == "__main__":
+    main(sys.argv)
diff --git a/tools/hiddenapi/generate_hiddenapi_lists_test.py b/tools/hiddenapi/generate_hiddenapi_lists_test.py
new file mode 100755
index 000000000000..8f793189650c
--- /dev/null
+++ b/tools/hiddenapi/generate_hiddenapi_lists_test.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Unit tests for Hidden API list generation."""
+import unittest
+from generate_hiddenapi_lists import *
+
+class TestHiddenapiListGeneration(unittest.TestCase):
+
+    def test_move_between_sets(self):
+        A = set([1, 2, 3, 4])
+        B = set([5, 6, 7, 8])
+        move_between_sets(set([2, 4]), A, B)
+        self.assertEqual(A, set([1, 3]))
+        self.assertEqual(B, set([2, 4, 5, 6, 7, 8]))
+
+    def test_move_between_sets_fail_not_superset(self):
+        A = set([1, 2, 3, 4])
+        B = set([5, 6, 7, 8])
+        with self.assertRaises(AssertionError) as ar:
+            move_between_sets(set([0, 2]), A, B)
+
+    def test_move_between_sets_fail_not_disjoint(self):
+        A = set([1, 2, 3, 4])
+        B = set([4, 5, 6, 7, 8])
+        with self.assertRaises(AssertionError) as ar:
+            move_between_sets(set([1, 4]), A, B)
+
+    def test_get_package_name(self):
+        self.assertEqual(get_package_name("Ljava/lang/String;->clone()V"), "Ljava/lang/")
+
+    def test_get_package_name_fail_no_arrow(self):
+        with self.assertRaises(AssertionError) as ar:
+            get_package_name("Ljava/lang/String;-clone()V")
+        with self.assertRaises(AssertionError) as ar:
+            get_package_name("Ljava/lang/String;>clone()V")
+        with self.assertRaises(AssertionError) as ar:
+            get_package_name("Ljava/lang/String;__clone()V")
+
+    def test_get_package_name_fail_no_package(self):
+        with self.assertRaises(AssertionError) as ar:
+            get_package_name("LString;->clone()V")
+
+    def test_all_package_names(self):
+        self.assertEqual(all_package_names(), set())
+        self.assertEqual(all_package_names(set(["Lfoo/Bar;->baz()V"])), set(["Lfoo/"]))
+        self.assertEqual(
+            all_package_names(set(["Lfoo/Bar;->baz()V", "Lfoo/BarX;->bazx()I"])),
+            set(["Lfoo/"]))
+        self.assertEqual(
+            all_package_names(
+                set(["Lfoo/Bar;->baz()V"]),
+                set(["Lfoo/BarX;->bazx()I", "Labc/xyz/Mno;->ijk()J"])),
+            set(["Lfoo/", "Labc/xyz/"]))
+
+    def test_move_all(self):
+        src = set([ "abc", "xyz" ])
+        dst = set([ "def" ])
+        move_all(src, dst)
+        self.assertEqual(src, set())
+        self.assertEqual(dst, set([ "abc", "def", "xyz" ]))
+
+    def test_move_from_packages(self):
+        src = set([ "Lfoo/bar/ClassA;->abc()J",        # will be moved
+                    "Lfoo/bar/ClassA;->def()J",        # will be moved
+                    "Lcom/pkg/example/ClassD;->ijk:J", # not moved: different package
+                    "Lfoo/bar/xyz/ClassC;->xyz()Z" ])  # not moved: subpackage
+        dst = set()
+        packages = set([ "Lfoo/bar/" ])
+        move_from_packages(packages, src, dst)
+        self.assertEqual(
+            src, set([ "Lfoo/bar/xyz/ClassC;->xyz()Z", "Lcom/pkg/example/ClassD;->ijk:J" ]))
+        self.assertEqual(
+            dst, set([ "Lfoo/bar/ClassA;->abc()J", "Lfoo/bar/ClassA;->def()J" ]))
+
+if __name__ == '__main__':
+    unittest.main()