diff options
Diffstat (limited to 'tools/hiddenapi')
| -rwxr-xr-x | tools/hiddenapi/generate_hiddenapi_lists.py | 127 | ||||
| -rwxr-xr-x | tools/hiddenapi/generate_hiddenapi_lists_test.py | 39 |
2 files changed, 113 insertions, 53 deletions
diff --git a/tools/hiddenapi/generate_hiddenapi_lists.py b/tools/hiddenapi/generate_hiddenapi_lists.py index de6b4785ec37..8a282e5f0821 100755 --- a/tools/hiddenapi/generate_hiddenapi_lists.py +++ b/tools/hiddenapi/generate_hiddenapi_lists.py @@ -13,9 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" -Generate API lists for non-SDK API enforcement. -""" +"""Generate API lists for non-SDK API enforcement.""" import argparse from collections import defaultdict import functools @@ -24,27 +22,47 @@ import re import sys # Names of flags recognized by the `hiddenapi` tool. -FLAG_WHITELIST = "whitelist" -FLAG_GREYLIST = "greylist" -FLAG_BLACKLIST = "blacklist" -FLAG_GREYLIST_MAX_O = "greylist-max-o" -FLAG_GREYLIST_MAX_P = "greylist-max-p" -FLAG_GREYLIST_MAX_Q = "greylist-max-q" -FLAG_GREYLIST_MAX_R = "greylist-max-r" -FLAG_CORE_PLATFORM_API = "core-platform-api" -FLAG_PUBLIC_API = "public-api" -FLAG_SYSTEM_API = "system-api" -FLAG_TEST_API = "test-api" +FLAG_SDK = 'sdk' +FLAG_UNSUPPORTED = 'unsupported' +FLAG_BLOCKED = 'blocked' +FLAG_MAX_TARGET_O = 'max-target-o' +FLAG_MAX_TARGET_P = 'max-target-p' +FLAG_MAX_TARGET_Q = 'max-target-q' +FLAG_MAX_TARGET_R = 'max-target-r' +FLAG_CORE_PLATFORM_API = 'core-platform-api' +FLAG_PUBLIC_API = 'public-api' +FLAG_SYSTEM_API = 'system-api' +FLAG_TEST_API = 'test-api' + +OLD_FLAG_SDK = "whitelist" +OLD_FLAG_UNSUPPORTED = "greylist" +OLD_FLAG_BLOCKED = "blacklist" +OLD_FLAG_MAX_TARGET_O = "greylist-max-o" +OLD_FLAG_MAX_TARGET_P = "greylist-max-p" +OLD_FLAG_MAX_TARGET_Q = "greylist-max-q" +OLD_FLAG_MAX_TARGET_R = "greylist-max-r" + +OLD_FLAGS_TO_NEW = { + OLD_FLAG_SDK: FLAG_SDK, + OLD_FLAG_UNSUPPORTED: FLAG_UNSUPPORTED, + OLD_FLAG_BLOCKED: FLAG_BLOCKED, + OLD_FLAG_MAX_TARGET_O: FLAG_MAX_TARGET_O, + OLD_FLAG_MAX_TARGET_P: FLAG_MAX_TARGET_P, + OLD_FLAG_MAX_TARGET_Q: FLAG_MAX_TARGET_Q, + OLD_FLAG_MAX_TARGET_R: FLAG_MAX_TARGET_R, +} + +NEW_FLAGS_TO_OLD = dict(zip(OLD_FLAGS_TO_NEW.values(), OLD_FLAGS_TO_NEW.keys())) # List of all known flags. FLAGS_API_LIST = [ - FLAG_WHITELIST, - FLAG_GREYLIST, - FLAG_BLACKLIST, - FLAG_GREYLIST_MAX_O, - FLAG_GREYLIST_MAX_P, - FLAG_GREYLIST_MAX_Q, - FLAG_GREYLIST_MAX_R, + FLAG_SDK, + FLAG_UNSUPPORTED, + FLAG_BLOCKED, + FLAG_MAX_TARGET_O, + FLAG_MAX_TARGET_P, + FLAG_MAX_TARGET_Q, + FLAG_MAX_TARGET_R, ] ALL_FLAGS = FLAGS_API_LIST + [ FLAG_CORE_PLATFORM_API, @@ -58,7 +76,7 @@ ALL_FLAGS_SET = set(ALL_FLAGS) # Suffix used in command line args to express that only known and # otherwise unassigned entries should be assign the given flag. -# For example, the P dark greylist is checked in as it was in P, +# For example, the max-target-P list is checked in as it was in P, # but signatures have changes since then. The flag instructs this # script to skip any entries which do not exist any more. FLAG_IGNORE_CONFLICTS_SUFFIX = "-ignore-conflicts" @@ -87,6 +105,7 @@ SERIALIZATION_REGEX = re.compile(r'.*->(' + '|'.join(SERIALIZATION_PATTERNS) + r HAS_NO_API_LIST_ASSIGNED = lambda api, flags: not FLAGS_API_LIST_SET.intersection(flags) IS_SERIALIZATION = lambda api, flags: SERIALIZATION_REGEX.match(api) + def get_args(): """Parses command line arguments. @@ -113,6 +132,7 @@ def get_args(): return parser.parse_args() + def read_lines(filename): """Reads entire file and return it as a list of lines. @@ -130,8 +150,9 @@ def read_lines(filename): lines = map(lambda line: line.strip(), lines) return set(lines) + def write_lines(filename, lines): - """Writes list of lines into a file, overwriting the file it it exists. + """Writes list of lines into a file, overwriting the file if it exists. Args: filename (string): Path to the file to be writting into. @@ -141,6 +162,7 @@ def write_lines(filename, lines): with open(filename, 'w') as f: f.writelines(lines) + def extract_package(signature): """Extracts the package from a signature. @@ -159,6 +181,7 @@ def extract_package(signature): package_name = full_class_name.rpartition("/")[0] return package_name.replace('/', '.') + class FlagsDict: def __init__(self): self._dict_keyset = set() @@ -182,6 +205,36 @@ class FlagsDict: "Please visit go/hiddenapi for more information.").format( source, "\n".join(flags_subset - ALL_FLAGS_SET)) + def convert_to_new_flag(self, flag): + """Converts old flag to a new variant. + + Flags that are considered old are replaced with new versions. + Otherwise, it is a no-op. + + Args: + flag: a string, representing SDK flag. + + Returns: + A string. Result of conversion. + + """ + return OLD_FLAGS_TO_NEW.get(flag, flag) + + def convert_to_old_flag(self, flag): + """Converts a new flag to a old variant. + + No-op if there is no suitable old flag. + Only used to support backwards compatibility. + + Args: + flag: a string, representing SDK flag. + + Returns: + A string. Result of conversion. + + """ + return NEW_FLAGS_TO_OLD.get(flag, flag) + def filter_apis(self, filter_fn): """Returns APIs which match a given predicate. @@ -212,10 +265,16 @@ class FlagsDict: def generate_csv(self): """Constructs CSV entries from a dictionary. + Old versions of flags are used to generate the file. + Returns: List of lines comprising a CSV file. See "parse_and_merge_csv" for format description. """ - return sorted(map(lambda api: ",".join([api] + sorted(self._dict[api])), self._dict)) + lines = [] + for api in self._dict: + flags = sorted([self.convert_to_old_flag(flag) for flag in self._dict[api]]) + lines.append(",".join([api] + flags)) + return sorted(lines) def parse_and_merge_csv(self, csv_lines, source = "<unknown>"): """Parses CSV entries and merges them into a given dictionary. @@ -237,17 +296,16 @@ class FlagsDict: self._dict_keyset.update([ csv[0] for csv in csv_values ]) # Check that all flags are known. - csv_flags = set(functools.reduce( - lambda x, y: set(x).union(y), - [ csv[1:] for csv in csv_values ], - [])) + csv_flags = set() + for csv in csv_values: + csv_flags.update([self.convert_to_new_flag(flag) for flag in csv[1:]]) self._check_flags_set(csv_flags, source) # Iterate over all CSV lines, find entry in dict and append flags to it. for csv in csv_values: - flags = csv[1:] + flags = [self.convert_to_new_flag(flag) for flag in csv[1:]] if (FLAG_PUBLIC_API in flags) or (FLAG_SYSTEM_API in flags): - flags.append(FLAG_WHITELIST) + flags.append(FLAG_SDK) self._dict[csv[0]].update(flags) def assign_flag(self, flag, apis, source="<unknown>"): @@ -271,6 +329,7 @@ class FlagsDict: for api in apis: self._dict[api].add(flag) + def main(argv): # Parse arguments. args = vars(get_args()) @@ -287,8 +346,8 @@ def main(argv): flags.parse_and_merge_csv(read_lines(filename), filename) # Combine inputs which do not require any particular order. - # (1) Assign serialization API to whitelist. - flags.assign_flag(FLAG_WHITELIST, flags.filter_apis(IS_SERIALIZATION)) + # (1) Assign serialization API to SDK. + flags.assign_flag(FLAG_SDK, flags.filter_apis(IS_SERIALIZATION)) # (2) Merge text files with a known flag into the dictionary. for flag in ALL_FLAGS: @@ -314,8 +373,8 @@ def main(argv): valid_entries = flags.filter_apis(should_add_signature_to_list) flags.assign_flag(flag, valid_entries) - # Assign all remaining entries to the blacklist. - flags.assign_flag(FLAG_BLACKLIST, flags.filter_apis(HAS_NO_API_LIST_ASSIGNED)) + # Mark all remaining entries as blocked. + flags.assign_flag(FLAG_BLOCKED, flags.filter_apis(HAS_NO_API_LIST_ASSIGNED)) # Write output. write_lines(args["output"], flags.generate_csv()) diff --git a/tools/hiddenapi/generate_hiddenapi_lists_test.py b/tools/hiddenapi/generate_hiddenapi_lists_test.py index 55c3a7d718db..321c400ef898 100755 --- a/tools/hiddenapi/generate_hiddenapi_lists_test.py +++ b/tools/hiddenapi/generate_hiddenapi_lists_test.py @@ -23,7 +23,7 @@ class TestHiddenapiListGeneration(unittest.TestCase): # Initialize flags so that A and B are put on the whitelist and # C, D, E are left unassigned. Try filtering for the unassigned ones. flags = FlagsDict() - flags.parse_and_merge_csv(['A,' + FLAG_WHITELIST, 'B,' + FLAG_WHITELIST, + flags.parse_and_merge_csv(['A,' + FLAG_SDK, 'B,' + FLAG_SDK, 'C', 'D', 'E']) filter_set = flags.filter_apis(lambda api, flags: not flags) self.assertTrue(isinstance(filter_set, set)) @@ -32,10 +32,10 @@ class TestHiddenapiListGeneration(unittest.TestCase): def test_get_valid_subset_of_unassigned_keys(self): # Create flags where only A is unassigned. flags = FlagsDict() - flags.parse_and_merge_csv(['A,' + FLAG_WHITELIST, 'B', 'C']) - flags.assign_flag(FLAG_GREYLIST, set(['C'])) + flags.parse_and_merge_csv(['A,' + FLAG_SDK, 'B', 'C']) + flags.assign_flag(FLAG_UNSUPPORTED, set(['C'])) self.assertEqual(flags.generate_csv(), - [ 'A,' + FLAG_WHITELIST, 'B', 'C,' + FLAG_GREYLIST ]) + [ 'A,' + OLD_FLAG_SDK, 'B', 'C,' + OLD_FLAG_UNSUPPORTED ]) # Check three things: # (1) B is selected as valid unassigned @@ -50,20 +50,21 @@ class TestHiddenapiListGeneration(unittest.TestCase): # Test empty CSV entry. self.assertEqual(flags.generate_csv(), []) - # Test new additions. + # Test new additions. CSV generator produces values with old flags + # to be backwards compatible. flags.parse_and_merge_csv([ - 'A,' + FLAG_GREYLIST, - 'B,' + FLAG_BLACKLIST + ',' + FLAG_GREYLIST_MAX_O, - 'C,' + FLAG_SYSTEM_API + ',' + FLAG_WHITELIST, - 'D,' + FLAG_GREYLIST+ ',' + FLAG_TEST_API, - 'E,' + FLAG_BLACKLIST+ ',' + FLAG_TEST_API, + 'A,' + FLAG_UNSUPPORTED, + 'B,' + FLAG_BLOCKED + ',' + FLAG_MAX_TARGET_O, + 'C,' + FLAG_SDK + ',' + FLAG_SYSTEM_API, + 'D,' + FLAG_UNSUPPORTED + ',' + FLAG_TEST_API, + 'E,' + FLAG_BLOCKED + ',' + FLAG_TEST_API, ]) self.assertEqual(flags.generate_csv(), [ - 'A,' + FLAG_GREYLIST, - 'B,' + FLAG_BLACKLIST + "," + FLAG_GREYLIST_MAX_O, - 'C,' + FLAG_SYSTEM_API + ',' + FLAG_WHITELIST, - 'D,' + FLAG_GREYLIST+ ',' + FLAG_TEST_API, - 'E,' + FLAG_BLACKLIST+ ',' + FLAG_TEST_API, + 'A,' + OLD_FLAG_UNSUPPORTED, + 'B,' + OLD_FLAG_BLOCKED + "," + OLD_FLAG_MAX_TARGET_O, + 'C,' + FLAG_SYSTEM_API + ',' + OLD_FLAG_SDK, + 'D,' + OLD_FLAG_UNSUPPORTED + ',' + FLAG_TEST_API, + 'E,' + OLD_FLAG_BLOCKED + ',' + FLAG_TEST_API, ]) # Test unknown flag. @@ -72,16 +73,16 @@ class TestHiddenapiListGeneration(unittest.TestCase): def test_assign_flag(self): flags = FlagsDict() - flags.parse_and_merge_csv(['A,' + FLAG_WHITELIST, 'B']) + flags.parse_and_merge_csv(['A,' + FLAG_SDK, 'B']) # Test new additions. - flags.assign_flag(FLAG_GREYLIST, set([ 'A', 'B' ])) + flags.assign_flag(FLAG_UNSUPPORTED, set([ 'A', 'B' ])) self.assertEqual(flags.generate_csv(), - [ 'A,' + FLAG_GREYLIST + "," + FLAG_WHITELIST, 'B,' + FLAG_GREYLIST ]) + [ 'A,' + OLD_FLAG_UNSUPPORTED + "," + OLD_FLAG_SDK, 'B,' + OLD_FLAG_UNSUPPORTED ]) # Test invalid API signature. with self.assertRaises(AssertionError): - flags.assign_flag(FLAG_WHITELIST, set([ 'C' ])) + flags.assign_flag(FLAG_SDK, set([ 'C' ])) # Test invalid flag. with self.assertRaises(AssertionError): |