diff options
| -rw-r--r-- | Android.mk | 153 | ||||
| -rwxr-xr-x | tools/hiddenapi/generate_hiddenapi_lists.py | 241 | ||||
| -rwxr-xr-x | tools/hiddenapi/generate_hiddenapi_lists_test.py | 89 |
3 files changed, 358 insertions, 125 deletions
diff --git a/Android.mk b/Android.mk index 84b708e53cdb..988c009bdbac 100644 --- a/Android.mk +++ b/Android.mk @@ -322,131 +322,34 @@ $(OUT_DOCS)/offline-sdk-timestamp: $(OUT_DOCS)/offline-sdk-docs-docs.zip ( unzip -qo $< -d $(OUT_DOCS)/offline-sdk && touch -f $@ ) || exit 1 # ==== hiddenapi lists ======================================= -include $(CLEAR_VARS) - -# File names of final API lists -LOCAL_WHITELIST := $(INTERNAL_PLATFORM_HIDDENAPI_WHITELIST) -LOCAL_LIGHT_GREYLIST := $(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST) -LOCAL_DARK_GREYLIST := $(INTERNAL_PLATFORM_HIDDENAPI_DARK_GREYLIST) -LOCAL_BLACKLIST := $(INTERNAL_PLATFORM_HIDDENAPI_BLACKLIST) - -# File names of source files we will use to generate the final API lists. -LOCAL_SRC_GREYLIST := frameworks/base/config/hiddenapi-light-greylist.txt -LOCAL_SRC_VENDOR_LIST := frameworks/base/config/hiddenapi-vendor-list.txt -LOCAL_SRC_FORCE_BLACKLIST := frameworks/base/config/hiddenapi-force-blacklist.txt -LOCAL_SRC_PUBLIC_API := $(INTERNAL_PLATFORM_HIDDENAPI_PUBLIC_LIST) -LOCAL_SRC_PRIVATE_API := $(INTERNAL_PLATFORM_HIDDENAPI_PRIVATE_LIST) -LOCAL_SRC_REMOVED_API := $(INTERNAL_PLATFORM_REMOVED_DEX_API_FILE) - -LOCAL_SRC_ALL := \ - $(LOCAL_SRC_GREYLIST) \ - $(LOCAL_SRC_VENDOR_LIST) \ - $(LOCAL_SRC_FORCE_BLACKLIST) \ - $(LOCAL_SRC_PUBLIC_API) \ - $(LOCAL_SRC_PRIVATE_API) \ - $(LOCAL_SRC_REMOVED_API) - -define assert-has-no-overlap -if [ ! -z "`comm -12 <(sort $(1)) <(sort $(2))`" ]; then \ - echo "$(1) and $(2) should not overlap" 1>&2; \ - comm -12 <(sort $(1)) <(sort $(2)) 1>&2; \ - exit 1; \ -fi -endef - -define assert-is-subset -if [ ! -z "`comm -23 <(sort $(1)) <(sort $(2))`" ]; then \ - echo "$(1) must be a subset of $(2)" 1>&2; \ - comm -23 <(sort $(1)) <(sort $(2)) 1>&2; \ - exit 1; \ -fi -endef - -define assert-has-no-duplicates -if [ ! -z "`sort $(1) | uniq -D`" ]; then \ - echo "$(1) has duplicate entries" 1>&2; \ - sort $(1) | uniq -D 1>&2; \ - exit 1; \ -fi -endef - -# The following rules build API lists in the build folder. -# By not using files from the source tree, ART buildbots can mock these lists -# or have alternative rules for building them. Other rules in the build system -# should depend on the files in the build folder. - -# Merge whitelist from: -# (1) public API stubs -# (2) whitelist entries generated by class2greylist (PRIVATE_WHITELIST_INPUTS) -$(LOCAL_WHITELIST): $(LOCAL_SRC_PUBLIC_API) - sort $(LOCAL_SRC_PUBLIC_API) $(PRIVATE_WHITELIST_INPUTS) > $@ - $(call assert-has-no-duplicates,$@) - -# Merge light greylist from multiple files: -# (1) manual greylist LOCAL_SRC_GREYLIST -# (2) list of usages from vendor apps LOCAL_SRC_VENDOR_LIST -# (3) list of removed APIs in LOCAL_SRC_REMOVED_API -# @removed does not imply private in Doclava. We must take the subset also -# in LOCAL_SRC_PRIVATE_API. -# (4) list of serialization APIs -# Automatically adds all methods which match the signatures in -# REGEX_SERIALIZATION. These are greylisted in order to allow applications -# to write their own serializers. -# (5) greylist entries generated by class2greylist (PRIVATE_GREYLIST_INPUTS) -$(LOCAL_LIGHT_GREYLIST): REGEX_SERIALIZATION := \ - "readObject\(Ljava/io/ObjectInputStream;\)V" \ - "readObjectNoData\(\)V" \ - "readResolve\(\)Ljava/lang/Object;" \ - "serialVersionUID:J" \ - "serialPersistentFields:\[Ljava/io/ObjectStreamField;" \ - "writeObject\(Ljava/io/ObjectOutputStream;\)V" \ - "writeReplace\(\)Ljava/lang/Object;" -$(LOCAL_LIGHT_GREYLIST): $(LOCAL_SRC_ALL) $(LOCAL_WHITELIST) - sort $(LOCAL_SRC_GREYLIST) $(LOCAL_SRC_VENDOR_LIST) $(PRIVATE_GREYLIST_INPUTS) \ - <(grep -E "\->("$(subst $(space),"|",$(REGEX_SERIALIZATION))")$$" \ - $(LOCAL_SRC_PRIVATE_API)) \ - <(comm -12 <(sort $(LOCAL_SRC_REMOVED_API)) <(sort $(LOCAL_SRC_PRIVATE_API))) \ - > $@ - $(call assert-has-no-duplicates,$@) - $(call assert-is-subset,$@,$(LOCAL_SRC_PRIVATE_API)) - $(call assert-has-no-overlap,$@,$(LOCAL_WHITELIST)) - $(call assert-has-no-overlap,$@,$(LOCAL_SRC_FORCE_BLACKLIST)) - -# Generate dark greylist as remaining classes and class members in the same -# package as classes listed in the light greylist. -# The algorithm is as follows: -# (1) extract the class descriptor from each entry in LOCAL_LIGHT_GREYLIST -# (2) strip everything after the last forward-slash, -# e.g. 'Lpackage/subpackage/class$inner;' turns into 'Lpackage/subpackage/' -# (3) insert all entries from LOCAL_SRC_PRIVATE_API which begin with the package -# name but do not contain another forward-slash in the class name, e.g. -# matching '^Lpackage/subpackage/[^/;]*;' -# (4) subtract entries shared with LOCAL_LIGHT_GREYLIST -$(LOCAL_DARK_GREYLIST): $(LOCAL_SRC_ALL) $(LOCAL_WHITELIST) $(LOCAL_LIGHT_GREYLIST) - comm -13 <(sort $(LOCAL_WHITELIST) $(LOCAL_LIGHT_GREYLIST) $(LOCAL_SRC_FORCE_BLACKLIST)) \ - <(cat $(LOCAL_WHITELIST) $(LOCAL_LIGHT_GREYLIST) | \ - sed 's/\->.*//' | sed 's/\(.*\/\).*/\1/' | sort | uniq | \ - while read PKG_NAME; do \ - grep -E "^$${PKG_NAME}[^/;]*;" $(LOCAL_SRC_PRIVATE_API); \ - done | sort | uniq) \ - > $@ - $(call assert-is-subset,$@,$(LOCAL_SRC_PRIVATE_API)) - $(call assert-has-no-duplicates,$@) - $(call assert-has-no-overlap,$@,$(LOCAL_WHITELIST)) - $(call assert-has-no-overlap,$@,$(LOCAL_LIGHT_GREYLIST)) - $(call assert-has-no-overlap,$@,$(LOCAL_SRC_FORCE_BLACKLIST)) - -# Generate blacklist as private API minus (light greylist plus dark greylist). -$(LOCAL_BLACKLIST): $(LOCAL_SRC_ALL) $(LOCAL_WHITELIST) $(LOCAL_LIGHT_GREYLIST) $(LOCAL_DARK_GREYLIST) - comm -13 <(sort $(LOCAL_WHITELIST) $(LOCAL_LIGHT_GREYLIST) $(LOCAL_DARK_GREYLIST)) \ - <(sort $(LOCAL_SRC_PRIVATE_API)) \ - > $@ - $(call assert-is-subset,$@,$(LOCAL_SRC_PRIVATE_API)) - $(call assert-has-no-duplicates,$@) - $(call assert-has-no-overlap,$@,$(LOCAL_WHITELIST)) - $(call assert-has-no-overlap,$@,$(LOCAL_LIGHT_GREYLIST)) - $(call assert-has-no-overlap,$@,$(LOCAL_DARK_GREYLIST)) - $(call assert-is-subset,$(LOCAL_SRC_FORCE_BLACKLIST),$@) +$(INTERNAL_PLATFORM_HIDDENAPI_WHITELIST): \ + .KATI_IMPLICIT_OUTPUTS := \ + $(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST) \ + $(INTERNAL_PLATFORM_HIDDENAPI_DARK_GREYLIST) \ + $(INTERNAL_PLATFORM_HIDDENAPI_BLACKLIST) +$(INTERNAL_PLATFORM_HIDDENAPI_WHITELIST): \ + frameworks/base/tools/hiddenapi/generate_hiddenapi_lists.py \ + frameworks/base/config/hiddenapi-light-greylist.txt \ + frameworks/base/config/hiddenapi-vendor-list.txt \ + frameworks/base/config/hiddenapi-force-blacklist.txt \ + $(INTERNAL_PLATFORM_HIDDENAPI_PUBLIC_LIST) \ + $(INTERNAL_PLATFORM_HIDDENAPI_PRIVATE_LIST) \ + $(INTERNAL_PLATFORM_REMOVED_DEX_API_FILE) + frameworks/base/tools/hiddenapi/generate_hiddenapi_lists.py \ + --input-public $(INTERNAL_PLATFORM_HIDDENAPI_PUBLIC_LIST) \ + --input-private $(INTERNAL_PLATFORM_HIDDENAPI_PRIVATE_LIST) \ + --input-whitelists $(PRIVATE_WHITELIST_INPUTS) \ + --input-greylists \ + frameworks/base/config/hiddenapi-light-greylist.txt \ + frameworks/base/config/hiddenapi-vendor-list.txt \ + <(comm -12 <(sort $(INTERNAL_PLATFORM_REMOVED_DEX_API_FILE)) \ + $(INTERNAL_PLATFORM_HIDDENAPI_PRIVATE_LIST)) \ + $(PRIVATE_GREYLIST_INPUTS) \ + --input-blacklists frameworks/base/config/hiddenapi-force-blacklist.txt \ + --output-whitelist $(INTERNAL_PLATFORM_HIDDENAPI_WHITELIST) \ + --output-light-greylist $(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST) \ + --output-dark-greylist $(INTERNAL_PLATFORM_HIDDENAPI_DARK_GREYLIST) \ + --output-blacklist $(INTERNAL_PLATFORM_HIDDENAPI_BLACKLIST) # Include subdirectory makefiles # ============================================================ diff --git a/tools/hiddenapi/generate_hiddenapi_lists.py b/tools/hiddenapi/generate_hiddenapi_lists.py new file mode 100755 index 000000000000..4a0931a149af --- /dev/null +++ b/tools/hiddenapi/generate_hiddenapi_lists.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python +# +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Generate API lists for non-SDK API enforcement. + +usage: generate-hiddenapi-lists.py [-h] + --input-public INPUT_PUBLIC + --input-private INPUT_PRIVATE + [--input-whitelists [INPUT_WHITELISTS [INPUT_WHITELISTS ...]]] + [--input-greylists [INPUT_GREYLISTS [INPUT_GREYLISTS ...]]] + [--input-blacklists [INPUT_BLACKLISTS [INPUT_BLACKLISTS ...]]] + --output-whitelist OUTPUT_WHITELIST + --output-light-greylist OUTPUT_LIGHT_GREYLIST + --output-dark-greylist OUTPUT_DARK_GREYLIST + --output-blacklist OUTPUT_BLACKLIST +""" +import argparse +import os +import sys +import re + +def get_args(): + """Parses command line arguments. + + Returns: + Namespace: dictionary of parsed arguments + """ + parser = argparse.ArgumentParser() + parser.add_argument('--input-public', required=True, help='List of all public members') + parser.add_argument('--input-private', required=True, help='List of all private members') + parser.add_argument( + '--input-whitelists', nargs='*', + help='Lists of members to force on whitelist') + parser.add_argument( + '--input-greylists', nargs='*', + help='Lists of members to force on light greylist') + parser.add_argument( + '--input-blacklists', nargs='*', + help='Lists of members to force on blacklist') + parser.add_argument('--output-whitelist', required=True) + parser.add_argument('--output-light-greylist', required=True) + parser.add_argument('--output-dark-greylist', required=True) + parser.add_argument('--output-blacklist', required=True) + return parser.parse_args() + +def read_lines(filename): + """Reads entire file and return it as a list of lines. + + Args: + filename (string): Path to the file to read from. + + Returns: + list: Lines of the loaded file as a list of strings. + """ + with open(filename, 'r') as f: + return f.readlines() + +def write_lines(filename, lines): + """Writes list of lines into a file, overwriting the file it it exists. + + Args: + filename (string): Path to the file to be writting into. + lines (list): List of strings to write into the file. + """ + with open(filename, 'w') as f: + f.writelines(lines) + +def move_between_sets(subset, src, dst, source = "<unknown>"): + """Removes a subset of elements from one set and add it to another. + + Args: + subset (set): The subset of `src` to be moved from `src` to `dst`. + src (set): Source set. Must be a superset of `subset`. + dst (set): Destination set. Must be disjoint with `subset`. + """ + assert src.issuperset(subset), ( + "Error processing: {}\n" + "The following entries were not found:\n" + "{}" + "Please visit go/hiddenapi for more information.").format( + source, "".join(map(lambda x: " " + str(x), subset.difference(src)))) + assert dst.isdisjoint(subset) + # Order matters if `src` and `subset` are the same object. + dst.update(subset) + src.difference_update(subset) + +def get_package_name(signature): + """Returns the package name prefix of a class member signature. + + Example: "Ljava/lang/String;->hashCode()J" --> "Ljava/lang/" + + Args: + signature (string): Member signature + + Returns + string: Package name of the given member + """ + class_name_end = signature.find("->") + assert class_name_end != -1, "Invalid signature: {}".format(signature) + package_name_end = signature.rfind("/", 0, class_name_end) + assert package_name_end != -1, "Invalid signature: {}".format(signature) + return signature[:package_name_end + 1] + +def all_package_names(*args): + """Returns a set of packages names in given lists of member signatures. + + Example: args = [ set([ "Lpkg1/ClassA;->foo()V", "Lpkg2/ClassB;->bar()J" ]), + set([ "Lpkg1/ClassC;->baz()Z" ]) ] + return value = set([ "Lpkg1/", "Lpkg2" ]) + + Args: + *args (list): List of sets to iterate over and extract the package names + of its elements (member signatures) + + Returns: + set: All package names extracted from the given lists of signatures. + """ + packages = set() + for arg in args: + packages = packages.union(map(get_package_name, arg)) + return packages + +def move_all(src, dst): + """Moves all elements of one set to another. + + Args: + src (set): Source set. Will become empty. + dst (set): Destination set. Will contain all elements of `src`. + """ + move_between_sets(src, src, dst) + +def move_from_files(filenames, src, dst): + """Loads member signatures from a list of files and moves them to a given set. + + Opens files in `filenames`, reads all their lines and moves those from `src` + set to `dst` set. + + Args: + filenames (list): List of paths to files to be loaded. + src (set): Set that loaded lines should be moved from. + dst (set): Set that loaded lines should be moved to. + """ + if filenames: + for filename in filenames: + move_between_sets(set(read_lines(filename)), src, dst, filename) + +def move_serialization(src, dst): + """Moves all members matching serialization API signatures between given sets. + + Args: + src (set): Set that will be searched for serialization API and that API + will be removed from it. + dst (set): Set that serialization API will be moved to. + """ + serialization_patterns = [ + r'readObject\(Ljava/io/ObjectInputStream;\)V', + r'readObjectNoData\(\)V', + r'readResolve\(\)Ljava/lang/Object;', + r'serialVersionUID:J', + r'serialPersistentFields:\[Ljava/io/ObjectStreamField;', + r'writeObject\(Ljava/io/ObjectOutputStream;\)V', + r'writeReplace\(\)Ljava/lang/Object;', + ] + regex = re.compile(r'.*->(' + '|'.join(serialization_patterns) + r')$') + move_between_sets(filter(lambda api: regex.match(api), src), src, dst) + +def move_from_packages(packages, src, dst): + """Moves all members of given package names from one set to another. + + Args: + packages (list): List of string package names. + src (set): Set that will be searched for API matching one of the given + package names. Surch API will be removed from the set. + dst (set): Set that matching API will be moved to. + """ + move_between_sets(filter(lambda api: get_package_name(api) in packages, src), src, dst) + +def main(argv): + args = get_args() + + # Initialize API sets by loading lists of public and private API. Public API + # are all members resolvable from SDK API stubs, other members are private. + # As an optimization, skip the step of moving public API from a full set of + # members and start with a populated whitelist. + whitelist = set(read_lines(args.input_public)) + uncategorized = set(read_lines(args.input_private)) + light_greylist = set() + dark_greylist = set() + blacklist = set() + + # Assert that there is no overlap between public and private API. + assert whitelist.isdisjoint(uncategorized) + num_all_api = len(whitelist) + len(uncategorized) + + # Read all files which manually assign members to specific lists. + move_from_files(args.input_whitelists, uncategorized, whitelist) + move_from_files(args.input_greylists, uncategorized, light_greylist) + move_from_files(args.input_blacklists, uncategorized, blacklist) + + # Iterate over all uncategorized members and move serialization API to light greylist. + move_serialization(uncategorized, light_greylist) + + # Extract package names of members from whitelist and light greylist, which + # are assumed to have been finalized at this point. Assign all uncategorized + # members from the same packages to the dark greylist. + dark_greylist_packages = all_package_names(whitelist, light_greylist) + move_from_packages(dark_greylist_packages, uncategorized, dark_greylist) + + # Assign all uncategorized members to the blacklist. + move_all(uncategorized, blacklist) + + # Assert we have not missed anything. + assert whitelist.isdisjoint(light_greylist) + assert whitelist.isdisjoint(dark_greylist) + assert whitelist.isdisjoint(blacklist) + assert light_greylist.isdisjoint(dark_greylist) + assert light_greylist.isdisjoint(blacklist) + assert dark_greylist.isdisjoint(blacklist) + assert num_all_api == len(whitelist) + len(light_greylist) + len(dark_greylist) + len(blacklist) + + # Write final lists to disk. + write_lines(args.output_whitelist, whitelist) + write_lines(args.output_light_greylist, light_greylist) + write_lines(args.output_dark_greylist, dark_greylist) + write_lines(args.output_blacklist, blacklist) + +if __name__ == "__main__": + main(sys.argv) diff --git a/tools/hiddenapi/generate_hiddenapi_lists_test.py b/tools/hiddenapi/generate_hiddenapi_lists_test.py new file mode 100755 index 000000000000..8f793189650c --- /dev/null +++ b/tools/hiddenapi/generate_hiddenapi_lists_test.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +# +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for Hidden API list generation.""" +import unittest +from generate_hiddenapi_lists import * + +class TestHiddenapiListGeneration(unittest.TestCase): + + def test_move_between_sets(self): + A = set([1, 2, 3, 4]) + B = set([5, 6, 7, 8]) + move_between_sets(set([2, 4]), A, B) + self.assertEqual(A, set([1, 3])) + self.assertEqual(B, set([2, 4, 5, 6, 7, 8])) + + def test_move_between_sets_fail_not_superset(self): + A = set([1, 2, 3, 4]) + B = set([5, 6, 7, 8]) + with self.assertRaises(AssertionError) as ar: + move_between_sets(set([0, 2]), A, B) + + def test_move_between_sets_fail_not_disjoint(self): + A = set([1, 2, 3, 4]) + B = set([4, 5, 6, 7, 8]) + with self.assertRaises(AssertionError) as ar: + move_between_sets(set([1, 4]), A, B) + + def test_get_package_name(self): + self.assertEqual(get_package_name("Ljava/lang/String;->clone()V"), "Ljava/lang/") + + def test_get_package_name_fail_no_arrow(self): + with self.assertRaises(AssertionError) as ar: + get_package_name("Ljava/lang/String;-clone()V") + with self.assertRaises(AssertionError) as ar: + get_package_name("Ljava/lang/String;>clone()V") + with self.assertRaises(AssertionError) as ar: + get_package_name("Ljava/lang/String;__clone()V") + + def test_get_package_name_fail_no_package(self): + with self.assertRaises(AssertionError) as ar: + get_package_name("LString;->clone()V") + + def test_all_package_names(self): + self.assertEqual(all_package_names(), set()) + self.assertEqual(all_package_names(set(["Lfoo/Bar;->baz()V"])), set(["Lfoo/"])) + self.assertEqual( + all_package_names(set(["Lfoo/Bar;->baz()V", "Lfoo/BarX;->bazx()I"])), + set(["Lfoo/"])) + self.assertEqual( + all_package_names( + set(["Lfoo/Bar;->baz()V"]), + set(["Lfoo/BarX;->bazx()I", "Labc/xyz/Mno;->ijk()J"])), + set(["Lfoo/", "Labc/xyz/"])) + + def test_move_all(self): + src = set([ "abc", "xyz" ]) + dst = set([ "def" ]) + move_all(src, dst) + self.assertEqual(src, set()) + self.assertEqual(dst, set([ "abc", "def", "xyz" ])) + + def test_move_from_packages(self): + src = set([ "Lfoo/bar/ClassA;->abc()J", # will be moved + "Lfoo/bar/ClassA;->def()J", # will be moved + "Lcom/pkg/example/ClassD;->ijk:J", # not moved: different package + "Lfoo/bar/xyz/ClassC;->xyz()Z" ]) # not moved: subpackage + dst = set() + packages = set([ "Lfoo/bar/" ]) + move_from_packages(packages, src, dst) + self.assertEqual( + src, set([ "Lfoo/bar/xyz/ClassC;->xyz()Z", "Lcom/pkg/example/ClassD;->ijk:J" ])) + self.assertEqual( + dst, set([ "Lfoo/bar/ClassA;->abc()J", "Lfoo/bar/ClassA;->def()J" ])) + +if __name__ == '__main__': + unittest.main() |