diff options
| -rwxr-xr-x | startop/scripts/app_startup/query_compiler_filter.py | 226 | ||||
| -rwxr-xr-x | startop/scripts/app_startup/query_compiler_filter_test.py | 116 | ||||
| -rwxr-xr-x | startop/scripts/app_startup/run_app_with_prefetch | 47 |
3 files changed, 384 insertions, 5 deletions
diff --git a/startop/scripts/app_startup/query_compiler_filter.py b/startop/scripts/app_startup/query_compiler_filter.py new file mode 100755 index 000000000000..dc97c6641f35 --- /dev/null +++ b/startop/scripts/app_startup/query_compiler_filter.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +# +# Copyright 2018, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# +# Query the current compiler filter for an application by its package name. +# (By parsing the results of the 'adb shell dumpsys package $package' command). +# The output is a string "$compilation_filter $compilation_reason $isa". +# +# See --help for more details. +# +# ----------------------------------- +# +# Sample usage: +# +# $> ./query_compiler_filter.py --package com.google.android.calculator +# speed-profile unknown arm64 +# + +import argparse +import sys +import re + +# TODO: refactor this with a common library file with analyze_metrics.py +import app_startup_runner +from app_startup_runner import _debug_print +from app_startup_runner import execute_arbitrary_command + +from typing import List, NamedTuple, Iterable + +_DEBUG_FORCE = None # Ignore -d/--debug if this is not none. + +def parse_options(argv: List[str] = None): + """Parse command line arguments and return an argparse Namespace object.""" + parser = argparse.ArgumentParser(description="Query the compiler filter for a package.") + # argparse considers args starting with - and -- optional in --help, even though required=True. + # by using a named argument group --help will clearly say that it's required instead of optional. + required_named = parser.add_argument_group('required named arguments') + required_named.add_argument('-p', '--package', action='store', dest='package', help='package of the application', required=True) + + # optional arguments + # use a group here to get the required arguments to appear 'above' the optional arguments in help. + optional_named = parser.add_argument_group('optional named arguments') + optional_named.add_argument('-i', '--isa', '--instruction-set', action='store', dest='instruction_set', help='which instruction set to select. defaults to the first one available if not specified.', choices=('arm64', 'arm', 'x86_64', 'x86')) + optional_named.add_argument('-s', '--simulate', dest='simulate', action='store_true', help='Print which commands will run, but don\'t run the apps') + optional_named.add_argument('-d', '--debug', dest='debug', action='store_true', help='Add extra debugging output') + + return parser.parse_args(argv) + +def remote_dumpsys_package(package: str, simulate: bool) -> str: + # --simulate is used for interactive debugging/development, but also for the unit test. + if simulate: + return """ +Dexopt state: + [%s] + path: /data/app/%s-D7s8PLidqqEq7Jc7UH_a5A==/base.apk + arm64: [status=speed-profile] [reason=unknown] + path: /data/app/%s-D7s8PLidqqEq7Jc7UH_a5A==/base.apk + arm: [status=speed] [reason=first-boot] + path: /data/app/%s-D7s8PLidqqEq7Jc7UH_a5A==/base.apk + x86: [status=quicken] [reason=install] +""" %(package, package, package, package) + + code, res = execute_arbitrary_command(['adb', 'shell', 'dumpsys', 'package', package], simulate=False, timeout=5) + if code: + return res + else: + raise AssertionError("Failed to dumpsys package, errors = %s", res) + +ParseTree = NamedTuple('ParseTree', [('label', str), ('children', List['ParseTree'])]) +DexoptState = ParseTree # With the Dexopt state: label +ParseResult = NamedTuple('ParseResult', [('remainder', List[str]), ('tree', ParseTree)]) + +def find_parse_subtree(parse_tree: ParseTree, match_regex: str) -> ParseTree: + if re.match(match_regex, parse_tree.label): + return parse_tree + + for node in parse_tree.children: + res = find_parse_subtree(node, match_regex) + if res: + return res + + return None + +def find_parse_children(parse_tree: ParseTree, match_regex: str) -> Iterable[ParseTree]: + for node in parse_tree.children: + if re.match(match_regex, node.label): + yield node + +def parse_tab_subtree(label: str, str_lines: List[str], separator=' ', indent=-1) -> ParseResult: + children = [] + + get_indent_level = lambda line: len(line) - len(line.lstrip()) + + line_num = 0 + + keep_going = True + while keep_going: + keep_going = False + + for line_num in range(len(str_lines)): + line = str_lines[line_num] + current_indent = get_indent_level(line) + + _debug_print("INDENT=%d, LINE=%s" %(current_indent, line)) + + current_label = line.lstrip() + + # skip empty lines + if line.lstrip() == "": + continue + + if current_indent > indent: + parse_result = parse_tab_subtree(current_label, str_lines[line_num+1::], separator, current_indent) + str_lines = parse_result.remainder + children.append(parse_result.tree) + keep_going = True + else: + # current_indent <= indent + keep_going = False + + break + + new_remainder = str_lines[line_num::] + _debug_print("NEW REMAINDER: ", new_remainder) + + parse_tree = ParseTree(label, children) + return ParseResult(new_remainder, parse_tree) + +def parse_tab_tree(str_tree: str, separator=' ', indentation_level=-1) -> ParseTree: + + label = None + lst = [] + + line_num = 0 + line_lst = str_tree.split("\n") + + return parse_tab_subtree("", line_lst, separator, indentation_level).tree + +def parse_dexopt_state(dumpsys_tree: ParseTree) -> DexoptState: + res = find_parse_subtree(dumpsys_tree, "Dexopt(\s+)state[:]?") + if not res: + raise AssertionError("Could not find the Dexopt state") + return res + +def find_first_compiler_filter(dexopt_state: DexoptState, package: str, instruction_set: str) -> str: + lst = find_all_compiler_filters(dexopt_state, package) + + _debug_print("all compiler filters: ", lst) + + for compiler_filter_info in lst: + if not instruction_set: + return compiler_filter_info + + if compiler_filter_info.isa == instruction_set: + return compiler_filter_info + + return None + +CompilerFilterInfo = NamedTuple('CompilerFilterInfo', [('isa', str), ('status', str), ('reason', str)]) + +def find_all_compiler_filters(dexopt_state: DexoptState, package: str) -> List[CompilerFilterInfo]: + + lst = [] + package_tree = find_parse_subtree(dexopt_state, re.escape("[%s]" %package)) + + if not package_tree: + raise AssertionError("Could not find any package subtree for package %s" %(package)) + + _debug_print("package tree: ", package_tree) + + for path_tree in find_parse_children(package_tree, "path: "): + _debug_print("path tree: ", path_tree) + + matchre = re.compile("([^:]+):\s+\[status=([^\]]+)\]\s+\[reason=([^\]]+)\]") + + for isa_node in find_parse_children(path_tree, matchre): + + matches = re.match(matchre, isa_node.label).groups() + + info = CompilerFilterInfo(*matches) + lst.append(info) + + return lst + +def main() -> int: + opts = parse_options() + app_startup_runner._debug = opts.debug + if _DEBUG_FORCE is not None: + app_startup_runner._debug = _DEBUG_FORCE + _debug_print("parsed options: ", opts) + + # Note: This can often 'fail' if the package isn't actually installed. + package_dumpsys = remote_dumpsys_package(opts.package, opts.simulate) + _debug_print("package dumpsys: ", package_dumpsys) + dumpsys_parse_tree = parse_tab_tree(package_dumpsys, package_dumpsys) + _debug_print("parse tree: ", dumpsys_parse_tree) + dexopt_state = parse_dexopt_state(dumpsys_parse_tree) + + filter = find_first_compiler_filter(dexopt_state, opts.package, opts.instruction_set) + + if filter: + print(filter.status, end=' ') + print(filter.reason, end=' ') + print(filter.isa) + else: + print("ERROR: Could not find any compiler-filter for package %s, isa %s" %(opts.package, opts.instruction_set), file=sys.stderr) + return 1 + + return 0 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/startop/scripts/app_startup/query_compiler_filter_test.py b/startop/scripts/app_startup/query_compiler_filter_test.py new file mode 100755 index 000000000000..a751a43d1d6c --- /dev/null +++ b/startop/scripts/app_startup/query_compiler_filter_test.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +# +# Copyright 2018, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +Unit tests for the query_compiler_filter.py script. + +Install: + $> sudo apt-get install python3-pytest ## OR + $> pip install -U pytest +See also https://docs.pytest.org/en/latest/getting-started.html + +Usage: + $> ./query_compiler_filter.py + $> pytest query_compiler_filter.py + $> python -m pytest query_compiler_filter.py + +See also https://docs.pytest.org/en/latest/usage.html +""" + +# global imports +from contextlib import contextmanager +import io +import shlex +import sys +import typing + +# pip imports +import pytest + +# local imports +import query_compiler_filter as qcf + +@contextmanager +def redirect_stdout_stderr(): + """Redirect stdout/stderr to a new StringIO for duration of context.""" + old_stdout = sys.stdout + old_stderr = sys.stderr + new_stdout = io.StringIO() + sys.stdout = new_stdout + new_stderr = io.StringIO() + sys.stderr = new_stderr + try: + yield (new_stdout, new_stderr) + finally: + sys.stdout = old_stdout + sys.stderr = old_stderr + # Seek back to the beginning so we can read whatever was written into it. + new_stdout.seek(0) + new_stderr.seek(0) + +@contextmanager +def replace_argv(argv): + """ Temporarily replace argv for duration of this context.""" + old_argv = sys.argv + sys.argv = [sys.argv[0]] + argv + try: + yield + finally: + sys.argv = old_argv + +def exec_main(argv): + """Run the query_compiler_filter main function with the provided arguments. + + Returns the stdout result when successful, assertion failure otherwise. + """ + try: + with redirect_stdout_stderr() as (the_stdout, the_stderr): + with replace_argv(argv): + code = qcf.main() + assert 0 == code, the_stderr.readlines() + + all_lines = the_stdout.readlines() + return "".join(all_lines) + finally: + the_stdout.close() + the_stderr.close() + +def test_query_compiler_filter(): + # no --instruction-set specified: provide whatever was the 'first' filter. + assert exec_main(['--simulate', + '--package', 'com.google.android.apps.maps']) == \ + "speed-profile unknown arm64\n" + + # specifying an instruction set finds the exact compiler filter match. + assert exec_main(['--simulate', + '--package', 'com.google.android.apps.maps', + '--instruction-set', 'arm64']) == \ + "speed-profile unknown arm64\n" + + assert exec_main(['--simulate', + '--package', 'com.google.android.apps.maps', + '--instruction-set', 'arm']) == \ + "speed first-boot arm\n" + + assert exec_main(['--simulate', + '--debug', + '--package', 'com.google.android.apps.maps', + '--instruction-set', 'x86']) == \ + "quicken install x86\n" + +if __name__ == '__main__': + pytest.main() diff --git a/startop/scripts/app_startup/run_app_with_prefetch b/startop/scripts/app_startup/run_app_with_prefetch index 1ff5fc64116f..ce63ff958613 100755 --- a/startop/scripts/app_startup/run_app_with_prefetch +++ b/startop/scripts/app_startup/run_app_with_prefetch @@ -92,8 +92,7 @@ parse_arguments() { shift ;; --compiler-filter) - # ignore any '--compiler-filter xyz' settings. - # FIXME: app_startup_runner.py should not be passing this flag. + compiler_filter="$2" shift ;; *) @@ -194,9 +193,6 @@ if [[ $? -ne 0 ]]; then fi verbose_print "Package was in path '$package_path'" - - - keep_application_trace_file=n application_trace_file_path="$package_path/TraceFile.pb" trace_file_directory="$package_path" @@ -286,6 +282,47 @@ perform_aot_cleanup() { fi } +configure_compiler_filter() { + local the_compiler_filter="$1" + local the_package="$2" + local the_activity="$3" + + if [[ -z $the_compiler_filter ]]; then + verbose_print "No --compiler-filter specified, don't need to force it." + return 0 + fi + + local current_compiler_filter_info="$("$DIR"/query_compiler_filter.py --package "$the_package")" + local res=$? + if [[ $res -ne 0 ]]; then + return $res + fi + + local current_compiler_filter + local current_reason + local current_isa + read current_compiler_filter current_reason current_isa <<< "$current_compiler_filter_info" + + verbose_print "Compiler Filter="$current_compiler_filter "Reason="$current_reason "Isa="$current_isa + + # Don't trust reasons that aren't 'unknown' because that means we didn't manually force the compilation filter. + # (e.g. if any automatic system-triggered compilations are not unknown). + if [[ $current_reason != "unknown" ]] || [[ $current_compiler_filter != $the_compiler_filter ]]; then + verbose_print "$DIR"/force_compiler_filter --compiler-filter "$the_compiler_filter" --package "$the_package" --activity "$the_activity" + "$DIR"/force_compiler_filter --compiler-filter "$the_compiler_filter" --package "$the_package" --activity "$the_activity" + res=$? + else + verbose_print "Queried compiler-filter matched requested compiler-filter, skip forcing." + res=0 + fi + + return $res +} + +# Ensure the APK is currently compiled with whatever we passed in via --compiler-filter. +# No-op if this option was not passed in. +configure_compiler_filter "$compiler_filter" "$package" "$activity" || exit 1 + # TODO: This loop logic could probably be moved into app_startup_runner.py for ((i=0;i<count;++i)) do verbose_print "==========================================" |