diff options
Diffstat (limited to 'startop')
| -rwxr-xr-x | startop/scripts/app_startup/app_startup_runner.py | 48 | ||||
| -rwxr-xr-x | startop/scripts/app_startup/app_startup_runner_test.py | 2 | ||||
| -rw-r--r--[-rwxr-xr-x] | startop/scripts/iorap/compiler.py | 356 | ||||
| -rw-r--r-- | startop/scripts/iorap/compiler_device.py | 68 | ||||
| -rwxr-xr-x | startop/scripts/iorap/compiler_ri.py | 323 | ||||
| -rw-r--r-- | startop/scripts/iorap/compiler_test.py | 2 | ||||
| -rw-r--r-- | startop/scripts/iorap/lib/iorapd_utils.py | 21 |
7 files changed, 494 insertions, 326 deletions
diff --git a/startop/scripts/app_startup/app_startup_runner.py b/startop/scripts/app_startup/app_startup_runner.py index eb582f946958..fa1c4e601f83 100755 --- a/startop/scripts/app_startup/app_startup_runner.py +++ b/startop/scripts/app_startup/app_startup_runner.py @@ -41,11 +41,12 @@ DIR = os.path.abspath(os.path.dirname(__file__)) sys.path.append(os.path.dirname(DIR)) import lib.cmd_utils as cmd_utils import lib.print_utils as print_utils -import iorap.compiler as compiler from app_startup.run_app_with_prefetch import PrefetchAppRunner import app_startup.lib.args_utils as args_utils from app_startup.lib.data_frame import DataFrame from app_startup.lib.perfetto_trace_collector import PerfettoTraceCollector +from iorap.compiler import CompilerType +import iorap.compiler as compiler # The following command line options participate in the combinatorial generation. # All other arguments have a global effect. @@ -58,8 +59,6 @@ _RUN_SCRIPT = os.path.join(os.path.dirname(os.path.realpath(__file__)), CollectorPackageInfo = NamedTuple('CollectorPackageInfo', [('package', str), ('compiler_filter', str)]) -_COMPILER_SCRIPT = os.path.join(os.path.dirname(os.path.dirname( - os.path.realpath(__file__))), 'iorap/compiler.py') # by 2; systrace starts up slowly. _UNLOCK_SCREEN_SCRIPT = os.path.join( @@ -135,6 +134,10 @@ def parse_options(argv: List[str] = None): action='append', help='The trace duration (milliseconds) in ' 'compilation') + optional_named.add_argument('--compiler-type', dest='compiler_type', + type=CompilerType, choices=list(CompilerType), + default=CompilerType.DEVICE, + help='The type of compiler.') return parser.parse_args(argv) @@ -211,26 +214,26 @@ def parse_run_script_csv_file(csv_file: TextIO) -> DataFrame: return DataFrame(d) -def compile_perfetto_trace(inodes_path: str, +def build_ri_compiler_argv(inodes_path: str, perfetto_trace_file: str, - trace_duration: Optional[timedelta]) -> TextIO: - compiler_trace_file = tempfile.NamedTemporaryFile() - argv = [_COMPILER_SCRIPT, '-i', inodes_path, '--perfetto-trace', - perfetto_trace_file, '-o', compiler_trace_file.name] + trace_duration: Optional[timedelta] + ) -> str: + argv = ['-i', inodes_path, '--perfetto-trace', + perfetto_trace_file] if trace_duration is not None: argv += ['--duration', str(int(trace_duration.total_seconds() - * PerfettoTraceCollector.MS_PER_SEC))] + * PerfettoTraceCollector.MS_PER_SEC))] print_utils.debug_print(argv) - compiler.main(argv) - return compiler_trace_file + return argv def execute_run_using_perfetto_trace(collector_info, run_combos: Iterable[RunCommandArgs], simulate: bool, inodes_path: str, - timeout: int) -> DataFrame: + timeout: int, + compiler_type: CompilerType) -> DataFrame: """ Executes run based on perfetto trace. """ passed, perfetto_trace_file = run_perfetto_collector(collector_info, timeout, @@ -244,9 +247,15 @@ def execute_run_using_perfetto_trace(collector_info, if simulate: compiler_trace_file = tempfile.NamedTemporaryFile() else: - compiler_trace_file = compile_perfetto_trace(inodes_path, - perfetto_trace_file.name, - combos.trace_duration) + ri_compiler_argv = build_ri_compiler_argv(inodes_path, + perfetto_trace_file.name, + combos.trace_duration) + compiler_trace_file = compiler.compile(compiler_type, + inodes_path, + ri_compiler_argv, + combos.package, + combos.activity) + with compiler_trace_file: combos = combos._replace(input=compiler_trace_file.name) print_utils.debug_print(combos) @@ -261,7 +270,8 @@ def execute_run_combos( grouped_run_combos: Iterable[Tuple[CollectorPackageInfo, Iterable[RunCommandArgs]]], simulate: bool, inodes_path: str, - timeout: int): + timeout: int, + compiler_type: CompilerType): # nothing will work if the screen isn't unlocked first. cmd_utils.execute_arbitrary_command([_UNLOCK_SCREEN_SCRIPT], timeout, @@ -273,7 +283,8 @@ def execute_run_combos( run_combos, simulate, inodes_path, - timeout) + timeout, + compiler_type) def gather_results(commands: Iterable[Tuple[DataFrame]], key_list: List[str], value_list: List[Tuple[str, ...]]): @@ -361,7 +372,8 @@ def main(): exec = execute_run_combos(grouped_combos(), opts.simulate, opts.inodes, - opts.timeout) + opts.timeout, + opts.compiler_type) results = gather_results(exec, _COMBINATORIAL_OPTIONS, combos()) diff --git a/startop/scripts/app_startup/app_startup_runner_test.py b/startop/scripts/app_startup/app_startup_runner_test.py index 42ea5f0e4bb9..382f6f3c70ff 100755 --- a/startop/scripts/app_startup/app_startup_runner_test.py +++ b/startop/scripts/app_startup/app_startup_runner_test.py @@ -92,7 +92,7 @@ def default_dict_for_parsed_args(**kwargs): """ d = {'compiler_filters': None, 'simulate': False, 'debug': False, 'output': None, 'timeout': 10, 'loop_count': 1, 'inodes': None, - 'trace_duration': None} + 'trace_duration': None, 'compiler_type': asr.CompilerType.HOST} d.update(kwargs) return d diff --git a/startop/scripts/iorap/compiler.py b/startop/scripts/iorap/compiler.py index 17b58c19ef28..1426d34f325d 100755..100644 --- a/startop/scripts/iorap/compiler.py +++ b/startop/scripts/iorap/compiler.py @@ -1,323 +1,73 @@ #!/usr/bin/env python3 - # -# Copyright (C) 2019 The Android Open Source Project +# Copyright 2019, The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# Dependencies: -# -# $> sudo apt-get install python3-pip -# $> pip3 install --user protobuf sqlalchemy sqlite3 -# - -import optparse +import importlib import os -import re import sys import tempfile -from pathlib import Path -from datetime import timedelta -from typing import Iterable, Optional, List +from enum import Enum +from typing import TextIO, List +# local import DIR = os.path.abspath(os.path.dirname(__file__)) sys.path.append(os.path.dirname(DIR)) -from iorap.generated.TraceFile_pb2 import * -from iorap.lib.inode2filename import Inode2Filename - -parent_dir_name = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -sys.path.append(parent_dir_name) -from trace_analyzer.lib.trace2db import Trace2Db, MmFilemapAddToPageCache, \ - RawFtraceEntry -import lib.cmd_utils as cmd_utils - -_PAGE_SIZE = 4096 # adb shell getconf PAGESIZE ## size of a memory page in bytes. -ANDROID_BUILD_TOP = Path(parent_dir_name).parents[3] -TRACECONV_BIN = ANDROID_BUILD_TOP.joinpath( - 'external/perfetto/tools/traceconv') - -class PageRun: - """ - Intermediate representation for a run of one or more pages. - """ - def __init__(self, device_number: int, inode: int, offset: int, length: int): - self.device_number = device_number - self.inode = inode - self.offset = offset - self.length = length - - def __str__(self): - return "PageRun(device_number=%d, inode=%d, offset=%d, length=%d)" \ - %(self.device_number, self.inode, self.offset, self.length) - -def debug_print(msg): - #print(msg) - pass - -UNDER_LAUNCH = False - -def page_cache_entries_to_runs(page_cache_entries: Iterable[MmFilemapAddToPageCache]): - global _PAGE_SIZE - - runs = [ - PageRun(device_number=pg_entry.dev, inode=pg_entry.ino, offset=pg_entry.ofs, - length=_PAGE_SIZE) - for pg_entry in page_cache_entries - ] - - for r in runs: - debug_print(r) - - print("Stats: Page runs totaling byte length: %d" %(len(runs) * _PAGE_SIZE)) - - return runs - -def optimize_page_runs(page_runs): - new_entries = [] - last_entry = None - for pg_entry in page_runs: - if last_entry: - if pg_entry.device_number == last_entry.device_number and pg_entry.inode == last_entry.inode: - # we are dealing with a run for the same exact file as a previous run. - if pg_entry.offset == last_entry.offset + last_entry.length: - # trivially contiguous entries. merge them together. - last_entry.length += pg_entry.length - continue - # Default: Add the run without merging it to a previous run. - last_entry = pg_entry - new_entries.append(pg_entry) - return new_entries - -def is_filename_matching_filter(file_name, filters=[]): - """ - Blacklist-style regular expression filters. - - :return: True iff file_name has an RE match in one of the filters. - """ - for filt in filters: - res = re.search(filt, file_name) - if res: - return True - - return False - -def build_protobuf(page_runs, inode2filename, filters=[]): - trace_file = TraceFile() - trace_file_index = trace_file.index - - file_id_counter = 0 - file_id_map = {} # filename -> id - - stats_length_total = 0 - filename_stats = {} # filename -> total size - - skipped_inode_map = {} - filtered_entry_map = {} # filename -> count - - for pg_entry in page_runs: - fn = inode2filename.resolve(pg_entry.device_number, pg_entry.inode) - if not fn: - skipped_inode_map[pg_entry.inode] = skipped_inode_map.get(pg_entry.inode, 0) + 1 - continue - - filename = fn - - if filters and not is_filename_matching_filter(filename, filters): - filtered_entry_map[filename] = filtered_entry_map.get(filename, 0) + 1 - continue - - file_id = file_id_map.get(filename) - if not file_id: - file_id = file_id_counter - file_id_map[filename] = file_id_counter - file_id_counter = file_id_counter + 1 - - file_index_entry = trace_file_index.entries.add() - file_index_entry.id = file_id - file_index_entry.file_name = filename - - # already in the file index, add the file entry. - file_entry = trace_file.list.entries.add() - file_entry.index_id = file_id - file_entry.file_length = pg_entry.length - stats_length_total += file_entry.file_length - file_entry.file_offset = pg_entry.offset - - filename_stats[filename] = filename_stats.get(filename, 0) + file_entry.file_length - - for inode, count in skipped_inode_map.items(): - print("WARNING: Skip inode %s because it's not in inode map (%d entries)" %(inode, count)) - - print("Stats: Sum of lengths %d" %(stats_length_total)) - - if filters: - print("Filter: %d total files removed." %(len(filtered_entry_map))) - - for fn, count in filtered_entry_map.items(): - print("Filter: File '%s' removed '%d' entries." %(fn, count)) - - for filename, file_size in filename_stats.items(): - print("%s,%s" %(filename, file_size)) - - return trace_file - -def calc_trace_end_time(trace2db: Trace2Db, - trace_duration: Optional[timedelta]) -> float: - """ - Calculates the end time based on the trace duration. - The start time is the first receiving mm file map event. - The end time is the start time plus the trace duration. - All of them are in milliseconds. - """ - # If the duration is not set, assume all time is acceptable. - if trace_duration is None: - # float('inf') - return RawFtraceEntry.__table__.c.timestamp.type.python_type('inf') - - first_event = trace2db.session.query(MmFilemapAddToPageCache).join( - MmFilemapAddToPageCache.raw_ftrace_entry).order_by( - RawFtraceEntry.timestamp).first() - - # total_seconds() will return a float number. - return first_event.raw_ftrace_entry.timestamp + trace_duration.total_seconds() - -def query_add_to_page_cache(trace2db: Trace2Db, trace_duration: Optional[timedelta]): - end_time = calc_trace_end_time(trace2db, trace_duration) - # SELECT * FROM tbl ORDER BY id; - return trace2db.session.query(MmFilemapAddToPageCache).join( - MmFilemapAddToPageCache.raw_ftrace_entry).filter( - RawFtraceEntry.timestamp <= end_time).order_by( - MmFilemapAddToPageCache.id).all() - -def transform_perfetto_trace_to_systrace(path_to_perfetto_trace: str, - path_to_tmp_systrace: str) -> None: - """ Transforms the systrace file from perfetto trace. """ - cmd_utils.run_command_nofail([str(TRACECONV_BIN), - 'systrace', - path_to_perfetto_trace, - path_to_tmp_systrace]) - - -def run(sql_db_path:str, - trace_file:str, - trace_duration:Optional[timedelta], - output_file:str, - inode_table:str, - filter:List[str]) -> int: - trace2db = Trace2Db(sql_db_path) - # Speed optimization: Skip any entries that aren't mm_filemap_add_to_pagecache. - trace2db.set_raw_ftrace_entry_filter(\ - lambda entry: entry['function'] == 'mm_filemap_add_to_page_cache') - # TODO: parse multiple trace files here. - parse_count = trace2db.parse_file_into_db(trace_file) - - mm_filemap_add_to_page_cache_rows = query_add_to_page_cache(trace2db, - trace_duration) - print("DONE. Parsed %d entries into sql db." %(len(mm_filemap_add_to_page_cache_rows))) - - page_runs = page_cache_entries_to_runs(mm_filemap_add_to_page_cache_rows) - print("DONE. Converted %d entries" %(len(page_runs))) - - # TODO: flags to select optimizations. - optimized_page_runs = optimize_page_runs(page_runs) - print("DONE. Optimized down to %d entries" %(len(optimized_page_runs))) - - print("Build protobuf...") - trace_file = build_protobuf(optimized_page_runs, inode_table, filter) - - print("Write protobuf to file...") - output_file = open(output_file, 'wb') - output_file.write(trace_file.SerializeToString()) - output_file.close() - - print("DONE") - - # TODO: Silent running mode [no output except on error] for build runs. - - return 0 - -def main(argv): - parser = optparse.OptionParser(usage="Usage: %prog [options]", description="Compile systrace file into TraceFile.pb") - parser.add_option('-i', dest='inode_data_file', metavar='FILE', - help='Read cached inode data from a file saved earlier with pagecache.py -d') - parser.add_option('-t', dest='trace_file', metavar='FILE', - help='Path to systrace file (trace.html) that will be parsed') - parser.add_option('--perfetto-trace', dest='perfetto_trace_file', - metavar='FILE', - help='Path to perfetto trace that will be parsed') - - parser.add_option('--db', dest='sql_db', metavar='FILE', - help='Path to intermediate sqlite3 database [default: in-memory].') - - parser.add_option('-f', dest='filter', action="append", default=[], - help="Add file filter. All file entries not matching one of the filters are discarded.") - - parser.add_option('-l', dest='launch_lock', action="store_true", default=False, - help="Exclude all events not inside launch_lock") - - parser.add_option('-o', dest='output_file', metavar='FILE', - help='Output protobuf file') - - parser.add_option('--duration', dest='trace_duration', action="store", - type=int, help='The duration of trace in milliseconds.') - - options, categories = parser.parse_args(argv[1:]) - - # TODO: OptionParser should have some flags to make these mandatory. - if not options.inode_data_file: - parser.error("-i is required") - if not options.trace_file and not options.perfetto_trace_file: - parser.error("one of -t or --perfetto-trace is required") - if options.trace_file and options.perfetto_trace_file: - parser.error("please enter either -t or --perfetto-trace, not both") - if not options.output_file: - parser.error("-o is required") - - if options.launch_lock: - print("INFO: Launch lock flag (-l) enabled; filtering all events not inside launch_lock.") - - inode_table = Inode2Filename.new_from_filename(options.inode_data_file) - - sql_db_path = ":memory:" - if options.sql_db: - sql_db_path = options.sql_db - - trace_duration = timedelta(milliseconds=options.trace_duration) if \ - options.trace_duration is not None else None - - # if the input is systrace - if options.trace_file: - return run(sql_db_path, - options.trace_file, - trace_duration, - options.output_file, - inode_table, - options.filter) - - # if the input is perfetto trace - # TODO python 3.7 switch to using nullcontext - with tempfile.NamedTemporaryFile() as trace_file: - transform_perfetto_trace_to_systrace(options.perfetto_trace_file, - trace_file.name) - return run(sql_db_path, - trace_file.name, - trace_duration, - options.output_file, - inode_table, - options.filter) - -if __name__ == '__main__': - print(sys.argv) - sys.exit(main(sys.argv)) +import lib.print_utils as print_utils + +# Type of compiler. +class CompilerType(Enum): + HOST = 1 # iorap.cmd.compiler on host + DEVICE = 2 # adb shell iorap.cmd.compiler + RI = 3 # compiler.py + +def compile_perfetto_trace_ri( + argv: List[str], + compiler) -> TextIO: + print_utils.debug_print('Compile using RI compiler.') + compiler_trace_file = tempfile.NamedTemporaryFile() + argv.extend(['-o', compiler_trace_file.name]) + print_utils.debug_print(argv) + compiler.main([''] + argv) + return compiler_trace_file + +def compile_perfetto_trace_device(inodes_path: str, + package: str, + activity: str, + compiler) -> TextIO: + print_utils.debug_print('Compile using on-device compiler.') + compiler_trace_file = tempfile.NamedTemporaryFile() + compiler.main(inodes_path, package, activity, compiler_trace_file.name) + return compiler_trace_file + +def compile(compiler_type: CompilerType, + inodes_path: str, + ri_compiler_argv, + package: str, + activity: str) -> TextIO: + if compiler_type == CompilerType.RI: + compiler = importlib.import_module('iorap.compiler_ri') + compiler_trace_file = compile_perfetto_trace_ri(ri_compiler_argv, + compiler) + return compiler_trace_file + if compiler_type == CompilerType.DEVICE: + compiler = importlib.import_module('iorap.compiler_device') + compiler_trace_file = compile_perfetto_trace_device(inodes_path, + package, + activity, + compiler) + return compiler_trace_file + + # Should not arrive here. + raise ValueError('Unknown compiler type') diff --git a/startop/scripts/iorap/compiler_device.py b/startop/scripts/iorap/compiler_device.py new file mode 100644 index 000000000000..d941cd913fe1 --- /dev/null +++ b/startop/scripts/iorap/compiler_device.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +# +# Copyright 2019, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import sys +from typing import List + +DIR = os.path.abspath(os.path.dirname(__file__)) +sys.path.append(os.path.dirname(DIR)) # framework/base/startop/script +import lib.print_utils as print_utils +import iorap.lib.iorapd_utils as iorapd_utils +from app_startup.lib.app_runner import AppRunner + +IORAP_COMMON_BASH_SCRIPT = os.path.join(DIR, 'common') + +def parse_options(argv: List[str] = None): + """Parses command line arguments and returns an argparse Namespace object.""" + parser = argparse.ArgumentParser(description="Compile perfetto trace file") + required_named = parser.add_argument_group('required named arguments') + + required_named.add_argument('-i', dest='inodes', metavar='FILE', + help='Read cached inode data from a file saved ' + 'earlier with pagecache.py -d') + required_named.add_argument('-p', dest='package', + help='Package of the app to be compiled') + + optional_named = parser.add_argument_group('optional named arguments') + optional_named.add_argument('-o', dest='output', + help='The compiled trace is stored into the output file') + optional_named.add_argument('-a', dest='activity', + help='Activity of the app to be compiled') + optional_named.add_argument('-d', dest='debug', action='store_true' + , help='Activity of the app to be compiled') + + return parser.parse_args(argv) + +def main(inodes, package, activity, output, **kwargs) -> int: + """Entries of the program.""" + if not activity: + activity = AppRunner.get_activity(package) + + passed = iorapd_utils.compile_perfetto_trace_on_device(package, activity, + inodes) + if passed and output: + iorapd_utils.get_iorapd_compiler_trace(package, activity, output) + + return 0 + +if __name__ == '__main__': + opts = parse_options() + if opts.debug: + print_utils.DEBUG = opts.debug + print_utils.debug_print(opts) + sys.exit(main(**(vars(opts)))) diff --git a/startop/scripts/iorap/compiler_ri.py b/startop/scripts/iorap/compiler_ri.py new file mode 100755 index 000000000000..17b58c19ef28 --- /dev/null +++ b/startop/scripts/iorap/compiler_ri.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 + +# +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# Dependencies: +# +# $> sudo apt-get install python3-pip +# $> pip3 install --user protobuf sqlalchemy sqlite3 +# + +import optparse +import os +import re +import sys +import tempfile +from pathlib import Path +from datetime import timedelta +from typing import Iterable, Optional, List + +DIR = os.path.abspath(os.path.dirname(__file__)) +sys.path.append(os.path.dirname(DIR)) +from iorap.generated.TraceFile_pb2 import * +from iorap.lib.inode2filename import Inode2Filename + +parent_dir_name = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) +sys.path.append(parent_dir_name) +from trace_analyzer.lib.trace2db import Trace2Db, MmFilemapAddToPageCache, \ + RawFtraceEntry +import lib.cmd_utils as cmd_utils + +_PAGE_SIZE = 4096 # adb shell getconf PAGESIZE ## size of a memory page in bytes. +ANDROID_BUILD_TOP = Path(parent_dir_name).parents[3] +TRACECONV_BIN = ANDROID_BUILD_TOP.joinpath( + 'external/perfetto/tools/traceconv') + +class PageRun: + """ + Intermediate representation for a run of one or more pages. + """ + def __init__(self, device_number: int, inode: int, offset: int, length: int): + self.device_number = device_number + self.inode = inode + self.offset = offset + self.length = length + + def __str__(self): + return "PageRun(device_number=%d, inode=%d, offset=%d, length=%d)" \ + %(self.device_number, self.inode, self.offset, self.length) + +def debug_print(msg): + #print(msg) + pass + +UNDER_LAUNCH = False + +def page_cache_entries_to_runs(page_cache_entries: Iterable[MmFilemapAddToPageCache]): + global _PAGE_SIZE + + runs = [ + PageRun(device_number=pg_entry.dev, inode=pg_entry.ino, offset=pg_entry.ofs, + length=_PAGE_SIZE) + for pg_entry in page_cache_entries + ] + + for r in runs: + debug_print(r) + + print("Stats: Page runs totaling byte length: %d" %(len(runs) * _PAGE_SIZE)) + + return runs + +def optimize_page_runs(page_runs): + new_entries = [] + last_entry = None + for pg_entry in page_runs: + if last_entry: + if pg_entry.device_number == last_entry.device_number and pg_entry.inode == last_entry.inode: + # we are dealing with a run for the same exact file as a previous run. + if pg_entry.offset == last_entry.offset + last_entry.length: + # trivially contiguous entries. merge them together. + last_entry.length += pg_entry.length + continue + # Default: Add the run without merging it to a previous run. + last_entry = pg_entry + new_entries.append(pg_entry) + return new_entries + +def is_filename_matching_filter(file_name, filters=[]): + """ + Blacklist-style regular expression filters. + + :return: True iff file_name has an RE match in one of the filters. + """ + for filt in filters: + res = re.search(filt, file_name) + if res: + return True + + return False + +def build_protobuf(page_runs, inode2filename, filters=[]): + trace_file = TraceFile() + trace_file_index = trace_file.index + + file_id_counter = 0 + file_id_map = {} # filename -> id + + stats_length_total = 0 + filename_stats = {} # filename -> total size + + skipped_inode_map = {} + filtered_entry_map = {} # filename -> count + + for pg_entry in page_runs: + fn = inode2filename.resolve(pg_entry.device_number, pg_entry.inode) + if not fn: + skipped_inode_map[pg_entry.inode] = skipped_inode_map.get(pg_entry.inode, 0) + 1 + continue + + filename = fn + + if filters and not is_filename_matching_filter(filename, filters): + filtered_entry_map[filename] = filtered_entry_map.get(filename, 0) + 1 + continue + + file_id = file_id_map.get(filename) + if not file_id: + file_id = file_id_counter + file_id_map[filename] = file_id_counter + file_id_counter = file_id_counter + 1 + + file_index_entry = trace_file_index.entries.add() + file_index_entry.id = file_id + file_index_entry.file_name = filename + + # already in the file index, add the file entry. + file_entry = trace_file.list.entries.add() + file_entry.index_id = file_id + file_entry.file_length = pg_entry.length + stats_length_total += file_entry.file_length + file_entry.file_offset = pg_entry.offset + + filename_stats[filename] = filename_stats.get(filename, 0) + file_entry.file_length + + for inode, count in skipped_inode_map.items(): + print("WARNING: Skip inode %s because it's not in inode map (%d entries)" %(inode, count)) + + print("Stats: Sum of lengths %d" %(stats_length_total)) + + if filters: + print("Filter: %d total files removed." %(len(filtered_entry_map))) + + for fn, count in filtered_entry_map.items(): + print("Filter: File '%s' removed '%d' entries." %(fn, count)) + + for filename, file_size in filename_stats.items(): + print("%s,%s" %(filename, file_size)) + + return trace_file + +def calc_trace_end_time(trace2db: Trace2Db, + trace_duration: Optional[timedelta]) -> float: + """ + Calculates the end time based on the trace duration. + The start time is the first receiving mm file map event. + The end time is the start time plus the trace duration. + All of them are in milliseconds. + """ + # If the duration is not set, assume all time is acceptable. + if trace_duration is None: + # float('inf') + return RawFtraceEntry.__table__.c.timestamp.type.python_type('inf') + + first_event = trace2db.session.query(MmFilemapAddToPageCache).join( + MmFilemapAddToPageCache.raw_ftrace_entry).order_by( + RawFtraceEntry.timestamp).first() + + # total_seconds() will return a float number. + return first_event.raw_ftrace_entry.timestamp + trace_duration.total_seconds() + +def query_add_to_page_cache(trace2db: Trace2Db, trace_duration: Optional[timedelta]): + end_time = calc_trace_end_time(trace2db, trace_duration) + # SELECT * FROM tbl ORDER BY id; + return trace2db.session.query(MmFilemapAddToPageCache).join( + MmFilemapAddToPageCache.raw_ftrace_entry).filter( + RawFtraceEntry.timestamp <= end_time).order_by( + MmFilemapAddToPageCache.id).all() + +def transform_perfetto_trace_to_systrace(path_to_perfetto_trace: str, + path_to_tmp_systrace: str) -> None: + """ Transforms the systrace file from perfetto trace. """ + cmd_utils.run_command_nofail([str(TRACECONV_BIN), + 'systrace', + path_to_perfetto_trace, + path_to_tmp_systrace]) + + +def run(sql_db_path:str, + trace_file:str, + trace_duration:Optional[timedelta], + output_file:str, + inode_table:str, + filter:List[str]) -> int: + trace2db = Trace2Db(sql_db_path) + # Speed optimization: Skip any entries that aren't mm_filemap_add_to_pagecache. + trace2db.set_raw_ftrace_entry_filter(\ + lambda entry: entry['function'] == 'mm_filemap_add_to_page_cache') + # TODO: parse multiple trace files here. + parse_count = trace2db.parse_file_into_db(trace_file) + + mm_filemap_add_to_page_cache_rows = query_add_to_page_cache(trace2db, + trace_duration) + print("DONE. Parsed %d entries into sql db." %(len(mm_filemap_add_to_page_cache_rows))) + + page_runs = page_cache_entries_to_runs(mm_filemap_add_to_page_cache_rows) + print("DONE. Converted %d entries" %(len(page_runs))) + + # TODO: flags to select optimizations. + optimized_page_runs = optimize_page_runs(page_runs) + print("DONE. Optimized down to %d entries" %(len(optimized_page_runs))) + + print("Build protobuf...") + trace_file = build_protobuf(optimized_page_runs, inode_table, filter) + + print("Write protobuf to file...") + output_file = open(output_file, 'wb') + output_file.write(trace_file.SerializeToString()) + output_file.close() + + print("DONE") + + # TODO: Silent running mode [no output except on error] for build runs. + + return 0 + +def main(argv): + parser = optparse.OptionParser(usage="Usage: %prog [options]", description="Compile systrace file into TraceFile.pb") + parser.add_option('-i', dest='inode_data_file', metavar='FILE', + help='Read cached inode data from a file saved earlier with pagecache.py -d') + parser.add_option('-t', dest='trace_file', metavar='FILE', + help='Path to systrace file (trace.html) that will be parsed') + parser.add_option('--perfetto-trace', dest='perfetto_trace_file', + metavar='FILE', + help='Path to perfetto trace that will be parsed') + + parser.add_option('--db', dest='sql_db', metavar='FILE', + help='Path to intermediate sqlite3 database [default: in-memory].') + + parser.add_option('-f', dest='filter', action="append", default=[], + help="Add file filter. All file entries not matching one of the filters are discarded.") + + parser.add_option('-l', dest='launch_lock', action="store_true", default=False, + help="Exclude all events not inside launch_lock") + + parser.add_option('-o', dest='output_file', metavar='FILE', + help='Output protobuf file') + + parser.add_option('--duration', dest='trace_duration', action="store", + type=int, help='The duration of trace in milliseconds.') + + options, categories = parser.parse_args(argv[1:]) + + # TODO: OptionParser should have some flags to make these mandatory. + if not options.inode_data_file: + parser.error("-i is required") + if not options.trace_file and not options.perfetto_trace_file: + parser.error("one of -t or --perfetto-trace is required") + if options.trace_file and options.perfetto_trace_file: + parser.error("please enter either -t or --perfetto-trace, not both") + if not options.output_file: + parser.error("-o is required") + + if options.launch_lock: + print("INFO: Launch lock flag (-l) enabled; filtering all events not inside launch_lock.") + + inode_table = Inode2Filename.new_from_filename(options.inode_data_file) + + sql_db_path = ":memory:" + if options.sql_db: + sql_db_path = options.sql_db + + trace_duration = timedelta(milliseconds=options.trace_duration) if \ + options.trace_duration is not None else None + + # if the input is systrace + if options.trace_file: + return run(sql_db_path, + options.trace_file, + trace_duration, + options.output_file, + inode_table, + options.filter) + + # if the input is perfetto trace + # TODO python 3.7 switch to using nullcontext + with tempfile.NamedTemporaryFile() as trace_file: + transform_perfetto_trace_to_systrace(options.perfetto_trace_file, + trace_file.name) + return run(sql_db_path, + trace_file.name, + trace_duration, + options.output_file, + inode_table, + options.filter) + +if __name__ == '__main__': + print(sys.argv) + sys.exit(main(sys.argv)) diff --git a/startop/scripts/iorap/compiler_test.py b/startop/scripts/iorap/compiler_test.py index 1a9f059fc6b6..d1f11c5da2a5 100644 --- a/startop/scripts/iorap/compiler_test.py +++ b/startop/scripts/iorap/compiler_test.py @@ -30,7 +30,7 @@ See also https://docs.pytest.org/en/latest/usage.html """ import os -import compiler +import compiler_host as compiler DIR = os.path.abspath(os.path.dirname(__file__)) TEXTCACHE = os.path.join(DIR, 'test_fixtures/compiler/common_textcache') diff --git a/startop/scripts/iorap/lib/iorapd_utils.py b/startop/scripts/iorap/lib/iorapd_utils.py index 0d62180a01e3..f6f21fd70005 100644 --- a/startop/scripts/iorap/lib/iorapd_utils.py +++ b/startop/scripts/iorap/lib/iorapd_utils.py @@ -18,10 +18,9 @@ import os import sys -from pathlib import Path -# up to two level, like '../../' -sys.path.append(Path(os.path.abspath(__file__)).parents[2]) +# up to two level +sys.path.append(os.path.join(os.path.abspath(__file__),'../..')) import lib.cmd_utils as cmd_utils IORAPID_LIB_DIR = os.path.abspath(os.path.dirname(__file__)) @@ -39,6 +38,22 @@ def _iorapd_path_to_data_file(package: str, activity: str, suffix: str) -> str: # Match logic of 'AppComponentName' in iorap::compiler C++ code. return '{}/{}%2F{}.{}'.format(IORAPD_DATA_PATH, package, activity, suffix) +def compile_perfetto_trace_on_device(package: str, activity: str, + inodes: str) -> bool: + """Compiles the perfetto trace using on-device compiler.""" + passed, _ = cmd_utils.run_shell_func(IORAP_COMMON_BASH_SCRIPT, + 'iorapd_compiler_for_app_trace', + [package, activity, inodes]) + return passed + +def get_iorapd_compiler_trace(package: str, activity: str, dest: str) -> str: + """Gets compiler trace to dest file.""" + src = _iorapd_path_to_data_file(package, activity, 'compiled_trace.pb') + passed, _ = cmd_utils.run_shell_command('adb pull "{}" "{}"'.format(src, dest)) + if not passed: + return False + return True + def iorapd_compiler_install_trace_file(package: str, activity: str, input_file: str) -> bool: """Installs a compiled trace file. |