From 06d017ee1913b20d5ebbb9c31e4c6061d993067e Mon Sep 17 00:00:00 2001 From: Igor Murashkin Date: Tue, 18 Jun 2019 16:03:20 -0700 Subject: startop: Add iorap compiler written in python Compiler will be used for experimentation purpose since it's both easier to develop in python and it accepts ftrace, making it very easy to write complicated-experimental changes that we aren't sure are worth it yet for the on-device C++/perfetto compiler. This 'new' compiler accepts ftrace/systrace files as input, then generates an in-memory sqlite3 database (using the trace_analyzer source code), and finally code-generates a TraceFile.pb protobuf. (Also refactor trace_analyzer into a library, and update it to parse systrace.html files) Limitations: currently does not accept perfetto_trace.pb files due to 'ofs' fields missing (see bug#135555191) Test: py.test-3 frameworks/base/startop/scripts Test: ./compiler.py -i tmp_sargo/textcache -t tmp_sargo/trace.html -o tmp/output.pb Test: ./compiler.py -i tmp_sargo/textcache -t tmp_sargo/trace.html -o tmp/output.pb -f '^/data' Test: ./trace_analyzer music_run.trace tmp_dbs/12345.db Bug: 135557978 Bug: 134789969 Change-Id: Ic8295900ee9e634b4cfd8cf99b671ae08d2ea4f7 --- .../scripts/trace_analyzer/lib/trace2db_test.py | 202 +++++++++++++++++++++ 1 file changed, 202 insertions(+) create mode 100755 startop/scripts/trace_analyzer/lib/trace2db_test.py (limited to 'startop/scripts/trace_analyzer/lib/trace2db_test.py') diff --git a/startop/scripts/trace_analyzer/lib/trace2db_test.py b/startop/scripts/trace_analyzer/lib/trace2db_test.py new file mode 100755 index 000000000000..b67cffa51cde --- /dev/null +++ b/startop/scripts/trace_analyzer/lib/trace2db_test.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +# +# Copyright 2019, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +Unit tests for inode2filename module. + +Install: + $> sudo apt-get install python3-pytest ## OR + $> pip install -U pytest +See also https://docs.pytest.org/en/latest/getting-started.html + +Usage: + $> ./inode2filename_test.py + $> pytest inode2filename_test.py + $> python -m pytest inode2filename_test.py + +See also https://docs.pytest.org/en/latest/usage.html +""" + +# global imports +from contextlib import contextmanager +import io +import shlex +import sys +import typing + +from copy import deepcopy + +# pip imports +import pytest + +# local imports +from trace2db import * + +# This pretty-prints the raw dictionary of the sqlalchemy object if it fails. +class EqualsSqlAlchemyObject: + # For convenience to write shorter tests, we also add 'ignore_fields' which allow us to specify + # which fields to ignore when doing the comparison. + def __init__(self_, self, ignore_fields=[]): + self_.self = self + self_.ignore_fields = ignore_fields + + # Do field-by-field comparison. + # It seems that SQLAlchemy does not implement __eq__ itself so we have to do it ourselves. + def __eq__(self_, other): + if isinstance(other, EqualsSqlAlchemyObject): + other = other.self + + self = self_.self + + classes_match = isinstance(other, self.__class__) + a, b = deepcopy(self.__dict__), deepcopy(other.__dict__) + + #compare based on equality our attributes, ignoring SQLAlchemy internal stuff + + a.pop('_sa_instance_state', None) + b.pop('_sa_instance_state', None) + + for f in self_.ignore_fields: + a.pop(f, None) + b.pop(f, None) + + attrs_match = (a == b) + return classes_match and attrs_match + + def __repr__(self): + return repr(self.self.__dict__) + + +def assert_eq_ignore_id(left, right): + # This pretty-prints the raw dictionary of the sqlalchemy object if it fails. + # It does field-by-field comparison, but ignores the 'id' field. + assert EqualsSqlAlchemyObject(left, ignore_fields=['id']) == EqualsSqlAlchemyObject(right) + +def parse_trace_file_to_db(*contents): + """ + Make temporary in-memory sqlite3 database by parsing the string contents as a trace. + + :return: Trace2Db instance + """ + buf = io.StringIO() + + for c in contents: + buf.write(c) + buf.write("\n") + + buf.seek(0) + + t2d = Trace2Db(":memory:") + t2d.parse_file_buf_into_db(buf) + + buf.close() + + return t2d + +def test_ftrace_mm_filemap_add_to_pagecache(): + test_contents = """ +MediaStoreImpor-27212 (27176) [000] .... 16136.595194: mm_filemap_add_to_page_cache: dev 253:6 ino 7580 page=0000000060e990c7 pfn=677646 ofs=159744 +MediaStoreImpor-27212 (27176) [000] .... 16136.595920: mm_filemap_add_to_page_cache: dev 253:6 ino 7580 page=0000000048e2e156 pfn=677645 ofs=126976 +MediaStoreImpor-27212 (27176) [000] .... 16136.597793: mm_filemap_add_to_page_cache: dev 253:6 ino 7580 page=0000000051eabfb2 pfn=677644 ofs=122880 +MediaStoreImpor-27212 (27176) [000] .... 16136.597815: mm_filemap_add_to_page_cache: dev 253:6 ino 7580 page=00000000ce7cd606 pfn=677643 ofs=131072 +MediaStoreImpor-27212 (27176) [000] .... 16136.603732: mm_filemap_add_to_page_cache: dev 253:6 ino 1 page=000000008ffd3030 pfn=730119 ofs=186482688 +MediaStoreImpor-27212 (27176) [000] .... 16136.604126: mm_filemap_add_to_page_cache: dev 253:6 ino b1d8 page=0000000098d4d2e2 pfn=829676 ofs=0 + <...>-27197 (-----) [002] .... 16136.613471: mm_filemap_add_to_page_cache: dev 253:6 ino 7580 page=00000000aca88a97 pfn=743346 ofs=241664 + <...>-27197 (-----) [002] .... 16136.615979: mm_filemap_add_to_page_cache: dev 253:6 ino 7580 page=00000000351f2bc1 pfn=777799 ofs=106496 + <...>-27224 (-----) [006] .... 16137.400090: mm_filemap_add_to_page_cache: dev 253:6 ino 712d page=000000006ff7ffdb pfn=754861 ofs=0 + <...>-1396 (-----) [000] .... 16137.451660: mm_filemap_add_to_page_cache: dev 253:6 ino 1 page=00000000ba0cbb34 pfn=769173 ofs=187191296 + <...>-1396 (-----) [000] .... 16137.453020: mm_filemap_add_to_page_cache: dev 253:6 ino b285 page=00000000f6ef038e pfn=820291 ofs=0 + <...>-1396 (-----) [000] .... 16137.453067: mm_filemap_add_to_page_cache: dev 253:6 ino b285 page=0000000083ebc446 pfn=956463 ofs=4096 + <...>-1396 (-----) [000] .... 16137.453101: mm_filemap_add_to_page_cache: dev 253:6 ino b285 page=000000009dc2cd25 pfn=822813 ofs=8192 + <...>-1396 (-----) [000] .... 16137.453113: mm_filemap_add_to_page_cache: dev 253:6 ino b285 page=00000000a11167fb pfn=928650 ofs=12288 + <...>-1396 (-----) [000] .... 16137.453126: mm_filemap_add_to_page_cache: dev 253:6 ino b285 page=00000000c1c3311b pfn=621110 ofs=16384 + <...>-1396 (-----) [000] .... 16137.453139: mm_filemap_add_to_page_cache: dev 253:6 ino b285 page=000000009aa78342 pfn=689370 ofs=20480 + <...>-1396 (-----) [000] .... 16137.453151: mm_filemap_add_to_page_cache: dev 253:6 ino b285 page=0000000082cddcd6 pfn=755584 ofs=24576 + <...>-1396 (-----) [000] .... 16137.453162: mm_filemap_add_to_page_cache: dev 253:6 ino b285 page=00000000b0249bc7 pfn=691431 ofs=28672 + <...>-1396 (-----) [000] .... 16137.453183: mm_filemap_add_to_page_cache: dev 253:6 ino b285 page=000000006a776ff0 pfn=795084 ofs=32768 + <...>-1396 (-----) [000] .... 16137.453203: mm_filemap_add_to_page_cache: dev 253:6 ino b285 page=000000001a4918a7 pfn=806998 ofs=36864 + <...>-2578 (-----) [002] .... 16137.561871: mm_filemap_add_to_page_cache: dev 253:6 ino 1 page=00000000d65af9d2 pfn=719246 ofs=187015168 + <...>-2578 (-----) [002] .... 16137.562846: mm_filemap_add_to_page_cache: dev 253:6 ino b25a page=000000002f6ba74f pfn=864982 ofs=0 + <...>-2578 (-----) [000] .... 16138.104500: mm_filemap_add_to_page_cache: dev 253:6 ino 1 page=00000000f888d0f6 pfn=805812 ofs=192794624 + <...>-2578 (-----) [000] .... 16138.105836: mm_filemap_add_to_page_cache: dev 253:6 ino b7dd page=000000003749523b pfn=977196 ofs=0 + <...>-27215 (-----) [001] .... 16138.256881: mm_filemap_add_to_page_cache: dev 253:6 ino 758f page=000000001b375de1 pfn=755928 ofs=0 + <...>-27215 (-----) [001] .... 16138.257526: mm_filemap_add_to_page_cache: dev 253:6 ino 7591 page=000000004e039481 pfn=841534 ofs=0 + NonUserFacing6-5246 ( 1322) [005] .... 16138.356491: mm_filemap_add_to_page_cache: dev 253:6 ino 1 page=00000000d65af9d2 pfn=719246 ofs=161890304 + NonUserFacing6-5246 ( 1322) [005] .... 16138.357538: mm_filemap_add_to_page_cache: dev 253:6 ino 9a64 page=000000002f6ba74f pfn=864982 ofs=0 + NonUserFacing6-5246 ( 1322) [005] .... 16138.357581: mm_filemap_add_to_page_cache: dev 253:6 ino 9a64 page=000000006e0f8322 pfn=797894 ofs=4096 + <...>-27197 (-----) [005] .... 16140.143224: mm_filemap_add_to_page_cache: dev 253:6 ino 7580 page=00000000a42527c6 pfn=1076669 ofs=32768 + """ + + t2d = parse_trace_file_to_db(test_contents) + session = t2d.session + + first_row = session.query(MmFilemapAddToPageCache).order_by(MmFilemapAddToPageCache.id).first() + + #dev 253:6 ino 7580 page=0000000060e990c7 pfn=677646 ofs=159744 + assert_eq_ignore_id(MmFilemapAddToPageCache(dev=64774, dev_major=253, dev_minor=6, + ino=0x7580, page=0x0000000060e990c7, pfn=677646, ofs=159744), first_row) + + second_to_last_row = session.query(MmFilemapAddToPageCache).filter(MmFilemapAddToPageCache.page.in_([0x000000006e0f8322])).first() + + # dev 253:6 ino 9a64 page=000000006e0f8322 pfn=797894 ofs=4096 + assert_eq_ignore_id(MmFilemapAddToPageCache(dev=64774, dev_major=253, dev_minor=6, + ino=0x9a64, page=0x000000006e0f8322, pfn=797894, ofs=4096), second_to_last_row) + +def test_systrace_mm_filemap_add_to_pagecache(): + test_contents = """ + + + + + +Android System Trace + + + + + + + """ + + t2d = parse_trace_file_to_db(test_contents) + session = t2d.session + + first_row = session.query(MmFilemapAddToPageCache).order_by(MmFilemapAddToPageCache.id).first() + + #dev 253:6 ino 7580 page=0000000060e990c7 pfn=677646 ofs=159744 + assert_eq_ignore_id(MmFilemapAddToPageCache(dev=64774, dev_major=253, dev_minor=6, + ino=0x7580, page=0x0000000060e990c7, pfn=677646, ofs=159744), first_row) + + second_to_last_row = session.query(MmFilemapAddToPageCache).filter(MmFilemapAddToPageCache.page.in_([0x000000006e0f8322])).first() + + # dev 253:6 ino 9a64 page=000000006e0f8322 pfn=797894 ofs=4096 + assert_eq_ignore_id(MmFilemapAddToPageCache(dev=64774, dev_major=253, dev_minor=6, + ino=0x9a64, page=0x000000006e0f8322, pfn=797894, ofs=4096), second_to_last_row) + + +if __name__ == '__main__': + pytest.main() -- cgit v1.2.3 From 315ae2d4c4396fa4f19f5a2be0d73afb9cfa7b31 Mon Sep 17 00:00:00 2001 From: Yan Wang Date: Tue, 16 Jul 2019 15:29:08 -0700 Subject: Add support of trace duration for host python compiler. The basic idea is add timestamp for MmFilemapAddToPageCache. Treat the first receiving MmFilemapAddToPageCache timestamp as the start time. The end time is the sum of start time and duration. Any MmFilemapAddToPageCache after end time is filtered out. Test: pytest trace2db_test.py Bug: 137398235 Change-Id: Ib9c439f3ae0ca666eacb08492361217d89adec34 --- .../scripts/trace_analyzer/lib/trace2db_test.py | 34 +++++++++++++++++----- 1 file changed, 27 insertions(+), 7 deletions(-) (limited to 'startop/scripts/trace_analyzer/lib/trace2db_test.py') diff --git a/startop/scripts/trace_analyzer/lib/trace2db_test.py b/startop/scripts/trace_analyzer/lib/trace2db_test.py index b67cffa51cde..3b326f000a7d 100755 --- a/startop/scripts/trace_analyzer/lib/trace2db_test.py +++ b/startop/scripts/trace_analyzer/lib/trace2db_test.py @@ -32,17 +32,10 @@ See also https://docs.pytest.org/en/latest/usage.html """ # global imports -from contextlib import contextmanager import io -import shlex -import sys -import typing - from copy import deepcopy # pip imports -import pytest - # local imports from trace2db import * @@ -197,6 +190,33 @@ NonUserFacing6-5246 ( 1322) [005] .... 16138.357581: mm_filemap_add_to_page_cac assert_eq_ignore_id(MmFilemapAddToPageCache(dev=64774, dev_major=253, dev_minor=6, ino=0x9a64, page=0x000000006e0f8322, pfn=797894, ofs=4096), second_to_last_row) +def test_timestamp_filter(): + test_contents = """ + MediaStoreImpor-27212 (27176) [000] .... 16136.595194: mm_filemap_add_to_page_cache: dev 253:6 ino 7580 page=0000000060e990c7 pfn=677646 ofs=159744 + NonUserFacing6-5246 ( 1322) [005] .... 16139.357581: mm_filemap_add_to_page_cache: dev 253:6 ino 9a64 page=000000006e0f8322 pfn=797894 ofs=4096 + MediaStoreImpor-27212 (27176) [000] .... 16136.604126: mm_filemap_add_to_page_cache: dev 253:6 ino b1d8 page=0000000098d4d2e2 pfn=829676 ofs=0 + """ + + t2d = parse_trace_file_to_db(test_contents) + session = t2d.session + + end_time = 16137.0 + + results = session.query(MmFilemapAddToPageCache).join( + MmFilemapAddToPageCache.raw_ftrace_entry).filter( + RawFtraceEntry.timestamp <= end_time).order_by( + MmFilemapAddToPageCache.id).all() + + assert len(results) == 2 + assert_eq_ignore_id( + MmFilemapAddToPageCache(dev=64774, dev_major=253, dev_minor=6, + ino=0x7580, page=0x0000000060e990c7, pfn=677646, + ofs=159744), results[0]) + assert_eq_ignore_id( + MmFilemapAddToPageCache(dev=64774, dev_major=253, dev_minor=6, + ino=0xb1d8, page=0x0000000098d4d2e2, pfn=829676, + ofs=0), results[1]) + if __name__ == '__main__': pytest.main() -- cgit v1.2.3