startop: Add iorap compiler written in python

Compiler will be used for experimentation purpose since it's both easier
to develop in python and it accepts ftrace, making it very easy to write
complicated-experimental changes that we aren't sure are worth it yet
for the on-device C++/perfetto compiler.

This 'new' compiler accepts ftrace/systrace files as input,
then generates an in-memory sqlite3 database (using the trace_analyzer
source code), and finally code-generates a TraceFile.pb protobuf.

(Also refactor trace_analyzer into a library, and update it to
parse systrace.html files)

Limitations: currently does not accept perfetto_trace.pb files due to
             'ofs' fields missing (see bug#135555191)

Test: py.test-3 frameworks/base/startop/scripts
Test: ./compiler.py -i tmp_sargo/textcache -t tmp_sargo/trace.html -o tmp/output.pb
Test: ./compiler.py -i tmp_sargo/textcache -t tmp_sargo/trace.html -o tmp/output.pb -f '^/data'
Test: ./trace_analyzer music_run.trace tmp_dbs/12345.db

Bug: 135557978
Bug: 134789969
Change-Id: Ic8295900ee9e634b4cfd8cf99b671ae08d2ea4f7

1 files changed, 259 insertions, 0 deletions

diff --git a/startop/scripts/iorap/generated/TraceFile_pb2.py b/startop/scripts/iorap/generated/TraceFile_pb2.py
new file mode 100644
index 000000000000..f005bed427ca
--- /dev/null
+++ b/startop/scripts/iorap/generated/TraceFile_pb2.py
@@ -0,0 +1,259 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: TraceFile.proto
+
+import sys
+_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf import descriptor_pb2
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='TraceFile.proto',
+  package='iorap.serialize.proto',
+  syntax='proto2',
+  serialized_pb=_b('\n\x0fTraceFile.proto\x12\x15iorap.serialize.proto\"u\n\tTraceFile\x12\x34\n\x05index\x18\x01 \x02(\x0b\x32%.iorap.serialize.proto.TraceFileIndex\x12\x32\n\x04list\x18\x02 \x02(\x0b\x32$.iorap.serialize.proto.TraceFileList\"M\n\x0eTraceFileIndex\x12;\n\x07\x65ntries\x18\x01 \x03(\x0b\x32*.iorap.serialize.proto.TraceFileIndexEntry\"4\n\x13TraceFileIndexEntry\x12\n\n\x02id\x18\x01 \x02(\x03\x12\x11\n\tfile_name\x18\x02 \x02(\t\"G\n\rTraceFileList\x12\x36\n\x07\x65ntries\x18\x01 \x03(\x0b\x32%.iorap.serialize.proto.TraceFileEntry\"L\n\x0eTraceFileEntry\x12\x10\n\x08index_id\x18\x01 \x02(\x03\x12\x13\n\x0b\x66ile_offset\x18\x02 \x02(\x03\x12\x13\n\x0b\x66ile_length\x18\x03 \x02(\x03\x42\x1c\n\x18\x63om.google.android.iorapH\x03')
+)
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+
+
+
+_TRACEFILE = _descriptor.Descriptor(
+  name='TraceFile',
+  full_name='iorap.serialize.proto.TraceFile',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='index', full_name='iorap.serialize.proto.TraceFile.index', index=0,
+      number=1, type=11, cpp_type=10, label=2,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='list', full_name='iorap.serialize.proto.TraceFile.list', index=1,
+      number=2, type=11, cpp_type=10, label=2,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto2',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=42,
+  serialized_end=159,
+)
+
+
+_TRACEFILEINDEX = _descriptor.Descriptor(
+  name='TraceFileIndex',
+  full_name='iorap.serialize.proto.TraceFileIndex',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='entries', full_name='iorap.serialize.proto.TraceFileIndex.entries', index=0,
+      number=1, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto2',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=161,
+  serialized_end=238,
+)
+
+
+_TRACEFILEINDEXENTRY = _descriptor.Descriptor(
+  name='TraceFileIndexEntry',
+  full_name='iorap.serialize.proto.TraceFileIndexEntry',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='id', full_name='iorap.serialize.proto.TraceFileIndexEntry.id', index=0,
+      number=1, type=3, cpp_type=2, label=2,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='file_name', full_name='iorap.serialize.proto.TraceFileIndexEntry.file_name', index=1,
+      number=2, type=9, cpp_type=9, label=2,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto2',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=240,
+  serialized_end=292,
+)
+
+
+_TRACEFILELIST = _descriptor.Descriptor(
+  name='TraceFileList',
+  full_name='iorap.serialize.proto.TraceFileList',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='entries', full_name='iorap.serialize.proto.TraceFileList.entries', index=0,
+      number=1, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto2',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=294,
+  serialized_end=365,
+)
+
+
+_TRACEFILEENTRY = _descriptor.Descriptor(
+  name='TraceFileEntry',
+  full_name='iorap.serialize.proto.TraceFileEntry',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='index_id', full_name='iorap.serialize.proto.TraceFileEntry.index_id', index=0,
+      number=1, type=3, cpp_type=2, label=2,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='file_offset', full_name='iorap.serialize.proto.TraceFileEntry.file_offset', index=1,
+      number=2, type=3, cpp_type=2, label=2,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='file_length', full_name='iorap.serialize.proto.TraceFileEntry.file_length', index=2,
+      number=3, type=3, cpp_type=2, label=2,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto2',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=367,
+  serialized_end=443,
+)
+
+_TRACEFILE.fields_by_name['index'].message_type = _TRACEFILEINDEX
+_TRACEFILE.fields_by_name['list'].message_type = _TRACEFILELIST
+_TRACEFILEINDEX.fields_by_name['entries'].message_type = _TRACEFILEINDEXENTRY
+_TRACEFILELIST.fields_by_name['entries'].message_type = _TRACEFILEENTRY
+DESCRIPTOR.message_types_by_name['TraceFile'] = _TRACEFILE
+DESCRIPTOR.message_types_by_name['TraceFileIndex'] = _TRACEFILEINDEX
+DESCRIPTOR.message_types_by_name['TraceFileIndexEntry'] = _TRACEFILEINDEXENTRY
+DESCRIPTOR.message_types_by_name['TraceFileList'] = _TRACEFILELIST
+DESCRIPTOR.message_types_by_name['TraceFileEntry'] = _TRACEFILEENTRY
+
+TraceFile = _reflection.GeneratedProtocolMessageType('TraceFile', (_message.Message,), dict(
+  DESCRIPTOR = _TRACEFILE,
+  __module__ = 'TraceFile_pb2'
+  # @@protoc_insertion_point(class_scope:iorap.serialize.proto.TraceFile)
+  ))
+_sym_db.RegisterMessage(TraceFile)
+
+TraceFileIndex = _reflection.GeneratedProtocolMessageType('TraceFileIndex', (_message.Message,), dict(
+  DESCRIPTOR = _TRACEFILEINDEX,
+  __module__ = 'TraceFile_pb2'
+  # @@protoc_insertion_point(class_scope:iorap.serialize.proto.TraceFileIndex)
+  ))
+_sym_db.RegisterMessage(TraceFileIndex)
+
+TraceFileIndexEntry = _reflection.GeneratedProtocolMessageType('TraceFileIndexEntry', (_message.Message,), dict(
+  DESCRIPTOR = _TRACEFILEINDEXENTRY,
+  __module__ = 'TraceFile_pb2'
+  # @@protoc_insertion_point(class_scope:iorap.serialize.proto.TraceFileIndexEntry)
+  ))
+_sym_db.RegisterMessage(TraceFileIndexEntry)
+
+TraceFileList = _reflection.GeneratedProtocolMessageType('TraceFileList', (_message.Message,), dict(
+  DESCRIPTOR = _TRACEFILELIST,
+  __module__ = 'TraceFile_pb2'
+  # @@protoc_insertion_point(class_scope:iorap.serialize.proto.TraceFileList)
+  ))
+_sym_db.RegisterMessage(TraceFileList)
+
+TraceFileEntry = _reflection.GeneratedProtocolMessageType('TraceFileEntry', (_message.Message,), dict(
+  DESCRIPTOR = _TRACEFILEENTRY,
+  __module__ = 'TraceFile_pb2'
+  # @@protoc_insertion_point(class_scope:iorap.serialize.proto.TraceFileEntry)
+  ))
+_sym_db.RegisterMessage(TraceFileEntry)
+
+
+DESCRIPTOR.has_options = True
+DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\030com.google.android.iorapH\003'))
+# @@protoc_insertion_point(module_scope)