update_engine: Adds BROTLI_BSDIFF operation

Brotli compression creates on average 10%-20% smaller output than bzip2
in addition to having faster decompressor. With recent changes in bsdiff
to compress the its patch with brotli, we can use it in the
update_engine as a new operation BROTLI_BSDIFF. This operation will be
turned on in minor version 4. However, this CL only adds support for it
in the client. It will not generate BROTLI_BSDIFF operations yet.

BUG=chromium:783437
TEST=unittests pass for both update_engine and update_payload;
'brillo_update_payload {generate|verify}' passes;
'scripts/paycheck.py payload.delta' passes;

Change-Id: Ie791ba5431561c95de6fbc031a8196dbfd912288
Reviewed-on: https://chromium-review.googlesource.com/764791
Commit-Ready: Amin Hassani <ahassani@chromium.org>
Tested-by: Amin Hassani <ahassani@chromium.org>
Reviewed-by: Ben Chan <benchan@chromium.org>

1 files changed, 20 insertions, 6 deletions

diff --git a/scripts/update_payload/checker.py b/scripts/update_payload/checker.py
index c6136428..c710c47b 100644
--- a/scripts/update_payload/checker.py
+++ b/scripts/update_payload/checker.py
@@ -833,10 +833,12 @@ class PayloadChecker(object):
     if op.data_offset:
       raise error.PayloadError('%s: contains data_offset.' % op_name)
 
-  def _CheckAnyDiffOperation(self, data_length, total_dst_blocks, op_name):
-    """Specific checks for BSDIFF, SOURCE_BSDIFF and PUFFDIFF operations.
+  def _CheckAnyDiffOperation(self, op, data_length, total_dst_blocks, op_name):
+    """Specific checks for BSDIFF, SOURCE_BSDIFF, PUFFDIFF and BROTLI_BSDIFF
+       operations.
 
     Args:
+      op: The operation.
       data_length: The length of the data blob associated with the operation.
       total_dst_blocks: Total number of blocks in dst_extents.
       op_name: Operation name for error reporting.
@@ -856,6 +858,15 @@ class PayloadChecker(object):
           (op_name, data_length, total_dst_blocks, self.block_size,
            total_dst_blocks * self.block_size))
 
+    # Check the existence of src_length and dst_length for legacy bsdiffs.
+    if (op.type == common.OpType.BSDIFF or
+        (op.type == common.OpType.SOURCE_BSDIFF and self.minor_version <= 3)):
+      if not op.HasField('src_length') or not op.HasField('dst_length'):
+        raise error.PayloadError('%s: require {src,dst}_length.' % op_name)
+    else:
+      if op.HasField('src_length') or op.HasField('dst_length'):
+        raise error.PayloadError('%s: unneeded {src,dst}_length.' % op_name)
+
   def _CheckSourceCopyOperation(self, data_offset, total_src_blocks,
                                 total_dst_blocks, op_name):
     """Specific checks for SOURCE_COPY.
@@ -993,16 +1004,17 @@ class PayloadChecker(object):
     elif op.type == common.OpType.ZERO and self.minor_version >= 4:
       self._CheckZeroOperation(op, op_name)
     elif op.type == common.OpType.BSDIFF and self.minor_version == 1:
-      self._CheckAnyDiffOperation(data_length, total_dst_blocks, op_name)
+      self._CheckAnyDiffOperation(op, data_length, total_dst_blocks, op_name)
     elif op.type == common.OpType.SOURCE_COPY and self.minor_version >= 2:
       self._CheckSourceCopyOperation(data_offset, total_src_blocks,
                                      total_dst_blocks, op_name)
       self._CheckAnySourceOperation(op, total_src_blocks, op_name)
     elif op.type == common.OpType.SOURCE_BSDIFF and self.minor_version >= 2:
-      self._CheckAnyDiffOperation(data_length, total_dst_blocks, op_name)
+      self._CheckAnyDiffOperation(op, data_length, total_dst_blocks, op_name)
       self._CheckAnySourceOperation(op, total_src_blocks, op_name)
-    elif op.type == common.OpType.PUFFDIFF and self.minor_version >= 4:
-      self._CheckAnyDiffOperation(data_length, total_dst_blocks, op_name)
+    elif (op.type in (common.OpType.PUFFDIFF, common.OpType.BROTLI_BSDIFF) and
+          self.minor_version >= 4):
+      self._CheckAnyDiffOperation(op, data_length, total_dst_blocks, op_name)
       self._CheckAnySourceOperation(op, total_src_blocks, op_name)
     else:
       raise error.PayloadError(
@@ -1064,6 +1076,7 @@ class PayloadChecker(object):
         common.OpType.SOURCE_COPY: 0,
         common.OpType.SOURCE_BSDIFF: 0,
         common.OpType.PUFFDIFF: 0,
+        common.OpType.BROTLI_BSDIFF: 0,
     }
     # Total blob sizes for each operation type.
     op_blob_totals = {
@@ -1074,6 +1087,7 @@ class PayloadChecker(object):
         # SOURCE_COPY operations don't have blobs.
         common.OpType.SOURCE_BSDIFF: 0,
         common.OpType.PUFFDIFF: 0,
+        common.OpType.BROTLI_BSDIFF: 0,
     }
     # Counts of hashed vs unhashed operations.
     blob_hash_counts = {