diff options
author | Adam Lesinski <adamlesinski@google.com> | 2017-04-03 18:12:45 -0700 |
---|---|---|
committer | Adam Lesinski <adamlesinski@google.com> | 2017-04-10 11:59:18 -0700 |
commit | d0f492db038c6210c1138865d816bfb134376538 (patch) | |
tree | 1d4ed362037758111a28ecb453d527403d5d6815 /tools/aapt2/cmd/Compile.cpp | |
parent | 0015a153536c51cdd66dce5e9f08d19eb517fab8 (diff) |
AAPT2: Share split functionality between link and optimize
Generating splits should be possible to do from the optimize command.
This means that a lot of infrastructure around split APKs can be
shared by both the optimize and link phase.
Bug: 35925830
Change-Id: Ia88b9e4bff300a56353b2f7a4a2547c8eb43a299
Test: manual
Diffstat (limited to 'tools/aapt2/cmd/Compile.cpp')
-rw-r--r-- | tools/aapt2/cmd/Compile.cpp | 740 |
1 files changed, 740 insertions, 0 deletions
diff --git a/tools/aapt2/cmd/Compile.cpp b/tools/aapt2/cmd/Compile.cpp new file mode 100644 index 000000000000..578a8fb423dc --- /dev/null +++ b/tools/aapt2/cmd/Compile.cpp @@ -0,0 +1,740 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <dirent.h> + +#include <fstream> +#include <string> + +#include "android-base/errors.h" +#include "android-base/file.h" +#include "androidfw/StringPiece.h" +#include "google/protobuf/io/coded_stream.h" +#include "google/protobuf/io/zero_copy_stream_impl_lite.h" + +#include "ConfigDescription.h" +#include "Diagnostics.h" +#include "Flags.h" +#include "ResourceParser.h" +#include "ResourceTable.h" +#include "compile/IdAssigner.h" +#include "compile/InlineXmlFormatParser.h" +#include "compile/Png.h" +#include "compile/PseudolocaleGenerator.h" +#include "compile/XmlIdCollector.h" +#include "flatten/Archive.h" +#include "flatten/XmlFlattener.h" +#include "io/BigBufferOutputStream.h" +#include "io/Util.h" +#include "proto/ProtoSerialize.h" +#include "util/Files.h" +#include "util/Maybe.h" +#include "util/Util.h" +#include "xml/XmlDom.h" +#include "xml/XmlPullParser.h" + +using android::StringPiece; +using google::protobuf::io::CopyingOutputStreamAdaptor; + +namespace aapt { + +struct ResourcePathData { + Source source; + std::string resource_dir; + std::string name; + std::string extension; + + // Original config str. We keep this because when we parse the config, we may + // add on + // version qualifiers. We want to preserve the original input so the output is + // easily + // computed before hand. + std::string config_str; + ConfigDescription config; +}; + +/** + * Resource file paths are expected to look like: + * [--/res/]type[-config]/name + */ +static Maybe<ResourcePathData> ExtractResourcePathData(const std::string& path, + std::string* out_error) { + std::vector<std::string> parts = util::Split(path, file::sDirSep); + if (parts.size() < 2) { + if (out_error) *out_error = "bad resource path"; + return {}; + } + + std::string& dir = parts[parts.size() - 2]; + StringPiece dir_str = dir; + + StringPiece config_str; + ConfigDescription config; + size_t dash_pos = dir.find('-'); + if (dash_pos != std::string::npos) { + config_str = dir_str.substr(dash_pos + 1, dir.size() - (dash_pos + 1)); + if (!ConfigDescription::Parse(config_str, &config)) { + if (out_error) { + std::stringstream err_str; + err_str << "invalid configuration '" << config_str << "'"; + *out_error = err_str.str(); + } + return {}; + } + dir_str = dir_str.substr(0, dash_pos); + } + + std::string& filename = parts[parts.size() - 1]; + StringPiece name = filename; + StringPiece extension; + size_t dot_pos = filename.find('.'); + if (dot_pos != std::string::npos) { + extension = name.substr(dot_pos + 1, filename.size() - (dot_pos + 1)); + name = name.substr(0, dot_pos); + } + + return ResourcePathData{Source(path), dir_str.to_string(), name.to_string(), + extension.to_string(), config_str.to_string(), config}; +} + +struct CompileOptions { + std::string output_path; + Maybe<std::string> res_dir; + bool pseudolocalize = false; + bool legacy_mode = false; + bool verbose = false; +}; + +static std::string BuildIntermediateFilename(const ResourcePathData& data) { + std::stringstream name; + name << data.resource_dir; + if (!data.config_str.empty()) { + name << "-" << data.config_str; + } + name << "_" << data.name; + if (!data.extension.empty()) { + name << "." << data.extension; + } + name << ".flat"; + return name.str(); +} + +static bool IsHidden(const StringPiece& filename) { + return util::StartsWith(filename, "."); +} + +/** + * Walks the res directory structure, looking for resource files. + */ +static bool LoadInputFilesFromDir(IAaptContext* context, const CompileOptions& options, + std::vector<ResourcePathData>* out_path_data) { + const std::string& root_dir = options.res_dir.value(); + std::unique_ptr<DIR, decltype(closedir)*> d(opendir(root_dir.data()), closedir); + if (!d) { + context->GetDiagnostics()->Error(DiagMessage() + << android::base::SystemErrorCodeToString(errno)); + return false; + } + + while (struct dirent* entry = readdir(d.get())) { + if (IsHidden(entry->d_name)) { + continue; + } + + std::string prefix_path = root_dir; + file::AppendPath(&prefix_path, entry->d_name); + + if (file::GetFileType(prefix_path) != file::FileType::kDirectory) { + continue; + } + + std::unique_ptr<DIR, decltype(closedir)*> subdir(opendir(prefix_path.data()), closedir); + if (!subdir) { + context->GetDiagnostics()->Error(DiagMessage() + << android::base::SystemErrorCodeToString(errno)); + return false; + } + + while (struct dirent* leaf_entry = readdir(subdir.get())) { + if (IsHidden(leaf_entry->d_name)) { + continue; + } + + std::string full_path = prefix_path; + file::AppendPath(&full_path, leaf_entry->d_name); + + std::string err_str; + Maybe<ResourcePathData> path_data = ExtractResourcePathData(full_path, &err_str); + if (!path_data) { + context->GetDiagnostics()->Error(DiagMessage() << err_str); + return false; + } + + out_path_data->push_back(std::move(path_data.value())); + } + } + return true; +} + +static bool CompileTable(IAaptContext* context, const CompileOptions& options, + const ResourcePathData& path_data, IArchiveWriter* writer, + const std::string& output_path) { + ResourceTable table; + { + std::ifstream fin(path_data.source.path, std::ifstream::binary); + if (!fin) { + context->GetDiagnostics()->Error(DiagMessage(path_data.source) + << android::base::SystemErrorCodeToString(errno)); + return false; + } + + // Parse the values file from XML. + xml::XmlPullParser xml_parser(fin); + + ResourceParserOptions parser_options; + parser_options.error_on_positional_arguments = !options.legacy_mode; + + // If the filename includes donottranslate, then the default translatable is + // false. + parser_options.translatable = path_data.name.find("donottranslate") == std::string::npos; + + ResourceParser res_parser(context->GetDiagnostics(), &table, path_data.source, path_data.config, + parser_options); + if (!res_parser.Parse(&xml_parser)) { + return false; + } + + fin.close(); + } + + if (options.pseudolocalize) { + // Generate pseudo-localized strings (en-XA and ar-XB). + // These are created as weak symbols, and are only generated from default + // configuration + // strings and plurals. + PseudolocaleGenerator pseudolocale_generator; + if (!pseudolocale_generator.Consume(context, &table)) { + return false; + } + } + + // Ensure we have the compilation package at least. + table.CreatePackage(context->GetCompilationPackage()); + + // Assign an ID to any package that has resources. + for (auto& pkg : table.packages) { + if (!pkg->id) { + // If no package ID was set while parsing (public identifiers), auto + // assign an ID. + pkg->id = context->GetPackageId(); + } + } + + // Create the file/zip entry. + if (!writer->StartEntry(output_path, 0)) { + context->GetDiagnostics()->Error(DiagMessage(output_path) << "failed to open"); + return false; + } + + // Make sure CopyingOutputStreamAdaptor is deleted before we call + // writer->FinishEntry(). + { + // Wrap our IArchiveWriter with an adaptor that implements the + // ZeroCopyOutputStream interface. + CopyingOutputStreamAdaptor copying_adaptor(writer); + + std::unique_ptr<pb::ResourceTable> pb_table = SerializeTableToPb(&table); + if (!pb_table->SerializeToZeroCopyStream(©ing_adaptor)) { + context->GetDiagnostics()->Error(DiagMessage(output_path) << "failed to write"); + return false; + } + } + + if (!writer->FinishEntry()) { + context->GetDiagnostics()->Error(DiagMessage(output_path) << "failed to finish entry"); + return false; + } + return true; +} + +static bool WriteHeaderAndBufferToWriter(const StringPiece& output_path, const ResourceFile& file, + const BigBuffer& buffer, IArchiveWriter* writer, + IDiagnostics* diag) { + // Start the entry so we can write the header. + if (!writer->StartEntry(output_path, 0)) { + diag->Error(DiagMessage(output_path) << "failed to open file"); + return false; + } + + // Make sure CopyingOutputStreamAdaptor is deleted before we call + // writer->FinishEntry(). + { + // Wrap our IArchiveWriter with an adaptor that implements the + // ZeroCopyOutputStream interface. + CopyingOutputStreamAdaptor copying_adaptor(writer); + CompiledFileOutputStream output_stream(©ing_adaptor); + + // Number of CompiledFiles. + output_stream.WriteLittleEndian32(1); + + std::unique_ptr<pb::CompiledFile> compiled_file = SerializeCompiledFileToPb(file); + output_stream.WriteCompiledFile(compiled_file.get()); + output_stream.WriteData(&buffer); + + if (output_stream.HadError()) { + diag->Error(DiagMessage(output_path) << "failed to write data"); + return false; + } + } + + if (!writer->FinishEntry()) { + diag->Error(DiagMessage(output_path) << "failed to finish writing data"); + return false; + } + return true; +} + +static bool WriteHeaderAndMmapToWriter(const StringPiece& output_path, const ResourceFile& file, + const android::FileMap& map, IArchiveWriter* writer, + IDiagnostics* diag) { + // Start the entry so we can write the header. + if (!writer->StartEntry(output_path, 0)) { + diag->Error(DiagMessage(output_path) << "failed to open file"); + return false; + } + + // Make sure CopyingOutputStreamAdaptor is deleted before we call + // writer->FinishEntry(). + { + // Wrap our IArchiveWriter with an adaptor that implements the + // ZeroCopyOutputStream interface. + CopyingOutputStreamAdaptor copying_adaptor(writer); + CompiledFileOutputStream output_stream(©ing_adaptor); + + // Number of CompiledFiles. + output_stream.WriteLittleEndian32(1); + + std::unique_ptr<pb::CompiledFile> compiled_file = SerializeCompiledFileToPb(file); + output_stream.WriteCompiledFile(compiled_file.get()); + output_stream.WriteData(map.getDataPtr(), map.getDataLength()); + + if (output_stream.HadError()) { + diag->Error(DiagMessage(output_path) << "failed to write data"); + return false; + } + } + + if (!writer->FinishEntry()) { + diag->Error(DiagMessage(output_path) << "failed to finish writing data"); + return false; + } + return true; +} + +static bool FlattenXmlToOutStream(IAaptContext* context, const StringPiece& output_path, + xml::XmlResource* xmlres, CompiledFileOutputStream* out) { + BigBuffer buffer(1024); + XmlFlattenerOptions xml_flattener_options; + xml_flattener_options.keep_raw_values = true; + XmlFlattener flattener(&buffer, xml_flattener_options); + if (!flattener.Consume(context, xmlres)) { + return false; + } + + std::unique_ptr<pb::CompiledFile> pb_compiled_file = SerializeCompiledFileToPb(xmlres->file); + out->WriteCompiledFile(pb_compiled_file.get()); + out->WriteData(&buffer); + + if (out->HadError()) { + context->GetDiagnostics()->Error(DiagMessage(output_path) << "failed to write data"); + return false; + } + return true; +} + +static bool CompileXml(IAaptContext* context, const CompileOptions& options, + const ResourcePathData& path_data, IArchiveWriter* writer, + const std::string& output_path) { + if (context->IsVerbose()) { + context->GetDiagnostics()->Note(DiagMessage(path_data.source) << "compiling XML"); + } + + std::unique_ptr<xml::XmlResource> xmlres; + { + std::ifstream fin(path_data.source.path, std::ifstream::binary); + if (!fin) { + context->GetDiagnostics()->Error(DiagMessage(path_data.source) + << android::base::SystemErrorCodeToString(errno)); + return false; + } + + xmlres = xml::Inflate(&fin, context->GetDiagnostics(), path_data.source); + + fin.close(); + } + + if (!xmlres) { + return false; + } + + xmlres->file.name = ResourceName({}, *ParseResourceType(path_data.resource_dir), path_data.name); + xmlres->file.config = path_data.config; + xmlres->file.source = path_data.source; + + // Collect IDs that are defined here. + XmlIdCollector collector; + if (!collector.Consume(context, xmlres.get())) { + return false; + } + + // Look for and process any <aapt:attr> tags and create sub-documents. + InlineXmlFormatParser inline_xml_format_parser; + if (!inline_xml_format_parser.Consume(context, xmlres.get())) { + return false; + } + + // Start the entry so we can write the header. + if (!writer->StartEntry(output_path, 0)) { + context->GetDiagnostics()->Error(DiagMessage(output_path) << "failed to open file"); + return false; + } + + // Make sure CopyingOutputStreamAdaptor is deleted before we call + // writer->FinishEntry(). + { + // Wrap our IArchiveWriter with an adaptor that implements the + // ZeroCopyOutputStream + // interface. + CopyingOutputStreamAdaptor copying_adaptor(writer); + CompiledFileOutputStream output_stream(©ing_adaptor); + + std::vector<std::unique_ptr<xml::XmlResource>>& inline_documents = + inline_xml_format_parser.GetExtractedInlineXmlDocuments(); + + // Number of CompiledFiles. + output_stream.WriteLittleEndian32(1 + inline_documents.size()); + + if (!FlattenXmlToOutStream(context, output_path, xmlres.get(), &output_stream)) { + return false; + } + + for (auto& inline_xml_doc : inline_documents) { + if (!FlattenXmlToOutStream(context, output_path, inline_xml_doc.get(), &output_stream)) { + return false; + } + } + } + + if (!writer->FinishEntry()) { + context->GetDiagnostics()->Error(DiagMessage(output_path) << "failed to finish writing data"); + return false; + } + return true; +} + +static bool CompilePng(IAaptContext* context, const CompileOptions& options, + const ResourcePathData& path_data, IArchiveWriter* writer, + const std::string& output_path) { + if (context->IsVerbose()) { + context->GetDiagnostics()->Note(DiagMessage(path_data.source) << "compiling PNG"); + } + + BigBuffer buffer(4096); + ResourceFile res_file; + res_file.name = ResourceName({}, *ParseResourceType(path_data.resource_dir), path_data.name); + res_file.config = path_data.config; + res_file.source = path_data.source; + + { + std::string content; + if (!android::base::ReadFileToString(path_data.source.path, &content)) { + context->GetDiagnostics()->Error(DiagMessage(path_data.source) + << android::base::SystemErrorCodeToString(errno)); + return false; + } + + BigBuffer crunched_png_buffer(4096); + io::BigBufferOutputStream crunched_png_buffer_out(&crunched_png_buffer); + + // Ensure that we only keep the chunks we care about if we end up + // using the original PNG instead of the crunched one. + PngChunkFilter png_chunk_filter(content); + std::unique_ptr<Image> image = ReadPng(context, &png_chunk_filter); + if (!image) { + return false; + } + + std::unique_ptr<NinePatch> nine_patch; + if (path_data.extension == "9.png") { + std::string err; + nine_patch = NinePatch::Create(image->rows.get(), image->width, image->height, &err); + if (!nine_patch) { + context->GetDiagnostics()->Error(DiagMessage() << err); + return false; + } + + // Remove the 1px border around the NinePatch. + // Basically the row array is shifted up by 1, and the length is treated + // as height - 2. + // For each row, shift the array to the left by 1, and treat the length as + // width - 2. + image->width -= 2; + image->height -= 2; + memmove(image->rows.get(), image->rows.get() + 1, image->height * sizeof(uint8_t**)); + for (int32_t h = 0; h < image->height; h++) { + memmove(image->rows[h], image->rows[h] + 4, image->width * 4); + } + + if (context->IsVerbose()) { + context->GetDiagnostics()->Note(DiagMessage(path_data.source) << "9-patch: " + << *nine_patch); + } + } + + // Write the crunched PNG. + if (!WritePng(context, image.get(), nine_patch.get(), &crunched_png_buffer_out, {})) { + return false; + } + + if (nine_patch != nullptr || + crunched_png_buffer_out.ByteCount() <= png_chunk_filter.ByteCount()) { + // No matter what, we must use the re-encoded PNG, even if it is larger. + // 9-patch images must be re-encoded since their borders are stripped. + buffer.AppendBuffer(std::move(crunched_png_buffer)); + } else { + // The re-encoded PNG is larger than the original, and there is + // no mandatory transformation. Use the original. + if (context->IsVerbose()) { + context->GetDiagnostics()->Note(DiagMessage(path_data.source) + << "original PNG is smaller than crunched PNG" + << ", using original"); + } + + png_chunk_filter.Rewind(); + BigBuffer filtered_png_buffer(4096); + io::BigBufferOutputStream filtered_png_buffer_out(&filtered_png_buffer); + io::Copy(&filtered_png_buffer_out, &png_chunk_filter); + buffer.AppendBuffer(std::move(filtered_png_buffer)); + } + + if (context->IsVerbose()) { + // For debugging only, use the legacy PNG cruncher and compare the resulting file sizes. + // This will help catch exotic cases where the new code may generate larger PNGs. + std::stringstream legacy_stream(content); + BigBuffer legacy_buffer(4096); + Png png(context->GetDiagnostics()); + if (!png.process(path_data.source, &legacy_stream, &legacy_buffer, {})) { + return false; + } + + context->GetDiagnostics()->Note(DiagMessage(path_data.source) + << "legacy=" << legacy_buffer.size() + << " new=" << buffer.size()); + } + } + + if (!WriteHeaderAndBufferToWriter(output_path, res_file, buffer, writer, + context->GetDiagnostics())) { + return false; + } + return true; +} + +static bool CompileFile(IAaptContext* context, const CompileOptions& options, + const ResourcePathData& path_data, IArchiveWriter* writer, + const std::string& output_path) { + if (context->IsVerbose()) { + context->GetDiagnostics()->Note(DiagMessage(path_data.source) << "compiling file"); + } + + BigBuffer buffer(256); + ResourceFile res_file; + res_file.name = ResourceName({}, *ParseResourceType(path_data.resource_dir), path_data.name); + res_file.config = path_data.config; + res_file.source = path_data.source; + + std::string error_str; + Maybe<android::FileMap> f = file::MmapPath(path_data.source.path, &error_str); + if (!f) { + context->GetDiagnostics()->Error(DiagMessage(path_data.source) << error_str); + return false; + } + + if (!WriteHeaderAndMmapToWriter(output_path, res_file, f.value(), writer, + context->GetDiagnostics())) { + return false; + } + return true; +} + +class CompileContext : public IAaptContext { + public: + void SetVerbose(bool val) { + verbose_ = val; + } + + bool IsVerbose() override { + return verbose_; + } + + IDiagnostics* GetDiagnostics() override { + return &diagnostics_; + } + + NameMangler* GetNameMangler() override { + abort(); + return nullptr; + } + + const std::string& GetCompilationPackage() override { + static std::string empty; + return empty; + } + + uint8_t GetPackageId() override { + return 0x0; + } + + SymbolTable* GetExternalSymbols() override { + abort(); + return nullptr; + } + + int GetMinSdkVersion() override { + return 0; + } + + private: + StdErrDiagnostics diagnostics_; + bool verbose_ = false; +}; + +/** + * Entry point for compilation phase. Parses arguments and dispatches to the + * correct steps. + */ +int Compile(const std::vector<StringPiece>& args) { + CompileContext context; + CompileOptions options; + + bool verbose = false; + Flags flags = + Flags() + .RequiredFlag("-o", "Output path", &options.output_path) + .OptionalFlag("--dir", "Directory to scan for resources", &options.res_dir) + .OptionalSwitch("--pseudo-localize", + "Generate resources for pseudo-locales " + "(en-XA and ar-XB)", + &options.pseudolocalize) + .OptionalSwitch("--legacy", "Treat errors that used to be valid in AAPT as warnings", + &options.legacy_mode) + .OptionalSwitch("-v", "Enables verbose logging", &verbose); + if (!flags.Parse("aapt2 compile", args, &std::cerr)) { + return 1; + } + + context.SetVerbose(verbose); + + std::unique_ptr<IArchiveWriter> archive_writer; + + std::vector<ResourcePathData> input_data; + if (options.res_dir) { + if (!flags.GetArgs().empty()) { + // Can't have both files and a resource directory. + context.GetDiagnostics()->Error(DiagMessage() << "files given but --dir specified"); + flags.Usage("aapt2 compile", &std::cerr); + return 1; + } + + if (!LoadInputFilesFromDir(&context, options, &input_data)) { + return 1; + } + + archive_writer = CreateZipFileArchiveWriter(context.GetDiagnostics(), options.output_path); + + } else { + input_data.reserve(flags.GetArgs().size()); + + // Collect data from the path for each input file. + for (const std::string& arg : flags.GetArgs()) { + std::string error_str; + if (Maybe<ResourcePathData> path_data = ExtractResourcePathData(arg, &error_str)) { + input_data.push_back(std::move(path_data.value())); + } else { + context.GetDiagnostics()->Error(DiagMessage() << error_str << " (" << arg << ")"); + return 1; + } + } + + archive_writer = CreateDirectoryArchiveWriter(context.GetDiagnostics(), options.output_path); + } + + if (!archive_writer) { + return 1; + } + + bool error = false; + for (ResourcePathData& path_data : input_data) { + if (options.verbose) { + context.GetDiagnostics()->Note(DiagMessage(path_data.source) << "processing"); + } + + if (path_data.resource_dir == "values") { + // Overwrite the extension. + path_data.extension = "arsc"; + + const std::string output_filename = BuildIntermediateFilename(path_data); + if (!CompileTable(&context, options, path_data, archive_writer.get(), output_filename)) { + error = true; + } + + } else { + const std::string output_filename = BuildIntermediateFilename(path_data); + if (const ResourceType* type = ParseResourceType(path_data.resource_dir)) { + if (*type != ResourceType::kRaw) { + if (path_data.extension == "xml") { + if (!CompileXml(&context, options, path_data, archive_writer.get(), output_filename)) { + error = true; + } + } else if (path_data.extension == "png" || path_data.extension == "9.png") { + if (!CompilePng(&context, options, path_data, archive_writer.get(), output_filename)) { + error = true; + } + } else { + if (!CompileFile(&context, options, path_data, archive_writer.get(), output_filename)) { + error = true; + } + } + } else { + if (!CompileFile(&context, options, path_data, archive_writer.get(), output_filename)) { + error = true; + } + } + } else { + context.GetDiagnostics()->Error(DiagMessage() << "invalid file path '" << path_data.source + << "'"); + error = true; + } + } + } + + if (error) { + return 1; + } + return 0; +} + +} // namespace aapt |