diff options
437 files changed, 67032 insertions, 13697 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..a09c56df5 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/.idea diff --git a/Android.bp b/Android.bp index 959cbf1ce..2df1afb66 100644 --- a/Android.bp +++ b/Android.bp @@ -11,14 +11,6 @@ subdirs = [ ] bootstrap_go_package { - name: "soong-env", - pkgPath: "android/soong/env", - srcs: [ - "env/env.go", - ], -} - -bootstrap_go_package { name: "soong", pkgPath: "android/soong", deps: [ @@ -29,416 +21,6 @@ bootstrap_go_package { ], } -bootstrap_go_package { - name: "soong-android", - pkgPath: "android/soong/android", - deps: [ - "blueprint", - "blueprint-bootstrap", - "soong", - "soong-env", - "soong-shared", - ], - srcs: [ - "android/androidmk.go", - "android/apex.go", - "android/api_levels.go", - "android/arch.go", - "android/config.go", - "android/defaults.go", - "android/defs.go", - "android/expand.go", - "android/filegroup.go", - "android/hooks.go", - "android/license.go", - "android/makevars.go", - "android/module.go", - "android/mutator.go", - "android/namespace.go", - "android/neverallow.go", - "android/notices.go", - "android/onceper.go", - "android/override_module.go", - "android/package.go", - "android/package_ctx.go", - "android/path_properties.go", - "android/paths.go", - "android/prebuilt.go", - "android/prebuilt_etc.go", - "android/proto.go", - "android/register.go", - "android/rule_builder.go", - "android/sh_binary.go", - "android/singleton.go", - "android/testing.go", - "android/util.go", - "android/variable.go", - "android/vts_config.go", - "android/writedocs.go", - - // Lock down environment access last - "android/env.go", - ], - testSrcs: [ - "android/arch_test.go", - "android/config_test.go", - "android/expand_test.go", - "android/license_test.go", - "android/namespace_test.go", - "android/neverallow_test.go", - "android/onceper_test.go", - "android/path_properties_test.go", - "android/package_test.go", - "android/paths_test.go", - "android/prebuilt_test.go", - "android/prebuilt_etc_test.go", - "android/rule_builder_test.go", - "android/util_test.go", - "android/variable_test.go", - "android/vts_config_test.go", - ], -} - -bootstrap_go_package { - name: "soong-cc-config", - pkgPath: "android/soong/cc/config", - deps: [ - "soong-android", - "soong-remoteexec", - ], - srcs: [ - "cc/config/clang.go", - "cc/config/global.go", - "cc/config/tidy.go", - "cc/config/toolchain.go", - "cc/config/vndk.go", - - "cc/config/arm_device.go", - "cc/config/arm64_device.go", - "cc/config/arm64_fuchsia_device.go", - "cc/config/mips_device.go", - "cc/config/mips64_device.go", - "cc/config/x86_device.go", - "cc/config/x86_64_device.go", - "cc/config/x86_64_fuchsia_device.go", - - "cc/config/x86_darwin_host.go", - "cc/config/x86_linux_host.go", - "cc/config/x86_linux_bionic_host.go", - "cc/config/x86_windows_host.go", - ], - testSrcs: [ - "cc/config/tidy_test.go", - ], -} - -bootstrap_go_package { - name: "soong-cc", - pkgPath: "android/soong/cc", - deps: [ - "blueprint", - "blueprint-pathtools", - "soong", - "soong-android", - "soong-cc-config", - "soong-genrule", - "soong-tradefed", - ], - srcs: [ - "cc/androidmk.go", - "cc/builder.go", - "cc/cc.go", - "cc/check.go", - "cc/coverage.go", - "cc/gen.go", - "cc/lto.go", - "cc/makevars.go", - "cc/pgo.go", - "cc/prebuilt.go", - "cc/proto.go", - "cc/rs.go", - "cc/sanitize.go", - "cc/sabi.go", - "cc/stl.go", - "cc/strip.go", - "cc/sysprop.go", - "cc/tidy.go", - "cc/util.go", - "cc/vndk.go", - "cc/vndk_prebuilt.go", - "cc/xom.go", - - "cc/cmakelists.go", - "cc/compdb.go", - "cc/compiler.go", - "cc/installer.go", - "cc/linker.go", - - "cc/binary.go", - "cc/library.go", - "cc/object.go", - "cc/test.go", - "cc/toolchain_library.go", - - "cc/ndk_prebuilt.go", - "cc/ndk_headers.go", - "cc/ndk_library.go", - "cc/ndk_sysroot.go", - - "cc/llndk_library.go", - - "cc/kernel_headers.go", - - "cc/genrule.go", - - "cc/vendor_public_library.go", - - "cc/testing.go", - ], - testSrcs: [ - "cc/cc_test.go", - "cc/gen_test.go", - "cc/genrule_test.go", - "cc/library_test.go", - "cc/prebuilt_test.go", - "cc/proto_test.go", - "cc/test_data_test.go", - "cc/util_test.go", - ], - pluginFor: ["soong_build"], -} - -bootstrap_go_package { - name: "soong-genrule", - pkgPath: "android/soong/genrule", - deps: [ - "blueprint", - "blueprint-pathtools", - "soong", - "soong-android", - "soong-shared", - ], - srcs: [ - "genrule/genrule.go", - ], - testSrcs: [ - "genrule/genrule_test.go", - ], - pluginFor: ["soong_build"], -} - -bootstrap_go_package { - name: "soong-phony", - pkgPath: "android/soong/phony", - deps: [ - "blueprint", - "soong-android", - ], - srcs: [ - "phony/phony.go", - ], - pluginFor: ["soong_build"], -} - -bootstrap_go_package { - name: "soong-java", - pkgPath: "android/soong/java", - deps: [ - "blueprint", - "blueprint-pathtools", - "soong", - "soong-android", - "soong-cc", - "soong-dexpreopt", - "soong-genrule", - "soong-java-config", - "soong-tradefed", - ], - srcs: [ - "java/aapt2.go", - "java/aar.go", - "java/android_manifest.go", - "java/android_resources.go", - "java/androidmk.go", - "java/app_builder.go", - "java/app.go", - "java/builder.go", - "java/device_host_converter.go", - "java/dex.go", - "java/dexpreopt.go", - "java/dexpreopt_bootjars.go", - "java/dexpreopt_config.go", - "java/droiddoc.go", - "java/gen.go", - "java/genrule.go", - "java/hiddenapi.go", - "java/hiddenapi_singleton.go", - "java/jacoco.go", - "java/java.go", - "java/jdeps.go", - "java/java_resources.go", - "java/kotlin.go", - "java/plugin.go", - "java/prebuilt_apis.go", - "java/proto.go", - "java/sdk.go", - "java/sdk_library.go", - "java/support_libraries.go", - "java/system_modules.go", - "java/testing.go", - ], - testSrcs: [ - "java/app_test.go", - "java/device_host_converter_test.go", - "java/dexpreopt_test.go", - "java/dexpreopt_bootjars_test.go", - "java/java_test.go", - "java/jdeps_test.go", - "java/kotlin_test.go", - "java/plugin_test.go", - "java/sdk_test.go", - ], - pluginFor: ["soong_build"], -} - -bootstrap_go_package { - name: "soong-java-config", - pkgPath: "android/soong/java/config", - deps: [ - "blueprint-proptools", - "soong-android", - "soong-remoteexec", - ], - srcs: [ - "java/config/config.go", - "java/config/error_prone.go", - "java/config/kotlin.go", - "java/config/makevars.go", - ], -} - -bootstrap_go_package { - name: "soong-python", - pkgPath: "android/soong/python", - deps: [ - "blueprint", - "soong-android", - "soong-tradefed", - ], - srcs: [ - "python/androidmk.go", - "python/binary.go", - "python/builder.go", - "python/defaults.go", - "python/installer.go", - "python/library.go", - "python/proto.go", - "python/python.go", - "python/test.go", - ], - testSrcs: [ - "python/python_test.go", - ], - pluginFor: ["soong_build"], -} - -bootstrap_go_package { - name: "soong-shared", - pkgPath: "android/soong/shared", - srcs: [ - "shared/paths.go", - ], -} - -bootstrap_go_package { - name: "soong-tradefed", - pkgPath: "android/soong/tradefed", - deps: [ - "blueprint", - "soong-android", - ], - srcs: [ - "tradefed/autogen.go", - "tradefed/config.go", - "tradefed/makevars.go", - ], - pluginFor: ["soong_build"], -} - -bootstrap_go_package { - name: "soong-xml", - pkgPath: "android/soong/xml", - deps: [ - "blueprint", - "blueprint-pathtools", - "soong", - "soong-android", - ], - srcs: [ - "xml/xml.go", - ], - testSrcs: [ - "xml/xml_test.go", - ], - pluginFor: ["soong_build"], -} - -bootstrap_go_package { - name: "soong-apex", - pkgPath: "android/soong/apex", - deps: [ - "blueprint", - "soong", - "soong-android", - "soong-cc", - "soong-java", - "soong-python", - ], - srcs: [ - "apex/apex.go", - "apex/key.go", - ], - testSrcs: [ - "apex/apex_test.go", - ], - pluginFor: ["soong_build"], -} - -bootstrap_go_package { - name: "soong-sysprop", - pkgPath: "android/soong/sysprop", - deps: [ - "blueprint", - "soong", - "soong-android", - "soong-cc", - "soong-java", - ], - srcs: [ - "sysprop/sysprop_library.go", - ], - testSrcs: [ - "sysprop/sysprop_test.go", - ], - pluginFor: ["soong_build"], -} - -bootstrap_go_package { - name: "soong-remoteexec", - pkgPath: "android/soong/remoteexec", - deps: [ - "blueprint", - "soong-android", - ], - srcs: [ - "remoteexec/remoteexec.go", - ], - testSrcs: [ - "remoteexec/remoteexec_test.go", - ], - pluginFor: ["soong_build"], -} - // // Defaults to enable various configurations of host bionic // @@ -464,7 +46,9 @@ toolchain_library { name: "libatomic", defaults: ["linux_bionic_supported"], vendor_available: true, + ramdisk_available: true, recovery_available: true, + native_bridge_supported: true, arch: { arm: { @@ -487,6 +71,7 @@ toolchain_library { defaults: ["linux_bionic_supported"], vendor_available: true, recovery_available: true, + native_bridge_supported: true, arch: { arm: { @@ -508,111 +93,29 @@ toolchain_library { name: "libgcc_stripped", defaults: ["linux_bionic_supported"], vendor_available: true, + ramdisk_available: true, recovery_available: true, + native_bridge_supported: true, + sdk_version: "current", arch: { arm: { src: "prebuilts/gcc/linux-x86/arm/arm-linux-androideabi-4.9/lib/gcc/arm-linux-androideabi/4.9.x/libgcc.a", - strip: { - keep_symbols_list: [ - // unwind-arm.o - "_Unwind_Complete", - "_Unwind_DeleteException", - "_Unwind_GetCFA", - "_Unwind_VRS_Get", - "_Unwind_VRS_Pop", - "_Unwind_VRS_Set", - "__aeabi_unwind_cpp_pr0", - "__aeabi_unwind_cpp_pr1", - "__aeabi_unwind_cpp_pr2", - "__gnu_Unwind_Backtrace", - "__gnu_Unwind_ForcedUnwind", - "__gnu_Unwind_RaiseException", - "__gnu_Unwind_Resume", - "__gnu_Unwind_Resume_or_Rethrow", - - // libunwind.o - "_Unwind_Backtrace", - "_Unwind_ForcedUnwind", - "_Unwind_RaiseException", - "_Unwind_Resume", - "_Unwind_Resume_or_Rethrow", - "___Unwind_Backtrace", - "___Unwind_ForcedUnwind", - "___Unwind_RaiseException", - "___Unwind_Resume", - "___Unwind_Resume_or_Rethrow", - "__gnu_Unwind_Restore_VFP", - "__gnu_Unwind_Restore_VFP_D", - "__gnu_Unwind_Restore_VFP_D_16_to_31", - "__gnu_Unwind_Restore_WMMXC", - "__gnu_Unwind_Restore_WMMXD", - "__gnu_Unwind_Save_VFP", - "__gnu_Unwind_Save_VFP_D", - "__gnu_Unwind_Save_VFP_D_16_to_31", - "__gnu_Unwind_Save_WMMXC", - "__gnu_Unwind_Save_WMMXD", - "__restore_core_regs", - "restore_core_regs", - - // pr-support.o - "_Unwind_GetDataRelBase", - "_Unwind_GetLanguageSpecificData", - "_Unwind_GetRegionStart", - "_Unwind_GetTextRelBase", - "__gnu_unwind_execute", - "__gnu_unwind_frame", - ], - use_gnu_strip: true, - }, + repack_objects_to_keep: ["unwind-arm.o", "libunwind.o", "pr-support.o"], }, arm64: { src: "prebuilts/gcc/linux-x86/aarch64/aarch64-linux-android-4.9/lib/gcc/aarch64-linux-android/4.9.x/libgcc.a", + repack_objects_to_keep: ["unwind-dw2.o", "unwind-dw2-fde-dip.o"], }, x86: { src: "prebuilts/gcc/linux-x86/x86/x86_64-linux-android-4.9/lib/gcc/x86_64-linux-android/4.9.x/32/libgcc.a", - + repack_objects_to_keep: ["unwind-dw2.o", "unwind-dw2-fde-dip.o"], }, x86_64: { src: "prebuilts/gcc/linux-x86/x86/x86_64-linux-android-4.9/lib/gcc/x86_64-linux-android/4.9.x/libgcc.a", + repack_objects_to_keep: ["unwind-dw2.o", "unwind-dw2-fde-dip.o"], }, }, - strip: { - keep_symbols_list: [ - // unwind-dw2.o - "_Unwind_Backtrace", - "_Unwind_DeleteException", - "_Unwind_FindEnclosingFunction", - "_Unwind_ForcedUnwind", - "_Unwind_GetCFA", - "_Unwind_GetDataRelBase", - "_Unwind_GetGR", - "_Unwind_GetIP", - "_Unwind_GetIPInfo", - "_Unwind_GetLanguageSpecificData", - "_Unwind_GetRegionStart", - "_Unwind_GetTextRelBase", - "_Unwind_RaiseException", - "_Unwind_Resume", - "_Unwind_Resume_or_Rethrow", - "_Unwind_SetGR", - "_Unwind_SetIP", - "__frame_state_for", - - // unwind-dw2-fde-dip.o - "_Unwind_Find_FDE", - "__deregister_frame", - "__deregister_frame_info", - "__deregister_frame_info_bases", - "__register_frame", - "__register_frame_info", - "__register_frame_info_bases", - "__register_frame_info_table", - "__register_frame_info_table_bases", - "__register_frame_table", - ], - use_gnu_strip: true, - }, } toolchain_library { @@ -36,13 +36,28 @@ all Android.bp files. For a list of valid module types and their properties see [$OUT_DIR/soong/docs/soong_build.html](https://ci.android.com/builds/latest/branches/aosp-build-tools/targets/linux/view/soong_build.html). -### Globs +### File lists -Properties that take a list of files can also take glob patterns. Glob -patterns can contain the normal Unix wildcard `*`, for example "*.java". Glob -patterns can also contain a single `**` wildcard as a path element, which will -match zero or more path elements. For example, `java/**/*.java` will match -`java/Main.java` and `java/com/android/Main.java`. +Properties that take a list of files can also take glob patterns and output path +expansions. + +* Glob patterns can contain the normal Unix wildcard `*`, for example `"*.java"`. + + Glob patterns can also contain a single `**` wildcard as a path element, which + will match zero or more path elements. For example, `java/**/*.java` will match + `java/Main.java` and `java/com/android/Main.java`. + +* Output path expansions take the format `:module` or `:module{.tag}`, where + `module` is the name of a module that produces output files, and it expands to + a list of those output files. With the optional `{.tag}` suffix, the module + may produce a different list of outputs according to `tag`. + + For example, a `droiddoc` module with the name "my-docs" would return its + `.stubs.srcjar` output with `":my-docs"`, and its `.doc.zip` file with + `":my-docs{.doc.zip}"`. + + This is commonly used to reference `filegroup` modules, whose output files + consist of their `srcs`. ### Variables @@ -59,11 +74,12 @@ cc_binary { ``` Variables are scoped to the remainder of the file they are declared in, as well -as any child blueprint files. Variables are immutable with one exception - they +as any child Android.bp files. Variables are immutable with one exception - they can be appended to with a += assignment, but only before they have been referenced. ### Comments + Android.bp files can contain C-style multiline `/* */` and C++ style single-line `//` comments. @@ -81,6 +97,8 @@ types are: Maps may values of any type, including nested maps. Lists and maps may have trailing commas after the last value. +Strings can contain double quotes using `\"`, for example `"cat \"a b\""`. + ### Operators Strings, lists of strings, and maps can be appended using the `+` operator. @@ -107,41 +125,228 @@ cc_binary { } ``` -### Name resolution +### Packages + +The build is organized into packages where each package is a collection of related files and a +specification of the dependencies among them in the form of modules. + +A package is defined as a directory containing a file named `Android.bp`, residing beneath the +top-level directory in the build and its name is its path relative to the top-level directory. A +package includes all files in its directory, plus all subdirectories beneath it, except those which +themselves contain an `Android.bp` file. + +The modules in a package's `Android.bp` and included files are part of the module. + +For example, in the following directory tree (where `.../android/` is the top-level Android +directory) there are two packages, `my/app`, and the subpackage `my/app/tests`. Note that +`my/app/data` is not a package, but a directory belonging to package `my/app`. + + .../android/my/app/Android.bp + .../android/my/app/app.cc + .../android/my/app/data/input.txt + .../android/my/app/tests/Android.bp + .../android/my/app/tests/test.cc + +This is based on the Bazel package concept. + +The `package` module type allows information to be specified about a package. Only a single +`package` module can be specified per package and in the case where there are multiple `.bp` files +in the same package directory it is highly recommended that the `package` module (if required) is +specified in the `Android.bp` file. + +Unlike most module type `package` does not have a `name` property. Instead the name is set to the +name of the package, e.g. if the package is in `top/intermediate/package` then the package name is +`//top/intermediate/package`. + +E.g. The following will set the default visibility for all the modules defined in the package and +any subpackages that do not set their own default visibility (irrespective of whether they are in +the same `.bp` file as the `package` module) to be visible to all the subpackages by default. + +``` +package { + default_visibility: [":__subpackages"] +} +``` + +### Referencing Modules + +A module `libfoo` can be referenced by its name + +``` +cc_binary { + name: "app", + shared_libs: ["libfoo"], +} +``` + +Obviously, this works only if there is only one `libfoo` module in the source +tree. Ensuring such name uniqueness for larger trees may become problematic. We +might also want to use the same name in multiple mutually exclusive subtrees +(for example, implementing different devices) deliberately in order to describe +a functionally equivalent module. Enter Soong namespaces. + +#### Namespaces -Soong provides the ability for modules in different directories to specify -the same name, as long as each module is declared within a separate namespace. -A namespace can be declared like this: +A presense of the `soong_namespace {..}` in an Android.bp file defines a +**namespace**. For instance, having ``` soong_namespace { - imports: ["path/to/otherNamespace1", "path/to/otherNamespace2"], + ... } +... ``` -Each Soong module is assigned a namespace based on its location in the tree. -Each Soong module is considered to be in the namespace defined by the -soong_namespace found in an Android.bp in the current directory or closest -ancestor directory, unless no such soong_namespace module is found, in which -case the module is considered to be in the implicit root namespace. +in `device/google/bonito/Android.bp` informs Soong that within the +`device/google/bonito` package the module names are unique, that is, all the +modules defined in the Android.bp files in the `device/google/bonito/` tree have +unique names. However, there may be modules with the same names outside +`device/google/bonito` tree. Indeed, there is a module `"pixelstats-vendor"` +both in `device/google/bonito/pixelstats` and in +`device/google/coral/pixelstats`. + +The name of a namespace is the path of its directory. The name of the namespace +in the example above is thus `device/google/bonito`. + +An implicit **global namespace** corresponds to the source tree as a whole. It +has empty name. + +A module name's **scope** is the smallest namespace containing it. Suppose a +source tree has `device/my` and `device/my/display` namespaces. If `libfoo` +module is defined in `device/co/display/lib/Android.bp`, its namespace is +`device/co/display`. -When Soong attempts to resolve dependency D declared my module M in namespace -N which imports namespaces I1, I2, I3..., then if D is a fully-qualified name -of the form "//namespace:module", only the specified namespace will be searched -for the specified module name. Otherwise, Soong will first look for a module -named D declared in namespace N. If that module does not exist, Soong will look -for a module named D in namespaces I1, I2, I3... Lastly, Soong will look in the -root namespace. +The name uniqueness thus means that module's name is unique within its scope. In +other words, "//_scope_:_name_" is globally unique module reference, e.g, +`"//device/google/bonito:pixelstats-vendor"`. _Note_ that the name of the +namespace for a module may be different from module's package name: `libfoo` +belongs to `device/my/display` namespace but is contained in +`device/my/display/lib` package. -Until we have fully converted from Make to Soong, it will be necessary for the -Make product config to specify a value of PRODUCT_SOONG_NAMESPACES. Its value -should be a space-separated list of namespaces that Soong export to Make to be -built by the `m` command. After we have fully converted from Make to Soong, the -details of enabling namespaces could potentially change. +#### Name Resolution + +The form of a module reference determines how Soong locates the module. + +For a **global reference** of the "//_scope_:_name_" form, Soong verifies there +is a namespace called "_scope_", then verifies it contains a "_name_" module and +uses it. Soong verifies there is only one "_name_" in "_scope_" at the beginning +when it parses Android.bp files. + +A **local reference** has "_name_" form, and resolving it involves looking for a +module "_name_" in one or more namespaces. By default only the global namespace +is searched for "_name_" (in other words, only the modules not belonging to an +explicitly defined scope are considered). The `imports` attribute of the +`soong_namespaces` allows to specify where to look for modules . For instance, +with `device/google/bonito/Android.bp` containing + +``` +soong_namespace { + imports: [ + "hardware/google/interfaces", + "hardware/google/pixel", + "hardware/qcom/bootctrl", + ], +} +``` + +a reference to `"libpixelstats"` will resolve to the module defined in +`hardware/google/pixel/pixelstats/Android.bp` because this module is in +`hardware/google/pixel` namespace. + +**TODO**: Conventionally, languages with similar concepts provide separate +constructs for namespace definition and name resolution (`namespace` and `using` +in C++, for instance). Should Soong do that, too? + +#### Referencing modules in makefiles + +While we are gradually converting makefiles to Android.bp files, Android build +is described by a mixture of Android.bp and Android.mk files, and a module +defined in an Android.mk file can reference a module defined in Android.bp file. +For instance, a binary still defined in an Android.mk file may have a library +defined in already converted Android.bp as a dependency. + +A module defined in an Android.bp file and belonging to the global namespace can +be referenced from a makefile without additional effort. If a module belongs to +an explicit namespace, it can be referenced from a makefile only after after the +name of the namespace has been added to the value of PRODUCT_SOONG_NAMESPACES +variable. + +Note that makefiles have no notion of namespaces and exposing namespaces with +the same modules via PRODUCT_SOONG_NAMESPACES may cause Make failure. For +instance, exposing both `device/google/bonito` and `device/google/coral` +namespaces will cause Make failure because it will see two targets for the +`pixelstats-vendor` module. + +### Visibility + +The `visibility` property on a module controls whether the module can be +used by other packages. Modules are always visible to other modules declared +in the same package. This is based on the Bazel visibility mechanism. + +If specified the `visibility` property must contain at least one rule. + +Each rule in the property must be in one of the following forms: +* `["//visibility:public"]`: Anyone can use this module. +* `["//visibility:private"]`: Only rules in the module's package (not its +subpackages) can use this module. +* `["//visibility:override"]`: Discards any rules inherited from defaults or a +creating module. Can only be used at the beginning of a list of visibility +rules. +* `["//some/package:__pkg__", "//other/package:__pkg__"]`: Only modules in +`some/package` and `other/package` (defined in `some/package/*.bp` and +`other/package/*.bp`) have access to this module. Note that sub-packages do not +have access to the rule; for example, `//some/package/foo:bar` or +`//other/package/testing:bla` wouldn't have access. `__pkg__` is a special +module and must be used verbatim. It represents all of the modules in the +package. +* `["//project:__subpackages__", "//other:__subpackages__"]`: Only modules in +packages `project` or `other` or in one of their sub-packages have access to +this module. For example, `//project:rule`, `//project/library:lib` or +`//other/testing/internal:munge` are allowed to depend on this rule (but not +`//independent:evil`) +* `["//project"]`: This is shorthand for `["//project:__pkg__"]` +* `[":__subpackages__"]`: This is shorthand for `["//project:__subpackages__"]` +where `//project` is the module's package, e.g. using `[":__subpackages__"]` in +`packages/apps/Settings/Android.bp` is equivalent to +`//packages/apps/Settings:__subpackages__`. +* `["//visibility:legacy_public"]`: The default visibility, behaves as +`//visibility:public` for now. It is an error if it is used in a module. + +The visibility rules of `//visibility:public` and `//visibility:private` cannot +be combined with any other visibility specifications, except +`//visibility:public` is allowed to override visibility specifications imported +through the `defaults` property. + +Packages outside `vendor/` cannot make themselves visible to specific packages +in `vendor/`, e.g. a module in `libcore` cannot declare that it is visible to +say `vendor/google`, instead it must make itself visible to all packages within +`vendor/` using `//vendor:__subpackages__`. + +If a module does not specify the `visibility` property then it uses the +`default_visibility` property of the `package` module in the module's package. + +If the `default_visibility` property is not set for the module's package then +it will use the `default_visibility` of its closest ancestor package for which +a `default_visibility` property is specified. + +If no `default_visibility` property can be found then the module uses the +global default of `//visibility:legacy_public`. + +The `visibility` property has no effect on a defaults module although it does +apply to any non-defaults module that uses it. To set the visibility of a +defaults module, use the `defaults_visibility` property on the defaults module; +not to be confused with the `default_visibility` property on the package module. + +Once the build has been completely switched over to soong it is possible that a +global refactoring will be done to change this to `//visibility:private` at +which point all packages that do not currently specify a `default_visibility` +property will be updated to have +`default_visibility = [//visibility:legacy_public]` added. It will then be the +owner's responsibility to replace that with a more appropriate visibility. ### Formatter -Soong includes a canonical formatter for blueprint files, similar to +Soong includes a canonical formatter for Android.bp files, similar to [gofmt](https://golang.org/cmd/gofmt/). To recursively reformat all Android.bp files in the current directory: ``` @@ -174,32 +379,17 @@ The androidmk converter will produce multiple conflicting modules, which must be resolved by hand to a single module with any differences inside `target: { android: { }, host: { } }` blocks. -## Build logic - -The build logic is written in Go using the -[blueprint](http://godoc.org/github.com/google/blueprint) framework. Build -logic receives module definitions parsed into Go structures using reflection -and produces build rules. The build rules are collected by blueprint and -written to a [ninja](http://ninja-build.org) build file. - -## Other documentation - -* [Best Practices](docs/best_practices.md) -* [Build Performance](docs/perf.md) -* [Generating CLion Projects](docs/clion.md) -* [Generating YouCompleteMe/VSCode compile\_commands.json file](docs/compdb.md) -* Make-specific documentation: [build/make/README.md](https://android.googlesource.com/platform/build/+/master/README.md) - -## FAQ +### Conditionals -### How do I write conditionals? +Soong deliberately does not support most conditionals in Android.bp files. We +suggest removing most conditionals from the build. See +[Best Practices](docs/best_practices.md#removing-conditionals) for some +examples on how to remove conditionals. -Soong deliberately does not support conditionals in Android.bp files. -Instead, complexity in build rules that would require conditionals are handled -in Go, where high level language features can be used and implicit dependencies -introduced by conditionals can be tracked. Most conditionals are converted -to a map property, where one of the values in the map will be selected and -appended to the top level properties. +Most conditionals supported natively by Soong are converted to a map +property. When building the module one of the properties in the map will be +selected, and its values appended to the property with the same name at the +top level of the module. For example, to support architecture specific files: ``` @@ -217,9 +407,112 @@ cc_library { } ``` -See [art/build/art.go](https://android.googlesource.com/platform/art/+/master/build/art.go) -or [external/llvm/soong/llvm.go](https://android.googlesource.com/platform/external/llvm/+/master/soong/llvm.go) -for examples of more complex conditionals on product variables or environment variables. +When building the module for arm the `generic.cpp` and `arm.cpp` sources will +be built. When building for x86 the `generic.cpp` and 'x86.cpp' sources will +be built. + +#### Soong Config Variables + +When converting vendor modules that contain conditionals, simple conditionals +can be supported through Soong config variables using `soong_config_*` +modules that describe the module types, variables and possible values: + +``` +soong_config_module_type { + name: "acme_cc_defaults", + module_type: "cc_defaults", + config_namespace: "acme", + variables: ["board"], + bool_variables: ["feature"], + value_variables: ["width"], + properties: ["cflags", "srcs"], +} + +soong_config_string_variable { + name: "board", + values: ["soc_a", "soc_b"], +} +``` + +This example describes a new `acme_cc_defaults` module type that extends the +`cc_defaults` module type, with three additional conditionals based on +variables `board`, `feature` and `width`, which can affect properties `cflags` +and `srcs`. + +The values of the variables can be set from a product's `BoardConfig.mk` file: +``` +SOONG_CONFIG_NAMESPACES += acme +SOONG_CONFIG_acme += \ + board \ + feature \ + +SOONG_CONFIG_acme_board := soc_a +SOONG_CONFIG_acme_feature := true +SOONG_CONFIG_acme_width := 200 +``` + +The `acme_cc_defaults` module type can be used anywhere after the definition in +the file where it is defined, or can be imported into another file with: +``` +soong_config_module_type_import { + from: "device/acme/Android.bp", + module_types: ["acme_cc_defaults"], +} +``` + +It can used like any other module type: +``` +acme_cc_defaults { + name: "acme_defaults", + cflags: ["-DGENERIC"], + soong_config_variables: { + board: { + soc_a: { + cflags: ["-DSOC_A"], + }, + soc_b: { + cflags: ["-DSOC_B"], + }, + }, + feature: { + cflags: ["-DFEATURE"], + }, + width: { + cflags: ["-DWIDTH=%s"], + }, + }, +} + +cc_library { + name: "libacme_foo", + defaults: ["acme_defaults"], + srcs: ["*.cpp"], +} +``` + +With the `BoardConfig.mk` snippet above, libacme_foo would build with +cflags "-DGENERIC -DSOC_A -DFEATURE -DWIDTH=200". + +`soong_config_module_type` modules will work best when used to wrap defaults +modules (`cc_defaults`, `java_defaults`, etc.), which can then be referenced +by all of the vendor's other modules using the normal namespace and visibility +rules. + +## Build logic + +The build logic is written in Go using the +[blueprint](http://godoc.org/github.com/google/blueprint) framework. Build +logic receives module definitions parsed into Go structures using reflection +and produces build rules. The build rules are collected by blueprint and +written to a [ninja](http://ninja-build.org) build file. + +## Other documentation + +* [Best Practices](docs/best_practices.md) +* [Build Performance](docs/perf.md) +* [Generating CLion Projects](docs/clion.md) +* [Generating YouCompleteMe/VSCode compile\_commands.json file](docs/compdb.md) +* Make-specific documentation: [build/make/README.md](https://android.googlesource.com/platform/build/+/master/README.md) ## Developing for Soong @@ -233,6 +526,31 @@ build/soong/scripts/setup_go_workspace_for_soong.sh This will bind mount the Soong source directories into the directory in the layout expected by the IDE. +### Running Soong in a debugger + +To run the soong_build process in a debugger, install `dlv` and then start the build with +`SOONG_DELVE=<listen addr>` in the environment. +For example: +```bash +SOONG_DELVE=:1234 m nothing +``` +and then in another terminal: +``` +dlv connect :1234 +``` + +If you see an error: +``` +Could not attach to pid 593: this could be caused by a kernel +security setting, try writing "0" to /proc/sys/kernel/yama/ptrace_scope +``` +you can temporarily disable +[Yama's ptrace protection](https://www.kernel.org/doc/Documentation/security/Yama.txt) +using: +```bash +sudo sysctl -w kernel.yama.ptrace_scope=0 +``` + ## Contact Email android-building@googlegroups.com (external) for any questions, or see diff --git a/android/Android.bp b/android/Android.bp new file mode 100644 index 000000000..bd72b7b14 --- /dev/null +++ b/android/Android.bp @@ -0,0 +1,83 @@ +bootstrap_go_package { + name: "soong-android", + pkgPath: "android/soong/android", + deps: [ + "blueprint", + "blueprint-bootstrap", + "soong", + "soong-android-soongconfig", + "soong-env", + "soong-shared", + ], + srcs: [ + "androidmk.go", + "apex.go", + "api_levels.go", + "arch.go", + "config.go", + "csuite_config.go", + "defaults.go", + "defs.go", + "depset.go", + "expand.go", + "filegroup.go", + "hooks.go", + "image.go", + "license.go", + "makevars.go", + "module.go", + "mutator.go", + "namespace.go", + "neverallow.go", + "notices.go", + "onceper.go", + "override_module.go", + "package.go", + "package_ctx.go", + "path_properties.go", + "paths.go", + "phony.go", + "prebuilt.go", + "proto.go", + "register.go", + "rule_builder.go", + "sandbox.go", + "sdk.go", + "singleton.go", + "soong_config_modules.go", + "testing.go", + "util.go", + "variable.go", + "visibility.go", + "vts_config.go", + "writedocs.go", + + // Lock down environment access last + "env.go", + ], + testSrcs: [ + "android_test.go", + "androidmk_test.go", + "arch_test.go", + "config_test.go", + "csuite_config_test.go", + "depset_test.go", + "expand_test.go", + "license_test.go", + "module_test.go", + "mutator_test.go", + "namespace_test.go", + "neverallow_test.go", + "onceper_test.go", + "package_test.go", + "path_properties_test.go", + "paths_test.go", + "prebuilt_test.go", + "rule_builder_test.go", + "soong_config_modules_test.go", + "util_test.go", + "variable_test.go", + "visibility_test.go", + "vts_config_test.go", + ], +} diff --git a/android/android_test.go b/android/android_test.go new file mode 100644 index 000000000..46b705468 --- /dev/null +++ b/android/android_test.go @@ -0,0 +1,46 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "io/ioutil" + "os" + "testing" +) + +var buildDir string + +func setUp() { + var err error + buildDir, err = ioutil.TempDir("", "soong_android_test") + if err != nil { + panic(err) + } +} + +func tearDown() { + os.RemoveAll(buildDir) +} + +func TestMain(m *testing.M) { + run := func() int { + setUp() + defer tearDown() + + return m.Run() + } + + os.Exit(run()) +} diff --git a/android/androidmk.go b/android/androidmk.go index bd49e4c6f..54a0c6465 100644 --- a/android/androidmk.go +++ b/android/androidmk.go @@ -29,22 +29,30 @@ import ( ) func init() { - RegisterSingletonType("androidmk", AndroidMkSingleton) + RegisterAndroidMkBuildComponents(InitRegistrationContext) } +func RegisterAndroidMkBuildComponents(ctx RegistrationContext) { + ctx.RegisterSingletonType("androidmk", AndroidMkSingleton) +} + +// Deprecated: consider using AndroidMkEntriesProvider instead, especially if you're not going to +// use the Custom function. type AndroidMkDataProvider interface { AndroidMk() AndroidMkData BaseModuleName() string } type AndroidMkData struct { - Class string - SubName string - DistFile OptionalPath - OutputFile OptionalPath - Disabled bool - Include string - Required []string + Class string + SubName string + DistFile OptionalPath + OutputFile OptionalPath + Disabled bool + Include string + Required []string + Host_required []string + Target_required []string Custom func(w io.Writer, name, prefix, moduleDir string, data AndroidMkData) @@ -55,6 +63,287 @@ type AndroidMkData struct { type AndroidMkExtraFunc func(w io.Writer, outputFile Path) +// Allows modules to customize their Android*.mk output. +type AndroidMkEntriesProvider interface { + AndroidMkEntries() []AndroidMkEntries + BaseModuleName() string +} + +type AndroidMkEntries struct { + Class string + SubName string + DistFile OptionalPath + OutputFile OptionalPath + Disabled bool + Include string + Required []string + Host_required []string + Target_required []string + + header bytes.Buffer + footer bytes.Buffer + + ExtraEntries []AndroidMkExtraEntriesFunc + ExtraFooters []AndroidMkExtraFootersFunc + + EntryMap map[string][]string + entryOrder []string +} + +type AndroidMkExtraEntriesFunc func(entries *AndroidMkEntries) +type AndroidMkExtraFootersFunc func(w io.Writer, name, prefix, moduleDir string, entries *AndroidMkEntries) + +func (a *AndroidMkEntries) SetString(name, value string) { + if _, ok := a.EntryMap[name]; !ok { + a.entryOrder = append(a.entryOrder, name) + } + a.EntryMap[name] = []string{value} +} + +func (a *AndroidMkEntries) SetPath(name string, path Path) { + if _, ok := a.EntryMap[name]; !ok { + a.entryOrder = append(a.entryOrder, name) + } + a.EntryMap[name] = []string{path.String()} +} + +func (a *AndroidMkEntries) SetOptionalPath(name string, path OptionalPath) { + if path.Valid() { + a.SetPath(name, path.Path()) + } +} + +func (a *AndroidMkEntries) AddPath(name string, path Path) { + if _, ok := a.EntryMap[name]; !ok { + a.entryOrder = append(a.entryOrder, name) + } + a.EntryMap[name] = append(a.EntryMap[name], path.String()) +} + +func (a *AndroidMkEntries) AddOptionalPath(name string, path OptionalPath) { + if path.Valid() { + a.AddPath(name, path.Path()) + } +} + +func (a *AndroidMkEntries) SetPaths(name string, paths Paths) { + if _, ok := a.EntryMap[name]; !ok { + a.entryOrder = append(a.entryOrder, name) + } + a.EntryMap[name] = paths.Strings() +} + +func (a *AndroidMkEntries) SetOptionalPaths(name string, paths Paths) { + if len(paths) > 0 { + a.SetPaths(name, paths) + } +} + +func (a *AndroidMkEntries) AddPaths(name string, paths Paths) { + if _, ok := a.EntryMap[name]; !ok { + a.entryOrder = append(a.entryOrder, name) + } + a.EntryMap[name] = append(a.EntryMap[name], paths.Strings()...) +} + +func (a *AndroidMkEntries) SetBoolIfTrue(name string, flag bool) { + if flag { + if _, ok := a.EntryMap[name]; !ok { + a.entryOrder = append(a.entryOrder, name) + } + a.EntryMap[name] = []string{"true"} + } +} + +func (a *AndroidMkEntries) SetBool(name string, flag bool) { + if _, ok := a.EntryMap[name]; !ok { + a.entryOrder = append(a.entryOrder, name) + } + if flag { + a.EntryMap[name] = []string{"true"} + } else { + a.EntryMap[name] = []string{"false"} + } +} + +func (a *AndroidMkEntries) AddStrings(name string, value ...string) { + if len(value) == 0 { + return + } + if _, ok := a.EntryMap[name]; !ok { + a.entryOrder = append(a.entryOrder, name) + } + a.EntryMap[name] = append(a.EntryMap[name], value...) +} + +func (a *AndroidMkEntries) fillInEntries(config Config, bpPath string, mod blueprint.Module) { + a.EntryMap = make(map[string][]string) + amod := mod.(Module).base() + name := amod.BaseModuleName() + + if a.Include == "" { + a.Include = "$(BUILD_PREBUILT)" + } + a.Required = append(a.Required, amod.commonProperties.Required...) + a.Host_required = append(a.Host_required, amod.commonProperties.Host_required...) + a.Target_required = append(a.Target_required, amod.commonProperties.Target_required...) + + // Fill in the header part. + if len(amod.commonProperties.Dist.Targets) > 0 { + distFile := a.DistFile + if !distFile.Valid() { + distFile = a.OutputFile + } + if distFile.Valid() { + dest := filepath.Base(distFile.String()) + + if amod.commonProperties.Dist.Dest != nil { + var err error + if dest, err = validateSafePath(*amod.commonProperties.Dist.Dest); err != nil { + // This was checked in ModuleBase.GenerateBuildActions + panic(err) + } + } + + if amod.commonProperties.Dist.Suffix != nil { + ext := filepath.Ext(dest) + suffix := *amod.commonProperties.Dist.Suffix + dest = strings.TrimSuffix(dest, ext) + suffix + ext + } + + if amod.commonProperties.Dist.Dir != nil { + var err error + if dest, err = validateSafePath(*amod.commonProperties.Dist.Dir, dest); err != nil { + // This was checked in ModuleBase.GenerateBuildActions + panic(err) + } + } + + goals := strings.Join(amod.commonProperties.Dist.Targets, " ") + fmt.Fprintln(&a.header, ".PHONY:", goals) + fmt.Fprintf(&a.header, "$(call dist-for-goals,%s,%s:%s)\n", + goals, distFile.String(), dest) + } + } + + fmt.Fprintln(&a.header, "\ninclude $(CLEAR_VARS)") + + // Collect make variable assignment entries. + a.SetString("LOCAL_PATH", filepath.Dir(bpPath)) + a.SetString("LOCAL_MODULE", name+a.SubName) + a.SetString("LOCAL_MODULE_CLASS", a.Class) + a.SetString("LOCAL_PREBUILT_MODULE_FILE", a.OutputFile.String()) + a.AddStrings("LOCAL_REQUIRED_MODULES", a.Required...) + a.AddStrings("LOCAL_HOST_REQUIRED_MODULES", a.Host_required...) + a.AddStrings("LOCAL_TARGET_REQUIRED_MODULES", a.Target_required...) + + if am, ok := mod.(ApexModule); ok { + a.SetBoolIfTrue("LOCAL_NOT_AVAILABLE_FOR_PLATFORM", am.NotAvailableForPlatform()) + } + + archStr := amod.Arch().ArchType.String() + host := false + switch amod.Os().Class { + case Host: + // Make cannot identify LOCAL_MODULE_HOST_ARCH:= common. + if amod.Arch().ArchType != Common { + a.SetString("LOCAL_MODULE_HOST_ARCH", archStr) + } + host = true + case HostCross: + // Make cannot identify LOCAL_MODULE_HOST_CROSS_ARCH:= common. + if amod.Arch().ArchType != Common { + a.SetString("LOCAL_MODULE_HOST_CROSS_ARCH", archStr) + } + host = true + case Device: + // Make cannot identify LOCAL_MODULE_TARGET_ARCH:= common. + if amod.Arch().ArchType != Common { + if amod.Target().NativeBridge { + hostArchStr := amod.Target().NativeBridgeHostArchName + if hostArchStr != "" { + a.SetString("LOCAL_MODULE_TARGET_ARCH", hostArchStr) + } + } else { + a.SetString("LOCAL_MODULE_TARGET_ARCH", archStr) + } + } + + a.AddStrings("LOCAL_INIT_RC", amod.commonProperties.Init_rc...) + a.AddStrings("LOCAL_VINTF_FRAGMENTS", amod.commonProperties.Vintf_fragments...) + a.SetBoolIfTrue("LOCAL_PROPRIETARY_MODULE", Bool(amod.commonProperties.Proprietary)) + if Bool(amod.commonProperties.Vendor) || Bool(amod.commonProperties.Soc_specific) { + a.SetString("LOCAL_VENDOR_MODULE", "true") + } + a.SetBoolIfTrue("LOCAL_ODM_MODULE", Bool(amod.commonProperties.Device_specific)) + a.SetBoolIfTrue("LOCAL_PRODUCT_MODULE", Bool(amod.commonProperties.Product_specific)) + a.SetBoolIfTrue("LOCAL_SYSTEM_EXT_MODULE", Bool(amod.commonProperties.System_ext_specific)) + if amod.commonProperties.Owner != nil { + a.SetString("LOCAL_MODULE_OWNER", *amod.commonProperties.Owner) + } + } + + if amod.noticeFile.Valid() { + a.SetString("LOCAL_NOTICE_FILE", amod.noticeFile.String()) + } + + if host { + makeOs := amod.Os().String() + if amod.Os() == Linux || amod.Os() == LinuxBionic { + makeOs = "linux" + } + a.SetString("LOCAL_MODULE_HOST_OS", makeOs) + a.SetString("LOCAL_IS_HOST_MODULE", "true") + } + + prefix := "" + if amod.ArchSpecific() { + switch amod.Os().Class { + case Host: + prefix = "HOST_" + case HostCross: + prefix = "HOST_CROSS_" + case Device: + prefix = "TARGET_" + + } + + if amod.Arch().ArchType != config.Targets[amod.Os()][0].Arch.ArchType { + prefix = "2ND_" + prefix + } + } + for _, extra := range a.ExtraEntries { + extra(a) + } + + // Write to footer. + fmt.Fprintln(&a.footer, "include "+a.Include) + blueprintDir := filepath.Dir(bpPath) + for _, footerFunc := range a.ExtraFooters { + footerFunc(&a.footer, name, prefix, blueprintDir, a) + } +} + +func (a *AndroidMkEntries) write(w io.Writer) { + if a.Disabled { + return + } + + if !a.OutputFile.Valid() { + return + } + + w.Write(a.header.Bytes()) + for _, name := range a.entryOrder { + fmt.Fprintln(w, name+" := "+strings.Join(a.EntryMap[name], " ")) + } + w.Write(a.footer.Bytes()) +} + +func (a *AndroidMkEntries) FooterLinesForTests() []string { + return strings.Split(string(a.footer.Bytes()), "\n") +} + func AndroidMkSingleton() Singleton { return &androidMkSingleton{} } @@ -81,7 +370,7 @@ func (c *androidMkSingleton) GenerateBuildActions(ctx SingletonContext) { return } - err := translateAndroidMk(ctx, transMk.String(), androidMkModulesList) + err := translateAndroidMk(ctx, absolutePath(transMk.String()), androidMkModulesList) if err != nil { ctx.Errorf(err.Error()) } @@ -122,8 +411,8 @@ func translateAndroidMk(ctx SingletonContext, mkFile string, mods []blueprint.Mo } // Don't write to the file if it hasn't changed - if _, err := os.Stat(mkFile); !os.IsNotExist(err) { - if data, err := ioutil.ReadFile(mkFile); err == nil { + if _, err := os.Stat(absolutePath(mkFile)); !os.IsNotExist(err) { + if data, err := ioutil.ReadFile(absolutePath(mkFile)); err == nil { matches := buf.Len() == len(data) if matches { @@ -141,7 +430,7 @@ func translateAndroidMk(ctx SingletonContext, mkFile string, mods []blueprint.Mo } } - return ioutil.WriteFile(mkFile, buf.Bytes(), 0666) + return ioutil.WriteFile(absolutePath(mkFile), buf.Bytes(), 0666) } func translateAndroidMkModule(ctx SingletonContext, w io.Writer, mod blueprint.Module) error { @@ -157,6 +446,8 @@ func translateAndroidMkModule(ctx SingletonContext, w io.Writer, mod blueprint.M return translateAndroidModule(ctx, w, mod, x) case bootstrap.GoBinaryTool: return translateGoBinaryModule(ctx, w, mod, x) + case AndroidMkEntriesProvider: + return translateAndroidMkEntriesModule(ctx, w, mod, x) default: return nil } @@ -173,38 +464,45 @@ func translateGoBinaryModule(ctx SingletonContext, w io.Writer, mod blueprint.Mo return nil } -func translateAndroidModule(ctx SingletonContext, w io.Writer, mod blueprint.Module, - provider AndroidMkDataProvider) error { +func (data *AndroidMkData) fillInData(config Config, bpPath string, mod blueprint.Module) { + // Get the preamble content through AndroidMkEntries logic. + entries := AndroidMkEntries{ + Class: data.Class, + SubName: data.SubName, + DistFile: data.DistFile, + OutputFile: data.OutputFile, + Disabled: data.Disabled, + Include: data.Include, + Required: data.Required, + Host_required: data.Host_required, + Target_required: data.Target_required, + } + entries.fillInEntries(config, bpPath, mod) - name := provider.BaseModuleName() - amod := mod.(Module).base() + // preamble doesn't need the footer content. + entries.footer = bytes.Buffer{} + entries.write(&data.preamble) - if !amod.Enabled() { - return nil - } + // copy entries back to data since it is used in Custom + data.Required = entries.Required + data.Host_required = entries.Host_required + data.Target_required = entries.Target_required +} - if amod.commonProperties.SkipInstall { - return nil - } +func translateAndroidModule(ctx SingletonContext, w io.Writer, mod blueprint.Module, + provider AndroidMkDataProvider) error { - if !amod.commonProperties.NamespaceExportedToMake { - // TODO(jeffrygaston) do we want to validate that there are no modules being - // exported to Kati that depend on this module? + amod := mod.(Module).base() + if shouldSkipAndroidMkProcessing(amod) { return nil } data := provider.AndroidMk() - if data.Include == "" { data.Include = "$(BUILD_PREBUILT)" } - data.Required = append(data.Required, amod.commonProperties.Required...) - - // Make does not understand LinuxBionic - if amod.Os() == LinuxBionic { - return nil - } + data.fillInData(ctx.Config(), ctx.BlueprintFile(mod), mod) prefix := "" if amod.ArchSpecific() { @@ -223,115 +521,7 @@ func translateAndroidModule(ctx SingletonContext, w io.Writer, mod blueprint.Mod } } - if len(amod.commonProperties.Dist.Targets) > 0 { - distFile := data.DistFile - if !distFile.Valid() { - distFile = data.OutputFile - } - if distFile.Valid() { - dest := filepath.Base(distFile.String()) - - if amod.commonProperties.Dist.Dest != nil { - var err error - dest, err = validateSafePath(*amod.commonProperties.Dist.Dest) - if err != nil { - // This was checked in ModuleBase.GenerateBuildActions - panic(err) - } - } - - if amod.commonProperties.Dist.Suffix != nil { - ext := filepath.Ext(dest) - suffix := *amod.commonProperties.Dist.Suffix - dest = strings.TrimSuffix(dest, ext) + suffix + ext - } - - if amod.commonProperties.Dist.Dir != nil { - var err error - dest, err = validateSafePath(*amod.commonProperties.Dist.Dir, dest) - if err != nil { - // This was checked in ModuleBase.GenerateBuildActions - panic(err) - } - } - - goals := strings.Join(amod.commonProperties.Dist.Targets, " ") - fmt.Fprintln(&data.preamble, ".PHONY:", goals) - fmt.Fprintf(&data.preamble, "$(call dist-for-goals,%s,%s:%s)\n", - goals, distFile.String(), dest) - } - } - - fmt.Fprintln(&data.preamble, "\ninclude $(CLEAR_VARS)") - fmt.Fprintln(&data.preamble, "LOCAL_PATH :=", filepath.Dir(ctx.BlueprintFile(mod))) - fmt.Fprintln(&data.preamble, "LOCAL_MODULE :=", name+data.SubName) - fmt.Fprintln(&data.preamble, "LOCAL_MODULE_CLASS :=", data.Class) - fmt.Fprintln(&data.preamble, "LOCAL_PREBUILT_MODULE_FILE :=", data.OutputFile.String()) - - if len(data.Required) > 0 { - fmt.Fprintln(&data.preamble, "LOCAL_REQUIRED_MODULES := "+strings.Join(data.Required, " ")) - } - - archStr := amod.Arch().ArchType.String() - host := false - switch amod.Os().Class { - case Host: - // Make cannot identify LOCAL_MODULE_HOST_ARCH:= common. - if archStr != "common" { - fmt.Fprintln(&data.preamble, "LOCAL_MODULE_HOST_ARCH :=", archStr) - } - host = true - case HostCross: - // Make cannot identify LOCAL_MODULE_HOST_CROSS_ARCH:= common. - if archStr != "common" { - fmt.Fprintln(&data.preamble, "LOCAL_MODULE_HOST_CROSS_ARCH :=", archStr) - } - host = true - case Device: - // Make cannot identify LOCAL_MODULE_TARGET_ARCH:= common. - if archStr != "common" { - fmt.Fprintln(&data.preamble, "LOCAL_MODULE_TARGET_ARCH :=", archStr) - } - - if len(amod.commonProperties.Init_rc) > 0 { - fmt.Fprintln(&data.preamble, "LOCAL_INIT_RC := ", strings.Join(amod.commonProperties.Init_rc, " ")) - } - if len(amod.commonProperties.Vintf_fragments) > 0 { - fmt.Fprintln(&data.preamble, "LOCAL_VINTF_FRAGMENTS := ", strings.Join(amod.commonProperties.Vintf_fragments, " ")) - } - if Bool(amod.commonProperties.Proprietary) { - fmt.Fprintln(&data.preamble, "LOCAL_PROPRIETARY_MODULE := true") - } - if Bool(amod.commonProperties.Vendor) || Bool(amod.commonProperties.Soc_specific) { - fmt.Fprintln(&data.preamble, "LOCAL_VENDOR_MODULE := true") - } - if Bool(amod.commonProperties.Device_specific) { - fmt.Fprintln(&data.preamble, "LOCAL_ODM_MODULE := true") - } - if Bool(amod.commonProperties.Product_specific) { - fmt.Fprintln(&data.preamble, "LOCAL_PRODUCT_MODULE := true") - } - if Bool(amod.commonProperties.Product_services_specific) { - fmt.Fprintln(&data.preamble, "LOCAL_PRODUCT_SERVICES_MODULE := true") - } - if amod.commonProperties.Owner != nil { - fmt.Fprintln(&data.preamble, "LOCAL_MODULE_OWNER :=", *amod.commonProperties.Owner) - } - } - - if amod.noticeFile.Valid() { - fmt.Fprintln(&data.preamble, "LOCAL_NOTICE_FILE :=", amod.noticeFile.String()) - } - - if host { - makeOs := amod.Os().String() - if amod.Os() == Linux || amod.Os() == LinuxBionic { - makeOs = "linux" - } - fmt.Fprintln(&data.preamble, "LOCAL_MODULE_HOST_OS :=", makeOs) - fmt.Fprintln(&data.preamble, "LOCAL_IS_HOST_MODULE := true") - } - + name := provider.BaseModuleName() blueprintDir := filepath.Dir(ctx.BlueprintFile(mod)) if data.Custom != nil { @@ -360,3 +550,30 @@ func WriteAndroidMkData(w io.Writer, data AndroidMkData) { fmt.Fprintln(w, "include "+data.Include) } + +func translateAndroidMkEntriesModule(ctx SingletonContext, w io.Writer, mod blueprint.Module, + provider AndroidMkEntriesProvider) error { + if shouldSkipAndroidMkProcessing(mod.(Module).base()) { + return nil + } + + for _, entries := range provider.AndroidMkEntries() { + entries.fillInEntries(ctx.Config(), ctx.BlueprintFile(mod), mod) + entries.write(w) + } + + return nil +} + +func shouldSkipAndroidMkProcessing(module *ModuleBase) bool { + if !module.commonProperties.NamespaceExportedToMake { + // TODO(jeffrygaston) do we want to validate that there are no modules being + // exported to Kati that depend on this module? + return true + } + + return !module.Enabled() || + module.commonProperties.SkipInstall || + // Make does not understand LinuxBionic + module.Os() == LinuxBionic +} diff --git a/android/androidmk_test.go b/android/androidmk_test.go new file mode 100644 index 000000000..71f802043 --- /dev/null +++ b/android/androidmk_test.go @@ -0,0 +1,78 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "io" + "reflect" + "testing" +) + +type customModule struct { + ModuleBase + data AndroidMkData +} + +func (m *customModule) GenerateAndroidBuildActions(ctx ModuleContext) { +} + +func (m *customModule) AndroidMk() AndroidMkData { + return AndroidMkData{ + Custom: func(w io.Writer, name, prefix, moduleDir string, data AndroidMkData) { + m.data = data + }, + } +} + +func customModuleFactory() Module { + module := &customModule{} + InitAndroidModule(module) + return module +} + +func TestAndroidMkSingleton_PassesUpdatedAndroidMkDataToCustomCallback(t *testing.T) { + bp := ` + custom { + name: "foo", + required: ["bar"], + host_required: ["baz"], + target_required: ["qux"], + } + ` + + config := TestConfig(buildDir, nil, bp, nil) + config.inMake = true // Enable androidmk Singleton + + ctx := NewTestContext() + ctx.RegisterSingletonType("androidmk", AndroidMkSingleton) + ctx.RegisterModuleType("custom", customModuleFactory) + ctx.Register(config) + + _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) + FailIfErrored(t, errs) + _, errs = ctx.PrepareBuildActions(config) + FailIfErrored(t, errs) + + m := ctx.ModuleForTests("foo", "").Module().(*customModule) + + assertEqual := func(expected interface{}, actual interface{}) { + if !reflect.DeepEqual(expected, actual) { + t.Errorf("%q expected, but got %q", expected, actual) + } + } + assertEqual([]string{"bar"}, m.data.Required) + assertEqual([]string{"baz"}, m.data.Host_required) + assertEqual([]string{"qux"}, m.data.Target_required) +} diff --git a/android/apex.go b/android/apex.go index 17df76240..30152db29 100644 --- a/android/apex.go +++ b/android/apex.go @@ -15,12 +15,35 @@ package android import ( + "fmt" "sort" + "strconv" + "strings" "sync" "github.com/google/blueprint" ) +const ( + SdkVersion_Android10 = 29 +) + +type ApexInfo struct { + // Name of the apex variant that this module is mutated into + ApexName string + + MinSdkVersion int + Updatable bool +} + +// Extracted from ApexModule to make it easier to define custom subsets of the +// ApexModule interface and improve code navigation within the IDE. +type DepIsInSameApex interface { + // DepIsInSameApex tests if the other module 'dep' is installed to the same + // APEX as this module + DepIsInSameApex(ctx BaseModuleContext, dep Module) bool +} + // ApexModule is the interface that a module type is expected to implement if // the module has to be built differently depending on whether the module // is destined for an apex or not (installed to one of the regular partitions). @@ -38,11 +61,16 @@ import ( // respectively. type ApexModule interface { Module + DepIsInSameApex + apexModuleBase() *ApexModuleBase - // Marks that this module should be built for the APEX of the specified name. + // Marks that this module should be built for the specified APEXes. // Call this before apex.apexMutator is run. - BuildForApex(apexName string) + BuildForApexes(apexes []ApexInfo) + + // Returns the APEXes that this module will be built for + ApexVariations() []ApexInfo // Returns the name of APEX that this module will be built for. Empty string // is returned when 'IsForPlatform() == true'. Note that a module can be @@ -68,17 +96,58 @@ type ApexModule interface { IsInstallableToApex() bool // Mutate this module into one or more variants each of which is built - // for an APEX marked via BuildForApex(). - CreateApexVariations(mctx BottomUpMutatorContext) []blueprint.Module - - // Sets the name of the apex variant of this module. Called inside - // CreateApexVariations. - setApexName(apexName string) + // for an APEX marked via BuildForApexes(). + CreateApexVariations(mctx BottomUpMutatorContext) []Module + + // Tests if this module is available for the specified APEX or ":platform" + AvailableFor(what string) bool + + // Return true if this module is not available to platform (i.e. apex_available + // property doesn't have "//apex_available:platform"), or shouldn't be available + // to platform, which is the case when this module depends on other module that + // isn't available to platform. + NotAvailableForPlatform() bool + + // Mark that this module is not available to platform. Set by the + // check-platform-availability mutator in the apex package. + SetNotAvailableForPlatform() + + // Returns the highest version which is <= maxSdkVersion. + // For example, with maxSdkVersion is 10 and versionList is [9,11] + // it returns 9 as string + ChooseSdkVersion(versionList []string, maxSdkVersion int) (string, error) + + // Tests if the module comes from an updatable APEX. + Updatable() bool + + // List of APEXes that this module tests. The module has access to + // the private part of the listed APEXes even when it is not included in the + // APEXes. + TestFor() []string } type ApexProperties struct { - // Name of the apex variant that this module is mutated into - ApexName string `blueprint:"mutated"` + // Availability of this module in APEXes. Only the listed APEXes can contain + // this module. If the module has stubs then other APEXes and the platform may + // access it through them (subject to visibility). + // + // "//apex_available:anyapex" is a pseudo APEX name that matches to any APEX. + // "//apex_available:platform" refers to non-APEX partitions like "system.img". + // Default is ["//apex_available:platform"]. + Apex_available []string + + Info ApexInfo `blueprint:"mutated"` + + NotAvailableForPlatform bool `blueprint:"mutated"` +} + +// Marker interface that identifies dependencies that are excluded from APEX +// contents. +type ExcludeFromApexContentsTag interface { + blueprint.DependencyTag + + // Method that differentiates this interface from others. + ExcludeFromApexContents() } // Provides default implementation for the ApexModule interface. APEX-aware @@ -89,31 +158,46 @@ type ApexModuleBase struct { canHaveApexVariants bool apexVariationsLock sync.Mutex // protects apexVariations during parallel apexDepsMutator - apexVariations []string + apexVariations []ApexInfo } func (m *ApexModuleBase) apexModuleBase() *ApexModuleBase { return m } -func (m *ApexModuleBase) BuildForApex(apexName string) { +func (m *ApexModuleBase) ApexAvailable() []string { + return m.ApexProperties.Apex_available +} + +func (m *ApexModuleBase) TestFor() []string { + // To be implemented by concrete types inheriting ApexModuleBase + return nil +} + +func (m *ApexModuleBase) BuildForApexes(apexes []ApexInfo) { m.apexVariationsLock.Lock() defer m.apexVariationsLock.Unlock() - if !InList(apexName, m.apexVariations) { - m.apexVariations = append(m.apexVariations, apexName) +nextApex: + for _, apex := range apexes { + for _, v := range m.apexVariations { + if v.ApexName == apex.ApexName { + continue nextApex + } + } + m.apexVariations = append(m.apexVariations, apex) } } -func (m *ApexModuleBase) ApexName() string { - return m.ApexProperties.ApexName +func (m *ApexModuleBase) ApexVariations() []ApexInfo { + return m.apexVariations } -func (m *ApexModuleBase) IsForPlatform() bool { - return m.ApexProperties.ApexName == "" +func (m *ApexModuleBase) ApexName() string { + return m.ApexProperties.Info.ApexName } -func (m *ApexModuleBase) setApexName(apexName string) { - m.ApexProperties.ApexName = apexName +func (m *ApexModuleBase) IsForPlatform() bool { + return m.ApexProperties.Info.ApexName == "" } func (m *ApexModuleBase) CanHaveApexVariants() bool { @@ -125,18 +209,94 @@ func (m *ApexModuleBase) IsInstallableToApex() bool { return false } -func (m *ApexModuleBase) CreateApexVariations(mctx BottomUpMutatorContext) []blueprint.Module { +const ( + AvailableToPlatform = "//apex_available:platform" + AvailableToAnyApex = "//apex_available:anyapex" +) + +func CheckAvailableForApex(what string, apex_available []string) bool { + if len(apex_available) == 0 { + // apex_available defaults to ["//apex_available:platform"], + // which means 'available to the platform but no apexes'. + return what == AvailableToPlatform + } + return InList(what, apex_available) || + (what != AvailableToPlatform && InList(AvailableToAnyApex, apex_available)) +} + +func (m *ApexModuleBase) AvailableFor(what string) bool { + return CheckAvailableForApex(what, m.ApexProperties.Apex_available) +} + +func (m *ApexModuleBase) NotAvailableForPlatform() bool { + return m.ApexProperties.NotAvailableForPlatform +} + +func (m *ApexModuleBase) SetNotAvailableForPlatform() { + m.ApexProperties.NotAvailableForPlatform = true +} + +func (m *ApexModuleBase) DepIsInSameApex(ctx BaseModuleContext, dep Module) bool { + // By default, if there is a dependency from A to B, we try to include both in the same APEX, + // unless B is explicitly from outside of the APEX (i.e. a stubs lib). Thus, returning true. + // This is overridden by some module types like apex.ApexBundle, cc.Module, java.Module, etc. + return true +} + +func (m *ApexModuleBase) ChooseSdkVersion(versionList []string, maxSdkVersion int) (string, error) { + for i := range versionList { + ver, _ := strconv.Atoi(versionList[len(versionList)-i-1]) + if ver <= maxSdkVersion { + return versionList[len(versionList)-i-1], nil + } + } + return "", fmt.Errorf("not found a version(<=%d) in versionList: %v", maxSdkVersion, versionList) +} + +func (m *ApexModuleBase) checkApexAvailableProperty(mctx BaseModuleContext) { + for _, n := range m.ApexProperties.Apex_available { + if n == AvailableToPlatform || n == AvailableToAnyApex { + continue + } + if !mctx.OtherModuleExists(n) && !mctx.Config().AllowMissingDependencies() { + mctx.PropertyErrorf("apex_available", "%q is not a valid module name", n) + } + } +} + +func (m *ApexModuleBase) Updatable() bool { + return m.ApexProperties.Info.Updatable +} + +type byApexName []ApexInfo + +func (a byApexName) Len() int { return len(a) } +func (a byApexName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a byApexName) Less(i, j int) bool { return a[i].ApexName < a[j].ApexName } + +func (m *ApexModuleBase) CreateApexVariations(mctx BottomUpMutatorContext) []Module { if len(m.apexVariations) > 0 { - sort.Strings(m.apexVariations) - variations := []string{""} // Original variation for platform - variations = append(variations, m.apexVariations...) + m.checkApexAvailableProperty(mctx) + + sort.Sort(byApexName(m.apexVariations)) + variations := []string{} + variations = append(variations, "") // Original variation for platform + for _, apex := range m.apexVariations { + variations = append(variations, apex.ApexName) + } + + defaultVariation := "" + mctx.SetDefaultDependencyVariation(&defaultVariation) modules := mctx.CreateVariations(variations...) - for i, m := range modules { - if i == 0 { - continue + for i, mod := range modules { + platformVariation := i == 0 + if platformVariation && !mctx.Host() && !mod.(ApexModule).AvailableFor(AvailableToPlatform) { + mod.SkipInstall() + } + if !platformVariation { + mod.(ApexModule).apexModuleBase().ApexProperties.Info = m.apexVariations[i-1] } - m.(ApexModule).setApexName(variations[i]) } return modules } @@ -160,18 +320,28 @@ func apexNamesMap() map[string]map[string]bool { } // Update the map to mark that a module named moduleName is directly or indirectly -// depended on by an APEX named apexName. Directly depending means that a module +// depended on by the specified APEXes. Directly depending means that a module // is explicitly listed in the build definition of the APEX via properties like // native_shared_libs, java_libs, etc. -func UpdateApexDependency(apexName string, moduleName string, directDep bool) { +func UpdateApexDependency(apexes []ApexInfo, moduleName string, directDep bool) { apexNamesMapMutex.Lock() defer apexNamesMapMutex.Unlock() - apexNames, ok := apexNamesMap()[moduleName] - if !ok { - apexNames = make(map[string]bool) - apexNamesMap()[moduleName] = apexNames + for _, apex := range apexes { + apexesForModule, ok := apexNamesMap()[moduleName] + if !ok { + apexesForModule = make(map[string]bool) + apexNamesMap()[moduleName] = apexesForModule + } + apexesForModule[apex.ApexName] = apexesForModule[apex.ApexName] || directDep + } +} + +// TODO(b/146393795): remove this when b/146393795 is fixed +func ClearApexDependency() { + m := apexNamesMap() + for k := range m { + delete(m, k) } - apexNames[apexName] = apexNames[apexName] || directDep } // Tests whether a module named moduleName is directly depended on by an APEX @@ -234,3 +404,76 @@ func InitApexModule(m ApexModule) { m.AddProperties(&base.ApexProperties) } + +// A dependency info for a single ApexModule, either direct or transitive. +type ApexModuleDepInfo struct { + // Name of the dependency + To string + // List of dependencies To belongs to. Includes APEX itself, if a direct dependency. + From []string + // Whether the dependency belongs to the final compiled APEX. + IsExternal bool + // min_sdk_version of the ApexModule + MinSdkVersion string +} + +// A map of a dependency name to its ApexModuleDepInfo +type DepNameToDepInfoMap map[string]ApexModuleDepInfo + +type ApexBundleDepsInfo struct { + flatListPath OutputPath + fullListPath OutputPath +} + +type ApexBundleDepsInfoIntf interface { + Updatable() bool + FlatListPath() Path + FullListPath() Path +} + +func (d *ApexBundleDepsInfo) FlatListPath() Path { + return d.flatListPath +} + +func (d *ApexBundleDepsInfo) FullListPath() Path { + return d.fullListPath +} + +// Generate two module out files: +// 1. FullList with transitive deps and their parents in the dep graph +// 2. FlatList with a flat list of transitive deps +func (d *ApexBundleDepsInfo) BuildDepsInfoLists(ctx ModuleContext, minSdkVersion string, depInfos DepNameToDepInfoMap) { + var fullContent strings.Builder + var flatContent strings.Builder + + fmt.Fprintf(&flatContent, "%s(minSdkVersion:%s):\\n", ctx.ModuleName(), minSdkVersion) + for _, key := range FirstUniqueStrings(SortedStringKeys(depInfos)) { + info := depInfos[key] + toName := fmt.Sprintf("%s(minSdkVersion:%s)", info.To, info.MinSdkVersion) + if info.IsExternal { + toName = toName + " (external)" + } + fmt.Fprintf(&fullContent, "%s <- %s\\n", toName, strings.Join(SortedUniqueStrings(info.From), ", ")) + fmt.Fprintf(&flatContent, " %s\\n", toName) + } + + d.fullListPath = PathForModuleOut(ctx, "depsinfo", "fulllist.txt").OutputPath + ctx.Build(pctx, BuildParams{ + Rule: WriteFile, + Description: "Full Dependency Info", + Output: d.fullListPath, + Args: map[string]string{ + "content": fullContent.String(), + }, + }) + + d.flatListPath = PathForModuleOut(ctx, "depsinfo", "flatlist.txt").OutputPath + ctx.Build(pctx, BuildParams{ + Rule: WriteFile, + Description: "Flat Dependency Info", + Output: d.flatListPath, + Args: map[string]string{ + "content": flatContent.String(), + }, + }) +} diff --git a/android/api_levels.go b/android/api_levels.go index 2f70f6200..087206633 100644 --- a/android/api_levels.go +++ b/android/api_levels.go @@ -16,6 +16,7 @@ package android import ( "encoding/json" + "fmt" "strconv" ) @@ -72,8 +73,9 @@ func getApiLevelsMap(config Config) map[string]int { "O-MR1": 27, "P": 28, "Q": 29, + "R": 30, } - for i, codename := range config.PlatformVersionCombinedCodenames() { + for i, codename := range config.PlatformVersionActiveCodenames() { apiLevelsMap[codename] = baseApiLevel + i } @@ -84,14 +86,19 @@ func getApiLevelsMap(config Config) map[string]int { // Converts an API level string into its numeric form. // * Codenames are decoded. // * Numeric API levels are simply converted. -// * "minimum" and "current" are not currently handled since the former is -// NDK specific and the latter has inconsistent meaning. -func ApiStrToNum(ctx BaseContext, apiLevel string) (int, error) { - num, ok := getApiLevelsMap(ctx.Config())[apiLevel] - if ok { +// * "current" is mapped to FutureApiLevel(10000) +// * "minimum" is NDK specific and not handled with this. (refer normalizeNdkApiLevel in cc.go) +func ApiStrToNum(ctx BaseModuleContext, apiLevel string) (int, error) { + if apiLevel == "current" { + return FutureApiLevel, nil + } + if num, ok := getApiLevelsMap(ctx.Config())[apiLevel]; ok { + return num, nil + } + if num, err := strconv.Atoi(apiLevel); err == nil { return num, nil } - return strconv.Atoi(apiLevel) + return 0, fmt.Errorf("SDK version should be one of \"current\", <number> or <codename>: %q", apiLevel) } func (a *apiLevelsSingleton) GenerateBuildActions(ctx SingletonContext) { diff --git a/android/arch.go b/android/arch.go index 957a659c6..18ff13e9c 100644 --- a/android/arch.go +++ b/android/arch.go @@ -22,9 +22,12 @@ import ( "strconv" "strings" + "github.com/google/blueprint" "github.com/google/blueprint/proptools" ) +const COMMON_VARIANT = "common" + var ( archTypeList []ArchType @@ -36,7 +39,7 @@ var ( X86_64 = newArch("x86_64", "lib64") Common = ArchType{ - Name: "common", + Name: COMMON_VARIANT, } ) @@ -527,7 +530,6 @@ type Arch struct { CpuVariant string Abi []string ArchFeatures []string - Native bool } func (a Arch) String() string { @@ -557,6 +559,10 @@ func newArch(name, multilib string) ArchType { return archType } +func ArchTypeList() []ArchType { + return append([]ArchType(nil), archTypeList...) +} + func (a ArchType) String() string { return a.Name } @@ -590,7 +596,7 @@ var BuildOs = func() OsType { }() var ( - osTypeList []OsType + OsTypeList []OsType commonTargetMap = make(map[string]Target) NoOsType OsType @@ -601,6 +607,10 @@ var ( Android = NewOsType("android", Device, false) Fuchsia = NewOsType("fuchsia", Device, false) + // A pseudo OSType for a common os variant, which is OSType agnostic and which + // has dependencies on all the OS variants. + CommonOS = NewOsType("common_os", Generic, false) + osArchTypeMap = map[OsType][]ArchType{ Linux: []ArchType{X86, X86_64}, LinuxBionic: []ArchType{X86_64}, @@ -662,7 +672,7 @@ func NewOsType(name string, class OsClass, defDisabled bool) OsType { DefaultDisabled: defDisabled, } - osTypeList = append(osTypeList, os) + OsTypeList = append(OsTypeList, os) if _, found := commonTargetMap[name]; found { panic(fmt.Errorf("Found Os type duplicate during OsType registration: %q", name)) @@ -674,7 +684,7 @@ func NewOsType(name string, class OsClass, defDisabled bool) OsType { } func osByName(name string) OsType { - for _, os := range osTypeList { + for _, os := range OsTypeList { if os.Name == name { return os } @@ -683,13 +693,150 @@ func osByName(name string) OsType { return NoOsType } +type NativeBridgeSupport bool + +const ( + NativeBridgeDisabled NativeBridgeSupport = false + NativeBridgeEnabled NativeBridgeSupport = true +) + type Target struct { - Os OsType - Arch Arch + Os OsType + Arch Arch + NativeBridge NativeBridgeSupport + NativeBridgeHostArchName string + NativeBridgeRelativePath string } func (target Target) String() string { - return target.Os.String() + "_" + target.Arch.String() + return target.OsVariation() + "_" + target.ArchVariation() +} + +func (target Target) OsVariation() string { + return target.Os.String() +} + +func (target Target) ArchVariation() string { + var variation string + if target.NativeBridge { + variation = "native_bridge_" + } + variation += target.Arch.String() + + return variation +} + +func (target Target) Variations() []blueprint.Variation { + return []blueprint.Variation{ + {Mutator: "os", Variation: target.OsVariation()}, + {Mutator: "arch", Variation: target.ArchVariation()}, + } +} + +func osMutator(mctx BottomUpMutatorContext) { + var module Module + var ok bool + if module, ok = mctx.Module().(Module); !ok { + return + } + + base := module.base() + + if !base.ArchSpecific() { + return + } + + osClasses := base.OsClassSupported() + + var moduleOSList []OsType + + for _, os := range OsTypeList { + supportedClass := false + for _, osClass := range osClasses { + if os.Class == osClass { + supportedClass = true + } + } + if !supportedClass { + continue + } + + if len(mctx.Config().Targets[os]) == 0 { + continue + } + + moduleOSList = append(moduleOSList, os) + } + + if len(moduleOSList) == 0 { + base.Disable() + return + } + + osNames := make([]string, len(moduleOSList)) + + for i, os := range moduleOSList { + osNames[i] = os.String() + } + + createCommonOSVariant := base.commonProperties.CreateCommonOSVariant + if createCommonOSVariant { + // A CommonOS variant was requested so add it to the list of OS's variants to + // create. It needs to be added to the end because it needs to depend on the + // the other variants in the list returned by CreateVariations(...) and inter + // variant dependencies can only be created from a later variant in that list to + // an earlier one. That is because variants are always processed in the order in + // which they are returned from CreateVariations(...). + osNames = append(osNames, CommonOS.Name) + moduleOSList = append(moduleOSList, CommonOS) + } + + modules := mctx.CreateVariations(osNames...) + for i, m := range modules { + m.base().commonProperties.CompileOS = moduleOSList[i] + m.base().setOSProperties(mctx) + } + + if createCommonOSVariant { + // A CommonOS variant was requested so add dependencies from it (the last one in + // the list) to the OS type specific variants. + last := len(modules) - 1 + commonOSVariant := modules[last] + commonOSVariant.base().commonProperties.CommonOSVariant = true + for _, module := range modules[0:last] { + // Ignore modules that are enabled. Note, this will only avoid adding + // dependencies on OsType variants that are explicitly disabled in their + // properties. The CommonOS variant will still depend on disabled variants + // if they are disabled afterwards, e.g. in archMutator if + if module.Enabled() { + mctx.AddInterVariantDependency(commonOsToOsSpecificVariantTag, commonOSVariant, module) + } + } + } +} + +// Identifies the dependency from CommonOS variant to the os specific variants. +type commonOSTag struct{ blueprint.BaseDependencyTag } + +var commonOsToOsSpecificVariantTag = commonOSTag{} + +// Get the OsType specific variants for the current CommonOS variant. +// +// The returned list will only contain enabled OsType specific variants of the +// module referenced in the supplied context. An empty list is returned if there +// are no enabled variants or the supplied context is not for an CommonOS +// variant. +func GetOsSpecificVariantsOfCommonOSVariant(mctx BaseModuleContext) []Module { + var variants []Module + mctx.VisitDirectDeps(func(m Module) { + if mctx.OtherModuleDependencyTag(m) == commonOsToOsSpecificVariantTag { + if m.Enabled() { + variants = append(variants, m) + } + } + }) + + return variants } // archMutator splits a module into a variant for each Target requested by the module. Target selection @@ -698,10 +845,10 @@ func (target Target) String() string { // - The HostOrDeviceSupported value passed in to InitAndroidArchModule by the module type factory, which selects // whether the module type can compile for host, device or both. // - The host_supported and device_supported properties on the module. -// If host is supported for the module, the Host and HostCross OsClasses are are selected. If device is supported +// If host is supported for the module, the Host and HostCross OsClasses are selected. If device is supported // for the module, the Device OsClass is selected. // Within each selected OsClass, the multilib selection is determined by: -// - The compile_multilib property if it set (which may be overriden by target.android.compile_multlib or +// - The compile_multilib property if it set (which may be overridden by target.android.compile_multilib or // target.host.compile_multilib). // - The default multilib passed to InitAndroidArchModule if compile_multilib was not set. // Valid multilib values include: @@ -729,76 +876,87 @@ func archMutator(mctx BottomUpMutatorContext) { return } - var moduleTargets []Target - moduleMultiTargets := make(map[int][]Target) - primaryModules := make(map[int]bool) - osClasses := base.OsClassSupported() + os := base.commonProperties.CompileOS + if os == CommonOS { + // Make sure that the target related properties are initialized for the + // CommonOS variant. + addTargetProperties(module, commonTargetMap[os.Name], nil, true) - for _, os := range osTypeList { - supportedClass := false - for _, osClass := range osClasses { - if os.Class == osClass { - supportedClass = true + // Do not create arch specific variants for the CommonOS variant. + return + } + + osTargets := mctx.Config().Targets[os] + image := base.commonProperties.ImageVariation + // Filter NativeBridge targets unless they are explicitly supported + // Skip creating native bridge variants for vendor modules + if os == Android && + !(Bool(base.commonProperties.Native_bridge_supported) && image == CoreVariation) { + + var targets []Target + for _, t := range osTargets { + if !t.NativeBridge { + targets = append(targets, t) } } - if !supportedClass { - continue - } - osTargets := mctx.Config().Targets[os] - if len(osTargets) == 0 { - continue - } + osTargets = targets + } - // only the primary arch in the recovery partition - if os == Android && module.InstallInRecovery() { - osTargets = []Target{osTargets[0]} - } + // only the primary arch in the ramdisk / recovery partition + if os == Android && (module.InstallInRecovery() || module.InstallInRamdisk()) { + osTargets = []Target{osTargets[0]} + } - prefer32 := false - if base.prefer32 != nil { - prefer32 = base.prefer32(mctx, base, os.Class) - } + prefer32 := false + if base.prefer32 != nil { + prefer32 = base.prefer32(mctx, base, os.Class) + } + + multilib, extraMultilib := decodeMultilib(base, os.Class) + targets, err := decodeMultilibTargets(multilib, osTargets, prefer32) + if err != nil { + mctx.ModuleErrorf("%s", err.Error()) + } - multilib, extraMultilib := decodeMultilib(base, os.Class) - targets, err := decodeMultilibTargets(multilib, osTargets, prefer32) + var multiTargets []Target + if extraMultilib != "" { + multiTargets, err = decodeMultilibTargets(extraMultilib, osTargets, prefer32) if err != nil { mctx.ModuleErrorf("%s", err.Error()) } + } - var multiTargets []Target - if extraMultilib != "" { - multiTargets, err = decodeMultilibTargets(extraMultilib, osTargets, prefer32) - if err != nil { - mctx.ModuleErrorf("%s", err.Error()) - } - } - - if len(targets) > 0 { - primaryModules[len(moduleTargets)] = true - moduleMultiTargets[len(moduleTargets)] = multiTargets - moduleTargets = append(moduleTargets, targets...) - } + if image == RecoveryVariation { + primaryArch := mctx.Config().DevicePrimaryArchType() + targets = filterToArch(targets, primaryArch) + multiTargets = filterToArch(multiTargets, primaryArch) } - if len(moduleTargets) == 0 { - base.commonProperties.Enabled = boolPtr(false) + if len(targets) == 0 { + base.Disable() return } - targetNames := make([]string, len(moduleTargets)) + targetNames := make([]string, len(targets)) - for i, target := range moduleTargets { - targetNames[i] = target.String() + for i, target := range targets { + targetNames[i] = target.ArchVariation() } modules := mctx.CreateVariations(targetNames...) for i, m := range modules { - m.(Module).base().SetTarget(moduleTargets[i], moduleMultiTargets[i], primaryModules[i]) + addTargetProperties(m, targets[i], multiTargets, i == 0) m.(Module).base().setArchProperties(mctx) } } +func addTargetProperties(m Module, target Target, multiTargets []Target, primaryTarget bool) { + m.base().commonProperties.CompileTarget = target + m.base().commonProperties.CompileMultiTargets = multiTargets + m.base().commonProperties.CompilePrimary = primaryTarget +} + func decodeMultilib(base *ModuleBase, class OsClass) (multilib, extraMultilib string) { switch class { case Device: @@ -825,154 +983,41 @@ func decodeMultilib(base *ModuleBase, class OsClass) (multilib, extraMultilib st } } -func filterArchStructFields(fields []reflect.StructField) (filteredFields []reflect.StructField, filtered bool) { - for _, field := range fields { - if !proptools.HasTag(field, "android", "arch_variant") { - filtered = true - continue - } - - // The arch_variant field isn't necessary past this point - // Instead of wasting space, just remove it. Go also has a - // 16-bit limit on structure name length. The name is constructed - // based on the Go source representation of the structure, so - // the tag names count towards that length. - // - // TODO: handle the uncommon case of other tags being involved - if field.Tag == `android:"arch_variant"` { - field.Tag = "" - } - - // Recurse into structs - switch field.Type.Kind() { - case reflect.Struct: - var subFiltered bool - field.Type, subFiltered = filterArchStruct(field.Type) - filtered = filtered || subFiltered - if field.Type == nil { - continue - } - case reflect.Ptr: - if field.Type.Elem().Kind() == reflect.Struct { - nestedType, subFiltered := filterArchStruct(field.Type.Elem()) - filtered = filtered || subFiltered - if nestedType == nil { - continue - } - field.Type = reflect.PtrTo(nestedType) - } - case reflect.Interface: - panic("Interfaces are not supported in arch_variant properties") - } - - filteredFields = append(filteredFields, field) - } - - return filteredFields, filtered -} - -// filterArchStruct takes a reflect.Type that is either a sturct or a pointer to a struct, and returns a reflect.Type -// that only contains the fields in the original type that have an `android:"arch_variant"` struct tag, and a bool -// that is true if the new struct type has fewer fields than the original type. If there are no fields in the -// original type with the struct tag it returns nil and true. -func filterArchStruct(prop reflect.Type) (filteredProp reflect.Type, filtered bool) { - var fields []reflect.StructField - - ptr := prop.Kind() == reflect.Ptr - if ptr { - prop = prop.Elem() - } - - for i := 0; i < prop.NumField(); i++ { - fields = append(fields, prop.Field(i)) - } - - filteredFields, filtered := filterArchStructFields(fields) - - if len(filteredFields) == 0 { - return nil, true - } - - if !filtered { - if ptr { - return reflect.PtrTo(prop), false +func filterToArch(targets []Target, arch ArchType) []Target { + for i := 0; i < len(targets); i++ { + if targets[i].Arch.ArchType != arch { + targets = append(targets[:i], targets[i+1:]...) + i-- } - return prop, false - } - - ret := reflect.StructOf(filteredFields) - if ptr { - ret = reflect.PtrTo(ret) } - - return ret, true + return targets } -// filterArchStruct takes a reflect.Type that is either a sturct or a pointer to a struct, and returns a list of -// reflect.Type that only contains the fields in the original type that have an `android:"arch_variant"` struct tag, -// and a bool that is true if the new struct type has fewer fields than the original type. If there are no fields in -// the original type with the struct tag it returns nil and true. Each returned struct type will have a maximum of -// 10 top level fields in it to attempt to avoid hitting the reflect.StructOf name length limit, although the limit -// can still be reached with a single struct field with many fields in it. -func filterArchStructSharded(prop reflect.Type) (filteredProp []reflect.Type, filtered bool) { - var fields []reflect.StructField - - ptr := prop.Kind() == reflect.Ptr - if ptr { - prop = prop.Elem() - } - - for i := 0; i < prop.NumField(); i++ { - fields = append(fields, prop.Field(i)) - } - - fields, filtered = filterArchStructFields(fields) - if !filtered { - if ptr { - return []reflect.Type{reflect.PtrTo(prop)}, false - } - return []reflect.Type{prop}, false - } - - if len(fields) == 0 { - return nil, true - } - - shards := shardFields(fields, 10) - - for _, shard := range shards { - s := reflect.StructOf(shard) - if ptr { - s = reflect.PtrTo(s) - } - filteredProp = append(filteredProp, s) - } - - return filteredProp, true +type archPropTypeDesc struct { + arch, multilib, target reflect.Type } -func shardFields(fields []reflect.StructField, shardSize int) [][]reflect.StructField { - ret := make([][]reflect.StructField, 0, (len(fields)+shardSize-1)/shardSize) - for len(fields) > shardSize { - ret = append(ret, fields[0:shardSize]) - fields = fields[shardSize:] - } - if len(fields) > 0 { - ret = append(ret, fields) - } - return ret +type archPropRoot struct { + Arch, Multilib, Target interface{} } -// createArchType takes a reflect.Type that is either a struct or a pointer to a struct, and returns a list of -// reflect.Type that contains the arch-variant properties inside structs for each architecture, os, target, multilib, -// etc. -func createArchType(props reflect.Type) []reflect.Type { - propShards, _ := filterArchStructSharded(props) +// createArchPropTypeDesc takes a reflect.Type that is either a struct or a pointer to a struct, and +// returns lists of reflect.Types that contains the arch-variant properties inside structs for each +// arch, multilib and target property. +func createArchPropTypeDesc(props reflect.Type) []archPropTypeDesc { + // Each property struct shard will be nested many times under the runtime generated arch struct, + // which can hit the limit of 64kB for the name of runtime generated structs. They are nested + // 97 times now, which may grow in the future, plus there is some overhead for the containing + // type. This number may need to be reduced if too many are added, but reducing it too far + // could cause problems if a single deeply nested property no longer fits in the name. + const maxArchTypeNameSize = 500 + + propShards, _ := proptools.FilterPropertyStructSharded(props, maxArchTypeNameSize, filterArchStruct) if len(propShards) == 0 { return nil } - var ret []reflect.Type + var ret []archPropTypeDesc for _, props := range propShards { variantFields := func(names []string) []reflect.StructField { @@ -1025,8 +1070,9 @@ func createArchType(props reflect.Type) []reflect.Type { "Not_windows", "Arm_on_x86", "Arm_on_x86_64", + "Native_bridge", } - for _, os := range osTypeList { + for _, os := range OsTypeList { targets = append(targets, os.Field) for _, archType := range osArchTypeMap[os] { @@ -1048,24 +1094,42 @@ func createArchType(props reflect.Type) []reflect.Type { } targetType := reflect.StructOf(variantFields(targets)) - ret = append(ret, reflect.StructOf([]reflect.StructField{ - { - Name: "Arch", - Type: archType, - }, - { - Name: "Multilib", - Type: multilibType, - }, - { - Name: "Target", - Type: targetType, - }, - })) + + ret = append(ret, archPropTypeDesc{ + arch: reflect.PtrTo(archType), + multilib: reflect.PtrTo(multilibType), + target: reflect.PtrTo(targetType), + }) } return ret } +func filterArchStruct(field reflect.StructField, prefix string) (bool, reflect.StructField) { + if proptools.HasTag(field, "android", "arch_variant") { + // The arch_variant field isn't necessary past this point + // Instead of wasting space, just remove it. Go also has a + // 16-bit limit on structure name length. The name is constructed + // based on the Go source representation of the structure, so + // the tag names count towards that length. + + androidTag := field.Tag.Get("android") + values := strings.Split(androidTag, ",") + + if string(field.Tag) != `android:"`+strings.Join(values, ",")+`"` { + panic(fmt.Errorf("unexpected tag format %q", field.Tag)) + } + // these tags don't need to be present in the runtime generated struct type. + values = RemoveListFromList(values, []string{"arch_variant", "variant_prepend", "path"}) + if len(values) > 0 { + panic(fmt.Errorf("unknown tags %q in field %q", values, prefix+field.Name)) + } + + field.Tag = "" + return true, field + } + return false, field +} + var archPropTypeMap OncePer func InitArchModule(m Module) { @@ -1089,12 +1153,16 @@ func InitArchModule(m Module) { } archPropTypes := archPropTypeMap.Once(NewCustomOnceKey(t), func() interface{} { - return createArchType(t) - }).([]reflect.Type) + return createArchPropTypeDesc(t) + }).([]archPropTypeDesc) var archProperties []interface{} for _, t := range archPropTypes { - archProperties = append(archProperties, reflect.New(t).Interface()) + archProperties = append(archProperties, &archPropRoot{ + Arch: reflect.Zero(t.arch).Interface(), + Multilib: reflect.Zero(t.multilib).Interface(), + Target: reflect.Zero(t.target).Interface(), + }) } base.archProperties = append(base.archProperties, archProperties) m.AddProperties(archProperties...) @@ -1105,9 +1173,16 @@ func InitArchModule(m Module) { var variantReplacer = strings.NewReplacer("-", "_", ".", "_") -func (a *ModuleBase) appendProperties(ctx BottomUpMutatorContext, +func (m *ModuleBase) appendProperties(ctx BottomUpMutatorContext, dst interface{}, src reflect.Value, field, srcPrefix string) reflect.Value { + if src.Kind() == reflect.Ptr { + if src.IsNil() { + return src + } + src = src.Elem() + } + src = src.FieldByName(field) if !src.IsValid() { ctx.ModuleErrorf("field %q does not exist", srcPrefix) @@ -1142,25 +1217,116 @@ func (a *ModuleBase) appendProperties(ctx BottomUpMutatorContext, return ret } -// Rewrite the module's properties structs to contain arch-specific values. -func (a *ModuleBase) setArchProperties(ctx BottomUpMutatorContext) { - arch := a.Arch() - os := a.Os() +// Rewrite the module's properties structs to contain os-specific values. +func (m *ModuleBase) setOSProperties(ctx BottomUpMutatorContext) { + os := m.commonProperties.CompileOS - for i := range a.generalProperties { - genProps := a.generalProperties[i] - if a.archProperties[i] == nil { + for i := range m.generalProperties { + genProps := m.generalProperties[i] + if m.archProperties[i] == nil { continue } - for _, archProperties := range a.archProperties[i] { + for _, archProperties := range m.archProperties[i] { archPropValues := reflect.ValueOf(archProperties).Elem() - archProp := archPropValues.FieldByName("Arch") - multilibProp := archPropValues.FieldByName("Multilib") - targetProp := archPropValues.FieldByName("Target") + targetProp := archPropValues.FieldByName("Target").Elem() + + // Handle host-specific properties in the form: + // target: { + // host: { + // key: value, + // }, + // }, + if os.Class == Host || os.Class == HostCross { + field := "Host" + prefix := "target.host" + m.appendProperties(ctx, genProps, targetProp, field, prefix) + } + + // Handle target OS generalities of the form: + // target: { + // bionic: { + // key: value, + // }, + // } + if os.Linux() { + field := "Linux" + prefix := "target.linux" + m.appendProperties(ctx, genProps, targetProp, field, prefix) + } + + if os.Bionic() { + field := "Bionic" + prefix := "target.bionic" + m.appendProperties(ctx, genProps, targetProp, field, prefix) + } + + // Handle target OS properties in the form: + // target: { + // linux_glibc: { + // key: value, + // }, + // not_windows: { + // key: value, + // }, + // android { + // key: value, + // }, + // }, + field := os.Field + prefix := "target." + os.Name + m.appendProperties(ctx, genProps, targetProp, field, prefix) + + if (os.Class == Host || os.Class == HostCross) && os != Windows { + field := "Not_windows" + prefix := "target.not_windows" + m.appendProperties(ctx, genProps, targetProp, field, prefix) + } + + // Handle 64-bit device properties in the form: + // target { + // android64 { + // key: value, + // }, + // android32 { + // key: value, + // }, + // }, + // WARNING: this is probably not what you want to use in your blueprints file, it selects + // options for all targets on a device that supports 64-bit binaries, not just the targets + // that are being compiled for 64-bit. Its expected use case is binaries like linker and + // debuggerd that need to know when they are a 32-bit process running on a 64-bit device + if os.Class == Device { + if ctx.Config().Android64() { + field := "Android64" + prefix := "target.android64" + m.appendProperties(ctx, genProps, targetProp, field, prefix) + } else { + field := "Android32" + prefix := "target.android32" + m.appendProperties(ctx, genProps, targetProp, field, prefix) + } + } + } + } +} + +// Rewrite the module's properties structs to contain arch-specific values. +func (m *ModuleBase) setArchProperties(ctx BottomUpMutatorContext) { + arch := m.Arch() + os := m.Os() + + for i := range m.generalProperties { + genProps := m.generalProperties[i] + if m.archProperties[i] == nil { + continue + } + for _, archProperties := range m.archProperties[i] { + archPropValues := reflect.ValueOf(archProperties).Elem() - var field string - var prefix string + archProp := archPropValues.FieldByName("Arch").Elem() + multilibProp := archPropValues.FieldByName("Multilib").Elem() + targetProp := archPropValues.FieldByName("Target").Elem() // Handle arch-specific properties in the form: // arch: { @@ -1173,7 +1339,7 @@ func (a *ModuleBase) setArchProperties(ctx BottomUpMutatorContext) { if arch.ArchType != Common { field := proptools.FieldNameForProperty(t.Name) prefix := "arch." + t.Name - archStruct := a.appendProperties(ctx, genProps, archProp, field, prefix) + archStruct := m.appendProperties(ctx, genProps, archProp, field, prefix) // Handle arch-variant-specific properties in the form: // arch: { @@ -1185,7 +1351,7 @@ func (a *ModuleBase) setArchProperties(ctx BottomUpMutatorContext) { if v != "" { field := proptools.FieldNameForProperty(v) prefix := "arch." + t.Name + "." + v - a.appendProperties(ctx, genProps, archStruct, field, prefix) + m.appendProperties(ctx, genProps, archStruct, field, prefix) } // Handle cpu-variant-specific properties in the form: @@ -1199,7 +1365,7 @@ func (a *ModuleBase) setArchProperties(ctx BottomUpMutatorContext) { if c != "" { field := proptools.FieldNameForProperty(c) prefix := "arch." + t.Name + "." + c - a.appendProperties(ctx, genProps, archStruct, field, prefix) + m.appendProperties(ctx, genProps, archStruct, field, prefix) } } @@ -1212,7 +1378,7 @@ func (a *ModuleBase) setArchProperties(ctx BottomUpMutatorContext) { for _, feature := range arch.ArchFeatures { field := proptools.FieldNameForProperty(feature) prefix := "arch." + t.Name + "." + feature - a.appendProperties(ctx, genProps, archStruct, field, prefix) + m.appendProperties(ctx, genProps, archStruct, field, prefix) } // Handle multilib-specific properties in the form: @@ -1223,71 +1389,35 @@ func (a *ModuleBase) setArchProperties(ctx BottomUpMutatorContext) { // }, field = proptools.FieldNameForProperty(t.Multilib) prefix = "multilib." + t.Multilib - a.appendProperties(ctx, genProps, multilibProp, field, prefix) + m.appendProperties(ctx, genProps, multilibProp, field, prefix) } - // Handle host-specific properties in the form: + // Handle combined OS-feature and arch specific properties in the form: // target: { - // host: { - // key: value, - // }, - // }, - if os.Class == Host || os.Class == HostCross { - field = "Host" - prefix = "target.host" - a.appendProperties(ctx, genProps, targetProp, field, prefix) - } - - // Handle target OS generalities of the form: - // target: { - // bionic: { - // key: value, - // }, // bionic_x86: { // key: value, // }, // } - if os.Linux() { - field = "Linux" - prefix = "target.linux" - a.appendProperties(ctx, genProps, targetProp, field, prefix) - - if arch.ArchType != Common { - field = "Linux_" + arch.ArchType.Name - prefix = "target.linux_" + arch.ArchType.Name - a.appendProperties(ctx, genProps, targetProp, field, prefix) - } + if os.Linux() && arch.ArchType != Common { + field := "Linux_" + arch.ArchType.Name + prefix := "target.linux_" + arch.ArchType.Name + m.appendProperties(ctx, genProps, targetProp, field, prefix) } - if os.Bionic() { - field = "Bionic" - prefix = "target.bionic" - a.appendProperties(ctx, genProps, targetProp, field, prefix) - - if arch.ArchType != Common { - field = "Bionic_" + t.Name - prefix = "target.bionic_" + t.Name - a.appendProperties(ctx, genProps, targetProp, field, prefix) - } + if os.Bionic() && arch.ArchType != Common { + field := "Bionic_" + t.Name + prefix := "target.bionic_" + t.Name + m.appendProperties(ctx, genProps, targetProp, field, prefix) } - // Handle target OS properties in the form: + // Handle combined OS and arch specific properties in the form: // target: { - // linux_glibc: { - // key: value, - // }, - // not_windows: { - // key: value, - // }, // linux_glibc_x86: { // key: value, // }, // linux_glibc_arm: { // key: value, // }, - // android { - // key: value, - // }, // android_arm { // key: value, // }, @@ -1295,61 +1425,38 @@ func (a *ModuleBase) setArchProperties(ctx BottomUpMutatorContext) { // key: value, // }, // }, - field = os.Field - prefix = "target." + os.Name - a.appendProperties(ctx, genProps, targetProp, field, prefix) - if arch.ArchType != Common { - field = os.Field + "_" + t.Name - prefix = "target." + os.Name + "_" + t.Name - a.appendProperties(ctx, genProps, targetProp, field, prefix) - } - - if (os.Class == Host || os.Class == HostCross) && os != Windows { - field := "Not_windows" - prefix := "target.not_windows" - a.appendProperties(ctx, genProps, targetProp, field, prefix) + field := os.Field + "_" + t.Name + prefix := "target." + os.Name + "_" + t.Name + m.appendProperties(ctx, genProps, targetProp, field, prefix) } - // Handle 64-bit device properties in the form: + // Handle arm on x86 properties in the form: // target { - // android64 { + // arm_on_x86 { // key: value, // }, - // android32 { + // arm_on_x86_64 { // key: value, // }, // }, - // WARNING: this is probably not what you want to use in your blueprints file, it selects - // options for all targets on a device that supports 64-bit binaries, not just the targets - // that are being compiled for 64-bit. Its expected use case is binaries like linker and - // debuggerd that need to know when they are a 32-bit process running on a 64-bit device if os.Class == Device { - if ctx.Config().Android64() { - field := "Android64" - prefix := "target.android64" - a.appendProperties(ctx, genProps, targetProp, field, prefix) - } else { - field := "Android32" - prefix := "target.android32" - a.appendProperties(ctx, genProps, targetProp, field, prefix) - } - - if (arch.ArchType == X86 && (hasArmAbi(arch) || - hasArmAndroidArch(ctx.Config().Targets[Android]))) || - (arch.ArchType == Arm && - hasX86AndroidArch(ctx.Config().Targets[Android])) { + if arch.ArchType == X86 && (hasArmAbi(arch) || + hasArmAndroidArch(ctx.Config().Targets[Android])) { field := "Arm_on_x86" prefix := "target.arm_on_x86" - a.appendProperties(ctx, genProps, targetProp, field, prefix) + m.appendProperties(ctx, genProps, targetProp, field, prefix) } - if (arch.ArchType == X86_64 && (hasArmAbi(arch) || - hasArmAndroidArch(ctx.Config().Targets[Android]))) || - (arch.ArchType == Arm && - hasX8664AndroidArch(ctx.Config().Targets[Android])) { + if arch.ArchType == X86_64 && (hasArmAbi(arch) || + hasArmAndroidArch(ctx.Config().Targets[Android])) { field := "Arm_on_x86_64" prefix := "target.arm_on_x86_64" - a.appendProperties(ctx, genProps, targetProp, field, prefix) + m.appendProperties(ctx, genProps, targetProp, field, prefix) + } + if os == Android && m.Target().NativeBridge == NativeBridgeEnabled { + field := "Native_bridge" + prefix := "target.native_bridge" + m.appendProperties(ctx, genProps, targetProp, field, prefix) } } } @@ -1378,7 +1485,9 @@ func decodeTargetProductVariables(config *config) (map[OsType][]Target, error) { targets := make(map[OsType][]Target) var targetErr error - addTarget := func(os OsType, archName string, archVariant, cpuVariant *string, abi []string) { + addTarget := func(os OsType, archName string, archVariant, cpuVariant *string, abi []string, + nativeBridgeEnabled NativeBridgeSupport, nativeBridgeHostArchName *string, + nativeBridgeRelativePath *string) { if targetErr != nil { return } @@ -1388,11 +1497,21 @@ func decodeTargetProductVariables(config *config) (map[OsType][]Target, error) { targetErr = err return } + nativeBridgeRelativePathStr := String(nativeBridgeRelativePath) + nativeBridgeHostArchNameStr := String(nativeBridgeHostArchName) + + // Use guest arch as relative install path by default + if nativeBridgeEnabled && nativeBridgeRelativePathStr == "" { + nativeBridgeRelativePathStr = arch.ArchType.String() + } targets[os] = append(targets[os], Target{ - Os: os, - Arch: arch, + Os: os, + Arch: arch, + NativeBridge: nativeBridgeEnabled, + NativeBridgeHostArchName: nativeBridgeHostArchNameStr, + NativeBridgeRelativePath: nativeBridgeRelativePathStr, }) } @@ -1400,14 +1519,14 @@ func decodeTargetProductVariables(config *config) (map[OsType][]Target, error) { return nil, fmt.Errorf("No host primary architecture set") } - addTarget(BuildOs, *variables.HostArch, nil, nil, nil) + addTarget(BuildOs, *variables.HostArch, nil, nil, nil, NativeBridgeDisabled, nil, nil) if variables.HostSecondaryArch != nil && *variables.HostSecondaryArch != "" { - addTarget(BuildOs, *variables.HostSecondaryArch, nil, nil, nil) + addTarget(BuildOs, *variables.HostSecondaryArch, nil, nil, nil, NativeBridgeDisabled, nil, nil) } if Bool(config.Host_bionic) { - addTarget(LinuxBionic, "x86_64", nil, nil, nil) + addTarget(LinuxBionic, "x86_64", nil, nil, nil, NativeBridgeDisabled, nil, nil) } if String(variables.CrossHost) != "" { @@ -1420,10 +1539,10 @@ func decodeTargetProductVariables(config *config) (map[OsType][]Target, error) { return nil, fmt.Errorf("No cross-host primary architecture set") } - addTarget(crossHostOs, *variables.CrossHostArch, nil, nil, nil) + addTarget(crossHostOs, *variables.CrossHostArch, nil, nil, nil, NativeBridgeDisabled, nil, nil) if variables.CrossHostSecondaryArch != nil && *variables.CrossHostSecondaryArch != "" { - addTarget(crossHostOs, *variables.CrossHostSecondaryArch, nil, nil, nil) + addTarget(crossHostOs, *variables.CrossHostSecondaryArch, nil, nil, nil, NativeBridgeDisabled, nil, nil) } } @@ -1434,17 +1553,30 @@ func decodeTargetProductVariables(config *config) (map[OsType][]Target, error) { } addTarget(target, *variables.DeviceArch, variables.DeviceArchVariant, - variables.DeviceCpuVariant, variables.DeviceAbi) + variables.DeviceCpuVariant, variables.DeviceAbi, NativeBridgeDisabled, nil, nil) if variables.DeviceSecondaryArch != nil && *variables.DeviceSecondaryArch != "" { addTarget(Android, *variables.DeviceSecondaryArch, variables.DeviceSecondaryArchVariant, variables.DeviceSecondaryCpuVariant, - variables.DeviceSecondaryAbi) + variables.DeviceSecondaryAbi, NativeBridgeDisabled, nil, nil) + } - deviceArches := targets[Android] - if deviceArches[0].Arch.ArchType.Multilib == deviceArches[1].Arch.ArchType.Multilib { - deviceArches[1].Arch.Native = false - } + if variables.NativeBridgeArch != nil && *variables.NativeBridgeArch != "" { + addTarget(Android, *variables.NativeBridgeArch, + variables.NativeBridgeArchVariant, variables.NativeBridgeCpuVariant, + variables.NativeBridgeAbi, NativeBridgeEnabled, variables.DeviceArch, + variables.NativeBridgeRelativePath) + } + + if variables.DeviceSecondaryArch != nil && *variables.DeviceSecondaryArch != "" && + variables.NativeBridgeSecondaryArch != nil && *variables.NativeBridgeSecondaryArch != "" { + addTarget(Android, *variables.NativeBridgeSecondaryArch, + variables.NativeBridgeSecondaryArchVariant, + variables.NativeBridgeSecondaryCpuVariant, + variables.NativeBridgeSecondaryAbi, + NativeBridgeEnabled, + variables.DeviceSecondaryArch, + variables.NativeBridgeSecondaryRelativePath) } } @@ -1457,15 +1589,10 @@ func decodeTargetProductVariables(config *config) (map[OsType][]Target, error) { // hasArmAbi returns true if arch has at least one arm ABI func hasArmAbi(arch Arch) bool { - for _, abi := range arch.Abi { - if strings.HasPrefix(abi, "arm") { - return true - } - } - return false + return PrefixInList(arch.Abi, "arm") } -// hasArmArch returns true if targets has at least arm Android arch +// hasArmArch returns true if targets has at least non-native_bridge arm Android arch func hasArmAndroidArch(targets []Target) bool { for _, target := range targets { if target.Os == Android && target.Arch.ArchType == Arm { @@ -1475,26 +1602,6 @@ func hasArmAndroidArch(targets []Target) bool { return false } -// hasX86Arch returns true if targets has at least x86 Android arch -func hasX86AndroidArch(targets []Target) bool { - for _, target := range targets { - if target.Os == Android && target.Arch.ArchType == X86 { - return true - } - } - return false -} - -// hasX8664Arch returns true if targets has at least x86_64 Android arch -func hasX8664AndroidArch(targets []Target) bool { - for _, target := range targets { - if target.Os == Android && target.Arch.ArchType == X86_64 { - return true - } - } - return false -} - type archConfig struct { arch string archVariant string @@ -1558,7 +1665,16 @@ func getMegaDeviceConfig() []archConfig { func getNdkAbisConfig() []archConfig { return []archConfig{ - {"arm", "armv7-a", "", []string{"armeabi"}}, + {"arm", "armv7-a", "", []string{"armeabi-v7a"}}, + {"arm64", "armv8-a", "", []string{"arm64-v8a"}}, + {"x86", "", "", []string{"x86"}}, + {"x86_64", "", "", []string{"x86_64"}}, + } +} + +func getAmlAbisConfig() []archConfig { + return []archConfig{ + {"arm", "armv7-a", "", []string{"armeabi-v7a"}}, {"arm64", "armv8-a", "", []string{"arm64-v8a"}}, {"x86", "", "", []string{"x86"}}, {"x86_64", "", "", []string{"x86_64"}}, @@ -1574,7 +1690,7 @@ func decodeArchSettings(os OsType, archConfigs []archConfig) ([]Target, error) { if err != nil { return nil, err } - arch.Native = false + ret = append(ret, Target{ Os: Android, Arch: arch, @@ -1603,7 +1719,6 @@ func decodeArch(os OsType, arch string, archVariant, cpuVariant *string, abi []s ArchVariant: stringPtr(archVariant), CpuVariant: stringPtr(cpuVariant), Abi: abi, - Native: true, } if a.ArchVariant == a.ArchType.Name || a.ArchVariant == "generic" { @@ -1644,6 +1759,8 @@ func filterMultilibTargets(targets []Target, multilib string) []Target { return ret } +// Return the set of Os specific common architecture targets for each Os in a list of +// targets. func getCommonTargets(targets []Target) []Target { var ret []Target set := make(map[string]bool) diff --git a/android/arch_test.go b/android/arch_test.go index 0589e6c3a..8525b0349 100644 --- a/android/arch_test.go +++ b/android/arch_test.go @@ -16,7 +16,10 @@ package android import ( "reflect" + "runtime" "testing" + + "github.com/google/blueprint/proptools" ) type Named struct { @@ -52,6 +55,24 @@ func TestFilterArchStruct(t *testing.T) { filtered: true, }, { + name: "tags", + in: &struct { + A *string `android:"arch_variant"` + B *string `android:"arch_variant,path"` + C *string `android:"arch_variant,path,variant_prepend"` + D *string `android:"path,variant_prepend,arch_variant"` + E *string `android:"path"` + F *string + }{}, + out: &struct { + A *string + B *string + C *string + D *string + }{}, + filtered: true, + }, + { name: "all filtered", in: &struct { A *string @@ -219,7 +240,7 @@ func TestFilterArchStruct(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - out, filtered := filterArchStruct(reflect.TypeOf(test.in)) + out, filtered := proptools.FilterPropertyStruct(reflect.TypeOf(test.in), filterArchStruct) if filtered != test.filtered { t.Errorf("expected filtered %v, got %v", test.filtered, filtered) } @@ -230,3 +251,220 @@ func TestFilterArchStruct(t *testing.T) { }) } } + +type archTestModule struct { + ModuleBase + props struct { + Deps []string + } +} + +func (m *archTestModule) GenerateAndroidBuildActions(ctx ModuleContext) { +} + +func (m *archTestModule) DepsMutator(ctx BottomUpMutatorContext) { + ctx.AddDependency(ctx.Module(), nil, m.props.Deps...) +} + +func archTestModuleFactory() Module { + m := &archTestModule{} + m.AddProperties(&m.props) + InitAndroidArchModule(m, HostAndDeviceSupported, MultilibBoth) + return m +} + +func TestArchMutator(t *testing.T) { + var buildOSVariants []string + var buildOS32Variants []string + switch runtime.GOOS { + case "linux": + buildOSVariants = []string{"linux_glibc_x86_64", "linux_glibc_x86"} + buildOS32Variants = []string{"linux_glibc_x86"} + case "darwin": + buildOSVariants = []string{"darwin_x86_64"} + buildOS32Variants = nil + } + + bp := ` + module { + name: "foo", + } + + module { + name: "bar", + host_supported: true, + } + + module { + name: "baz", + device_supported: false, + } + + module { + name: "qux", + host_supported: true, + compile_multilib: "32", + } + ` + + testCases := []struct { + name string + config func(Config) + fooVariants []string + barVariants []string + bazVariants []string + quxVariants []string + }{ + { + name: "normal", + config: nil, + fooVariants: []string{"android_arm64_armv8-a", "android_arm_armv7-a-neon"}, + barVariants: append(buildOSVariants, "android_arm64_armv8-a", "android_arm_armv7-a-neon"), + bazVariants: nil, + quxVariants: append(buildOS32Variants, "android_arm_armv7-a-neon"), + }, + { + name: "host-only", + config: func(config Config) { + config.BuildOSTarget = Target{} + config.BuildOSCommonTarget = Target{} + config.Targets[Android] = nil + }, + fooVariants: nil, + barVariants: buildOSVariants, + bazVariants: nil, + quxVariants: buildOS32Variants, + }, + } + + enabledVariants := func(ctx *TestContext, name string) []string { + var ret []string + variants := ctx.ModuleVariantsForTests(name) + for _, variant := range variants { + m := ctx.ModuleForTests(name, variant) + if m.Module().Enabled() { + ret = append(ret, variant) + } + } + return ret + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + config := TestArchConfig(buildDir, nil, bp, nil) + + ctx := NewTestArchContext() + ctx.RegisterModuleType("module", archTestModuleFactory) + ctx.Register(config) + if tt.config != nil { + tt.config(config) + } + + _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) + FailIfErrored(t, errs) + _, errs = ctx.PrepareBuildActions(config) + FailIfErrored(t, errs) + + if g, w := enabledVariants(ctx, "foo"), tt.fooVariants; !reflect.DeepEqual(w, g) { + t.Errorf("want foo variants:\n%q\ngot:\n%q\n", w, g) + } + + if g, w := enabledVariants(ctx, "bar"), tt.barVariants; !reflect.DeepEqual(w, g) { + t.Errorf("want bar variants:\n%q\ngot:\n%q\n", w, g) + } + + if g, w := enabledVariants(ctx, "baz"), tt.bazVariants; !reflect.DeepEqual(w, g) { + t.Errorf("want baz variants:\n%q\ngot:\n%q\n", w, g) + } + + if g, w := enabledVariants(ctx, "qux"), tt.quxVariants; !reflect.DeepEqual(w, g) { + t.Errorf("want qux variants:\n%q\ngot:\n%q\n", w, g) + } + }) + } +} + +func TestArchMutatorNativeBridge(t *testing.T) { + bp := ` + // This module is only enabled for x86. + module { + name: "foo", + } + + // This module is enabled for x86 and arm (via native bridge). + module { + name: "bar", + native_bridge_supported: true, + } + + // This module is enabled for arm (native_bridge) only. + module { + name: "baz", + native_bridge_supported: true, + enabled: false, + target: { + native_bridge: { + enabled: true, + } + } + } + ` + + testCases := []struct { + name string + config func(Config) + fooVariants []string + barVariants []string + bazVariants []string + }{ + { + name: "normal", + config: nil, + fooVariants: []string{"android_x86_64_silvermont", "android_x86_silvermont"}, + barVariants: []string{"android_x86_64_silvermont", "android_native_bridge_arm64_armv8-a", "android_x86_silvermont", "android_native_bridge_arm_armv7-a-neon"}, + bazVariants: []string{"android_native_bridge_arm64_armv8-a", "android_native_bridge_arm_armv7-a-neon"}, + }, + } + + enabledVariants := func(ctx *TestContext, name string) []string { + var ret []string + variants := ctx.ModuleVariantsForTests(name) + for _, variant := range variants { + m := ctx.ModuleForTests(name, variant) + if m.Module().Enabled() { + ret = append(ret, variant) + } + } + return ret + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + config := TestArchConfigNativeBridge(buildDir, nil, bp, nil) + + ctx := NewTestArchContext() + ctx.RegisterModuleType("module", archTestModuleFactory) + ctx.Register(config) + if tt.config != nil { + tt.config(config) + } + + _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) + FailIfErrored(t, errs) + _, errs = ctx.PrepareBuildActions(config) + FailIfErrored(t, errs) + + if g, w := enabledVariants(ctx, "foo"), tt.fooVariants; !reflect.DeepEqual(w, g) { + t.Errorf("want foo variants:\n%q\ngot:\n%q\n", w, g) + } + + if g, w := enabledVariants(ctx, "bar"), tt.barVariants; !reflect.DeepEqual(w, g) { + t.Errorf("want bar variants:\n%q\ngot:\n%q\n", w, g) + } + + if g, w := enabledVariants(ctx, "baz"), tt.bazVariants; !reflect.DeepEqual(w, g) { + t.Errorf("want qux variants:\n%q\ngot:\n%q\n", w, g) + } + }) + } +} diff --git a/android/config.go b/android/config.go index 2e0c247bb..3541f831e 100644 --- a/android/config.go +++ b/android/config.go @@ -25,13 +25,18 @@ import ( "strings" "sync" + "github.com/google/blueprint" "github.com/google/blueprint/bootstrap" + "github.com/google/blueprint/pathtools" "github.com/google/blueprint/proptools" + + "android/soong/android/soongconfig" ) var Bool = proptools.Bool var String = proptools.String -var FutureApiLevel = 10000 + +const FutureApiLevel = 10000 // The configuration file name const configFileName = "soong.config" @@ -64,19 +69,7 @@ type DeviceConfig struct { *deviceConfig } -type VendorConfig interface { - // Bool interprets the variable named `name` as a boolean, returning true if, after - // lowercasing, it matches one of "1", "y", "yes", "on", or "true". Unset, or any other - // value will return false. - Bool(name string) bool - - // String returns the string value of `name`. If the variable was not set, it will - // return the empty string. - String(name string) string - - // IsSet returns whether the variable `name` was set by Make. - IsSet(name string) bool -} +type VendorConfig soongconfig.SoongConfig type config struct { FileConfigurableOptions @@ -89,9 +82,14 @@ type config struct { ConfigFileName string ProductVariablesFileName string - Targets map[OsType][]Target - BuildOsVariant string - BuildOsCommonVariant string + Targets map[OsType][]Target + BuildOSTarget Target // the Target for tools run on the build machine + BuildOSCommonTarget Target // the Target for common (java) tools run on the build machine + AndroidCommonTarget Target // the Target for common modules for the Android device + + // multilibConflicts for an ArchType is true if there is earlier configured device architecture with the same + // multilib value. + multilibConflicts map[ArchType]bool deviceConfig *deviceConfig @@ -108,10 +106,15 @@ type config struct { captureBuild bool // true for tests, saves build parameters for each module ignoreEnvironment bool // true for tests, returns empty from all Getenv calls - targetOpenJDK9 bool // Target 1.9 - stopBefore bootstrap.StopBefore + fs pathtools.FileSystem + mockBpList string + + // If testAllowNonExistentPaths is true then PathForSource and PathForModuleSrc won't error + // in tests when a path doesn't exist. + testAllowNonExistentPaths bool + OncePer } @@ -120,19 +123,17 @@ type deviceConfig struct { OncePer } -type vendorConfig map[string]string - type jsonConfigurable interface { SetDefaultConfig() } func loadConfig(config *config) error { - err := loadFromConfigFile(&config.FileConfigurableOptions, config.ConfigFileName) + err := loadFromConfigFile(&config.FileConfigurableOptions, absolutePath(config.ConfigFileName)) if err != nil { return err } - return loadFromConfigFile(&config.productVariables, config.ProductVariablesFileName) + return loadFromConfigFile(&config.productVariables, absolutePath(config.ProductVariablesFileName)) } // loads configuration options from a JSON file in the cwd. @@ -196,14 +197,33 @@ func saveToConfigFile(config jsonConfigurable, filename string) error { return nil } +// NullConfig returns a mostly empty Config for use by standalone tools like dexpreopt_gen that +// use the android package. +func NullConfig(buildDir string) Config { + return Config{ + config: &config{ + buildDir: buildDir, + fs: pathtools.OsFs, + }, + } +} + // TestConfig returns a Config object suitable for using for tests -func TestConfig(buildDir string, env map[string]string) Config { +func TestConfig(buildDir string, env map[string]string, bp string, fs map[string][]byte) Config { + envCopy := make(map[string]string) + for k, v := range env { + envCopy[k] = v + } + + // Copy the real PATH value to the test environment, it's needed by HostSystemTool() used in x86_darwin_host.go + envCopy["PATH"] = originalEnv["PATH"] + config := &config{ productVariables: productVariables{ DeviceName: stringPtr("test_device"), - Platform_sdk_version: intPtr(26), + Platform_sdk_version: intPtr(30), DeviceSystemSdkVersions: []string{"14", "15"}, - Platform_systemsdk_versions: []string{"25", "26"}, + Platform_systemsdk_versions: []string{"29", "30"}, AAPTConfig: []string{"normal", "large", "xlarge", "hdpi", "xhdpi", "xxhdpi"}, AAPTPreferredConfig: stringPtr("xhdpi"), AAPTCharacteristics: stringPtr("nosdcard"), @@ -213,13 +233,19 @@ func TestConfig(buildDir string, env map[string]string) Config { buildDir: buildDir, captureBuild: true, - env: env, + env: envCopy, + + // Set testAllowNonExistentPaths so that test contexts don't need to specify every path + // passed to PathForSource or PathForModuleSrc. + testAllowNonExistentPaths: true, } config.deviceConfig = &deviceConfig{ config: config, } config.TestProductVariables = &config.productVariables + config.mockFileSystem(bp, fs) + if err := config.fromEnv(); err != nil { panic(err) } @@ -227,16 +253,30 @@ func TestConfig(buildDir string, env map[string]string) Config { return Config{config} } -func TestArchConfigFuchsia(buildDir string, env map[string]string) Config { - testConfig := TestConfig(buildDir, env) +func TestArchConfigNativeBridge(buildDir string, env map[string]string, bp string, fs map[string][]byte) Config { + testConfig := TestArchConfig(buildDir, env, bp, fs) + config := testConfig.config + + config.Targets[Android] = []Target{ + {Android, Arch{ArchType: X86_64, ArchVariant: "silvermont", Abi: []string{"arm64-v8a"}}, NativeBridgeDisabled, "", ""}, + {Android, Arch{ArchType: X86, ArchVariant: "silvermont", Abi: []string{"armeabi-v7a"}}, NativeBridgeDisabled, "", ""}, + {Android, Arch{ArchType: Arm64, ArchVariant: "armv8-a", Abi: []string{"arm64-v8a"}}, NativeBridgeEnabled, "x86_64", "arm64"}, + {Android, Arch{ArchType: Arm, ArchVariant: "armv7-a-neon", Abi: []string{"armeabi-v7a"}}, NativeBridgeEnabled, "x86", "arm"}, + } + + return testConfig +} + +func TestArchConfigFuchsia(buildDir string, env map[string]string, bp string, fs map[string][]byte) Config { + testConfig := TestConfig(buildDir, env, bp, fs) config := testConfig.config config.Targets = map[OsType][]Target{ Fuchsia: []Target{ - {Fuchsia, Arch{ArchType: Arm64, ArchVariant: "", Native: true}}, + {Fuchsia, Arch{ArchType: Arm64, ArchVariant: "", Abi: []string{"arm64-v8a"}}, NativeBridgeDisabled, "", ""}, }, BuildOs: []Target{ - {BuildOs, Arch{ArchType: X86_64}}, + {BuildOs, Arch{ArchType: X86_64}, NativeBridgeDisabled, "", ""}, }, } @@ -244,23 +284,32 @@ func TestArchConfigFuchsia(buildDir string, env map[string]string) Config { } // TestConfig returns a Config object suitable for using for tests that need to run the arch mutator -func TestArchConfig(buildDir string, env map[string]string) Config { - testConfig := TestConfig(buildDir, env) +func TestArchConfig(buildDir string, env map[string]string, bp string, fs map[string][]byte) Config { + testConfig := TestConfig(buildDir, env, bp, fs) config := testConfig.config config.Targets = map[OsType][]Target{ Android: []Target{ - {Android, Arch{ArchType: Arm64, ArchVariant: "armv8-a", Native: true, Abi: []string{"arm64-v8a"}}}, - {Android, Arch{ArchType: Arm, ArchVariant: "armv7-a-neon", Native: true, Abi: []string{"armeabi-v7a"}}}, + {Android, Arch{ArchType: Arm64, ArchVariant: "armv8-a", Abi: []string{"arm64-v8a"}}, NativeBridgeDisabled, "", ""}, + {Android, Arch{ArchType: Arm, ArchVariant: "armv7-a-neon", Abi: []string{"armeabi-v7a"}}, NativeBridgeDisabled, "", ""}, }, BuildOs: []Target{ - {BuildOs, Arch{ArchType: X86_64}}, - {BuildOs, Arch{ArchType: X86}}, + {BuildOs, Arch{ArchType: X86_64}, NativeBridgeDisabled, "", ""}, + {BuildOs, Arch{ArchType: X86}, NativeBridgeDisabled, "", ""}, }, } - config.BuildOsVariant = config.Targets[BuildOs][0].String() - config.BuildOsCommonVariant = getCommonTargets(config.Targets[BuildOs])[0].String() + if runtime.GOOS == "darwin" { + config.Targets[BuildOs] = config.Targets[BuildOs][:1] + } + + config.BuildOSTarget = config.Targets[BuildOs][0] + config.BuildOSCommonTarget = getCommonTargets(config.Targets[BuildOs])[0] + config.AndroidCommonTarget = getCommonTargets(config.Targets[Android])[0] + config.TestProductVariables.DeviceArch = proptools.StringPtr("arm64") + config.TestProductVariables.DeviceArchVariant = proptools.StringPtr("armv8-a") + config.TestProductVariables.DeviceSecondaryArch = proptools.StringPtr("arm") + config.TestProductVariables.DeviceSecondaryArchVariant = proptools.StringPtr("armv7-a-neon") return testConfig } @@ -275,8 +324,11 @@ func NewConfig(srcDir, buildDir string) (Config, error) { env: originalEnv, - srcDir: srcDir, - buildDir: buildDir, + srcDir: srcDir, + buildDir: buildDir, + multilibConflicts: make(map[ArchType]bool), + + fs: pathtools.NewOsFs(absSrcDir), } config.deviceConfig = &deviceConfig{ @@ -306,7 +358,7 @@ func NewConfig(srcDir, buildDir string) (Config, error) { } inMakeFile := filepath.Join(buildDir, ".soong.in_make") - if _, err := os.Stat(inMakeFile); err == nil { + if _, err := os.Stat(absolutePath(inMakeFile)); err == nil { config.inMake = true } @@ -315,11 +367,16 @@ func NewConfig(srcDir, buildDir string) (Config, error) { return Config{}, err } + // Make the CommonOS OsType available for all products. + targets[CommonOS] = []Target{commonTargetMap[CommonOS.Name]} + var archConfig []archConfig if Bool(config.Mega_device) { archConfig = getMegaDeviceConfig() } else if config.NdkAbis() { archConfig = getNdkAbisConfig() + } else if config.AmlAbis() { + archConfig = getAmlAbisConfig() } if archConfig != nil { @@ -330,26 +387,74 @@ func NewConfig(srcDir, buildDir string) (Config, error) { targets[Android] = androidTargets } + multilib := make(map[string]bool) + for _, target := range targets[Android] { + if seen := multilib[target.Arch.ArchType.Multilib]; seen { + config.multilibConflicts[target.Arch.ArchType] = true + } + multilib[target.Arch.ArchType.Multilib] = true + } + config.Targets = targets - config.BuildOsVariant = targets[BuildOs][0].String() - config.BuildOsCommonVariant = getCommonTargets(targets[BuildOs])[0].String() + config.BuildOSTarget = config.Targets[BuildOs][0] + config.BuildOSCommonTarget = getCommonTargets(config.Targets[BuildOs])[0] + if len(config.Targets[Android]) > 0 { + config.AndroidCommonTarget = getCommonTargets(config.Targets[Android])[0] + } if err := config.fromEnv(); err != nil { return Config{}, err } + if Bool(config.productVariables.GcovCoverage) && Bool(config.productVariables.ClangCoverage) { + return Config{}, fmt.Errorf("GcovCoverage and ClangCoverage cannot both be set") + } + + config.productVariables.Native_coverage = proptools.BoolPtr( + Bool(config.productVariables.GcovCoverage) || + Bool(config.productVariables.ClangCoverage)) + return Config{config}, nil } +var TestConfigOsFs = map[string][]byte{} + +// mockFileSystem replaces all reads with accesses to the provided map of +// filenames to contents stored as a byte slice. +func (c *config) mockFileSystem(bp string, fs map[string][]byte) { + mockFS := map[string][]byte{} + + if _, exists := mockFS["Android.bp"]; !exists { + mockFS["Android.bp"] = []byte(bp) + } + + for k, v := range fs { + mockFS[k] = v + } + + // no module list file specified; find every file named Blueprints or Android.bp + pathsToParse := []string{} + for candidate := range mockFS { + base := filepath.Base(candidate) + if base == "Blueprints" || base == "Android.bp" { + pathsToParse = append(pathsToParse, candidate) + } + } + if len(pathsToParse) < 1 { + panic(fmt.Sprintf("No Blueprint or Android.bp files found in mock filesystem: %v\n", mockFS)) + } + mockFS[blueprint.MockModuleListFile] = []byte(strings.Join(pathsToParse, "\n")) + + c.fs = pathtools.MockFs(mockFS) + c.mockBpList = blueprint.MockModuleListFile +} + func (c *config) fromEnv() error { - switch c.Getenv("EXPERIMENTAL_USE_OPENJDK9") { - case "", "1.8": - // Nothing, we always use OpenJDK9 - case "true": - // Use OpenJDK9 and target 1.9 - c.targetOpenJDK9 = true + switch c.Getenv("EXPERIMENTAL_JAVA_LANGUAGE_LEVEL_9") { + case "", "true": + // Do nothing default: - return fmt.Errorf(`Invalid value for EXPERIMENTAL_USE_OPENJDK9, should be "", "1.8", or "true"`) + return fmt.Errorf("The environment variable EXPERIMENTAL_JAVA_LANGUAGE_LEVEL_9 is no longer supported. Java language level 9 is now the global default.") } return nil @@ -375,6 +480,18 @@ func (c *config) HostToolPath(ctx PathContext, tool string) Path { return PathForOutput(ctx, "host", c.PrebuiltOS(), "bin", tool) } +func (c *config) HostJNIToolPath(ctx PathContext, path string) Path { + ext := ".so" + if runtime.GOOS == "darwin" { + ext = ".dylib" + } + return PathForOutput(ctx, "host", c.PrebuiltOS(), "lib64", path+ext) +} + +func (c *config) HostJavaToolPath(ctx PathContext, path string) Path { + return PathForOutput(ctx, "host", c.PrebuiltOS(), "framework", path) +} + // HostSystemTool looks for non-hermetic tools from the system we're running on. // Generally shouldn't be used, but useful to find the XCode SDK, etc. func (c *config) HostSystemTool(name string) string { @@ -406,6 +523,10 @@ func (c *config) GoRoot() string { return fmt.Sprintf("%s/prebuilts/go/%s", c.srcDir, c.PrebuiltOS()) } +func (c *config) PrebuiltBuildTool(ctx PathContext, tool string) Path { + return PathForSource(ctx, "prebuilts/build-tools", c.PrebuiltOS(), "bin", tool) +} + func (c *config) CpPreserveSymlinksFlags() string { switch runtime.GOOS { case "darwin": @@ -468,8 +589,8 @@ func (c *config) BuildId() string { return String(c.productVariables.BuildId) } -func (c *config) BuildNumberFromFile() string { - return String(c.productVariables.BuildNumberFromFile) +func (c *config) BuildNumberFile(ctx PathContext) Path { + return PathForOutput(ctx, String(c.productVariables.BuildNumberFile)) } // DeviceName returns the name of the current device target @@ -547,22 +668,6 @@ func (c *config) PlatformVersionActiveCodenames() []string { return c.productVariables.Platform_version_active_codenames } -// Codenames that are available in the branch but not included in the current -// lunch target. -func (c *config) PlatformVersionFutureCodenames() []string { - return c.productVariables.Platform_version_future_codenames -} - -// All possible codenames in the current branch. NB: Not named AllCodenames -// because "all" has historically meant "active" in make, and still does in -// build.prop. -func (c *config) PlatformVersionCombinedCodenames() []string { - combined := []string{} - combined = append(combined, c.PlatformVersionActiveCodenames()...) - combined = append(combined, c.PlatformVersionFutureCodenames()...) - return combined -} - func (c *config) ProductAAPTConfig() []string { return c.productVariables.AAPTConfig } @@ -584,7 +689,7 @@ func (c *config) DefaultAppCertificateDir(ctx PathContext) SourcePath { if defaultCert != "" { return PathForSource(ctx, filepath.Dir(defaultCert)) } else { - return PathForSource(ctx, "build/target/product/security") + return PathForSource(ctx, "build/make/target/product/security") } } @@ -601,7 +706,7 @@ func (c *config) DefaultAppCertificate(ctx PathContext) (pem, key SourcePath) { func (c *config) ApexKeyDir(ctx ModuleContext) SourcePath { // TODO(b/121224311): define another variable such as TARGET_APEX_KEY_OVERRIDE defaultCert := String(c.productVariables.DefaultAppCertificate) - if defaultCert == "" || filepath.Dir(defaultCert) == "build/target/product/security" { + if defaultCert == "" || filepath.Dir(defaultCert) == "build/make/target/product/security" { // When defaultCert is unset or is set to the testkeys path, use the APEX keys // that is under the module dir return pathForModuleSrc(ctx) @@ -655,10 +760,6 @@ func (c *config) DevicePrimaryArchType() ArchType { return c.Targets[Android][0].Arch.ArchType } -func (c *config) SkipDeviceInstall() bool { - return c.EmbeddedInMake() -} - func (c *config) SkipMegaDeviceInstall(path string) bool { return Bool(c.Mega_device) && strings.HasPrefix(path, filepath.Join(c.buildDir, "target", "product")) @@ -692,14 +793,6 @@ func (c *config) DisableScudo() bool { return Bool(c.productVariables.DisableScudo) } -func (c *config) EnableXOM() bool { - if c.productVariables.EnableXOM == nil { - return true - } else { - return Bool(c.productVariables.EnableXOM) - } -} - func (c *config) Android64() bool { for _, t := range c.Targets[Android] { if t.Arch.ArchType.Multilib == "lib64" { @@ -738,9 +831,20 @@ func (c *config) RunErrorProne() bool { return c.IsEnvTrue("RUN_ERROR_PRONE") } -// Returns true if -source 1.9 -target 1.9 is being passed to javac -func (c *config) TargetOpenJDK9() bool { - return c.targetOpenJDK9 +func (c *config) XrefCorpusName() string { + return c.Getenv("XREF_CORPUS") +} + +// Returns Compilation Unit encoding to use. Can be 'json' (default), 'proto' or 'all'. +func (c *config) XrefCuEncoding() string { + if enc := c.Getenv("KYTHE_KZIP_ENCODING"); enc != "" { + return enc + } + return "json" +} + +func (c *config) EmitXrefRules() bool { + return c.XrefCorpusName() != "" } func (c *config) ClangTidy() bool { @@ -777,8 +881,15 @@ func (c *config) ArtUseReadBarrier() bool { func (c *config) EnforceRROForModule(name string) bool { enforceList := c.productVariables.EnforceRROTargets + // TODO(b/150820813) Some modules depend on static overlay, remove this after eliminating the dependency. + exemptedList := c.productVariables.EnforceRROExemptedTargets + if exemptedList != nil { + if InList(name, exemptedList) { + return false + } + } if enforceList != nil { - if len(enforceList) == 1 && (enforceList)[0] == "*" { + if InList("*", enforceList) { return true } return InList(name, enforceList) @@ -789,11 +900,7 @@ func (c *config) EnforceRROForModule(name string) bool { func (c *config) EnforceRROExcludedOverlay(path string) bool { excluded := c.productVariables.EnforceRROExcludedOverlays if excluded != nil { - for _, exclude := range excluded { - if strings.HasPrefix(path, exclude) { - return true - } - } + return HasAnyPrefix(path, excluded) } return false } @@ -814,18 +921,46 @@ func (c *config) ModulesLoadedByPrivilegedModules() []string { return c.productVariables.ModulesLoadedByPrivilegedModules } +// Expected format for apexJarValue = <apex name>:<jar name> +func SplitApexJarPair(apexJarValue string) (string, string) { + var apexJarPair []string = strings.SplitN(apexJarValue, ":", 2) + if apexJarPair == nil || len(apexJarPair) != 2 { + panic(fmt.Errorf("malformed apexJarValue: %q, expected format: <apex>:<jar>", + apexJarValue)) + } + return apexJarPair[0], apexJarPair[1] +} + func (c *config) BootJars() []string { - return c.productVariables.BootJars + jars := c.productVariables.BootJars + for _, p := range c.productVariables.UpdatableBootJars { + _, jar := SplitApexJarPair(p) + jars = append(jars, jar) + } + return jars } -func (c *config) DexpreoptGlobalConfig() string { - return String(c.productVariables.DexpreoptGlobalConfig) +func (c *config) DexpreoptGlobalConfig(ctx PathContext) ([]byte, error) { + if c.productVariables.DexpreoptGlobalConfig == nil { + return nil, nil + } + path := absolutePath(*c.productVariables.DexpreoptGlobalConfig) + ctx.AddNinjaFileDeps(path) + return ioutil.ReadFile(path) } func (c *config) FrameworksBaseDirExists(ctx PathContext) bool { return ExistentPathForSource(ctx, "frameworks", "base").Valid() } +func (c *config) VndkSnapshotBuildArtifacts() bool { + return Bool(c.productVariables.VndkSnapshotBuildArtifacts) +} + +func (c *config) HasMultilibConflict(arch ArchType) bool { + return c.multilibConflicts[arch] +} + func (c *deviceConfig) Arches() []Arch { var arches []Arch for _, target := range c.config.Targets[Android] { @@ -857,6 +992,10 @@ func (c *deviceConfig) PlatformVndkVersion() string { return String(c.config.productVariables.Platform_vndk_version) } +func (c *deviceConfig) ProductVndkVersion() string { + return String(c.config.productVariables.ProductVndkVersion) +} + func (c *deviceConfig) ExtraVndkVersions() []string { return c.config.productVariables.ExtraVndkVersions } @@ -887,11 +1026,11 @@ func (c *deviceConfig) ProductPath() string { return "product" } -func (c *deviceConfig) ProductServicesPath() string { - if c.config.productVariables.ProductServicesPath != nil { - return *c.config.productVariables.ProductServicesPath +func (c *deviceConfig) SystemExtPath() string { + if c.config.productVariables.SystemExtPath != nil { + return *c.config.productVariables.SystemExtPath } - return "product_services" + return "system_ext" } func (c *deviceConfig) BtConfigIncludeDir() string { @@ -902,19 +1041,59 @@ func (c *deviceConfig) DeviceKernelHeaderDirs() []string { return c.config.productVariables.DeviceKernelHeaders } +func (c *deviceConfig) SamplingPGO() bool { + return Bool(c.config.productVariables.SamplingPGO) +} + +// JavaCoverageEnabledForPath returns whether Java code coverage is enabled for +// path. Coverage is enabled by default when the product variable +// JavaCoveragePaths is empty. If JavaCoveragePaths is not empty, coverage is +// enabled for any path which is part of this variable (and not part of the +// JavaCoverageExcludePaths product variable). Value "*" in JavaCoveragePaths +// represents any path. +func (c *deviceConfig) JavaCoverageEnabledForPath(path string) bool { + coverage := false + if len(c.config.productVariables.JavaCoveragePaths) == 0 || + InList("*", c.config.productVariables.JavaCoveragePaths) || + HasAnyPrefix(path, c.config.productVariables.JavaCoveragePaths) { + coverage = true + } + if coverage && c.config.productVariables.JavaCoverageExcludePaths != nil { + if HasAnyPrefix(path, c.config.productVariables.JavaCoverageExcludePaths) { + coverage = false + } + } + return coverage +} + +// Returns true if gcov or clang coverage is enabled. func (c *deviceConfig) NativeCoverageEnabled() bool { - return Bool(c.config.productVariables.NativeCoverage) + return Bool(c.config.productVariables.GcovCoverage) || + Bool(c.config.productVariables.ClangCoverage) } -func (c *deviceConfig) CoverageEnabledForPath(path string) bool { +func (c *deviceConfig) ClangCoverageEnabled() bool { + return Bool(c.config.productVariables.ClangCoverage) +} + +func (c *deviceConfig) GcovCoverageEnabled() bool { + return Bool(c.config.productVariables.GcovCoverage) +} + +// NativeCoverageEnabledForPath returns whether (GCOV- or Clang-based) native +// code coverage is enabled for path. By default, coverage is not enabled for a +// given path unless it is part of the NativeCoveragePaths product variable (and +// not part of the NativeCoverageExcludePaths product variable). Value "*" in +// NativeCoveragePaths represents any path. +func (c *deviceConfig) NativeCoverageEnabledForPath(path string) bool { coverage := false - if c.config.productVariables.CoveragePaths != nil { - if InList("*", c.config.productVariables.CoveragePaths) || PrefixInList(path, c.config.productVariables.CoveragePaths) { + if c.config.productVariables.NativeCoveragePaths != nil { + if InList("*", c.config.productVariables.NativeCoveragePaths) || HasAnyPrefix(path, c.config.productVariables.NativeCoveragePaths) { coverage = true } } - if coverage && c.config.productVariables.CoverageExcludePaths != nil { - if PrefixInList(path, c.config.productVariables.CoverageExcludePaths) { + if coverage && c.config.productVariables.NativeCoverageExcludePaths != nil { + if HasAnyPrefix(path, c.config.productVariables.NativeCoverageExcludePaths) { coverage = false } } @@ -941,6 +1120,10 @@ func (c *deviceConfig) PlatPrivateSepolicyDirs() []string { return c.config.productVariables.BoardPlatPrivateSepolicyDirs } +func (c *deviceConfig) SepolicyM4Defs() []string { + return c.config.productVariables.BoardSepolicyM4Defs +} + func (c *deviceConfig) OverrideManifestPackageNameFor(name string) (manifestName string, overridden bool) { return findOverrideValue(c.config.productVariables.ManifestPackageNameOverrides, name, "invalid override rule %q in PRODUCT_MANIFEST_PACKAGE_NAME_OVERRIDES should be <module_name>:<manifest_name>") @@ -979,83 +1162,61 @@ func findOverrideValue(overrides []string, name string, errorMsg string) (newVal return "", false } -// SecondArchIsTranslated returns true if the primary device arch is X86 or X86_64 and the device also has an arch -// that is Arm or Arm64. -func (c *config) SecondArchIsTranslated() bool { - deviceTargets := c.Targets[Android] - if len(deviceTargets) < 2 { - return false - } - - arch := deviceTargets[0].Arch - - return (arch.ArchType == X86 || arch.ArchType == X86_64) && hasArmAndroidArch(deviceTargets) -} - func (c *config) IntegerOverflowDisabledForPath(path string) bool { if c.productVariables.IntegerOverflowExcludePaths == nil { return false } - return PrefixInList(path, c.productVariables.IntegerOverflowExcludePaths) + return HasAnyPrefix(path, c.productVariables.IntegerOverflowExcludePaths) } func (c *config) CFIDisabledForPath(path string) bool { if c.productVariables.CFIExcludePaths == nil { return false } - return PrefixInList(path, c.productVariables.CFIExcludePaths) + return HasAnyPrefix(path, c.productVariables.CFIExcludePaths) } func (c *config) CFIEnabledForPath(path string) bool { if c.productVariables.CFIIncludePaths == nil { return false } - return PrefixInList(path, c.productVariables.CFIIncludePaths) -} - -func (c *config) XOMDisabledForPath(path string) bool { - if c.productVariables.XOMExcludePaths == nil { - return false - } - return PrefixInList(path, c.productVariables.XOMExcludePaths) + return HasAnyPrefix(path, c.productVariables.CFIIncludePaths) } func (c *config) VendorConfig(name string) VendorConfig { - return vendorConfig(c.productVariables.VendorVars[name]) -} - -func (c vendorConfig) Bool(name string) bool { - v := strings.ToLower(c[name]) - return v == "1" || v == "y" || v == "yes" || v == "on" || v == "true" -} - -func (c vendorConfig) String(name string) string { - return c[name] -} - -func (c vendorConfig) IsSet(name string) bool { - _, ok := c[name] - return ok + return soongconfig.Config(c.productVariables.VendorVars[name]) } func (c *config) NdkAbis() bool { return Bool(c.productVariables.Ndk_abis) } +func (c *config) AmlAbis() bool { + return Bool(c.productVariables.Aml_abis) +} + func (c *config) ExcludeDraftNdkApis() bool { return Bool(c.productVariables.Exclude_draft_ndk_apis) } func (c *config) FlattenApex() bool { - return Bool(c.productVariables.FlattenApex) + return Bool(c.productVariables.Flatten_apex) } func (c *config) EnforceSystemCertificate() bool { return Bool(c.productVariables.EnforceSystemCertificate) } -func (c *config) EnforceSystemCertificateWhitelist() []string { - return c.productVariables.EnforceSystemCertificateWhitelist +func (c *config) EnforceSystemCertificateAllowList() []string { + return c.productVariables.EnforceSystemCertificateAllowList +} + +func (c *config) EnforceProductPartitionInterface() bool { + return Bool(c.productVariables.EnforceProductPartitionInterface) +} + +func (c *config) InstallExtraFlattenedApexes() bool { + return Bool(c.productVariables.InstallExtraFlattenedApexes) } func (c *config) ProductHiddenAPIStubs() []string { @@ -1074,10 +1235,42 @@ func (c *deviceConfig) TargetFSConfigGen() []string { return c.config.productVariables.TargetFSConfigGen } +func (c *config) ProductPublicSepolicyDirs() []string { + return c.productVariables.ProductPublicSepolicyDirs +} + +func (c *config) ProductPrivateSepolicyDirs() []string { + return c.productVariables.ProductPrivateSepolicyDirs +} + +func (c *config) ProductCompatibleProperty() bool { + return Bool(c.productVariables.ProductCompatibleProperty) +} + +func (c *config) MissingUsesLibraries() []string { + return c.productVariables.MissingUsesLibraries +} + +func (c *deviceConfig) BoardVndkRuntimeDisable() bool { + return Bool(c.config.productVariables.BoardVndkRuntimeDisable) +} + func (c *deviceConfig) DeviceArch() string { return String(c.config.productVariables.DeviceArch) } +func (c *deviceConfig) DeviceArchVariant() string { + return String(c.config.productVariables.DeviceArchVariant) +} + func (c *deviceConfig) DeviceSecondaryArch() string { return String(c.config.productVariables.DeviceSecondaryArch) } + +func (c *deviceConfig) DeviceSecondaryArchVariant() string { + return String(c.config.productVariables.DeviceSecondaryArchVariant) +} + +func (c *deviceConfig) BoardUsesRecoveryAsBoot() bool { + return Bool(c.config.productVariables.BoardUsesRecoveryAsBoot) +} diff --git a/android/csuite_config.go b/android/csuite_config.go new file mode 100644 index 000000000..15c518a07 --- /dev/null +++ b/android/csuite_config.go @@ -0,0 +1,70 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "fmt" + "io" +) + +func init() { + RegisterModuleType("csuite_config", CSuiteConfigFactory) +} + +type csuiteConfigProperties struct { + // Override the default (AndroidTest.xml) test manifest file name. + Test_config *string +} + +type CSuiteConfig struct { + ModuleBase + properties csuiteConfigProperties + OutputFilePath OutputPath +} + +func (me *CSuiteConfig) GenerateAndroidBuildActions(ctx ModuleContext) { + me.OutputFilePath = PathForModuleOut(ctx, me.BaseModuleName()).OutputPath +} + +func (me *CSuiteConfig) AndroidMk() AndroidMkData { + androidMkData := AndroidMkData{ + Class: "FAKE", + Include: "$(BUILD_SYSTEM)/suite_host_config.mk", + OutputFile: OptionalPathForPath(me.OutputFilePath), + } + androidMkData.Extra = []AndroidMkExtraFunc{ + func(w io.Writer, outputFile Path) { + if me.properties.Test_config != nil { + fmt.Fprintf(w, "LOCAL_TEST_CONFIG := %s\n", + *me.properties.Test_config) + } + fmt.Fprintln(w, "LOCAL_COMPATIBILITY_SUITE := csuite") + }, + } + return androidMkData +} + +func InitCSuiteConfigModule(me *CSuiteConfig) { + me.AddProperties(&me.properties) +} + +// csuite_config generates an App Compatibility Test Suite (C-Suite) configuration file from the +// <test_config> xml file and stores it in a subdirectory of $(HOST_OUT). +func CSuiteConfigFactory() Module { + module := &CSuiteConfig{} + InitCSuiteConfigModule(module) + InitAndroidArchModule(module, HostSupported, MultilibFirst) + return module +} diff --git a/android/csuite_config_test.go b/android/csuite_config_test.go new file mode 100644 index 000000000..bf1a19a46 --- /dev/null +++ b/android/csuite_config_test.go @@ -0,0 +1,49 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "testing" +) + +func testCSuiteConfig(test *testing.T, bpFileContents string) *TestContext { + config := TestArchConfig(buildDir, nil, bpFileContents, nil) + + ctx := NewTestArchContext() + ctx.RegisterModuleType("csuite_config", CSuiteConfigFactory) + ctx.Register(config) + _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) + FailIfErrored(test, errs) + _, errs = ctx.PrepareBuildActions(config) + FailIfErrored(test, errs) + return ctx +} + +func TestCSuiteConfig(t *testing.T) { + ctx := testCSuiteConfig(t, ` +csuite_config { name: "plain"} +csuite_config { name: "with_manifest", test_config: "manifest.xml" } +`) + + variants := ctx.ModuleVariantsForTests("plain") + if len(variants) > 1 { + t.Errorf("expected 1, got %d", len(variants)) + } + expectedOutputFilename := ctx.ModuleForTests( + "plain", variants[0]).Module().(*CSuiteConfig).OutputFilePath.Base() + if expectedOutputFilename != "plain" { + t.Errorf("expected plain, got %q", expectedOutputFilename) + } +} diff --git a/android/defaults.go b/android/defaults.go index d4fbf487d..81e340e8e 100644 --- a/android/defaults.go +++ b/android/defaults.go @@ -15,6 +15,8 @@ package android import ( + "reflect" + "github.com/google/blueprint" "github.com/google/blueprint/proptools" ) @@ -30,22 +32,53 @@ type defaultsProperties struct { } type DefaultableModuleBase struct { - defaultsProperties defaultsProperties - defaultableProperties []interface{} + defaultsProperties defaultsProperties + defaultableProperties []interface{} + defaultableVariableProperties interface{} + + // The optional hook to call after any defaults have been applied. + hook DefaultableHook } func (d *DefaultableModuleBase) defaults() *defaultsProperties { return &d.defaultsProperties } -func (d *DefaultableModuleBase) setProperties(props []interface{}) { +func (d *DefaultableModuleBase) setProperties(props []interface{}, variableProperties interface{}) { d.defaultableProperties = props + d.defaultableVariableProperties = variableProperties } +func (d *DefaultableModuleBase) SetDefaultableHook(hook DefaultableHook) { + d.hook = hook +} + +func (d *DefaultableModuleBase) callHookIfAvailable(ctx DefaultableHookContext) { + if d.hook != nil { + d.hook(ctx) + } +} + +// Interface that must be supported by any module to which defaults can be applied. type Defaultable interface { + // Get a pointer to the struct containing the Defaults property. defaults() *defaultsProperties - setProperties([]interface{}) + + // Set the property structures into which defaults will be added. + setProperties(props []interface{}, variableProperties interface{}) + + // Apply defaults from the supplied Defaults to the property structures supplied to + // setProperties(...). applyDefaults(TopDownMutatorContext, []Defaults) + + // Set the hook to be called after any defaults have been applied. + // + // Should be used in preference to a AddLoadHook when the behavior of the load + // hook is dependent on properties supplied in the Android.bp file. + SetDefaultableHook(hook DefaultableHook) + + // Call the hook if specified. + callHookIfAvailable(context DefaultableHookContext) } type DefaultableModule interface { @@ -56,42 +89,138 @@ type DefaultableModule interface { var _ Defaultable = (*DefaultableModuleBase)(nil) func InitDefaultableModule(module DefaultableModule) { - module.(Defaultable).setProperties(module.(Module).GetProperties()) + if module.(Module).base().module == nil { + panic("InitAndroidModule must be called before InitDefaultableModule") + } + module.setProperties(module.(Module).GetProperties(), module.(Module).base().variableProperties) module.AddProperties(module.defaults()) } +// A restricted subset of context methods, similar to LoadHookContext. +type DefaultableHookContext interface { + EarlyModuleContext + + CreateModule(ModuleFactory, ...interface{}) Module +} + +type DefaultableHook func(ctx DefaultableHookContext) + +// The Defaults_visibility property. +type DefaultsVisibilityProperties struct { + + // Controls the visibility of the defaults module itself. + Defaults_visibility []string +} + type DefaultsModuleBase struct { DefaultableModuleBase - defaultProperties []interface{} + + // Container for defaults of the common properties + commonProperties commonProperties + + defaultsVisibilityProperties DefaultsVisibilityProperties } +// The common pattern for defaults modules is to register separate instances of +// the xxxProperties structs in the AddProperties calls, rather than reusing the +// ones inherited from Module. +// +// The effect is that e.g. myDefaultsModuleInstance.base().xxxProperties won't +// contain the values that have been set for the defaults module. Rather, to +// retrieve the values it is necessary to iterate over properties(). E.g. to get +// the commonProperties instance that have the real values: +// +// d := myModule.(Defaults) +// for _, props := range d.properties() { +// if cp, ok := props.(*commonProperties); ok { +// ... access property values in cp ... +// } +// } +// +// The rationale is that the properties on a defaults module apply to the +// defaultable modules using it, not to the defaults module itself. E.g. setting +// the "enabled" property false makes inheriting modules disabled by default, +// rather than disabling the defaults module itself. type Defaults interface { Defaultable + + // Although this function is unused it is actually needed to ensure that only modules that embed + // DefaultsModuleBase will type-assert to the Defaults interface. isDefaults() bool + + // Get the structures containing the properties for which defaults can be provided. properties() []interface{} + + productVariableProperties() interface{} + + // Return the defaults common properties. + common() *commonProperties + + // Return the defaults visibility properties. + defaultsVisibility() *DefaultsVisibilityProperties } func (d *DefaultsModuleBase) isDefaults() bool { return true } +type DefaultsModule interface { + Module + Defaults +} + func (d *DefaultsModuleBase) properties() []interface{} { return d.defaultableProperties } -func InitDefaultsModule(module DefaultableModule) { +func (d *DefaultsModuleBase) productVariableProperties() interface{} { + return d.defaultableVariableProperties +} + +func (d *DefaultsModuleBase) common() *commonProperties { + return &d.commonProperties +} + +func (d *DefaultsModuleBase) defaultsVisibility() *DefaultsVisibilityProperties { + return &d.defaultsVisibilityProperties +} + +func (d *DefaultsModuleBase) GenerateAndroidBuildActions(ctx ModuleContext) { +} + +func InitDefaultsModule(module DefaultsModule) { + commonProperties := module.common() + module.AddProperties( &hostAndDeviceProperties{}, - &commonProperties{}, - &variableProperties{}) + commonProperties, + &ApexProperties{}) + initAndroidModuleBase(module) + initProductVariableModule(module) InitArchModule(module) InitDefaultableModule(module) - module.AddProperties(&module.base().nameProperties) + // Add properties that will not have defaults applied to them. + base := module.base() + defaultsVisibility := module.defaultsVisibility() + module.AddProperties(&base.nameProperties, defaultsVisibility) + + // Unlike non-defaults modules the visibility property is not stored in m.base().commonProperties. + // Instead it is stored in a separate instance of commonProperties created above so clear the + // existing list of properties. + clearVisibilityProperties(module) + + // The defaults_visibility property controls the visibility of a defaults module so it must be + // set as the primary property, which also adds it to the list. + setPrimaryVisibilityProperty(module, "defaults_visibility", &defaultsVisibility.Defaults_visibility) + + // The visibility property needs to be checked (but not parsed) by the visibility module during + // its checking phase and parsing phase so add it to the list as a normal property. + AddVisibilityProperty(module, "visibility", &commonProperties.Visibility) - module.base().module = module + base.module = module } var _ Defaults = (*DefaultsModuleBase)(nil) @@ -101,16 +230,57 @@ func (defaultable *DefaultableModuleBase) applyDefaults(ctx TopDownMutatorContex for _, defaults := range defaultsList { for _, prop := range defaultable.defaultableProperties { - for _, def := range defaults.properties() { - if proptools.TypeEqual(prop, def) { - err := proptools.PrependProperties(prop, def, nil) - if err != nil { - if propertyErr, ok := err.(*proptools.ExtendPropertyError); ok { - ctx.PropertyErrorf(propertyErr.Property, "%s", propertyErr.Err.Error()) - } else { - panic(err) - } - } + if prop == defaultable.defaultableVariableProperties { + defaultable.applyDefaultVariableProperties(ctx, defaults, prop) + } else { + defaultable.applyDefaultProperties(ctx, defaults, prop) + } + } + } +} + +// Product variable properties need special handling, the type of the filtered product variable +// property struct may not be identical between the defaults module and the defaultable module. +// Use PrependMatchingProperties to apply whichever properties match. +func (defaultable *DefaultableModuleBase) applyDefaultVariableProperties(ctx TopDownMutatorContext, + defaults Defaults, defaultableProp interface{}) { + if defaultableProp == nil { + return + } + + defaultsProp := defaults.productVariableProperties() + if defaultsProp == nil { + return + } + + dst := []interface{}{ + defaultableProp, + // Put an empty copy of the src properties into dst so that properties in src that are not in dst + // don't cause a "failed to find property to extend" error. + proptools.CloneEmptyProperties(reflect.ValueOf(defaultsProp)).Interface(), + } + + err := proptools.PrependMatchingProperties(dst, defaultsProp, nil) + if err != nil { + if propertyErr, ok := err.(*proptools.ExtendPropertyError); ok { + ctx.PropertyErrorf(propertyErr.Property, "%s", propertyErr.Err.Error()) + } else { + panic(err) + } + } +} + +func (defaultable *DefaultableModuleBase) applyDefaultProperties(ctx TopDownMutatorContext, + defaults Defaults, defaultableProp interface{}) { + + for _, def := range defaults.properties() { + if proptools.TypeEqual(defaultableProp, def) { + err := proptools.PrependProperties(defaultableProp, def, nil) + if err != nil { + if propertyErr, ok := err.(*proptools.ExtendPropertyError); ok { + ctx.PropertyErrorf(propertyErr.Property, "%s", propertyErr.Err.Error()) + } else { + panic(err) } } } @@ -129,25 +299,29 @@ func defaultsDepsMutator(ctx BottomUpMutatorContext) { } func defaultsMutator(ctx TopDownMutatorContext) { - if defaultable, ok := ctx.Module().(Defaultable); ok && len(defaultable.defaults().Defaults) > 0 { - var defaultsList []Defaults - seen := make(map[Defaults]bool) - - ctx.WalkDeps(func(module, parent Module) bool { - if ctx.OtherModuleDependencyTag(module) == DefaultsDepTag { - if defaults, ok := module.(Defaults); ok { - if !seen[defaults] { - seen[defaults] = true - defaultsList = append(defaultsList, defaults) - return len(defaults.defaults().Defaults) > 0 + if defaultable, ok := ctx.Module().(Defaultable); ok { + if len(defaultable.defaults().Defaults) > 0 { + var defaultsList []Defaults + seen := make(map[Defaults]bool) + + ctx.WalkDeps(func(module, parent Module) bool { + if ctx.OtherModuleDependencyTag(module) == DefaultsDepTag { + if defaults, ok := module.(Defaults); ok { + if !seen[defaults] { + seen[defaults] = true + defaultsList = append(defaultsList, defaults) + return len(defaults.defaults().Defaults) > 0 + } + } else { + ctx.PropertyErrorf("defaults", "module %s is not an defaults module", + ctx.OtherModuleName(module)) } - } else { - ctx.PropertyErrorf("defaults", "module %s is not an defaults module", - ctx.OtherModuleName(module)) } - } - return false - }) - defaultable.applyDefaults(ctx, defaultsList) + return false + }) + defaultable.applyDefaults(ctx, defaultsList) + } + + defaultable.callHookIfAvailable(ctx) } } diff --git a/android/defaults_test.go b/android/defaults_test.go new file mode 100644 index 000000000..d096b2f92 --- /dev/null +++ b/android/defaults_test.go @@ -0,0 +1,156 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "reflect" + "testing" + + "github.com/google/blueprint/proptools" +) + +type defaultsTestProperties struct { + Foo []string +} + +type defaultsTestModule struct { + ModuleBase + DefaultableModuleBase + properties defaultsTestProperties +} + +func (d *defaultsTestModule) GenerateAndroidBuildActions(ctx ModuleContext) { + ctx.Build(pctx, BuildParams{ + Rule: Touch, + Output: PathForModuleOut(ctx, "out"), + }) +} + +func defaultsTestModuleFactory() Module { + module := &defaultsTestModule{} + module.AddProperties(&module.properties) + InitAndroidModule(module) + InitDefaultableModule(module) + return module +} + +type defaultsTestDefaults struct { + ModuleBase + DefaultsModuleBase +} + +func defaultsTestDefaultsFactory() Module { + defaults := &defaultsTestDefaults{} + defaults.AddProperties(&defaultsTestProperties{}) + InitDefaultsModule(defaults) + return defaults +} + +func TestDefaults(t *testing.T) { + bp := ` + defaults { + name: "transitive", + foo: ["transitive"], + } + + defaults { + name: "defaults", + defaults: ["transitive"], + foo: ["defaults"], + } + + test { + name: "foo", + defaults: ["defaults"], + foo: ["module"], + } + ` + + config := TestConfig(buildDir, nil, bp, nil) + + ctx := NewTestContext() + + ctx.RegisterModuleType("test", defaultsTestModuleFactory) + ctx.RegisterModuleType("defaults", defaultsTestDefaultsFactory) + + ctx.PreArchMutators(RegisterDefaultsPreArchMutators) + + ctx.Register(config) + + _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) + FailIfErrored(t, errs) + _, errs = ctx.PrepareBuildActions(config) + FailIfErrored(t, errs) + + foo := ctx.ModuleForTests("foo", "").Module().(*defaultsTestModule) + + if g, w := foo.properties.Foo, []string{"transitive", "defaults", "module"}; !reflect.DeepEqual(g, w) { + t.Errorf("expected foo %q, got %q", w, g) + } +} + +func TestDefaultsAllowMissingDependencies(t *testing.T) { + bp := ` + defaults { + name: "defaults", + defaults: ["missing"], + foo: ["defaults"], + } + + test { + name: "missing_defaults", + defaults: ["missing"], + foo: ["module"], + } + + test { + name: "missing_transitive_defaults", + defaults: ["defaults"], + foo: ["module"], + } + ` + + config := TestConfig(buildDir, nil, bp, nil) + config.TestProductVariables.Allow_missing_dependencies = proptools.BoolPtr(true) + + ctx := NewTestContext() + ctx.SetAllowMissingDependencies(true) + + ctx.RegisterModuleType("test", defaultsTestModuleFactory) + ctx.RegisterModuleType("defaults", defaultsTestDefaultsFactory) + + ctx.PreArchMutators(RegisterDefaultsPreArchMutators) + + ctx.Register(config) + + _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) + FailIfErrored(t, errs) + _, errs = ctx.PrepareBuildActions(config) + FailIfErrored(t, errs) + + missingDefaults := ctx.ModuleForTests("missing_defaults", "").Output("out") + missingTransitiveDefaults := ctx.ModuleForTests("missing_transitive_defaults", "").Output("out") + + if missingDefaults.Rule != ErrorRule { + t.Errorf("expected missing_defaults rule to be ErrorRule, got %#v", missingDefaults.Rule) + } + + if g, w := missingDefaults.Args["error"], "module missing_defaults missing dependencies: missing\n"; g != w { + t.Errorf("want error %q, got %q", w, g) + } + + // TODO: missing transitive defaults is currently not handled + _ = missingTransitiveDefaults +} diff --git a/android/defs.go b/android/defs.go index 4890c6618..45522246f 100644 --- a/android/defs.go +++ b/android/defs.go @@ -99,6 +99,12 @@ var ( // Used only when USE_GOMA=true is set, to restrict non-goma jobs to the local parallelism value localPool = blueprint.NewBuiltinPool("local_pool") + + // Used only by RuleBuilder to identify remoteable rules. Does not actually get created in ninja. + remotePool = blueprint.NewBuiltinPool("remote_pool") + + // Used for processes that need significant RAM to ensure there are not too many running in parallel. + highmemPool = blueprint.NewBuiltinPool("highmem_pool") ) func init() { diff --git a/android/depset.go b/android/depset.go new file mode 100644 index 000000000..f70709423 --- /dev/null +++ b/android/depset.go @@ -0,0 +1,190 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import "fmt" + +// DepSet is designed to be conceptually compatible with Bazel's depsets: +// https://docs.bazel.build/versions/master/skylark/depsets.html + +// A DepSet efficiently stores Paths from transitive dependencies without copying. It is stored +// as a DAG of DepSet nodes, each of which has some direct contents and a list of dependency +// DepSet nodes. +// +// A DepSet has an order that will be used to walk the DAG when ToList() is called. The order +// can be POSTORDER, PREORDER, or TOPOLOGICAL. POSTORDER and PREORDER orders return a postordered +// or preordered left to right flattened list. TOPOLOGICAL returns a list that guarantees that +// elements of children are listed after all of their parents (unless there are duplicate direct +// elements in the DepSet or any of its transitive dependencies, in which case the ordering of the +// duplicated element is not guaranteed). +// +// A DepSet is created by NewDepSet or NewDepSetBuilder.Build from the Paths for direct contents +// and the *DepSets of dependencies. A DepSet is immutable once created. +type DepSet struct { + preorder bool + reverse bool + order DepSetOrder + direct Paths + transitive []*DepSet +} + +// DepSetBuilder is used to create an immutable DepSet. +type DepSetBuilder struct { + order DepSetOrder + direct Paths + transitive []*DepSet +} + +type DepSetOrder int + +const ( + PREORDER DepSetOrder = iota + POSTORDER + TOPOLOGICAL +) + +func (o DepSetOrder) String() string { + switch o { + case PREORDER: + return "PREORDER" + case POSTORDER: + return "POSTORDER" + case TOPOLOGICAL: + return "TOPOLOGICAL" + default: + panic(fmt.Errorf("Invalid DepSetOrder %d", o)) + } +} + +// NewDepSet returns an immutable DepSet with the given order, direct and transitive contents. +func NewDepSet(order DepSetOrder, direct Paths, transitive []*DepSet) *DepSet { + var directCopy Paths + var transitiveCopy []*DepSet + if order == TOPOLOGICAL { + directCopy = ReversePaths(direct) + transitiveCopy = reverseDepSets(transitive) + } else { + // Use copy instead of append(nil, ...) to make a slice that is exactly the size of the input + // slice. The DepSet is immutable, there is no need for additional capacity. + directCopy = make(Paths, len(direct)) + copy(directCopy, direct) + transitiveCopy = make([]*DepSet, len(transitive)) + copy(transitiveCopy, transitive) + } + + for _, dep := range transitive { + if dep.order != order { + panic(fmt.Errorf("incompatible order, new DepSet is %s but transitive DepSet is %s", + order, dep.order)) + } + } + + return &DepSet{ + preorder: order == PREORDER, + reverse: order == TOPOLOGICAL, + order: order, + direct: directCopy, + transitive: transitiveCopy, + } +} + +// NewDepSetBuilder returns a DepSetBuilder to create an immutable DepSet with the given order. +func NewDepSetBuilder(order DepSetOrder) *DepSetBuilder { + return &DepSetBuilder{order: order} +} + +// Direct adds direct contents to the DepSet being built by a DepSetBuilder. Newly added direct +// contents are to the right of any existing direct contents. +func (b *DepSetBuilder) Direct(direct ...Path) *DepSetBuilder { + b.direct = append(b.direct, direct...) + return b +} + +// Transitive adds transitive contents to the DepSet being built by a DepSetBuilder. Newly added +// transitive contents are to the right of any existing transitive contents. +func (b *DepSetBuilder) Transitive(transitive ...*DepSet) *DepSetBuilder { + b.transitive = append(b.transitive, transitive...) + return b +} + +// Returns the DepSet being built by this DepSetBuilder. The DepSetBuilder retains its contents +// for creating more DepSets. +func (b *DepSetBuilder) Build() *DepSet { + return NewDepSet(b.order, b.direct, b.transitive) +} + +// walk calls the visit method in depth-first order on a DepSet, preordered if d.preorder is set, +// otherwise postordered. +func (d *DepSet) walk(visit func(Paths)) { + visited := make(map[*DepSet]bool) + + var dfs func(d *DepSet) + dfs = func(d *DepSet) { + visited[d] = true + if d.preorder { + visit(d.direct) + } + for _, dep := range d.transitive { + if !visited[dep] { + dfs(dep) + } + } + + if !d.preorder { + visit(d.direct) + } + } + + dfs(d) +} + +// ToList returns the DepSet flattened to a list. The order in the list is based on the order +// of the DepSet. POSTORDER and PREORDER orders return a postordered or preordered left to right +// flattened list. TOPOLOGICAL returns a list that guarantees that elements of children are listed +// after all of their parents (unless there are duplicate direct elements in the DepSet or any of +// its transitive dependencies, in which case the ordering of the duplicated element is not +// guaranteed). +func (d *DepSet) ToList() Paths { + var list Paths + d.walk(func(paths Paths) { + list = append(list, paths...) + }) + list = FirstUniquePaths(list) + if d.reverse { + reversePathsInPlace(list) + } + return list +} + +// ToSortedList returns the direct and transitive contents of a DepSet in lexically sorted order +// with duplicates removed. +func (d *DepSet) ToSortedList() Paths { + list := d.ToList() + return SortedUniquePaths(list) +} + +func reversePathsInPlace(list Paths) { + for i, j := 0, len(list)-1; i < j; i, j = i+1, j-1 { + list[i], list[j] = list[j], list[i] + } +} + +func reverseDepSets(list []*DepSet) []*DepSet { + ret := make([]*DepSet, len(list)) + for i := range list { + ret[i] = list[len(list)-1-i] + } + return ret +} diff --git a/android/depset_test.go b/android/depset_test.go new file mode 100644 index 000000000..c328127b1 --- /dev/null +++ b/android/depset_test.go @@ -0,0 +1,304 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +func ExampleDepSet_ToList_postordered() { + a := NewDepSetBuilder(POSTORDER).Direct(PathForTesting("a")).Build() + b := NewDepSetBuilder(POSTORDER).Direct(PathForTesting("b")).Transitive(a).Build() + c := NewDepSetBuilder(POSTORDER).Direct(PathForTesting("c")).Transitive(a).Build() + d := NewDepSetBuilder(POSTORDER).Direct(PathForTesting("d")).Transitive(b, c).Build() + + fmt.Println(d.ToList().Strings()) + // Output: [a b c d] +} + +func ExampleDepSet_ToList_preordered() { + a := NewDepSetBuilder(PREORDER).Direct(PathForTesting("a")).Build() + b := NewDepSetBuilder(PREORDER).Direct(PathForTesting("b")).Transitive(a).Build() + c := NewDepSetBuilder(PREORDER).Direct(PathForTesting("c")).Transitive(a).Build() + d := NewDepSetBuilder(PREORDER).Direct(PathForTesting("d")).Transitive(b, c).Build() + + fmt.Println(d.ToList().Strings()) + // Output: [d b a c] +} + +func ExampleDepSet_ToList_topological() { + a := NewDepSetBuilder(TOPOLOGICAL).Direct(PathForTesting("a")).Build() + b := NewDepSetBuilder(TOPOLOGICAL).Direct(PathForTesting("b")).Transitive(a).Build() + c := NewDepSetBuilder(TOPOLOGICAL).Direct(PathForTesting("c")).Transitive(a).Build() + d := NewDepSetBuilder(TOPOLOGICAL).Direct(PathForTesting("d")).Transitive(b, c).Build() + + fmt.Println(d.ToList().Strings()) + // Output: [d b c a] +} + +func ExampleDepSet_ToSortedList() { + a := NewDepSetBuilder(POSTORDER).Direct(PathForTesting("a")).Build() + b := NewDepSetBuilder(POSTORDER).Direct(PathForTesting("b")).Transitive(a).Build() + c := NewDepSetBuilder(POSTORDER).Direct(PathForTesting("c")).Transitive(a).Build() + d := NewDepSetBuilder(POSTORDER).Direct(PathForTesting("d")).Transitive(b, c).Build() + + fmt.Println(d.ToSortedList().Strings()) + // Output: [a b c d] +} + +// Tests based on Bazel's ExpanderTestBase.java to ensure compatibility +// https://github.com/bazelbuild/bazel/blob/master/src/test/java/com/google/devtools/build/lib/collect/nestedset/ExpanderTestBase.java +func TestDepSet(t *testing.T) { + a := PathForTesting("a") + b := PathForTesting("b") + c := PathForTesting("c") + c2 := PathForTesting("c2") + d := PathForTesting("d") + e := PathForTesting("e") + + tests := []struct { + name string + depSet func(t *testing.T, order DepSetOrder) *DepSet + postorder, preorder, topological []string + }{ + { + name: "simple", + depSet: func(t *testing.T, order DepSetOrder) *DepSet { + return NewDepSet(order, Paths{c, a, b}, nil) + }, + postorder: []string{"c", "a", "b"}, + preorder: []string{"c", "a", "b"}, + topological: []string{"c", "a", "b"}, + }, + { + name: "simpleNoDuplicates", + depSet: func(t *testing.T, order DepSetOrder) *DepSet { + return NewDepSet(order, Paths{c, a, a, a, b}, nil) + }, + postorder: []string{"c", "a", "b"}, + preorder: []string{"c", "a", "b"}, + topological: []string{"c", "a", "b"}, + }, + { + name: "nesting", + depSet: func(t *testing.T, order DepSetOrder) *DepSet { + subset := NewDepSet(order, Paths{c, a, e}, nil) + return NewDepSet(order, Paths{b, d}, []*DepSet{subset}) + }, + postorder: []string{"c", "a", "e", "b", "d"}, + preorder: []string{"b", "d", "c", "a", "e"}, + topological: []string{"b", "d", "c", "a", "e"}, + }, + { + name: "builderReuse", + depSet: func(t *testing.T, order DepSetOrder) *DepSet { + assertEquals := func(t *testing.T, w, g Paths) { + if !reflect.DeepEqual(w, g) { + t.Errorf("want %q, got %q", w, g) + } + } + builder := NewDepSetBuilder(order) + assertEquals(t, nil, builder.Build().ToList()) + + builder.Direct(b) + assertEquals(t, Paths{b}, builder.Build().ToList()) + + builder.Direct(d) + assertEquals(t, Paths{b, d}, builder.Build().ToList()) + + child := NewDepSetBuilder(order).Direct(c, a, e).Build() + builder.Transitive(child) + return builder.Build() + }, + postorder: []string{"c", "a", "e", "b", "d"}, + preorder: []string{"b", "d", "c", "a", "e"}, + topological: []string{"b", "d", "c", "a", "e"}, + }, + { + name: "builderChaining", + depSet: func(t *testing.T, order DepSetOrder) *DepSet { + return NewDepSetBuilder(order).Direct(b).Direct(d). + Transitive(NewDepSetBuilder(order).Direct(c, a, e).Build()).Build() + }, + postorder: []string{"c", "a", "e", "b", "d"}, + preorder: []string{"b", "d", "c", "a", "e"}, + topological: []string{"b", "d", "c", "a", "e"}, + }, + { + name: "transitiveDepsHandledSeparately", + depSet: func(t *testing.T, order DepSetOrder) *DepSet { + subset := NewDepSetBuilder(order).Direct(c, a, e).Build() + builder := NewDepSetBuilder(order) + // The fact that we add the transitive subset between the Direct(b) and Direct(d) + // calls should not change the result. + builder.Direct(b) + builder.Transitive(subset) + builder.Direct(d) + return builder.Build() + }, + postorder: []string{"c", "a", "e", "b", "d"}, + preorder: []string{"b", "d", "c", "a", "e"}, + topological: []string{"b", "d", "c", "a", "e"}, + }, + { + name: "nestingNoDuplicates", + depSet: func(t *testing.T, order DepSetOrder) *DepSet { + subset := NewDepSetBuilder(order).Direct(c, a, e).Build() + return NewDepSetBuilder(order).Direct(b, d, e).Transitive(subset).Build() + }, + postorder: []string{"c", "a", "e", "b", "d"}, + preorder: []string{"b", "d", "e", "c", "a"}, + topological: []string{"b", "d", "c", "a", "e"}, + }, + { + name: "chain", + depSet: func(t *testing.T, order DepSetOrder) *DepSet { + c := NewDepSetBuilder(order).Direct(c).Build() + b := NewDepSetBuilder(order).Direct(b).Transitive(c).Build() + a := NewDepSetBuilder(order).Direct(a).Transitive(b).Build() + + return a + }, + postorder: []string{"c", "b", "a"}, + preorder: []string{"a", "b", "c"}, + topological: []string{"a", "b", "c"}, + }, + { + name: "diamond", + depSet: func(t *testing.T, order DepSetOrder) *DepSet { + d := NewDepSetBuilder(order).Direct(d).Build() + c := NewDepSetBuilder(order).Direct(c).Transitive(d).Build() + b := NewDepSetBuilder(order).Direct(b).Transitive(d).Build() + a := NewDepSetBuilder(order).Direct(a).Transitive(b).Transitive(c).Build() + + return a + }, + postorder: []string{"d", "b", "c", "a"}, + preorder: []string{"a", "b", "d", "c"}, + topological: []string{"a", "b", "c", "d"}, + }, + { + name: "extendedDiamond", + depSet: func(t *testing.T, order DepSetOrder) *DepSet { + d := NewDepSetBuilder(order).Direct(d).Build() + e := NewDepSetBuilder(order).Direct(e).Build() + b := NewDepSetBuilder(order).Direct(b).Transitive(d).Transitive(e).Build() + c := NewDepSetBuilder(order).Direct(c).Transitive(e).Transitive(d).Build() + a := NewDepSetBuilder(order).Direct(a).Transitive(b).Transitive(c).Build() + return a + }, + postorder: []string{"d", "e", "b", "c", "a"}, + preorder: []string{"a", "b", "d", "e", "c"}, + topological: []string{"a", "b", "c", "e", "d"}, + }, + { + name: "extendedDiamondRightArm", + depSet: func(t *testing.T, order DepSetOrder) *DepSet { + d := NewDepSetBuilder(order).Direct(d).Build() + e := NewDepSetBuilder(order).Direct(e).Build() + b := NewDepSetBuilder(order).Direct(b).Transitive(d).Transitive(e).Build() + c2 := NewDepSetBuilder(order).Direct(c2).Transitive(e).Transitive(d).Build() + c := NewDepSetBuilder(order).Direct(c).Transitive(c2).Build() + a := NewDepSetBuilder(order).Direct(a).Transitive(b).Transitive(c).Build() + return a + }, + postorder: []string{"d", "e", "b", "c2", "c", "a"}, + preorder: []string{"a", "b", "d", "e", "c", "c2"}, + topological: []string{"a", "b", "c", "c2", "e", "d"}, + }, + { + name: "orderConflict", + depSet: func(t *testing.T, order DepSetOrder) *DepSet { + child1 := NewDepSetBuilder(order).Direct(a, b).Build() + child2 := NewDepSetBuilder(order).Direct(b, a).Build() + parent := NewDepSetBuilder(order).Transitive(child1).Transitive(child2).Build() + return parent + }, + postorder: []string{"a", "b"}, + preorder: []string{"a", "b"}, + topological: []string{"b", "a"}, + }, + { + name: "orderConflictNested", + depSet: func(t *testing.T, order DepSetOrder) *DepSet { + a := NewDepSetBuilder(order).Direct(a).Build() + b := NewDepSetBuilder(order).Direct(b).Build() + child1 := NewDepSetBuilder(order).Transitive(a).Transitive(b).Build() + child2 := NewDepSetBuilder(order).Transitive(b).Transitive(a).Build() + parent := NewDepSetBuilder(order).Transitive(child1).Transitive(child2).Build() + return parent + }, + postorder: []string{"a", "b"}, + preorder: []string{"a", "b"}, + topological: []string{"b", "a"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Run("postorder", func(t *testing.T) { + depSet := tt.depSet(t, POSTORDER) + if g, w := depSet.ToList().Strings(), tt.postorder; !reflect.DeepEqual(g, w) { + t.Errorf("expected ToList() = %q, got %q", w, g) + } + }) + t.Run("preorder", func(t *testing.T) { + depSet := tt.depSet(t, PREORDER) + if g, w := depSet.ToList().Strings(), tt.preorder; !reflect.DeepEqual(g, w) { + t.Errorf("expected ToList() = %q, got %q", w, g) + } + }) + t.Run("topological", func(t *testing.T) { + depSet := tt.depSet(t, TOPOLOGICAL) + if g, w := depSet.ToList().Strings(), tt.topological; !reflect.DeepEqual(g, w) { + t.Errorf("expected ToList() = %q, got %q", w, g) + } + }) + }) + } +} + +func TestDepSetInvalidOrder(t *testing.T) { + orders := []DepSetOrder{POSTORDER, PREORDER, TOPOLOGICAL} + + run := func(t *testing.T, order1, order2 DepSetOrder) { + defer func() { + if r := recover(); r != nil { + if err, ok := r.(error); !ok { + t.Fatalf("expected panic error, got %v", err) + } else if !strings.Contains(err.Error(), "incompatible order") { + t.Fatalf("expected incompatible order error, got %v", err) + } + } + }() + NewDepSet(order1, nil, []*DepSet{NewDepSet(order2, nil, nil)}) + t.Fatal("expected panic") + } + + for _, order1 := range orders { + t.Run(order1.String(), func(t *testing.T) { + for _, order2 := range orders { + t.Run(order2.String(), func(t *testing.T) { + if order1 != order2 { + run(t, order1, order2) + } + }) + } + }) + } +} diff --git a/android/env.go b/android/env.go index 469dfffed..46bd3d6c5 100644 --- a/android/env.go +++ b/android/env.go @@ -16,6 +16,7 @@ package android import ( "os" + "os/exec" "strings" "android/soong/env" @@ -29,8 +30,16 @@ import ( // a manifest regeneration. var originalEnv map[string]string +var SoongDelveListen string +var SoongDelvePath string func init() { + // Delve support needs to read this environment variable very early, before NewConfig has created a way to + // access originalEnv with dependencies. Store the value where soong_build can find it, it will manually + // ensure the dependencies are created. + SoongDelveListen = os.Getenv("SOONG_DELVE") + SoongDelvePath, _ = exec.LookPath("dlv") + originalEnv = make(map[string]string) for _, env := range os.Environ() { idx := strings.IndexRune(env, '=') @@ -38,9 +47,22 @@ func init() { originalEnv[env[:idx]] = env[idx+1:] } } + // Clear the environment to prevent use of os.Getenv(), which would not provide dependencies on environment + // variable values. The environment is available through ctx.Config().Getenv, ctx.Config().IsEnvTrue, etc. os.Clearenv() } +// getenv checks either os.Getenv or originalEnv so that it works before or after the init() +// function above. It doesn't add any dependencies on the environment variable, so it should +// only be used for values that won't change. For values that might change use ctx.Config().Getenv. +func getenv(key string) string { + if originalEnv == nil { + return os.Getenv(key) + } else { + return originalEnv[key] + } +} + func EnvSingleton() Singleton { return &envSingleton{} } @@ -55,7 +77,12 @@ func (c *envSingleton) GenerateBuildActions(ctx SingletonContext) { return } - err := env.WriteEnvFile(envFile.String(), envDeps) + data, err := env.EnvFileContents(envDeps) + if err != nil { + ctx.Errorf(err.Error()) + } + + err = WriteFileToOutputDir(envFile, data, 0666) if err != nil { ctx.Errorf(err.Error()) } diff --git a/android/expand.go b/android/expand.go index 527c4ac6d..67fb4ee6e 100644 --- a/android/expand.go +++ b/android/expand.go @@ -18,12 +18,30 @@ import ( "fmt" "strings" "unicode" + + "github.com/google/blueprint/proptools" ) +// ExpandNinjaEscaped substitutes $() variables in a string +// $(var) is passed to mapping(var), which should return the expanded value, a bool for whether the result should +// be left unescaped when using in a ninja value (generally false, true if the expanded value is a ninja variable like +// '${in}'), and an error. +// $$ is converted to $, which is escaped back to $$. +func ExpandNinjaEscaped(s string, mapping func(string) (string, bool, error)) (string, error) { + return expand(s, true, mapping) +} + // Expand substitutes $() variables in a string -// $(var) is passed to Expander(var) -// $$ is converted to $ +// $(var) is passed to mapping(var), which should return the expanded value and an error. +// $$ is converted to $. func Expand(s string, mapping func(string) (string, error)) (string, error) { + return expand(s, false, func(s string) (string, bool, error) { + s, err := mapping(s) + return s, false, err + }) +} + +func expand(s string, ninjaEscape bool, mapping func(string) (string, bool, error)) (string, error) { // based on os.Expand buf := make([]byte, 0, 2*len(s)) i := 0 @@ -33,10 +51,13 @@ func Expand(s string, mapping func(string) (string, error)) (string, error) { return "", fmt.Errorf("expected character after '$'") } buf = append(buf, s[i:j]...) - value, w, err := getMapping(s[j+1:], mapping) + value, ninjaVariable, w, err := getMapping(s[j+1:], mapping) if err != nil { return "", err } + if !ninjaVariable && ninjaEscape { + value = proptools.NinjaEscape(value) + } buf = append(buf, value...) j += w i = j + 1 @@ -45,26 +66,26 @@ func Expand(s string, mapping func(string) (string, error)) (string, error) { return string(buf) + s[i:], nil } -func getMapping(s string, mapping func(string) (string, error)) (string, int, error) { +func getMapping(s string, mapping func(string) (string, bool, error)) (string, bool, int, error) { switch s[0] { case '(': // Scan to closing brace for i := 1; i < len(s); i++ { if s[i] == ')' { - ret, err := mapping(strings.TrimSpace(s[1:i])) - return ret, i + 1, err + ret, ninjaVariable, err := mapping(strings.TrimSpace(s[1:i])) + return ret, ninjaVariable, i + 1, err } } - return "", len(s), fmt.Errorf("missing )") + return "", false, len(s), fmt.Errorf("missing )") case '$': - return "$$", 1, nil + return "$", false, 1, nil default: i := strings.IndexFunc(s, unicode.IsSpace) if i == 0 { - return "", 0, fmt.Errorf("unexpected character '%c' after '$'", s[0]) + return "", false, 0, fmt.Errorf("unexpected character '%c' after '$'", s[0]) } else if i == -1 { i = len(s) } - return "", 0, fmt.Errorf("expected '(' after '$', did you mean $(%s)?", s[:i]) + return "", false, 0, fmt.Errorf("expected '(' after '$', did you mean $(%s)?", s[:i]) } } diff --git a/android/expand_test.go b/android/expand_test.go index 128de8a4e..12179eddf 100644 --- a/android/expand_test.go +++ b/android/expand_test.go @@ -20,88 +20,111 @@ import ( ) var vars = map[string]string{ - "var1": "abc", - "var2": "", - "var3": "def", - "💩": "😃", + "var1": "abc", + "var2": "", + "var3": "def", + "💩": "😃", + "escape": "${in}", } -func expander(s string) (string, error) { +func expander(s string) (string, bool, error) { if val, ok := vars[s]; ok { - return val, nil + return val, s == "escape", nil } else { - return "", fmt.Errorf("unknown variable %q", s) + return "", false, fmt.Errorf("unknown variable %q", s) } } var expandTestCases = []struct { - in string - out string - err bool + in string + out string + out_escaped string + err bool }{ { - in: "$(var1)", - out: "abc", + in: "$(var1)", + out: "abc", + out_escaped: "abc", }, { - in: "$( var1 )", - out: "abc", + in: "$( var1 )", + out: "abc", + out_escaped: "abc", }, { - in: "def$(var1)", - out: "defabc", + in: "def$(var1)", + out: "defabc", + out_escaped: "defabc", }, { - in: "$(var1)def", - out: "abcdef", + in: "$(var1)def", + out: "abcdef", + out_escaped: "abcdef", }, { - in: "def$(var1)def", - out: "defabcdef", + in: "def$(var1)def", + out: "defabcdef", + out_escaped: "defabcdef", }, { - in: "$(var2)", - out: "", + in: "$(var2)", + out: "", + out_escaped: "", }, { - in: "def$(var2)", - out: "def", + in: "def$(var2)", + out: "def", + out_escaped: "def", }, { - in: "$(var2)def", - out: "def", + in: "$(var2)def", + out: "def", + out_escaped: "def", }, { - in: "def$(var2)def", - out: "defdef", + in: "def$(var2)def", + out: "defdef", + out_escaped: "defdef", }, { - in: "$(var1)$(var3)", - out: "abcdef", + in: "$(var1)$(var3)", + out: "abcdef", + out_escaped: "abcdef", }, { - in: "$(var1)g$(var3)", - out: "abcgdef", + in: "$(var1)g$(var3)", + out: "abcgdef", + out_escaped: "abcgdef", }, { - in: "$$", - out: "$$", + in: "$$", + out: "$", + out_escaped: "$$", }, { - in: "$$(var1)", - out: "$$(var1)", + in: "$$(var1)", + out: "$(var1)", + out_escaped: "$$(var1)", }, { - in: "$$$(var1)", - out: "$$abc", + in: "$$$(var1)", + out: "$abc", + out_escaped: "$$abc", }, { - in: "$(var1)$$", - out: "abc$$", + in: "$(var1)$$", + out: "abc$", + out_escaped: "abc$$", }, { - in: "$(💩)", - out: "😃", + in: "$(💩)", + out: "😃", + out_escaped: "😃", + }, + { + in: "$$a$(escape)$$b", + out: "$a${in}$b", + out_escaped: "$$a${in}$$b", }, // Errors @@ -141,7 +164,10 @@ var expandTestCases = []struct { func TestExpand(t *testing.T) { for _, test := range expandTestCases { - got, err := Expand(test.in, expander) + got, err := Expand(test.in, func(s string) (string, error) { + s, _, err := expander(s) + return s, err + }) if err != nil && !test.err { t.Errorf("%q: unexpected error %s", test.in, err.Error()) } else if err == nil && test.err { @@ -151,3 +177,16 @@ func TestExpand(t *testing.T) { } } } + +func TestExpandNinjaEscaped(t *testing.T) { + for _, test := range expandTestCases { + got, err := ExpandNinjaEscaped(test.in, expander) + if err != nil && !test.err { + t.Errorf("%q: unexpected error %s", test.in, err.Error()) + } else if err == nil && test.err { + t.Errorf("%q: expected error, got %q", test.in, got) + } else if !test.err && got != test.out_escaped { + t.Errorf("%q: expected %q, got %q", test.in, test.out, got) + } + } +} diff --git a/android/hooks.go b/android/hooks.go index 6b2468dc5..f43a007a0 100644 --- a/android/hooks.go +++ b/android/hooks.go @@ -15,7 +15,10 @@ package android import ( + "reflect" + "github.com/google/blueprint" + "github.com/google/blueprint/proptools" ) // This file implements hooks that external module types can use to inject logic into existing @@ -26,56 +29,98 @@ import ( // before the module has been split into architecture variants, and before defaults modules have // been applied. type LoadHookContext interface { - // TODO: a new context that includes Config() but not Target(), etc.? - BaseContext - AppendProperties(...interface{}) - PrependProperties(...interface{}) - CreateModule(blueprint.ModuleFactory, ...interface{}) -} + EarlyModuleContext -// Arch hooks are run after the module has been split into architecture variants, and can be used -// to add architecture-specific properties. -type ArchHookContext interface { - BaseContext AppendProperties(...interface{}) PrependProperties(...interface{}) + CreateModule(ModuleFactory, ...interface{}) Module + + registerScopedModuleType(name string, factory blueprint.ModuleFactory) + moduleFactories() map[string]blueprint.ModuleFactory } +// Add a hook that will be called once the module has been loaded, i.e. its +// properties have been initialized from the Android.bp file. +// +// Consider using SetDefaultableHook to register a hook for any module that implements +// DefaultableModule as the hook is called after any defaults have been applied to the +// module which could reduce duplication and make it easier to use. func AddLoadHook(m blueprint.Module, hook func(LoadHookContext)) { - h := &m.(Module).base().hooks - h.load = append(h.load, hook) + blueprint.AddLoadHook(m, func(ctx blueprint.LoadHookContext) { + actx := &loadHookContext{ + earlyModuleContext: m.(Module).base().earlyModuleContextFactory(ctx), + bp: ctx, + } + hook(actx) + }) } -func AddArchHook(m blueprint.Module, hook func(ArchHookContext)) { - h := &m.(Module).base().hooks - h.arch = append(h.arch, hook) +type loadHookContext struct { + earlyModuleContext + bp blueprint.LoadHookContext + module Module } -func (x *hooks) runLoadHooks(ctx LoadHookContext, m *ModuleBase) { - if len(x.load) > 0 { - for _, x := range x.load { - x(ctx) - if ctx.Failed() { - return +func (l *loadHookContext) moduleFactories() map[string]blueprint.ModuleFactory { + return l.bp.ModuleFactories() +} + +func (l *loadHookContext) AppendProperties(props ...interface{}) { + for _, p := range props { + err := proptools.AppendMatchingProperties(l.Module().base().customizableProperties, + p, nil) + if err != nil { + if propertyErr, ok := err.(*proptools.ExtendPropertyError); ok { + l.PropertyErrorf(propertyErr.Property, "%s", propertyErr.Err.Error()) + } else { + panic(err) } } } } -func (x *hooks) runArchHooks(ctx ArchHookContext, m *ModuleBase) { - if len(x.arch) > 0 { - for _, x := range x.arch { - x(ctx) - if ctx.Failed() { - return +func (l *loadHookContext) PrependProperties(props ...interface{}) { + for _, p := range props { + err := proptools.PrependMatchingProperties(l.Module().base().customizableProperties, + p, nil) + if err != nil { + if propertyErr, ok := err.(*proptools.ExtendPropertyError); ok { + l.PropertyErrorf(propertyErr.Property, "%s", propertyErr.Err.Error()) + } else { + panic(err) } } } } +func (l *loadHookContext) CreateModule(factory ModuleFactory, props ...interface{}) Module { + inherited := []interface{}{&l.Module().base().commonProperties} + module := l.bp.CreateModule(ModuleFactoryAdaptor(factory), append(inherited, props...)...).(Module) + + if l.Module().base().variableProperties != nil && module.base().variableProperties != nil { + src := l.Module().base().variableProperties + dst := []interface{}{ + module.base().variableProperties, + // Put an empty copy of the src properties into dst so that properties in src that are not in dst + // don't cause a "failed to find property to extend" error. + proptools.CloneEmptyProperties(reflect.ValueOf(src)).Interface(), + } + err := proptools.AppendMatchingProperties(dst, src, nil) + if err != nil { + panic(err) + } + } + + return module +} + +func (l *loadHookContext) registerScopedModuleType(name string, factory blueprint.ModuleFactory) { + l.bp.RegisterScopedModuleType(name, factory) +} + type InstallHookContext interface { ModuleContext - Path() OutputPath + Path() InstallPath Symlink() bool } @@ -89,11 +134,11 @@ func AddInstallHook(m blueprint.Module, hook func(InstallHookContext)) { type installHookContext struct { ModuleContext - path OutputPath + path InstallPath symlink bool } -func (x *installHookContext) Path() OutputPath { +func (x *installHookContext) Path() InstallPath { return x.path } @@ -101,7 +146,7 @@ func (x *installHookContext) Symlink() bool { return x.symlink } -func (x *hooks) runInstallHooks(ctx ModuleContext, path OutputPath, symlink bool) { +func (x *hooks) runInstallHooks(ctx ModuleContext, path InstallPath, symlink bool) { if len(x.install) > 0 { mctx := &installHookContext{ ModuleContext: ctx, @@ -118,25 +163,5 @@ func (x *hooks) runInstallHooks(ctx ModuleContext, path OutputPath, symlink bool } type hooks struct { - load []func(LoadHookContext) - arch []func(ArchHookContext) install []func(InstallHookContext) } - -func LoadHookMutator(ctx TopDownMutatorContext) { - if m, ok := ctx.Module().(Module); ok { - // Cast through *androidTopDownMutatorContext because AppendProperties is implemented - // on *androidTopDownMutatorContext but not exposed through TopDownMutatorContext - var loadHookCtx LoadHookContext = ctx.(*androidTopDownMutatorContext) - m.base().hooks.runLoadHooks(loadHookCtx, m.base()) - } -} - -func archHookMutator(ctx TopDownMutatorContext) { - if m, ok := ctx.Module().(Module); ok { - // Cast through *androidTopDownMutatorContext because AppendProperties is implemented - // on *androidTopDownMutatorContext but not exposed through TopDownMutatorContext - var archHookCtx ArchHookContext = ctx.(*androidTopDownMutatorContext) - m.base().hooks.runArchHooks(archHookCtx, m.base()) - } -} diff --git a/android/image.go b/android/image.go new file mode 100644 index 000000000..061bfa594 --- /dev/null +++ b/android/image.go @@ -0,0 +1,93 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +// ImageInterface is implemented by modules that need to be split by the imageMutator. +type ImageInterface interface { + // ImageMutatorBegin is called before any other method in the ImageInterface. + ImageMutatorBegin(ctx BaseModuleContext) + + // CoreVariantNeeded should return true if the module needs a core variant (installed on the system image). + CoreVariantNeeded(ctx BaseModuleContext) bool + + // RamdiskVariantNeeded should return true if the module needs a ramdisk variant (installed on the + // ramdisk partition). + RamdiskVariantNeeded(ctx BaseModuleContext) bool + + // RecoveryVariantNeeded should return true if the module needs a recovery variant (installed on the + // recovery partition). + RecoveryVariantNeeded(ctx BaseModuleContext) bool + + // ExtraImageVariations should return a list of the additional variations needed for the module. After the + // variants are created the SetImageVariation method will be called on each newly created variant with the + // its variation. + ExtraImageVariations(ctx BaseModuleContext) []string + + // SetImageVariation will be passed a newly created recovery variant of the module. ModuleBase implements + // SetImageVariation, most module types will not need to override it, and those that do must call the + // overridden method. Implementors of SetImageVariation must be careful to modify the module argument + // and not the receiver. + SetImageVariation(ctx BaseModuleContext, variation string, module Module) +} + +const ( + // CoreVariation is the variant used for framework-private libraries, or + // SDK libraries. (which framework-private libraries can use), which + // will be installed to the system image. + CoreVariation string = "" + + // RecoveryVariation means a module to be installed to recovery image. + RecoveryVariation string = "recovery" + + // RamdiskVariation means a module to be installed to ramdisk image. + RamdiskVariation string = "ramdisk" +) + +// imageMutator creates variants for modules that implement the ImageInterface that +// allow them to build differently for each partition (recovery, core, vendor, etc.). +func imageMutator(ctx BottomUpMutatorContext) { + if ctx.Os() != Android { + return + } + + if m, ok := ctx.Module().(ImageInterface); ok { + m.ImageMutatorBegin(ctx) + + var variations []string + + if m.CoreVariantNeeded(ctx) { + variations = append(variations, CoreVariation) + } + if m.RamdiskVariantNeeded(ctx) { + variations = append(variations, RamdiskVariation) + } + if m.RecoveryVariantNeeded(ctx) { + variations = append(variations, RecoveryVariation) + } + + extraVariations := m.ExtraImageVariations(ctx) + variations = append(variations, extraVariations...) + + if len(variations) == 0 { + return + } + + mod := ctx.CreateVariations(variations...) + for i, v := range variations { + mod[i].base().setImageVariation(v) + m.SetImageVariation(ctx, v, mod[i]) + } + } +} diff --git a/android/license.go b/android/license.go index 6a7c31338..00ddacd41 100644 --- a/android/license.go +++ b/android/license.go @@ -14,12 +14,13 @@ package android -import ( - "github.com/google/blueprint" -) - func init() { - RegisterModuleType("license", LicenseFactory) + RegisterLicenseBuildComponents(InitRegistrationContext) +} + +// Register the license module type. +func RegisterLicenseBuildComponents(ctx RegistrationContext) { + ctx.RegisterModuleType("license", LicenseFactory) } type licenseProperties struct { @@ -50,20 +51,20 @@ func (m *licenseModule) GenerateAndroidBuildActions(ctx ModuleContext) { // Nothing to do. } -func (m *licenseModule) GenerateBuildActions(ctx blueprint.ModuleContext) { - // Nothing to do. -} - func LicenseFactory() Module { module := &licenseModule{} base := module.base() - module.AddProperties( - &base.nameProperties, - &module.properties) + module.AddProperties(&base.nameProperties, &module.properties) base.generalProperties = module.GetProperties() base.customizableProperties = module.GetProperties() + // The visibility property needs to be checked and parsed by the visibility module. + setPrimaryVisibilityProperty(module, "visibility", &module.properties.Visibility) + + initAndroidModuleBase(module) + InitDefaultableModule(module) + return module } diff --git a/android/license_test.go b/android/license_test.go index cd2cb64ff..e80ba5e94 100644 --- a/android/license_test.go +++ b/android/license_test.go @@ -1,8 +1,6 @@ package android import ( - "io/ioutil" - "os" "testing" ) @@ -106,23 +104,20 @@ func TestLicense(t *testing.T) { } } func testLicense(fs map[string][]byte) (*TestContext, []error) { - buildDir, err := ioutil.TempDir("", "license_test") - if err != nil { - return nil, []error{err} - } - defer os.RemoveAll(buildDir) - // Create a new config per test as visibility information is stored in the config. env := make(map[string]string) env["ANDROID_REQUIRE_LICENSES"] = "1" - config := TestArchConfig(buildDir, env) + config := TestArchConfig(buildDir, env, "", fs) ctx := NewTestArchContext() - ctx.MockFileSystem(fs) - ctx.RegisterModuleType("package", ModuleFactoryAdaptor(PackageFactory)) - ctx.RegisterModuleType("license", ModuleFactoryAdaptor(LicenseFactory)) - ctx.RegisterModuleType("rule", ModuleFactoryAdaptor(newMockRuleModule)) - ctx.PreArchMutators(registerPackageRenamer) - ctx.Register() + RegisterPackageBuildComponents(ctx) + registerTestPrebuiltBuildComponents(ctx) + RegisterLicenseBuildComponents(ctx) + ctx.RegisterModuleType("rule", newMockRuleModule) + ctx.PreArchMutators(RegisterVisibilityRuleChecker) + ctx.PreArchMutators(RegisterDefaultsPreArchMutators) + ctx.PreArchMutators(RegisterVisibilityRuleGatherer) + ctx.PostDepsMutators(RegisterVisibilityRuleEnforcer) + ctx.Register(config) _, errs := ctx.ParseBlueprintsFiles(".") if len(errs) > 0 { return ctx, errs @@ -145,6 +140,3 @@ func newMockRuleModule() Module { func (p *mockRuleModule) GenerateAndroidBuildActions(ModuleContext) { } - -func (p *mockRuleModule) DepsMutator(BottomUpMutatorContext) { -} diff --git a/android/makevars.go b/android/makevars.go index c011ea61f..ff7c8e4a7 100644 --- a/android/makevars.go +++ b/android/makevars.go @@ -17,8 +17,6 @@ package android import ( "bytes" "fmt" - "io/ioutil" - "os" "strconv" "strings" @@ -41,7 +39,6 @@ type MakeVarsContext interface { Config() Config DeviceConfig() DeviceConfig AddNinjaFileDeps(deps ...string) - Fs() pathtools.FileSystem ModuleName(module blueprint.Module) string ModuleDir(module blueprint.Module) string @@ -80,6 +77,35 @@ type MakeVarsContext interface { // Eval(). StrictRaw(name, value string) CheckRaw(name, value string) + + // GlobWithDeps returns a list of files that match the specified pattern but do not match any + // of the patterns in excludes. It also adds efficient dependencies to rerun the primary + // builder whenever a file matching the pattern as added or removed, without rerunning if a + // file that does not match the pattern is added to a searched directory. + GlobWithDeps(pattern string, excludes []string) ([]string, error) + + // Phony creates a phony rule in Make, which will allow additional DistForGoal + // dependencies to be added to it. Phony can be called on the same name multiple + // times to add additional dependencies. + Phony(names string, deps ...Path) + + // DistForGoal creates a rule to copy one or more Paths to the artifacts + // directory on the build server when the specified goal is built. + DistForGoal(goal string, paths ...Path) + + // DistForGoalWithFilename creates a rule to copy a Path to the artifacts + // directory on the build server with the given filename when the specified + // goal is built. + DistForGoalWithFilename(goal string, path Path, filename string) + + // DistForGoals creates a rule to copy one or more Paths to the artifacts + // directory on the build server when any of the specified goals are built. + DistForGoals(goals []string, paths ...Path) + + // DistForGoalsWithFilename creates a rule to copy a Path to the artifacts + // directory on the build server with the given filename when any of the + // specified goals are built. + DistForGoalsWithFilename(goals []string, path Path, filename string) } var _ PathContext = MakeVarsContext(nil) @@ -126,9 +152,11 @@ var makeVarsProviders []makeVarsProvider type makeVarsContext struct { SingletonContext - config Config - pctx PackageContext - vars []makeVarsVariable + config Config + pctx PackageContext + vars []makeVarsVariable + phonies []phony + dists []dist } var _ MakeVarsContext = &makeVarsContext{} @@ -140,18 +168,34 @@ type makeVarsVariable struct { strict bool } +type phony struct { + name string + deps []string +} + +type dist struct { + goals []string + paths []string +} + func (s *makeVarsSingleton) GenerateBuildActions(ctx SingletonContext) { if !ctx.Config().EmbeddedInMake() { return } - outFile := PathForOutput(ctx, "make_vars"+proptools.String(ctx.Config().productVariables.Make_suffix)+".mk").String() + outFile := absolutePath(PathForOutput(ctx, + "make_vars"+proptools.String(ctx.Config().productVariables.Make_suffix)+".mk").String()) + + lateOutFile := absolutePath(PathForOutput(ctx, + "late"+proptools.String(ctx.Config().productVariables.Make_suffix)+".mk").String()) if ctx.Failed() { return } - vars := []makeVarsVariable{} + var vars []makeVarsVariable + var dists []dist + var phonies []phony for _, provider := range makeVarsProviders { mctx := &makeVarsContext{ SingletonContext: ctx, @@ -161,6 +205,8 @@ func (s *makeVarsSingleton) GenerateBuildActions(ctx SingletonContext) { provider.call(mctx) vars = append(vars, mctx.vars...) + phonies = append(phonies, mctx.phonies...) + dists = append(dists, mctx.dists...) } if ctx.Failed() { @@ -169,17 +215,16 @@ func (s *makeVarsSingleton) GenerateBuildActions(ctx SingletonContext) { outBytes := s.writeVars(vars) - if _, err := os.Stat(outFile); err == nil { - if data, err := ioutil.ReadFile(outFile); err == nil { - if bytes.Equal(data, outBytes) { - return - } - } + if err := pathtools.WriteFileIfChanged(outFile, outBytes, 0666); err != nil { + ctx.Errorf(err.Error()) } - if err := ioutil.WriteFile(outFile, outBytes, 0666); err != nil { + lateOutBytes := s.writeLate(phonies, dists) + + if err := pathtools.WriteFileIfChanged(lateOutFile, lateOutBytes, 0666); err != nil { ctx.Errorf(err.Error()) } + } func (s *makeVarsSingleton) writeVars(vars []makeVarsVariable) []byte { @@ -258,6 +303,33 @@ my_check_failed := fmt.Fprintln(buf, "\nsoong-compare-var :=") + fmt.Fprintln(buf) + + return buf.Bytes() +} + +func (s *makeVarsSingleton) writeLate(phonies []phony, dists []dist) []byte { + buf := &bytes.Buffer{} + + fmt.Fprint(buf, `# Autogenerated file + +# Values written by Soong read after parsing all Android.mk files. + + +`) + + for _, phony := range phonies { + fmt.Fprintf(buf, ".PHONY: %s\n", phony.name) + fmt.Fprintf(buf, "%s: %s\n", phony.name, strings.Join(phony.deps, "\\\n ")) + } + + fmt.Fprintln(buf) + + for _, dist := range dists { + fmt.Fprintf(buf, "$(call dist-for-goals,%s,%s)\n", + strings.Join(dist.goals, " "), strings.Join(dist.paths, " ")) + } + return buf.Bytes() } @@ -294,6 +366,17 @@ func (c *makeVarsContext) addVariable(name, ninjaStr string, strict, sort bool) c.addVariableRaw(name, value, strict, sort) } +func (c *makeVarsContext) addPhony(name string, deps []string) { + c.phonies = append(c.phonies, phony{name, deps}) +} + +func (c *makeVarsContext) addDist(goals []string, paths []string) { + c.dists = append(c.dists, dist{ + goals: goals, + paths: paths, + }) +} + func (c *makeVarsContext) Strict(name, ninjaStr string) { c.addVariable(name, ninjaStr, true, false) } @@ -313,3 +396,23 @@ func (c *makeVarsContext) CheckSorted(name, ninjaStr string) { func (c *makeVarsContext) CheckRaw(name, value string) { c.addVariableRaw(name, value, false, false) } + +func (c *makeVarsContext) Phony(name string, deps ...Path) { + c.addPhony(name, Paths(deps).Strings()) +} + +func (c *makeVarsContext) DistForGoal(goal string, paths ...Path) { + c.DistForGoals([]string{goal}, paths...) +} + +func (c *makeVarsContext) DistForGoalWithFilename(goal string, path Path, filename string) { + c.DistForGoalsWithFilename([]string{goal}, path, filename) +} + +func (c *makeVarsContext) DistForGoals(goals []string, paths ...Path) { + c.addDist(goals, Paths(paths).Strings()) +} + +func (c *makeVarsContext) DistForGoalsWithFilename(goals []string, path Path, filename string) { + c.addDist(goals, []string{path.String() + ":" + filename}) +} diff --git a/android/module.go b/android/module.go index 7f4e30817..a498839ce 100644 --- a/android/module.go +++ b/android/module.go @@ -16,14 +16,13 @@ package android import ( "fmt" + "os" "path" "path/filepath" - "sort" "strings" "text/scanner" "github.com/google/blueprint" - "github.com/google/blueprint/pathtools" "github.com/google/blueprint/proptools" ) @@ -56,40 +55,14 @@ type BuildParams struct { type ModuleBuildParams BuildParams -type androidBaseContext interface { - Target() Target - TargetPrimary() bool - MultiTargets() []Target - Arch() Arch - Os() OsType - Host() bool - Device() bool - Darwin() bool - Fuchsia() bool - Windows() bool - Debug() bool - PrimaryArch() bool - Platform() bool - DeviceSpecific() bool - SocSpecific() bool - ProductSpecific() bool - ProductServicesSpecific() bool - AConfig() Config - DeviceConfig() DeviceConfig -} - -type BaseContext interface { - BaseModuleContext - androidBaseContext -} - -// BaseModuleContext is the same as blueprint.BaseModuleContext except that Config() returns -// a Config instead of an interface{}. -type BaseModuleContext interface { +// EarlyModuleContext provides methods that can be called early, as soon as the properties have +// been parsed into the module and before any mutators have run. +type EarlyModuleContext interface { + Module() Module ModuleName() string ModuleDir() string ModuleType() string - Config() Config + BlueprintsFile() string ContainsProperty(name string) bool Errorf(pos scanner.Position, fmt string, args ...interface{}) @@ -97,55 +70,50 @@ type BaseModuleContext interface { PropertyErrorf(property, fmt string, args ...interface{}) Failed() bool + AddNinjaFileDeps(deps ...string) + + DeviceSpecific() bool + SocSpecific() bool + ProductSpecific() bool + SystemExtSpecific() bool + Platform() bool + + Config() Config + DeviceConfig() DeviceConfig + + // Deprecated: use Config() + AConfig() Config + // GlobWithDeps returns a list of files that match the specified pattern but do not match any // of the patterns in excludes. It also adds efficient dependencies to rerun the primary // builder whenever a file matching the pattern as added or removed, without rerunning if a // file that does not match the pattern is added to a searched directory. GlobWithDeps(pattern string, excludes []string) ([]string, error) - Fs() pathtools.FileSystem - AddNinjaFileDeps(deps ...string) -} - -type ModuleContext interface { - androidBaseContext - BaseModuleContext - - // Deprecated: use ModuleContext.Build instead. - ModuleBuild(pctx PackageContext, params ModuleBuildParams) - - ExpandSources(srcFiles, excludes []string) Paths - ExpandSource(srcFile, prop string) Path - ExpandOptionalSource(srcFile *string, prop string) OptionalPath Glob(globPattern string, excludes []string) Paths GlobFiles(globPattern string, excludes []string) Paths + IsSymlink(path Path) bool + Readlink(path Path) string +} - InstallExecutable(installPath OutputPath, name string, srcPath Path, deps ...Path) OutputPath - InstallFile(installPath OutputPath, name string, srcPath Path, deps ...Path) OutputPath - InstallSymlink(installPath OutputPath, name string, srcPath OutputPath) OutputPath - InstallAbsoluteSymlink(installPath OutputPath, name string, absPath string) OutputPath - CheckbuildFile(srcPath Path) - - AddMissingDependencies(deps []string) - - InstallInData() bool - InstallInSanitizerDir() bool - InstallInRecovery() bool - - RequiredModuleNames() []string +// BaseModuleContext is the same as blueprint.BaseModuleContext except that Config() returns +// a Config instead of an interface{}, and some methods have been wrapped to use an android.Module +// instead of a blueprint.Module, plus some extra methods that return Android-specific information +// about the current module. +type BaseModuleContext interface { + EarlyModuleContext - // android.ModuleContext methods - // These are duplicated instead of embedded so that can eventually be wrapped to take an - // android.Module instead of a blueprint.Module OtherModuleName(m blueprint.Module) string + OtherModuleDir(m blueprint.Module) string OtherModuleErrorf(m blueprint.Module, fmt string, args ...interface{}) OtherModuleDependencyTag(m blueprint.Module) blueprint.DependencyTag + OtherModuleExists(name string) bool + OtherModuleType(m blueprint.Module) string + GetDirectDepsWithTag(tag blueprint.DependencyTag) []Module GetDirectDepWithTag(name string, tag blueprint.DependencyTag) blueprint.Module GetDirectDep(name string) (blueprint.Module, blueprint.DependencyTag) - ModuleSubDir() string - VisitDirectDepsBlueprint(visit func(blueprint.Module)) VisitDirectDeps(visit func(Module)) VisitDirectDepsWithTag(tag blueprint.DependencyTag, visit func(Module)) @@ -156,12 +124,80 @@ type ModuleContext interface { VisitDepsDepthFirstIf(pred func(Module) bool, visit func(Module)) WalkDeps(visit func(Module, Module) bool) WalkDepsBlueprint(visit func(blueprint.Module, blueprint.Module) bool) + // GetWalkPath is supposed to be called in visit function passed in WalkDeps() + // and returns a top-down dependency path from a start module to current child module. + GetWalkPath() []Module + + // GetTagPath is supposed to be called in visit function passed in WalkDeps() + // and returns a top-down dependency tags path from a start module to current child module. + // It has one less entry than GetWalkPath() as it contains the dependency tags that + // exist between each adjacent pair of modules in the GetWalkPath(). + // GetTagPath()[i] is the tag between GetWalkPath()[i] and GetWalkPath()[i+1] + GetTagPath() []blueprint.DependencyTag + + AddMissingDependencies(missingDeps []string) + + Target() Target + TargetPrimary() bool + + // The additional arch specific targets (e.g. 32/64 bit) that this module variant is + // responsible for creating. + MultiTargets() []Target + Arch() Arch + Os() OsType + Host() bool + Device() bool + Darwin() bool + Fuchsia() bool + Windows() bool + Debug() bool + PrimaryArch() bool +} + +// Deprecated: use EarlyModuleContext instead +type BaseContext interface { + EarlyModuleContext +} + +type ModuleContext interface { + BaseModuleContext + + // Deprecated: use ModuleContext.Build instead. + ModuleBuild(pctx PackageContext, params ModuleBuildParams) + + ExpandSources(srcFiles, excludes []string) Paths + ExpandSource(srcFile, prop string) Path + ExpandOptionalSource(srcFile *string, prop string) OptionalPath + + InstallExecutable(installPath InstallPath, name string, srcPath Path, deps ...Path) InstallPath + InstallFile(installPath InstallPath, name string, srcPath Path, deps ...Path) InstallPath + InstallSymlink(installPath InstallPath, name string, srcPath InstallPath) InstallPath + InstallAbsoluteSymlink(installPath InstallPath, name string, absPath string) InstallPath + CheckbuildFile(srcPath Path) + + InstallInData() bool + InstallInTestcases() bool + InstallInSanitizerDir() bool + InstallInRamdisk() bool + InstallInRecovery() bool + InstallInRoot() bool + InstallBypassMake() bool + + RequiredModuleNames() []string + HostRequiredModuleNames() []string + TargetRequiredModuleNames() []string + + ModuleSubDir() string Variable(pctx PackageContext, name, value string) Rule(pctx PackageContext, name string, params blueprint.RuleParams, argNames ...string) blueprint.Rule // Similar to blueprint.ModuleContext.Build, but takes Paths instead of []string, // and performs more verification. Build(pctx PackageContext, params BuildParams) + // Phony creates a Make-style phony rule, a rule with no commands that can depend on other + // phony rules or real files. Phony can be called on the same name multiple times to add + // additional dependencies. + Phony(phony string, deps ...Path) PrimaryModule() Module FinalModule() Module @@ -182,13 +218,22 @@ type Module interface { DepsMutator(BottomUpMutatorContext) base() *ModuleBase + Disable() Enabled() bool Target() Target + Owner() string InstallInData() bool + InstallInTestcases() bool InstallInSanitizerDir() bool + InstallInRamdisk() bool InstallInRecovery() bool + InstallInRoot() bool + InstallBypassMake() bool SkipInstall() + IsSkipInstall() bool ExportedToMake() bool + InitRc() Paths + VintfFragments() Paths NoticeFile() OptionalPath AddProperties(props ...interface{}) @@ -197,6 +242,62 @@ type Module interface { BuildParamsForTests() []BuildParams RuleParamsForTests() map[blueprint.Rule]blueprint.RuleParams VariablesForTests() map[string]string + + // String returns a string that includes the module name and variants for printing during debugging. + String() string + + // Get the qualified module id for this module. + qualifiedModuleId(ctx BaseModuleContext) qualifiedModuleName + + // Get information about the properties that can contain visibility rules. + visibilityProperties() []visibilityProperty + + RequiredModuleNames() []string + HostRequiredModuleNames() []string + TargetRequiredModuleNames() []string +} + +// Qualified id for a module +type qualifiedModuleName struct { + // The package (i.e. directory) in which the module is defined, without trailing / + pkg string + + // The name of the module, empty string if package. + name string +} + +func (q qualifiedModuleName) String() string { + if q.name == "" { + return "//" + q.pkg + } + return "//" + q.pkg + ":" + q.name +} + +func (q qualifiedModuleName) isRootPackage() bool { + return q.pkg == "" && q.name == "" +} + +// Get the id for the package containing this module. +func (q qualifiedModuleName) getContainingPackageId() qualifiedModuleName { + pkg := q.pkg + if q.name == "" { + if pkg == "" { + panic(fmt.Errorf("Cannot get containing package id of root package")) + } + + index := strings.LastIndex(pkg, "/") + if index == -1 { + pkg = "" + } else { + pkg = pkg[:index] + } + } + return newPackageId(pkg) +} + +func newPackageId(pkg string) qualifiedModuleName { + // A qualified id for a package module has no name. + return qualifiedModuleName{pkg: pkg, name: ""} } type nameProperties struct { @@ -206,8 +307,59 @@ type nameProperties struct { type commonProperties struct { // emit build rules for this module + // + // Disabling a module should only be done for those modules that cannot be built + // in the current environment. Modules that can build in the current environment + // but are not usually required (e.g. superceded by a prebuilt) should not be + // disabled as that will prevent them from being built by the checkbuild target + // and so prevent early detection of changes that have broken those modules. Enabled *bool `android:"arch_variant"` + // Controls the visibility of this module to other modules. Allowable values are one or more of + // these formats: + // + // ["//visibility:public"]: Anyone can use this module. + // ["//visibility:private"]: Only rules in the module's package (not its subpackages) can use + // this module. + // ["//visibility:override"]: Discards any rules inherited from defaults or a creating module. + // Can only be used at the beginning of a list of visibility rules. + // ["//some/package:__pkg__", "//other/package:__pkg__"]: Only modules in some/package and + // other/package (defined in some/package/*.bp and other/package/*.bp) have access to + // this module. Note that sub-packages do not have access to the rule; for example, + // //some/package/foo:bar or //other/package/testing:bla wouldn't have access. __pkg__ + // is a special module and must be used verbatim. It represents all of the modules in the + // package. + // ["//project:__subpackages__", "//other:__subpackages__"]: Only modules in packages project + // or other or in one of their sub-packages have access to this module. For example, + // //project:rule, //project/library:lib or //other/testing/internal:munge are allowed + // to depend on this rule (but not //independent:evil) + // ["//project"]: This is shorthand for ["//project:__pkg__"] + // [":__subpackages__"]: This is shorthand for ["//project:__subpackages__"] where + // //project is the module's package. e.g. using [":__subpackages__"] in + // packages/apps/Settings/Android.bp is equivalent to + // //packages/apps/Settings:__subpackages__. + // ["//visibility:legacy_public"]: The default visibility, behaves as //visibility:public + // for now. It is an error if it is used in a module. + // + // If a module does not specify the `visibility` property then it uses the + // `default_visibility` property of the `package` module in the module's package. + // + // If the `default_visibility` property is not set for the module's package then + // it will use the `default_visibility` of its closest ancestor package for which + // a `default_visibility` property is specified. + // + // If no `default_visibility` property can be found then the module uses the + // global default of `//visibility:legacy_public`. + // + // The `visibility` property has no effect on a defaults module although it does + // apply to any non-defaults module that uses it. To set the visibility of a + // defaults module, use the `defaults_visibility` property on the defaults module; + // not to be confused with the `default_visibility` property on the package module. + // + // See https://android.googlesource.com/platform/build/soong/+/master/README.md#visibility for + // more details. + Visibility []string + // Names of the licenses that apply to this module. Licenses []string @@ -226,6 +378,10 @@ type commonProperties struct { } } + // If set to true then the archMutator will create variants for each arch specific target + // (e.g. 32/64) that the module is required to produce. If set to false then it will only + // create a variant for the architecture and will list the additional arch specific targets + // that the variant needs to produce in the CompileMultiTargets property. UseTargetVariants bool `blueprint:"mutated"` Default_multilib string `blueprint:"mutated"` @@ -255,14 +411,19 @@ type commonProperties struct { // /system/product if product partition does not exist). Product_specific *bool - // whether this module provides services owned by the OS provider to the core platform. When set - // to true, it is installed into /product_services (or /system/product_services if - // product_services partition does not exist). - Product_services_specific *bool + // whether this module extends system. When set to true, it is installed into /system_ext + // (or /system/system_ext if system_ext partition does not exist). + System_ext_specific *bool // Whether this module is installed to recovery partition Recovery *bool + // Whether this module is installed to ramdisk + Ramdisk *bool + + // Whether this module is built for non-native architecures (also known as native bridge binary) + Native_bridge_supported *bool `android:"arch_variant"` + // init.rc files to be installed if this module is installed Init_rc []string `android:"path"` @@ -272,6 +433,12 @@ type commonProperties struct { // names of other modules to install if this module is installed Required []string `android:"arch_variant"` + // names of other modules to install on host if this module is installed + Host_required []string `android:"arch_variant"` + + // names of other modules to install on target if this module is installed + Target_required []string `android:"arch_variant"` + // relative path to a file to include in the list of notices for the device Notice *string `android:"path"` @@ -293,18 +460,69 @@ type commonProperties struct { Suffix *string `android:"arch_variant"` } `android:"arch_variant"` - // Set by TargetMutator - CompileTarget Target `blueprint:"mutated"` + // The OsType of artifacts that this module variant is responsible for creating. + // + // Set by osMutator + CompileOS OsType `blueprint:"mutated"` + + // The Target of artifacts that this module variant is responsible for creating. + // + // Set by archMutator + CompileTarget Target `blueprint:"mutated"` + + // The additional arch specific targets (e.g. 32/64 bit) that this module variant is + // responsible for creating. + // + // By default this is nil as, where necessary, separate variants are created for the + // different multilib types supported and that information is encapsulated in the + // CompileTarget so the module variant simply needs to create artifacts for that. + // + // However, if UseTargetVariants is set to false (e.g. by + // InitAndroidMultiTargetsArchModule) then no separate variants are created for the + // multilib targets. Instead a single variant is created for the architecture and + // this contains the multilib specific targets that this variant should create. + // + // Set by archMutator CompileMultiTargets []Target `blueprint:"mutated"` - CompilePrimary bool `blueprint:"mutated"` + + // True if the module variant's CompileTarget is the primary target + // + // Set by archMutator + CompilePrimary bool `blueprint:"mutated"` // Set by InitAndroidModule HostOrDeviceSupported HostOrDeviceSupported `blueprint:"mutated"` ArchSpecific bool `blueprint:"mutated"` + // If set to true then a CommonOS variant will be created which will have dependencies + // on all its OsType specific variants. Used by sdk/module_exports to create a snapshot + // that covers all os and architecture variants. + // + // The OsType specific variants can be retrieved by calling + // GetOsSpecificVariantsOfCommonOSVariant + // + // Set at module initialization time by calling InitCommonOSAndroidMultiTargetsArchModule + CreateCommonOSVariant bool `blueprint:"mutated"` + + // If set to true then this variant is the CommonOS variant that has dependencies on its + // OsType specific variants. + // + // Set by osMutator. + CommonOSVariant bool `blueprint:"mutated"` + SkipInstall bool `blueprint:"mutated"` NamespaceExportedToMake bool `blueprint:"mutated"` + + MissingDeps []string `blueprint:"mutated"` + + // Name and variant strings stored by mutators to enable Module.String() + DebugName string `blueprint:"mutated"` + DebugMutators []string `blueprint:"mutated"` + DebugVariations []string `blueprint:"mutated"` + + // set by ImageMutator + ImageVariation string `blueprint:"mutated"` } type hostAndDeviceProperties struct { @@ -358,7 +576,7 @@ const ( deviceSpecificModule socSpecificModule productSpecificModule - productServicesSpecificModule + systemExtSpecificModule ) func (k moduleKind) String() string { @@ -371,23 +589,33 @@ func (k moduleKind) String() string { return "soc-specific" case productSpecificModule: return "product-specific" - case productServicesSpecificModule: - return "productservices-specific" + case systemExtSpecificModule: + return "systemext-specific" default: panic(fmt.Errorf("unknown module kind %d", k)) } } +func initAndroidModuleBase(m Module) { + m.base().module = m +} + func InitAndroidModule(m Module) { + initAndroidModuleBase(m) base := m.base() - base.module = m m.AddProperties( &base.nameProperties, - &base.commonProperties, - &base.variableProperties) + &base.commonProperties) + + initProductVariableModule(m) + base.generalProperties = m.GetProperties() base.customizableProperties = m.GetProperties() + + // The default_visibility property needs to be checked and parsed by the visibility module during + // its checking and parsing phases so make it the primary visibility property. + setPrimaryVisibilityProperty(m, "visibility", &base.commonProperties.Visibility) } func InitAndroidArchModule(m Module, hod HostOrDeviceSupported, defaultMultilib Multilib) { @@ -412,6 +640,14 @@ func InitAndroidMultiTargetsArchModule(m Module, hod HostOrDeviceSupported, defa m.base().commonProperties.UseTargetVariants = false } +// As InitAndroidMultiTargetsArchModule except it creates an additional CommonOS variant that +// has dependencies on all the OsType specific variants. +func InitCommonOSAndroidMultiTargetsArchModule(m Module, hod HostOrDeviceSupported, defaultMultilib Multilib) { + InitAndroidArchModule(m, hod, defaultMultilib) + m.base().commonProperties.UseTargetVariants = false + m.base().commonProperties.CreateCommonOSVariant = true +} + // A ModuleBase object contains the properties that are common to all Android // modules. It should be included as an anonymous field in every module // struct definition. InitAndroidModule should then be called from the module's @@ -421,9 +657,9 @@ func InitAndroidMultiTargetsArchModule(m Module, hod HostOrDeviceSupported, defa // The ModuleBase type is responsible for implementing the GenerateBuildActions // method to support the blueprint.Module interface. This method will then call // the module's GenerateAndroidBuildActions method once for each build variant -// that is to be built. GenerateAndroidBuildActions is passed a -// AndroidModuleContext rather than the usual blueprint.ModuleContext. -// AndroidModuleContext exposes extra functionality specific to the Android build +// that is to be built. GenerateAndroidBuildActions is passed a ModuleContext +// rather than the usual blueprint.ModuleContext. +// ModuleContext exposes extra functionality specific to the Android build // system including details about the particular build variant that is to be // generated. // @@ -461,16 +697,24 @@ type ModuleBase struct { nameProperties nameProperties commonProperties commonProperties - variableProperties variableProperties + variableProperties interface{} hostAndDeviceProperties hostAndDeviceProperties generalProperties []interface{} archProperties [][]interface{} customizableProperties []interface{} + // Information about all the properties on the module that contains visibility rules that need + // checking. + visibilityPropertyInfo []visibilityProperty + + // The primary visibility property, may be nil, that controls access to the module. + primaryVisibilityProperty visibilityProperty + noAddressSanitizer bool installFiles Paths checkbuildFiles Paths noticeFile OptionalPath + phonies map[string]Paths // Used by buildTargetSingleton to create checkbuild and per-directory build targets // Only set on the final variant of each module @@ -487,86 +731,117 @@ type ModuleBase struct { ruleParams map[blueprint.Rule]blueprint.RuleParams variables map[string]string + initRcPaths Paths + vintfFragmentsPaths Paths + prefer32 func(ctx BaseModuleContext, base *ModuleBase, class OsClass) bool } -func (a *ModuleBase) DepsMutator(BottomUpMutatorContext) {} +func (m *ModuleBase) DepsMutator(BottomUpMutatorContext) {} -func (a *ModuleBase) AddProperties(props ...interface{}) { - a.registerProps = append(a.registerProps, props...) +func (m *ModuleBase) AddProperties(props ...interface{}) { + m.registerProps = append(m.registerProps, props...) } -func (a *ModuleBase) GetProperties() []interface{} { - return a.registerProps +func (m *ModuleBase) GetProperties() []interface{} { + return m.registerProps } -func (a *ModuleBase) BuildParamsForTests() []BuildParams { - return a.buildParams +func (m *ModuleBase) BuildParamsForTests() []BuildParams { + return m.buildParams } -func (a *ModuleBase) RuleParamsForTests() map[blueprint.Rule]blueprint.RuleParams { - return a.ruleParams +func (m *ModuleBase) RuleParamsForTests() map[blueprint.Rule]blueprint.RuleParams { + return m.ruleParams } -func (a *ModuleBase) VariablesForTests() map[string]string { - return a.variables +func (m *ModuleBase) VariablesForTests() map[string]string { + return m.variables } -func (a *ModuleBase) Prefer32(prefer32 func(ctx BaseModuleContext, base *ModuleBase, class OsClass) bool) { - a.prefer32 = prefer32 +func (m *ModuleBase) Prefer32(prefer32 func(ctx BaseModuleContext, base *ModuleBase, class OsClass) bool) { + m.prefer32 = prefer32 } // Name returns the name of the module. It may be overridden by individual module types, for // example prebuilts will prepend prebuilt_ to the name. -func (a *ModuleBase) Name() string { - return String(a.nameProperties.Name) +func (m *ModuleBase) Name() string { + return String(m.nameProperties.Name) +} + +// String returns a string that includes the module name and variants for printing during debugging. +func (m *ModuleBase) String() string { + sb := strings.Builder{} + sb.WriteString(m.commonProperties.DebugName) + sb.WriteString("{") + for i := range m.commonProperties.DebugMutators { + if i != 0 { + sb.WriteString(",") + } + sb.WriteString(m.commonProperties.DebugMutators[i]) + sb.WriteString(":") + sb.WriteString(m.commonProperties.DebugVariations[i]) + } + sb.WriteString("}") + return sb.String() } // BaseModuleName returns the name of the module as specified in the blueprints file. -func (a *ModuleBase) BaseModuleName() string { - return String(a.nameProperties.Name) +func (m *ModuleBase) BaseModuleName() string { + return String(m.nameProperties.Name) +} + +func (m *ModuleBase) base() *ModuleBase { + return m +} + +func (m *ModuleBase) qualifiedModuleId(ctx BaseModuleContext) qualifiedModuleName { + return qualifiedModuleName{pkg: ctx.ModuleDir(), name: ctx.ModuleName()} +} + +func (m *ModuleBase) visibilityProperties() []visibilityProperty { + return m.visibilityPropertyInfo } -func (a *ModuleBase) base() *ModuleBase { - return a +func (m *ModuleBase) Target() Target { + return m.commonProperties.CompileTarget } -func (a *ModuleBase) SetTarget(target Target, multiTargets []Target, primary bool) { - a.commonProperties.CompileTarget = target - a.commonProperties.CompileMultiTargets = multiTargets - a.commonProperties.CompilePrimary = primary +func (m *ModuleBase) TargetPrimary() bool { + return m.commonProperties.CompilePrimary } -func (a *ModuleBase) Target() Target { - return a.commonProperties.CompileTarget +func (m *ModuleBase) MultiTargets() []Target { + return m.commonProperties.CompileMultiTargets } -func (a *ModuleBase) TargetPrimary() bool { - return a.commonProperties.CompilePrimary +func (m *ModuleBase) Os() OsType { + return m.Target().Os } -func (a *ModuleBase) MultiTargets() []Target { - return a.commonProperties.CompileMultiTargets +func (m *ModuleBase) Host() bool { + return m.Os().Class == Host || m.Os().Class == HostCross } -func (a *ModuleBase) Os() OsType { - return a.Target().Os +func (m *ModuleBase) Device() bool { + return m.Os().Class == Device } -func (a *ModuleBase) Host() bool { - return a.Os().Class == Host || a.Os().Class == HostCross +func (m *ModuleBase) Arch() Arch { + return m.Target().Arch } -func (a *ModuleBase) Arch() Arch { - return a.Target().Arch +func (m *ModuleBase) ArchSpecific() bool { + return m.commonProperties.ArchSpecific } -func (a *ModuleBase) ArchSpecific() bool { - return a.commonProperties.ArchSpecific +// True if the current variant is a CommonOS variant, false otherwise. +func (m *ModuleBase) IsCommonOSVariant() bool { + return m.commonProperties.CommonOSVariant } -func (a *ModuleBase) OsClassSupported() []OsClass { - switch a.commonProperties.HostOrDeviceSupported { +func (m *ModuleBase) OsClassSupported() []OsClass { + switch m.commonProperties.HostOrDeviceSupported { case HostSupported: return []OsClass{Host, HostCross} case HostSupportedNoCross: @@ -575,13 +850,13 @@ func (a *ModuleBase) OsClassSupported() []OsClass { return []OsClass{Device} case HostAndDeviceSupported, HostAndDeviceDefault: var supported []OsClass - if Bool(a.hostAndDeviceProperties.Host_supported) || - (a.commonProperties.HostOrDeviceSupported == HostAndDeviceDefault && - a.hostAndDeviceProperties.Host_supported == nil) { + if Bool(m.hostAndDeviceProperties.Host_supported) || + (m.commonProperties.HostOrDeviceSupported == HostAndDeviceDefault && + m.hostAndDeviceProperties.Host_supported == nil) { supported = append(supported, Host, HostCross) } - if a.hostAndDeviceProperties.Device_supported == nil || - *a.hostAndDeviceProperties.Device_supported { + if m.hostAndDeviceProperties.Device_supported == nil || + *m.hostAndDeviceProperties.Device_supported { supported = append(supported, Device) } return supported @@ -590,31 +865,45 @@ func (a *ModuleBase) OsClassSupported() []OsClass { } } -func (a *ModuleBase) DeviceSupported() bool { - return a.commonProperties.HostOrDeviceSupported == DeviceSupported || - a.commonProperties.HostOrDeviceSupported == HostAndDeviceSupported && - (a.hostAndDeviceProperties.Device_supported == nil || - *a.hostAndDeviceProperties.Device_supported) +func (m *ModuleBase) DeviceSupported() bool { + return m.commonProperties.HostOrDeviceSupported == DeviceSupported || + m.commonProperties.HostOrDeviceSupported == HostAndDeviceSupported && + (m.hostAndDeviceProperties.Device_supported == nil || + *m.hostAndDeviceProperties.Device_supported) +} + +func (m *ModuleBase) HostSupported() bool { + return m.commonProperties.HostOrDeviceSupported == HostSupported || + m.commonProperties.HostOrDeviceSupported == HostAndDeviceSupported && + (m.hostAndDeviceProperties.Host_supported != nil && + *m.hostAndDeviceProperties.Host_supported) +} + +func (m *ModuleBase) Platform() bool { + return !m.DeviceSpecific() && !m.SocSpecific() && !m.ProductSpecific() && !m.SystemExtSpecific() } -func (a *ModuleBase) Platform() bool { - return !a.DeviceSpecific() && !a.SocSpecific() && !a.ProductSpecific() && !a.ProductServicesSpecific() +func (m *ModuleBase) DeviceSpecific() bool { + return Bool(m.commonProperties.Device_specific) } -func (a *ModuleBase) DeviceSpecific() bool { - return Bool(a.commonProperties.Device_specific) +func (m *ModuleBase) SocSpecific() bool { + return Bool(m.commonProperties.Vendor) || Bool(m.commonProperties.Proprietary) || Bool(m.commonProperties.Soc_specific) } -func (a *ModuleBase) SocSpecific() bool { - return Bool(a.commonProperties.Vendor) || Bool(a.commonProperties.Proprietary) || Bool(a.commonProperties.Soc_specific) +func (m *ModuleBase) ProductSpecific() bool { + return Bool(m.commonProperties.Product_specific) } -func (a *ModuleBase) ProductSpecific() bool { - return Bool(a.commonProperties.Product_specific) +func (m *ModuleBase) SystemExtSpecific() bool { + return Bool(m.commonProperties.System_ext_specific) } -func (a *ModuleBase) ProductServicesSpecific() bool { - return Bool(a.commonProperties.Product_services_specific) +// RequiresStableAPIs returns true if the module will be installed to a partition that may +// be updated separately from the system image. +func (m *ModuleBase) RequiresStableAPIs(ctx BaseModuleContext) bool { + return m.SocSpecific() || m.DeviceSpecific() || + (m.ProductSpecific() && ctx.Config().EnforceProductPartitionInterface()) } func (m *ModuleBase) PartitionTag(config DeviceConfig) string { @@ -640,26 +929,41 @@ func (m *ModuleBase) PartitionTag(config DeviceConfig) string { if config.ProductPath() == "product" { partition = "product" } + } else if m.SystemExtSpecific() { + // A system_ext-specific module could be on the system_ext + // partition at "system_ext" or the system partition at + // "system/system_ext". + if config.SystemExtPath() == "system_ext" { + partition = "system_ext" + } } return partition } -func (a *ModuleBase) Enabled() bool { - if a.commonProperties.Enabled == nil { - return !a.Os().DefaultDisabled +func (m *ModuleBase) Enabled() bool { + if m.commonProperties.Enabled == nil { + return !m.Os().DefaultDisabled } - return *a.commonProperties.Enabled + return *m.commonProperties.Enabled +} + +func (m *ModuleBase) Disable() { + m.commonProperties.Enabled = proptools.BoolPtr(false) +} + +func (m *ModuleBase) SkipInstall() { + m.commonProperties.SkipInstall = true } -func (a *ModuleBase) SkipInstall() { - a.commonProperties.SkipInstall = true +func (m *ModuleBase) IsSkipInstall() bool { + return m.commonProperties.SkipInstall == true } -func (a *ModuleBase) ExportedToMake() bool { - return a.commonProperties.NamespaceExportedToMake +func (m *ModuleBase) ExportedToMake() bool { + return m.commonProperties.NamespaceExportedToMake } -func (a *ModuleBase) computeInstallDeps( +func (m *ModuleBase) computeInstallDeps( ctx blueprint.ModuleContext) Paths { result := Paths{} @@ -674,35 +978,100 @@ func (a *ModuleBase) computeInstallDeps( return result } -func (a *ModuleBase) filesToInstall() Paths { - return a.installFiles +func (m *ModuleBase) filesToInstall() Paths { + return m.installFiles } -func (p *ModuleBase) NoAddressSanitizer() bool { - return p.noAddressSanitizer +func (m *ModuleBase) NoAddressSanitizer() bool { + return m.noAddressSanitizer } -func (p *ModuleBase) InstallInData() bool { +func (m *ModuleBase) InstallInData() bool { return false } -func (p *ModuleBase) InstallInSanitizerDir() bool { +func (m *ModuleBase) InstallInTestcases() bool { return false } -func (p *ModuleBase) InstallInRecovery() bool { - return Bool(p.commonProperties.Recovery) +func (m *ModuleBase) InstallInSanitizerDir() bool { + return false +} + +func (m *ModuleBase) InstallInRamdisk() bool { + return Bool(m.commonProperties.Ramdisk) +} + +func (m *ModuleBase) InstallInRecovery() bool { + return Bool(m.commonProperties.Recovery) +} + +func (m *ModuleBase) InstallInRoot() bool { + return false +} + +func (m *ModuleBase) InstallBypassMake() bool { + return false +} + +func (m *ModuleBase) Owner() string { + return String(m.commonProperties.Owner) +} + +func (m *ModuleBase) NoticeFile() OptionalPath { + return m.noticeFile +} + +func (m *ModuleBase) setImageVariation(variant string) { + m.commonProperties.ImageVariation = variant +} + +func (m *ModuleBase) ImageVariation() blueprint.Variation { + return blueprint.Variation{ + Mutator: "image", + Variation: m.base().commonProperties.ImageVariation, + } +} + +func (m *ModuleBase) getVariationByMutatorName(mutator string) string { + for i, v := range m.commonProperties.DebugMutators { + if v == mutator { + return m.commonProperties.DebugVariations[i] + } + } + + return "" +} + +func (m *ModuleBase) InRamdisk() bool { + return m.base().commonProperties.ImageVariation == RamdiskVariation +} + +func (m *ModuleBase) InRecovery() bool { + return m.base().commonProperties.ImageVariation == RecoveryVariation +} + +func (m *ModuleBase) RequiredModuleNames() []string { + return m.base().commonProperties.Required } -func (a *ModuleBase) Owner() string { - return String(a.commonProperties.Owner) +func (m *ModuleBase) HostRequiredModuleNames() []string { + return m.base().commonProperties.Host_required } -func (a *ModuleBase) NoticeFile() OptionalPath { - return a.noticeFile +func (m *ModuleBase) TargetRequiredModuleNames() []string { + return m.base().commonProperties.Target_required } -func (a *ModuleBase) generateModuleTarget(ctx ModuleContext) { +func (m *ModuleBase) InitRc() Paths { + return append(Paths{}, m.initRcPaths...) +} + +func (m *ModuleBase) VintfFragments() Paths { + return append(Paths{}, m.vintfFragmentsPaths...) +} + +func (m *ModuleBase) generateModuleTarget(ctx ModuleContext) { allInstalledFiles := Paths{} allCheckbuildFiles := Paths{} ctx.VisitAllModuleVariants(func(module Module) { @@ -719,26 +1088,17 @@ func (a *ModuleBase) generateModuleTarget(ctx ModuleContext) { } if len(allInstalledFiles) > 0 { - name := PathForPhony(ctx, namespacePrefix+ctx.ModuleName()+"-install") - ctx.Build(pctx, BuildParams{ - Rule: blueprint.Phony, - Output: name, - Implicits: allInstalledFiles, - Default: !ctx.Config().EmbeddedInMake(), - }) - deps = append(deps, name) - a.installTarget = name + name := namespacePrefix + ctx.ModuleName() + "-install" + ctx.Phony(name, allInstalledFiles...) + m.installTarget = PathForPhony(ctx, name) + deps = append(deps, m.installTarget) } if len(allCheckbuildFiles) > 0 { - name := PathForPhony(ctx, namespacePrefix+ctx.ModuleName()+"-checkbuild") - ctx.Build(pctx, BuildParams{ - Rule: blueprint.Phony, - Output: name, - Implicits: allCheckbuildFiles, - }) - deps = append(deps, name) - a.checkbuildTarget = name + name := namespacePrefix + ctx.ModuleName() + "-checkbuild" + ctx.Phony(name, allCheckbuildFiles...) + m.checkbuildTarget = PathForPhony(ctx, name) + deps = append(deps, m.checkbuildTarget) } if len(deps) > 0 { @@ -747,58 +1107,53 @@ func (a *ModuleBase) generateModuleTarget(ctx ModuleContext) { suffix = "-soong" } - name := PathForPhony(ctx, namespacePrefix+ctx.ModuleName()+suffix) - ctx.Build(pctx, BuildParams{ - Rule: blueprint.Phony, - Outputs: []WritablePath{name}, - Implicits: deps, - }) + ctx.Phony(namespacePrefix+ctx.ModuleName()+suffix, deps...) - a.blueprintDir = ctx.ModuleDir() + m.blueprintDir = ctx.ModuleDir() } } -func determineModuleKind(a *ModuleBase, ctx blueprint.BaseModuleContext) moduleKind { - var socSpecific = Bool(a.commonProperties.Vendor) || Bool(a.commonProperties.Proprietary) || Bool(a.commonProperties.Soc_specific) - var deviceSpecific = Bool(a.commonProperties.Device_specific) - var productSpecific = Bool(a.commonProperties.Product_specific) - var productServicesSpecific = Bool(a.commonProperties.Product_services_specific) +func determineModuleKind(m *ModuleBase, ctx blueprint.EarlyModuleContext) moduleKind { + var socSpecific = Bool(m.commonProperties.Vendor) || Bool(m.commonProperties.Proprietary) || Bool(m.commonProperties.Soc_specific) + var deviceSpecific = Bool(m.commonProperties.Device_specific) + var productSpecific = Bool(m.commonProperties.Product_specific) + var systemExtSpecific = Bool(m.commonProperties.System_ext_specific) msg := "conflicting value set here" if socSpecific && deviceSpecific { ctx.PropertyErrorf("device_specific", "a module cannot be specific to SoC and device at the same time.") - if Bool(a.commonProperties.Vendor) { + if Bool(m.commonProperties.Vendor) { ctx.PropertyErrorf("vendor", msg) } - if Bool(a.commonProperties.Proprietary) { + if Bool(m.commonProperties.Proprietary) { ctx.PropertyErrorf("proprietary", msg) } - if Bool(a.commonProperties.Soc_specific) { + if Bool(m.commonProperties.Soc_specific) { ctx.PropertyErrorf("soc_specific", msg) } } - if productSpecific && productServicesSpecific { - ctx.PropertyErrorf("product_specific", "a module cannot be specific to product and product_services at the same time.") - ctx.PropertyErrorf("product_services_specific", msg) + if productSpecific && systemExtSpecific { + ctx.PropertyErrorf("product_specific", "a module cannot be specific to product and system_ext at the same time.") + ctx.PropertyErrorf("system_ext_specific", msg) } - if (socSpecific || deviceSpecific) && (productSpecific || productServicesSpecific) { + if (socSpecific || deviceSpecific) && (productSpecific || systemExtSpecific) { if productSpecific { ctx.PropertyErrorf("product_specific", "a module cannot be specific to SoC or device and product at the same time.") } else { - ctx.PropertyErrorf("product_services_specific", "a module cannot be specific to SoC or device and product_services at the same time.") + ctx.PropertyErrorf("system_ext_specific", "a module cannot be specific to SoC or device and system_ext at the same time.") } if deviceSpecific { ctx.PropertyErrorf("device_specific", msg) } else { - if Bool(a.commonProperties.Vendor) { + if Bool(m.commonProperties.Vendor) { ctx.PropertyErrorf("vendor", msg) } - if Bool(a.commonProperties.Proprietary) { + if Bool(m.commonProperties.Proprietary) { ctx.PropertyErrorf("proprietary", msg) } - if Bool(a.commonProperties.Soc_specific) { + if Bool(m.commonProperties.Soc_specific) { ctx.PropertyErrorf("soc_specific", msg) } } @@ -806,8 +1161,8 @@ func determineModuleKind(a *ModuleBase, ctx blueprint.BaseModuleContext) moduleK if productSpecific { return productSpecificModule - } else if productServicesSpecific { - return productServicesSpecificModule + } else if systemExtSpecific { + return systemExtSpecificModule } else if deviceSpecific { return deviceSpecificModule } else if socSpecific { @@ -817,27 +1172,47 @@ func determineModuleKind(a *ModuleBase, ctx blueprint.BaseModuleContext) moduleK } } -func (a *ModuleBase) androidBaseContextFactory(ctx blueprint.BaseModuleContext) androidBaseContextImpl { - return androidBaseContextImpl{ - target: a.commonProperties.CompileTarget, - targetPrimary: a.commonProperties.CompilePrimary, - multiTargets: a.commonProperties.CompileMultiTargets, - kind: determineModuleKind(a, ctx), - config: ctx.Config().(Config), +func (m *ModuleBase) earlyModuleContextFactory(ctx blueprint.EarlyModuleContext) earlyModuleContext { + return earlyModuleContext{ + EarlyModuleContext: ctx, + kind: determineModuleKind(m, ctx), + config: ctx.Config().(Config), + } +} + +func (m *ModuleBase) baseModuleContextFactory(ctx blueprint.BaseModuleContext) baseModuleContext { + return baseModuleContext{ + bp: ctx, + earlyModuleContext: m.earlyModuleContextFactory(ctx), + os: m.commonProperties.CompileOS, + target: m.commonProperties.CompileTarget, + targetPrimary: m.commonProperties.CompilePrimary, + multiTargets: m.commonProperties.CompileMultiTargets, } } -func (a *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext) { - ctx := &androidModuleContext{ - module: a.module, - ModuleContext: blueprintCtx, - androidBaseContextImpl: a.androidBaseContextFactory(blueprintCtx), - installDeps: a.computeInstallDeps(blueprintCtx), - installFiles: a.installFiles, - missingDeps: blueprintCtx.GetMissingDependencies(), - variables: make(map[string]string), +func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext) { + ctx := &moduleContext{ + module: m.module, + bp: blueprintCtx, + baseModuleContext: m.baseModuleContextFactory(blueprintCtx), + installDeps: m.computeInstallDeps(blueprintCtx), + installFiles: m.installFiles, + variables: make(map[string]string), } + // Temporarily continue to call blueprintCtx.GetMissingDependencies() to maintain the previous behavior of never + // reporting missing dependency errors in Blueprint when AllowMissingDependencies == true. + // TODO: This will be removed once defaults modules handle missing dependency errors + blueprintCtx.GetMissingDependencies() + + // For the final GenerateAndroidBuildActions pass, require that all visited dependencies Soong modules and + // are enabled. Unless the module is a CommonOS variant which may have dependencies on disabled variants + // (because the dependencies are added before the modules are disabled). The + // GetOsSpecificVariantsOfCommonOSVariant(...) method will ensure that the disabled variants are + // ignored. + ctx.baseModuleContext.strictVisitDeps = !m.IsCommonOSVariant() + if ctx.config.captureBuild { ctx.ruleParams = make(map[blueprint.Rule]blueprint.RuleParams) } @@ -850,6 +1225,9 @@ func (a *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext) if !ctx.PrimaryArch() { suffix = append(suffix, ctx.Arch().ArchType.String()) } + if apex, ok := m.module.(ApexModule); ok && !apex.IsForPlatform() { + suffix = append(suffix, apex.ApexName()) + } ctx.Variable(pctx, "moduleDesc", desc) @@ -860,71 +1238,190 @@ func (a *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext) ctx.Variable(pctx, "moduleDescSuffix", s) // Some common property checks for properties that will be used later in androidmk.go - if a.commonProperties.Dist.Dest != nil { - _, err := validateSafePath(*a.commonProperties.Dist.Dest) + if m.commonProperties.Dist.Dest != nil { + _, err := validateSafePath(*m.commonProperties.Dist.Dest) if err != nil { ctx.PropertyErrorf("dist.dest", "%s", err.Error()) } } - if a.commonProperties.Dist.Dir != nil { - _, err := validateSafePath(*a.commonProperties.Dist.Dir) + if m.commonProperties.Dist.Dir != nil { + _, err := validateSafePath(*m.commonProperties.Dist.Dir) if err != nil { ctx.PropertyErrorf("dist.dir", "%s", err.Error()) } } - if a.commonProperties.Dist.Suffix != nil { - if strings.Contains(*a.commonProperties.Dist.Suffix, "/") { + if m.commonProperties.Dist.Suffix != nil { + if strings.Contains(*m.commonProperties.Dist.Suffix, "/") { ctx.PropertyErrorf("dist.suffix", "Suffix may not contain a '/' character.") } } - if a.Enabled() { - notice := proptools.StringDefault(a.commonProperties.Notice, "NOTICE") - if m := SrcIsModule(notice); m != "" { - a.noticeFile = ctx.ExpandOptionalSource(¬ice, "notice") + if m.Enabled() { + // ensure all direct android.Module deps are enabled + ctx.VisitDirectDepsBlueprint(func(bm blueprint.Module) { + if _, ok := bm.(Module); ok { + ctx.validateAndroidModule(bm, ctx.baseModuleContext.strictVisitDeps) + } + }) + + notice := proptools.StringDefault(m.commonProperties.Notice, "NOTICE") + if module := SrcIsModule(notice); module != "" { + m.noticeFile = ctx.ExpandOptionalSource(¬ice, "notice") } else { noticePath := filepath.Join(ctx.ModuleDir(), notice) - a.noticeFile = ExistentPathForSource(ctx, noticePath) + m.noticeFile = ExistentPathForSource(ctx, noticePath) } - a.module.GenerateAndroidBuildActions(ctx) + m.module.GenerateAndroidBuildActions(ctx) if ctx.Failed() { return } - a.installFiles = append(a.installFiles, ctx.installFiles...) - a.checkbuildFiles = append(a.checkbuildFiles, ctx.checkbuildFiles...) + m.installFiles = append(m.installFiles, ctx.installFiles...) + m.checkbuildFiles = append(m.checkbuildFiles, ctx.checkbuildFiles...) + m.initRcPaths = PathsForModuleSrc(ctx, m.commonProperties.Init_rc) + m.vintfFragmentsPaths = PathsForModuleSrc(ctx, m.commonProperties.Vintf_fragments) + for k, v := range ctx.phonies { + m.phonies[k] = append(m.phonies[k], v...) + } + } else if ctx.Config().AllowMissingDependencies() { + // If the module is not enabled it will not create any build rules, nothing will call + // ctx.GetMissingDependencies(), and blueprint will consider the missing dependencies to be unhandled + // and report them as an error even when AllowMissingDependencies = true. Call + // ctx.GetMissingDependencies() here to tell blueprint not to handle them. + ctx.GetMissingDependencies() } - if a == ctx.FinalModule().(Module).base() { - a.generateModuleTarget(ctx) + if m == ctx.FinalModule().(Module).base() { + m.generateModuleTarget(ctx) if ctx.Failed() { return } } - a.buildParams = ctx.buildParams - a.ruleParams = ctx.ruleParams - a.variables = ctx.variables + m.buildParams = ctx.buildParams + m.ruleParams = ctx.ruleParams + m.variables = ctx.variables +} + +type earlyModuleContext struct { + blueprint.EarlyModuleContext + + kind moduleKind + config Config +} + +func (e *earlyModuleContext) Glob(globPattern string, excludes []string) Paths { + ret, err := e.GlobWithDeps(globPattern, excludes) + if err != nil { + e.ModuleErrorf("glob: %s", err.Error()) + } + return pathsForModuleSrcFromFullPath(e, ret, true) +} + +func (e *earlyModuleContext) GlobFiles(globPattern string, excludes []string) Paths { + ret, err := e.GlobWithDeps(globPattern, excludes) + if err != nil { + e.ModuleErrorf("glob: %s", err.Error()) + } + return pathsForModuleSrcFromFullPath(e, ret, false) +} + +func (b *earlyModuleContext) IsSymlink(path Path) bool { + fileInfo, err := b.config.fs.Lstat(path.String()) + if err != nil { + b.ModuleErrorf("os.Lstat(%q) failed: %s", path.String(), err) + } + return fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink +} + +func (b *earlyModuleContext) Readlink(path Path) string { + dest, err := b.config.fs.Readlink(path.String()) + if err != nil { + b.ModuleErrorf("os.Readlink(%q) failed: %s", path.String(), err) + } + return dest +} + +func (e *earlyModuleContext) Module() Module { + module, _ := e.EarlyModuleContext.Module().(Module) + return module +} + +func (e *earlyModuleContext) Config() Config { + return e.EarlyModuleContext.Config().(Config) +} + +func (e *earlyModuleContext) AConfig() Config { + return e.config +} + +func (e *earlyModuleContext) DeviceConfig() DeviceConfig { + return DeviceConfig{e.config.deviceConfig} +} + +func (e *earlyModuleContext) Platform() bool { + return e.kind == platformModule +} + +func (e *earlyModuleContext) DeviceSpecific() bool { + return e.kind == deviceSpecificModule +} + +func (e *earlyModuleContext) SocSpecific() bool { + return e.kind == socSpecificModule +} + +func (e *earlyModuleContext) ProductSpecific() bool { + return e.kind == productSpecificModule +} + +func (e *earlyModuleContext) SystemExtSpecific() bool { + return e.kind == systemExtSpecificModule } -type androidBaseContextImpl struct { +type baseModuleContext struct { + bp blueprint.BaseModuleContext + earlyModuleContext + os OsType target Target multiTargets []Target targetPrimary bool debug bool - kind moduleKind - config Config + + walkPath []Module + tagPath []blueprint.DependencyTag + + strictVisitDeps bool // If true, enforce that all dependencies are enabled } -type androidModuleContext struct { - blueprint.ModuleContext - androidBaseContextImpl +func (b *baseModuleContext) OtherModuleName(m blueprint.Module) string { + return b.bp.OtherModuleName(m) +} +func (b *baseModuleContext) OtherModuleDir(m blueprint.Module) string { return b.bp.OtherModuleDir(m) } +func (b *baseModuleContext) OtherModuleErrorf(m blueprint.Module, fmt string, args ...interface{}) { + b.bp.OtherModuleErrorf(m, fmt, args...) +} +func (b *baseModuleContext) OtherModuleDependencyTag(m blueprint.Module) blueprint.DependencyTag { + return b.bp.OtherModuleDependencyTag(m) +} +func (b *baseModuleContext) OtherModuleExists(name string) bool { return b.bp.OtherModuleExists(name) } +func (b *baseModuleContext) OtherModuleType(m blueprint.Module) string { + return b.bp.OtherModuleType(m) +} + +func (b *baseModuleContext) GetDirectDepWithTag(name string, tag blueprint.DependencyTag) blueprint.Module { + return b.bp.GetDirectDepWithTag(name, tag) +} + +type moduleContext struct { + bp blueprint.ModuleContext + baseModuleContext installDeps Paths installFiles Paths checkbuildFiles Paths - missingDeps []string module Module + phonies map[string]Paths // For tests buildParams []BuildParams @@ -932,25 +1429,22 @@ type androidModuleContext struct { variables map[string]string } -func (a *androidModuleContext) ninjaError(desc string, outputs []string, err error) { - a.ModuleContext.Build(pctx.PackageContext, blueprint.BuildParams{ - Rule: ErrorRule, - Description: desc, - Outputs: outputs, - Optional: true, +func (m *moduleContext) ninjaError(params BuildParams, err error) (PackageContext, BuildParams) { + return pctx, BuildParams{ + Rule: ErrorRule, + Description: params.Description, + Output: params.Output, + Outputs: params.Outputs, + ImplicitOutput: params.ImplicitOutput, + ImplicitOutputs: params.ImplicitOutputs, Args: map[string]string{ "error": err.Error(), }, - }) - return -} - -func (a *androidModuleContext) Config() Config { - return a.ModuleContext.Config().(Config) + } } -func (a *androidModuleContext) ModuleBuild(pctx PackageContext, params ModuleBuildParams) { - a.Build(pctx, BuildParams(params)) +func (m *moduleContext) ModuleBuild(pctx PackageContext, params ModuleBuildParams) { + m.Build(pctx, BuildParams(params)) } func convertBuildParams(params BuildParams) blueprint.BuildParams { @@ -993,92 +1487,108 @@ func convertBuildParams(params BuildParams) blueprint.BuildParams { return bparams } -func (a *androidModuleContext) Variable(pctx PackageContext, name, value string) { - if a.config.captureBuild { - a.variables[name] = value +func (m *moduleContext) Variable(pctx PackageContext, name, value string) { + if m.config.captureBuild { + m.variables[name] = value } - a.ModuleContext.Variable(pctx.PackageContext, name, value) + m.bp.Variable(pctx.PackageContext, name, value) } -func (a *androidModuleContext) Rule(pctx PackageContext, name string, params blueprint.RuleParams, +func (m *moduleContext) Rule(pctx PackageContext, name string, params blueprint.RuleParams, argNames ...string) blueprint.Rule { - if (a.config.UseGoma() || a.config.UseRBE()) && params.Pool == nil { - // When USE_GOMA=true or USE_RBE=true are set and the rule is not supported by goma/RBE, restrict - // jobs to the local parallelism value - params.Pool = localPool + if m.config.UseRemoteBuild() { + if params.Pool == nil { + // When USE_GOMA=true or USE_RBE=true are set and the rule is not supported by goma/RBE, restrict + // jobs to the local parallelism value + params.Pool = localPool + } else if params.Pool == remotePool { + // remotePool is a fake pool used to identify rule that are supported for remoting. If the rule's + // pool is the remotePool, replace with nil so that ninja runs it at NINJA_REMOTE_NUM_JOBS + // parallelism. + params.Pool = nil + } } - rule := a.ModuleContext.Rule(pctx.PackageContext, name, params, argNames...) + rule := m.bp.Rule(pctx.PackageContext, name, params, argNames...) - if a.config.captureBuild { - a.ruleParams[rule] = params + if m.config.captureBuild { + m.ruleParams[rule] = params } return rule } -func (a *androidModuleContext) Build(pctx PackageContext, params BuildParams) { - if a.config.captureBuild { - a.buildParams = append(a.buildParams, params) +func (m *moduleContext) Build(pctx PackageContext, params BuildParams) { + if params.Description != "" { + params.Description = "${moduleDesc}" + params.Description + "${moduleDescSuffix}" } - bparams := convertBuildParams(params) - - if bparams.Description != "" { - bparams.Description = "${moduleDesc}" + params.Description + "${moduleDescSuffix}" + if missingDeps := m.GetMissingDependencies(); len(missingDeps) > 0 { + pctx, params = m.ninjaError(params, fmt.Errorf("module %s missing dependencies: %s\n", + m.ModuleName(), strings.Join(missingDeps, ", "))) } - if a.missingDeps != nil { - a.ninjaError(bparams.Description, bparams.Outputs, - fmt.Errorf("module %s missing dependencies: %s\n", - a.ModuleName(), strings.Join(a.missingDeps, ", "))) - return + if m.config.captureBuild { + m.buildParams = append(m.buildParams, params) } - a.ModuleContext.Build(pctx.PackageContext, bparams) + m.bp.Build(pctx.PackageContext, convertBuildParams(params)) +} + +func (m *moduleContext) Phony(name string, deps ...Path) { + addPhony(m.config, name, deps...) } -func (a *androidModuleContext) GetMissingDependencies() []string { - return a.missingDeps +func (m *moduleContext) GetMissingDependencies() []string { + var missingDeps []string + missingDeps = append(missingDeps, m.Module().base().commonProperties.MissingDeps...) + missingDeps = append(missingDeps, m.bp.GetMissingDependencies()...) + missingDeps = FirstUniqueStrings(missingDeps) + return missingDeps } -func (a *androidModuleContext) AddMissingDependencies(deps []string) { +func (b *baseModuleContext) AddMissingDependencies(deps []string) { if deps != nil { - a.missingDeps = append(a.missingDeps, deps...) - a.missingDeps = FirstUniqueStrings(a.missingDeps) + missingDeps := &b.Module().base().commonProperties.MissingDeps + *missingDeps = append(*missingDeps, deps...) + *missingDeps = FirstUniqueStrings(*missingDeps) } } -func (a *androidModuleContext) validateAndroidModule(module blueprint.Module) Module { +func (b *baseModuleContext) validateAndroidModule(module blueprint.Module, strict bool) Module { aModule, _ := module.(Module) + + if !strict { + return aModule + } + if aModule == nil { - a.ModuleErrorf("module %q not an android module", a.OtherModuleName(aModule)) + b.ModuleErrorf("module %q not an android module", b.OtherModuleName(module)) return nil } if !aModule.Enabled() { - if a.Config().AllowMissingDependencies() { - a.AddMissingDependencies([]string{a.OtherModuleName(aModule)}) + if b.Config().AllowMissingDependencies() { + b.AddMissingDependencies([]string{b.OtherModuleName(aModule)}) } else { - a.ModuleErrorf("depends on disabled module %q", a.OtherModuleName(aModule)) + b.ModuleErrorf("depends on disabled module %q", b.OtherModuleName(aModule)) } return nil } - return aModule } -func (a *androidModuleContext) getDirectDepInternal(name string, tag blueprint.DependencyTag) (blueprint.Module, blueprint.DependencyTag) { +func (b *baseModuleContext) getDirectDepInternal(name string, tag blueprint.DependencyTag) (blueprint.Module, blueprint.DependencyTag) { type dep struct { mod blueprint.Module tag blueprint.DependencyTag } var deps []dep - a.VisitDirectDepsBlueprint(func(m blueprint.Module) { - if aModule, _ := m.(Module); aModule != nil && aModule.base().BaseModuleName() == name { - returnedTag := a.ModuleContext.OtherModuleDependencyTag(aModule) + b.VisitDirectDepsBlueprint(func(module blueprint.Module) { + if aModule, _ := module.(Module); aModule != nil && aModule.base().BaseModuleName() == name { + returnedTag := b.bp.OtherModuleDependencyTag(aModule) if tag == nil || returnedTag == tag { deps = append(deps, dep{aModule, returnedTag}) } @@ -1088,48 +1598,60 @@ func (a *androidModuleContext) getDirectDepInternal(name string, tag blueprint.D return deps[0].mod, deps[0].tag } else if len(deps) >= 2 { panic(fmt.Errorf("Multiple dependencies having same BaseModuleName() %q found from %q", - name, a.ModuleName())) + name, b.ModuleName())) } else { return nil, nil } } -func (a *androidModuleContext) GetDirectDepWithTag(name string, tag blueprint.DependencyTag) blueprint.Module { - m, _ := a.getDirectDepInternal(name, tag) - return m +func (b *baseModuleContext) GetDirectDepsWithTag(tag blueprint.DependencyTag) []Module { + var deps []Module + b.VisitDirectDepsBlueprint(func(module blueprint.Module) { + if aModule, _ := module.(Module); aModule != nil { + if b.bp.OtherModuleDependencyTag(aModule) == tag { + deps = append(deps, aModule) + } + } + }) + return deps +} + +func (m *moduleContext) GetDirectDepWithTag(name string, tag blueprint.DependencyTag) blueprint.Module { + module, _ := m.getDirectDepInternal(name, tag) + return module } -func (a *androidModuleContext) GetDirectDep(name string) (blueprint.Module, blueprint.DependencyTag) { - return a.getDirectDepInternal(name, nil) +func (b *baseModuleContext) GetDirectDep(name string) (blueprint.Module, blueprint.DependencyTag) { + return b.getDirectDepInternal(name, nil) } -func (a *androidModuleContext) VisitDirectDepsBlueprint(visit func(blueprint.Module)) { - a.ModuleContext.VisitDirectDeps(visit) +func (b *baseModuleContext) VisitDirectDepsBlueprint(visit func(blueprint.Module)) { + b.bp.VisitDirectDeps(visit) } -func (a *androidModuleContext) VisitDirectDeps(visit func(Module)) { - a.ModuleContext.VisitDirectDeps(func(module blueprint.Module) { - if aModule := a.validateAndroidModule(module); aModule != nil { +func (b *baseModuleContext) VisitDirectDeps(visit func(Module)) { + b.bp.VisitDirectDeps(func(module blueprint.Module) { + if aModule := b.validateAndroidModule(module, b.strictVisitDeps); aModule != nil { visit(aModule) } }) } -func (a *androidModuleContext) VisitDirectDepsWithTag(tag blueprint.DependencyTag, visit func(Module)) { - a.ModuleContext.VisitDirectDeps(func(module blueprint.Module) { - if aModule := a.validateAndroidModule(module); aModule != nil { - if a.ModuleContext.OtherModuleDependencyTag(aModule) == tag { +func (b *baseModuleContext) VisitDirectDepsWithTag(tag blueprint.DependencyTag, visit func(Module)) { + b.bp.VisitDirectDeps(func(module blueprint.Module) { + if aModule := b.validateAndroidModule(module, b.strictVisitDeps); aModule != nil { + if b.bp.OtherModuleDependencyTag(aModule) == tag { visit(aModule) } } }) } -func (a *androidModuleContext) VisitDirectDepsIf(pred func(Module) bool, visit func(Module)) { - a.ModuleContext.VisitDirectDepsIf( +func (b *baseModuleContext) VisitDirectDepsIf(pred func(Module) bool, visit func(Module)) { + b.bp.VisitDirectDepsIf( // pred func(module blueprint.Module) bool { - if aModule := a.validateAndroidModule(module); aModule != nil { + if aModule := b.validateAndroidModule(module, b.strictVisitDeps); aModule != nil { return pred(aModule) } else { return false @@ -1141,19 +1663,19 @@ func (a *androidModuleContext) VisitDirectDepsIf(pred func(Module) bool, visit f }) } -func (a *androidModuleContext) VisitDepsDepthFirst(visit func(Module)) { - a.ModuleContext.VisitDepsDepthFirst(func(module blueprint.Module) { - if aModule := a.validateAndroidModule(module); aModule != nil { +func (b *baseModuleContext) VisitDepsDepthFirst(visit func(Module)) { + b.bp.VisitDepsDepthFirst(func(module blueprint.Module) { + if aModule := b.validateAndroidModule(module, b.strictVisitDeps); aModule != nil { visit(aModule) } }) } -func (a *androidModuleContext) VisitDepsDepthFirstIf(pred func(Module) bool, visit func(Module)) { - a.ModuleContext.VisitDepsDepthFirstIf( +func (b *baseModuleContext) VisitDepsDepthFirstIf(pred func(Module) bool, visit func(Module)) { + b.bp.VisitDepsDepthFirstIf( // pred func(module blueprint.Module) bool { - if aModule := a.validateAndroidModule(module); aModule != nil { + if aModule := b.validateAndroidModule(module, b.strictVisitDeps); aModule != nil { return pred(aModule) } else { return false @@ -1165,15 +1687,24 @@ func (a *androidModuleContext) VisitDepsDepthFirstIf(pred func(Module) bool, vis }) } -func (a *androidModuleContext) WalkDepsBlueprint(visit func(blueprint.Module, blueprint.Module) bool) { - a.ModuleContext.WalkDeps(visit) +func (b *baseModuleContext) WalkDepsBlueprint(visit func(blueprint.Module, blueprint.Module) bool) { + b.bp.WalkDeps(visit) } -func (a *androidModuleContext) WalkDeps(visit func(Module, Module) bool) { - a.ModuleContext.WalkDeps(func(child, parent blueprint.Module) bool { - childAndroidModule := a.validateAndroidModule(child) - parentAndroidModule := a.validateAndroidModule(parent) +func (b *baseModuleContext) WalkDeps(visit func(Module, Module) bool) { + b.walkPath = []Module{b.Module()} + b.tagPath = []blueprint.DependencyTag{} + b.bp.WalkDeps(func(child, parent blueprint.Module) bool { + childAndroidModule, _ := child.(Module) + parentAndroidModule, _ := parent.(Module) if childAndroidModule != nil && parentAndroidModule != nil { + // record walkPath before visit + for b.walkPath[len(b.walkPath)-1] != parentAndroidModule { + b.walkPath = b.walkPath[0 : len(b.walkPath)-1] + b.tagPath = b.tagPath[0 : len(b.tagPath)-1] + } + b.walkPath = append(b.walkPath, childAndroidModule) + b.tagPath = append(b.tagPath, b.OtherModuleDependencyTag(childAndroidModule)) return visit(childAndroidModule, parentAndroidModule) } else { return false @@ -1181,139 +1712,156 @@ func (a *androidModuleContext) WalkDeps(visit func(Module, Module) bool) { }) } -func (a *androidModuleContext) VisitAllModuleVariants(visit func(Module)) { - a.ModuleContext.VisitAllModuleVariants(func(module blueprint.Module) { +func (b *baseModuleContext) GetWalkPath() []Module { + return b.walkPath +} + +func (b *baseModuleContext) GetTagPath() []blueprint.DependencyTag { + return b.tagPath +} + +func (m *moduleContext) VisitAllModuleVariants(visit func(Module)) { + m.bp.VisitAllModuleVariants(func(module blueprint.Module) { visit(module.(Module)) }) } -func (a *androidModuleContext) PrimaryModule() Module { - return a.ModuleContext.PrimaryModule().(Module) +func (m *moduleContext) PrimaryModule() Module { + return m.bp.PrimaryModule().(Module) } -func (a *androidModuleContext) FinalModule() Module { - return a.ModuleContext.FinalModule().(Module) +func (m *moduleContext) FinalModule() Module { + return m.bp.FinalModule().(Module) } -func (a *androidBaseContextImpl) Target() Target { - return a.target +func (m *moduleContext) ModuleSubDir() string { + return m.bp.ModuleSubDir() } -func (a *androidBaseContextImpl) TargetPrimary() bool { - return a.targetPrimary +func (b *baseModuleContext) Target() Target { + return b.target } -func (a *androidBaseContextImpl) MultiTargets() []Target { - return a.multiTargets +func (b *baseModuleContext) TargetPrimary() bool { + return b.targetPrimary } -func (a *androidBaseContextImpl) Arch() Arch { - return a.target.Arch +func (b *baseModuleContext) MultiTargets() []Target { + return b.multiTargets } -func (a *androidBaseContextImpl) Os() OsType { - return a.target.Os +func (b *baseModuleContext) Arch() Arch { + return b.target.Arch } -func (a *androidBaseContextImpl) Host() bool { - return a.target.Os.Class == Host || a.target.Os.Class == HostCross +func (b *baseModuleContext) Os() OsType { + return b.os } -func (a *androidBaseContextImpl) Device() bool { - return a.target.Os.Class == Device +func (b *baseModuleContext) Host() bool { + return b.os.Class == Host || b.os.Class == HostCross } -func (a *androidBaseContextImpl) Darwin() bool { - return a.target.Os == Darwin +func (b *baseModuleContext) Device() bool { + return b.os.Class == Device } -func (a *androidBaseContextImpl) Fuchsia() bool { - return a.target.Os == Fuchsia +func (b *baseModuleContext) Darwin() bool { + return b.os == Darwin } -func (a *androidBaseContextImpl) Windows() bool { - return a.target.Os == Windows +func (b *baseModuleContext) Fuchsia() bool { + return b.os == Fuchsia } -func (a *androidBaseContextImpl) Debug() bool { - return a.debug +func (b *baseModuleContext) Windows() bool { + return b.os == Windows } -func (a *androidBaseContextImpl) PrimaryArch() bool { - if len(a.config.Targets[a.target.Os]) <= 1 { +func (b *baseModuleContext) Debug() bool { + return b.debug +} + +func (b *baseModuleContext) PrimaryArch() bool { + if len(b.config.Targets[b.target.Os]) <= 1 { return true } - return a.target.Arch.ArchType == a.config.Targets[a.target.Os][0].Arch.ArchType + return b.target.Arch.ArchType == b.config.Targets[b.target.Os][0].Arch.ArchType } -func (a *androidBaseContextImpl) AConfig() Config { - return a.config +// Makes this module a platform module, i.e. not specific to soc, device, +// product, or system_ext. +func (m *ModuleBase) MakeAsPlatform() { + m.commonProperties.Vendor = boolPtr(false) + m.commonProperties.Proprietary = boolPtr(false) + m.commonProperties.Soc_specific = boolPtr(false) + m.commonProperties.Product_specific = boolPtr(false) + m.commonProperties.System_ext_specific = boolPtr(false) } -func (a *androidBaseContextImpl) DeviceConfig() DeviceConfig { - return DeviceConfig{a.config.deviceConfig} +func (m *ModuleBase) EnableNativeBridgeSupportByDefault() { + m.commonProperties.Native_bridge_supported = boolPtr(true) } -func (a *androidBaseContextImpl) Platform() bool { - return a.kind == platformModule +func (m *ModuleBase) MakeAsSystemExt() { + m.commonProperties.Vendor = boolPtr(false) + m.commonProperties.Proprietary = boolPtr(false) + m.commonProperties.Soc_specific = boolPtr(false) + m.commonProperties.Product_specific = boolPtr(false) + m.commonProperties.System_ext_specific = boolPtr(true) } -func (a *androidBaseContextImpl) DeviceSpecific() bool { - return a.kind == deviceSpecificModule +// IsNativeBridgeSupported returns true if "native_bridge_supported" is explicitly set as "true" +func (m *ModuleBase) IsNativeBridgeSupported() bool { + return proptools.Bool(m.commonProperties.Native_bridge_supported) } -func (a *androidBaseContextImpl) SocSpecific() bool { - return a.kind == socSpecificModule +func (m *moduleContext) InstallInData() bool { + return m.module.InstallInData() } -func (a *androidBaseContextImpl) ProductSpecific() bool { - return a.kind == productSpecificModule +func (m *moduleContext) InstallInTestcases() bool { + return m.module.InstallInTestcases() } -func (a *androidBaseContextImpl) ProductServicesSpecific() bool { - return a.kind == productServicesSpecificModule +func (m *moduleContext) InstallInSanitizerDir() bool { + return m.module.InstallInSanitizerDir() } -// Makes this module a platform module, i.e. not specific to soc, device, -// product, or product_services. -func (a *ModuleBase) MakeAsPlatform() { - a.commonProperties.Vendor = boolPtr(false) - a.commonProperties.Proprietary = boolPtr(false) - a.commonProperties.Soc_specific = boolPtr(false) - a.commonProperties.Product_specific = boolPtr(false) - a.commonProperties.Product_services_specific = boolPtr(false) +func (m *moduleContext) InstallInRamdisk() bool { + return m.module.InstallInRamdisk() } -func (a *androidModuleContext) InstallInData() bool { - return a.module.InstallInData() +func (m *moduleContext) InstallInRecovery() bool { + return m.module.InstallInRecovery() } -func (a *androidModuleContext) InstallInSanitizerDir() bool { - return a.module.InstallInSanitizerDir() +func (m *moduleContext) InstallInRoot() bool { + return m.module.InstallInRoot() } -func (a *androidModuleContext) InstallInRecovery() bool { - return a.module.InstallInRecovery() +func (m *moduleContext) InstallBypassMake() bool { + return m.module.InstallBypassMake() } -func (a *androidModuleContext) skipInstall(fullInstallPath OutputPath) bool { - if a.module.base().commonProperties.SkipInstall { +func (m *moduleContext) skipInstall(fullInstallPath InstallPath) bool { + if m.module.base().commonProperties.SkipInstall { return true } // We'll need a solution for choosing which of modules with the same name in different // namespaces to install. For now, reuse the list of namespaces exported to Make as the // list of namespaces to install in a Soong-only build. - if !a.module.base().commonProperties.NamespaceExportedToMake { + if !m.module.base().commonProperties.NamespaceExportedToMake { return true } - if a.Device() { - if a.Config().SkipDeviceInstall() { + if m.Device() { + if m.Config().EmbeddedInMake() && !m.InstallBypassMake() { return true } - if a.Config().SkipMegaDeviceInstall(fullInstallPath.String()) { + if m.Config().SkipMegaDeviceInstall(fullInstallPath.String()) { return true } } @@ -1321,29 +1869,29 @@ func (a *androidModuleContext) skipInstall(fullInstallPath OutputPath) bool { return false } -func (a *androidModuleContext) InstallFile(installPath OutputPath, name string, srcPath Path, - deps ...Path) OutputPath { - return a.installFile(installPath, name, srcPath, Cp, deps) +func (m *moduleContext) InstallFile(installPath InstallPath, name string, srcPath Path, + deps ...Path) InstallPath { + return m.installFile(installPath, name, srcPath, Cp, deps) } -func (a *androidModuleContext) InstallExecutable(installPath OutputPath, name string, srcPath Path, - deps ...Path) OutputPath { - return a.installFile(installPath, name, srcPath, CpExecutable, deps) +func (m *moduleContext) InstallExecutable(installPath InstallPath, name string, srcPath Path, + deps ...Path) InstallPath { + return m.installFile(installPath, name, srcPath, CpExecutable, deps) } -func (a *androidModuleContext) installFile(installPath OutputPath, name string, srcPath Path, - rule blueprint.Rule, deps []Path) OutputPath { +func (m *moduleContext) installFile(installPath InstallPath, name string, srcPath Path, + rule blueprint.Rule, deps []Path) InstallPath { - fullInstallPath := installPath.Join(a, name) - a.module.base().hooks.runInstallHooks(a, fullInstallPath, false) + fullInstallPath := installPath.Join(m, name) + m.module.base().hooks.runInstallHooks(m, fullInstallPath, false) - if !a.skipInstall(fullInstallPath) { + if !m.skipInstall(fullInstallPath) { - deps = append(deps, a.installDeps...) + deps = append(deps, m.installDeps...) var implicitDeps, orderOnlyDeps Paths - if a.Host() { + if m.Host() { // Installed host modules might be used during the build, depend directly on their // dependencies so their timestamp is updated whenever their dependency is updated implicitDeps = deps @@ -1351,73 +1899,73 @@ func (a *androidModuleContext) installFile(installPath OutputPath, name string, orderOnlyDeps = deps } - a.Build(pctx, BuildParams{ + m.Build(pctx, BuildParams{ Rule: rule, Description: "install " + fullInstallPath.Base(), Output: fullInstallPath, Input: srcPath, Implicits: implicitDeps, OrderOnly: orderOnlyDeps, - Default: !a.Config().EmbeddedInMake(), + Default: !m.Config().EmbeddedInMake(), }) - a.installFiles = append(a.installFiles, fullInstallPath) + m.installFiles = append(m.installFiles, fullInstallPath) } - a.checkbuildFiles = append(a.checkbuildFiles, srcPath) + m.checkbuildFiles = append(m.checkbuildFiles, srcPath) return fullInstallPath } -func (a *androidModuleContext) InstallSymlink(installPath OutputPath, name string, srcPath OutputPath) OutputPath { - fullInstallPath := installPath.Join(a, name) - a.module.base().hooks.runInstallHooks(a, fullInstallPath, true) +func (m *moduleContext) InstallSymlink(installPath InstallPath, name string, srcPath InstallPath) InstallPath { + fullInstallPath := installPath.Join(m, name) + m.module.base().hooks.runInstallHooks(m, fullInstallPath, true) - if !a.skipInstall(fullInstallPath) { + if !m.skipInstall(fullInstallPath) { relPath, err := filepath.Rel(path.Dir(fullInstallPath.String()), srcPath.String()) if err != nil { panic(fmt.Sprintf("Unable to generate symlink between %q and %q: %s", fullInstallPath.Base(), srcPath.Base(), err)) } - a.Build(pctx, BuildParams{ + m.Build(pctx, BuildParams{ Rule: Symlink, Description: "install symlink " + fullInstallPath.Base(), Output: fullInstallPath, - OrderOnly: Paths{srcPath}, - Default: !a.Config().EmbeddedInMake(), + Input: srcPath, + Default: !m.Config().EmbeddedInMake(), Args: map[string]string{ "fromPath": relPath, }, }) - a.installFiles = append(a.installFiles, fullInstallPath) - a.checkbuildFiles = append(a.checkbuildFiles, srcPath) + m.installFiles = append(m.installFiles, fullInstallPath) + m.checkbuildFiles = append(m.checkbuildFiles, srcPath) } return fullInstallPath } // installPath/name -> absPath where absPath might be a path that is available only at runtime // (e.g. /apex/...) -func (a *androidModuleContext) InstallAbsoluteSymlink(installPath OutputPath, name string, absPath string) OutputPath { - fullInstallPath := installPath.Join(a, name) - a.module.base().hooks.runInstallHooks(a, fullInstallPath, true) +func (m *moduleContext) InstallAbsoluteSymlink(installPath InstallPath, name string, absPath string) InstallPath { + fullInstallPath := installPath.Join(m, name) + m.module.base().hooks.runInstallHooks(m, fullInstallPath, true) - if !a.skipInstall(fullInstallPath) { - a.Build(pctx, BuildParams{ + if !m.skipInstall(fullInstallPath) { + m.Build(pctx, BuildParams{ Rule: Symlink, Description: "install symlink " + fullInstallPath.Base() + " -> " + absPath, Output: fullInstallPath, - Default: !a.Config().EmbeddedInMake(), + Default: !m.Config().EmbeddedInMake(), Args: map[string]string{ "fromPath": absPath, }, }) - a.installFiles = append(a.installFiles, fullInstallPath) + m.installFiles = append(m.installFiles, fullInstallPath) } return fullInstallPath } -func (a *androidModuleContext) CheckbuildFile(srcPath Path) { - a.checkbuildFiles = append(a.checkbuildFiles, srcPath) +func (m *moduleContext) CheckbuildFile(srcPath Path) { + m.checkbuildFiles = append(m.checkbuildFiles, srcPath) } type fileInstaller interface { @@ -1443,39 +1991,60 @@ func findStringInSlice(str string, slice []string) int { return -1 } -func SrcIsModule(s string) string { +// SrcIsModule decodes module references in the format ":name" into the module name, or empty string if the input +// was not a module reference. +func SrcIsModule(s string) (module string) { if len(s) > 1 && s[0] == ':' { return s[1:] } return "" } -type sourceDependencyTag struct { +// SrcIsModule decodes module references in the format ":name{.tag}" into the module name and tag, ":name" into the +// module name and an empty string for the tag, or empty strings if the input was not a module reference. +func SrcIsModuleWithTag(s string) (module, tag string) { + if len(s) > 1 && s[0] == ':' { + module = s[1:] + if tagStart := strings.IndexByte(module, '{'); tagStart > 0 { + if module[len(module)-1] == '}' { + tag = module[tagStart+1 : len(module)-1] + module = module[:tagStart] + return module, tag + } + } + return module, "" + } + return "", "" +} + +type sourceOrOutputDependencyTag struct { blueprint.BaseDependencyTag + tag string } -var SourceDepTag sourceDependencyTag +func sourceOrOutputDepTag(tag string) blueprint.DependencyTag { + return sourceOrOutputDependencyTag{tag: tag} +} + +var SourceDepTag = sourceOrOutputDepTag("") // Adds necessary dependencies to satisfy filegroup or generated sources modules listed in srcFiles // using ":module" syntax, if any. // // Deprecated: tag the property with `android:"path"` instead. func ExtractSourcesDeps(ctx BottomUpMutatorContext, srcFiles []string) { - var deps []string set := make(map[string]bool) for _, s := range srcFiles { - if m := SrcIsModule(s); m != "" { - if _, found := set[m]; found { - ctx.ModuleErrorf("found source dependency duplicate: %q!", m) + if m, t := SrcIsModuleWithTag(s); m != "" { + if _, found := set[s]; found { + ctx.ModuleErrorf("found source dependency duplicate: %q!", s) } else { - set[m] = true - deps = append(deps, m) + set[s] = true + ctx.AddDependency(ctx.Module(), sourceOrOutputDepTag(t), m) } } } - - ctx.AddDependency(ctx.Module(), SourceDepTag, deps...) } // Adds necessary dependencies to satisfy filegroup or generated sources modules specified in s @@ -1484,16 +2053,68 @@ func ExtractSourcesDeps(ctx BottomUpMutatorContext, srcFiles []string) { // Deprecated: tag the property with `android:"path"` instead. func ExtractSourceDeps(ctx BottomUpMutatorContext, s *string) { if s != nil { - if m := SrcIsModule(*s); m != "" { - ctx.AddDependency(ctx.Module(), SourceDepTag, m) + if m, t := SrcIsModuleWithTag(*s); m != "" { + ctx.AddDependency(ctx.Module(), sourceOrOutputDepTag(t), m) } } } +// A module that implements SourceFileProducer can be referenced from any property that is tagged with `android:"path"` +// using the ":module" syntax and provides a list of paths to be used as if they were listed in the property. type SourceFileProducer interface { Srcs() Paths } +// A module that implements OutputFileProducer can be referenced from any property that is tagged with `android:"path"` +// using the ":module" syntax or ":module{.tag}" syntax and provides a list of output files to be used as if they were +// listed in the property. +type OutputFileProducer interface { + OutputFiles(tag string) (Paths, error) +} + +// OutputFilesForModule returns the paths from an OutputFileProducer with the given tag. On error, including if the +// module produced zero paths, it reports errors to the ctx and returns nil. +func OutputFilesForModule(ctx PathContext, module blueprint.Module, tag string) Paths { + paths, err := outputFilesForModule(ctx, module, tag) + if err != nil { + reportPathError(ctx, err) + return nil + } + return paths +} + +// OutputFileForModule returns the path from an OutputFileProducer with the given tag. On error, including if the +// module produced zero or multiple paths, it reports errors to the ctx and returns nil. +func OutputFileForModule(ctx PathContext, module blueprint.Module, tag string) Path { + paths, err := outputFilesForModule(ctx, module, tag) + if err != nil { + reportPathError(ctx, err) + return nil + } + if len(paths) > 1 { + reportPathErrorf(ctx, "got multiple output files from module %q, expected exactly one", + pathContextName(ctx, module)) + return nil + } + return paths[0] +} + +func outputFilesForModule(ctx PathContext, module blueprint.Module, tag string) (Paths, error) { + if outputFileProducer, ok := module.(OutputFileProducer); ok { + paths, err := outputFileProducer.OutputFiles(tag) + if err != nil { + return nil, fmt.Errorf("failed to get output file from module %q: %s", + pathContextName(ctx, module), err.Error()) + } + if len(paths) == 0 { + return nil, fmt.Errorf("failed to get output files from module %q", pathContextName(ctx, module)) + } + return paths, nil + } else { + return nil, fmt.Errorf("module %q is not an OutputFileProducer", pathContextName(ctx, module)) + } +} + type HostToolProvider interface { HostToolPath() OptionalPath } @@ -1502,46 +2123,38 @@ type HostToolProvider interface { // be tagged with `android:"path" to support automatic source module dependency resolution. // // Deprecated: use PathsForModuleSrc or PathsForModuleSrcExcludes instead. -func (ctx *androidModuleContext) ExpandSources(srcFiles, excludes []string) Paths { - return PathsForModuleSrcExcludes(ctx, srcFiles, excludes) +func (m *moduleContext) ExpandSources(srcFiles, excludes []string) Paths { + return PathsForModuleSrcExcludes(m, srcFiles, excludes) } // Returns a single path expanded from globs and modules referenced using ":module" syntax. The property must // be tagged with `android:"path" to support automatic source module dependency resolution. // // Deprecated: use PathForModuleSrc instead. -func (ctx *androidModuleContext) ExpandSource(srcFile, prop string) Path { - return PathForModuleSrc(ctx, srcFile) +func (m *moduleContext) ExpandSource(srcFile, prop string) Path { + return PathForModuleSrc(m, srcFile) } // Returns an optional single path expanded from globs and modules referenced using ":module" syntax if // the srcFile is non-nil. The property must be tagged with `android:"path" to support automatic source module // dependency resolution. -func (ctx *androidModuleContext) ExpandOptionalSource(srcFile *string, prop string) OptionalPath { +func (m *moduleContext) ExpandOptionalSource(srcFile *string, prop string) OptionalPath { if srcFile != nil { - return OptionalPathForPath(PathForModuleSrc(ctx, *srcFile)) + return OptionalPathForPath(PathForModuleSrc(m, *srcFile)) } return OptionalPath{} } -func (ctx *androidModuleContext) RequiredModuleNames() []string { - return ctx.module.base().commonProperties.Required +func (m *moduleContext) RequiredModuleNames() []string { + return m.module.RequiredModuleNames() } -func (ctx *androidModuleContext) Glob(globPattern string, excludes []string) Paths { - ret, err := ctx.GlobWithDeps(globPattern, excludes) - if err != nil { - ctx.ModuleErrorf("glob: %s", err.Error()) - } - return pathsForModuleSrcFromFullPath(ctx, ret, true) +func (m *moduleContext) HostRequiredModuleNames() []string { + return m.module.HostRequiredModuleNames() } -func (ctx *androidModuleContext) GlobFiles(globPattern string, excludes []string) Paths { - ret, err := ctx.GlobWithDeps(globPattern, excludes) - if err != nil { - ctx.ModuleErrorf("glob: %s", err.Error()) - } - return pathsForModuleSrcFromFullPath(ctx, ret, false) +func (m *moduleContext) TargetRequiredModuleNames() []string { + return m.module.TargetRequiredModuleNames() } func init() { @@ -1562,9 +2175,8 @@ type buildTargetSingleton struct{} func (c *buildTargetSingleton) GenerateBuildActions(ctx SingletonContext) { var checkbuildDeps Paths - mmTarget := func(dir string) WritablePath { - return PathForPhony(ctx, - "MODULES-IN-"+strings.Replace(filepath.Clean(dir), "/", "-", -1)) + mmTarget := func(dir string) string { + return "MODULES-IN-" + strings.Replace(filepath.Clean(dir), "/", "-", -1) } modulesInDir := make(map[string]Paths) @@ -1590,28 +2202,15 @@ func (c *buildTargetSingleton) GenerateBuildActions(ctx SingletonContext) { } // Create a top-level checkbuild target that depends on all modules - ctx.Build(pctx, BuildParams{ - Rule: blueprint.Phony, - Output: PathForPhony(ctx, "checkbuild"+suffix), - Implicits: checkbuildDeps, - }) + ctx.Phony("checkbuild"+suffix, checkbuildDeps...) // Make will generate the MODULES-IN-* targets if ctx.Config().EmbeddedInMake() { return } - sortedKeys := func(m map[string]Paths) []string { - s := make([]string, 0, len(m)) - for k := range m { - s = append(s, k) - } - sort.Strings(s) - return s - } - // Ensure ancestor directories are in modulesInDir - dirs := sortedKeys(modulesInDir) + dirs := SortedStringKeys(modulesInDir) for _, dir := range dirs { dir := parentDir(dir) for dir != "." && dir != "/" { @@ -1624,11 +2223,10 @@ func (c *buildTargetSingleton) GenerateBuildActions(ctx SingletonContext) { } // Make directories build their direct subdirectories - dirs = sortedKeys(modulesInDir) for _, dir := range dirs { p := parentDir(dir) if p != "." && p != "/" { - modulesInDir[p] = append(modulesInDir[p], mmTarget(dir)) + modulesInDir[p] = append(modulesInDir[p], PathForPhony(ctx, mmTarget(dir))) } } @@ -1636,14 +2234,7 @@ func (c *buildTargetSingleton) GenerateBuildActions(ctx SingletonContext) { // depends on the MODULES-IN-* targets of all of its subdirectories that contain Android.bp // files. for _, dir := range dirs { - ctx.Build(pctx, BuildParams{ - Rule: blueprint.Phony, - Output: mmTarget(dir), - Implicits: modulesInDir[dir], - // HACK: checkbuild should be an optional build, but force it - // enabled for now in standalone builds - Default: !ctx.Config().EmbeddedInMake(), - }) + ctx.Phony(mmTarget(dir), modulesInDir[dir]...) } // Create (host|host-cross|target)-<OS> phony rules to build a reduced checkbuild. @@ -1670,24 +2261,15 @@ func (c *buildTargetSingleton) GenerateBuildActions(ctx SingletonContext) { continue } - name := PathForPhony(ctx, className+"-"+os.Name) - osClass[className] = append(osClass[className], name) + name := className + "-" + os.Name + osClass[className] = append(osClass[className], PathForPhony(ctx, name)) - ctx.Build(pctx, BuildParams{ - Rule: blueprint.Phony, - Output: name, - Implicits: deps, - }) + ctx.Phony(name, deps...) } // Wrap those into host|host-cross|target phony rules - osClasses := sortedKeys(osClass) - for _, class := range osClasses { - ctx.Build(pctx, BuildParams{ - Rule: blueprint.Phony, - Output: PathForPhony(ctx, class), - Implicits: osClass[class], - }) + for _, class := range SortedStringKeys(osClass) { + ctx.Phony(class, osClass[class]...) } } @@ -1713,4 +2295,5 @@ type IdeInfo struct { Jars []string `json:"jars,omitempty"` Classes []string `json:"class,omitempty"` Installed_paths []string `json:"installed,omitempty"` + SrcJars []string `json:"srcjars,omitempty"` } diff --git a/android/module_test.go b/android/module_test.go new file mode 100644 index 000000000..6e648d7bb --- /dev/null +++ b/android/module_test.go @@ -0,0 +1,189 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "testing" +) + +func TestSrcIsModule(t *testing.T) { + type args struct { + s string + } + tests := []struct { + name string + args args + wantModule string + }{ + { + name: "file", + args: args{ + s: "foo", + }, + wantModule: "", + }, + { + name: "module", + args: args{ + s: ":foo", + }, + wantModule: "foo", + }, + { + name: "tag", + args: args{ + s: ":foo{.bar}", + }, + wantModule: "foo{.bar}", + }, + { + name: "extra colon", + args: args{ + s: ":foo:bar", + }, + wantModule: "foo:bar", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if gotModule := SrcIsModule(tt.args.s); gotModule != tt.wantModule { + t.Errorf("SrcIsModule() = %v, want %v", gotModule, tt.wantModule) + } + }) + } +} + +func TestSrcIsModuleWithTag(t *testing.T) { + type args struct { + s string + } + tests := []struct { + name string + args args + wantModule string + wantTag string + }{ + { + name: "file", + args: args{ + s: "foo", + }, + wantModule: "", + wantTag: "", + }, + { + name: "module", + args: args{ + s: ":foo", + }, + wantModule: "foo", + wantTag: "", + }, + { + name: "tag", + args: args{ + s: ":foo{.bar}", + }, + wantModule: "foo", + wantTag: ".bar", + }, + { + name: "empty tag", + args: args{ + s: ":foo{}", + }, + wantModule: "foo", + wantTag: "", + }, + { + name: "extra colon", + args: args{ + s: ":foo:bar", + }, + wantModule: "foo:bar", + }, + { + name: "invalid tag", + args: args{ + s: ":foo{.bar", + }, + wantModule: "foo{.bar", + }, + { + name: "invalid tag 2", + args: args{ + s: ":foo.bar}", + }, + wantModule: "foo.bar}", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotModule, gotTag := SrcIsModuleWithTag(tt.args.s) + if gotModule != tt.wantModule { + t.Errorf("SrcIsModuleWithTag() gotModule = %v, want %v", gotModule, tt.wantModule) + } + if gotTag != tt.wantTag { + t.Errorf("SrcIsModuleWithTag() gotTag = %v, want %v", gotTag, tt.wantTag) + } + }) + } +} + +type depsModule struct { + ModuleBase + props struct { + Deps []string + } +} + +func (m *depsModule) GenerateAndroidBuildActions(ctx ModuleContext) { +} + +func (m *depsModule) DepsMutator(ctx BottomUpMutatorContext) { + ctx.AddDependency(ctx.Module(), nil, m.props.Deps...) +} + +func depsModuleFactory() Module { + m := &depsModule{} + m.AddProperties(&m.props) + InitAndroidModule(m) + return m +} + +func TestErrorDependsOnDisabledModule(t *testing.T) { + ctx := NewTestContext() + ctx.RegisterModuleType("deps", depsModuleFactory) + + bp := ` + deps { + name: "foo", + deps: ["bar"], + } + deps { + name: "bar", + enabled: false, + } + ` + + config := TestConfig(buildDir, nil, bp, nil) + + ctx.Register(config) + + _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) + FailIfErrored(t, errs) + _, errs = ctx.PrepareBuildActions(config) + FailIfNoMatchingErrors(t, `module "foo": depends on disabled module "bar"`, errs) +} diff --git a/android/mutator.go b/android/mutator.go index 68ba6f476..77d5f433e 100644 --- a/android/mutator.go +++ b/android/mutator.go @@ -15,6 +15,8 @@ package android import ( + "reflect" + "github.com/google/blueprint" "github.com/google/blueprint/proptools" ) @@ -25,6 +27,7 @@ import ( // run Pre-deps mutators // run depsMutator // run PostDeps mutators +// run FinalDeps mutators (CreateVariations disallowed in this phase) // continue on to GenerateAndroidBuildActions func registerMutatorsToContext(ctx *blueprint.Context, mutators []*mutator) { @@ -41,7 +44,7 @@ func registerMutatorsToContext(ctx *blueprint.Context, mutators []*mutator) { } } -func registerMutators(ctx *blueprint.Context, preArch, preDeps, postDeps []RegisterMutatorFunc) { +func registerMutators(ctx *blueprint.Context, preArch, preDeps, postDeps, finalDeps []RegisterMutatorFunc) { mctx := ®isterMutatorsContext{} register := func(funcs []RegisterMutatorFunc) { @@ -58,34 +61,79 @@ func registerMutators(ctx *blueprint.Context, preArch, preDeps, postDeps []Regis register(postDeps) + mctx.finalPhase = true + register(finalDeps) + registerMutatorsToContext(ctx, mctx.mutators) } type registerMutatorsContext struct { - mutators []*mutator + mutators []*mutator + finalPhase bool } type RegisterMutatorsContext interface { - TopDown(name string, m AndroidTopDownMutator) MutatorHandle - BottomUp(name string, m AndroidBottomUpMutator) MutatorHandle + TopDown(name string, m TopDownMutator) MutatorHandle + BottomUp(name string, m BottomUpMutator) MutatorHandle } type RegisterMutatorFunc func(RegisterMutatorsContext) var preArch = []RegisterMutatorFunc{ - func(ctx RegisterMutatorsContext) { - ctx.TopDown("load_hooks", LoadHookMutator).Parallel() - }, RegisterNamespaceMutator, - registerPackageRenamer, - RegisterPrebuiltsPreArchMutators, + + // Check the visibility rules are valid. + // + // This must run after the package renamer mutators so that any issues found during + // validation of the package's default_visibility property are reported using the + // correct package name and not the synthetic name. + // + // This must also be run before defaults mutators as the rules for validation are + // different before checking the rules than they are afterwards. e.g. + // visibility: ["//visibility:private", "//visibility:public"] + // would be invalid if specified in a module definition but is valid if it results + // from something like this: + // + // defaults { + // name: "defaults", + // // Be inaccessible outside a package by default. + // visibility: ["//visibility:private"] + // } + // + // defaultable_module { + // name: "defaultable_module", + // defaults: ["defaults"], + // // Override the default. + // visibility: ["//visibility:public"] + // } + // + RegisterVisibilityRuleChecker, + + // Apply properties from defaults modules to the referencing modules. + // + // Any mutators that are added before this will not see any modules created by + // a DefaultableHook. RegisterDefaultsPreArchMutators, - RegisterOverridePreArchMutators, + + // Create an association between prebuilt modules and their corresponding source + // modules (if any). + // + // Must be run after defaults mutators to ensure that any modules created by + // a DefaultableHook can be either a prebuilt or a source module with a matching + // prebuilt. + RegisterPrebuiltsPreArchMutators, + + // Gather the visibility rules for all modules for us during visibility enforcement. + // + // This must come after the defaults mutators to ensure that any visibility supplied + // in a defaults module has been successfully applied before the rules are gathered. + RegisterVisibilityRuleGatherer, } func registerArchMutator(ctx RegisterMutatorsContext) { + ctx.BottomUp("os", osMutator).Parallel() + ctx.BottomUp("image", imageMutator).Parallel() ctx.BottomUp("arch", archMutator).Parallel() - ctx.TopDown("arch_hooks", archHookMutator).Parallel() } var preDeps = []RegisterMutatorFunc{ @@ -95,9 +143,13 @@ var preDeps = []RegisterMutatorFunc{ var postDeps = []RegisterMutatorFunc{ registerPathDepsMutator, RegisterPrebuiltsPostDepsMutators, - registerNeverallowMutator, + RegisterVisibilityRuleEnforcer, + RegisterNeverallowMutator, + RegisterOverridePostDepsMutators, } +var finalDeps = []RegisterMutatorFunc{} + func PreArchMutators(f RegisterMutatorFunc) { preArch = append(preArch, f) } @@ -110,75 +162,63 @@ func PostDepsMutators(f RegisterMutatorFunc) { postDeps = append(postDeps, f) } -type AndroidTopDownMutator func(TopDownMutatorContext) +func FinalDepsMutators(f RegisterMutatorFunc) { + finalDeps = append(finalDeps, f) +} + +type TopDownMutator func(TopDownMutatorContext) type TopDownMutatorContext interface { BaseModuleContext - androidBaseContext - - OtherModuleExists(name string) bool - Rename(name string) - Module() Module - OtherModuleName(m blueprint.Module) string - OtherModuleErrorf(m blueprint.Module, fmt string, args ...interface{}) - OtherModuleDependencyTag(m blueprint.Module) blueprint.DependencyTag + MutatorName() string - CreateModule(blueprint.ModuleFactory, ...interface{}) - - GetDirectDepWithTag(name string, tag blueprint.DependencyTag) blueprint.Module - GetDirectDep(name string) (blueprint.Module, blueprint.DependencyTag) + Rename(name string) - VisitDirectDeps(visit func(Module)) - VisitDirectDepsWithTag(tag blueprint.DependencyTag, visit func(Module)) - VisitDirectDepsIf(pred func(Module) bool, visit func(Module)) - VisitDepsDepthFirst(visit func(Module)) - VisitDepsDepthFirstIf(pred func(Module) bool, visit func(Module)) - WalkDeps(visit func(Module, Module) bool) - // GetWalkPath is supposed to be called in visit function passed in WalkDeps() - // and returns a top-down dependency path from a start module to current child module. - GetWalkPath() []Module + CreateModule(ModuleFactory, ...interface{}) Module } -type androidTopDownMutatorContext struct { - blueprint.TopDownMutatorContext - androidBaseContextImpl - walkPath []Module +type topDownMutatorContext struct { + bp blueprint.TopDownMutatorContext + baseModuleContext } -type AndroidBottomUpMutator func(BottomUpMutatorContext) +type BottomUpMutator func(BottomUpMutatorContext) type BottomUpMutatorContext interface { BaseModuleContext - androidBaseContext - OtherModuleExists(name string) bool + MutatorName() string + Rename(name string) - Module() blueprint.Module AddDependency(module blueprint.Module, tag blueprint.DependencyTag, name ...string) AddReverseDependency(module blueprint.Module, tag blueprint.DependencyTag, name string) - CreateVariations(...string) []blueprint.Module - CreateLocalVariations(...string) []blueprint.Module + CreateVariations(...string) []Module + CreateLocalVariations(...string) []Module SetDependencyVariation(string) SetDefaultDependencyVariation(*string) AddVariationDependencies([]blueprint.Variation, blueprint.DependencyTag, ...string) AddFarVariationDependencies([]blueprint.Variation, blueprint.DependencyTag, ...string) AddInterVariantDependency(tag blueprint.DependencyTag, from, to blueprint.Module) ReplaceDependencies(string) + AliasVariation(variationName string) } -type androidBottomUpMutatorContext struct { - blueprint.BottomUpMutatorContext - androidBaseContextImpl +type bottomUpMutatorContext struct { + bp blueprint.BottomUpMutatorContext + baseModuleContext + finalPhase bool } -func (x *registerMutatorsContext) BottomUp(name string, m AndroidBottomUpMutator) MutatorHandle { +func (x *registerMutatorsContext) BottomUp(name string, m BottomUpMutator) MutatorHandle { + finalPhase := x.finalPhase f := func(ctx blueprint.BottomUpMutatorContext) { if a, ok := ctx.Module().(Module); ok { - actx := &androidBottomUpMutatorContext{ - BottomUpMutatorContext: ctx, - androidBaseContextImpl: a.base().androidBaseContextFactory(ctx), + actx := &bottomUpMutatorContext{ + bp: ctx, + baseModuleContext: a.base().baseModuleContextFactory(ctx), + finalPhase: finalPhase, } m(actx) } @@ -188,12 +228,12 @@ func (x *registerMutatorsContext) BottomUp(name string, m AndroidBottomUpMutator return mutator } -func (x *registerMutatorsContext) TopDown(name string, m AndroidTopDownMutator) MutatorHandle { +func (x *registerMutatorsContext) TopDown(name string, m TopDownMutator) MutatorHandle { f := func(ctx blueprint.TopDownMutatorContext) { if a, ok := ctx.Module().(Module); ok { - actx := &androidTopDownMutatorContext{ - TopDownMutatorContext: ctx, - androidBaseContextImpl: a.base().androidBaseContextFactory(ctx), + actx := &topDownMutatorContext{ + bp: ctx, + baseModuleContext: a.base().baseModuleContextFactory(ctx), } m(actx) } @@ -218,123 +258,151 @@ func depsMutator(ctx BottomUpMutatorContext) { } } -func (a *androidTopDownMutatorContext) Config() Config { - return a.config +func (t *topDownMutatorContext) AppendProperties(props ...interface{}) { + for _, p := range props { + err := proptools.AppendMatchingProperties(t.Module().base().customizableProperties, + p, nil) + if err != nil { + if propertyErr, ok := err.(*proptools.ExtendPropertyError); ok { + t.PropertyErrorf(propertyErr.Property, "%s", propertyErr.Err.Error()) + } else { + panic(err) + } + } + } } -func (a *androidBottomUpMutatorContext) Config() Config { - return a.config +func (t *topDownMutatorContext) PrependProperties(props ...interface{}) { + for _, p := range props { + err := proptools.PrependMatchingProperties(t.Module().base().customizableProperties, + p, nil) + if err != nil { + if propertyErr, ok := err.(*proptools.ExtendPropertyError); ok { + t.PropertyErrorf(propertyErr.Property, "%s", propertyErr.Err.Error()) + } else { + panic(err) + } + } + } } -func (a *androidTopDownMutatorContext) Module() Module { - module, _ := a.TopDownMutatorContext.Module().(Module) - return module +// android.topDownMutatorContext either has to embed blueprint.TopDownMutatorContext, in which case every method that +// has an overridden version in android.BaseModuleContext has to be manually forwarded to BaseModuleContext to avoid +// ambiguous method errors, or it has to store a blueprint.TopDownMutatorContext non-embedded, in which case every +// non-overridden method has to be forwarded. There are fewer non-overridden methods, so use the latter. The following +// methods forward to the identical blueprint versions for topDownMutatorContext and bottomUpMutatorContext. + +func (t *topDownMutatorContext) MutatorName() string { + return t.bp.MutatorName() } -func (a *androidTopDownMutatorContext) VisitDirectDeps(visit func(Module)) { - a.TopDownMutatorContext.VisitDirectDeps(func(module blueprint.Module) { - if aModule, _ := module.(Module); aModule != nil { - visit(aModule) - } - }) +func (t *topDownMutatorContext) Rename(name string) { + t.bp.Rename(name) + t.Module().base().commonProperties.DebugName = name } -func (a *androidTopDownMutatorContext) VisitDirectDepsWithTag(tag blueprint.DependencyTag, visit func(Module)) { - a.TopDownMutatorContext.VisitDirectDeps(func(module blueprint.Module) { - if aModule, _ := module.(Module); aModule != nil { - if a.TopDownMutatorContext.OtherModuleDependencyTag(aModule) == tag { - visit(aModule) - } +func (t *topDownMutatorContext) CreateModule(factory ModuleFactory, props ...interface{}) Module { + inherited := []interface{}{&t.Module().base().commonProperties} + module := t.bp.CreateModule(ModuleFactoryAdaptor(factory), append(inherited, props...)...).(Module) + + if t.Module().base().variableProperties != nil && module.base().variableProperties != nil { + src := t.Module().base().variableProperties + dst := []interface{}{ + module.base().variableProperties, + // Put an empty copy of the src properties into dst so that properties in src that are not in dst + // don't cause a "failed to find property to extend" error. + proptools.CloneEmptyProperties(reflect.ValueOf(src)).Interface(), + } + err := proptools.AppendMatchingProperties(dst, src, nil) + if err != nil { + panic(err) } - }) + } + + return module } -func (a *androidTopDownMutatorContext) VisitDirectDepsIf(pred func(Module) bool, visit func(Module)) { - a.TopDownMutatorContext.VisitDirectDepsIf( - // pred - func(module blueprint.Module) bool { - if aModule, _ := module.(Module); aModule != nil { - return pred(aModule) - } else { - return false - } - }, - // visit - func(module blueprint.Module) { - visit(module.(Module)) - }) +func (b *bottomUpMutatorContext) MutatorName() string { + return b.bp.MutatorName() } -func (a *androidTopDownMutatorContext) VisitDepsDepthFirst(visit func(Module)) { - a.TopDownMutatorContext.VisitDepsDepthFirst(func(module blueprint.Module) { - if aModule, _ := module.(Module); aModule != nil { - visit(aModule) - } - }) +func (b *bottomUpMutatorContext) Rename(name string) { + b.bp.Rename(name) + b.Module().base().commonProperties.DebugName = name } -func (a *androidTopDownMutatorContext) VisitDepsDepthFirstIf(pred func(Module) bool, visit func(Module)) { - a.TopDownMutatorContext.VisitDepsDepthFirstIf( - // pred - func(module blueprint.Module) bool { - if aModule, _ := module.(Module); aModule != nil { - return pred(aModule) - } else { - return false - } - }, - // visit - func(module blueprint.Module) { - visit(module.(Module)) - }) -} - -func (a *androidTopDownMutatorContext) WalkDeps(visit func(Module, Module) bool) { - a.walkPath = []Module{a.Module()} - a.TopDownMutatorContext.WalkDeps(func(child, parent blueprint.Module) bool { - childAndroidModule, _ := child.(Module) - parentAndroidModule, _ := parent.(Module) - if childAndroidModule != nil && parentAndroidModule != nil { - // record walkPath before visit - for a.walkPath[len(a.walkPath)-1] != parentAndroidModule { - a.walkPath = a.walkPath[0 : len(a.walkPath)-1] - } - a.walkPath = append(a.walkPath, childAndroidModule) - return visit(childAndroidModule, parentAndroidModule) - } else { - return false - } - }) +func (b *bottomUpMutatorContext) AddDependency(module blueprint.Module, tag blueprint.DependencyTag, name ...string) { + b.bp.AddDependency(module, tag, name...) } -func (a *androidTopDownMutatorContext) GetWalkPath() []Module { - return a.walkPath +func (b *bottomUpMutatorContext) AddReverseDependency(module blueprint.Module, tag blueprint.DependencyTag, name string) { + b.bp.AddReverseDependency(module, tag, name) } -func (a *androidTopDownMutatorContext) AppendProperties(props ...interface{}) { - for _, p := range props { - err := proptools.AppendMatchingProperties(a.Module().base().customizableProperties, - p, nil) - if err != nil { - if propertyErr, ok := err.(*proptools.ExtendPropertyError); ok { - a.PropertyErrorf(propertyErr.Property, "%s", propertyErr.Err.Error()) - } else { - panic(err) - } - } +func (b *bottomUpMutatorContext) CreateVariations(variations ...string) []Module { + if b.finalPhase { + panic("CreateVariations not allowed in FinalDepsMutators") + } + + modules := b.bp.CreateVariations(variations...) + + aModules := make([]Module, len(modules)) + for i := range variations { + aModules[i] = modules[i].(Module) + base := aModules[i].base() + base.commonProperties.DebugMutators = append(base.commonProperties.DebugMutators, b.MutatorName()) + base.commonProperties.DebugVariations = append(base.commonProperties.DebugVariations, variations[i]) } + + return aModules } -func (a *androidTopDownMutatorContext) PrependProperties(props ...interface{}) { - for _, p := range props { - err := proptools.PrependMatchingProperties(a.Module().base().customizableProperties, - p, nil) - if err != nil { - if propertyErr, ok := err.(*proptools.ExtendPropertyError); ok { - a.PropertyErrorf(propertyErr.Property, "%s", propertyErr.Err.Error()) - } else { - panic(err) - } - } +func (b *bottomUpMutatorContext) CreateLocalVariations(variations ...string) []Module { + if b.finalPhase { + panic("CreateLocalVariations not allowed in FinalDepsMutators") + } + + modules := b.bp.CreateLocalVariations(variations...) + + aModules := make([]Module, len(modules)) + for i := range variations { + aModules[i] = modules[i].(Module) + base := aModules[i].base() + base.commonProperties.DebugMutators = append(base.commonProperties.DebugMutators, b.MutatorName()) + base.commonProperties.DebugVariations = append(base.commonProperties.DebugVariations, variations[i]) } + + return aModules +} + +func (b *bottomUpMutatorContext) SetDependencyVariation(variation string) { + b.bp.SetDependencyVariation(variation) +} + +func (b *bottomUpMutatorContext) SetDefaultDependencyVariation(variation *string) { + b.bp.SetDefaultDependencyVariation(variation) +} + +func (b *bottomUpMutatorContext) AddVariationDependencies(variations []blueprint.Variation, tag blueprint.DependencyTag, + names ...string) { + + b.bp.AddVariationDependencies(variations, tag, names...) +} + +func (b *bottomUpMutatorContext) AddFarVariationDependencies(variations []blueprint.Variation, + tag blueprint.DependencyTag, names ...string) { + + b.bp.AddFarVariationDependencies(variations, tag, names...) +} + +func (b *bottomUpMutatorContext) AddInterVariantDependency(tag blueprint.DependencyTag, from, to blueprint.Module) { + b.bp.AddInterVariantDependency(tag, from, to) +} + +func (b *bottomUpMutatorContext) ReplaceDependencies(name string) { + b.bp.ReplaceDependencies(name) +} + +func (b *bottomUpMutatorContext) AliasVariation(variationName string) { + b.bp.AliasVariation(variationName) } diff --git a/android/mutator_test.go b/android/mutator_test.go new file mode 100644 index 000000000..191b535a3 --- /dev/null +++ b/android/mutator_test.go @@ -0,0 +1,297 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "fmt" + "reflect" + "strings" + "testing" + + "github.com/google/blueprint" + "github.com/google/blueprint/proptools" +) + +type mutatorTestModule struct { + ModuleBase + props struct { + Deps_missing_deps []string + Mutator_missing_deps []string + } + + missingDeps []string +} + +func mutatorTestModuleFactory() Module { + module := &mutatorTestModule{} + module.AddProperties(&module.props) + InitAndroidModule(module) + return module +} + +func (m *mutatorTestModule) GenerateAndroidBuildActions(ctx ModuleContext) { + ctx.Build(pctx, BuildParams{ + Rule: Touch, + Output: PathForModuleOut(ctx, "output"), + }) + + m.missingDeps = ctx.GetMissingDependencies() +} + +func (m *mutatorTestModule) DepsMutator(ctx BottomUpMutatorContext) { + ctx.AddDependency(ctx.Module(), nil, m.props.Deps_missing_deps...) +} + +func addMissingDependenciesMutator(ctx TopDownMutatorContext) { + ctx.AddMissingDependencies(ctx.Module().(*mutatorTestModule).props.Mutator_missing_deps) +} + +func TestMutatorAddMissingDependencies(t *testing.T) { + bp := ` + test { + name: "foo", + deps_missing_deps: ["regular_missing_dep"], + mutator_missing_deps: ["added_missing_dep"], + } + ` + + config := TestConfig(buildDir, nil, bp, nil) + config.TestProductVariables.Allow_missing_dependencies = proptools.BoolPtr(true) + + ctx := NewTestContext() + ctx.SetAllowMissingDependencies(true) + + ctx.RegisterModuleType("test", mutatorTestModuleFactory) + ctx.PreDepsMutators(func(ctx RegisterMutatorsContext) { + ctx.TopDown("add_missing_dependencies", addMissingDependenciesMutator) + }) + + ctx.Register(config) + _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) + FailIfErrored(t, errs) + _, errs = ctx.PrepareBuildActions(config) + FailIfErrored(t, errs) + + foo := ctx.ModuleForTests("foo", "").Module().(*mutatorTestModule) + + if g, w := foo.missingDeps, []string{"added_missing_dep", "regular_missing_dep"}; !reflect.DeepEqual(g, w) { + t.Errorf("want foo missing deps %q, got %q", w, g) + } +} + +func TestModuleString(t *testing.T) { + ctx := NewTestContext() + + var moduleStrings []string + + ctx.PreArchMutators(func(ctx RegisterMutatorsContext) { + ctx.BottomUp("pre_arch", func(ctx BottomUpMutatorContext) { + moduleStrings = append(moduleStrings, ctx.Module().String()) + ctx.CreateVariations("a", "b") + }) + ctx.TopDown("rename_top_down", func(ctx TopDownMutatorContext) { + moduleStrings = append(moduleStrings, ctx.Module().String()) + ctx.Rename(ctx.Module().base().Name() + "_renamed1") + }) + }) + + ctx.PreDepsMutators(func(ctx RegisterMutatorsContext) { + ctx.BottomUp("pre_deps", func(ctx BottomUpMutatorContext) { + moduleStrings = append(moduleStrings, ctx.Module().String()) + ctx.CreateVariations("c", "d") + }) + }) + + ctx.PostDepsMutators(func(ctx RegisterMutatorsContext) { + ctx.BottomUp("post_deps", func(ctx BottomUpMutatorContext) { + moduleStrings = append(moduleStrings, ctx.Module().String()) + ctx.CreateLocalVariations("e", "f") + }) + ctx.BottomUp("rename_bottom_up", func(ctx BottomUpMutatorContext) { + moduleStrings = append(moduleStrings, ctx.Module().String()) + ctx.Rename(ctx.Module().base().Name() + "_renamed2") + }) + ctx.BottomUp("final", func(ctx BottomUpMutatorContext) { + moduleStrings = append(moduleStrings, ctx.Module().String()) + }) + }) + + ctx.RegisterModuleType("test", mutatorTestModuleFactory) + + bp := ` + test { + name: "foo", + } + ` + + config := TestConfig(buildDir, nil, bp, nil) + + ctx.Register(config) + + _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) + FailIfErrored(t, errs) + _, errs = ctx.PrepareBuildActions(config) + FailIfErrored(t, errs) + + want := []string{ + // Initial name. + "foo{}", + + // After pre_arch (reversed because rename_top_down is TopDown so it visits in reverse order). + "foo{pre_arch:b}", + "foo{pre_arch:a}", + + // After rename_top_down. + "foo_renamed1{pre_arch:a}", + "foo_renamed1{pre_arch:b}", + + // After pre_deps. + "foo_renamed1{pre_arch:a,pre_deps:c}", + "foo_renamed1{pre_arch:a,pre_deps:d}", + "foo_renamed1{pre_arch:b,pre_deps:c}", + "foo_renamed1{pre_arch:b,pre_deps:d}", + + // After post_deps. + "foo_renamed1{pre_arch:a,pre_deps:c,post_deps:e}", + "foo_renamed1{pre_arch:a,pre_deps:c,post_deps:f}", + "foo_renamed1{pre_arch:a,pre_deps:d,post_deps:e}", + "foo_renamed1{pre_arch:a,pre_deps:d,post_deps:f}", + "foo_renamed1{pre_arch:b,pre_deps:c,post_deps:e}", + "foo_renamed1{pre_arch:b,pre_deps:c,post_deps:f}", + "foo_renamed1{pre_arch:b,pre_deps:d,post_deps:e}", + "foo_renamed1{pre_arch:b,pre_deps:d,post_deps:f}", + + // After rename_bottom_up. + "foo_renamed2{pre_arch:a,pre_deps:c,post_deps:e}", + "foo_renamed2{pre_arch:a,pre_deps:c,post_deps:f}", + "foo_renamed2{pre_arch:a,pre_deps:d,post_deps:e}", + "foo_renamed2{pre_arch:a,pre_deps:d,post_deps:f}", + "foo_renamed2{pre_arch:b,pre_deps:c,post_deps:e}", + "foo_renamed2{pre_arch:b,pre_deps:c,post_deps:f}", + "foo_renamed2{pre_arch:b,pre_deps:d,post_deps:e}", + "foo_renamed2{pre_arch:b,pre_deps:d,post_deps:f}", + } + + if !reflect.DeepEqual(moduleStrings, want) { + t.Errorf("want module String() values:\n%q\ngot:\n%q", want, moduleStrings) + } +} + +func TestFinalDepsPhase(t *testing.T) { + ctx := NewTestContext() + + finalGot := map[string]int{} + + dep1Tag := struct { + blueprint.BaseDependencyTag + }{} + dep2Tag := struct { + blueprint.BaseDependencyTag + }{} + + ctx.PostDepsMutators(func(ctx RegisterMutatorsContext) { + ctx.BottomUp("far_deps_1", func(ctx BottomUpMutatorContext) { + if !strings.HasPrefix(ctx.ModuleName(), "common_dep") { + ctx.AddFarVariationDependencies([]blueprint.Variation{}, dep1Tag, "common_dep_1") + } + }) + ctx.BottomUp("variant", func(ctx BottomUpMutatorContext) { + ctx.CreateLocalVariations("a", "b") + }) + }) + + ctx.FinalDepsMutators(func(ctx RegisterMutatorsContext) { + ctx.BottomUp("far_deps_2", func(ctx BottomUpMutatorContext) { + if !strings.HasPrefix(ctx.ModuleName(), "common_dep") { + ctx.AddFarVariationDependencies([]blueprint.Variation{}, dep2Tag, "common_dep_2") + } + }) + ctx.BottomUp("final", func(ctx BottomUpMutatorContext) { + finalGot[ctx.Module().String()] += 1 + ctx.VisitDirectDeps(func(mod Module) { + finalGot[fmt.Sprintf("%s -> %s", ctx.Module().String(), mod)] += 1 + }) + }) + }) + + ctx.RegisterModuleType("test", mutatorTestModuleFactory) + + bp := ` + test { + name: "common_dep_1", + } + test { + name: "common_dep_2", + } + test { + name: "foo", + } + ` + + config := TestConfig(buildDir, nil, bp, nil) + ctx.Register(config) + + _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) + FailIfErrored(t, errs) + _, errs = ctx.PrepareBuildActions(config) + FailIfErrored(t, errs) + + finalWant := map[string]int{ + "common_dep_1{variant:a}": 1, + "common_dep_1{variant:b}": 1, + "common_dep_2{variant:a}": 1, + "common_dep_2{variant:b}": 1, + "foo{variant:a}": 1, + "foo{variant:a} -> common_dep_1{variant:a}": 1, + "foo{variant:a} -> common_dep_2{variant:a}": 1, + "foo{variant:b}": 1, + "foo{variant:b} -> common_dep_1{variant:b}": 1, + "foo{variant:b} -> common_dep_2{variant:a}": 1, + } + + if !reflect.DeepEqual(finalWant, finalGot) { + t.Errorf("want:\n%q\ngot:\n%q", finalWant, finalGot) + } +} + +func TestNoCreateVariationsInFinalDeps(t *testing.T) { + ctx := NewTestContext() + + checkErr := func() { + if err := recover(); err == nil || !strings.Contains(fmt.Sprintf("%s", err), "not allowed in FinalDepsMutators") { + panic("Expected FinalDepsMutators consistency check to fail") + } + } + + ctx.FinalDepsMutators(func(ctx RegisterMutatorsContext) { + ctx.BottomUp("vars", func(ctx BottomUpMutatorContext) { + defer checkErr() + ctx.CreateVariations("a", "b") + }) + ctx.BottomUp("local_vars", func(ctx BottomUpMutatorContext) { + defer checkErr() + ctx.CreateLocalVariations("a", "b") + }) + }) + + ctx.RegisterModuleType("test", mutatorTestModuleFactory) + config := TestConfig(buildDir, nil, `test {name: "foo"}`, nil) + ctx.Register(config) + + _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) + FailIfErrored(t, errs) + _, errs = ctx.PrepareBuildActions(config) + FailIfErrored(t, errs) +} diff --git a/android/namespace.go b/android/namespace.go index 50bdcba7f..9d7e8acf4 100644 --- a/android/namespace.go +++ b/android/namespace.go @@ -26,12 +26,6 @@ import ( "github.com/google/blueprint" ) -// This file implements namespaces -const ( - namespacePrefix = "//" - modulePrefix = ":" -) - func init() { RegisterModuleType("soong_namespace", NamespaceFactory) } @@ -168,6 +162,12 @@ func (r *NameResolver) findNamespace(path string) (namespace *Namespace) { return namespace } +// A NamelessModule can never be looked up by name. It must still implement Name(), but the return +// value doesn't have to be unique. +type NamelessModule interface { + Nameless() +} + func (r *NameResolver) NewModule(ctx blueprint.NamespaceContext, moduleGroup blueprint.ModuleGroup, module blueprint.Module) (namespace blueprint.Namespace, errs []error) { // if this module is a namespace, then save it to our list of namespaces newNamespace, ok := module.(*NamespaceModule) @@ -179,6 +179,10 @@ func (r *NameResolver) NewModule(ctx blueprint.NamespaceContext, moduleGroup blu return nil, nil } + if _, ok := module.(NamelessModule); ok { + return nil, nil + } + // if this module is not a namespace, then save it into the appropriate namespace ns := r.findNamespaceFromCtx(ctx) @@ -191,6 +195,7 @@ func (r *NameResolver) NewModule(ctx blueprint.NamespaceContext, moduleGroup blu if ok { // inform the module whether its namespace is one that we want to export to Make amod.base().commonProperties.NamespaceExportedToMake = ns.exportToKati + amod.base().commonProperties.DebugName = module.Name() } return ns, nil @@ -215,11 +220,11 @@ func (r *NameResolver) AllModules() []blueprint.ModuleGroup { // parses a fully-qualified path (like "//namespace_path:module_name") into a namespace name and a // module name func (r *NameResolver) parseFullyQualifiedName(name string) (namespaceName string, moduleName string, ok bool) { - if !strings.HasPrefix(name, namespacePrefix) { + if !strings.HasPrefix(name, "//") { return "", "", false } - name = strings.TrimPrefix(name, namespacePrefix) - components := strings.Split(name, modulePrefix) + name = strings.TrimPrefix(name, "//") + components := strings.Split(name, ":") if len(components) != 2 { return "", "", false } @@ -228,6 +233,11 @@ func (r *NameResolver) parseFullyQualifiedName(name string) (namespaceName strin } func (r *NameResolver) getNamespacesToSearchForModule(sourceNamespace *Namespace) (searchOrder []*Namespace) { + if sourceNamespace.visibleNamespaces == nil { + // When handling dependencies before namespaceMutator, assume they are non-Soong Blueprint modules and give + // access to all namespaces. + return r.sortedNamespaces.sortedItems() + } return sourceNamespace.visibleNamespaces } diff --git a/android/namespace_test.go b/android/namespace_test.go index 9a791a534..66c0d895a 100644 --- a/android/namespace_test.go +++ b/android/namespace_test.go @@ -16,8 +16,6 @@ package android import ( "errors" - "io/ioutil" - "os" "path/filepath" "reflect" "testing" @@ -93,6 +91,28 @@ func TestImplicitlyImportRootNamespace(t *testing.T) { // setupTest will report any errors } +func TestDependingOnBlueprintModuleInRootNamespace(t *testing.T) { + _ = setupTest(t, + map[string]string{ + ".": ` + blueprint_test_module { + name: "a", + } + `, + "dir1": ` + soong_namespace { + } + blueprint_test_module { + name: "b", + deps: ["a"], + } + `, + }, + ) + + // setupTest will report any errors +} + func TestDependingOnModuleInImportedNamespace(t *testing.T) { ctx := setupTest(t, map[string]string{ @@ -613,23 +633,17 @@ func mockFiles(bps map[string]string) (files map[string][]byte) { } func setupTestFromFiles(bps map[string][]byte) (ctx *TestContext, errs []error) { - buildDir, err := ioutil.TempDir("", "soong_namespace_test") - if err != nil { - return nil, []error{err} - } - defer os.RemoveAll(buildDir) - - config := TestConfig(buildDir, nil) + config := TestConfig(buildDir, nil, "", bps) ctx = NewTestContext() - ctx.MockFileSystem(bps) - ctx.RegisterModuleType("test_module", ModuleFactoryAdaptor(newTestModule)) - ctx.RegisterModuleType("soong_namespace", ModuleFactoryAdaptor(NamespaceFactory)) + ctx.RegisterModuleType("test_module", newTestModule) + ctx.RegisterModuleType("soong_namespace", NamespaceFactory) + ctx.Context.RegisterModuleType("blueprint_test_module", newBlueprintTestModule) ctx.PreArchMutators(RegisterNamespaceMutator) ctx.PreDepsMutators(func(ctx RegisterMutatorsContext) { ctx.BottomUp("rename", renameMutator) }) - ctx.Register() + ctx.Register(config) _, errs = ctx.ParseBlueprintsFiles("Android.bp") if len(errs) > 0 { @@ -649,6 +663,7 @@ func setupTestExpectErrs(bps map[string]string) (ctx *TestContext, errs []error) } func setupTest(t *testing.T, bps map[string]string) (ctx *TestContext) { + t.Helper() ctx, errs := setupTestExpectErrs(bps) FailIfErrored(t, errs) return ctx @@ -726,3 +741,22 @@ func newTestModule() Module { InitAndroidModule(m) return m } + +type blueprintTestModule struct { + blueprint.SimpleName + properties struct { + Deps []string + } +} + +func (b *blueprintTestModule) DynamicDependencies(ctx blueprint.DynamicDependerModuleContext) []string { + return b.properties.Deps +} + +func (b *blueprintTestModule) GenerateBuildActions(blueprint.ModuleContext) { +} + +func newBlueprintTestModule() (blueprint.Module, []interface{}) { + m := &blueprintTestModule{} + return m, []interface{}{&m.properties, &m.SimpleName.Properties} +} diff --git a/android/neverallow.go b/android/neverallow.go index b90fe43d3..2eba4a950 100644 --- a/android/neverallow.go +++ b/android/neverallow.go @@ -17,6 +17,7 @@ package android import ( "path/filepath" "reflect" + "regexp" "strconv" "strings" @@ -31,62 +32,107 @@ import ( // work regardless of these restrictions. // // A module is disallowed if all of the following are true: -// - it is in one of the "in" paths -// - it is not in one of the "notIn" paths -// - it has all "with" properties matched +// - it is in one of the "In" paths +// - it is not in one of the "NotIn" paths +// - it has all "With" properties matched // - - values are matched in their entirety // - - nil is interpreted as an empty string // - - nested properties are separated with a '.' // - - if the property is a list, any of the values in the list being matches // counts as a match -// - it has none of the "without" properties matched (same rules as above) +// - it has none of the "Without" properties matched (same rules as above) -func registerNeverallowMutator(ctx RegisterMutatorsContext) { +func RegisterNeverallowMutator(ctx RegisterMutatorsContext) { ctx.BottomUp("neverallow", neverallowMutator).Parallel() } -var neverallows = createNeverAllows() +var neverallows = []Rule{} + +func init() { + AddNeverAllowRules(createIncludeDirsRules()...) + AddNeverAllowRules(createTrebleRules()...) + AddNeverAllowRules(createLibcoreRules()...) + AddNeverAllowRules(createMediaRules()...) + AddNeverAllowRules(createJavaDeviceForHostRules()...) + AddNeverAllowRules(createCcSdkVariantRules()...) + AddNeverAllowRules(createUncompressDexRules()...) +} + +// Add a NeverAllow rule to the set of rules to apply. +func AddNeverAllowRules(rules ...Rule) { + neverallows = append(neverallows, rules...) +} + +func createIncludeDirsRules() []Rule { + // The list of paths that cannot be referenced using include_dirs + paths := []string{ + "art", + "art/libnativebridge", + "art/libnativeloader", + "libcore", + "libnativehelper", + "external/apache-harmony", + "external/apache-xml", + "external/boringssl", + "external/bouncycastle", + "external/conscrypt", + "external/icu", + "external/okhttp", + "external/vixl", + "external/wycheproof", + } + + // Create a composite matcher that will match if the value starts with any of the restricted + // paths. A / is appended to the prefix to ensure that restricting path X does not affect paths + // XY. + rules := make([]Rule, 0, len(paths)) + for _, path := range paths { + rule := + NeverAllow(). + WithMatcher("include_dirs", StartsWith(path+"/")). + Because("include_dirs is deprecated, all usages of '" + path + "' have been migrated" + + " to use alternate mechanisms and so can no longer be used.") + + rules = append(rules, rule) + } -func createNeverAllows() []*rule { - rules := []*rule{} - rules = append(rules, createTrebleRules()...) - rules = append(rules, createLibcoreRules()...) - rules = append(rules, createMediaRules()...) - rules = append(rules, createJavaDeviceForHostRules()...) return rules } -func createTrebleRules() []*rule { - return []*rule{ - neverallow(). - in("vendor", "device"). - with("vndk.enabled", "true"). - without("vendor", "true"). - because("the VNDK can never contain a library that is device dependent."), - neverallow(). - with("vndk.enabled", "true"). - without("vendor", "true"). - without("owner", ""). - because("a VNDK module can never have an owner."), +func createTrebleRules() []Rule { + return []Rule{ + NeverAllow(). + In("vendor", "device"). + With("vndk.enabled", "true"). + Without("vendor", "true"). + Without("product_specific", "true"). + Because("the VNDK can never contain a library that is device dependent."), + NeverAllow(). + With("vndk.enabled", "true"). + Without("vendor", "true"). + Without("owner", ""). + Because("a VNDK module can never have an owner."), // TODO(b/67974785): always enforce the manifest - neverallow(). - without("name", "libhidltransport-impl-internal"). - with("product_variables.enforce_vintf_manifest.cflags", "*"). - because("manifest enforcement should be independent of ."), + NeverAllow(). + Without("name", "libhidlbase-combined-impl"). + Without("name", "libhidlbase"). + Without("name", "libhidlbase_pgo"). + With("product_variables.enforce_vintf_manifest.cflags", "*"). + Because("manifest enforcement should be independent of ."), // TODO(b/67975799): vendor code should always use /vendor/bin/sh - neverallow(). - without("name", "libc_bionic_ndk"). - with("product_variables.treble_linker_namespaces.cflags", "*"). - because("nothing should care if linker namespaces are enabled or not"), + NeverAllow(). + Without("name", "libc_bionic_ndk"). + With("product_variables.treble_linker_namespaces.cflags", "*"). + Because("nothing should care if linker namespaces are enabled or not"), // Example: - // *neverallow().with("Srcs", "main.cpp")) + // *NeverAllow().with("Srcs", "main.cpp")) } } -func createLibcoreRules() []*rule { +func createLibcoreRules() []Rule { var coreLibraryProjects = []string{ "libcore", "external/apache-harmony", @@ -96,56 +142,81 @@ func createLibcoreRules() []*rule { "external/icu", "external/okhttp", "external/wycheproof", + "prebuilts", } - var coreModules = []string{ - "core-all", - "core-oj", - "core-libart", - "okhttp", - "bouncycastle", - "conscrypt", - "apache-xml", - } - - // Core library constraints. Prevent targets adding dependencies on core - // library internals, which could lead to compatibility issues with the ART - // mainline module. They should use core.platform.api.stubs instead. - rules := []*rule{ - neverallow(). - notIn(append(coreLibraryProjects, "development")...). - with("no_standard_libs", "true"), + // Core library constraints. The sdk_version: "none" can only be used in core library projects. + // Access to core library targets is restricted using visibility rules. + rules := []Rule{ + NeverAllow(). + NotIn(coreLibraryProjects...). + With("sdk_version", "none"). + WithoutMatcher("name", Regexp("^android_.*stubs_current$")), } - for _, m := range coreModules { - r := neverallow(). - notIn(coreLibraryProjects...). - with("libs", m). - because("Only core libraries projects can depend on " + m) - rules = append(rules, r) - } return rules } -func createMediaRules() []*rule { - return []*rule{ - neverallow(). - with("libs", "updatable-media"). - because("updatable-media includes private APIs. Use updatable_media_stubs instead."), +func createMediaRules() []Rule { + return []Rule{ + NeverAllow(). + With("libs", "updatable-media"). + Because("updatable-media includes private APIs. Use updatable_media_stubs instead."), } } -func createJavaDeviceForHostRules() []*rule { - javaDeviceForHostProjectsWhitelist := []string{ +func createJavaDeviceForHostRules() []Rule { + javaDeviceForHostProjectsAllowedList := []string{ + "external/guava", "external/robolectric-shadows", "framework/layoutlib", } - return []*rule{ - neverallow(). - notIn(javaDeviceForHostProjectsWhitelist...). - moduleType("java_device_for_host", "java_host_for_device"). - because("java_device_for_host can only be used in whitelisted projects"), + return []Rule{ + NeverAllow(). + NotIn(javaDeviceForHostProjectsAllowedList...). + ModuleType("java_device_for_host", "java_host_for_device"). + Because("java_device_for_host can only be used in allowed projects"), + } +} + +func createCcSdkVariantRules() []Rule { + sdkVersionOnlyAllowedList := []string{ + // derive_sdk_prefer32 has stem: "derive_sdk" which conflicts with the derive_sdk. + // This sometimes works because the APEX modules that contain derive_sdk and + // derive_sdk_prefer32 suppress the platform installation rules, but fails when + // the APEX modules contain the SDK variant and the platform variant still exists. + "frameworks/base/apex/sdkextensions/derive_sdk", + } + + platformVariantPropertiesAllowedList := []string{ + // android_native_app_glue and libRSSupport use native_window.h but target old + // sdk versions (minimum and 9 respectively) where libnativewindow didn't exist, + // so they can't add libnativewindow to shared_libs to get the header directory + // for the platform variant. Allow them to use the platform variant + // property to set shared_libs. + "prebuilts/ndk", + "frameworks/rs", + } + + return []Rule{ + NeverAllow(). + NotIn(sdkVersionOnlyAllowedList...). + WithMatcher("sdk_variant_only", isSetMatcherInstance). + Because("sdk_variant_only can only be used in allowed projects"), + NeverAllow(). + NotIn(platformVariantPropertiesAllowedList...). + WithMatcher("platform.shared_libs", isSetMatcherInstance). + Because("platform variant properties can only be used in allowed projects"), + } +} + +func createUncompressDexRules() []Rule { + return []Rule{ + NeverAllow(). + NotIn("art"). + WithMatcher("uncompress_dex", isSetMatcherInstance). + Because("uncompress_dex is only allowed for certain jars for test in art."), } } @@ -158,7 +229,10 @@ func neverallowMutator(ctx BottomUpMutatorContext) { dir := ctx.ModuleDir() + "/" properties := m.GetProperties() - for _, n := range neverallows { + osClass := ctx.Module().Target().Os.Class + + for _, r := range neverallowRules(ctx.Config()) { + n := r.(*rule) if !n.appliesToPath(dir) { continue } @@ -171,13 +245,130 @@ func neverallowMutator(ctx BottomUpMutatorContext) { continue } + if !n.appliesToOsClass(osClass) { + continue + } + + if !n.appliesToDirectDeps(ctx) { + continue + } + + if !n.appliesToBootclasspathJar(ctx) { + continue + } + ctx.ModuleErrorf("violates " + n.String()) } } +type ValueMatcher interface { + Test(string) bool + String() string +} + +type equalMatcher struct { + expected string +} + +func (m *equalMatcher) Test(value string) bool { + return m.expected == value +} + +func (m *equalMatcher) String() string { + return "=" + m.expected +} + +type anyMatcher struct { +} + +func (m *anyMatcher) Test(value string) bool { + return true +} + +func (m *anyMatcher) String() string { + return "=*" +} + +var anyMatcherInstance = &anyMatcher{} + +type startsWithMatcher struct { + prefix string +} + +func (m *startsWithMatcher) Test(value string) bool { + return strings.HasPrefix(value, m.prefix) +} + +func (m *startsWithMatcher) String() string { + return ".starts-with(" + m.prefix + ")" +} + +type regexMatcher struct { + re *regexp.Regexp +} + +func (m *regexMatcher) Test(value string) bool { + return m.re.MatchString(value) +} + +func (m *regexMatcher) String() string { + return ".regexp(" + m.re.String() + ")" +} + +type notInListMatcher struct { + allowed []string +} + +func (m *notInListMatcher) Test(value string) bool { + return !InList(value, m.allowed) +} + +func (m *notInListMatcher) String() string { + return ".not-in-list(" + strings.Join(m.allowed, ",") + ")" +} + +type isSetMatcher struct{} + +func (m *isSetMatcher) Test(value string) bool { + return value != "" +} + +func (m *isSetMatcher) String() string { + return ".is-set" +} + +var isSetMatcherInstance = &isSetMatcher{} + type ruleProperty struct { - fields []string // e.x.: Vndk.Enabled - value string // e.x.: true + fields []string // e.x.: Vndk.Enabled + matcher ValueMatcher +} + +// A NeverAllow rule. +type Rule interface { + In(path ...string) Rule + + NotIn(path ...string) Rule + + InDirectDeps(deps ...string) Rule + + WithOsClass(osClasses ...OsClass) Rule + + ModuleType(types ...string) Rule + + NotModuleType(types ...string) Rule + + BootclasspathJar() Rule + + With(properties, value string) Rule + + WithMatcher(properties string, matcher ValueMatcher) Rule + + Without(properties, value string) Rule + + WithoutMatcher(properties string, matcher ValueMatcher) Rule + + Because(reason string) Rule } type rule struct { @@ -187,58 +378,97 @@ type rule struct { paths []string unlessPaths []string + directDeps map[string]bool + + osClasses []OsClass + moduleTypes []string unlessModuleTypes []string props []ruleProperty unlessProps []ruleProperty + + onlyBootclasspathJar bool } -func neverallow() *rule { - return &rule{} +// Create a new NeverAllow rule. +func NeverAllow() Rule { + return &rule{directDeps: make(map[string]bool)} } -func (r *rule) in(path ...string) *rule { +func (r *rule) In(path ...string) Rule { r.paths = append(r.paths, cleanPaths(path)...) return r } -func (r *rule) notIn(path ...string) *rule { +func (r *rule) NotIn(path ...string) Rule { r.unlessPaths = append(r.unlessPaths, cleanPaths(path)...) return r } -func (r *rule) moduleType(types ...string) *rule { +func (r *rule) InDirectDeps(deps ...string) Rule { + for _, d := range deps { + r.directDeps[d] = true + } + return r +} + +func (r *rule) WithOsClass(osClasses ...OsClass) Rule { + r.osClasses = append(r.osClasses, osClasses...) + return r +} + +func (r *rule) ModuleType(types ...string) Rule { r.moduleTypes = append(r.moduleTypes, types...) return r } -func (r *rule) notModuleType(types ...string) *rule { +func (r *rule) NotModuleType(types ...string) Rule { r.unlessModuleTypes = append(r.unlessModuleTypes, types...) return r } -func (r *rule) with(properties, value string) *rule { +func (r *rule) With(properties, value string) Rule { + return r.WithMatcher(properties, selectMatcher(value)) +} + +func (r *rule) WithMatcher(properties string, matcher ValueMatcher) Rule { r.props = append(r.props, ruleProperty{ - fields: fieldNamesForProperties(properties), - value: value, + fields: fieldNamesForProperties(properties), + matcher: matcher, }) return r } -func (r *rule) without(properties, value string) *rule { +func (r *rule) Without(properties, value string) Rule { + return r.WithoutMatcher(properties, selectMatcher(value)) +} + +func (r *rule) WithoutMatcher(properties string, matcher ValueMatcher) Rule { r.unlessProps = append(r.unlessProps, ruleProperty{ - fields: fieldNamesForProperties(properties), - value: value, + fields: fieldNamesForProperties(properties), + matcher: matcher, }) return r } -func (r *rule) because(reason string) *rule { +func selectMatcher(expected string) ValueMatcher { + if expected == "*" { + return anyMatcherInstance + } + return &equalMatcher{expected: expected} +} + +func (r *rule) Because(reason string) Rule { r.reason = reason return r } +func (r *rule) BootclasspathJar() Rule { + r.onlyBootclasspathJar = true + return r +} + func (r *rule) String() string { s := "neverallow" for _, v := range r.paths { @@ -254,10 +484,19 @@ func (r *rule) String() string { s += " -type:" + v } for _, v := range r.props { - s += " " + strings.Join(v.fields, ".") + "=" + v.value + s += " " + strings.Join(v.fields, ".") + v.matcher.String() } for _, v := range r.unlessProps { - s += " -" + strings.Join(v.fields, ".") + "=" + v.value + s += " -" + strings.Join(v.fields, ".") + v.matcher.String() + } + for k := range r.directDeps { + s += " deps:" + k + } + for _, v := range r.osClasses { + s += " os:" + v.String() + } + if r.onlyBootclasspathJar { + s += " inBcp" } if len(r.reason) != 0 { s += " which is restricted because " + r.reason @@ -266,11 +505,49 @@ func (r *rule) String() string { } func (r *rule) appliesToPath(dir string) bool { - includePath := len(r.paths) == 0 || hasAnyPrefix(dir, r.paths) - excludePath := hasAnyPrefix(dir, r.unlessPaths) + includePath := len(r.paths) == 0 || HasAnyPrefix(dir, r.paths) + excludePath := HasAnyPrefix(dir, r.unlessPaths) return includePath && !excludePath } +func (r *rule) appliesToDirectDeps(ctx BottomUpMutatorContext) bool { + if len(r.directDeps) == 0 { + return true + } + + matches := false + ctx.VisitDirectDeps(func(m Module) { + if !matches { + name := ctx.OtherModuleName(m) + matches = r.directDeps[name] + } + }) + + return matches +} + +func (r *rule) appliesToBootclasspathJar(ctx BottomUpMutatorContext) bool { + if !r.onlyBootclasspathJar { + return true + } + + return InList(ctx.ModuleName(), ctx.Config().BootJars()) +} + +func (r *rule) appliesToOsClass(osClass OsClass) bool { + if len(r.osClasses) == 0 { + return true + } + + for _, c := range r.osClasses { + if c == osClass { + return true + } + } + + return false +} + func (r *rule) appliesToModuleType(moduleType string) bool { return (len(r.moduleTypes) == 0 || InList(moduleType, r.moduleTypes)) && !InList(moduleType, r.unlessModuleTypes) } @@ -281,6 +558,22 @@ func (r *rule) appliesToProperties(properties []interface{}) bool { return includeProps && !excludeProps } +func StartsWith(prefix string) ValueMatcher { + return &startsWithMatcher{prefix} +} + +func Regexp(re string) ValueMatcher { + r, err := regexp.Compile(re) + if err != nil { + panic(err) + } + return ®exMatcher{r} +} + +func NotInList(allowed []string) ValueMatcher { + return ¬InListMatcher{allowed} +} + // assorted utils func cleanPaths(paths []string) []string { @@ -299,15 +592,6 @@ func fieldNamesForProperties(propertyNames string) []string { return names } -func hasAnyPrefix(s string, prefixes []string) bool { - for _, prefix := range prefixes { - if strings.HasPrefix(s, prefix) { - return true - } - } - return false -} - func hasAnyProperty(properties []interface{}, props []ruleProperty) bool { for _, v := range props { if hasProperty(properties, v) { @@ -339,8 +623,8 @@ func hasProperty(properties []interface{}, prop ruleProperty) bool { continue } - check := func(v string) bool { - return prop.value == "*" || prop.value == v + check := func(value string) bool { + return prop.matcher.Test(value) } if matchValue(propertiesValue, check) { @@ -384,3 +668,19 @@ func matchValue(value reflect.Value, check func(string) bool) bool { panic("Can't handle type: " + value.Kind().String()) } + +var neverallowRulesKey = NewOnceKey("neverallowRules") + +func neverallowRules(config Config) []Rule { + return config.Once(neverallowRulesKey, func() interface{} { + // No test rules were set by setTestNeverallowRules, use the global rules + return neverallows + }).([]Rule) +} + +// Overrides the default neverallow rules for the supplied config. +// +// For testing only. +func SetTestNeverallowRules(config Config, testRules []Rule) { + config.Once(neverallowRulesKey, func() interface{} { return testRules }) +} diff --git a/android/neverallow_test.go b/android/neverallow_test.go index 6fd2cd71d..45d36a69b 100644 --- a/android/neverallow_test.go +++ b/android/neverallow_test.go @@ -15,20 +15,81 @@ package android import ( - "io/ioutil" - "os" "testing" + + "github.com/google/blueprint" ) var neverallowTests = []struct { - name string - fs map[string][]byte - expectedError string + // The name of the test. + name string + + // Optional test specific rules. If specified then they are used instead of the default rules. + rules []Rule + + // Additional contents to add to the virtual filesystem used by the tests. + fs map[string][]byte + + // The expected error patterns. If empty then no errors are expected, otherwise each error + // reported must be matched by at least one of these patterns. A pattern matches if the error + // message contains the pattern. A pattern does not have to match the whole error message. + expectedErrors []string }{ + // Test General Functionality + + // in direct deps tests + { + name: "not_allowed_in_direct_deps", + rules: []Rule{ + NeverAllow().InDirectDeps("not_allowed_in_direct_deps"), + }, + fs: map[string][]byte{ + "top/Android.bp": []byte(` + cc_library { + name: "not_allowed_in_direct_deps", + }`), + "other/Android.bp": []byte(` + cc_library { + name: "libother", + static_libs: ["not_allowed_in_direct_deps"], + }`), + }, + expectedErrors: []string{ + `module "libother": violates neverallow deps:not_allowed_in_direct_deps`, + }, + }, + + // Test android specific rules + + // include_dir rule tests + { + name: "include_dir not allowed to reference art", + fs: map[string][]byte{ + "other/Android.bp": []byte(` + cc_library { + name: "libother", + include_dirs: ["art/libdexfile/include"], + }`), + }, + expectedErrors: []string{ + "all usages of 'art' have been migrated", + }, + }, + { + name: "include_dir can reference another location", + fs: map[string][]byte{ + "other/Android.bp": []byte(` + cc_library { + name: "libother", + include_dirs: ["another/include"], + }`), + }, + }, + // Treble rule tests { name: "no vndk.enabled under vendor directory", fs: map[string][]byte{ - "vendor/Blueprints": []byte(` + "vendor/Android.bp": []byte(` cc_library { name: "libvndk", vendor_available: true, @@ -37,12 +98,14 @@ var neverallowTests = []struct { }, }`), }, - expectedError: "VNDK can never contain a library that is device dependent", + expectedErrors: []string{ + "VNDK can never contain a library that is device dependent", + }, }, { name: "no vndk.enabled under device directory", fs: map[string][]byte{ - "device/Blueprints": []byte(` + "device/Android.bp": []byte(` cc_library { name: "libvndk", vendor_available: true, @@ -51,12 +114,14 @@ var neverallowTests = []struct { }, }`), }, - expectedError: "VNDK can never contain a library that is device dependent", + expectedErrors: []string{ + "VNDK can never contain a library that is device dependent", + }, }, { name: "vndk-ext under vendor or device directory", fs: map[string][]byte{ - "device/Blueprints": []byte(` + "device/Android.bp": []byte(` cc_library { name: "libvndk1_ext", vendor: true, @@ -64,7 +129,7 @@ var neverallowTests = []struct { enabled: true, }, }`), - "vendor/Blueprints": []byte(` + "vendor/Android.bp": []byte(` cc_library { name: "libvndk2_ext", vendor: true, @@ -73,13 +138,12 @@ var neverallowTests = []struct { }, }`), }, - expectedError: "", }, { name: "no enforce_vintf_manifest.cflags", fs: map[string][]byte{ - "Blueprints": []byte(` + "Android.bp": []byte(` cc_library { name: "libexample", product_variables: { @@ -89,13 +153,15 @@ var neverallowTests = []struct { }, }`), }, - expectedError: "manifest enforcement should be independent", + expectedErrors: []string{ + "manifest enforcement should be independent", + }, }, { name: "no treble_linker_namespaces.cflags", fs: map[string][]byte{ - "Blueprints": []byte(` + "Android.bp": []byte(` cc_library { name: "libexample", product_variables: { @@ -105,12 +171,14 @@ var neverallowTests = []struct { }, }`), }, - expectedError: "nothing should care if linker namespaces are enabled or not", + expectedErrors: []string{ + "nothing should care if linker namespaces are enabled or not", + }, }, { name: "libc_bionic_ndk treble_linker_namespaces.cflags", fs: map[string][]byte{ - "Blueprints": []byte(` + "Android.bp": []byte(` cc_library { name: "libc_bionic_ndk", product_variables: { @@ -120,76 +188,172 @@ var neverallowTests = []struct { }, }`), }, - expectedError: "", - }, - { - name: "dependency on core-libart", - fs: map[string][]byte{ - "Blueprints": []byte(` - java_library { - name: "needs_core_libart", - libs: ["core-libart"], - }`), - }, - expectedError: "Only core libraries projects can depend on core-libart", }, { name: "dependency on updatable-media", fs: map[string][]byte{ - "Blueprints": []byte(` + "Android.bp": []byte(` java_library { name: "needs_updatable_media", libs: ["updatable-media"], }`), }, - expectedError: "updatable-media includes private APIs. Use updatable_media_stubs instead.", + expectedErrors: []string{ + "updatable-media includes private APIs. Use updatable_media_stubs instead.", + }, }, { name: "java_device_for_host", fs: map[string][]byte{ - "Blueprints": []byte(` + "Android.bp": []byte(` java_device_for_host { name: "device_for_host", libs: ["core-libart"], }`), }, - expectedError: "java_device_for_host can only be used in whitelisted projects", + expectedErrors: []string{ + "java_device_for_host can only be used in allowed projects", + }, + }, + // Libcore rule tests + { + name: "sdk_version: \"none\" inside core libraries", + fs: map[string][]byte{ + "libcore/Android.bp": []byte(` + java_library { + name: "inside_core_libraries", + sdk_version: "none", + }`), + }, + }, + { + name: "sdk_version: \"none\" on android_*stubs_current stub", + fs: map[string][]byte{ + "frameworks/base/Android.bp": []byte(` + java_library { + name: "android_stubs_current", + sdk_version: "none", + }`), + }, + }, + { + name: "sdk_version: \"none\" outside core libraries", + fs: map[string][]byte{ + "Android.bp": []byte(` + java_library { + name: "outside_core_libraries", + sdk_version: "none", + }`), + }, + expectedErrors: []string{ + "module \"outside_core_libraries\": violates neverallow", + }, + }, + { + name: "sdk_version: \"current\"", + fs: map[string][]byte{ + "Android.bp": []byte(` + java_library { + name: "outside_core_libraries", + sdk_version: "current", + }`), + }, + }, + // CC sdk rule tests + { + name: `"sdk_variant_only" outside allowed list`, + fs: map[string][]byte{ + "Android.bp": []byte(` + cc_library { + name: "outside_allowed_list", + sdk_version: "current", + sdk_variant_only: true, + }`), + }, + expectedErrors: []string{ + `module "outside_allowed_list": violates neverallow`, + }, + }, + { + name: `"sdk_variant_only: false" outside allowed list`, + fs: map[string][]byte{ + "Android.bp": []byte(` + cc_library { + name: "outside_allowed_list", + sdk_version: "current", + sdk_variant_only: false, + }`), + }, + expectedErrors: []string{ + `module "outside_allowed_list": violates neverallow`, + }, + }, + { + name: `"platform" outside allowed list`, + fs: map[string][]byte{ + "Android.bp": []byte(` + cc_library { + name: "outside_allowed_list", + platform: { + shared_libs: ["libfoo"], + }, + }`), + }, + expectedErrors: []string{ + `module "outside_allowed_list": violates neverallow`, + }, + }, + { + name: "uncompress_dex inside art", + fs: map[string][]byte{ + "art/Android.bp": []byte(` + java_library { + name: "inside_art_libraries", + uncompress_dex: true, + }`), + }, + }, + { + name: "uncompress_dex outside art", + fs: map[string][]byte{ + "other/Android.bp": []byte(` + java_library { + name: "outside_art_libraries", + uncompress_dex: true, + }`), + }, + expectedErrors: []string{ + "module \"outside_art_libraries\": violates neverallow", + }, }, } func TestNeverallow(t *testing.T) { - buildDir, err := ioutil.TempDir("", "soong_neverallow_test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(buildDir) - - config := TestConfig(buildDir, nil) - for _, test := range neverallowTests { - t.Run(test.name, func(t *testing.T) { - _, errs := testNeverallow(t, config, test.fs) + // Create a test per config to allow for test specific config, e.g. test rules. + config := TestConfig(buildDir, nil, "", test.fs) - if test.expectedError == "" { - FailIfErrored(t, errs) - } else { - FailIfNoMatchingErrors(t, test.expectedError, errs) + t.Run(test.name, func(t *testing.T) { + // If the test has its own rules then use them instead of the default ones. + if test.rules != nil { + SetTestNeverallowRules(config, test.rules) } + _, errs := testNeverallow(config) + CheckErrorsAgainstExpectations(t, errs, test.expectedErrors) }) } } -func testNeverallow(t *testing.T, config Config, fs map[string][]byte) (*TestContext, []error) { +func testNeverallow(config Config) (*TestContext, []error) { ctx := NewTestContext() - ctx.RegisterModuleType("cc_library", ModuleFactoryAdaptor(newMockCcLibraryModule)) - ctx.RegisterModuleType("java_library", ModuleFactoryAdaptor(newMockJavaLibraryModule)) - ctx.RegisterModuleType("java_device_for_host", ModuleFactoryAdaptor(newMockJavaLibraryModule)) - ctx.PostDepsMutators(registerNeverallowMutator) - ctx.Register() - - ctx.MockFileSystem(fs) + ctx.RegisterModuleType("cc_library", newMockCcLibraryModule) + ctx.RegisterModuleType("java_library", newMockJavaLibraryModule) + ctx.RegisterModuleType("java_library_host", newMockJavaLibraryModule) + ctx.RegisterModuleType("java_device_for_host", newMockJavaLibraryModule) + ctx.PostDepsMutators(RegisterNeverallowMutator) + ctx.Register(config) - _, errs := ctx.ParseBlueprintsFiles("Blueprints") + _, errs := ctx.ParseBlueprintsFiles("Android.bp") if len(errs) > 0 { return ctx, errs } @@ -199,7 +363,11 @@ func testNeverallow(t *testing.T, config Config, fs map[string][]byte) (*TestCon } type mockCcLibraryProperties struct { + Include_dirs []string Vendor_available *bool + Static_libs []string + Sdk_version *string + Sdk_variant_only *bool Vndk struct { Enabled *bool @@ -216,6 +384,10 @@ type mockCcLibraryProperties struct { Cflags []string } } + + Platform struct { + Shared_libs []string + } } type mockCcLibraryModule struct { @@ -230,11 +402,26 @@ func newMockCcLibraryModule() Module { return m } +type neverallowTestDependencyTag struct { + blueprint.BaseDependencyTag + name string +} + +var staticDepTag = neverallowTestDependencyTag{name: "static"} + +func (c *mockCcLibraryModule) DepsMutator(ctx BottomUpMutatorContext) { + for _, lib := range c.properties.Static_libs { + ctx.AddDependency(ctx.Module(), staticDepTag, lib) + } +} + func (p *mockCcLibraryModule) GenerateAndroidBuildActions(ModuleContext) { } type mockJavaLibraryProperties struct { - Libs []string + Libs []string + Sdk_version *string + Uncompress_dex *bool } type mockJavaLibraryModule struct { diff --git a/android/notices.go b/android/notices.go index dbb88fc1a..bf273b544 100644 --- a/android/notices.go +++ b/android/notices.go @@ -27,6 +27,13 @@ func init() { pctx.HostBinToolVariable("minigzip", "minigzip") } +type NoticeOutputs struct { + Merged OptionalPath + TxtOutput OptionalPath + HtmlOutput OptionalPath + HtmlGzOutput OptionalPath +} + var ( mergeNoticesRule = pctx.AndroidStaticRule("mergeNoticesRule", blueprint.RuleParams{ Command: `${merge_notices} --output $out $in`, @@ -35,13 +42,13 @@ var ( }) generateNoticeRule = pctx.AndroidStaticRule("generateNoticeRule", blueprint.RuleParams{ - Command: `rm -rf $tmpDir $$(dirname $out) && ` + - `mkdir -p $tmpDir $$(dirname $out) && ` + - `${generate_notice} --text-output $tmpDir/NOTICE.txt --html-output $tmpDir/NOTICE.html -t "$title" -s $inputDir && ` + - `${minigzip} -c $tmpDir/NOTICE.html > $out`, + Command: `rm -rf $$(dirname $txtOut) $$(dirname $htmlOut) $$(dirname $out) && ` + + `mkdir -p $$(dirname $txtOut) $$(dirname $htmlOut) $$(dirname $out) && ` + + `${generate_notice} --text-output $txtOut --html-output $htmlOut -t "$title" -s $inputDir && ` + + `${minigzip} -c $htmlOut > $out`, CommandDeps: []string{"${generate_notice}", "${minigzip}"}, Description: "produce notice file $out", - }, "tmpDir", "title", "inputDir") + }, "txtOut", "htmlOut", "title", "inputDir") ) func MergeNotices(ctx ModuleContext, mergedNotice WritablePath, noticePaths []Path) { @@ -53,8 +60,8 @@ func MergeNotices(ctx ModuleContext, mergedNotice WritablePath, noticePaths []Pa }) } -func BuildNoticeOutput(ctx ModuleContext, installPath OutputPath, installFilename string, - noticePaths []Path) ModuleOutPath { +func BuildNoticeOutput(ctx ModuleContext, installPath InstallPath, installFilename string, + noticePaths []Path) NoticeOutputs { // Merge all NOTICE files into one. // TODO(jungjw): We should just produce a well-formatted NOTICE.html file in a single pass. // @@ -68,20 +75,28 @@ func BuildNoticeOutput(ctx ModuleContext, installPath OutputPath, installFilenam MergeNotices(ctx, mergedNotice, noticePaths) // Transform the merged NOTICE file into a gzipped HTML file. - noticeOutput := PathForModuleOut(ctx, "NOTICE", "NOTICE.html.gz") - tmpDir := PathForModuleOut(ctx, "NOTICE_tmp") + txtOuptut := PathForModuleOut(ctx, "NOTICE_txt", "NOTICE.txt") + htmlOutput := PathForModuleOut(ctx, "NOTICE_html", "NOTICE.html") + htmlGzOutput := PathForModuleOut(ctx, "NOTICE", "NOTICE.html.gz") title := "Notices for " + ctx.ModuleName() ctx.Build(pctx, BuildParams{ - Rule: generateNoticeRule, - Description: "generate notice output", - Input: mergedNotice, - Output: noticeOutput, + Rule: generateNoticeRule, + Description: "generate notice output", + Input: mergedNotice, + Output: htmlGzOutput, + ImplicitOutputs: WritablePaths{txtOuptut, htmlOutput}, Args: map[string]string{ - "tmpDir": tmpDir.String(), + "txtOut": txtOuptut.String(), + "htmlOut": htmlOutput.String(), "title": title, "inputDir": PathForModuleOut(ctx, "NOTICE_FILES/src").String(), }, }) - return noticeOutput + return NoticeOutputs{ + Merged: OptionalPathForPath(mergedNotice), + TxtOutput: OptionalPathForPath(txtOuptut), + HtmlOutput: OptionalPathForPath(htmlOutput), + HtmlGzOutput: OptionalPathForPath(htmlGzOutput), + } } diff --git a/android/onceper.go b/android/onceper.go index 5ad17fa91..481cdea05 100644 --- a/android/onceper.go +++ b/android/onceper.go @@ -40,7 +40,8 @@ func (once *OncePer) maybeWaitFor(key OnceKey, value interface{}) interface{} { } // Once computes a value the first time it is called with a given key per OncePer, and returns the -// value without recomputing when called with the same key. key must be hashable. +// value without recomputing when called with the same key. key must be hashable. If value panics +// the panic will be propagated but the next call to Once with the same key will return nil. func (once *OncePer) Once(key OnceKey, value func() interface{}) interface{} { // Fast path: check if the key is already in the map if v, ok := once.values.Load(key); ok { @@ -54,10 +55,15 @@ func (once *OncePer) Once(key OnceKey, value func() interface{}) interface{} { return once.maybeWaitFor(key, v) } - // The waiter is inserted, call the value constructor, store it, and signal the waiter - v := value() - once.values.Store(key, v) - close(waiter) + // The waiter is inserted, call the value constructor, store it, and signal the waiter. Use defer in case + // the function panics. + var v interface{} + defer func() { + once.values.Store(key, v) + close(waiter) + }() + + v = value() return v } @@ -89,6 +95,16 @@ func (once *OncePer) Once2StringSlice(key OnceKey, value func() ([]string, []str return s[0], s[1] } +// OncePath is the same as Once, but returns the value cast to a Path +func (once *OncePer) OncePath(key OnceKey, value func() Path) Path { + return once.Once(key, func() interface{} { return value() }).(Path) +} + +// OncePath is the same as Once, but returns the value cast to a SourcePath +func (once *OncePer) OnceSourcePath(key OnceKey, value func() SourcePath) SourcePath { + return once.Once(key, func() interface{} { return value() }).(SourcePath) +} + // OnceKey is an opaque type to be used as the key in calls to Once. type OnceKey struct { key interface{} diff --git a/android/onceper_test.go b/android/onceper_test.go index 95303bafd..1a55ff4b1 100644 --- a/android/onceper_test.go +++ b/android/onceper_test.go @@ -175,3 +175,43 @@ func TestOncePerReentrant(t *testing.T) { t.Errorf(`reentrant Once should return "a": %q`, a) } } + +// Test that a recovered panic in a Once function doesn't deadlock +func TestOncePerPanic(t *testing.T) { + once := OncePer{} + key := NewOnceKey("key") + + ch := make(chan interface{}) + + var a interface{} + + go func() { + defer func() { + ch <- recover() + }() + + a = once.Once(key, func() interface{} { + panic("foo") + }) + }() + + p := <-ch + + if p.(string) != "foo" { + t.Errorf(`expected panic with "foo", got %#v`, p) + } + + if a != nil { + t.Errorf(`expected a to be nil, got %#v`, a) + } + + // If the call to Once that panicked leaves the key in a bad state this will deadlock + b := once.Once(key, func() interface{} { + return "bar" + }) + + // The second call to Once should return nil inserted by the first call that panicked. + if b != nil { + t.Errorf(`expected b to be nil, got %#v`, b) + } +} diff --git a/android/override_module.go b/android/override_module.go index ba66182ec..90ddf5049 100644 --- a/android/override_module.go +++ b/android/override_module.go @@ -42,6 +42,11 @@ type OverrideModule interface { setOverridingProperties(properties []interface{}) getOverrideModuleProperties() *OverrideModuleProperties + + // Internal funcs to handle interoperability between override modules and prebuilts. + // i.e. cases where an overriding module, too, is overridden by a prebuilt module. + setOverriddenByPrebuilt(overridden bool) + getOverriddenByPrebuilt() bool } // Base module struct for override module types @@ -49,6 +54,8 @@ type OverrideModuleBase struct { moduleProperties OverrideModuleProperties overridingProperties []interface{} + + overriddenByPrebuilt bool } type OverrideModuleProperties struct { @@ -70,6 +77,18 @@ func (o *OverrideModuleBase) getOverrideModuleProperties() *OverrideModuleProper return &o.moduleProperties } +func (o *OverrideModuleBase) GetOverriddenModuleName() string { + return proptools.String(o.moduleProperties.Base) +} + +func (o *OverrideModuleBase) setOverriddenByPrebuilt(overridden bool) { + o.overriddenByPrebuilt = overridden +} + +func (o *OverrideModuleBase) getOverriddenByPrebuilt() bool { + return o.overriddenByPrebuilt +} + func InitOverrideModule(m OverrideModule) { m.setOverridingProperties(m.GetProperties()) @@ -78,14 +97,26 @@ func InitOverrideModule(m OverrideModule) { // Interface for overridable module types, e.g. android_app, apex type OverridableModule interface { + Module + moduleBase() *OverridableModuleBase + setOverridableProperties(prop []interface{}) addOverride(o OverrideModule) getOverrides() []OverrideModule override(ctx BaseModuleContext, o OverrideModule) + GetOverriddenBy() string setOverridesProperty(overridesProperties *[]string) + + // Due to complications with incoming dependencies, overrides are processed after DepsMutator. + // So, overridable properties need to be handled in a separate, dedicated deps mutator. + OverridablePropertiesDepsMutator(ctx BottomUpMutatorContext) +} + +type overridableModuleProperties struct { + OverriddenBy string `blueprint:"mutated"` } // Base module struct for overridable module types @@ -104,11 +135,18 @@ type OverridableModuleBase struct { // set this to a pointer to the property through the InitOverridableModule function, so that // override information is propagated and aggregated correctly. overridesProperty *[]string + + overridableModuleProperties overridableModuleProperties } func InitOverridableModule(m OverridableModule, overridesProperty *[]string) { m.setOverridableProperties(m.(Module).GetProperties()) m.setOverridesProperty(overridesProperty) + m.AddProperties(&m.moduleBase().overridableModuleProperties) +} + +func (o *OverridableModuleBase) moduleBase() *OverridableModuleBase { + return o } func (b *OverridableModuleBase) setOverridableProperties(prop []interface{}) { @@ -132,15 +170,10 @@ func (b *OverridableModuleBase) setOverridesProperty(overridesProperty *[]string // Overrides a base module with the given OverrideModule. func (b *OverridableModuleBase) override(ctx BaseModuleContext, o OverrideModule) { - // Adds the base module to the overrides property, if exists, of the overriding module. See the - // comment on OverridableModuleBase.overridesProperty for details. - if b.overridesProperty != nil { - *b.overridesProperty = append(*b.overridesProperty, ctx.ModuleName()) - } for _, p := range b.overridableProperties { for _, op := range o.getOverridingProperties() { if proptools.TypeEqual(p, op) { - err := proptools.AppendProperties(p, op, nil) + err := proptools.ExtendProperties(p, op, nil, proptools.OrderReplace) if err != nil { if propertyErr, ok := err.(*proptools.ExtendPropertyError); ok { ctx.PropertyErrorf(propertyErr.Property, "%s", propertyErr.Err.Error()) @@ -151,14 +184,33 @@ func (b *OverridableModuleBase) override(ctx BaseModuleContext, o OverrideModule } } } + // Adds the base module to the overrides property, if exists, of the overriding module. See the + // comment on OverridableModuleBase.overridesProperty for details. + if b.overridesProperty != nil { + *b.overridesProperty = append(*b.overridesProperty, ctx.ModuleName()) + } + b.overridableModuleProperties.OverriddenBy = o.Name() +} + +// GetOverriddenBy returns the name of the override module that has overridden this module. +// For example, if an override module foo has its 'base' property set to bar, then another local variant +// of bar is created and its properties are overriden by foo. This method returns bar when called from +// the new local variant. It returns "" when called from the original variant of bar. +func (b *OverridableModuleBase) GetOverriddenBy() string { + return b.overridableModuleProperties.OverriddenBy +} + +func (b *OverridableModuleBase) OverridablePropertiesDepsMutator(ctx BottomUpMutatorContext) { } // Mutators for override/overridable modules. All the fun happens in these functions. It is critical // to keep them in this order and not put any order mutators between them. -func RegisterOverridePreArchMutators(ctx RegisterMutatorsContext) { +func RegisterOverridePostDepsMutators(ctx RegisterMutatorsContext) { ctx.BottomUp("override_deps", overrideModuleDepsMutator).Parallel() ctx.TopDown("register_override", registerOverrideMutator).Parallel() ctx.BottomUp("perform_override", performOverrideMutator).Parallel() + ctx.BottomUp("overridable_deps", overridableModuleDepsMutator).Parallel() + ctx.BottomUp("replace_deps_on_override", replaceDepsOnOverridingModuleMutator).Parallel() } type overrideBaseDependencyTag struct { @@ -171,6 +223,18 @@ var overrideBaseDepTag overrideBaseDependencyTag // next phase. func overrideModuleDepsMutator(ctx BottomUpMutatorContext) { if module, ok := ctx.Module().(OverrideModule); ok { + // See if there's a prebuilt module that overrides this override module with prefer flag, + // in which case we call SkipInstall on the corresponding variant later. + ctx.VisitDirectDepsWithTag(PrebuiltDepTag, func(dep Module) { + prebuilt, ok := dep.(PrebuiltInterface) + if !ok { + panic("PrebuiltDepTag leads to a non-prebuilt module " + dep.Name()) + } + if prebuilt.Prebuilt().UsePrebuilt() { + module.setOverriddenByPrebuilt(true) + return + } + }) ctx.AddDependency(ctx.Module(), overrideBaseDepTag, *module.getOverrideModuleProperties().Base) } } @@ -202,8 +266,39 @@ func performOverrideMutator(ctx BottomUpMutatorContext) { variants[i+1] = o.(Module).Name() } mods := ctx.CreateLocalVariations(variants...) + // Make the original variation the default one to depend on if no other override module variant + // is specified. + ctx.AliasVariation(variants[0]) for i, o := range overrides { mods[i+1].(OverridableModule).override(ctx, o) + if o.getOverriddenByPrebuilt() { + // The overriding module itself, too, is overridden by a prebuilt. Skip its installation. + mods[i+1].SkipInstall() + } + } + } else if o, ok := ctx.Module().(OverrideModule); ok { + // Create a variant of the overriding module with its own name. This matches the above local + // variant name rule for overridden modules, and thus allows ReplaceDependencies to match the + // two. + ctx.CreateLocalVariations(o.Name()) + // To allow dependencies to be added without having to know the above variation. + ctx.AliasVariation(o.Name()) + } +} + +func overridableModuleDepsMutator(ctx BottomUpMutatorContext) { + if b, ok := ctx.Module().(OverridableModule); ok { + b.OverridablePropertiesDepsMutator(ctx) + } +} + +func replaceDepsOnOverridingModuleMutator(ctx BottomUpMutatorContext) { + if b, ok := ctx.Module().(OverridableModule); ok { + if o := b.GetOverriddenBy(); o != "" { + // Redirect dependencies on the overriding module to this overridden module. Overriding + // modules are basically pseudo modules, and all build actions are associated to overridden + // modules. Therefore, dependencies on overriding modules need to be forwarded there as well. + ctx.ReplaceDependencies(o) } } } diff --git a/android/package.go b/android/package.go index 6949c4e2d..053fec286 100644 --- a/android/package.go +++ b/android/package.go @@ -15,14 +15,17 @@ package android import ( - "fmt" - "sync/atomic" - "github.com/google/blueprint" + "github.com/google/blueprint/proptools" ) func init() { - RegisterModuleType("package", PackageFactory) + RegisterPackageBuildComponents(InitRegistrationContext) +} + +// Register the package module type. +func RegisterPackageBuildComponents(ctx RegistrationContext) { + ctx.RegisterModuleType("package", PackageFactory) } type packageProperties struct { @@ -36,8 +39,6 @@ type packageModule struct { ModuleBase properties packageProperties - // The module dir - name string `blueprint:"mutated"` } func (p *packageModule) GenerateAndroidBuildActions(ModuleContext) { @@ -48,46 +49,26 @@ func (p *packageModule) GenerateBuildActions(ctx blueprint.ModuleContext) { // Nothing to do. } -func (p *packageModule) DepsMutator(ctx BottomUpMutatorContext) { - // Nothing to do. +func (p *packageModule) qualifiedModuleId(ctx BaseModuleContext) qualifiedModuleName { + // Override to create a package id. + return newPackageId(ctx.ModuleDir()) } -func (p *packageModule) Name() string { - return p.name -} - -func registerPackageRenamer(ctx RegisterMutatorsContext) { - ctx.BottomUp("packages", packageRenamer).Parallel() -} - -// packageRenamer ensures that every package gets named -func packageRenamer(ctx BottomUpMutatorContext) { - if p, ok := ctx.Module().(*packageModule); ok { - p.name = "//" + ctx.ModuleDir() - ctx.Rename("//" + ctx.ModuleDir()) - } -} - -// Counter to ensure package modules are created with a unique name within whatever namespace they -// belong. -var packageCount uint32 = 0 - func PackageFactory() Module { module := &packageModule{} - // Get a unique if for the package. Has to be done atomically as the creation of the modules are - // done in parallel. - id := atomic.AddUint32(&packageCount, 1) - module.name = fmt.Sprintf("soong_package_%d", id) - module.AddProperties(&module.properties) // The name is the relative path from build root to the directory containing this // module. Set that name at the earliest possible moment that information is available // which is in a LoadHook. AddLoadHook(module, func(ctx LoadHookContext) { - module.name = "//" + ctx.ModuleDir() + module.nameProperties.Name = proptools.StringPtr("//" + ctx.ModuleDir()) }) + // The default_visibility property needs to be checked and parsed by the visibility module during + // its checking and parsing phases so make it the primary visibility property. + setPrimaryVisibilityProperty(module, "default_visibility", &module.properties.Default_visibility) + return module } diff --git a/android/package_ctx.go b/android/package_ctx.go index 166346930..0de356e1c 100644 --- a/android/package_ctx.go +++ b/android/package_ctx.go @@ -16,11 +16,9 @@ package android import ( "fmt" - "runtime" "strings" "github.com/google/blueprint" - "github.com/google/blueprint/pathtools" ) // PackageContext is a wrapper for blueprint.PackageContext that adds @@ -61,10 +59,6 @@ func (e *configErrorWrapper) AddNinjaFileDeps(deps ...string) { e.pctx.AddNinjaFileDeps(deps...) } -func (e *configErrorWrapper) Fs() pathtools.FileSystem { - return nil -} - type PackageVarContext interface { PathContext errorfContext @@ -115,7 +109,7 @@ func (p PackageContext) RuleFunc(name string, if len(ctx.errors) > 0 { return params, ctx.errors[0] } - if (ctx.Config().UseGoma() || ctx.Config().UseRBE()) && params.Pool == nil { + if ctx.Config().UseRemoteBuild() && params.Pool == nil { // When USE_GOMA=true or USE_RBE=true are set and the rule is not supported by // goma/RBE, restrict jobs to the local parallelism value params.Pool = localPool @@ -177,46 +171,30 @@ func (p PackageContext) SourcePathVariableWithEnvOverride(name, path, env string // package-scoped variable's initialization. func (p PackageContext) HostBinToolVariable(name, path string) blueprint.Variable { return p.VariableFunc(name, func(ctx PackageVarContext) string { - return p.HostBinToolPath(ctx, path).String() + return ctx.Config().HostToolPath(ctx, path).String() }) } -func (p PackageContext) HostBinToolPath(ctx PackageVarContext, path string) Path { - return PathForOutput(ctx, "host", ctx.Config().PrebuiltOS(), "bin", path) -} - // HostJNIToolVariable returns a Variable whose value is the path to a host tool // in the lib directory for host targets. It may only be called during a Go // package's initialization - either from the init() function or as part of a // package-scoped variable's initialization. func (p PackageContext) HostJNIToolVariable(name, path string) blueprint.Variable { return p.VariableFunc(name, func(ctx PackageVarContext) string { - return p.HostJNIToolPath(ctx, path).String() + return ctx.Config().HostJNIToolPath(ctx, path).String() }) } -func (p PackageContext) HostJNIToolPath(ctx PackageVarContext, path string) Path { - ext := ".so" - if runtime.GOOS == "darwin" { - ext = ".dylib" - } - return PathForOutput(ctx, "host", ctx.Config().PrebuiltOS(), "lib64", path+ext) -} - // HostJavaToolVariable returns a Variable whose value is the path to a host // tool in the frameworks directory for host targets. It may only be called // during a Go package's initialization - either from the init() function or as // part of a package-scoped variable's initialization. func (p PackageContext) HostJavaToolVariable(name, path string) blueprint.Variable { return p.VariableFunc(name, func(ctx PackageVarContext) string { - return p.HostJavaToolPath(ctx, path).String() + return ctx.Config().HostJavaToolPath(ctx, path).String() }) } -func (p PackageContext) HostJavaToolPath(ctx PackageVarContext, path string) Path { - return PathForOutput(ctx, "host", ctx.Config().PrebuiltOS(), "framework", path) -} - // IntermediatesPathVariable returns a Variable whose value is the intermediate // directory appended with the supplied path. It may only be called during a Go // package's initialization - either from the init() function or as part of a diff --git a/android/package_test.go b/android/package_test.go index c7e9c553f..f25599cf8 100644 --- a/android/package_test.go +++ b/android/package_test.go @@ -1,8 +1,6 @@ package android import ( - "io/ioutil" - "os" "testing" ) @@ -53,7 +51,7 @@ var packageTests = []struct { }`), }, expectedErrors: []string{ - `"//top" conflicts with existing module`, + `module "//top" already defined`, }, }, } @@ -85,20 +83,13 @@ func TestPackage(t *testing.T) { } func testPackage(fs map[string][]byte) (*TestContext, []error) { - buildDir, err := ioutil.TempDir("", "package_test") - if err != nil { - return nil, []error{err} - } - defer os.RemoveAll(buildDir) // Create a new config per test as visibility information is stored in the config. - config := TestArchConfig(buildDir, nil) + config := TestArchConfig(buildDir, nil, "", fs) ctx := NewTestArchContext() - ctx.MockFileSystem(fs) - ctx.RegisterModuleType("package", ModuleFactoryAdaptor(PackageFactory)) - ctx.PreArchMutators(registerPackageRenamer) - ctx.Register() + RegisterPackageBuildComponents(ctx) + ctx.Register(config) _, errs := ctx.ParseBlueprintsFiles(".") if len(errs) > 0 { diff --git a/android/path_properties.go b/android/path_properties.go index 1a12290d3..6b1cdb308 100644 --- a/android/path_properties.go +++ b/android/path_properties.go @@ -35,18 +35,17 @@ func pathDepsMutator(ctx BottomUpMutatorContext) { props := m.base().generalProperties + var pathProperties []string for _, ps := range props { - pathProperties := pathPropertiesForPropertyStruct(ctx, ps) - pathProperties = FirstUniqueStrings(pathProperties) + pathProperties = append(pathProperties, pathPropertiesForPropertyStruct(ctx, ps)...) + } - var deps []string - for _, s := range pathProperties { - if m := SrcIsModule(s); m != "" { - deps = append(deps, m) - } - } + pathProperties = FirstUniqueStrings(pathProperties) - ctx.AddDependency(ctx.Module(), SourceDepTag, deps...) + for _, s := range pathProperties { + if m, t := SrcIsModuleWithTag(s); m != "" { + ctx.AddDependency(ctx.Module(), sourceOrOutputDepTag(t), m) + } } } diff --git a/android/path_properties_test.go b/android/path_properties_test.go index ecc2d2122..f367b82b8 100644 --- a/android/path_properties_test.go +++ b/android/path_properties_test.go @@ -15,8 +15,6 @@ package android import ( - "io/ioutil" - "os" "reflect" "testing" ) @@ -30,20 +28,34 @@ type pathDepsMutatorTestModule struct { Qux string } + // A second property struct with a duplicate property name + props2 struct { + Foo string `android:"path"` + } + sourceDeps []string } func pathDepsMutatorTestModuleFactory() Module { module := &pathDepsMutatorTestModule{} - module.AddProperties(&module.props) + module.AddProperties(&module.props, &module.props2) InitAndroidArchModule(module, DeviceSupported, MultilibBoth) return module } func (p *pathDepsMutatorTestModule) GenerateAndroidBuildActions(ctx ModuleContext) { - ctx.VisitDirectDepsWithTag(SourceDepTag, func(dep Module) { - p.sourceDeps = append(p.sourceDeps, ctx.OtherModuleName(dep)) + ctx.VisitDirectDeps(func(dep Module) { + if _, ok := ctx.OtherModuleDependencyTag(dep).(sourceOrOutputDependencyTag); ok { + p.sourceDeps = append(p.sourceDeps, ctx.OtherModuleName(dep)) + } }) + + if p.props.Foo != "" { + // Make sure there is only one dependency on a module listed in a property present in multiple property structs + if ctx.GetDirectDepWithTag(SrcIsModule(p.props.Foo), sourceOrOutputDepTag("")) == nil { + ctx.ModuleErrorf("GetDirectDepWithTag failed") + } + } } func TestPathDepsMutator(t *testing.T) { @@ -59,7 +71,7 @@ func TestPathDepsMutator(t *testing.T) { name: "foo", foo: ":a", bar: [":b"], - baz: ":c", + baz: ":c{.bar}", qux: ":d", }`, deps: []string{"a", "b", "c"}, @@ -83,20 +95,8 @@ func TestPathDepsMutator(t *testing.T) { }, } - buildDir, err := ioutil.TempDir("", "soong_path_properties_test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(buildDir) - for _, test := range tests { t.Run(test.name, func(t *testing.T) { - config := TestArchConfig(buildDir, nil) - ctx := NewTestArchContext() - - ctx.RegisterModuleType("test", ModuleFactoryAdaptor(pathDepsMutatorTestModuleFactory)) - ctx.RegisterModuleType("filegroup", ModuleFactoryAdaptor(FileGroupFactory)) - bp := test.bp + ` filegroup { name: "a", @@ -115,13 +115,13 @@ func TestPathDepsMutator(t *testing.T) { } ` - mockFS := map[string][]byte{ - "Android.bp": []byte(bp), - } + config := TestArchConfig(buildDir, nil, bp, nil) + ctx := NewTestArchContext() - ctx.MockFileSystem(mockFS) + ctx.RegisterModuleType("test", pathDepsMutatorTestModuleFactory) + ctx.RegisterModuleType("filegroup", FileGroupFactory) - ctx.Register() + ctx.Register(config) _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) FailIfErrored(t, errs) _, errs = ctx.PrepareBuildActions(config) diff --git a/android/paths.go b/android/paths.go index 0f20b844d..ddbeed3c5 100644 --- a/android/paths.go +++ b/android/paths.go @@ -16,6 +16,8 @@ package android import ( "fmt" + "io/ioutil" + "os" "path/filepath" "reflect" "sort" @@ -25,10 +27,11 @@ import ( "github.com/google/blueprint/pathtools" ) +var absSrcDir string + // PathContext is the subset of a (Module|Singleton)Context required by the // Path methods. type PathContext interface { - Fs() pathtools.FileSystem Config() Config AddNinjaFileDeps(deps ...string) } @@ -41,13 +44,15 @@ var _ PathContext = SingletonContext(nil) var _ PathContext = ModuleContext(nil) type ModuleInstallPathContext interface { - PathContext - - androidBaseContext + BaseModuleContext InstallInData() bool + InstallInTestcases() bool InstallInSanitizerDir() bool + InstallInRamdisk() bool InstallInRecovery() bool + InstallInRoot() bool + InstallBypassMake() bool } var _ ModuleInstallPathContext = ModuleContext(nil) @@ -88,6 +93,15 @@ func reportPathErrorf(ctx PathContext, format string, args ...interface{}) { } } +func pathContextName(ctx PathContext, module blueprint.Module) string { + if x, ok := ctx.(interface{ ModuleName(blueprint.Module) string }); ok { + return x.ModuleName(module) + } else if x, ok := ctx.(interface{ OtherModuleName(blueprint.Module) string }); ok { + return x.OtherModuleName(module) + } + return "unknown" +} + type Path interface { // Returns the path in string form String() string @@ -108,6 +122,9 @@ type Path interface { type WritablePath interface { Path + // return the path to the build directory. + buildDir() string + // the writablePath method doesn't directly do anything, // but it allows a struct to distinguish between whether or not it implements the WritablePath interface writablePath() @@ -217,21 +234,23 @@ func ExistentPathsForSources(ctx PathContext, paths []string) Paths { return ret } -// PathsForModuleSrc returns Paths rooted from the module's local source directory. It expands globs and references -// to SourceFileProducer modules using the ":name" syntax. Properties passed as the paths argument must have been -// annotated with struct tag `android:"path"` so that dependencies on SourceFileProducer modules will have already -// been handled by the path_properties mutator. If ctx.Config().AllowMissingDependencies() is true, then any missing -// SourceFileProducer dependencies will cause the module to be marked as having missing dependencies. +// PathsForModuleSrc returns Paths rooted from the module's local source directory. It expands globs, references to +// SourceFileProducer modules using the ":name" syntax, and references to OutputFileProducer modules using the +// ":name{.tag}" syntax. Properties passed as the paths argument must have been annotated with struct tag +// `android:"path"` so that dependencies on SourceFileProducer modules will have already been handled by the +// path_properties mutator. If ctx.Config().AllowMissingDependencies() is true then any missing SourceFileProducer or +// OutputFileProducer dependencies will cause the module to be marked as having missing dependencies. func PathsForModuleSrc(ctx ModuleContext, paths []string) Paths { return PathsForModuleSrcExcludes(ctx, paths, nil) } // PathsForModuleSrcExcludes returns Paths rooted from the module's local source directory, excluding paths listed in -// the excludes arguments. It expands globs and references to SourceFileProducer modules in both paths and excludes -// using the ":name" syntax. Properties passed as the paths or excludes argument must have been annotated with struct -// tag `android:"path"` so that dependencies on SourceFileProducer modules will have already been handled by the -// path_properties mutator. If ctx.Config().AllowMissingDependencies() is true, then any missing SourceFileProducer -// dependencies will cause the module to be marked as having missing dependencies. +// the excludes arguments. It expands globs, references to SourceFileProducer modules using the ":name" syntax, and +// references to OutputFileProducer modules using the ":name{.tag}" syntax. Properties passed as the paths or excludes +// argument must have been annotated with struct tag `android:"path"` so that dependencies on SourceFileProducer modules +// will have already been handled by the path_properties mutator. If ctx.Config().AllowMissingDependencies() is +// true then any missing SourceFileProducer or OutputFileProducer dependencies will cause the module to be marked as +// having missing dependencies. func PathsForModuleSrcExcludes(ctx ModuleContext, paths, excludes []string) Paths { ret, missingDeps := PathsAndMissingDepsForModuleSrcExcludes(ctx, paths, excludes) if ctx.Config().AllowMissingDependencies() { @@ -244,13 +263,41 @@ func PathsForModuleSrcExcludes(ctx ModuleContext, paths, excludes []string) Path return ret } +// OutputPaths is a slice of OutputPath objects, with helpers to operate on the collection. +type OutputPaths []OutputPath + +// Paths returns the OutputPaths as a Paths +func (p OutputPaths) Paths() Paths { + if p == nil { + return nil + } + ret := make(Paths, len(p)) + for i, path := range p { + ret[i] = path + } + return ret +} + +// Strings returns the string forms of the writable paths. +func (p OutputPaths) Strings() []string { + if p == nil { + return nil + } + ret := make([]string, len(p)) + for i, path := range p { + ret[i] = path.String() + } + return ret +} + // PathsAndMissingDepsForModuleSrcExcludes returns Paths rooted from the module's local source directory, excluding -// paths listed in the excludes arguments, and a list of missing dependencies. It expands globs and references to -// SourceFileProducer modules in both paths and excludes using the ":name" syntax. Properties passed as the paths or -// excludes argument must have been annotated with struct tag `android:"path"` so that dependencies on -// SourceFileProducer modules will have already been handled by the path_properties mutator. If -// ctx.Config().AllowMissingDependencies() is true, then any missing SourceFileProducer dependencies will be returned, -// and they will NOT cause the module to be marked as having missing dependencies. +// paths listed in the excludes arguments, and a list of missing dependencies. It expands globs, references to +// SourceFileProducer modules using the ":name" syntax, and references to OutputFileProducer modules using the +// ":name{.tag}" syntax. Properties passed as the paths or excludes argument must have been annotated with struct tag +// `android:"path"` so that dependencies on SourceFileProducer modules will have already been handled by the +// path_properties mutator. If ctx.Config().AllowMissingDependencies() is true then any missing SourceFileProducer or +// OutputFileProducer dependencies will be returned, and they will NOT cause the module to be marked as having missing +// dependencies. func PathsAndMissingDepsForModuleSrcExcludes(ctx ModuleContext, paths, excludes []string) (Paths, []string) { prefix := pathForModuleSrc(ctx).String() @@ -262,16 +309,24 @@ func PathsAndMissingDepsForModuleSrcExcludes(ctx ModuleContext, paths, excludes var missingExcludeDeps []string for _, e := range excludes { - if m := SrcIsModule(e); m != "" { - module := ctx.GetDirectDepWithTag(m, SourceDepTag) + if m, t := SrcIsModuleWithTag(e); m != "" { + module := ctx.GetDirectDepWithTag(m, sourceOrOutputDepTag(t)) if module == nil { missingExcludeDeps = append(missingExcludeDeps, m) continue } - if srcProducer, ok := module.(SourceFileProducer); ok { + if outProducer, ok := module.(OutputFileProducer); ok { + outputFiles, err := outProducer.OutputFiles(t) + if err != nil { + ctx.ModuleErrorf("path dependency %q: %s", e, err) + } + expandedExcludes = append(expandedExcludes, outputFiles.Strings()...) + } else if t != "" { + ctx.ModuleErrorf("path dependency %q is not an output file producing module", e) + } else if srcProducer, ok := module.(SourceFileProducer); ok { expandedExcludes = append(expandedExcludes, srcProducer.Srcs().Strings()...) } else { - ctx.ModuleErrorf("srcs dependency %q is not a source file producing module", m) + ctx.ModuleErrorf("path dependency %q is not a source file producing module", e) } } else { expandedExcludes = append(expandedExcludes, filepath.Join(prefix, e)) @@ -307,12 +362,20 @@ func (e missingDependencyError) Error() string { } func expandOneSrcPath(ctx ModuleContext, s string, expandedExcludes []string) (Paths, error) { - if m := SrcIsModule(s); m != "" { - module := ctx.GetDirectDepWithTag(m, SourceDepTag) + if m, t := SrcIsModuleWithTag(s); m != "" { + module := ctx.GetDirectDepWithTag(m, sourceOrOutputDepTag(t)) if module == nil { return nil, missingDependencyError{[]string{m}} } - if srcProducer, ok := module.(SourceFileProducer); ok { + if outProducer, ok := module.(OutputFileProducer); ok { + outputFiles, err := outProducer.OutputFiles(t) + if err != nil { + return nil, fmt.Errorf("path dependency %q: %s", s, err) + } + return outputFiles, nil + } else if t != "" { + return nil, fmt.Errorf("path dependency %q is not an output file producing module", s) + } else if srcProducer, ok := module.(SourceFileProducer); ok { moduleSrcs := srcProducer.Srcs() for _, e := range expandedExcludes { for j := 0; j < len(moduleSrcs); j++ { @@ -324,16 +387,16 @@ func expandOneSrcPath(ctx ModuleContext, s string, expandedExcludes []string) (P } return moduleSrcs, nil } else { - return nil, fmt.Errorf("path dependency %q is not a source file producing module", m) + return nil, fmt.Errorf("path dependency %q is not a source file producing module", s) } } else if pathtools.IsGlob(s) { paths := ctx.GlobFiles(pathForModuleSrc(ctx, s).String(), expandedExcludes) return PathsWithModuleSrcSubDir(ctx, paths, ""), nil } else { p := pathForModuleSrc(ctx, s) - if exists, _, err := ctx.Fs().Exists(p.String()); err != nil { + if exists, _, err := ctx.Config().fs.Exists(p.String()); err != nil { reportPathErrorf(ctx, "%s: %s", p, err.Error()) - } else if !exists { + } else if !exists && !ctx.Config().testAllowNonExistentPaths { reportPathErrorf(ctx, "module source path %q does not exist", p) } @@ -350,7 +413,7 @@ func expandOneSrcPath(ctx ModuleContext, s string, expandedExcludes []string) (P // each string. If incDirs is false, strip paths with a trailing '/' from the list. // It intended for use in globs that only list files that exist, so it allows '$' in // filenames. -func pathsForModuleSrcFromFullPath(ctx ModuleContext, paths []string, incDirs bool) Paths { +func pathsForModuleSrcFromFullPath(ctx EarlyModuleContext, paths []string, incDirs bool) Paths { prefix := filepath.Join(ctx.Config().srcDir, ctx.ModuleDir()) + "/" if prefix == "./" { prefix = "" @@ -403,6 +466,10 @@ func (p Paths) Strings() []string { return ret } +func CopyOfPaths(paths Paths) Paths { + return append(Paths(nil), paths...) +} + // FirstUniquePaths returns all unique elements of a Paths, keeping the first copy of each. It // modifies the Paths slice contents in place, and returns a subslice of the original slice. func FirstUniquePaths(list Paths) Paths { @@ -420,6 +487,15 @@ outer: return list[:k] } +// SortedUniquePaths returns what its name says +func SortedUniquePaths(list Paths) Paths { + unique := FirstUniquePaths(list) + sort.Slice(unique, func(i, j int) bool { + return unique[i].String() < unique[j].String() + }) + return unique +} + // LastUniquePaths returns all unique elements of a Paths, keeping the last copy of each. It // modifies the Paths slice contents in place, and returns a subslice of the original slice. func LastUniquePaths(list Paths) Paths { @@ -465,8 +541,12 @@ func inPathList(p Path, list []Path) bool { } func FilterPathList(list []Path, filter []Path) (remainder []Path, filtered []Path) { + return FilterPathListPredicate(list, func(p Path) bool { return inPathList(p, filter) }) +} + +func FilterPathListPredicate(list []Path, predicate func(Path) bool) (remainder []Path, filtered []Path) { for _, l := range list { - if inPathList(l, filter) { + if predicate(l) { filtered = append(filtered, l) } else { remainder = append(remainder, l) @@ -657,7 +737,7 @@ func existsWithDependencies(ctx PathContext, path SourcePath) (exists bool, err var deps []string // We cannot add build statements in this context, so we fall back to // AddNinjaFileDeps - files, deps, err = pathtools.Glob(path.String(), nil, pathtools.FollowSymlinks) + files, deps, err = ctx.Config().fs.Glob(path.String(), nil, pathtools.FollowSymlinks) ctx.AddNinjaFileDeps(deps...) } @@ -689,9 +769,9 @@ func PathForSource(ctx PathContext, pathComponents ...string) SourcePath { if !exists { modCtx.AddMissingDependencies([]string{path.String()}) } - } else if exists, _, err := ctx.Fs().Exists(path.String()); err != nil { + } else if exists, _, err := ctx.Config().fs.Exists(path.String()); err != nil { reportPathErrorf(ctx, "%s: %s", path, err.Error()) - } else if !exists { + } else if !exists && !ctx.Config().testAllowNonExistentPaths { reportPathErrorf(ctx, "source path %q does not exist", path) } return path @@ -773,13 +853,15 @@ func (p SourcePath) OverlayPath(ctx ModuleContext, path Path) OptionalPath { return OptionalPathForPath(PathForSource(ctx, relPath)) } -// OutputPath is a Path representing a file path rooted from the build directory +// OutputPath is a Path representing an intermediates file path rooted from the build directory type OutputPath struct { basePath + fullPath string } func (p OutputPath) withRel(rel string) OutputPath { p.basePath = p.basePath.withRel(rel) + p.fullPath = filepath.Join(p.fullPath, rel) return p } @@ -788,7 +870,12 @@ func (p OutputPath) WithoutRel() OutputPath { return p } +func (p OutputPath) buildDir() string { + return p.config.buildDir +} + var _ Path = OutputPath{} +var _ WritablePath = OutputPath{} // PathForOutput joins the provided paths and returns an OutputPath that is // validated to not escape the build dir. @@ -798,7 +885,9 @@ func PathForOutput(ctx PathContext, pathComponents ...string) OutputPath { if err != nil { reportPathError(ctx, err) } - return OutputPath{basePath{path, ctx.Config(), ""}} + fullPath := filepath.Join(ctx.Config().buildDir, path) + path = fullPath[len(fullPath)-len(path):] + return OutputPath{basePath{path, ctx.Config(), ""}, fullPath} } // PathsForOutput returns Paths rooted from buildDir @@ -813,11 +902,7 @@ func PathsForOutput(ctx PathContext, paths []string) WritablePaths { func (p OutputPath) writablePath() {} func (p OutputPath) String() string { - return filepath.Join(p.config.buildDir, p.path) -} - -func (p OutputPath) RelPathString() string { - return p.path + return p.fullPath } // Join creates a new OutputPath with paths... joined with the current path. The @@ -960,6 +1045,10 @@ type ModuleOutPath struct { var _ Path = ModuleOutPath{} +func (p ModuleOutPath) objPathWithExt(ctx ModuleContext, subdir, ext string) ModuleObjPath { + return PathForModuleObj(ctx, subdir, pathtools.ReplaceExtension(p.path, ext)) +} + func pathForModule(ctx ModuleContext) OutputPath { return PathForOutput(ctx, ".intermediates", ctx.ModuleDir(), ctx.ModuleName(), ctx.ModuleSubDir()) } @@ -967,7 +1056,7 @@ func pathForModule(ctx ModuleContext) OutputPath { // PathForVndkRefAbiDump returns an OptionalPath representing the path of the // reference abi dump for the given module. This is not guaranteed to be valid. func PathForVndkRefAbiDump(ctx ModuleContext, version, fileName string, - isLlndk, isGzip bool) OptionalPath { + isNdk, isLlndkOrVndk, isGzip bool) OptionalPath { arches := ctx.DeviceConfig().Arches() if len(arches) == 0 { @@ -980,10 +1069,12 @@ func PathForVndkRefAbiDump(ctx ModuleContext, version, fileName string, } var dirName string - if isLlndk { + if isNdk { dirName = "ndk" - } else { + } else if isLlndkOrVndk { dirName = "vndk" + } else { + dirName = "platform" // opt-in libs } binderBitness := ctx.DeviceConfig().BinderBitness() @@ -1082,9 +1173,51 @@ func PathForModuleRes(ctx ModuleContext, pathComponents ...string) ModuleResPath return ModuleResPath{PathForModuleOut(ctx, "res", p)} } +// InstallPath is a Path representing a installed file path rooted from the build directory +type InstallPath struct { + basePath + + baseDir string // "../" for Make paths to convert "out/soong" to "out", "" for Soong paths +} + +func (p InstallPath) buildDir() string { + return p.config.buildDir +} + +var _ Path = InstallPath{} +var _ WritablePath = InstallPath{} + +func (p InstallPath) writablePath() {} + +func (p InstallPath) String() string { + return filepath.Join(p.config.buildDir, p.baseDir, p.path) +} + +// Join creates a new InstallPath with paths... joined with the current path. The +// provided paths... may not use '..' to escape from the current path. +func (p InstallPath) Join(ctx PathContext, paths ...string) InstallPath { + path, err := validatePath(paths...) + if err != nil { + reportPathError(ctx, err) + } + return p.withRel(path) +} + +func (p InstallPath) withRel(rel string) InstallPath { + p.basePath = p.basePath.withRel(rel) + return p +} + +// ToMakePath returns a new InstallPath that points to Make's install directory instead of Soong's, +// i.e. out/ instead of out/soong/. +func (p InstallPath) ToMakePath() InstallPath { + p.baseDir = "../" + return p +} + // PathForModuleInstall returns a Path representing the install path for the // module appended with paths... -func PathForModuleInstall(ctx ModuleInstallPathContext, pathComponents ...string) OutputPath { +func PathForModuleInstall(ctx ModuleInstallPathContext, pathComponents ...string) InstallPath { var outPaths []string if ctx.Device() { partition := modulePartition(ctx) @@ -1104,10 +1237,38 @@ func PathForModuleInstall(ctx ModuleInstallPathContext, pathComponents ...string outPaths = append([]string{"debug"}, outPaths...) } outPaths = append(outPaths, pathComponents...) - return PathForOutput(ctx, outPaths...) + + path, err := validatePath(outPaths...) + if err != nil { + reportPathError(ctx, err) + } + + ret := InstallPath{basePath{path, ctx.Config(), ""}, ""} + if ctx.InstallBypassMake() && ctx.Config().EmbeddedInMake() { + ret = ret.ToMakePath() + } + + return ret +} + +func pathForNdkOrSdkInstall(ctx PathContext, prefix string, paths []string) InstallPath { + paths = append([]string{prefix}, paths...) + path, err := validatePath(paths...) + if err != nil { + reportPathError(ctx, err) + } + return InstallPath{basePath{path, ctx.Config(), ""}, ""} +} + +func PathForNdkInstall(ctx PathContext, paths ...string) InstallPath { + return pathForNdkOrSdkInstall(ctx, "ndk", paths) } -func InstallPathToOnDevicePath(ctx PathContext, path OutputPath) string { +func PathForMainlineSdksInstall(ctx PathContext, paths ...string) InstallPath { + return pathForNdkOrSdkInstall(ctx, "mainline-sdks", paths) +} + +func InstallPathToOnDevicePath(ctx PathContext, path InstallPath) string { rel := Rel(ctx, PathForOutput(ctx, "target", "product", ctx.Config().DeviceName()).String(), path.String()) return "/" + rel @@ -1117,17 +1278,34 @@ func modulePartition(ctx ModuleInstallPathContext) string { var partition string if ctx.InstallInData() { partition = "data" + } else if ctx.InstallInTestcases() { + partition = "testcases" + } else if ctx.InstallInRamdisk() { + if ctx.DeviceConfig().BoardUsesRecoveryAsBoot() { + partition = "recovery/root/first_stage_ramdisk" + } else { + partition = "ramdisk" + } + if !ctx.InstallInRoot() { + partition += "/system" + } } else if ctx.InstallInRecovery() { - // the layout of recovery partion is the same as that of system partition - partition = "recovery/root/system" + if ctx.InstallInRoot() { + partition = "recovery/root" + } else { + // the layout of recovery partion is the same as that of system partition + partition = "recovery/root/system" + } } else if ctx.SocSpecific() { partition = ctx.DeviceConfig().VendorPath() } else if ctx.DeviceSpecific() { partition = ctx.DeviceConfig().OdmPath() } else if ctx.ProductSpecific() { partition = ctx.DeviceConfig().ProductPath() - } else if ctx.ProductServicesSpecific() { - partition = ctx.DeviceConfig().ProductServicesPath() + } else if ctx.SystemExtSpecific() { + partition = ctx.DeviceConfig().SystemExtPath() + } else if ctx.InstallInRoot() { + partition = "root" } else { partition = "system" } @@ -1177,6 +1355,10 @@ type PhonyPath struct { func (p PhonyPath) writablePath() {} +func (p PhonyPath) buildDir() string { + return p.config.buildDir +} + var _ Path = PhonyPath{} var _ WritablePath = PhonyPath{} @@ -1188,12 +1370,6 @@ func (p testPath) String() string { return p.path } -type testWritablePath struct { - testPath -} - -func (p testPath) writablePath() {} - // PathForTesting returns a Path constructed from joining the elements of paths with '/'. It should only be used from // within tests. func PathForTesting(paths ...string) Path { @@ -1214,42 +1390,18 @@ func PathsForTesting(strs ...string) Paths { return p } -// WritablePathForTesting returns a Path constructed from joining the elements of paths with '/'. It should only be -// used from within tests. -func WritablePathForTesting(paths ...string) WritablePath { - p, err := validateSafePath(paths...) - if err != nil { - panic(err) - } - return testWritablePath{testPath{basePath{path: p, rel: p}}} -} - -// WritablePathsForTesting returns a Path constructed from each element in strs. It should only be used from within -// tests. -func WritablePathsForTesting(strs ...string) WritablePaths { - p := make(WritablePaths, len(strs)) - for i, s := range strs { - p[i] = WritablePathForTesting(s) - } - - return p -} - type testPathContext struct { config Config - fs pathtools.FileSystem } -func (x *testPathContext) Fs() pathtools.FileSystem { return x.fs } func (x *testPathContext) Config() Config { return x.config } func (x *testPathContext) AddNinjaFileDeps(...string) {} // PathContextForTesting returns a PathContext that can be used in tests, for example to create an OutputPath with // PathForOutput. -func PathContextForTesting(config Config, fs map[string][]byte) PathContext { +func PathContextForTesting(config Config) PathContext { return &testPathContext{ config: config, - fs: pathtools.MockFs(fs), } } @@ -1287,3 +1439,16 @@ func maybeRelErr(basePath string, targetPath string) (string, bool, error) { } return rel, true, nil } + +// Writes a file to the output directory. Attempting to write directly to the output directory +// will fail due to the sandbox of the soong_build process. +func WriteFileToOutputDir(path WritablePath, data []byte, perm os.FileMode) error { + return ioutil.WriteFile(absolutePath(path.String()), data, perm) +} + +func absolutePath(path string) string { + if filepath.IsAbs(path) { + return path + } + return filepath.Join(absSrcDir, path) +} diff --git a/android/paths_test.go b/android/paths_test.go index b52d71337..7a3202613 100644 --- a/android/paths_test.go +++ b/android/paths_test.go @@ -17,13 +17,10 @@ package android import ( "errors" "fmt" - "io/ioutil" - "os" "reflect" "strings" "testing" - "github.com/google/blueprint/pathtools" "github.com/google/blueprint/proptools" ) @@ -200,19 +197,18 @@ func p(in interface{}) string { } type moduleInstallPathContextImpl struct { - androidBaseContextImpl + baseModuleContext inData bool + inTestcases bool inSanitizerDir bool + inRamdisk bool inRecovery bool -} - -func (moduleInstallPathContextImpl) Fs() pathtools.FileSystem { - return pathtools.MockFs(nil) + inRoot bool } func (m moduleInstallPathContextImpl) Config() Config { - return m.androidBaseContextImpl.config + return m.baseModuleContext.config } func (moduleInstallPathContextImpl) AddNinjaFileDeps(deps ...string) {} @@ -221,16 +217,36 @@ func (m moduleInstallPathContextImpl) InstallInData() bool { return m.inData } +func (m moduleInstallPathContextImpl) InstallInTestcases() bool { + return m.inTestcases +} + func (m moduleInstallPathContextImpl) InstallInSanitizerDir() bool { return m.inSanitizerDir } +func (m moduleInstallPathContextImpl) InstallInRamdisk() bool { + return m.inRamdisk +} + func (m moduleInstallPathContextImpl) InstallInRecovery() bool { return m.inRecovery } +func (m moduleInstallPathContextImpl) InstallInRoot() bool { + return m.inRoot +} + +func (m moduleInstallPathContextImpl) InstallBypassMake() bool { + return false +} + +func pathTestConfig(buildDir string) Config { + return TestConfig(buildDir, nil, "", nil) +} + func TestPathForModuleInstall(t *testing.T) { - testConfig := TestConfig("", nil) + testConfig := pathTestConfig("") hostTarget := Target{Os: Linux} deviceTarget := Target{Os: Android} @@ -244,7 +260,8 @@ func TestPathForModuleInstall(t *testing.T) { { name: "host binary", ctx: &moduleInstallPathContextImpl{ - androidBaseContextImpl: androidBaseContextImpl{ + baseModuleContext: baseModuleContext{ + os: hostTarget.Os, target: hostTarget, }, }, @@ -255,7 +272,8 @@ func TestPathForModuleInstall(t *testing.T) { { name: "system binary", ctx: &moduleInstallPathContextImpl{ - androidBaseContextImpl: androidBaseContextImpl{ + baseModuleContext: baseModuleContext{ + os: deviceTarget.Os, target: deviceTarget, }, }, @@ -265,9 +283,12 @@ func TestPathForModuleInstall(t *testing.T) { { name: "vendor binary", ctx: &moduleInstallPathContextImpl{ - androidBaseContextImpl: androidBaseContextImpl{ + baseModuleContext: baseModuleContext{ + os: deviceTarget.Os, target: deviceTarget, - kind: socSpecificModule, + earlyModuleContext: earlyModuleContext{ + kind: socSpecificModule, + }, }, }, in: []string{"bin", "my_test"}, @@ -276,9 +297,12 @@ func TestPathForModuleInstall(t *testing.T) { { name: "odm binary", ctx: &moduleInstallPathContextImpl{ - androidBaseContextImpl: androidBaseContextImpl{ + baseModuleContext: baseModuleContext{ + os: deviceTarget.Os, target: deviceTarget, - kind: deviceSpecificModule, + earlyModuleContext: earlyModuleContext{ + kind: deviceSpecificModule, + }, }, }, in: []string{"bin", "my_test"}, @@ -287,30 +311,74 @@ func TestPathForModuleInstall(t *testing.T) { { name: "product binary", ctx: &moduleInstallPathContextImpl{ - androidBaseContextImpl: androidBaseContextImpl{ + baseModuleContext: baseModuleContext{ + os: deviceTarget.Os, target: deviceTarget, - kind: productSpecificModule, + earlyModuleContext: earlyModuleContext{ + kind: productSpecificModule, + }, }, }, in: []string{"bin", "my_test"}, out: "target/product/test_device/product/bin/my_test", }, { - name: "product_services binary", + name: "system_ext binary", ctx: &moduleInstallPathContextImpl{ - androidBaseContextImpl: androidBaseContextImpl{ + baseModuleContext: baseModuleContext{ + os: deviceTarget.Os, target: deviceTarget, - kind: productServicesSpecificModule, + earlyModuleContext: earlyModuleContext{ + kind: systemExtSpecificModule, + }, }, }, in: []string{"bin", "my_test"}, - out: "target/product/test_device/product_services/bin/my_test", + out: "target/product/test_device/system_ext/bin/my_test", + }, + { + name: "root binary", + ctx: &moduleInstallPathContextImpl{ + baseModuleContext: baseModuleContext{ + os: deviceTarget.Os, + target: deviceTarget, + }, + inRoot: true, + }, + in: []string{"my_test"}, + out: "target/product/test_device/root/my_test", + }, + { + name: "recovery binary", + ctx: &moduleInstallPathContextImpl{ + baseModuleContext: baseModuleContext{ + os: deviceTarget.Os, + target: deviceTarget, + }, + inRecovery: true, + }, + in: []string{"bin/my_test"}, + out: "target/product/test_device/recovery/root/system/bin/my_test", + }, + { + name: "recovery root binary", + ctx: &moduleInstallPathContextImpl{ + baseModuleContext: baseModuleContext{ + os: deviceTarget.Os, + target: deviceTarget, + }, + inRecovery: true, + inRoot: true, + }, + in: []string{"my_test"}, + out: "target/product/test_device/recovery/root/my_test", }, { name: "system native test binary", ctx: &moduleInstallPathContextImpl{ - androidBaseContextImpl: androidBaseContextImpl{ + baseModuleContext: baseModuleContext{ + os: deviceTarget.Os, target: deviceTarget, }, inData: true, @@ -321,9 +389,12 @@ func TestPathForModuleInstall(t *testing.T) { { name: "vendor native test binary", ctx: &moduleInstallPathContextImpl{ - androidBaseContextImpl: androidBaseContextImpl{ + baseModuleContext: baseModuleContext{ + os: deviceTarget.Os, target: deviceTarget, - kind: socSpecificModule, + earlyModuleContext: earlyModuleContext{ + kind: socSpecificModule, + }, }, inData: true, }, @@ -333,9 +404,12 @@ func TestPathForModuleInstall(t *testing.T) { { name: "odm native test binary", ctx: &moduleInstallPathContextImpl{ - androidBaseContextImpl: androidBaseContextImpl{ + baseModuleContext: baseModuleContext{ + os: deviceTarget.Os, target: deviceTarget, - kind: deviceSpecificModule, + earlyModuleContext: earlyModuleContext{ + kind: deviceSpecificModule, + }, }, inData: true, }, @@ -345,9 +419,12 @@ func TestPathForModuleInstall(t *testing.T) { { name: "product native test binary", ctx: &moduleInstallPathContextImpl{ - androidBaseContextImpl: androidBaseContextImpl{ + baseModuleContext: baseModuleContext{ + os: deviceTarget.Os, target: deviceTarget, - kind: productSpecificModule, + earlyModuleContext: earlyModuleContext{ + kind: productSpecificModule, + }, }, inData: true, }, @@ -356,11 +433,14 @@ func TestPathForModuleInstall(t *testing.T) { }, { - name: "product_services native test binary", + name: "system_ext native test binary", ctx: &moduleInstallPathContextImpl{ - androidBaseContextImpl: androidBaseContextImpl{ + baseModuleContext: baseModuleContext{ + os: deviceTarget.Os, target: deviceTarget, - kind: productServicesSpecificModule, + earlyModuleContext: earlyModuleContext{ + kind: systemExtSpecificModule, + }, }, inData: true, }, @@ -371,7 +451,8 @@ func TestPathForModuleInstall(t *testing.T) { { name: "sanitized system binary", ctx: &moduleInstallPathContextImpl{ - androidBaseContextImpl: androidBaseContextImpl{ + baseModuleContext: baseModuleContext{ + os: deviceTarget.Os, target: deviceTarget, }, inSanitizerDir: true, @@ -382,9 +463,12 @@ func TestPathForModuleInstall(t *testing.T) { { name: "sanitized vendor binary", ctx: &moduleInstallPathContextImpl{ - androidBaseContextImpl: androidBaseContextImpl{ + baseModuleContext: baseModuleContext{ + os: deviceTarget.Os, target: deviceTarget, - kind: socSpecificModule, + earlyModuleContext: earlyModuleContext{ + kind: socSpecificModule, + }, }, inSanitizerDir: true, }, @@ -394,9 +478,12 @@ func TestPathForModuleInstall(t *testing.T) { { name: "sanitized odm binary", ctx: &moduleInstallPathContextImpl{ - androidBaseContextImpl: androidBaseContextImpl{ + baseModuleContext: baseModuleContext{ + os: deviceTarget.Os, target: deviceTarget, - kind: deviceSpecificModule, + earlyModuleContext: earlyModuleContext{ + kind: deviceSpecificModule, + }, }, inSanitizerDir: true, }, @@ -406,9 +493,12 @@ func TestPathForModuleInstall(t *testing.T) { { name: "sanitized product binary", ctx: &moduleInstallPathContextImpl{ - androidBaseContextImpl: androidBaseContextImpl{ + baseModuleContext: baseModuleContext{ + os: deviceTarget.Os, target: deviceTarget, - kind: productSpecificModule, + earlyModuleContext: earlyModuleContext{ + kind: productSpecificModule, + }, }, inSanitizerDir: true, }, @@ -417,22 +507,26 @@ func TestPathForModuleInstall(t *testing.T) { }, { - name: "sanitized product_services binary", + name: "sanitized system_ext binary", ctx: &moduleInstallPathContextImpl{ - androidBaseContextImpl: androidBaseContextImpl{ + baseModuleContext: baseModuleContext{ + os: deviceTarget.Os, target: deviceTarget, - kind: productServicesSpecificModule, + earlyModuleContext: earlyModuleContext{ + kind: systemExtSpecificModule, + }, }, inSanitizerDir: true, }, in: []string{"bin", "my_test"}, - out: "target/product/test_device/data/asan/product_services/bin/my_test", + out: "target/product/test_device/data/asan/system_ext/bin/my_test", }, { name: "sanitized system native test binary", ctx: &moduleInstallPathContextImpl{ - androidBaseContextImpl: androidBaseContextImpl{ + baseModuleContext: baseModuleContext{ + os: deviceTarget.Os, target: deviceTarget, }, inData: true, @@ -444,9 +538,12 @@ func TestPathForModuleInstall(t *testing.T) { { name: "sanitized vendor native test binary", ctx: &moduleInstallPathContextImpl{ - androidBaseContextImpl: androidBaseContextImpl{ + baseModuleContext: baseModuleContext{ + os: deviceTarget.Os, target: deviceTarget, - kind: socSpecificModule, + earlyModuleContext: earlyModuleContext{ + kind: socSpecificModule, + }, }, inData: true, inSanitizerDir: true, @@ -457,9 +554,12 @@ func TestPathForModuleInstall(t *testing.T) { { name: "sanitized odm native test binary", ctx: &moduleInstallPathContextImpl{ - androidBaseContextImpl: androidBaseContextImpl{ + baseModuleContext: baseModuleContext{ + os: deviceTarget.Os, target: deviceTarget, - kind: deviceSpecificModule, + earlyModuleContext: earlyModuleContext{ + kind: deviceSpecificModule, + }, }, inData: true, inSanitizerDir: true, @@ -470,9 +570,12 @@ func TestPathForModuleInstall(t *testing.T) { { name: "sanitized product native test binary", ctx: &moduleInstallPathContextImpl{ - androidBaseContextImpl: androidBaseContextImpl{ + baseModuleContext: baseModuleContext{ + os: deviceTarget.Os, target: deviceTarget, - kind: productSpecificModule, + earlyModuleContext: earlyModuleContext{ + kind: productSpecificModule, + }, }, inData: true, inSanitizerDir: true, @@ -481,11 +584,14 @@ func TestPathForModuleInstall(t *testing.T) { out: "target/product/test_device/data/asan/data/nativetest/my_test", }, { - name: "sanitized product_services native test binary", + name: "sanitized system_ext native test binary", ctx: &moduleInstallPathContextImpl{ - androidBaseContextImpl: androidBaseContextImpl{ + baseModuleContext: baseModuleContext{ + os: deviceTarget.Os, target: deviceTarget, - kind: productServicesSpecificModule, + earlyModuleContext: earlyModuleContext{ + kind: systemExtSpecificModule, + }, }, inData: true, inSanitizerDir: true, @@ -497,7 +603,7 @@ func TestPathForModuleInstall(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - tc.ctx.androidBaseContextImpl.config = testConfig + tc.ctx.baseModuleContext.config = testConfig output := PathForModuleInstall(tc.ctx, tc.in...) if output.basePath.path != tc.out { t.Errorf("unexpected path:\n got: %q\nwant: %q\n", @@ -509,18 +615,19 @@ func TestPathForModuleInstall(t *testing.T) { } func TestDirectorySortedPaths(t *testing.T) { - config := TestConfig("out", nil) - - ctx := PathContextForTesting(config, map[string][]byte{ - "a.txt": nil, - "a/txt": nil, - "a/b/c": nil, - "a/b/d": nil, - "b": nil, - "b/b.txt": nil, - "a/a.txt": nil, + config := TestConfig("out", nil, "", map[string][]byte{ + "Android.bp": nil, + "a.txt": nil, + "a/txt": nil, + "a/b/c": nil, + "a/b/d": nil, + "b": nil, + "b/b.txt": nil, + "a/a.txt": nil, }) + ctx := PathContextForTesting(config) + makePaths := func() Paths { return Paths{ PathForSource(ctx, "a.txt"), @@ -684,7 +791,7 @@ func TestPathForSource(t *testing.T) { t.Run(f.name, func(t *testing.T) { for _, test := range testCases { t.Run(test.name, func(t *testing.T) { - testConfig := TestConfig(test.buildDir, nil) + testConfig := pathTestConfig(test.buildDir) ctx := &configErrorWrapper{config: testConfig} _, err := f.f(ctx, test.src) if len(ctx.errors) > 0 { @@ -758,6 +865,50 @@ func (p *pathForModuleSrcTestModule) GenerateAndroidBuildActions(ctx ModuleConte if !p.props.Module_handles_missing_deps { p.missingDeps = ctx.GetMissingDependencies() } + + ctx.Build(pctx, BuildParams{ + Rule: Touch, + Output: PathForModuleOut(ctx, "output"), + }) +} + +type pathForModuleSrcOutputFileProviderModule struct { + ModuleBase + props struct { + Outs []string + Tagged []string + } + + outs Paths + tagged Paths +} + +func pathForModuleSrcOutputFileProviderModuleFactory() Module { + module := &pathForModuleSrcOutputFileProviderModule{} + module.AddProperties(&module.props) + InitAndroidModule(module) + return module +} + +func (p *pathForModuleSrcOutputFileProviderModule) GenerateAndroidBuildActions(ctx ModuleContext) { + for _, out := range p.props.Outs { + p.outs = append(p.outs, PathForModuleOut(ctx, out)) + } + + for _, tagged := range p.props.Tagged { + p.tagged = append(p.tagged, PathForModuleOut(ctx, tagged)) + } +} + +func (p *pathForModuleSrcOutputFileProviderModule) OutputFiles(tag string) (Paths, error) { + switch tag { + case "": + return p.outs, nil + case ".tagged": + return p.tagged, nil + default: + return nil, fmt.Errorf("unsupported tag %q", tag) + } } type pathForModuleSrcTestCase struct { @@ -772,11 +923,11 @@ type pathForModuleSrcTestCase struct { func testPathForModuleSrc(t *testing.T, buildDir string, tests []pathForModuleSrcTestCase) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - config := TestConfig(buildDir, nil) ctx := NewTestContext() - ctx.RegisterModuleType("test", ModuleFactoryAdaptor(pathForModuleSrcTestModuleFactory)) - ctx.RegisterModuleType("filegroup", ModuleFactoryAdaptor(FileGroupFactory)) + ctx.RegisterModuleType("test", pathForModuleSrcTestModuleFactory) + ctx.RegisterModuleType("output_file_provider", pathForModuleSrcOutputFileProviderModuleFactory) + ctx.RegisterModuleType("filegroup", FileGroupFactory) fgBp := ` filegroup { @@ -785,9 +936,18 @@ func testPathForModuleSrc(t *testing.T, buildDir string, tests []pathForModuleSr } ` + ofpBp := ` + output_file_provider { + name: "b", + outs: ["gen/b"], + tagged: ["gen/c"], + } + ` + mockFS := map[string][]byte{ "fg/Android.bp": []byte(fgBp), "foo/Android.bp": []byte(test.bp), + "ofp/Android.bp": []byte(ofpBp), "fg/src/a": nil, "foo/src/b": nil, "foo/src/c": nil, @@ -796,10 +956,10 @@ func testPathForModuleSrc(t *testing.T, buildDir string, tests []pathForModuleSr "foo/src_special/$": nil, } - ctx.MockFileSystem(mockFS) + config := TestConfig(buildDir, nil, "", mockFS) - ctx.Register() - _, errs := ctx.ParseFileList(".", []string{"fg/Android.bp", "foo/Android.bp"}) + ctx.Register(config) + _, errs := ctx.ParseFileList(".", []string{"fg/Android.bp", "foo/Android.bp", "ofp/Android.bp"}) FailIfErrored(t, errs) _, errs = ctx.PrepareBuildActions(config) FailIfErrored(t, errs) @@ -871,6 +1031,26 @@ func TestPathsForModuleSrc(t *testing.T) { rels: []string{"src/a"}, }, { + name: "output file provider", + bp: ` + test { + name: "foo", + srcs: [":b"], + }`, + srcs: []string{buildDir + "/.intermediates/ofp/b/gen/b"}, + rels: []string{"gen/b"}, + }, + { + name: "output file provider tagged", + bp: ` + test { + name: "foo", + srcs: [":b{.tagged}"], + }`, + srcs: []string{buildDir + "/.intermediates/ofp/b/gen/c"}, + rels: []string{"gen/c"}, + }, + { name: "special characters glob", bp: ` test { @@ -882,12 +1062,6 @@ func TestPathsForModuleSrc(t *testing.T) { }, } - buildDir, err := ioutil.TempDir("", "soong_paths_for_module_src_test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(buildDir) - testPathForModuleSrc(t, buildDir, tests) } @@ -924,6 +1098,26 @@ func TestPathForModuleSrc(t *testing.T) { rel: "src/a", }, { + name: "output file provider", + bp: ` + test { + name: "foo", + src: ":b", + }`, + src: buildDir + "/.intermediates/ofp/b/gen/b", + rel: "gen/b", + }, + { + name: "output file provider tagged", + bp: ` + test { + name: "foo", + src: ":b{.tagged}", + }`, + src: buildDir + "/.intermediates/ofp/b/gen/c", + rel: "gen/c", + }, + { name: "special characters glob", bp: ` test { @@ -935,30 +1129,10 @@ func TestPathForModuleSrc(t *testing.T) { }, } - buildDir, err := ioutil.TempDir("", "soong_path_for_module_src_test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(buildDir) - testPathForModuleSrc(t, buildDir, tests) } func TestPathsForModuleSrc_AllowMissingDependencies(t *testing.T) { - buildDir, err := ioutil.TempDir("", "soong_paths_for_module_src_allow_missing_dependencies_test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(buildDir) - - config := TestConfig(buildDir, nil) - config.TestProductVariables.Allow_missing_dependencies = proptools.BoolPtr(true) - - ctx := NewTestContext() - ctx.SetAllowMissingDependencies(true) - - ctx.RegisterModuleType("test", ModuleFactoryAdaptor(pathForModuleSrcTestModuleFactory)) - bp := ` test { name: "foo", @@ -975,13 +1149,16 @@ func TestPathsForModuleSrc_AllowMissingDependencies(t *testing.T) { } ` - mockFS := map[string][]byte{ - "Android.bp": []byte(bp), - } + config := TestConfig(buildDir, nil, bp, nil) + config.TestProductVariables.Allow_missing_dependencies = proptools.BoolPtr(true) + + ctx := NewTestContext() + ctx.SetAllowMissingDependencies(true) + + ctx.RegisterModuleType("test", pathForModuleSrcTestModuleFactory) - ctx.MockFileSystem(mockFS) + ctx.Register(config) - ctx.Register() _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) FailIfErrored(t, errs) _, errs = ctx.PrepareBuildActions(config) @@ -1014,7 +1191,7 @@ func TestPathsForModuleSrc_AllowMissingDependencies(t *testing.T) { func ExampleOutputPath_ReplaceExtension() { ctx := &configErrorWrapper{ - config: TestConfig("out", nil), + config: TestConfig("out", nil, "", nil), } p := PathForOutput(ctx, "system/framework").Join(ctx, "boot.art") p2 := p.ReplaceExtension(ctx, "oat") @@ -1028,7 +1205,7 @@ func ExampleOutputPath_ReplaceExtension() { func ExampleOutputPath_FileInSameDir() { ctx := &configErrorWrapper{ - config: TestConfig("out", nil), + config: TestConfig("out", nil, "", nil), } p := PathForOutput(ctx, "system/framework").Join(ctx, "boot.art") p2 := p.InSameDir(ctx, "oat", "arm", "boot.vdex") diff --git a/android/phony.go b/android/phony.go new file mode 100644 index 000000000..f8e5a4401 --- /dev/null +++ b/android/phony.go @@ -0,0 +1,75 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "sync" + + "github.com/google/blueprint" +) + +var phonyMapOnceKey = NewOnceKey("phony") + +type phonyMap map[string]Paths + +var phonyMapLock sync.Mutex + +func getPhonyMap(config Config) phonyMap { + return config.Once(phonyMapOnceKey, func() interface{} { + return make(phonyMap) + }).(phonyMap) +} + +func addPhony(config Config, name string, deps ...Path) { + phonyMap := getPhonyMap(config) + phonyMapLock.Lock() + defer phonyMapLock.Unlock() + phonyMap[name] = append(phonyMap[name], deps...) +} + +type phonySingleton struct { + phonyMap phonyMap + phonyList []string +} + +var _ SingletonMakeVarsProvider = (*phonySingleton)(nil) + +func (p *phonySingleton) GenerateBuildActions(ctx SingletonContext) { + p.phonyMap = getPhonyMap(ctx.Config()) + p.phonyList = SortedStringKeys(p.phonyMap) + for _, phony := range p.phonyList { + p.phonyMap[phony] = SortedUniquePaths(p.phonyMap[phony]) + } + + if !ctx.Config().EmbeddedInMake() { + for _, phony := range p.phonyList { + ctx.Build(pctx, BuildParams{ + Rule: blueprint.Phony, + Outputs: []WritablePath{PathForPhony(ctx, phony)}, + Implicits: p.phonyMap[phony], + }) + } + } +} + +func (p phonySingleton) MakeVars(ctx MakeVarsContext) { + for _, phony := range p.phonyList { + ctx.Phony(phony, p.phonyMap[phony]...) + } +} + +func phonySingletonFactory() Singleton { + return &phonySingleton{} +} diff --git a/android/prebuilt.go b/android/prebuilt.go index 3be10f723..ee4a13af2 100644 --- a/android/prebuilt.go +++ b/android/prebuilt.go @@ -16,6 +16,7 @@ package android import ( "fmt" + "reflect" "github.com/google/blueprint" "github.com/google/blueprint/proptools" @@ -24,11 +25,25 @@ import ( // This file implements common functionality for handling modules that may exist as prebuilts, // source, or both. +func RegisterPrebuiltMutators(ctx RegistrationContext) { + ctx.PreArchMutators(RegisterPrebuiltsPreArchMutators) + ctx.PostDepsMutators(RegisterPrebuiltsPostDepsMutators) +} + type prebuiltDependencyTag struct { blueprint.BaseDependencyTag } -var prebuiltDepTag prebuiltDependencyTag +var PrebuiltDepTag prebuiltDependencyTag + +// Mark this tag so dependencies that use it are excluded from visibility enforcement. +func (t prebuiltDependencyTag) ExcludeFromVisibilityEnforcement() {} + +// Mark this tag so dependencies that use it are excluded from APEX contents. +func (t prebuiltDependencyTag) ExcludeFromApexContents() {} + +var _ ExcludeFromVisibilityEnforcementTag = PrebuiltDepTag +var _ ExcludeFromApexContentsTag = PrebuiltDepTag type PrebuiltProperties struct { // When prefer is set to true the prebuilt will be used instead of any source module with @@ -41,36 +56,49 @@ type PrebuiltProperties struct { type Prebuilt struct { properties PrebuiltProperties - module Module - srcs *[]string - src *string + + srcsSupplier PrebuiltSrcsSupplier + srcsPropertyName string } func (p *Prebuilt) Name(name string) string { return "prebuilt_" + name } +func (p *Prebuilt) ForcePrefer() { + p.properties.Prefer = proptools.BoolPtr(true) +} + +func (p *Prebuilt) Prefer() bool { + return proptools.Bool(p.properties.Prefer) +} + +// The below source-related functions and the srcs, src fields are based on an assumption that +// prebuilt modules have a static source property at the moment. Currently there is only one +// exception, android_app_import, which chooses a source file depending on the product's DPI +// preference configs. We'll want to add native support for dynamic source cases if we end up having +// more modules like this. func (p *Prebuilt) SingleSourcePath(ctx ModuleContext) Path { - if p.srcs != nil { - if len(*p.srcs) == 0 { - ctx.PropertyErrorf("srcs", "missing prebuilt source file") + if p.srcsSupplier != nil { + srcs := p.srcsSupplier() + + if len(srcs) == 0 { + ctx.PropertyErrorf(p.srcsPropertyName, "missing prebuilt source file") return nil } - if len(*p.srcs) > 1 { - ctx.PropertyErrorf("srcs", "multiple prebuilt source files") + if len(srcs) > 1 { + ctx.PropertyErrorf(p.srcsPropertyName, "multiple prebuilt source files") return nil } // Return the singleton source after expanding any filegroup in the // sources. - return PathForModuleSrc(ctx, (*p.srcs)[0]) + src := srcs[0] + return PathForModuleSrc(ctx, src) } else { - if proptools.String(p.src) == "" { - ctx.PropertyErrorf("src", "missing prebuilt source file") - return nil - } - return PathForModuleSrc(ctx, *p.src) + ctx.ModuleErrorf("prebuilt source was not set") + return nil } } @@ -78,16 +106,80 @@ func (p *Prebuilt) UsePrebuilt() bool { return p.properties.UsePrebuilt } -func InitPrebuiltModule(module PrebuiltInterface, srcs *[]string) { +// Called to provide the srcs value for the prebuilt module. +// +// Return the src value or nil if it is not available. +type PrebuiltSrcsSupplier func() []string + +// Initialize the module as a prebuilt module that uses the provided supplier to access the +// prebuilt sources of the module. +// +// The supplier will be called multiple times and must return the same values each time it +// is called. If it returns an empty array (or nil) then the prebuilt module will not be used +// as a replacement for a source module with the same name even if prefer = true. +// +// If the Prebuilt.SingleSourcePath() is called on the module then this must return an array +// containing exactly one source file. +// +// The provided property name is used to provide helpful error messages in the event that +// a problem arises, e.g. calling SingleSourcePath() when more than one source is provided. +func InitPrebuiltModuleWithSrcSupplier(module PrebuiltInterface, srcsSupplier PrebuiltSrcsSupplier, srcsPropertyName string) { p := module.Prebuilt() module.AddProperties(&p.properties) - p.srcs = srcs + + if srcsSupplier == nil { + panic(fmt.Errorf("srcsSupplier must not be nil")) + } + if srcsPropertyName == "" { + panic(fmt.Errorf("srcsPropertyName must not be empty")) + } + + p.srcsSupplier = srcsSupplier + p.srcsPropertyName = srcsPropertyName } -func InitSingleSourcePrebuiltModule(module PrebuiltInterface, src *string) { - p := module.Prebuilt() - module.AddProperties(&p.properties) - p.src = src +func InitPrebuiltModule(module PrebuiltInterface, srcs *[]string) { + if srcs == nil { + panic(fmt.Errorf("srcs must not be nil")) + } + + srcsSupplier := func() []string { + return *srcs + } + + InitPrebuiltModuleWithSrcSupplier(module, srcsSupplier, "srcs") +} + +func InitSingleSourcePrebuiltModule(module PrebuiltInterface, srcProps interface{}, srcField string) { + srcPropsValue := reflect.ValueOf(srcProps).Elem() + srcStructField, _ := srcPropsValue.Type().FieldByName(srcField) + if !srcPropsValue.IsValid() || srcStructField.Name == "" { + panic(fmt.Errorf("invalid single source prebuilt %+v", module)) + } + + if srcPropsValue.Kind() != reflect.Struct && srcPropsValue.Kind() != reflect.Interface { + panic(fmt.Errorf("invalid single source prebuilt %+v", srcProps)) + } + + srcFieldIndex := srcStructField.Index + srcPropertyName := proptools.PropertyNameForField(srcField) + + srcsSupplier := func() []string { + value := srcPropsValue.FieldByIndex(srcFieldIndex) + if value.Kind() == reflect.Ptr { + value = value.Elem() + } + if value.Kind() != reflect.String { + panic(fmt.Errorf("prebuilt src field %q should be a string or a pointer to one but was %d %q", srcPropertyName, value.Kind(), value)) + } + src := value.String() + if src == "" { + return nil + } + return []string{src} + } + + InitPrebuiltModuleWithSrcSupplier(module, srcsSupplier, srcPropertyName) } type PrebuiltInterface interface { @@ -111,7 +203,7 @@ func PrebuiltMutator(ctx BottomUpMutatorContext) { p := m.Prebuilt() name := m.base().BaseModuleName() if ctx.OtherModuleExists(name) { - ctx.AddReverseDependency(ctx.Module(), prebuiltDepTag, name) + ctx.AddReverseDependency(ctx.Module(), PrebuiltDepTag, name) p.properties.SourceExists = true } else { ctx.Rename(name) @@ -124,14 +216,14 @@ func PrebuiltMutator(ctx BottomUpMutatorContext) { func PrebuiltSelectModuleMutator(ctx TopDownMutatorContext) { if m, ok := ctx.Module().(PrebuiltInterface); ok && m.Prebuilt() != nil { p := m.Prebuilt() - if p.srcs == nil && p.src == nil { + if p.srcsSupplier == nil { panic(fmt.Errorf("prebuilt module did not have InitPrebuiltModule called on it")) } if !p.properties.SourceExists { p.properties.UsePrebuilt = p.usePrebuilt(ctx, nil) } } else if s, ok := ctx.Module().(Module); ok { - ctx.VisitDirectDepsWithTag(prebuiltDepTag, func(m Module) { + ctx.VisitDirectDepsWithTag(PrebuiltDepTag, func(m Module) { p := m.(PrebuiltInterface).Prebuilt() if p.usePrebuilt(ctx, s) { p.properties.UsePrebuilt = true @@ -163,11 +255,7 @@ func PrebuiltPostDepsMutator(ctx BottomUpMutatorContext) { // usePrebuilt returns true if a prebuilt should be used instead of the source module. The prebuilt // will be used if it is marked "prefer" or if the source module is disabled. func (p *Prebuilt) usePrebuilt(ctx TopDownMutatorContext, source Module) bool { - if p.srcs != nil && len(*p.srcs) == 0 { - return false - } - - if p.src != nil && *p.src == "" { + if p.srcsSupplier != nil && len(p.srcsSupplier()) == 0 { return false } diff --git a/android/prebuilt_etc.go b/android/prebuilt_etc.go deleted file mode 100644 index 8f23d78e6..000000000 --- a/android/prebuilt_etc.go +++ /dev/null @@ -1,259 +0,0 @@ -// Copyright 2016 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package android - -import ( - "fmt" - "io" - "strings" -) - -// TODO(jungw): Now that it handles more than the ones in etc/, consider renaming this file. - -func init() { - RegisterModuleType("prebuilt_etc", PrebuiltEtcFactory) - RegisterModuleType("prebuilt_etc_host", PrebuiltEtcHostFactory) - RegisterModuleType("prebuilt_usr_share", PrebuiltUserShareFactory) - RegisterModuleType("prebuilt_usr_share_host", PrebuiltUserShareHostFactory) - - PreDepsMutators(func(ctx RegisterMutatorsContext) { - ctx.BottomUp("prebuilt_etc", prebuiltEtcMutator).Parallel() - }) -} - -type prebuiltEtcProperties struct { - // Source file of this prebuilt. - Src *string `android:"path,arch_variant"` - - // optional subdirectory under which this file is installed into - Sub_dir *string `android:"arch_variant"` - - // optional name for the installed file. If unspecified, name of the module is used as the file name - Filename *string `android:"arch_variant"` - - // when set to true, and filename property is not set, the name for the installed file - // is the same as the file name of the source file. - Filename_from_src *bool `android:"arch_variant"` - - // Make this module available when building for recovery. - Recovery_available *bool - - InRecovery bool `blueprint:"mutated"` - - // Whether this module is directly installable to one of the partitions. Default: true. - Installable *bool -} - -type PrebuiltEtc struct { - ModuleBase - - properties prebuiltEtcProperties - - sourceFilePath Path - outputFilePath OutputPath - // The base install location, e.g. "etc" for prebuilt_etc, "usr/share" for prebuilt_usr_share. - installDirBase string - installDirPath OutputPath - additionalDependencies *Paths -} - -func (p *PrebuiltEtc) inRecovery() bool { - return p.properties.InRecovery || p.ModuleBase.InstallInRecovery() -} - -func (p *PrebuiltEtc) onlyInRecovery() bool { - return p.ModuleBase.InstallInRecovery() -} - -func (p *PrebuiltEtc) InstallInRecovery() bool { - return p.inRecovery() -} - -func (p *PrebuiltEtc) DepsMutator(ctx BottomUpMutatorContext) { - if p.properties.Src == nil { - ctx.PropertyErrorf("src", "missing prebuilt source file") - } -} - -func (p *PrebuiltEtc) SourceFilePath(ctx ModuleContext) Path { - return PathForModuleSrc(ctx, String(p.properties.Src)) -} - -// This allows other derivative modules (e.g. prebuilt_etc_xml) to perform -// additional steps (like validating the src) before the file is installed. -func (p *PrebuiltEtc) SetAdditionalDependencies(paths Paths) { - p.additionalDependencies = &paths -} - -func (p *PrebuiltEtc) OutputFile() OutputPath { - return p.outputFilePath -} - -func (p *PrebuiltEtc) SubDir() string { - return String(p.properties.Sub_dir) -} - -func (p *PrebuiltEtc) Installable() bool { - return p.properties.Installable == nil || Bool(p.properties.Installable) -} - -func (p *PrebuiltEtc) GenerateAndroidBuildActions(ctx ModuleContext) { - p.sourceFilePath = PathForModuleSrc(ctx, String(p.properties.Src)) - filename := String(p.properties.Filename) - filename_from_src := Bool(p.properties.Filename_from_src) - if filename == "" { - if filename_from_src { - filename = p.sourceFilePath.Base() - } else { - filename = ctx.ModuleName() - } - } else if filename_from_src { - ctx.PropertyErrorf("filename_from_src", "filename is set. filename_from_src can't be true") - return - } - p.outputFilePath = PathForModuleOut(ctx, filename).OutputPath - p.installDirPath = PathForModuleInstall(ctx, p.installDirBase, String(p.properties.Sub_dir)) - - // This ensures that outputFilePath has the correct name for others to - // use, as the source file may have a different name. - ctx.Build(pctx, BuildParams{ - Rule: Cp, - Output: p.outputFilePath, - Input: p.sourceFilePath, - }) -} - -func (p *PrebuiltEtc) AndroidMk() AndroidMkData { - return AndroidMkData{ - Custom: func(w io.Writer, name, prefix, moduleDir string, data AndroidMkData) { - nameSuffix := "" - if p.inRecovery() && !p.onlyInRecovery() { - nameSuffix = ".recovery" - } - fmt.Fprintln(w, "\ninclude $(CLEAR_VARS)") - fmt.Fprintln(w, "LOCAL_PATH :=", moduleDir) - fmt.Fprintln(w, "LOCAL_MODULE :=", name+nameSuffix) - fmt.Fprintln(w, "LOCAL_MODULE_CLASS := ETC") - if p.commonProperties.Owner != nil { - fmt.Fprintln(w, "LOCAL_MODULE_OWNER :=", *p.commonProperties.Owner) - } - fmt.Fprintln(w, "LOCAL_MODULE_TAGS := optional") - if p.Host() { - fmt.Fprintln(w, "LOCAL_IS_HOST_MODULE := true") - } - fmt.Fprintln(w, "LOCAL_PREBUILT_MODULE_FILE :=", p.outputFilePath.String()) - fmt.Fprintln(w, "LOCAL_MODULE_PATH :=", "$(OUT_DIR)/"+p.installDirPath.RelPathString()) - fmt.Fprintln(w, "LOCAL_INSTALLED_MODULE_STEM :=", p.outputFilePath.Base()) - fmt.Fprintln(w, "LOCAL_UNINSTALLABLE_MODULE :=", !p.Installable()) - fmt.Fprintln(w, "LOCAL_REQUIRED_MODULES :=", strings.Join(data.Required, " ")) - fmt.Fprintln(w, "LOCAL_MODULE_TARGET_ARCH :=", p.Arch().ArchType.String()) - if p.additionalDependencies != nil { - fmt.Fprint(w, "LOCAL_ADDITIONAL_DEPENDENCIES :=") - for _, path := range *p.additionalDependencies { - fmt.Fprint(w, " "+path.String()) - } - fmt.Fprintln(w, "") - } - fmt.Fprintln(w, "include $(BUILD_PREBUILT)") - }, - } -} - -func InitPrebuiltEtcModule(p *PrebuiltEtc) { - p.AddProperties(&p.properties) -} - -// prebuilt_etc is for a prebuilt artifact that is installed in -// <partition>/etc/<sub_dir> directory. -func PrebuiltEtcFactory() Module { - module := &PrebuiltEtc{installDirBase: "etc"} - InitPrebuiltEtcModule(module) - // This module is device-only - InitAndroidArchModule(module, DeviceSupported, MultilibFirst) - return module -} - -// prebuilt_etc_host is for a host prebuilt artifact that is installed in -// $(HOST_OUT)/etc/<sub_dir> directory. -func PrebuiltEtcHostFactory() Module { - module := &PrebuiltEtc{installDirBase: "etc"} - InitPrebuiltEtcModule(module) - // This module is host-only - InitAndroidArchModule(module, HostSupported, MultilibCommon) - return module -} - -// prebuilt_usr_share is for a prebuilt artifact that is installed in -// <partition>/usr/share/<sub_dir> directory. -func PrebuiltUserShareFactory() Module { - module := &PrebuiltEtc{installDirBase: "usr/share"} - InitPrebuiltEtcModule(module) - // This module is device-only - InitAndroidArchModule(module, DeviceSupported, MultilibFirst) - return module -} - -// prebuild_usr_share_host is for a host prebuilt artifact that is installed in -// $(HOST_OUT)/usr/share/<sub_dir> directory. -func PrebuiltUserShareHostFactory() Module { - module := &PrebuiltEtc{installDirBase: "usr/share"} - InitPrebuiltEtcModule(module) - // This module is host-only - InitAndroidArchModule(module, HostSupported, MultilibCommon) - return module -} - -const ( - // coreMode is the variant for modules to be installed to system. - coreMode = "core" - - // recoveryMode means a module to be installed to recovery image. - recoveryMode = "recovery" -) - -// prebuiltEtcMutator creates the needed variants to install the module to -// system or recovery. -func prebuiltEtcMutator(mctx BottomUpMutatorContext) { - m, ok := mctx.Module().(*PrebuiltEtc) - if !ok || m.Host() { - return - } - - var coreVariantNeeded bool = true - var recoveryVariantNeeded bool = false - if Bool(m.properties.Recovery_available) { - recoveryVariantNeeded = true - } - - if m.ModuleBase.InstallInRecovery() { - recoveryVariantNeeded = true - coreVariantNeeded = false - } - - var variants []string - if coreVariantNeeded { - variants = append(variants, coreMode) - } - if recoveryVariantNeeded { - variants = append(variants, recoveryMode) - } - mod := mctx.CreateVariations(variants...) - for i, v := range variants { - if v == recoveryMode { - m := mod[i].(*PrebuiltEtc) - m.properties.InRecovery = true - } - } -} diff --git a/android/prebuilt_etc_test.go b/android/prebuilt_etc_test.go deleted file mode 100644 index e0ade7e50..000000000 --- a/android/prebuilt_etc_test.go +++ /dev/null @@ -1,231 +0,0 @@ -// Copyright 2018 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package android - -import ( - "bufio" - "bytes" - "io/ioutil" - "os" - "path/filepath" - "strings" - "testing" -) - -func testPrebuiltEtc(t *testing.T, bp string) (*TestContext, Config) { - config, buildDir := setUp(t) - defer tearDown(buildDir) - ctx := NewTestArchContext() - ctx.RegisterModuleType("prebuilt_etc", ModuleFactoryAdaptor(PrebuiltEtcFactory)) - ctx.RegisterModuleType("prebuilt_etc_host", ModuleFactoryAdaptor(PrebuiltEtcHostFactory)) - ctx.RegisterModuleType("prebuilt_usr_share", ModuleFactoryAdaptor(PrebuiltUserShareFactory)) - ctx.RegisterModuleType("prebuilt_usr_share_host", ModuleFactoryAdaptor(PrebuiltUserShareHostFactory)) - ctx.PreDepsMutators(func(ctx RegisterMutatorsContext) { - ctx.BottomUp("prebuilt_etc", prebuiltEtcMutator).Parallel() - }) - ctx.Register() - mockFiles := map[string][]byte{ - "Android.bp": []byte(bp), - "foo.conf": nil, - "bar.conf": nil, - "baz.conf": nil, - } - ctx.MockFileSystem(mockFiles) - _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) - FailIfErrored(t, errs) - _, errs = ctx.PrepareBuildActions(config) - FailIfErrored(t, errs) - - return ctx, config -} - -func setUp(t *testing.T) (config Config, buildDir string) { - buildDir, err := ioutil.TempDir("", "soong_prebuilt_etc_test") - if err != nil { - t.Fatal(err) - } - - config = TestArchConfig(buildDir, nil) - return -} - -func tearDown(buildDir string) { - os.RemoveAll(buildDir) -} - -func TestPrebuiltEtcVariants(t *testing.T) { - ctx, _ := testPrebuiltEtc(t, ` - prebuilt_etc { - name: "foo.conf", - src: "foo.conf", - } - prebuilt_etc { - name: "bar.conf", - src: "bar.conf", - recovery_available: true, - } - prebuilt_etc { - name: "baz.conf", - src: "baz.conf", - recovery: true, - } - `) - - foo_variants := ctx.ModuleVariantsForTests("foo.conf") - if len(foo_variants) != 1 { - t.Errorf("expected 1, got %#v", foo_variants) - } - - bar_variants := ctx.ModuleVariantsForTests("bar.conf") - if len(bar_variants) != 2 { - t.Errorf("expected 2, got %#v", bar_variants) - } - - baz_variants := ctx.ModuleVariantsForTests("baz.conf") - if len(baz_variants) != 1 { - t.Errorf("expected 1, got %#v", bar_variants) - } -} - -func TestPrebuiltEtcOutputPath(t *testing.T) { - ctx, _ := testPrebuiltEtc(t, ` - prebuilt_etc { - name: "foo.conf", - src: "foo.conf", - filename: "foo.installed.conf", - } - `) - - p := ctx.ModuleForTests("foo.conf", "android_arm64_armv8-a_core").Module().(*PrebuiltEtc) - if p.outputFilePath.Base() != "foo.installed.conf" { - t.Errorf("expected foo.installed.conf, got %q", p.outputFilePath.Base()) - } -} - -func TestPrebuiltEtcGlob(t *testing.T) { - ctx, _ := testPrebuiltEtc(t, ` - prebuilt_etc { - name: "my_foo", - src: "foo.*", - } - prebuilt_etc { - name: "my_bar", - src: "bar.*", - filename_from_src: true, - } - `) - - p := ctx.ModuleForTests("my_foo", "android_arm64_armv8-a_core").Module().(*PrebuiltEtc) - if p.outputFilePath.Base() != "my_foo" { - t.Errorf("expected my_foo, got %q", p.outputFilePath.Base()) - } - - p = ctx.ModuleForTests("my_bar", "android_arm64_armv8-a_core").Module().(*PrebuiltEtc) - if p.outputFilePath.Base() != "bar.conf" { - t.Errorf("expected bar.conf, got %q", p.outputFilePath.Base()) - } -} - -func TestPrebuiltEtcAndroidMk(t *testing.T) { - ctx, _ := testPrebuiltEtc(t, ` - prebuilt_etc { - name: "foo", - src: "foo.conf", - owner: "abc", - filename_from_src: true, - } - `) - - data := AndroidMkData{} - data.Required = append(data.Required, "modA", "moduleB") - - expected := map[string]string{ - "LOCAL_MODULE": "foo", - "LOCAL_MODULE_CLASS": "ETC", - "LOCAL_MODULE_OWNER": "abc", - "LOCAL_INSTALLED_MODULE_STEM": "foo.conf", - "LOCAL_REQUIRED_MODULES": "modA moduleB", - } - - mod := ctx.ModuleForTests("foo", "android_arm64_armv8-a_core").Module().(*PrebuiltEtc) - buf := &bytes.Buffer{} - mod.AndroidMk().Custom(buf, "foo", "", "", data) - for k, expected := range expected { - found := false - scanner := bufio.NewScanner(bytes.NewReader(buf.Bytes())) - for scanner.Scan() { - line := scanner.Text() - tok := strings.Split(line, " := ") - if tok[0] == k { - found = true - if tok[1] != expected { - t.Errorf("Incorrect %s '%s', expected '%s'", k, tok[1], expected) - } - } - } - - if !found { - t.Errorf("No %s defined, saw %s", k, buf.String()) - } - } -} - -func TestPrebuiltEtcHost(t *testing.T) { - ctx, _ := testPrebuiltEtc(t, ` - prebuilt_etc_host { - name: "foo.conf", - src: "foo.conf", - } - `) - - buildOS := BuildOs.String() - p := ctx.ModuleForTests("foo.conf", buildOS+"_common").Module().(*PrebuiltEtc) - if !p.Host() { - t.Errorf("host bit is not set for a prebuilt_etc_host module.") - } -} - -func TestPrebuiltUserShareInstallDirPath(t *testing.T) { - ctx, _ := testPrebuiltEtc(t, ` - prebuilt_usr_share { - name: "foo.conf", - src: "foo.conf", - sub_dir: "bar", - } - `) - - p := ctx.ModuleForTests("foo.conf", "android_arm64_armv8-a_core").Module().(*PrebuiltEtc) - expected := "target/product/test_device/system/usr/share/bar" - if p.installDirPath.RelPathString() != expected { - t.Errorf("expected %q, got %q", expected, p.installDirPath.RelPathString()) - } -} - -func TestPrebuiltUserShareHostInstallDirPath(t *testing.T) { - ctx, config := testPrebuiltEtc(t, ` - prebuilt_usr_share_host { - name: "foo.conf", - src: "foo.conf", - sub_dir: "bar", - } - `) - - buildOS := BuildOs.String() - p := ctx.ModuleForTests("foo.conf", buildOS+"_common").Module().(*PrebuiltEtc) - expected := filepath.Join("host", config.PrebuiltOS(), "usr", "share", "bar") - if p.installDirPath.RelPathString() != expected { - t.Errorf("expected %q, got %q", expected, p.installDirPath.RelPathString()) - } -} diff --git a/android/prebuilt_test.go b/android/prebuilt_test.go index e1826414d..e8c5121f0 100644 --- a/android/prebuilt_test.go +++ b/android/prebuilt_test.go @@ -15,8 +15,7 @@ package android import ( - "io/ioutil" - "os" + "fmt" "testing" "github.com/google/blueprint" @@ -61,7 +60,7 @@ var prebuiltsTests = []struct { source { name: "bar", } - + prebuilt { name: "bar", prefer: false, @@ -75,7 +74,7 @@ var prebuiltsTests = []struct { source { name: "bar", } - + prebuilt { name: "bar", prefer: true, @@ -89,7 +88,7 @@ var prebuiltsTests = []struct { source { name: "bar", } - + prebuilt { name: "bar", prefer: false, @@ -102,7 +101,7 @@ var prebuiltsTests = []struct { source { name: "bar", } - + prebuilt { name: "bar", prefer: true, @@ -123,38 +122,68 @@ var prebuiltsTests = []struct { }`, prebuilt: true, }, + { + name: "prebuilt override not preferred", + modules: ` + source { + name: "baz", + } + + override_source { + name: "bar", + base: "baz", + } + + prebuilt { + name: "bar", + prefer: false, + srcs: ["prebuilt_file"], + }`, + prebuilt: false, + }, + { + name: "prebuilt override preferred", + modules: ` + source { + name: "baz", + } + + override_source { + name: "bar", + base: "baz", + } + + prebuilt { + name: "bar", + prefer: true, + srcs: ["prebuilt_file"], + }`, + prebuilt: true, + }, } func TestPrebuilts(t *testing.T) { - buildDir, err := ioutil.TempDir("", "soong_prebuilt_test") - if err != nil { - t.Fatal(err) + fs := map[string][]byte{ + "prebuilt_file": nil, + "source_file": nil, } - defer os.RemoveAll(buildDir) - - config := TestConfig(buildDir, nil) for _, test := range prebuiltsTests { t.Run(test.name, func(t *testing.T) { + bp := ` + source { + name: "foo", + deps: [":bar"], + } + ` + test.modules + config := TestConfig(buildDir, nil, bp, fs) + ctx := NewTestContext() - ctx.PreArchMutators(RegisterPrebuiltsPreArchMutators) - ctx.PostDepsMutators(RegisterPrebuiltsPostDepsMutators) - ctx.RegisterModuleType("filegroup", ModuleFactoryAdaptor(FileGroupFactory)) - ctx.RegisterModuleType("prebuilt", ModuleFactoryAdaptor(newPrebuiltModule)) - ctx.RegisterModuleType("source", ModuleFactoryAdaptor(newSourceModule)) - ctx.Register() - ctx.MockFileSystem(map[string][]byte{ - "prebuilt_file": nil, - "source_file": nil, - "Blueprints": []byte(` - source { - name: "foo", - deps: [":bar"], - } - ` + test.modules), - }) + registerTestPrebuiltBuildComponents(ctx) + ctx.RegisterModuleType("filegroup", FileGroupFactory) + ctx.Register(config) - _, errs := ctx.ParseBlueprintsFiles("Blueprints") + _, errs := ctx.ParseBlueprintsFiles("Android.bp") FailIfErrored(t, errs) _, errs = ctx.PrepareBuildActions(config) FailIfErrored(t, errs) @@ -219,6 +248,15 @@ func TestPrebuilts(t *testing.T) { } } +func registerTestPrebuiltBuildComponents(ctx RegistrationContext) { + ctx.RegisterModuleType("prebuilt", newPrebuiltModule) + ctx.RegisterModuleType("source", newSourceModule) + ctx.RegisterModuleType("override_source", newOverrideSourceModule) + + RegisterPrebuiltMutators(ctx) + ctx.PostDepsMutators(RegisterOverridePostDepsMutators) +} + type prebuiltModule struct { ModuleBase prebuilt Prebuilt @@ -250,15 +288,24 @@ func (p *prebuiltModule) Prebuilt() *Prebuilt { return &p.prebuilt } -func (p *prebuiltModule) Srcs() Paths { - return Paths{p.src} +func (p *prebuiltModule) OutputFiles(tag string) (Paths, error) { + switch tag { + case "": + return Paths{p.src}, nil + default: + return nil, fmt.Errorf("unsupported module reference tag %q", tag) + } +} + +type sourceModuleProperties struct { + Deps []string `android:"path"` } type sourceModule struct { ModuleBase - properties struct { - Deps []string `android:"path"` - } + OverridableModuleBase + + properties sourceModuleProperties dependsOnSourceModule, dependsOnPrebuiltModule bool deps Paths src Path @@ -268,10 +315,11 @@ func newSourceModule() Module { m := &sourceModule{} m.AddProperties(&m.properties) InitAndroidModule(m) + InitOverridableModule(m, nil) return m } -func (s *sourceModule) DepsMutator(ctx BottomUpMutatorContext) { +func (s *sourceModule) OverridablePropertiesDepsMutator(ctx BottomUpMutatorContext) { // s.properties.Deps are annotated with android:path, so they are // automatically added to the dependency by pathDeps mutator } @@ -284,3 +332,20 @@ func (s *sourceModule) GenerateAndroidBuildActions(ctx ModuleContext) { func (s *sourceModule) Srcs() Paths { return Paths{s.src} } + +type overrideSourceModule struct { + ModuleBase + OverrideModuleBase +} + +func (o *overrideSourceModule) GenerateAndroidBuildActions(_ ModuleContext) { +} + +func newOverrideSourceModule() Module { + m := &overrideSourceModule{} + m.AddProperties(&sourceModuleProperties{}) + + InitAndroidModule(m) + InitOverrideModule(m) + return m +} diff --git a/android/proto.go b/android/proto.go index 5247c68dc..b712258eb 100644 --- a/android/proto.go +++ b/android/proto.go @@ -52,9 +52,8 @@ func ProtoDeps(ctx BottomUpMutatorContext, p *ProtoProperties) { } if plugin := String(p.Proto.Plugin); plugin != "" { - ctx.AddFarVariationDependencies([]blueprint.Variation{ - {Mutator: "arch", Variation: ctx.Config().BuildOsVariant}, - }, ProtoPluginDepTag, "protoc-gen-"+plugin) + ctx.AddFarVariationDependencies(ctx.Config().BuildOSTarget.Variations(), + ProtoPluginDepTag, "protoc-gen-"+plugin) } } @@ -135,7 +134,7 @@ func ProtoRule(ctx ModuleContext, rule *RuleBuilder, protoFile Path, flags Proto } rule.Command(). - Tool(ctx.Config().HostToolPath(ctx, "aprotoc")). + BuiltTool(ctx, "aprotoc"). FlagWithArg(flags.OutTypeFlag+"=", strings.Join(flags.OutParams, ",")+":"+outDir.String()). FlagWithDepFile("--dependency_out=", depFile). FlagWithArg("-I ", protoBase). @@ -145,5 +144,5 @@ func ProtoRule(ctx ModuleContext, rule *RuleBuilder, protoFile Path, flags Proto ImplicitOutputs(outputs) rule.Command(). - Tool(ctx.Config().HostToolPath(ctx, "dep_fixer")).Flag(depFile.String()) + BuiltTool(ctx, "dep_fixer").Flag(depFile.String()) } diff --git a/android/register.go b/android/register.go index d79982a31..036a8113f 100644 --- a/android/register.go +++ b/android/register.go @@ -15,6 +15,8 @@ package android import ( + "fmt" + "github.com/google/blueprint" ) @@ -82,7 +84,9 @@ type Context struct { } func NewContext() *Context { - return &Context{blueprint.NewContext()} + ctx := &Context{blueprint.NewContext()} + ctx.SetSrcDir(absSrcDir) + return ctx } func (ctx *Context) Register() { @@ -98,7 +102,10 @@ func (ctx *Context) Register() { ctx.RegisterSingletonType(t.name, t.factory) } - registerMutators(ctx.Context, preArch, preDeps, postDeps) + registerMutators(ctx.Context, preArch, preDeps, postDeps, finalDeps) + + // Register phony just before makevars so it can write out its phony rules as Make rules + ctx.RegisterSingletonType("phony", SingletonFactoryAdaptor(phonySingletonFactory)) // Register makevars after other singletons so they can export values through makevars ctx.RegisterSingletonType("makevars", SingletonFactoryAdaptor(makeVarsSingletonFunc)) @@ -114,3 +121,87 @@ func ModuleTypeFactories() map[string]ModuleFactory { } return ret } + +// Interface for registering build components. +// +// Provided to allow registration of build components to be shared between the runtime +// and test environments. +type RegistrationContext interface { + RegisterModuleType(name string, factory ModuleFactory) + RegisterSingletonType(name string, factory SingletonFactory) + PreArchMutators(f RegisterMutatorFunc) + + // Register pre arch mutators that are hard coded into mutator.go. + // + // Only registers mutators for testing, is a noop on the InitRegistrationContext. + HardCodedPreArchMutators(f RegisterMutatorFunc) + + PreDepsMutators(f RegisterMutatorFunc) + PostDepsMutators(f RegisterMutatorFunc) + FinalDepsMutators(f RegisterMutatorFunc) +} + +// Used to register build components from an init() method, e.g. +// +// init() { +// RegisterBuildComponents(android.InitRegistrationContext) +// } +// +// func RegisterBuildComponents(ctx android.RegistrationContext) { +// ctx.RegisterModuleType(...) +// ... +// } +// +// Extracting the actual registration into a separate RegisterBuildComponents(ctx) function +// allows it to be used to initialize test context, e.g. +// +// ctx := android.NewTestContext() +// RegisterBuildComponents(ctx) +var InitRegistrationContext RegistrationContext = &initRegistrationContext{ + moduleTypes: make(map[string]ModuleFactory), + singletonTypes: make(map[string]SingletonFactory), +} + +// Make sure the TestContext implements RegistrationContext. +var _ RegistrationContext = (*TestContext)(nil) + +type initRegistrationContext struct { + moduleTypes map[string]ModuleFactory + singletonTypes map[string]SingletonFactory +} + +func (ctx *initRegistrationContext) RegisterModuleType(name string, factory ModuleFactory) { + if _, present := ctx.moduleTypes[name]; present { + panic(fmt.Sprintf("module type %q is already registered", name)) + } + ctx.moduleTypes[name] = factory + RegisterModuleType(name, factory) +} + +func (ctx *initRegistrationContext) RegisterSingletonType(name string, factory SingletonFactory) { + if _, present := ctx.singletonTypes[name]; present { + panic(fmt.Sprintf("singleton type %q is already registered", name)) + } + ctx.singletonTypes[name] = factory + RegisterSingletonType(name, factory) +} + +func (ctx *initRegistrationContext) PreArchMutators(f RegisterMutatorFunc) { + PreArchMutators(f) +} + +func (ctx *initRegistrationContext) HardCodedPreArchMutators(f RegisterMutatorFunc) { + // Nothing to do as the mutators are hard code in preArch in mutator.go +} + +func (ctx *initRegistrationContext) PreDepsMutators(f RegisterMutatorFunc) { + PreDepsMutators(f) +} + +func (ctx *initRegistrationContext) PostDepsMutators(f RegisterMutatorFunc) { + PostDepsMutators(f) +} + +func (ctx *initRegistrationContext) FinalDepsMutators(f RegisterMutatorFunc) { + FinalDepsMutators(f) +} diff --git a/android/rule_builder.go b/android/rule_builder.go index b3fccea36..afb5f4e4a 100644 --- a/android/rule_builder.go +++ b/android/rule_builder.go @@ -33,6 +33,8 @@ type RuleBuilder struct { temporariesSet map[WritablePath]bool restat bool sbox bool + highmem bool + remoteable RemoteRuleSupports sboxOutDir WritablePath missingDeps []string } @@ -87,6 +89,19 @@ func (r *RuleBuilder) Restat() *RuleBuilder { return r } +// HighMem marks the rule as a high memory rule, which will limit how many run in parallel with other high memory +// rules. +func (r *RuleBuilder) HighMem() *RuleBuilder { + r.highmem = true + return r +} + +// Remoteable marks the rule as supporting remote execution. +func (r *RuleBuilder) Remoteable(supports RemoteRuleSupports) *RuleBuilder { + r.remoteable = supports + return r +} + // Sbox marks the rule as needing to be wrapped by sbox. The WritablePath should point to the output // directory that sbox will wipe. It should not be written to by any other rule. sbox will ensure // that all outputs have been written, and will discard any output files that were not specified. @@ -147,16 +162,17 @@ func (r *RuleBuilder) DeleteTemporaryFiles() { r.Command().Text("rm").Flag("-f").Outputs(temporariesList) } -// Inputs returns the list of paths that were passed to the RuleBuilderCommand methods that take input paths, such -// as RuleBuilderCommand.Input, RuleBuilderComand.Implicit, or RuleBuilderCommand.FlagWithInput. Inputs to a command -// that are also outputs of another command in the same RuleBuilder are filtered out. +// Inputs returns the list of paths that were passed to the RuleBuilderCommand methods that take +// input paths, such as RuleBuilderCommand.Input, RuleBuilderComand.Implicit, or +// RuleBuilderCommand.FlagWithInput. Inputs to a command that are also outputs of another command +// in the same RuleBuilder are filtered out. The list is sorted and duplicates removed. func (r *RuleBuilder) Inputs() Paths { outputs := r.outputSet() depFiles := r.depFileSet() inputs := make(map[string]Path) for _, c := range r.commands { - for _, input := range c.inputs { + for _, input := range append(c.inputs, c.implicits...) { inputStr := input.String() if _, isOutput := outputs[inputStr]; !isOutput { if _, isDepFile := depFiles[inputStr]; !isDepFile { @@ -178,6 +194,28 @@ func (r *RuleBuilder) Inputs() Paths { return inputList } +// OrderOnlys returns the list of paths that were passed to the RuleBuilderCommand.OrderOnly or +// RuleBuilderCommand.OrderOnlys. The list is sorted and duplicates removed. +func (r *RuleBuilder) OrderOnlys() Paths { + orderOnlys := make(map[string]Path) + for _, c := range r.commands { + for _, orderOnly := range c.orderOnlys { + orderOnlys[orderOnly.String()] = orderOnly + } + } + + var orderOnlyList Paths + for _, orderOnly := range orderOnlys { + orderOnlyList = append(orderOnlyList, orderOnly) + } + + sort.Slice(orderOnlyList, func(i, j int) bool { + return orderOnlyList[i].String() < orderOnlyList[j].String() + }) + + return orderOnlyList +} + func (r *RuleBuilder) outputSet() map[string]WritablePath { outputs := make(map[string]WritablePath) for _, c := range r.commands { @@ -188,8 +226,9 @@ func (r *RuleBuilder) outputSet() map[string]WritablePath { return outputs } -// Outputs returns the list of paths that were passed to the RuleBuilderCommand methods that take output paths, such -// as RuleBuilderCommand.Output, RuleBuilderCommand.ImplicitOutput, or RuleBuilderCommand.FlagWithInput. +// Outputs returns the list of paths that were passed to the RuleBuilderCommand methods that take +// output paths, such as RuleBuilderCommand.Output, RuleBuilderCommand.ImplicitOutput, or +// RuleBuilderCommand.FlagWithInput. The list is sorted and duplicates removed. func (r *RuleBuilder) Outputs() WritablePaths { outputs := r.outputSet() @@ -247,7 +286,8 @@ func (r *RuleBuilder) toolsSet() map[string]Path { return tools } -// Tools returns the list of paths that were passed to the RuleBuilderCommand.Tool method. +// Tools returns the list of paths that were passed to the RuleBuilderCommand.Tool method. The +// list is sorted and duplicates removed. func (r *RuleBuilder) Tools() Paths { toolsSet := r.toolsSet() @@ -263,11 +303,36 @@ func (r *RuleBuilder) Tools() Paths { return toolsList } -// Commands returns a slice containing a the built command line for each call to RuleBuilder.Command. +// RspFileInputs returns the list of paths that were passed to the RuleBuilderCommand.FlagWithRspFileInputList method. +func (r *RuleBuilder) RspFileInputs() Paths { + var rspFileInputs Paths + for _, c := range r.commands { + if c.rspFileInputs != nil { + if rspFileInputs != nil { + panic("Multiple commands in a rule may not have rsp file inputs") + } + rspFileInputs = c.rspFileInputs + } + } + + return rspFileInputs +} + +// Commands returns a slice containing the built command line for each call to RuleBuilder.Command. func (r *RuleBuilder) Commands() []string { var commands []string for _, c := range r.commands { - commands = append(commands, string(c.buf)) + commands = append(commands, c.String()) + } + return commands +} + +// NinjaEscapedCommands returns a slice containin the built command line after ninja escaping for each call to +// RuleBuilder.Command. +func (r *RuleBuilder) NinjaEscapedCommands() []string { + var commands []string + for _, c := range r.commands { + commands = append(commands, c.NinjaEscapedString()) } return commands } @@ -284,7 +349,7 @@ var _ BuilderContext = SingletonContext(nil) func (r *RuleBuilder) depFileMergerCmd(ctx PathContext, depFiles WritablePaths) *RuleBuilderCommand { return r.Command(). - Tool(ctx.Config().HostToolPath(ctx, "dep_fixer")). + BuiltTool(ctx, "dep_fixer"). Inputs(depFiles.Paths()) } @@ -297,6 +362,7 @@ func (r *RuleBuilder) Build(pctx PackageContext, ctx BuilderContext, name string ctx.Build(pctx, BuildParams{ Rule: ErrorRule, Outputs: r.Outputs(), + OrderOnly: r.OrderOnlys(), Description: desc, Args: map[string]string{ "error": "missing dependencies: " + strings.Join(r.missingDeps, ", "), @@ -324,7 +390,7 @@ func (r *RuleBuilder) Build(pctx PackageContext, ctx BuilderContext, name string } tools := r.Tools() - commands := r.Commands() + commands := r.NinjaEscapedCommands() outputs := r.Outputs() if len(commands) == 0 { @@ -334,7 +400,7 @@ func (r *RuleBuilder) Build(pctx PackageContext, ctx BuilderContext, name string panic("No outputs specified from any Commands") } - commandString := strings.Join(proptools.NinjaEscapeList(commands), " && ") + commandString := strings.Join(commands, " && ") if r.sbox { sboxOutputs := make([]string, len(outputs)) @@ -348,7 +414,7 @@ func (r *RuleBuilder) Build(pctx PackageContext, ctx BuilderContext, name string } sboxCmd := &RuleBuilderCommand{} - sboxCmd.Tool(ctx.Config().HostToolPath(ctx, "sbox")). + sboxCmd.BuiltTool(ctx, "sbox"). Flag("-c").Text(commandString). Flag("--sandbox-path").Text(shared.TempDirForOutDir(PathForOutput(ctx).String())). Flag("--output-root").Text(r.sboxOutDir.String()) @@ -359,22 +425,45 @@ func (r *RuleBuilder) Build(pctx PackageContext, ctx BuilderContext, name string sboxCmd.Flags(sboxOutputs) - commandString = string(sboxCmd.buf) + commandString = sboxCmd.buf.String() tools = append(tools, sboxCmd.tools...) } // Ninja doesn't like multiple outputs when depfiles are enabled, move all but the first output to - // ImplicitOutputs. RuleBuilder never uses "$out", so the distinction between Outputs and ImplicitOutputs - // doesn't matter. + // ImplicitOutputs. RuleBuilder only uses "$out" for the rsp file location, so the distinction between Outputs and + // ImplicitOutputs doesn't matter. output := outputs[0] implicitOutputs := outputs[1:] + var rspFile, rspFileContent string + rspFileInputs := r.RspFileInputs() + if rspFileInputs != nil { + rspFile = "$out.rsp" + rspFileContent = "$in" + } + + var pool blueprint.Pool + if ctx.Config().UseGoma() && r.remoteable.Goma { + // When USE_GOMA=true is set and the rule is supported by goma, allow jobs to run outside the local pool. + } else if ctx.Config().UseRBE() && r.remoteable.RBE { + // When USE_RBE=true is set and the rule is supported by RBE, use the remotePool. + pool = remotePool + } else if r.highmem { + pool = highmemPool + } else if ctx.Config().UseRemoteBuild() { + pool = localPool + } + ctx.Build(pctx, BuildParams{ Rule: ctx.Rule(pctx, name, blueprint.RuleParams{ - Command: commandString, - CommandDeps: tools.Strings(), - Restat: r.restat, + Command: commandString, + CommandDeps: tools.Strings(), + Restat: r.restat, + Rspfile: rspFile, + RspfileContent: rspFileContent, + Pool: pool, }), + Inputs: rspFileInputs, Implicits: r.Inputs(), Output: output, ImplicitOutputs: implicitOutputs, @@ -389,11 +478,17 @@ func (r *RuleBuilder) Build(pctx PackageContext, ctx BuilderContext, name string // RuleBuilderCommand, so they can be used chained or unchained. All methods that add text implicitly add a single // space as a separator from the previous method. type RuleBuilderCommand struct { - buf []byte - inputs Paths - outputs WritablePaths - depFiles WritablePaths - tools Paths + buf strings.Builder + inputs Paths + implicits Paths + orderOnlys Paths + outputs WritablePaths + depFiles WritablePaths + tools Paths + rspFileInputs Paths + + // spans [start,end) of the command that should not be ninja escaped + unescapedSpans [][2]int sbox bool sboxOutDir WritablePath @@ -409,6 +504,20 @@ func (c *RuleBuilderCommand) addInput(path Path) string { return path.String() } +func (c *RuleBuilderCommand) addImplicit(path Path) string { + if c.sbox { + if rel, isRel, _ := maybeRelErr(c.sboxOutDir.String(), path.String()); isRel { + return "__SBOX_OUT_DIR__/" + rel + } + } + c.implicits = append(c.implicits, path) + return path.String() +} + +func (c *RuleBuilderCommand) addOrderOnly(path Path) { + c.orderOnlys = append(c.orderOnlys, path) +} + func (c *RuleBuilderCommand) outputStr(path Path) string { if c.sbox { // Errors will be handled in RuleBuilder.Build where we have a context to report them @@ -421,10 +530,10 @@ func (c *RuleBuilderCommand) outputStr(path Path) string { // Text adds the specified raw text to the command line. The text should not contain input or output paths or the // rule will not have them listed in its dependencies or outputs. func (c *RuleBuilderCommand) Text(text string) *RuleBuilderCommand { - if len(c.buf) > 0 { - c.buf = append(c.buf, ' ') + if c.buf.Len() > 0 { + c.buf.WriteByte(' ') } - c.buf = append(c.buf, text...) + c.buf.WriteString(text) return c } @@ -440,6 +549,16 @@ func (c *RuleBuilderCommand) Flag(flag string) *RuleBuilderCommand { return c.Text(flag) } +// OptionalFlag adds the specified raw text to the command line if it is not nil. The text should not contain input or +// output paths or the rule will not have them listed in its dependencies or outputs. +func (c *RuleBuilderCommand) OptionalFlag(flag *string) *RuleBuilderCommand { + if flag != nil { + c.Text(*flag) + } + + return c +} + // Flags adds the specified raw text to the command line. The text should not contain input or output paths or the // rule will not have them listed in its dependencies or outputs. func (c *RuleBuilderCommand) Flags(flags []string) *RuleBuilderCommand { @@ -479,6 +598,24 @@ func (c *RuleBuilderCommand) Tool(path Path) *RuleBuilderCommand { return c.Text(path.String()) } +// BuiltTool adds the specified tool path that was built using a host Soong module to the command line. The path will +// be also added to the dependencies returned by RuleBuilder.Tools. +// +// It is equivalent to: +// cmd.Tool(ctx.Config().HostToolPath(ctx, tool)) +func (c *RuleBuilderCommand) BuiltTool(ctx PathContext, tool string) *RuleBuilderCommand { + return c.Tool(ctx.Config().HostToolPath(ctx, tool)) +} + +// PrebuiltBuildTool adds the specified tool path from prebuils/build-tools. The path will be also added to the +// dependencies returned by RuleBuilder.Tools. +// +// It is equivalent to: +// cmd.Tool(ctx.Config().PrebuiltBuildTool(ctx, tool)) +func (c *RuleBuilderCommand) PrebuiltBuildTool(ctx PathContext, tool string) *RuleBuilderCommand { + return c.Tool(ctx.Config().PrebuiltBuildTool(ctx, tool)) +} + // Input adds the specified input path to the command line. The path will also be added to the dependencies returned by // RuleBuilder.Inputs. func (c *RuleBuilderCommand) Input(path Path) *RuleBuilderCommand { @@ -497,7 +634,7 @@ func (c *RuleBuilderCommand) Inputs(paths Paths) *RuleBuilderCommand { // Implicit adds the specified input path to the dependencies returned by RuleBuilder.Inputs without modifying the // command line. func (c *RuleBuilderCommand) Implicit(path Path) *RuleBuilderCommand { - c.addInput(path) + c.addImplicit(path) return c } @@ -505,7 +642,28 @@ func (c *RuleBuilderCommand) Implicit(path Path) *RuleBuilderCommand { // command line. func (c *RuleBuilderCommand) Implicits(paths Paths) *RuleBuilderCommand { for _, path := range paths { - c.addInput(path) + c.addImplicit(path) + } + return c +} + +// GetImplicits returns the command's implicit inputs. +func (c *RuleBuilderCommand) GetImplicits() Paths { + return c.implicits +} + +// OrderOnly adds the specified input path to the dependencies returned by RuleBuilder.OrderOnlys +// without modifying the command line. +func (c *RuleBuilderCommand) OrderOnly(path Path) *RuleBuilderCommand { + c.addOrderOnly(path) + return c +} + +// OrderOnlys adds the specified input paths to the dependencies returned by RuleBuilder.OrderOnlys +// without modifying the command line. +func (c *RuleBuilderCommand) OrderOnlys(paths Paths) *RuleBuilderCommand { + for _, path := range paths { + c.addOrderOnly(path) } return c } @@ -526,6 +684,15 @@ func (c *RuleBuilderCommand) Outputs(paths WritablePaths) *RuleBuilderCommand { return c } +// OutputDir adds the output directory to the command line. This is only available when used with RuleBuilder.Sbox, +// and will be the temporary output directory managed by sbox, not the final one. +func (c *RuleBuilderCommand) OutputDir() *RuleBuilderCommand { + if !c.sbox { + panic("OutputDir only valid with Sbox") + } + return c.Text("__SBOX_OUT_DIR__") +} + // DepFile adds the specified depfile path to the paths returned by RuleBuilder.DepFiles and adds it to the command // line, and causes RuleBuilder.Build file to set the depfile flag for ninja. If multiple depfiles are added to // commands in a single RuleBuilder then RuleBuilder.Build will add an extra command to merge the depfiles together. @@ -598,9 +765,54 @@ func (c *RuleBuilderCommand) FlagWithDepFile(flag string, path WritablePath) *Ru return c.Text(flag + c.outputStr(path)) } +// FlagWithRspFileInputList adds the specified flag and path to an rspfile to the command line, with no separator +// between them. The paths will be written to the rspfile. +func (c *RuleBuilderCommand) FlagWithRspFileInputList(flag string, paths Paths) *RuleBuilderCommand { + if c.rspFileInputs != nil { + panic("FlagWithRspFileInputList cannot be called if rsp file inputs have already been provided") + } + + // Use an empty slice if paths is nil, the non-nil slice is used as an indicator that the rsp file must be + // generated. + if paths == nil { + paths = Paths{} + } + + c.rspFileInputs = paths + + rspFile := "$out.rsp" + c.FlagWithArg(flag, rspFile) + c.unescapedSpans = append(c.unescapedSpans, [2]int{c.buf.Len() - len(rspFile), c.buf.Len()}) + return c +} + // String returns the command line. func (c *RuleBuilderCommand) String() string { - return string(c.buf) + return c.buf.String() +} + +// String returns the command line. +func (c *RuleBuilderCommand) NinjaEscapedString() string { + return ninjaEscapeExceptForSpans(c.String(), c.unescapedSpans) +} + +func ninjaEscapeExceptForSpans(s string, spans [][2]int) string { + if len(spans) == 0 { + return proptools.NinjaEscape(s) + } + + sb := strings.Builder{} + sb.Grow(len(s) * 11 / 10) + + i := 0 + for _, span := range spans { + sb.WriteString(proptools.NinjaEscape(s[i:span[0]])) + sb.WriteString(s[span[0]:span[1]]) + i = span[1] + } + sb.WriteString(proptools.NinjaEscape(s[i:])) + + return sb.String() } func ninjaNameEscape(s string) string { diff --git a/android/rule_builder_test.go b/android/rule_builder_test.go index d122a4294..c41b06734 100644 --- a/android/rule_builder_test.go +++ b/android/rule_builder_test.go @@ -16,8 +16,6 @@ package android import ( "fmt" - "io/ioutil" - "os" "path/filepath" "reflect" "strings" @@ -29,18 +27,18 @@ import ( ) func pathContext() PathContext { - return PathContextForTesting(TestConfig("out", nil), - map[string][]byte{ - "ld": nil, - "a.o": nil, - "b.o": nil, - "cp": nil, - "a": nil, - "b": nil, - "ls": nil, - "turbine": nil, - "java": nil, - }) + return PathContextForTesting(TestConfig("out", nil, "", map[string][]byte{ + "ld": nil, + "a.o": nil, + "b.o": nil, + "cp": nil, + "a": nil, + "b": nil, + "ls": nil, + "turbine": nil, + "java": nil, + "javac": nil, + })) } func ExampleRuleBuilder() { @@ -237,19 +235,49 @@ func ExampleRuleBuilderCommand_FlagWithList() { // ls --sort=time,size } +func ExampleRuleBuilderCommand_FlagWithRspFileInputList() { + ctx := pathContext() + fmt.Println(NewRuleBuilder().Command(). + Tool(PathForSource(ctx, "javac")). + FlagWithRspFileInputList("@", PathsForTesting("a.java", "b.java")). + NinjaEscapedString()) + // Output: + // javac @$out.rsp +} + +func ExampleRuleBuilderCommand_String() { + fmt.Println(NewRuleBuilder().Command(). + Text("FOO=foo"). + Text("echo $FOO"). + String()) + // Output: + // FOO=foo echo $FOO +} + +func ExampleRuleBuilderCommand_NinjaEscapedString() { + fmt.Println(NewRuleBuilder().Command(). + Text("FOO=foo"). + Text("echo $FOO"). + NinjaEscapedString()) + // Output: + // FOO=foo echo $$FOO +} + func TestRuleBuilder(t *testing.T) { fs := map[string][]byte{ - "dep_fixer": nil, - "input": nil, - "Implicit": nil, - "Input": nil, - "Tool": nil, - "input2": nil, - "tool2": nil, - "input3": nil, + "dep_fixer": nil, + "input": nil, + "Implicit": nil, + "Input": nil, + "OrderOnly": nil, + "OrderOnlys": nil, + "Tool": nil, + "input2": nil, + "tool2": nil, + "input3": nil, } - ctx := PathContextForTesting(TestConfig("out", nil), fs) + ctx := PathContextForTesting(TestConfig("out", nil, "", fs)) addCommands := func(rule *RuleBuilder) { cmd := rule.Command(). @@ -264,6 +292,7 @@ func TestRuleBuilder(t *testing.T) { ImplicitOutput(PathForOutput(ctx, "ImplicitOutput")). Input(PathForSource(ctx, "Input")). Output(PathForOutput(ctx, "Output")). + OrderOnly(PathForSource(ctx, "OrderOnly")). Text("Text"). Tool(PathForSource(ctx, "Tool")) @@ -272,6 +301,7 @@ func TestRuleBuilder(t *testing.T) { DepFile(PathForOutput(ctx, "depfile2")). Input(PathForSource(ctx, "input2")). Output(PathForOutput(ctx, "output2")). + OrderOnlys(PathsForSource(ctx, []string{"OrderOnlys"})). Tool(PathForSource(ctx, "tool2")) // Test updates to the first command after the second command has been started @@ -291,6 +321,7 @@ func TestRuleBuilder(t *testing.T) { wantOutputs := PathsForOutput(ctx, []string{"ImplicitOutput", "Output", "output", "output2", "output3"}) wantDepFiles := PathsForOutput(ctx, []string{"DepFile", "depfile", "ImplicitDepFile", "depfile2"}) wantTools := PathsForSource(ctx, []string{"Tool", "tool2"}) + wantOrderOnlys := PathsForSource(ctx, []string{"OrderOnly", "OrderOnlys"}) t.Run("normal", func(t *testing.T) { rule := NewRuleBuilder() @@ -320,6 +351,9 @@ func TestRuleBuilder(t *testing.T) { if g, w := rule.Tools(), wantTools; !reflect.DeepEqual(w, g) { t.Errorf("\nwant rule.Tools() = %#v\n got %#v", w, g) } + if g, w := rule.OrderOnlys(), wantOrderOnlys; !reflect.DeepEqual(w, g) { + t.Errorf("\nwant rule.OrderOnlys() = %#v\n got %#v", w, g) + } if g, w := rule.depFileMergerCmd(ctx, rule.DepFiles()).String(), wantDepMergerCommand; g != w { t.Errorf("\nwant rule.depFileMergerCmd() = %#v\n got %#v", w, g) @@ -354,6 +388,9 @@ func TestRuleBuilder(t *testing.T) { if g, w := rule.Tools(), wantTools; !reflect.DeepEqual(w, g) { t.Errorf("\nwant rule.Tools() = %#v\n got %#v", w, g) } + if g, w := rule.OrderOnlys(), wantOrderOnlys; !reflect.DeepEqual(w, g) { + t.Errorf("\nwant rule.OrderOnlys() = %#v\n got %#v", w, g) + } if g, w := rule.depFileMergerCmd(ctx, rule.DepFiles()).String(), wantDepMergerCommand; g != w { t.Errorf("\nwant rule.depFileMergerCmd() = %#v\n got %#v", w, g) @@ -418,11 +455,10 @@ func testRuleBuilder_Build(ctx BuilderContext, in Path, out, outDep, outDir Writ } func TestRuleBuilder_Build(t *testing.T) { - buildDir, err := ioutil.TempDir("", "soong_test_rule_builder") - if err != nil { - t.Fatal(err) + fs := map[string][]byte{ + "bar": nil, + "cp": nil, } - defer os.RemoveAll(buildDir) bp := ` rule_builder_test { @@ -437,16 +473,11 @@ func TestRuleBuilder_Build(t *testing.T) { } ` - config := TestConfig(buildDir, nil) + config := TestConfig(buildDir, nil, bp, fs) ctx := NewTestContext() - ctx.MockFileSystem(map[string][]byte{ - "Android.bp": []byte(bp), - "bar": nil, - "cp": nil, - }) - ctx.RegisterModuleType("rule_builder_test", ModuleFactoryAdaptor(testRuleBuilderFactory)) - ctx.RegisterSingletonType("rule_builder_test", SingletonFactoryAdaptor(testRuleBuilderSingletonFactory)) - ctx.Register() + ctx.RegisterModuleType("rule_builder_test", testRuleBuilderFactory) + ctx.RegisterSingletonType("rule_builder_test", testRuleBuilderSingletonFactory) + ctx.Register(config) _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) FailIfErrored(t, errs) @@ -513,3 +544,77 @@ func TestRuleBuilder_Build(t *testing.T) { "cp bar "+outFile, outFile, outFile+".d", true, nil) }) } + +func Test_ninjaEscapeExceptForSpans(t *testing.T) { + type args struct { + s string + spans [][2]int + } + tests := []struct { + name string + args args + want string + }{ + { + name: "empty", + args: args{ + s: "", + }, + want: "", + }, + { + name: "unescape none", + args: args{ + s: "$abc", + }, + want: "$$abc", + }, + { + name: "unescape all", + args: args{ + s: "$abc", + spans: [][2]int{{0, 4}}, + }, + want: "$abc", + }, + { + name: "unescape first", + args: args{ + s: "$abc$", + spans: [][2]int{{0, 1}}, + }, + want: "$abc$$", + }, + { + name: "unescape last", + args: args{ + s: "$abc$", + spans: [][2]int{{4, 5}}, + }, + want: "$$abc$", + }, + { + name: "unescape middle", + args: args{ + s: "$a$b$c$", + spans: [][2]int{{2, 5}}, + }, + want: "$$a$b$c$$", + }, + { + name: "unescape multiple", + args: args{ + s: "$a$b$c$", + spans: [][2]int{{2, 3}, {4, 5}}, + }, + want: "$$a$b$c$$", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ninjaEscapeExceptForSpans(tt.args.s, tt.args.spans); got != tt.want { + t.Errorf("ninjaEscapeExceptForSpans() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/android/sandbox.go b/android/sandbox.go new file mode 100644 index 000000000..ed022fb87 --- /dev/null +++ b/android/sandbox.go @@ -0,0 +1,47 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "fmt" + "os" +) + +func init() { + // Stash the working directory in a private variable and then change the working directory + // to "/", which will prevent untracked accesses to files by Go Soong plugins. The + // SOONG_SANDBOX_SOONG_BUILD environment variable is set by soong_ui, and is not + // overrideable on the command line. + + orig, err := os.Getwd() + if err != nil { + panic(fmt.Errorf("failed to get working directory: %s", err)) + } + absSrcDir = orig + + if getenv("SOONG_SANDBOX_SOONG_BUILD") == "true" { + err = os.Chdir("/") + if err != nil { + panic(fmt.Errorf("failed to change working directory to '/': %s", err)) + } + } +} + +// DO NOT USE THIS FUNCTION IN NEW CODE. +// Deprecated: This function will be removed as soon as the existing use cases that use it have been +// replaced. +func AbsSrcDirForExistingUseCases() string { + return absSrcDir +} diff --git a/android/sdk.go b/android/sdk.go new file mode 100644 index 000000000..e823106e8 --- /dev/null +++ b/android/sdk.go @@ -0,0 +1,537 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "sort" + "strings" + + "github.com/google/blueprint" + "github.com/google/blueprint/proptools" +) + +// Extracted from SdkAware to make it easier to define custom subsets of the +// SdkAware interface and improve code navigation within the IDE. +// +// In addition to its use in SdkAware this interface must also be implemented by +// APEX to specify the SDKs required by that module and its contents. e.g. APEX +// is expected to implement RequiredSdks() by reading its own properties like +// `uses_sdks`. +type RequiredSdks interface { + // The set of SDKs required by an APEX and its contents. + RequiredSdks() SdkRefs +} + +// SdkAware is the interface that must be supported by any module to become a member of SDK or to be +// built with SDK +type SdkAware interface { + Module + RequiredSdks + + sdkBase() *SdkBase + MakeMemberOf(sdk SdkRef) + IsInAnySdk() bool + ContainingSdk() SdkRef + MemberName() string + BuildWithSdks(sdks SdkRefs) +} + +// SdkRef refers to a version of an SDK +type SdkRef struct { + Name string + Version string +} + +// Unversioned determines if the SdkRef is referencing to the unversioned SDK module +func (s SdkRef) Unversioned() bool { + return s.Version == "" +} + +// String returns string representation of this SdkRef for debugging purpose +func (s SdkRef) String() string { + if s.Name == "" { + return "(No Sdk)" + } + if s.Unversioned() { + return s.Name + } + return s.Name + string(SdkVersionSeparator) + s.Version +} + +// SdkVersionSeparator is a character used to separate an sdk name and its version +const SdkVersionSeparator = '@' + +// ParseSdkRef parses a `name@version` style string into a corresponding SdkRef struct +func ParseSdkRef(ctx BaseModuleContext, str string, property string) SdkRef { + tokens := strings.Split(str, string(SdkVersionSeparator)) + if len(tokens) < 1 || len(tokens) > 2 { + ctx.PropertyErrorf(property, "%q does not follow name#version syntax", str) + return SdkRef{Name: "invalid sdk name", Version: "invalid sdk version"} + } + + name := tokens[0] + + var version string + if len(tokens) == 2 { + version = tokens[1] + } + + return SdkRef{Name: name, Version: version} +} + +type SdkRefs []SdkRef + +// Contains tells if the given SdkRef is in this list of SdkRef's +func (refs SdkRefs) Contains(s SdkRef) bool { + for _, r := range refs { + if r == s { + return true + } + } + return false +} + +type sdkProperties struct { + // The SDK that this module is a member of. nil if it is not a member of any SDK + ContainingSdk *SdkRef `blueprint:"mutated"` + + // The list of SDK names and versions that are used to build this module + RequiredSdks SdkRefs `blueprint:"mutated"` + + // Name of the module that this sdk member is representing + Sdk_member_name *string +} + +// SdkBase is a struct that is expected to be included in module types to implement the SdkAware +// interface. InitSdkAwareModule should be called to initialize this struct. +type SdkBase struct { + properties sdkProperties + module SdkAware +} + +func (s *SdkBase) sdkBase() *SdkBase { + return s +} + +// MakeMemberOf sets this module to be a member of a specific SDK +func (s *SdkBase) MakeMemberOf(sdk SdkRef) { + s.properties.ContainingSdk = &sdk +} + +// IsInAnySdk returns true if this module is a member of any SDK +func (s *SdkBase) IsInAnySdk() bool { + return s.properties.ContainingSdk != nil +} + +// ContainingSdk returns the SDK that this module is a member of +func (s *SdkBase) ContainingSdk() SdkRef { + if s.properties.ContainingSdk != nil { + return *s.properties.ContainingSdk + } + return SdkRef{Name: "", Version: ""} +} + +// MemberName returns the name of the module that this SDK member is overriding +func (s *SdkBase) MemberName() string { + return proptools.String(s.properties.Sdk_member_name) +} + +// BuildWithSdks is used to mark that this module has to be built with the given SDK(s). +func (s *SdkBase) BuildWithSdks(sdks SdkRefs) { + s.properties.RequiredSdks = sdks +} + +// RequiredSdks returns the SDK(s) that this module has to be built with +func (s *SdkBase) RequiredSdks() SdkRefs { + return s.properties.RequiredSdks +} + +// InitSdkAwareModule initializes the SdkBase struct. This must be called by all modules including +// SdkBase. +func InitSdkAwareModule(m SdkAware) { + base := m.sdkBase() + base.module = m + m.AddProperties(&base.properties) +} + +// Provide support for generating the build rules which will build the snapshot. +type SnapshotBuilder interface { + // Copy src to the dest (which is a snapshot relative path) and add the dest + // to the zip + CopyToSnapshot(src Path, dest string) + + // Unzip the supplied zip into the snapshot relative directory destDir. + UnzipToSnapshot(zipPath Path, destDir string) + + // Add a new prebuilt module to the snapshot. The returned module + // must be populated with the module type specific properties. The following + // properties will be automatically populated. + // + // * name + // * sdk_member_name + // * prefer + // + // This will result in two Soong modules being generated in the Android. One + // that is versioned, coupled to the snapshot version and marked as + // prefer=true. And one that is not versioned, not marked as prefer=true and + // will only be used if the equivalently named non-prebuilt module is not + // present. + AddPrebuiltModule(member SdkMember, moduleType string) BpModule + + // The property tag to use when adding a property to a BpModule that contains + // references to other sdk members. Using this will ensure that the reference + // is correctly output for both versioned and unversioned prebuilts in the + // snapshot. + // + // "required: true" means that the property must only contain references + // to other members of the sdk. Passing a reference to a module that is not a + // member of the sdk will result in a build error. + // + // "required: false" means that the property can contain references to modules + // that are either members or not members of the sdk. If a reference is to a + // module that is a non member then the reference is left unchanged, i.e. it + // is not transformed as references to members are. + // + // The handling of the member names is dependent on whether it is an internal or + // exported member. An exported member is one whose name is specified in one of + // the member type specific properties. An internal member is one that is added + // due to being a part of an exported (or other internal) member and is not itself + // an exported member. + // + // Member names are handled as follows: + // * When creating the unversioned form of the module the name is left unchecked + // unless the member is internal in which case it is transformed into an sdk + // specific name, i.e. by prefixing with the sdk name. + // + // * When creating the versioned form of the module the name is transformed into + // a versioned sdk specific name, i.e. by prefixing with the sdk name and + // suffixing with the version. + // + // e.g. + // bpPropertySet.AddPropertyWithTag("libs", []string{"member1", "member2"}, builder.SdkMemberReferencePropertyTag(true)) + SdkMemberReferencePropertyTag(required bool) BpPropertyTag +} + +type BpPropertyTag interface{} + +// A set of properties for use in a .bp file. +type BpPropertySet interface { + // Add a property, the value can be one of the following types: + // * string + // * array of the above + // * bool + // * BpPropertySet + // + // It is an error if multiple properties with the same name are added. + AddProperty(name string, value interface{}) + + // Add a property with an associated tag + AddPropertyWithTag(name string, value interface{}, tag BpPropertyTag) + + // Add a property set with the specified name and return so that additional + // properties can be added. + AddPropertySet(name string) BpPropertySet +} + +// A .bp module definition. +type BpModule interface { + BpPropertySet +} + +// An individual member of the SDK, includes all of the variants that the SDK +// requires. +type SdkMember interface { + // The name of the member. + Name() string + + // All the variants required by the SDK. + Variants() []SdkAware +} + +type SdkMemberTypeDependencyTag interface { + blueprint.DependencyTag + + SdkMemberType() SdkMemberType +} + +type sdkMemberDependencyTag struct { + blueprint.BaseDependencyTag + memberType SdkMemberType +} + +func (t *sdkMemberDependencyTag) SdkMemberType() SdkMemberType { + return t.memberType +} + +func DependencyTagForSdkMemberType(memberType SdkMemberType) SdkMemberTypeDependencyTag { + return &sdkMemberDependencyTag{memberType: memberType} +} + +// Interface that must be implemented for every type that can be a member of an +// sdk. +// +// The basic implementation should look something like this, where ModuleType is +// the name of the module type being supported. +// +// type moduleTypeSdkMemberType struct { +// android.SdkMemberTypeBase +// } +// +// func init() { +// android.RegisterSdkMemberType(&moduleTypeSdkMemberType{ +// SdkMemberTypeBase: android.SdkMemberTypeBase{ +// PropertyName: "module_types", +// }, +// } +// } +// +// ...methods... +// +type SdkMemberType interface { + // The name of the member type property on an sdk module. + SdkPropertyName() string + + // True if the member type supports the sdk/sdk_snapshot, false otherwise. + UsableWithSdkAndSdkSnapshot() bool + + // Return true if modules of this type can have dependencies which should be + // treated as if they are sdk members. + // + // Any dependency that is to be treated as a member of the sdk needs to implement + // SdkAware and be added with an SdkMemberTypeDependencyTag tag. + HasTransitiveSdkMembers() bool + + // Add dependencies from the SDK module to all the module variants the member + // type contributes to the SDK. `names` is the list of module names given in + // the member type property (as returned by SdkPropertyName()) in the SDK + // module. The exact set of variants required is determined by the SDK and its + // properties. The dependencies must be added with the supplied tag. + // + // The BottomUpMutatorContext provided is for the SDK module. + AddDependencies(mctx BottomUpMutatorContext, dependencyTag blueprint.DependencyTag, names []string) + + // Return true if the supplied module is an instance of this member type. + // + // This is used to check the type of each variant before added to the + // SdkMember. Returning false will cause an error to be logged expaining that + // the module is not allowed in whichever sdk property it was added. + IsInstance(module Module) bool + + // Add a prebuilt module that the sdk will populate. + // + // The sdk module code generates the snapshot as follows: + // + // * A properties struct of type SdkMemberProperties is created for each variant and + // populated with information from the variant by calling PopulateFromVariant(SdkAware) + // on the struct. + // + // * An additional properties struct is created into which the common properties will be + // added. + // + // * The variant property structs are analysed to find exported (capitalized) fields which + // have common values. Those fields are cleared and the common value added to the common + // properties. + // + // A field annotated with a tag of `sdk:"keep"` will be treated as if it + // was not capitalized, i.e. not optimized for common values. + // + // A field annotated with a tag of `android:"arch_variant"` will be allowed to have + // values that differ by arch, fields not tagged as such must have common values across + // all variants. + // + // * Additional field tags can be specified on a field that will ignore certain values + // for the purpose of common value optimization. A value that is ignored must have the + // default value for the property type. This is to ensure that significant value are not + // ignored by accident. The purpose of this is to allow the snapshot generation to reflect + // the behavior of the runtime. e.g. if a property is ignored on the host then a property + // that is common for android can be treated as if it was common for android and host as + // the setting for host is ignored anyway. + // * `sdk:"ignored-on-host" - this indicates the property is ignored on the host variant. + // + // * The sdk module type populates the BpModule structure, creating the arch specific + // structure and calls AddToPropertySet(...) on the properties struct to add the member + // specific properties in the correct place in the structure. + // + AddPrebuiltModule(ctx SdkMemberContext, member SdkMember) BpModule + + // Create a structure into which variant specific properties can be added. + CreateVariantPropertiesStruct() SdkMemberProperties +} + +// Base type for SdkMemberType implementations. +type SdkMemberTypeBase struct { + PropertyName string + SupportsSdk bool + TransitiveSdkMembers bool +} + +func (b *SdkMemberTypeBase) SdkPropertyName() string { + return b.PropertyName +} + +func (b *SdkMemberTypeBase) UsableWithSdkAndSdkSnapshot() bool { + return b.SupportsSdk +} + +func (b *SdkMemberTypeBase) HasTransitiveSdkMembers() bool { + return b.TransitiveSdkMembers +} + +// Encapsulates the information about registered SdkMemberTypes. +type SdkMemberTypesRegistry struct { + // The list of types sorted by property name. + list []SdkMemberType + + // The key that uniquely identifies this registry instance. + key OnceKey +} + +func (r *SdkMemberTypesRegistry) copyAndAppend(memberType SdkMemberType) *SdkMemberTypesRegistry { + oldList := r.list + + // Copy the slice just in case this is being read while being modified, e.g. when testing. + list := make([]SdkMemberType, 0, len(oldList)+1) + list = append(list, oldList...) + list = append(list, memberType) + + // Sort the member types by their property name to ensure that registry order has no effect + // on behavior. + sort.Slice(list, func(i1, i2 int) bool { + t1 := list[i1] + t2 := list[i2] + + return t1.SdkPropertyName() < t2.SdkPropertyName() + }) + + // Generate a key that identifies the slice of SdkMemberTypes by joining the property names + // from all the SdkMemberType . + var properties []string + for _, t := range list { + properties = append(properties, t.SdkPropertyName()) + } + key := NewOnceKey(strings.Join(properties, "|")) + + // Create a new registry so the pointer uniquely identifies the set of registered types. + return &SdkMemberTypesRegistry{ + list: list, + key: key, + } +} + +func (r *SdkMemberTypesRegistry) RegisteredTypes() []SdkMemberType { + return r.list +} + +func (r *SdkMemberTypesRegistry) UniqueOnceKey() OnceKey { + // Use the pointer to the registry as the unique key. + return NewCustomOnceKey(r) +} + +// The set of registered SdkMemberTypes, one for sdk module and one for module_exports. +var ModuleExportsMemberTypes = &SdkMemberTypesRegistry{} +var SdkMemberTypes = &SdkMemberTypesRegistry{} + +// Register an SdkMemberType object to allow them to be used in the sdk and sdk_snapshot module +// types. +func RegisterSdkMemberType(memberType SdkMemberType) { + // All member types are usable with module_exports. + ModuleExportsMemberTypes = ModuleExportsMemberTypes.copyAndAppend(memberType) + + // Only those that explicitly indicate it are usable with sdk. + if memberType.UsableWithSdkAndSdkSnapshot() { + SdkMemberTypes = SdkMemberTypes.copyAndAppend(memberType) + } +} + +// Base structure for all implementations of SdkMemberProperties. +// +// Contains common properties that apply across many different member types. These +// are not affected by the optimization to extract common values. +type SdkMemberPropertiesBase struct { + // The number of unique os types supported by the member variants. + // + // If a member has a variant with more than one os type then it will need to differentiate + // the locations of any of their prebuilt files in the snapshot by os type to prevent them + // from colliding. See OsPrefix(). + // + // This property is the same for all variants of a member and so would be optimized away + // if it was not explicitly kept. + Os_count int `sdk:"keep"` + + // The os type for which these properties refer. + // + // Provided to allow a member to differentiate between os types in the locations of their + // prebuilt files when it supports more than one os type. + // + // This property is the same for all os type specific variants of a member and so would be + // optimized away if it was not explicitly kept. + Os OsType `sdk:"keep"` + + // The setting to use for the compile_multilib property. + // + // This property is set after optimization so there is no point in trying to optimize it. + Compile_multilib string `sdk:"keep"` +} + +// The os prefix to use for any file paths in the sdk. +// +// Is an empty string if the member only provides variants for a single os type, otherwise +// is the OsType.Name. +func (b *SdkMemberPropertiesBase) OsPrefix() string { + if b.Os_count == 1 { + return "" + } else { + return b.Os.Name + } +} + +func (b *SdkMemberPropertiesBase) Base() *SdkMemberPropertiesBase { + return b +} + +// Interface to be implemented on top of a structure that contains variant specific +// information. +// +// Struct fields that are capitalized are examined for common values to extract. Fields +// that are not capitalized are assumed to be arch specific. +type SdkMemberProperties interface { + // Access the base structure. + Base() *SdkMemberPropertiesBase + + // Populate this structure with information from the variant. + PopulateFromVariant(ctx SdkMemberContext, variant Module) + + // Add the information from this structure to the property set. + AddToPropertySet(ctx SdkMemberContext, propertySet BpPropertySet) +} + +// Provides access to information common to a specific member. +type SdkMemberContext interface { + + // The module context of the sdk common os variant which is creating the snapshot. + SdkModuleContext() ModuleContext + + // The builder of the snapshot. + SnapshotBuilder() SnapshotBuilder + + // The type of the member. + MemberType() SdkMemberType + + // The name of the member. + // + // Provided for use by sdk members to create a member specific location within the snapshot + // into which to copy the prebuilt files. + Name() string +} diff --git a/android/sh_binary.go b/android/sh_binary.go deleted file mode 100644 index cf415c560..000000000 --- a/android/sh_binary.go +++ /dev/null @@ -1,180 +0,0 @@ -// Copyright 2019 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package android - -import ( - "fmt" - "io" - "strings" -) - -// sh_binary is for shell scripts (and batch files) that are installed as -// executable files into .../bin/ -// -// Do not use them for prebuilt C/C++/etc files. Use cc_prebuilt_binary -// instead. - -func init() { - RegisterModuleType("sh_binary", ShBinaryFactory) - RegisterModuleType("sh_binary_host", ShBinaryHostFactory) - RegisterModuleType("sh_test", ShTestFactory) -} - -type shBinaryProperties struct { - // Source file of this prebuilt. - Src *string `android:"path,arch_variant"` - - // optional subdirectory under which this file is installed into - Sub_dir *string `android:"arch_variant"` - - // optional name for the installed file. If unspecified, name of the module is used as the file name - Filename *string `android:"arch_variant"` - - // when set to true, and filename property is not set, the name for the installed file - // is the same as the file name of the source file. - Filename_from_src *bool `android:"arch_variant"` - - // Whether this module is directly installable to one of the partitions. Default: true. - Installable *bool -} - -type TestProperties struct { - // list of compatibility suites (for example "cts", "vts") that the module should be - // installed into. - Test_suites []string `android:"arch_variant"` - - // the name of the test configuration (for example "AndroidTest.xml") that should be - // installed with the module. - Test_config *string `android:"arch_variant"` -} - -type ShBinary struct { - ModuleBase - - properties shBinaryProperties - - sourceFilePath Path - outputFilePath OutputPath -} - -type ShTest struct { - ShBinary - - testProperties TestProperties -} - -func (s *ShBinary) DepsMutator(ctx BottomUpMutatorContext) { - if s.properties.Src == nil { - ctx.PropertyErrorf("src", "missing prebuilt source file") - } -} - -func (s *ShBinary) SourceFilePath(ctx ModuleContext) Path { - return PathForModuleSrc(ctx, String(s.properties.Src)) -} - -func (s *ShBinary) OutputFile() OutputPath { - return s.outputFilePath -} - -func (s *ShBinary) SubDir() string { - return String(s.properties.Sub_dir) -} - -func (s *ShBinary) Installable() bool { - return s.properties.Installable == nil || Bool(s.properties.Installable) -} - -func (s *ShBinary) GenerateAndroidBuildActions(ctx ModuleContext) { - s.sourceFilePath = PathForModuleSrc(ctx, String(s.properties.Src)) - filename := String(s.properties.Filename) - filename_from_src := Bool(s.properties.Filename_from_src) - if filename == "" { - if filename_from_src { - filename = s.sourceFilePath.Base() - } else { - filename = ctx.ModuleName() - } - } else if filename_from_src { - ctx.PropertyErrorf("filename_from_src", "filename is set. filename_from_src can't be true") - return - } - s.outputFilePath = PathForModuleOut(ctx, filename).OutputPath - - // This ensures that outputFilePath has the correct name for others to - // use, as the source file may have a different name. - ctx.Build(pctx, BuildParams{ - Rule: CpExecutable, - Output: s.outputFilePath, - Input: s.sourceFilePath, - }) -} - -func (s *ShBinary) AndroidMk() AndroidMkData { - return AndroidMkData{ - Class: "EXECUTABLES", - OutputFile: OptionalPathForPath(s.outputFilePath), - Include: "$(BUILD_SYSTEM)/soong_cc_prebuilt.mk", - Extra: []AndroidMkExtraFunc{ - func(w io.Writer, outputFile Path) { - fmt.Fprintln(w, "LOCAL_MODULE_RELATIVE_PATH :=", String(s.properties.Sub_dir)) - fmt.Fprintln(w, "LOCAL_MODULE_SUFFIX :=") - fmt.Fprintln(w, "LOCAL_MODULE_STEM :=", s.outputFilePath.Rel()) - }, - }, - } -} - -func (s *ShTest) AndroidMk() AndroidMkData { - data := s.ShBinary.AndroidMk() - data.Class = "NATIVE_TESTS" - data.Extra = append(data.Extra, func(w io.Writer, outputFile Path) { - fmt.Fprintln(w, "LOCAL_COMPATIBILITY_SUITE :=", - strings.Join(s.testProperties.Test_suites, " ")) - fmt.Fprintln(w, "LOCAL_TEST_CONFIG :=", String(s.testProperties.Test_config)) - }) - return data -} - -func InitShBinaryModule(s *ShBinary) { - s.AddProperties(&s.properties) -} - -// sh_binary is for a shell script or batch file to be installed as an -// executable binary to <partition>/bin. -func ShBinaryFactory() Module { - module := &ShBinary{} - InitShBinaryModule(module) - InitAndroidArchModule(module, HostAndDeviceSupported, MultilibFirst) - return module -} - -// sh_binary_host is for a shell script to be installed as an executable binary -// to $(HOST_OUT)/bin. -func ShBinaryHostFactory() Module { - module := &ShBinary{} - InitShBinaryModule(module) - InitAndroidArchModule(module, HostSupported, MultilibFirst) - return module -} - -func ShTestFactory() Module { - module := &ShTest{} - InitShBinaryModule(&module.ShBinary) - module.AddProperties(&module.testProperties) - - InitAndroidArchModule(module, HostAndDeviceSupported, MultilibFirst) - return module -} diff --git a/android/singleton.go b/android/singleton.go index 0266d7798..2c51c6c48 100644 --- a/android/singleton.go +++ b/android/singleton.go @@ -16,7 +16,6 @@ package android import ( "github.com/google/blueprint" - "github.com/google/blueprint/pathtools" ) // SingletonContext @@ -37,6 +36,12 @@ type SingletonContext interface { Variable(pctx PackageContext, name, value string) Rule(pctx PackageContext, name string, params blueprint.RuleParams, argNames ...string) blueprint.Rule Build(pctx PackageContext, params BuildParams) + + // Phony creates a Make-style phony rule, a rule with no commands that can depend on other + // phony rules or real files. Phony can be called on the same name multiple times to add + // additional dependencies. + Phony(name string, deps ...Path) + RequireNinjaVersion(major, minor, micro int) // SetNinjaBuildDir sets the value of the top-level "builddir" Ninja variable @@ -52,6 +57,10 @@ type SingletonContext interface { VisitAllModulesBlueprint(visit func(blueprint.Module)) VisitAllModules(visit func(Module)) VisitAllModulesIf(pred func(Module) bool, visit func(Module)) + + VisitDirectDeps(module Module, visit func(Module)) + VisitDirectDepsIf(module Module, pred func(Module) bool, visit func(Module)) + // Deprecated: use WalkDeps instead to support multiple dependency tags on the same module VisitDepsDepthFirst(module Module, visit func(Module)) // Deprecated: use WalkDeps instead to support multiple dependency tags on the same module @@ -70,8 +79,6 @@ type SingletonContext interface { // builder whenever a file matching the pattern as added or removed, without rerunning if a // file that does not match the pattern is added to a searched directory. GlobWithDeps(pattern string, excludes []string) ([]string, error) - - Fs() pathtools.FileSystem } type singletonAdaptor struct { @@ -127,10 +134,17 @@ func (s *singletonContextAdaptor) Variable(pctx PackageContext, name, value stri } func (s *singletonContextAdaptor) Rule(pctx PackageContext, name string, params blueprint.RuleParams, argNames ...string) blueprint.Rule { - if (s.Config().UseGoma() || s.Config().UseRBE()) && params.Pool == nil { - // When USE_GOMA=true or USE_RBE=true are set and the rule is not supported by goma/RBE, restrict - // jobs to the local parallelism value - params.Pool = localPool + if s.Config().UseRemoteBuild() { + if params.Pool == nil { + // When USE_GOMA=true or USE_RBE=true are set and the rule is not supported by goma/RBE, restrict + // jobs to the local parallelism value + params.Pool = localPool + } else if params.Pool == remotePool { + // remotePool is a fake pool used to identify rule that are supported for remoting. If the rule's + // pool is the remotePool, replace with nil so that ninja runs it at NINJA_REMOTE_NUM_JOBS + // parallelism. + params.Pool = nil + } } rule := s.SingletonContext.Rule(pctx.PackageContext, name, params, argNames...) if s.Config().captureBuild { @@ -148,6 +162,10 @@ func (s *singletonContextAdaptor) Build(pctx PackageContext, params BuildParams) } +func (s *singletonContextAdaptor) Phony(name string, deps ...Path) { + addPhony(s.Config(), name, deps...) +} + func (s *singletonContextAdaptor) SetNinjaBuildDir(pctx PackageContext, value string) { s.SingletonContext.SetNinjaBuildDir(pctx.PackageContext, value) } @@ -192,6 +210,14 @@ func (s *singletonContextAdaptor) VisitAllModulesIf(pred func(Module) bool, visi s.SingletonContext.VisitAllModulesIf(predAdaptor(pred), visitAdaptor(visit)) } +func (s *singletonContextAdaptor) VisitDirectDeps(module Module, visit func(Module)) { + s.SingletonContext.VisitDirectDeps(module, visitAdaptor(visit)) +} + +func (s *singletonContextAdaptor) VisitDirectDepsIf(module Module, pred func(Module) bool, visit func(Module)) { + s.SingletonContext.VisitDirectDepsIf(module, predAdaptor(pred), visitAdaptor(visit)) +} + func (s *singletonContextAdaptor) VisitDepsDepthFirst(module Module, visit func(Module)) { s.SingletonContext.VisitDepsDepthFirst(module, visitAdaptor(visit)) } diff --git a/android/soong_config_modules.go b/android/soong_config_modules.go new file mode 100644 index 000000000..619cf8615 --- /dev/null +++ b/android/soong_config_modules.go @@ -0,0 +1,380 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +// This file provides module types that implement wrapper module types that add conditionals on +// Soong config variables. + +import ( + "fmt" + "path/filepath" + "strings" + "text/scanner" + + "github.com/google/blueprint" + "github.com/google/blueprint/parser" + "github.com/google/blueprint/proptools" + + "android/soong/android/soongconfig" +) + +func init() { + RegisterModuleType("soong_config_module_type_import", soongConfigModuleTypeImportFactory) + RegisterModuleType("soong_config_module_type", soongConfigModuleTypeFactory) + RegisterModuleType("soong_config_string_variable", soongConfigStringVariableDummyFactory) + RegisterModuleType("soong_config_bool_variable", soongConfigBoolVariableDummyFactory) +} + +type soongConfigModuleTypeImport struct { + ModuleBase + properties soongConfigModuleTypeImportProperties +} + +type soongConfigModuleTypeImportProperties struct { + From string + Module_types []string +} + +// soong_config_module_type_import imports module types with conditionals on Soong config +// variables from another Android.bp file. The imported module type will exist for all +// modules after the import in the Android.bp file. +// +// For example, an Android.bp file could have: +// +// soong_config_module_type_import { +// from: "device/acme/Android.bp", +// module_types: ["acme_cc_defaults"], +// } +// +// acme_cc_defaults { +// name: "acme_defaults", +// cflags: ["-DGENERIC"], +// soong_config_variables: { +// board: { +// soc_a: { +// cflags: ["-DSOC_A"], +// }, +// soc_b: { +// cflags: ["-DSOC_B"], +// }, +// }, +// feature: { +// cflags: ["-DFEATURE"], +// }, +// width: { +// cflags: ["-DWIDTH=%s"], +// }, +// }, +// } +// +// cc_library { +// name: "libacme_foo", +// defaults: ["acme_defaults"], +// srcs: ["*.cpp"], +// } +// +// And device/acme/Android.bp could have: +// +// soong_config_module_type { +// name: "acme_cc_defaults", +// module_type: "cc_defaults", +// config_namespace: "acme", +// variables: ["board"], +// bool_variables: ["feature"], +// value_variables: ["width"], +// properties: ["cflags", "srcs"], +// } +// +// soong_config_string_variable { +// name: "board", +// values: ["soc_a", "soc_b"], +// } +// +// If an acme BoardConfig.mk file contained: +// +// SOONG_CONFIG_NAMESPACES += acme +// SOONG_CONFIG_acme += \ +// board \ +// feature \ +// +// SOONG_CONFIG_acme_board := soc_a +// SOONG_CONFIG_acme_feature := true +// SOONG_CONFIG_acme_width := 200 +// +// Then libacme_foo would build with cflags "-DGENERIC -DSOC_A -DFEATURE -DWIDTH=200". +func soongConfigModuleTypeImportFactory() Module { + module := &soongConfigModuleTypeImport{} + + module.AddProperties(&module.properties) + AddLoadHook(module, func(ctx LoadHookContext) { + importModuleTypes(ctx, module.properties.From, module.properties.Module_types...) + }) + + initAndroidModuleBase(module) + return module +} + +func (m *soongConfigModuleTypeImport) Name() string { + // The generated name is non-deterministic, but it does not + // matter because this module does not emit any rules. + return soongconfig.CanonicalizeToProperty(m.properties.From) + + "soong_config_module_type_import_" + fmt.Sprintf("%p", m) +} + +func (*soongConfigModuleTypeImport) Nameless() {} +func (*soongConfigModuleTypeImport) GenerateAndroidBuildActions(ModuleContext) {} + +// Create dummy modules for soong_config_module_type and soong_config_*_variable + +type soongConfigModuleTypeModule struct { + ModuleBase + properties soongconfig.ModuleTypeProperties +} + +// soong_config_module_type defines module types with conditionals on Soong config +// variables. The new module type will exist for all modules after the definition +// in an Android.bp file, and can be imported into other Android.bp files using +// soong_config_module_type_import. +// +// For example, an Android.bp file could have: +// +// soong_config_module_type { +// name: "acme_cc_defaults", +// module_type: "cc_defaults", +// config_namespace: "acme", +// variables: ["board"], +// bool_variables: ["feature"], +// value_variables: ["width"], +// properties: ["cflags", "srcs"], +// } +// +// soong_config_string_variable { +// name: "board", +// values: ["soc_a", "soc_b"], +// } +// +// acme_cc_defaults { +// name: "acme_defaults", +// cflags: ["-DGENERIC"], +// soong_config_variables: { +// board: { +// soc_a: { +// cflags: ["-DSOC_A"], +// }, +// soc_b: { +// cflags: ["-DSOC_B"], +// }, +// }, +// feature: { +// cflags: ["-DFEATURE"], +// }, +// width: { +// cflags: ["-DWIDTH=%s"], +// }, +// }, +// } +// +// cc_library { +// name: "libacme_foo", +// defaults: ["acme_defaults"], +// srcs: ["*.cpp"], +// } +// +// If an acme BoardConfig.mk file contained: +// +// SOONG_CONFIG_NAMESPACES += acme +// SOONG_CONFIG_acme += \ +// board \ +// feature \ +// +// SOONG_CONFIG_acme_board := soc_a +// SOONG_CONFIG_acme_feature := true +// SOONG_CONFIG_acme_width := 200 +// +// Then libacme_foo would build with cflags "-DGENERIC -DSOC_A -DFEATURE". +func soongConfigModuleTypeFactory() Module { + module := &soongConfigModuleTypeModule{} + + module.AddProperties(&module.properties) + + AddLoadHook(module, func(ctx LoadHookContext) { + // A soong_config_module_type module should implicitly import itself. + importModuleTypes(ctx, ctx.BlueprintsFile(), module.properties.Name) + }) + + initAndroidModuleBase(module) + + return module +} + +func (m *soongConfigModuleTypeModule) Name() string { + return m.properties.Name +} +func (*soongConfigModuleTypeModule) Nameless() {} +func (*soongConfigModuleTypeModule) GenerateAndroidBuildActions(ctx ModuleContext) {} + +type soongConfigStringVariableDummyModule struct { + ModuleBase + properties soongconfig.VariableProperties + stringProperties soongconfig.StringVariableProperties +} + +type soongConfigBoolVariableDummyModule struct { + ModuleBase + properties soongconfig.VariableProperties +} + +// soong_config_string_variable defines a variable and a set of possible string values for use +// in a soong_config_module_type definition. +func soongConfigStringVariableDummyFactory() Module { + module := &soongConfigStringVariableDummyModule{} + module.AddProperties(&module.properties, &module.stringProperties) + initAndroidModuleBase(module) + return module +} + +// soong_config_string_variable defines a variable with true or false values for use +// in a soong_config_module_type definition. +func soongConfigBoolVariableDummyFactory() Module { + module := &soongConfigBoolVariableDummyModule{} + module.AddProperties(&module.properties) + initAndroidModuleBase(module) + return module +} + +func (m *soongConfigStringVariableDummyModule) Name() string { + return m.properties.Name +} +func (*soongConfigStringVariableDummyModule) Nameless() {} +func (*soongConfigStringVariableDummyModule) GenerateAndroidBuildActions(ctx ModuleContext) {} + +func (m *soongConfigBoolVariableDummyModule) Name() string { + return m.properties.Name +} +func (*soongConfigBoolVariableDummyModule) Nameless() {} +func (*soongConfigBoolVariableDummyModule) GenerateAndroidBuildActions(ctx ModuleContext) {} + +func importModuleTypes(ctx LoadHookContext, from string, moduleTypes ...string) { + from = filepath.Clean(from) + if filepath.Ext(from) != ".bp" { + ctx.PropertyErrorf("from", "%q must be a file with extension .bp", from) + return + } + + if strings.HasPrefix(from, "../") { + ctx.PropertyErrorf("from", "%q must not use ../ to escape the source tree", + from) + return + } + + moduleTypeDefinitions := loadSoongConfigModuleTypeDefinition(ctx, from) + if moduleTypeDefinitions == nil { + return + } + for _, moduleType := range moduleTypes { + if factory, ok := moduleTypeDefinitions[moduleType]; ok { + ctx.registerScopedModuleType(moduleType, factory) + } else { + ctx.PropertyErrorf("module_types", "module type %q not defined in %q", + moduleType, from) + } + } +} + +// loadSoongConfigModuleTypeDefinition loads module types from an Android.bp file. It caches the +// result so each file is only parsed once. +func loadSoongConfigModuleTypeDefinition(ctx LoadHookContext, from string) map[string]blueprint.ModuleFactory { + type onceKeyType string + key := NewCustomOnceKey(onceKeyType(filepath.Clean(from))) + + reportErrors := func(ctx LoadHookContext, filename string, errs ...error) { + for _, err := range errs { + if parseErr, ok := err.(*parser.ParseError); ok { + ctx.Errorf(parseErr.Pos, "%s", parseErr.Err) + } else { + ctx.Errorf(scanner.Position{Filename: filename}, "%s", err) + } + } + } + + return ctx.Config().Once(key, func() interface{} { + ctx.AddNinjaFileDeps(from) + r, err := ctx.Config().fs.Open(from) + if err != nil { + ctx.PropertyErrorf("from", "failed to open %q: %s", from, err) + return (map[string]blueprint.ModuleFactory)(nil) + } + + mtDef, errs := soongconfig.Parse(r, from) + + if len(errs) > 0 { + reportErrors(ctx, from, errs...) + return (map[string]blueprint.ModuleFactory)(nil) + } + + globalModuleTypes := ctx.moduleFactories() + + factories := make(map[string]blueprint.ModuleFactory) + + for name, moduleType := range mtDef.ModuleTypes { + factory := globalModuleTypes[moduleType.BaseModuleType] + if factory != nil { + factories[name] = soongConfigModuleFactory(factory, moduleType) + } else { + reportErrors(ctx, from, + fmt.Errorf("missing global module type factory for %q", moduleType.BaseModuleType)) + } + } + + if ctx.Failed() { + return (map[string]blueprint.ModuleFactory)(nil) + } + + return factories + }).(map[string]blueprint.ModuleFactory) +} + +// soongConfigModuleFactory takes an existing soongConfigModuleFactory and a ModuleType and returns +// a new soongConfigModuleFactory that wraps the existing soongConfigModuleFactory and adds conditional on Soong config +// variables. +func soongConfigModuleFactory(factory blueprint.ModuleFactory, + moduleType *soongconfig.ModuleType) blueprint.ModuleFactory { + + conditionalFactoryProps := soongconfig.CreateProperties(factory, moduleType) + if conditionalFactoryProps.IsValid() { + return func() (blueprint.Module, []interface{}) { + module, props := factory() + + conditionalProps := proptools.CloneEmptyProperties(conditionalFactoryProps) + props = append(props, conditionalProps.Interface()) + + AddLoadHook(module, func(ctx LoadHookContext) { + config := ctx.Config().VendorConfig(moduleType.ConfigNamespace) + newProps, err := soongconfig.PropertiesToApply(moduleType, conditionalProps, config) + if err != nil { + ctx.ModuleErrorf("%s", err) + return + } + for _, ps := range newProps { + ctx.AppendProperties(ps) + } + }) + + return module, props + } + } else { + return factory + } +} diff --git a/android/soong_config_modules_test.go b/android/soong_config_modules_test.go new file mode 100644 index 000000000..f905b1ab2 --- /dev/null +++ b/android/soong_config_modules_test.go @@ -0,0 +1,143 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "reflect" + "testing" +) + +type soongConfigTestModule struct { + ModuleBase + props soongConfigTestModuleProperties +} + +type soongConfigTestModuleProperties struct { + Cflags []string +} + +func soongConfigTestModuleFactory() Module { + m := &soongConfigTestModule{} + m.AddProperties(&m.props) + InitAndroidModule(m) + return m +} + +func (t soongConfigTestModule) GenerateAndroidBuildActions(ModuleContext) {} + +func TestSoongConfigModule(t *testing.T) { + configBp := ` + soong_config_module_type { + name: "acme_test_defaults", + module_type: "test_defaults", + config_namespace: "acme", + variables: ["board", "feature1", "FEATURE3"], + bool_variables: ["feature2"], + value_variables: ["size"], + properties: ["cflags", "srcs"], + } + + soong_config_string_variable { + name: "board", + values: ["soc_a", "soc_b"], + } + + soong_config_bool_variable { + name: "feature1", + } + + soong_config_bool_variable { + name: "FEATURE3", + } + ` + + importBp := ` + soong_config_module_type_import { + from: "SoongConfig.bp", + module_types: ["acme_test_defaults"], + } + ` + + bp := ` + acme_test_defaults { + name: "foo", + cflags: ["-DGENERIC"], + soong_config_variables: { + board: { + soc_a: { + cflags: ["-DSOC_A"], + }, + soc_b: { + cflags: ["-DSOC_B"], + }, + }, + size: { + cflags: ["-DSIZE=%s"], + }, + feature1: { + cflags: ["-DFEATURE1"], + }, + feature2: { + cflags: ["-DFEATURE2"], + }, + FEATURE3: { + cflags: ["-DFEATURE3"], + }, + }, + } + ` + + run := func(t *testing.T, bp string, fs map[string][]byte) { + config := TestConfig(buildDir, nil, bp, fs) + + config.TestProductVariables.VendorVars = map[string]map[string]string{ + "acme": map[string]string{ + "board": "soc_a", + "size": "42", + "feature1": "true", + "feature2": "false", + // FEATURE3 unset + }, + } + + ctx := NewTestContext() + ctx.RegisterModuleType("soong_config_module_type_import", soongConfigModuleTypeImportFactory) + ctx.RegisterModuleType("soong_config_module_type", soongConfigModuleTypeFactory) + ctx.RegisterModuleType("soong_config_string_variable", soongConfigStringVariableDummyFactory) + ctx.RegisterModuleType("soong_config_bool_variable", soongConfigBoolVariableDummyFactory) + ctx.RegisterModuleType("test_defaults", soongConfigTestModuleFactory) + ctx.Register(config) + + _, errs := ctx.ParseBlueprintsFiles("Android.bp") + FailIfErrored(t, errs) + _, errs = ctx.PrepareBuildActions(config) + FailIfErrored(t, errs) + + foo := ctx.ModuleForTests("foo", "").Module().(*soongConfigTestModule) + if g, w := foo.props.Cflags, []string{"-DGENERIC", "-DSIZE=42", "-DSOC_A", "-DFEATURE1"}; !reflect.DeepEqual(g, w) { + t.Errorf("wanted foo cflags %q, got %q", w, g) + } + } + + t.Run("single file", func(t *testing.T) { + run(t, configBp+bp, nil) + }) + + t.Run("import", func(t *testing.T) { + run(t, importBp+bp, map[string][]byte{ + "SoongConfig.bp": []byte(configBp), + }) + }) +} diff --git a/android/soongconfig/Android.bp b/android/soongconfig/Android.bp new file mode 100644 index 000000000..df912e62e --- /dev/null +++ b/android/soongconfig/Android.bp @@ -0,0 +1,13 @@ +bootstrap_go_package { + name: "soong-android-soongconfig", + pkgPath: "android/soong/android/soongconfig", + deps: [ + "blueprint", + "blueprint-parser", + "blueprint-proptools", + ], + srcs: [ + "config.go", + "modules.go", + ], +} diff --git a/android/soongconfig/config.go b/android/soongconfig/config.go new file mode 100644 index 000000000..39a776c6d --- /dev/null +++ b/android/soongconfig/config.go @@ -0,0 +1,51 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package soongconfig + +import "strings" + +type SoongConfig interface { + // Bool interprets the variable named `name` as a boolean, returning true if, after + // lowercasing, it matches one of "1", "y", "yes", "on", or "true". Unset, or any other + // value will return false. + Bool(name string) bool + + // String returns the string value of `name`. If the variable was not set, it will + // return the empty string. + String(name string) string + + // IsSet returns whether the variable `name` was set by Make. + IsSet(name string) bool +} + +func Config(vars map[string]string) SoongConfig { + return soongConfig(vars) +} + +type soongConfig map[string]string + +func (c soongConfig) Bool(name string) bool { + v := strings.ToLower(c[name]) + return v == "1" || v == "y" || v == "yes" || v == "on" || v == "true" +} + +func (c soongConfig) String(name string) string { + return c[name] +} + +func (c soongConfig) IsSet(name string) bool { + _, ok := c[name] + return ok +} diff --git a/android/soongconfig/modules.go b/android/soongconfig/modules.go new file mode 100644 index 000000000..142a81387 --- /dev/null +++ b/android/soongconfig/modules.go @@ -0,0 +1,622 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package soongconfig + +import ( + "fmt" + "io" + "reflect" + "sort" + "strings" + + "github.com/google/blueprint" + "github.com/google/blueprint/parser" + "github.com/google/blueprint/proptools" +) + +var soongConfigProperty = proptools.FieldNameForProperty("soong_config_variables") + +// loadSoongConfigModuleTypeDefinition loads module types from an Android.bp file. It caches the +// result so each file is only parsed once. +func Parse(r io.Reader, from string) (*SoongConfigDefinition, []error) { + scope := parser.NewScope(nil) + file, errs := parser.ParseAndEval(from, r, scope) + + if len(errs) > 0 { + return nil, errs + } + + mtDef := &SoongConfigDefinition{ + ModuleTypes: make(map[string]*ModuleType), + variables: make(map[string]soongConfigVariable), + } + + for _, def := range file.Defs { + switch def := def.(type) { + case *parser.Module: + newErrs := processImportModuleDef(mtDef, def) + + if len(newErrs) > 0 { + errs = append(errs, newErrs...) + } + + case *parser.Assignment: + // Already handled via Scope object + default: + panic("unknown definition type") + } + } + + if len(errs) > 0 { + return nil, errs + } + + for name, moduleType := range mtDef.ModuleTypes { + for _, varName := range moduleType.variableNames { + if v, ok := mtDef.variables[varName]; ok { + moduleType.Variables = append(moduleType.Variables, v) + } else { + return nil, []error{ + fmt.Errorf("unknown variable %q in module type %q", varName, name), + } + } + } + } + + return mtDef, nil +} + +func processImportModuleDef(v *SoongConfigDefinition, def *parser.Module) (errs []error) { + switch def.Type { + case "soong_config_module_type": + return processModuleTypeDef(v, def) + case "soong_config_string_variable": + return processStringVariableDef(v, def) + case "soong_config_bool_variable": + return processBoolVariableDef(v, def) + default: + // Unknown module types will be handled when the file is parsed as a normal + // Android.bp file. + } + + return nil +} + +type ModuleTypeProperties struct { + // the name of the new module type. Unlike most modules, this name does not need to be unique, + // although only one module type with any name will be importable into an Android.bp file. + Name string + + // the module type that this module type will extend. + Module_type string + + // the SOONG_CONFIG_NAMESPACE value from a BoardConfig.mk that this module type will read + // configuration variables from. + Config_namespace string + + // the list of SOONG_CONFIG variables that this module type will read + Variables []string + + // the list of boolean SOONG_CONFIG variables that this module type will read + Bool_variables []string + + // the list of SOONG_CONFIG variables that this module type will read. The value will be + // inserted into the properties with %s substitution. + Value_variables []string + + // the list of properties that this module type will extend. + Properties []string +} + +func processModuleTypeDef(v *SoongConfigDefinition, def *parser.Module) (errs []error) { + + props := &ModuleTypeProperties{} + + _, errs = proptools.UnpackProperties(def.Properties, props) + if len(errs) > 0 { + return errs + } + + if props.Name == "" { + errs = append(errs, fmt.Errorf("name property must be set")) + } + + if props.Config_namespace == "" { + errs = append(errs, fmt.Errorf("config_namespace property must be set")) + } + + if props.Module_type == "" { + errs = append(errs, fmt.Errorf("module_type property must be set")) + } + + if len(errs) > 0 { + return errs + } + + mt := &ModuleType{ + affectableProperties: props.Properties, + ConfigNamespace: props.Config_namespace, + BaseModuleType: props.Module_type, + variableNames: props.Variables, + } + v.ModuleTypes[props.Name] = mt + + for _, name := range props.Bool_variables { + if name == "" { + return []error{fmt.Errorf("bool_variable name must not be blank")} + } + + mt.Variables = append(mt.Variables, &boolVariable{ + baseVariable: baseVariable{ + variable: name, + }, + }) + } + + for _, name := range props.Value_variables { + if name == "" { + return []error{fmt.Errorf("value_variables entry must not be blank")} + } + + mt.Variables = append(mt.Variables, &valueVariable{ + baseVariable: baseVariable{ + variable: name, + }, + }) + } + + return nil +} + +type VariableProperties struct { + Name string +} + +type StringVariableProperties struct { + Values []string +} + +func processStringVariableDef(v *SoongConfigDefinition, def *parser.Module) (errs []error) { + stringProps := &StringVariableProperties{} + + base, errs := processVariableDef(def, stringProps) + if len(errs) > 0 { + return errs + } + + if len(stringProps.Values) == 0 { + return []error{fmt.Errorf("values property must be set")} + } + + v.variables[base.variable] = &stringVariable{ + baseVariable: base, + values: CanonicalizeToProperties(stringProps.Values), + } + + return nil +} + +func processBoolVariableDef(v *SoongConfigDefinition, def *parser.Module) (errs []error) { + base, errs := processVariableDef(def) + if len(errs) > 0 { + return errs + } + + v.variables[base.variable] = &boolVariable{ + baseVariable: base, + } + + return nil +} + +func processVariableDef(def *parser.Module, + extraProps ...interface{}) (cond baseVariable, errs []error) { + + props := &VariableProperties{} + + allProps := append([]interface{}{props}, extraProps...) + + _, errs = proptools.UnpackProperties(def.Properties, allProps...) + if len(errs) > 0 { + return baseVariable{}, errs + } + + if props.Name == "" { + return baseVariable{}, []error{fmt.Errorf("name property must be set")} + } + + return baseVariable{ + variable: props.Name, + }, nil +} + +type SoongConfigDefinition struct { + ModuleTypes map[string]*ModuleType + + variables map[string]soongConfigVariable +} + +// CreateProperties returns a reflect.Value of a newly constructed type that contains the desired +// property layout for the Soong config variables, with each possible value an interface{} that +// contains a nil pointer to another newly constructed type that contains the affectable properties. +// The reflect.Value will be cloned for each call to the Soong config module type's factory method. +// +// For example, the acme_cc_defaults example above would +// produce a reflect.Value whose type is: +// *struct { +// Soong_config_variables struct { +// Board struct { +// Soc_a interface{} +// Soc_b interface{} +// } +// } +// } +// And whose value is: +// &{ +// Soong_config_variables: { +// Board: { +// Soc_a: (*struct{ Cflags []string })(nil), +// Soc_b: (*struct{ Cflags []string })(nil), +// }, +// }, +// } +func CreateProperties(factory blueprint.ModuleFactory, moduleType *ModuleType) reflect.Value { + var fields []reflect.StructField + + _, factoryProps := factory() + affectablePropertiesType := createAffectablePropertiesType(moduleType.affectableProperties, factoryProps) + if affectablePropertiesType == nil { + return reflect.Value{} + } + + for _, c := range moduleType.Variables { + fields = append(fields, reflect.StructField{ + Name: proptools.FieldNameForProperty(c.variableProperty()), + Type: c.variableValuesType(), + }) + } + + typ := reflect.StructOf([]reflect.StructField{{ + Name: soongConfigProperty, + Type: reflect.StructOf(fields), + }}) + + props := reflect.New(typ) + structConditions := props.Elem().FieldByName(soongConfigProperty) + + for i, c := range moduleType.Variables { + c.initializeProperties(structConditions.Field(i), affectablePropertiesType) + } + + return props +} + +// createAffectablePropertiesType creates a reflect.Type of a struct that has a field for each affectable property +// that exists in factoryProps. +func createAffectablePropertiesType(affectableProperties []string, factoryProps []interface{}) reflect.Type { + affectableProperties = append([]string(nil), affectableProperties...) + sort.Strings(affectableProperties) + + var recurse func(prefix string, aps []string) ([]string, reflect.Type) + recurse = func(prefix string, aps []string) ([]string, reflect.Type) { + var fields []reflect.StructField + + for len(affectableProperties) > 0 { + p := affectableProperties[0] + if !strings.HasPrefix(affectableProperties[0], prefix) { + break + } + affectableProperties = affectableProperties[1:] + + nestedProperty := strings.TrimPrefix(p, prefix) + if i := strings.IndexRune(nestedProperty, '.'); i >= 0 { + var nestedType reflect.Type + nestedPrefix := nestedProperty[:i+1] + + affectableProperties, nestedType = recurse(prefix+nestedPrefix, affectableProperties) + + if nestedType != nil { + nestedFieldName := proptools.FieldNameForProperty(strings.TrimSuffix(nestedPrefix, ".")) + + fields = append(fields, reflect.StructField{ + Name: nestedFieldName, + Type: nestedType, + }) + } + } else { + typ := typeForPropertyFromPropertyStructs(factoryProps, p) + if typ != nil { + fields = append(fields, reflect.StructField{ + Name: proptools.FieldNameForProperty(nestedProperty), + Type: typ, + }) + } + } + } + + var typ reflect.Type + if len(fields) > 0 { + typ = reflect.StructOf(fields) + } + return affectableProperties, typ + } + + affectableProperties, typ := recurse("", affectableProperties) + if len(affectableProperties) > 0 { + panic(fmt.Errorf("didn't handle all affectable properties")) + } + + if typ != nil { + return reflect.PtrTo(typ) + } + + return nil +} + +func typeForPropertyFromPropertyStructs(psList []interface{}, property string) reflect.Type { + for _, ps := range psList { + if typ := typeForPropertyFromPropertyStruct(ps, property); typ != nil { + return typ + } + } + + return nil +} + +func typeForPropertyFromPropertyStruct(ps interface{}, property string) reflect.Type { + v := reflect.ValueOf(ps) + for len(property) > 0 { + if !v.IsValid() { + return nil + } + + if v.Kind() == reflect.Interface { + if v.IsNil() { + return nil + } else { + v = v.Elem() + } + } + + if v.Kind() == reflect.Ptr { + if v.IsNil() { + v = reflect.Zero(v.Type().Elem()) + } else { + v = v.Elem() + } + } + + if v.Kind() != reflect.Struct { + return nil + } + + if index := strings.IndexRune(property, '.'); index >= 0 { + prefix := property[:index] + property = property[index+1:] + + v = v.FieldByName(proptools.FieldNameForProperty(prefix)) + } else { + f := v.FieldByName(proptools.FieldNameForProperty(property)) + if !f.IsValid() { + return nil + } + return f.Type() + } + } + return nil +} + +// PropertiesToApply returns the applicable properties from a ModuleType that should be applied +// based on SoongConfig values. +func PropertiesToApply(moduleType *ModuleType, props reflect.Value, config SoongConfig) ([]interface{}, error) { + var ret []interface{} + props = props.Elem().FieldByName(soongConfigProperty) + for i, c := range moduleType.Variables { + if ps, err := c.PropertiesToApply(config, props.Field(i)); err != nil { + return nil, err + } else if ps != nil { + ret = append(ret, ps) + } + } + return ret, nil +} + +type ModuleType struct { + BaseModuleType string + ConfigNamespace string + Variables []soongConfigVariable + + affectableProperties []string + variableNames []string +} + +type soongConfigVariable interface { + // variableProperty returns the name of the variable. + variableProperty() string + + // conditionalValuesType returns a reflect.Type that contains an interface{} for each possible value. + variableValuesType() reflect.Type + + // initializeProperties is passed a reflect.Value of the reflect.Type returned by conditionalValuesType and a + // reflect.Type of the affectable properties, and should initialize each interface{} in the reflect.Value with + // the zero value of the affectable properties type. + initializeProperties(v reflect.Value, typ reflect.Type) + + // PropertiesToApply should return one of the interface{} values set by initializeProperties to be applied + // to the module. + PropertiesToApply(config SoongConfig, values reflect.Value) (interface{}, error) +} + +type baseVariable struct { + variable string +} + +func (c *baseVariable) variableProperty() string { + return CanonicalizeToProperty(c.variable) +} + +type stringVariable struct { + baseVariable + values []string +} + +func (s *stringVariable) variableValuesType() reflect.Type { + var fields []reflect.StructField + + for _, v := range s.values { + fields = append(fields, reflect.StructField{ + Name: proptools.FieldNameForProperty(v), + Type: emptyInterfaceType, + }) + } + + return reflect.StructOf(fields) +} + +func (s *stringVariable) initializeProperties(v reflect.Value, typ reflect.Type) { + for i := range s.values { + v.Field(i).Set(reflect.Zero(typ)) + } +} + +func (s *stringVariable) PropertiesToApply(config SoongConfig, values reflect.Value) (interface{}, error) { + for j, v := range s.values { + if config.String(s.variable) == v { + return values.Field(j).Interface(), nil + } + } + + return nil, nil +} + +type boolVariable struct { + baseVariable +} + +func (b boolVariable) variableValuesType() reflect.Type { + return emptyInterfaceType +} + +func (b boolVariable) initializeProperties(v reflect.Value, typ reflect.Type) { + v.Set(reflect.Zero(typ)) +} + +func (b boolVariable) PropertiesToApply(config SoongConfig, values reflect.Value) (interface{}, error) { + if config.Bool(b.variable) { + return values.Interface(), nil + } + + return nil, nil +} + +type valueVariable struct { + baseVariable +} + +func (s *valueVariable) variableValuesType() reflect.Type { + return emptyInterfaceType +} + +func (s *valueVariable) initializeProperties(v reflect.Value, typ reflect.Type) { + v.Set(reflect.Zero(typ)) +} + +func (s *valueVariable) PropertiesToApply(config SoongConfig, values reflect.Value) (interface{}, error) { + if !config.IsSet(s.variable) { + return nil, nil + } + configValue := config.String(s.variable) + + propStruct := values.Elem().Elem() + for i := 0; i < propStruct.NumField(); i++ { + field := propStruct.Field(i) + kind := field.Kind() + if kind == reflect.Ptr { + if field.IsNil() { + continue + } + field = field.Elem() + } + switch kind { + case reflect.String: + err := printfIntoProperty(field, configValue) + if err != nil { + return nil, fmt.Errorf("soong_config_variables.%s.%s: %s", s.variable, propStruct.Type().Field(i).Name, err) + } + case reflect.Slice: + for j := 0; j < field.Len(); j++ { + err := printfIntoProperty(field.Index(j), configValue) + if err != nil { + return nil, fmt.Errorf("soong_config_variables.%s.%s: %s", s.variable, propStruct.Type().Field(i).Name, err) + } + } + case reflect.Bool: + // Nothing to do + default: + return nil, fmt.Errorf("soong_config_variables.%s.%s: unsupported property type %q", s.variable, propStruct.Type().Field(i).Name, kind) + } + } + + return values.Interface(), nil +} + +func printfIntoProperty(propertyValue reflect.Value, configValue string) error { + s := propertyValue.String() + + count := strings.Count(s, "%") + if count == 0 { + return nil + } + + if count > 1 { + return fmt.Errorf("value variable properties only support a single '%%'") + } + + if !strings.Contains(s, "%s") { + return fmt.Errorf("unsupported %% in value variable property") + } + + propertyValue.Set(reflect.ValueOf(fmt.Sprintf(s, configValue))) + + return nil +} + +func CanonicalizeToProperty(v string) string { + return strings.Map(func(r rune) rune { + switch { + case r >= 'A' && r <= 'Z', + r >= 'a' && r <= 'z', + r >= '0' && r <= '9', + r == '_': + return r + default: + return '_' + } + }, v) +} + +func CanonicalizeToProperties(values []string) []string { + ret := make([]string, len(values)) + for i, v := range values { + ret[i] = CanonicalizeToProperty(v) + } + return ret +} + +type emptyInterfaceStruct struct { + i interface{} +} + +var emptyInterfaceType = reflect.TypeOf(emptyInterfaceStruct{}).Field(0).Type diff --git a/android/soongconfig/modules_test.go b/android/soongconfig/modules_test.go new file mode 100644 index 000000000..419001670 --- /dev/null +++ b/android/soongconfig/modules_test.go @@ -0,0 +1,249 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package soongconfig + +import ( + "reflect" + "testing" +) + +func Test_CanonicalizeToProperty(t *testing.T) { + tests := []struct { + name string + arg string + want string + }{ + { + name: "lowercase", + arg: "board", + want: "board", + }, + { + name: "uppercase", + arg: "BOARD", + want: "BOARD", + }, + { + name: "numbers", + arg: "BOARD123", + want: "BOARD123", + }, + { + name: "underscore", + arg: "TARGET_BOARD", + want: "TARGET_BOARD", + }, + { + name: "dash", + arg: "TARGET-BOARD", + want: "TARGET_BOARD", + }, + { + name: "unicode", + arg: "boardλ", + want: "board_", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := CanonicalizeToProperty(tt.arg); got != tt.want { + t.Errorf("canonicalizeToProperty() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_typeForPropertyFromPropertyStruct(t *testing.T) { + tests := []struct { + name string + ps interface{} + property string + want string + }{ + { + name: "string", + ps: struct { + A string + }{}, + property: "a", + want: "string", + }, + { + name: "list", + ps: struct { + A []string + }{}, + property: "a", + want: "[]string", + }, + { + name: "missing", + ps: struct { + A []string + }{}, + property: "b", + want: "", + }, + { + name: "nested", + ps: struct { + A struct { + B string + } + }{}, + property: "a.b", + want: "string", + }, + { + name: "missing nested", + ps: struct { + A struct { + B string + } + }{}, + property: "a.c", + want: "", + }, + { + name: "not a struct", + ps: struct { + A string + }{}, + property: "a.b", + want: "", + }, + { + name: "nested pointer", + ps: struct { + A *struct { + B string + } + }{}, + property: "a.b", + want: "string", + }, + { + name: "nested interface", + ps: struct { + A interface{} + }{ + A: struct { + B string + }{}, + }, + property: "a.b", + want: "string", + }, + { + name: "nested interface pointer", + ps: struct { + A interface{} + }{ + A: &struct { + B string + }{}, + }, + property: "a.b", + want: "string", + }, + { + name: "nested interface nil pointer", + ps: struct { + A interface{} + }{ + A: (*struct { + B string + })(nil), + }, + property: "a.b", + want: "string", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + typ := typeForPropertyFromPropertyStruct(tt.ps, tt.property) + got := "" + if typ != nil { + got = typ.String() + } + if got != tt.want { + t.Errorf("typeForPropertyFromPropertyStruct() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_createAffectablePropertiesType(t *testing.T) { + tests := []struct { + name string + affectableProperties []string + factoryProps interface{} + want string + }{ + { + name: "string", + affectableProperties: []string{"cflags"}, + factoryProps: struct { + Cflags string + }{}, + want: "*struct { Cflags string }", + }, + { + name: "list", + affectableProperties: []string{"cflags"}, + factoryProps: struct { + Cflags []string + }{}, + want: "*struct { Cflags []string }", + }, + { + name: "string pointer", + affectableProperties: []string{"cflags"}, + factoryProps: struct { + Cflags *string + }{}, + want: "*struct { Cflags *string }", + }, + { + name: "subset", + affectableProperties: []string{"cflags"}, + factoryProps: struct { + Cflags string + Ldflags string + }{}, + want: "*struct { Cflags string }", + }, + { + name: "none", + affectableProperties: []string{"cflags"}, + factoryProps: struct { + Ldflags string + }{}, + want: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + typ := createAffectablePropertiesType(tt.affectableProperties, []interface{}{tt.factoryProps}) + got := "" + if typ != nil { + got = typ.String() + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("createAffectablePropertiesType() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/android/testing.go b/android/testing.go index 0ec5af58a..90989ef53 100644 --- a/android/testing.go +++ b/android/testing.go @@ -50,14 +50,20 @@ func NewTestArchContext() *TestContext { type TestContext struct { *Context - preArch, preDeps, postDeps []RegisterMutatorFunc - NameResolver *NameResolver + preArch, preDeps, postDeps, finalDeps []RegisterMutatorFunc + NameResolver *NameResolver + config Config } func (ctx *TestContext) PreArchMutators(f RegisterMutatorFunc) { ctx.preArch = append(ctx.preArch, f) } +func (ctx *TestContext) HardCodedPreArchMutators(f RegisterMutatorFunc) { + // Register mutator function as normal for testing. + ctx.PreArchMutators(f) +} + func (ctx *TestContext) PreDepsMutators(f RegisterMutatorFunc) { ctx.preDeps = append(ctx.preDeps, f) } @@ -66,10 +72,40 @@ func (ctx *TestContext) PostDepsMutators(f RegisterMutatorFunc) { ctx.postDeps = append(ctx.postDeps, f) } -func (ctx *TestContext) Register() { - registerMutators(ctx.Context.Context, ctx.preArch, ctx.preDeps, ctx.postDeps) +func (ctx *TestContext) FinalDepsMutators(f RegisterMutatorFunc) { + ctx.finalDeps = append(ctx.finalDeps, f) +} + +func (ctx *TestContext) Register(config Config) { + ctx.SetFs(config.fs) + if config.mockBpList != "" { + ctx.SetModuleListFile(config.mockBpList) + } + registerMutators(ctx.Context.Context, ctx.preArch, ctx.preDeps, ctx.postDeps, ctx.finalDeps) + + ctx.RegisterSingletonType("env", EnvSingleton) + + ctx.config = config +} + +func (ctx *TestContext) ParseFileList(rootDir string, filePaths []string) (deps []string, errs []error) { + // This function adapts the old style ParseFileList calls that are spread throughout the tests + // to the new style that takes a config. + return ctx.Context.ParseFileList(rootDir, filePaths, ctx.config) +} + +func (ctx *TestContext) ParseBlueprintsFiles(rootDir string) (deps []string, errs []error) { + // This function adapts the old style ParseBlueprintsFiles calls that are spread throughout the + // tests to the new style that takes a config. + return ctx.Context.ParseBlueprintsFiles(rootDir, ctx.config) +} + +func (ctx *TestContext) RegisterModuleType(name string, factory ModuleFactory) { + ctx.Context.RegisterModuleType(name, ModuleFactoryAdaptor(factory)) +} - ctx.RegisterSingletonType("env", SingletonFactoryAdaptor(EnvSingleton)) +func (ctx *TestContext) RegisterSingletonType(name string, factory SingletonFactory) { + ctx.Context.RegisterSingletonType(name, SingletonFactoryAdaptor(factory)) } func (ctx *TestContext) ModuleForTests(name, variant string) TestingModule { @@ -122,25 +158,6 @@ func (ctx *TestContext) SingletonForTests(name string) TestingSingleton { "\nall singletons: %v", name, allSingletonNames)) } -// MockFileSystem causes the Context to replace all reads with accesses to the provided map of -// filenames to contents stored as a byte slice. -func (ctx *TestContext) MockFileSystem(files map[string][]byte) { - // no module list file specified; find every file named Blueprints or Android.bp - pathsToParse := []string{} - for candidate := range files { - base := filepath.Base(candidate) - if base == "Blueprints" || base == "Android.bp" { - pathsToParse = append(pathsToParse, candidate) - } - } - if len(pathsToParse) < 1 { - panic(fmt.Sprintf("No Blueprint or Android.bp files found in mock filesystem: %v\n", files)) - } - files[blueprint.MockModuleListFile] = []byte(strings.Join(pathsToParse, "\n")) - - ctx.Context.MockFileSystem(files) -} - type testBuildProvider interface { BuildParamsForTests() []BuildParams RuleParamsForTests() map[blueprint.Rule]blueprint.RuleParams @@ -177,7 +194,7 @@ func buildParamsFromRule(provider testBuildProvider, rule string) TestingBuildPa func maybeBuildParamsFromDescription(provider testBuildProvider, desc string) TestingBuildParams { for _, p := range provider.BuildParamsForTests() { - if p.Description == desc { + if strings.Contains(p.Description, desc) { return newTestingBuildParams(provider, p) } } @@ -369,3 +386,85 @@ func FailIfNoMatchingErrors(t *testing.T, pattern string, errs []error) { } } } + +func CheckErrorsAgainstExpectations(t *testing.T, errs []error, expectedErrorPatterns []string) { + t.Helper() + + if expectedErrorPatterns == nil { + FailIfErrored(t, errs) + } else { + for _, expectedError := range expectedErrorPatterns { + FailIfNoMatchingErrors(t, expectedError, errs) + } + if len(errs) > len(expectedErrorPatterns) { + t.Errorf("additional errors found, expected %d, found %d", + len(expectedErrorPatterns), len(errs)) + for i, expectedError := range expectedErrorPatterns { + t.Errorf("expectedErrors[%d] = %s", i, expectedError) + } + for i, err := range errs { + t.Errorf("errs[%d] = %s", i, err) + } + } + } + +} + +func SetInMakeForTests(config Config) { + config.inMake = true +} + +func AndroidMkEntriesForTest(t *testing.T, config Config, bpPath string, mod blueprint.Module) []AndroidMkEntries { + var p AndroidMkEntriesProvider + var ok bool + if p, ok = mod.(AndroidMkEntriesProvider); !ok { + t.Errorf("module does not implement AndroidMkEntriesProvider: " + mod.Name()) + } + + entriesList := p.AndroidMkEntries() + for i, _ := range entriesList { + entriesList[i].fillInEntries(config, bpPath, mod) + } + return entriesList +} + +func AndroidMkDataForTest(t *testing.T, config Config, bpPath string, mod blueprint.Module) AndroidMkData { + var p AndroidMkDataProvider + var ok bool + if p, ok = mod.(AndroidMkDataProvider); !ok { + t.Errorf("module does not implement AndroidMkDataProvider: " + mod.Name()) + } + data := p.AndroidMk() + data.fillInData(config, bpPath, mod) + return data +} + +// Normalize the path for testing. +// +// If the path is relative to the build directory then return the relative path +// to avoid tests having to deal with the dynamically generated build directory. +// +// Otherwise, return the supplied path as it is almost certainly a source path +// that is relative to the root of the source tree. +// +// The build and source paths should be distinguishable based on their contents. +func NormalizePathForTesting(path Path) string { + p := path.String() + if w, ok := path.(WritablePath); ok { + rel, err := filepath.Rel(w.buildDir(), p) + if err != nil { + panic(err) + } + return rel + } + return p +} + +func NormalizePathsForTesting(paths Paths) []string { + var result []string + for _, path := range paths { + relative := NormalizePathForTesting(path) + result = append(result, relative) + } + return result +} diff --git a/android/util.go b/android/util.go index b17422df3..ade851eff 100644 --- a/android/util.go +++ b/android/util.go @@ -16,6 +16,8 @@ package android import ( "fmt" + "path/filepath" + "reflect" "regexp" "runtime" "sort" @@ -52,10 +54,54 @@ func JoinWithPrefix(strs []string, prefix string) string { return string(ret) } -func sortedKeys(m map[string][]string) []string { - s := make([]string, 0, len(m)) - for k := range m { - s = append(s, k) +func JoinWithSuffix(strs []string, suffix string, separator string) string { + if len(strs) == 0 { + return "" + } + + if len(strs) == 1 { + return strs[0] + suffix + } + + n := len(" ") * (len(strs) - 1) + for _, s := range strs { + n += len(suffix) + len(s) + } + + ret := make([]byte, 0, n) + for i, s := range strs { + if i != 0 { + ret = append(ret, separator...) + } + ret = append(ret, s...) + ret = append(ret, suffix...) + } + return string(ret) +} + +func SortedStringKeys(m interface{}) []string { + v := reflect.ValueOf(m) + if v.Kind() != reflect.Map { + panic(fmt.Sprintf("%#v is not a map", m)) + } + keys := v.MapKeys() + s := make([]string, 0, len(keys)) + for _, key := range keys { + s = append(s, key.String()) + } + sort.Strings(s) + return s +} + +func SortedStringMapValues(m interface{}) []string { + v := reflect.ValueOf(m) + if v.Kind() != reflect.Map { + panic(fmt.Sprintf("%#v is not a map", m)) + } + keys := v.MapKeys() + s := make([]string, 0, len(keys)) + for _, key := range keys { + s = append(s, v.MapIndex(key).String()) } sort.Strings(s) return s @@ -75,8 +121,9 @@ func InList(s string, list []string) bool { return IndexList(s, list) != -1 } -func PrefixInList(s string, list []string) bool { - for _, prefix := range list { +// Returns true if the given string s is prefixed with any string in the given prefix list. +func HasAnyPrefix(s string, prefixList []string) bool { + for _, prefix := range prefixList { if strings.HasPrefix(s, prefix) { return true } @@ -84,6 +131,27 @@ func PrefixInList(s string, list []string) bool { return false } +// Returns true if any string in the given list has the given prefix. +func PrefixInList(list []string, prefix string) bool { + for _, s := range list { + if strings.HasPrefix(s, prefix) { + return true + } + } + return false +} + +// IndexListPred returns the index of the element which in the given `list` satisfying the predicate, or -1 if there is no such element. +func IndexListPred(pred func(s string) bool, list []string) int { + for i, l := range list { + if pred(l) { + return i + } + } + + return -1 +} + func FilterList(list []string, filter []string) (remainder []string, filtered []string) { for _, l := range list { if InList(l, filter) { @@ -157,6 +225,13 @@ func LastUniqueStrings(list []string) []string { return list[totalSkip:] } +// SortedUniqueStrings returns what the name says +func SortedUniqueStrings(list []string) []string { + unique := FirstUniqueStrings(list) + sort.Strings(unique) + return unique +} + // checkCalledFromInit panics if a Go package's init function is not on the // call stack. func checkCalledFromInit() { @@ -244,6 +319,32 @@ func matchPattern(pat, str string) bool { return strings.HasPrefix(str, pat[:i]) && strings.HasSuffix(str, pat[i+1:]) } +var shlibVersionPattern = regexp.MustCompile("(?:\\.\\d+(?:svn)?)+") + +// splitFileExt splits a file name into root, suffix and ext. root stands for the file name without +// the file extension and the version number (e.g. "libexample"). suffix stands for the +// concatenation of the file extension and the version number (e.g. ".so.1.0"). ext stands for the +// file extension after the version numbers are trimmed (e.g. ".so"). +func SplitFileExt(name string) (string, string, string) { + // Extract and trim the shared lib version number if the file name ends with dot digits. + suffix := "" + matches := shlibVersionPattern.FindAllStringIndex(name, -1) + if len(matches) > 0 { + lastMatch := matches[len(matches)-1] + if lastMatch[1] == len(name) { + suffix = name[lastMatch[0]:lastMatch[1]] + name = name[0:lastMatch[0]] + } + } + + // Extract the file name root and the file extension. + ext := filepath.Ext(name) + root := strings.TrimSuffix(name, ext) + suffix = ext + suffix + + return root, suffix, ext +} + // ShardPaths takes a Paths, and returns a slice of Paths where each one has at most shardSize paths. func ShardPaths(paths Paths, shardSize int) []Paths { if len(paths) == 0 { @@ -276,3 +377,14 @@ func ShardStrings(s []string, shardSize int) [][]string { } return ret } + +func CheckDuplicate(values []string) (duplicate string, found bool) { + seen := make(map[string]string) + for _, v := range values { + if duplicate, found = seen[v]; found { + return + } + seen[v] = v + } + return +} diff --git a/android/util_test.go b/android/util_test.go index 7f0d331c2..1f9ca361c 100644 --- a/android/util_test.go +++ b/android/util_test.go @@ -252,7 +252,7 @@ func TestPrefixInList(t *testing.T) { for _, testCase := range testcases { t.Run(testCase.str, func(t *testing.T) { - out := PrefixInList(testCase.str, prefixes) + out := HasAnyPrefix(testCase.str, prefixes) if out != testCase.expected { t.Errorf("incorrect output:") t.Errorf(" str: %#v", testCase.str) @@ -405,6 +405,71 @@ func ExampleCopyOf_append() { // c = ["foo" "baz"] } +func TestSplitFileExt(t *testing.T) { + t.Run("soname with version", func(t *testing.T) { + root, suffix, ext := SplitFileExt("libtest.so.1.0.30") + expected := "libtest" + if root != expected { + t.Errorf("root should be %q but got %q", expected, root) + } + expected = ".so.1.0.30" + if suffix != expected { + t.Errorf("suffix should be %q but got %q", expected, suffix) + } + expected = ".so" + if ext != expected { + t.Errorf("ext should be %q but got %q", expected, ext) + } + }) + + t.Run("soname with svn version", func(t *testing.T) { + root, suffix, ext := SplitFileExt("libtest.so.1svn") + expected := "libtest" + if root != expected { + t.Errorf("root should be %q but got %q", expected, root) + } + expected = ".so.1svn" + if suffix != expected { + t.Errorf("suffix should be %q but got %q", expected, suffix) + } + expected = ".so" + if ext != expected { + t.Errorf("ext should be %q but got %q", expected, ext) + } + }) + + t.Run("version numbers in the middle should be ignored", func(t *testing.T) { + root, suffix, ext := SplitFileExt("libtest.1.0.30.so") + expected := "libtest.1.0.30" + if root != expected { + t.Errorf("root should be %q but got %q", expected, root) + } + expected = ".so" + if suffix != expected { + t.Errorf("suffix should be %q but got %q", expected, suffix) + } + expected = ".so" + if ext != expected { + t.Errorf("ext should be %q but got %q", expected, ext) + } + }) + + t.Run("no known file extension", func(t *testing.T) { + root, suffix, ext := SplitFileExt("test.exe") + expected := "test" + if root != expected { + t.Errorf("root should be %q but got %q", expected, root) + } + expected = ".exe" + if suffix != expected { + t.Errorf("suffix should be %q but got %q", expected, suffix) + } + if ext != expected { + t.Errorf("ext should be %q but got %q", expected, ext) + } + }) +} + func Test_Shard(t *testing.T) { type args struct { strings []string diff --git a/android/variable.go b/android/variable.go index a88fc9d51..983c23567 100644 --- a/android/variable.go +++ b/android/variable.go @@ -25,7 +25,7 @@ import ( func init() { PreDepsMutators(func(ctx RegisterMutatorsContext) { - ctx.BottomUp("variable", variableMutator).Parallel() + ctx.BottomUp("variable", VariableMutator).Parallel() }) } @@ -43,8 +43,10 @@ type variableProperties struct { } `android:"arch_variant"` Malloc_not_svelte struct { - Cflags []string `android:"arch_variant"` - Shared_libs []string `android:"arch_variant"` + Cflags []string `android:"arch_variant"` + Shared_libs []string `android:"arch_variant"` + Whole_static_libs []string `android:"arch_variant"` + Exclude_static_libs []string `android:"arch_variant"` } `android:"arch_variant"` Safestack struct { @@ -59,17 +61,6 @@ type variableProperties struct { Cflags []string } - // Product_is_iot is true for Android Things devices. - Product_is_iot struct { - Cflags []string - Enabled bool - Exclude_srcs []string - Init_rc []string - Shared_libs []string - Srcs []string - Static_libs []string - } - // treble_linker_namespaces is true when the system/vendor linker namespace separation is // enabled. Treble_linker_namespaces struct { @@ -85,10 +76,12 @@ type variableProperties struct { // are used for dogfooding and performance testing, and should be as similar to user builds // as possible. Debuggable struct { - Cflags []string - Cppflags []string - Init_rc []string - Required []string + Cflags []string + Cppflags []string + Init_rc []string + Required []string + Host_required []string + Target_required []string } // eng is true for -eng builds, and can be used to turn on additionaly heavyweight debugging @@ -124,25 +117,37 @@ type variableProperties struct { Static_libs []string Srcs []string } + + Flatten_apex struct { + Enabled *bool + } + + Experimental_mte struct { + Cflags []string `android:"arch_variant"` + } `android:"arch_variant"` + + Native_coverage struct { + Src *string `android:"arch_variant"` + Srcs []string `android:"arch_variant"` + Exclude_srcs []string `android:"arch_variant"` + } `android:"arch_variant"` } `android:"arch_variant"` } -var zeroProductVariables variableProperties +var defaultProductVariables interface{} = variableProperties{} type productVariables struct { // Suffix to add to generated Makefiles Make_suffix *string `json:",omitempty"` - BuildId *string `json:",omitempty"` - BuildNumberFromFile *string `json:",omitempty"` - DateFromFile *string `json:",omitempty"` + BuildId *string `json:",omitempty"` + BuildNumberFile *string `json:",omitempty"` Platform_version_name *string `json:",omitempty"` Platform_sdk_version *int `json:",omitempty"` Platform_sdk_codename *string `json:",omitempty"` Platform_sdk_final *bool `json:",omitempty"` Platform_version_active_codenames []string `json:",omitempty"` - Platform_version_future_codenames []string `json:",omitempty"` Platform_vndk_version *string `json:",omitempty"` Platform_systemsdk_versions []string `json:",omitempty"` Platform_security_patch *string `json:",omitempty"` @@ -163,6 +168,18 @@ type productVariables struct { DeviceSecondaryCpuVariant *string `json:",omitempty"` DeviceSecondaryAbi []string `json:",omitempty"` + NativeBridgeArch *string `json:",omitempty"` + NativeBridgeArchVariant *string `json:",omitempty"` + NativeBridgeCpuVariant *string `json:",omitempty"` + NativeBridgeAbi []string `json:",omitempty"` + NativeBridgeRelativePath *string `json:",omitempty"` + + NativeBridgeSecondaryArch *string `json:",omitempty"` + NativeBridgeSecondaryArchVariant *string `json:",omitempty"` + NativeBridgeSecondaryCpuVariant *string `json:",omitempty"` + NativeBridgeSecondaryAbi []string `json:",omitempty"` + NativeBridgeSecondaryRelativePath *string `json:",omitempty"` + HostArch *string `json:",omitempty"` HostSecondaryArch *string `json:",omitempty"` @@ -170,9 +187,11 @@ type productVariables struct { CrossHostArch *string `json:",omitempty"` CrossHostSecondaryArch *string `json:",omitempty"` - DeviceResourceOverlays []string `json:",omitempty"` - ProductResourceOverlays []string `json:",omitempty"` - EnforceRROTargets []string `json:",omitempty"` + DeviceResourceOverlays []string `json:",omitempty"` + ProductResourceOverlays []string `json:",omitempty"` + EnforceRROTargets []string `json:",omitempty"` + // TODO(b/150820813) Some modules depend on static overlay, remove this after eliminating the dependency. + EnforceRROExemptedTargets []string `json:",omitempty"` EnforceRROExcludedOverlays []string `json:",omitempty"` AAPTCharacteristics *string `json:",omitempty"` @@ -211,7 +230,8 @@ type productVariables struct { UncompressPrivAppDex *bool `json:",omitempty"` ModulesLoadedByPrivilegedModules []string `json:",omitempty"` - BootJars []string `json:",omitempty"` + BootJars []string `json:",omitempty"` + UpdatableBootJars []string `json:",omitempty"` IntegerOverflowExcludePaths []string `json:",omitempty"` @@ -221,20 +241,28 @@ type productVariables struct { DisableScudo *bool `json:",omitempty"` - EnableXOM *bool `json:",omitempty"` - XOMExcludePaths []string `json:",omitempty"` + Experimental_mte *bool `json:",omitempty"` - VendorPath *string `json:",omitempty"` - OdmPath *string `json:",omitempty"` - ProductPath *string `json:",omitempty"` - ProductServicesPath *string `json:",omitempty"` + VendorPath *string `json:",omitempty"` + OdmPath *string `json:",omitempty"` + ProductPath *string `json:",omitempty"` + SystemExtPath *string `json:",omitempty"` ClangTidy *bool `json:",omitempty"` TidyChecks *string `json:",omitempty"` - NativeCoverage *bool `json:",omitempty"` - CoveragePaths []string `json:",omitempty"` - CoverageExcludePaths []string `json:",omitempty"` + SamplingPGO *bool `json:",omitempty"` + + JavaCoveragePaths []string `json:",omitempty"` + JavaCoverageExcludePaths []string `json:",omitempty"` + + GcovCoverage *bool `json:",omitempty"` + ClangCoverage *bool `json:",omitempty"` + NativeCoveragePaths []string `json:",omitempty"` + NativeCoverageExcludePaths []string `json:",omitempty"` + + // Set by NewConfig + Native_coverage *bool DevicePrefer32BitApps *bool `json:",omitempty"` DevicePrefer32BitExecutables *bool `json:",omitempty"` @@ -251,8 +279,6 @@ type productVariables struct { Override_rs_driver *string `json:",omitempty"` - Product_is_iot *bool `json:",omitempty"` - Fuchsia *bool `json:",omitempty"` DeviceKernelHeaders []string `json:",omitempty"` @@ -263,19 +289,24 @@ type productVariables struct { PgoAdditionalProfileDirs []string `json:",omitempty"` - VndkUseCoreVariant *bool `json:",omitempty"` + VndkUseCoreVariant *bool `json:",omitempty"` + VndkSnapshotBuildArtifacts *bool `json:",omitempty"` BoardVendorSepolicyDirs []string `json:",omitempty"` BoardOdmSepolicyDirs []string `json:",omitempty"` BoardPlatPublicSepolicyDirs []string `json:",omitempty"` BoardPlatPrivateSepolicyDirs []string `json:",omitempty"` + BoardSepolicyM4Defs []string `json:",omitempty"` + + BoardVndkRuntimeDisable *bool `json:",omitempty"` VendorVars map[string]map[string]string `json:",omitempty"` Ndk_abis *bool `json:",omitempty"` Exclude_draft_ndk_apis *bool `json:",omitempty"` - FlattenApex *bool `json:",omitempty"` + Flatten_apex *bool `json:",omitempty"` + Aml_abis *bool `json:",omitempty"` DexpreoptGlobalConfig *string `json:",omitempty"` @@ -284,13 +315,27 @@ type productVariables struct { PackageNameOverrides []string `json:",omitempty"` EnforceSystemCertificate *bool `json:",omitempty"` - EnforceSystemCertificateWhitelist []string `json:",omitempty"` + EnforceSystemCertificateAllowList []string `json:",omitempty"` ProductHiddenAPIStubs []string `json:",omitempty"` ProductHiddenAPIStubsSystem []string `json:",omitempty"` ProductHiddenAPIStubsTest []string `json:",omitempty"` + ProductPublicSepolicyDirs []string `json:",omitempty"` + ProductPrivateSepolicyDirs []string `json:",omitempty"` + ProductCompatibleProperty *bool `json:",omitempty"` + + ProductVndkVersion *string `json:",omitempty"` + TargetFSConfigGen []string `json:",omitempty"` + + MissingUsesLibraries []string `json:",omitempty"` + + EnforceProductPartitionInterface *bool `json:",omitempty"` + + InstallExtraFlattenedApexes *bool `json:",omitempty"` + + BoardUsesRecoveryAsBoot *bool `json:",omitempty"` } func boolPtr(v bool) *bool { @@ -307,9 +352,14 @@ func stringPtr(v string) *string { func (v *productVariables) SetDefaultConfig() { *v = productVariables{ - Platform_sdk_version: intPtr(26), - Platform_version_active_codenames: []string{"P"}, - Platform_version_future_codenames: []string{"P"}, + BuildNumberFile: stringPtr("build_number.txt"), + + Platform_version_name: stringPtr("Q"), + Platform_sdk_version: intPtr(28), + Platform_sdk_codename: stringPtr("Q"), + Platform_sdk_final: boolPtr(false), + Platform_version_active_codenames: []string{"Q"}, + Platform_vndk_version: stringPtr("Q"), HostArch: stringPtr("x86_64"), HostSecondaryArch: stringPtr("x86"), @@ -339,7 +389,7 @@ func (v *productVariables) SetDefaultConfig() { } } -func variableMutator(mctx BottomUpMutatorContext) { +func VariableMutator(mctx BottomUpMutatorContext) { var module Module var ok bool if module, ok = mctx.Module().(Module); !ok { @@ -348,12 +398,15 @@ func variableMutator(mctx BottomUpMutatorContext) { // TODO: depend on config variable, create variants, propagate variants up tree a := module.base() - variableValues := reflect.ValueOf(&a.variableProperties.Product_variables).Elem() - zeroValues := reflect.ValueOf(zeroProductVariables.Product_variables) + + if a.variableProperties == nil { + return + } + + variableValues := reflect.ValueOf(a.variableProperties).Elem().FieldByName("Product_variables") for i := 0; i < variableValues.NumField(); i++ { variableValue := variableValues.Field(i) - zeroValue := zeroValues.Field(i) name := variableValues.Type().Field(i).Name property := "product_variables." + proptools.PropertyNameForField(name) @@ -371,20 +424,19 @@ func variableMutator(mctx BottomUpMutatorContext) { } // Check if any properties were set for the module - if reflect.DeepEqual(variableValue.Interface(), zeroValue.Interface()) { + if variableValue.IsZero() { continue } - a.setVariableProperties(mctx, property, variableValue, val.Interface()) } } -func (a *ModuleBase) setVariableProperties(ctx BottomUpMutatorContext, +func (m *ModuleBase) setVariableProperties(ctx BottomUpMutatorContext, prefix string, productVariablePropertyValue reflect.Value, variableValue interface{}) { printfIntoProperties(ctx, prefix, productVariablePropertyValue, variableValue) - err := proptools.AppendMatchingProperties(a.generalProperties, + err := proptools.AppendMatchingProperties(m.generalProperties, productVariablePropertyValue.Addr().Interface(), nil) if err != nil { if propertyErr, ok := err.(*proptools.ExtendPropertyError); ok { @@ -478,3 +530,120 @@ func printfIntoProperty(propertyValue reflect.Value, variableValue interface{}) return nil } + +var variablePropTypeMap OncePer + +// sliceToTypeArray takes a slice of property structs and returns a reflection created array containing the +// reflect.Types of each property struct. The result can be used as a key in a map. +func sliceToTypeArray(s []interface{}) interface{} { + // Create an array using reflection whose length is the length of the input slice + ret := reflect.New(reflect.ArrayOf(len(s), reflect.TypeOf(reflect.TypeOf(0)))).Elem() + for i, e := range s { + ret.Index(i).Set(reflect.ValueOf(reflect.TypeOf(e))) + } + return ret.Interface() +} + +func initProductVariableModule(m Module) { + base := m.base() + + // Allow tests to override the default product variables + if base.variableProperties == nil { + base.variableProperties = defaultProductVariables + } + // Filter the product variables properties to the ones that exist on this module + base.variableProperties = createVariableProperties(m.GetProperties(), base.variableProperties) + if base.variableProperties != nil { + m.AddProperties(base.variableProperties) + } +} + +// createVariableProperties takes the list of property structs for a module and returns a property struct that +// contains the product variable properties that exist in the property structs, or nil if there are none. It +// caches the result. +func createVariableProperties(moduleTypeProps []interface{}, productVariables interface{}) interface{} { + // Convert the moduleTypeProps to an array of reflect.Types that can be used as a key in the OncePer. + key := sliceToTypeArray(moduleTypeProps) + + // Use the variablePropTypeMap OncePer to cache the result for each set of property struct types. + typ, _ := variablePropTypeMap.Once(NewCustomOnceKey(key), func() interface{} { + // Compute the filtered property struct type. + return createVariablePropertiesType(moduleTypeProps, productVariables) + }).(reflect.Type) + + if typ == nil { + return nil + } + + // Create a new pointer to a filtered property struct. + return reflect.New(typ).Interface() +} + +// createVariablePropertiesType creates a new type that contains only the product variable properties that exist in +// a list of property structs. +func createVariablePropertiesType(moduleTypeProps []interface{}, productVariables interface{}) reflect.Type { + typ, _ := proptools.FilterPropertyStruct(reflect.TypeOf(productVariables), + func(field reflect.StructField, prefix string) (bool, reflect.StructField) { + // Filter function, returns true if the field should be in the resulting struct + if prefix == "" { + // Keep the top level Product_variables field + return true, field + } + _, rest := splitPrefix(prefix) + if rest == "" { + // Keep the 2nd level field (i.e. Product_variables.Eng) + return true, field + } + + // Strip off the first 2 levels of the prefix + _, prefix = splitPrefix(rest) + + for _, p := range moduleTypeProps { + if fieldExistsByNameRecursive(reflect.TypeOf(p).Elem(), prefix, field.Name) { + // Keep any fields that exist in one of the property structs + return true, field + } + } + + return false, field + }) + return typ +} + +func splitPrefix(prefix string) (first, rest string) { + index := strings.IndexByte(prefix, '.') + if index == -1 { + return prefix, "" + } + return prefix[:index], prefix[index+1:] +} + +func fieldExistsByNameRecursive(t reflect.Type, prefix, name string) bool { + if t.Kind() != reflect.Struct { + panic(fmt.Errorf("fieldExistsByNameRecursive can only be called on a reflect.Struct")) + } + + if prefix != "" { + split := strings.SplitN(prefix, ".", 2) + firstPrefix := split[0] + rest := "" + if len(split) > 1 { + rest = split[1] + } + f, exists := t.FieldByName(firstPrefix) + if !exists { + return false + } + ft := f.Type + if ft.Kind() == reflect.Ptr { + ft = ft.Elem() + } + if ft.Kind() != reflect.Struct { + panic(fmt.Errorf("field %q in %q is not a struct", firstPrefix, t)) + } + return fieldExistsByNameRecursive(ft, rest, name) + } else { + _, exists := t.FieldByName(name) + return exists + } +} diff --git a/android/variable_test.go b/android/variable_test.go index ce9ba5485..9cafedd50 100644 --- a/android/variable_test.go +++ b/android/variable_test.go @@ -16,7 +16,10 @@ package android import ( "reflect" + "strconv" "testing" + + "github.com/google/blueprint/proptools" ) type printfIntoPropertyTestCase struct { @@ -122,3 +125,213 @@ func TestPrintfIntoProperty(t *testing.T) { } } } + +type testProductVariableModule struct { + ModuleBase +} + +func (m *testProductVariableModule) GenerateAndroidBuildActions(ctx ModuleContext) { +} + +var testProductVariableProperties = struct { + Product_variables struct { + Eng struct { + Srcs []string + Cflags []string + } + } +}{} + +func testProductVariableModuleFactoryFactory(props interface{}) func() Module { + return func() Module { + m := &testProductVariableModule{} + clonedProps := proptools.CloneProperties(reflect.ValueOf(props)).Interface() + m.AddProperties(clonedProps) + + // Set a default soongConfigVariableProperties, this will be used as the input to the property struct filter + // for this test module. + m.variableProperties = testProductVariableProperties + InitAndroidModule(m) + return m + } +} + +func TestProductVariables(t *testing.T) { + ctx := NewTestContext() + // A module type that has a srcs property but not a cflags property. + ctx.RegisterModuleType("module1", testProductVariableModuleFactoryFactory(&struct { + Srcs []string + }{})) + // A module type that has a cflags property but not a srcs property. + ctx.RegisterModuleType("module2", testProductVariableModuleFactoryFactory(&struct { + Cflags []string + }{})) + // A module type that does not have any properties that match product_variables. + ctx.RegisterModuleType("module3", testProductVariableModuleFactoryFactory(&struct { + Foo []string + }{})) + ctx.PreDepsMutators(func(ctx RegisterMutatorsContext) { + ctx.BottomUp("variable", VariableMutator).Parallel() + }) + + // Test that a module can use one product variable even if it doesn't have all the properties + // supported by that product variable. + bp := ` + module1 { + name: "foo", + product_variables: { + eng: { + srcs: ["foo.c"], + }, + }, + } + module2 { + name: "bar", + product_variables: { + eng: { + cflags: ["-DBAR"], + }, + }, + } + + module3 { + name: "baz", + } + ` + config := TestConfig(buildDir, nil, bp, nil) + config.TestProductVariables.Eng = proptools.BoolPtr(true) + + ctx.Register(config) + + _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) + FailIfErrored(t, errs) + _, errs = ctx.PrepareBuildActions(config) + FailIfErrored(t, errs) +} + +var testProductVariableDefaultsProperties = struct { + Product_variables struct { + Eng struct { + Foo []string + Bar []string + } + } +}{} + +type productVariablesDefaultsTestProperties struct { + Foo []string +} + +type productVariablesDefaultsTestProperties2 struct { + Foo []string + Bar []string +} + +type productVariablesDefaultsTestModule struct { + ModuleBase + DefaultableModuleBase + properties productVariablesDefaultsTestProperties +} + +func (d *productVariablesDefaultsTestModule) GenerateAndroidBuildActions(ctx ModuleContext) { + ctx.Build(pctx, BuildParams{ + Rule: Touch, + Output: PathForModuleOut(ctx, "out"), + }) +} + +func productVariablesDefaultsTestModuleFactory() Module { + module := &productVariablesDefaultsTestModule{} + module.AddProperties(&module.properties) + module.variableProperties = testProductVariableDefaultsProperties + InitAndroidModule(module) + InitDefaultableModule(module) + return module +} + +type productVariablesDefaultsTestDefaults struct { + ModuleBase + DefaultsModuleBase +} + +func productVariablesDefaultsTestDefaultsFactory() Module { + defaults := &productVariablesDefaultsTestDefaults{} + defaults.AddProperties(&productVariablesDefaultsTestProperties{}) + defaults.AddProperties(&productVariablesDefaultsTestProperties2{}) + defaults.variableProperties = testProductVariableDefaultsProperties + InitDefaultsModule(defaults) + return defaults +} + +// Test a defaults module that supports more product variable properties than the target module. +func TestProductVariablesDefaults(t *testing.T) { + bp := ` + defaults { + name: "defaults", + product_variables: { + eng: { + foo: ["product_variable_defaults"], + bar: ["product_variable_defaults"], + }, + }, + foo: ["defaults"], + bar: ["defaults"], + } + + test { + name: "foo", + defaults: ["defaults"], + foo: ["module"], + product_variables: { + eng: { + foo: ["product_variable_module"], + }, + }, + } + ` + + config := TestConfig(buildDir, nil, bp, nil) + config.TestProductVariables.Eng = boolPtr(true) + + ctx := NewTestContext() + + ctx.RegisterModuleType("test", productVariablesDefaultsTestModuleFactory) + ctx.RegisterModuleType("defaults", productVariablesDefaultsTestDefaultsFactory) + + ctx.PreArchMutators(RegisterDefaultsPreArchMutators) + ctx.PreDepsMutators(func(ctx RegisterMutatorsContext) { + ctx.BottomUp("variable", VariableMutator).Parallel() + }) + + ctx.Register(config) + + _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) + FailIfErrored(t, errs) + _, errs = ctx.PrepareBuildActions(config) + FailIfErrored(t, errs) + + foo := ctx.ModuleForTests("foo", "").Module().(*productVariablesDefaultsTestModule) + + want := []string{"defaults", "module", "product_variable_defaults", "product_variable_module"} + if g, w := foo.properties.Foo, want; !reflect.DeepEqual(g, w) { + t.Errorf("expected foo %q, got %q", w, g) + } +} + +func BenchmarkSliceToTypeArray(b *testing.B) { + for _, n := range []int{1, 2, 4, 8, 100} { + var propStructs []interface{} + for i := 0; i < n; i++ { + propStructs = append(propStructs, &struct { + A *string + B string + }{}) + + } + b.Run(strconv.Itoa(n), func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = sliceToTypeArray(propStructs) + } + }) + } +} diff --git a/android/visibility.go b/android/visibility.go new file mode 100644 index 000000000..68da1c475 --- /dev/null +++ b/android/visibility.go @@ -0,0 +1,543 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "fmt" + "regexp" + "strings" + "sync" + + "github.com/google/blueprint" +) + +// Enforces visibility rules between modules. +// +// Multi stage process: +// * First stage works bottom up, before defaults expansion, to check the syntax of the visibility +// rules that have been specified. +// +// * Second stage works bottom up to extract the package info for each package and store them in a +// map by package name. See package.go for functionality for this. +// +// * Third stage works bottom up to extract visibility information from the modules, parse it, +// create visibilityRule structures and store them in a map keyed by the module's +// qualifiedModuleName instance, i.e. //<pkg>:<name>. The map is stored in the context rather +// than a global variable for testing. Each test has its own Config so they do not share a map +// and so can be run in parallel. If a module has no visibility specified then it uses the +// default package visibility if specified. +// +// * Fourth stage works top down and iterates over all the deps for each module. If the dep is in +// the same package then it is automatically visible. Otherwise, for each dep it first extracts +// its visibilityRule from the config map. If one could not be found then it assumes that it is +// publicly visible. Otherwise, it calls the visibility rule to check that the module can see +// the dependency. If it cannot then an error is reported. +// +// TODO(b/130631145) - Make visibility work properly with prebuilts. +// TODO(b/130796911) - Make visibility work properly with defaults. + +// Patterns for the values that can be specified in visibility property. +const ( + packagePattern = `//([^/:]+(?:/[^/:]+)*)` + namePattern = `:([^/:]+)` + visibilityRulePattern = `^(?:` + packagePattern + `)?(?:` + namePattern + `)?$` +) + +var visibilityRuleRegexp = regexp.MustCompile(visibilityRulePattern) + +// A visibility rule is associated with a module and determines which other modules it is visible +// to, i.e. which other modules can depend on the rule's module. +type visibilityRule interface { + // Check to see whether this rules matches m. + // Returns true if it does, false otherwise. + matches(m qualifiedModuleName) bool + + String() string +} + +// Describes the properties provided by a module that contain visibility rules. +type visibilityPropertyImpl struct { + name string + stringsProperty *[]string +} + +type visibilityProperty interface { + getName() string + getStrings() []string +} + +func newVisibilityProperty(name string, stringsProperty *[]string) visibilityProperty { + return visibilityPropertyImpl{ + name: name, + stringsProperty: stringsProperty, + } +} + +func (p visibilityPropertyImpl) getName() string { + return p.name +} + +func (p visibilityPropertyImpl) getStrings() []string { + return *p.stringsProperty +} + +// A compositeRule is a visibility rule composed from a list of atomic visibility rules. +// +// The list corresponds to the list of strings in the visibility property after defaults expansion. +// Even though //visibility:public is not allowed together with other rules in the visibility list +// of a single module, it is allowed here to permit a module to override an inherited visibility +// spec with public visibility. +// +// //visibility:private is not allowed in the same way, since we'd need to check for it during the +// defaults expansion to make that work. No non-private visibility rules are allowed in a +// compositeRule containing a privateRule. +// +// This array will only be [] if all the rules are invalid and will behave as if visibility was +// ["//visibility:private"]. +type compositeRule []visibilityRule + +// A compositeRule matches if and only if any of its rules matches. +func (c compositeRule) matches(m qualifiedModuleName) bool { + for _, r := range c { + if r.matches(m) { + return true + } + } + return false +} + +func (c compositeRule) String() string { + return "[" + strings.Join(c.Strings(), ", ") + "]" +} + +func (c compositeRule) Strings() []string { + s := make([]string, 0, len(c)) + for _, r := range c { + s = append(s, r.String()) + } + return s +} + +// A packageRule is a visibility rule that matches modules in a specific package (i.e. directory). +type packageRule struct { + pkg string +} + +func (r packageRule) matches(m qualifiedModuleName) bool { + return m.pkg == r.pkg +} + +func (r packageRule) String() string { + return fmt.Sprintf("//%s", r.pkg) // :__pkg__ is the default, so skip it. +} + +// A subpackagesRule is a visibility rule that matches modules in a specific package (i.e. +// directory) or any of its subpackages (i.e. subdirectories). +type subpackagesRule struct { + pkgPrefix string +} + +func (r subpackagesRule) matches(m qualifiedModuleName) bool { + return isAncestor(r.pkgPrefix, m.pkg) +} + +func isAncestor(p1 string, p2 string) bool { + return strings.HasPrefix(p2+"/", p1+"/") +} + +func (r subpackagesRule) String() string { + return fmt.Sprintf("//%s:__subpackages__", r.pkgPrefix) +} + +// visibilityRule for //visibility:public +type publicRule struct{} + +func (r publicRule) matches(_ qualifiedModuleName) bool { + return true +} + +func (r publicRule) String() string { + return "//visibility:public" +} + +// visibilityRule for //visibility:private +type privateRule struct{} + +func (r privateRule) matches(_ qualifiedModuleName) bool { + return false +} + +func (r privateRule) String() string { + return "//visibility:private" +} + +var visibilityRuleMap = NewOnceKey("visibilityRuleMap") + +// The map from qualifiedModuleName to visibilityRule. +func moduleToVisibilityRuleMap(config Config) *sync.Map { + return config.Once(visibilityRuleMap, func() interface{} { + return &sync.Map{} + }).(*sync.Map) +} + +// Marker interface that identifies dependencies that are excluded from visibility +// enforcement. +type ExcludeFromVisibilityEnforcementTag interface { + blueprint.DependencyTag + + // Method that differentiates this interface from others. + ExcludeFromVisibilityEnforcement() +} + +// The rule checker needs to be registered before defaults expansion to correctly check that +// //visibility:xxx isn't combined with other packages in the same list in any one module. +func RegisterVisibilityRuleChecker(ctx RegisterMutatorsContext) { + ctx.BottomUp("visibilityRuleChecker", visibilityRuleChecker).Parallel() +} + +// Registers the function that gathers the visibility rules for each module. +// +// Visibility is not dependent on arch so this must be registered before the arch phase to avoid +// having to process multiple variants for each module. This goes after defaults expansion to gather +// the complete visibility lists from flat lists and after the package info is gathered to ensure +// that default_visibility is available. +func RegisterVisibilityRuleGatherer(ctx RegisterMutatorsContext) { + ctx.BottomUp("visibilityRuleGatherer", visibilityRuleGatherer).Parallel() +} + +// This must be registered after the deps have been resolved. +func RegisterVisibilityRuleEnforcer(ctx RegisterMutatorsContext) { + ctx.TopDown("visibilityRuleEnforcer", visibilityRuleEnforcer).Parallel() +} + +// Checks the per-module visibility rule lists before defaults expansion. +func visibilityRuleChecker(ctx BottomUpMutatorContext) { + qualified := createQualifiedModuleName(ctx) + if m, ok := ctx.Module().(Module); ok { + visibilityProperties := m.visibilityProperties() + for _, p := range visibilityProperties { + if visibility := p.getStrings(); visibility != nil { + checkRules(ctx, qualified.pkg, p.getName(), visibility) + } + } + } +} + +func checkRules(ctx BaseModuleContext, currentPkg, property string, visibility []string) { + ruleCount := len(visibility) + if ruleCount == 0 { + // This prohibits an empty list as its meaning is unclear, e.g. it could mean no visibility and + // it could mean public visibility. Requiring at least one rule makes the owner's intent + // clearer. + ctx.PropertyErrorf(property, "must contain at least one visibility rule") + return + } + + for i, v := range visibility { + ok, pkg, name := splitRule(ctx, v, currentPkg, property) + if !ok { + continue + } + + if pkg == "visibility" { + switch name { + case "private", "public": + case "legacy_public": + ctx.PropertyErrorf(property, "//visibility:legacy_public must not be used") + continue + case "override": + // This keyword does not create a rule so pretend it does not exist. + ruleCount -= 1 + default: + ctx.PropertyErrorf(property, "unrecognized visibility rule %q", v) + continue + } + if name == "override" { + if i != 0 { + ctx.PropertyErrorf(property, `"%v" may only be used at the start of the visibility rules`, v) + } + } else if ruleCount != 1 { + ctx.PropertyErrorf(property, "cannot mix %q with any other visibility rules", v) + continue + } + } + + // If the current directory is not in the vendor tree then there are some additional + // restrictions on the rules. + if !isAncestor("vendor", currentPkg) { + if !isAllowedFromOutsideVendor(pkg, name) { + ctx.PropertyErrorf(property, + "%q is not allowed. Packages outside //vendor cannot make themselves visible to specific"+ + " targets within //vendor, they can only use //vendor:__subpackages__.", v) + continue + } + } + } +} + +// Gathers the flattened visibility rules after defaults expansion, parses the visibility +// properties, stores them in a map by qualifiedModuleName for retrieval during enforcement. +// +// See ../README.md#Visibility for information on the format of the visibility rules. +func visibilityRuleGatherer(ctx BottomUpMutatorContext) { + m, ok := ctx.Module().(Module) + if !ok { + return + } + + qualifiedModuleId := m.qualifiedModuleId(ctx) + currentPkg := qualifiedModuleId.pkg + + // Parse the visibility rules that control access to the module and store them by id + // for use when enforcing the rules. + primaryProperty := m.base().primaryVisibilityProperty + if primaryProperty != nil { + if visibility := primaryProperty.getStrings(); visibility != nil { + rule := parseRules(ctx, currentPkg, primaryProperty.getName(), visibility) + if rule != nil { + moduleToVisibilityRuleMap(ctx.Config()).Store(qualifiedModuleId, rule) + } + } + } +} + +func parseRules(ctx BaseModuleContext, currentPkg, property string, visibility []string) compositeRule { + rules := make(compositeRule, 0, len(visibility)) + hasPrivateRule := false + hasPublicRule := false + hasNonPrivateRule := false + for _, v := range visibility { + ok, pkg, name := splitRule(ctx, v, currentPkg, property) + if !ok { + continue + } + + var r visibilityRule + isPrivateRule := false + if pkg == "visibility" { + switch name { + case "private": + r = privateRule{} + isPrivateRule = true + case "public": + r = publicRule{} + hasPublicRule = true + case "override": + // Discard all preceding rules and any state based on them. + rules = nil + hasPrivateRule = false + hasPublicRule = false + hasNonPrivateRule = false + // This does not actually create a rule so continue onto the next rule. + continue + } + } else { + switch name { + case "__pkg__": + r = packageRule{pkg} + case "__subpackages__": + r = subpackagesRule{pkg} + default: + continue + } + } + + if isPrivateRule { + hasPrivateRule = true + } else { + hasNonPrivateRule = true + } + + rules = append(rules, r) + } + + if hasPrivateRule && hasNonPrivateRule { + ctx.PropertyErrorf("visibility", + "cannot mix \"//visibility:private\" with any other visibility rules") + return compositeRule{privateRule{}} + } + + if hasPublicRule { + // Public overrides all other rules so just return it. + return compositeRule{publicRule{}} + } + + return rules +} + +func isAllowedFromOutsideVendor(pkg string, name string) bool { + if pkg == "vendor" { + if name == "__subpackages__" { + return true + } + return false + } + + return !isAncestor("vendor", pkg) +} + +func splitRule(ctx BaseModuleContext, ruleExpression string, currentPkg, property string) (bool, string, string) { + // Make sure that the rule is of the correct format. + matches := visibilityRuleRegexp.FindStringSubmatch(ruleExpression) + if ruleExpression == "" || matches == nil { + // Visibility rule is invalid so ignore it. Keep going rather than aborting straight away to + // ensure all the rules on this module are checked. + ctx.PropertyErrorf(property, + "invalid visibility pattern %q must match"+ + " //<package>:<module>, //<package> or :<module>", + ruleExpression) + return false, "", "" + } + + // Extract the package and name. + pkg := matches[1] + name := matches[2] + + // Normalize the short hands + if pkg == "" { + pkg = currentPkg + } + if name == "" { + name = "__pkg__" + } + + return true, pkg, name +} + +func visibilityRuleEnforcer(ctx TopDownMutatorContext) { + if _, ok := ctx.Module().(Module); !ok { + return + } + + qualified := createQualifiedModuleName(ctx) + + // Visit all the dependencies making sure that this module has access to them all. + ctx.VisitDirectDeps(func(dep Module) { + // Ignore dependencies that have an ExcludeFromVisibilityEnforcementTag + tag := ctx.OtherModuleDependencyTag(dep) + if _, ok := tag.(ExcludeFromVisibilityEnforcementTag); ok { + return + } + + depName := ctx.OtherModuleName(dep) + depDir := ctx.OtherModuleDir(dep) + depQualified := qualifiedModuleName{depDir, depName} + + // Targets are always visible to other targets in their own package. + if depQualified.pkg == qualified.pkg { + return + } + + rule := effectiveVisibilityRules(ctx.Config(), depQualified) + if rule != nil && !rule.matches(qualified) { + ctx.ModuleErrorf("depends on %s which is not visible to this module", depQualified) + } + }) +} + +func effectiveVisibilityRules(config Config, qualified qualifiedModuleName) compositeRule { + moduleToVisibilityRule := moduleToVisibilityRuleMap(config) + value, ok := moduleToVisibilityRule.Load(qualified) + var rule compositeRule + if ok { + rule = value.(compositeRule) + } else { + rule = packageDefaultVisibility(config, qualified) + } + return rule +} + +func createQualifiedModuleName(ctx BaseModuleContext) qualifiedModuleName { + moduleName := ctx.ModuleName() + dir := ctx.ModuleDir() + qualified := qualifiedModuleName{dir, moduleName} + return qualified +} + +func packageDefaultVisibility(config Config, moduleId qualifiedModuleName) compositeRule { + moduleToVisibilityRule := moduleToVisibilityRuleMap(config) + packageQualifiedId := moduleId.getContainingPackageId() + for { + value, ok := moduleToVisibilityRule.Load(packageQualifiedId) + if ok { + return value.(compositeRule) + } + + if packageQualifiedId.isRootPackage() { + return nil + } + + packageQualifiedId = packageQualifiedId.getContainingPackageId() + } +} + +// Get the effective visibility rules, i.e. the actual rules that affect the visibility of the +// property irrespective of where they are defined. +// +// Includes visibility rules specified by package default_visibility and/or on defaults. +// Short hand forms, e.g. //:__subpackages__ are replaced with their full form, e.g. +// //package/containing/rule:__subpackages__. +func EffectiveVisibilityRules(ctx BaseModuleContext, module Module) []string { + moduleName := ctx.OtherModuleName(module) + dir := ctx.OtherModuleDir(module) + qualified := qualifiedModuleName{dir, moduleName} + + rule := effectiveVisibilityRules(ctx.Config(), qualified) + + // Modules are implicitly visible to other modules in the same package, + // without checking the visibility rules. Here we need to add that visibility + // explicitly. + if rule != nil && !rule.matches(qualified) { + if len(rule) == 1 { + if _, ok := rule[0].(privateRule); ok { + // If the rule is //visibility:private we can't append another + // visibility to it. Semantically we need to convert it to a package + // visibility rule for the location where the result is used, but since + // modules are implicitly visible within the package we get the same + // result without any rule at all, so just make it an empty list to be + // appended below. + rule = compositeRule{} + } + } + rule = append(rule, packageRule{dir}) + } + + return rule.Strings() +} + +// Clear the default visibility properties so they can be replaced. +func clearVisibilityProperties(module Module) { + module.base().visibilityPropertyInfo = nil +} + +// Add a property that contains visibility rules so that they are checked for +// correctness. +func AddVisibilityProperty(module Module, name string, stringsProperty *[]string) { + addVisibilityProperty(module, name, stringsProperty) +} + +func addVisibilityProperty(module Module, name string, stringsProperty *[]string) visibilityProperty { + base := module.base() + property := newVisibilityProperty(name, stringsProperty) + base.visibilityPropertyInfo = append(base.visibilityPropertyInfo, property) + return property +} + +// Set the primary visibility property. +// +// Also adds the property to the list of properties to be validated. +func setPrimaryVisibilityProperty(module Module, name string, stringsProperty *[]string) { + module.base().primaryVisibilityProperty = addVisibilityProperty(module, name, stringsProperty) +} diff --git a/android/visibility_test.go b/android/visibility_test.go new file mode 100644 index 000000000..ca0934557 --- /dev/null +++ b/android/visibility_test.go @@ -0,0 +1,1272 @@ +package android + +import ( + "reflect" + "testing" + + "github.com/google/blueprint" +) + +var visibilityTests = []struct { + name string + fs map[string][]byte + expectedErrors []string + effectiveVisibility map[qualifiedModuleName][]string +}{ + { + name: "invalid visibility: empty list", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_library { + name: "libexample", + visibility: [], + }`), + }, + expectedErrors: []string{`visibility: must contain at least one visibility rule`}, + }, + { + name: "invalid visibility: empty rule", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_library { + name: "libexample", + visibility: [""], + }`), + }, + expectedErrors: []string{`visibility: invalid visibility pattern ""`}, + }, + { + name: "invalid visibility: unqualified", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_library { + name: "libexample", + visibility: ["target"], + }`), + }, + expectedErrors: []string{`visibility: invalid visibility pattern "target"`}, + }, + { + name: "invalid visibility: empty namespace", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_library { + name: "libexample", + visibility: ["//"], + }`), + }, + expectedErrors: []string{`visibility: invalid visibility pattern "//"`}, + }, + { + name: "invalid visibility: empty module", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_library { + name: "libexample", + visibility: [":"], + }`), + }, + expectedErrors: []string{`visibility: invalid visibility pattern ":"`}, + }, + { + name: "invalid visibility: empty namespace and module", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_library { + name: "libexample", + visibility: ["//:"], + }`), + }, + expectedErrors: []string{`visibility: invalid visibility pattern "//:"`}, + }, + { + name: "//visibility:unknown", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_library { + name: "libexample", + visibility: ["//visibility:unknown"], + }`), + }, + expectedErrors: []string{`unrecognized visibility rule "//visibility:unknown"`}, + }, + { + name: "//visibility:xxx mixed", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_library { + name: "libexample", + visibility: ["//visibility:public", "//namespace"], + } + + mock_library { + name: "libother", + visibility: ["//visibility:private", "//namespace"], + }`), + }, + expectedErrors: []string{ + `module "libother": visibility: cannot mix "//visibility:private"` + + ` with any other visibility rules`, + `module "libexample": visibility: cannot mix "//visibility:public"` + + ` with any other visibility rules`, + }, + }, + { + name: "//visibility:legacy_public", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_library { + name: "libexample", + visibility: ["//visibility:legacy_public"], + }`), + }, + expectedErrors: []string{ + `module "libexample": visibility: //visibility:legacy_public must` + + ` not be used`, + }, + }, + { + // Verify that //visibility:public will allow the module to be referenced from anywhere, e.g. + // the current directory, a nested directory and a directory in a separate tree. + name: "//visibility:public", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_library { + name: "libexample", + visibility: ["//visibility:public"], + } + + mock_library { + name: "libsamepackage", + deps: ["libexample"], + }`), + "top/nested/Blueprints": []byte(` + mock_library { + name: "libnested", + deps: ["libexample"], + }`), + "other/Blueprints": []byte(` + mock_library { + name: "libother", + deps: ["libexample"], + }`), + }, + }, + { + // Verify that //visibility:private allows the module to be referenced from the current + // directory only. + name: "//visibility:private", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_library { + name: "libexample", + visibility: ["//visibility:private"], + } + + mock_library { + name: "libsamepackage", + deps: ["libexample"], + }`), + "top/nested/Blueprints": []byte(` + mock_library { + name: "libnested", + deps: ["libexample"], + }`), + "other/Blueprints": []byte(` + mock_library { + name: "libother", + deps: ["libexample"], + }`), + }, + expectedErrors: []string{ + `module "libnested" variant "android_common": depends on //top:libexample which is not` + + ` visible to this module`, + `module "libother" variant "android_common": depends on //top:libexample which is not` + + ` visible to this module`, + }, + }, + { + // Verify that :__pkg__ allows the module to be referenced from the current directory only. + name: ":__pkg__", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_library { + name: "libexample", + visibility: [":__pkg__"], + } + + mock_library { + name: "libsamepackage", + deps: ["libexample"], + }`), + "top/nested/Blueprints": []byte(` + mock_library { + name: "libnested", + deps: ["libexample"], + }`), + "other/Blueprints": []byte(` + mock_library { + name: "libother", + deps: ["libexample"], + }`), + }, + expectedErrors: []string{ + `module "libnested" variant "android_common": depends on //top:libexample which is not` + + ` visible to this module`, + `module "libother" variant "android_common": depends on //top:libexample which is not` + + ` visible to this module`, + }, + }, + { + // Verify that //top/nested allows the module to be referenced from the current directory and + // the top/nested directory only, not a subdirectory of top/nested and not peak directory. + name: "//top/nested", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_library { + name: "libexample", + visibility: ["//top/nested"], + } + + mock_library { + name: "libsamepackage", + deps: ["libexample"], + }`), + "top/nested/Blueprints": []byte(` + mock_library { + name: "libnested", + deps: ["libexample"], + }`), + "top/nested/again/Blueprints": []byte(` + mock_library { + name: "libnestedagain", + deps: ["libexample"], + }`), + "peak/Blueprints": []byte(` + mock_library { + name: "libother", + deps: ["libexample"], + }`), + }, + expectedErrors: []string{ + `module "libother" variant "android_common": depends on //top:libexample which is not` + + ` visible to this module`, + `module "libnestedagain" variant "android_common": depends on //top:libexample which is not` + + ` visible to this module`, + }, + }, + { + // Verify that :__subpackages__ allows the module to be referenced from the current directory + // and sub directories but nowhere else. + name: ":__subpackages__", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_library { + name: "libexample", + visibility: [":__subpackages__"], + } + + mock_library { + name: "libsamepackage", + deps: ["libexample"], + }`), + "top/nested/Blueprints": []byte(` + mock_library { + name: "libnested", + deps: ["libexample"], + }`), + "peak/other/Blueprints": []byte(` + mock_library { + name: "libother", + deps: ["libexample"], + }`), + }, + expectedErrors: []string{ + `module "libother" variant "android_common": depends on //top:libexample which is not` + + ` visible to this module`, + }, + }, + { + // Verify that //top/nested:__subpackages__ allows the module to be referenced from the current + // directory and sub directories but nowhere else. + name: "//top/nested:__subpackages__", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_library { + name: "libexample", + visibility: ["//top/nested:__subpackages__", "//other"], + } + + mock_library { + name: "libsamepackage", + deps: ["libexample"], + }`), + "top/nested/Blueprints": []byte(` + mock_library { + name: "libnested", + deps: ["libexample"], + }`), + "top/other/Blueprints": []byte(` + mock_library { + name: "libother", + deps: ["libexample"], + }`), + }, + expectedErrors: []string{ + `module "libother" variant "android_common": depends on //top:libexample which is not` + + ` visible to this module`, + }, + }, + { + // Verify that ["//top/nested", "//peak:__subpackages"] allows the module to be referenced from + // the current directory, top/nested and peak and all its subpackages. + name: `["//top/nested", "//peak:__subpackages__"]`, + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_library { + name: "libexample", + visibility: ["//top/nested", "//peak:__subpackages__"], + } + + mock_library { + name: "libsamepackage", + deps: ["libexample"], + }`), + "top/nested/Blueprints": []byte(` + mock_library { + name: "libnested", + deps: ["libexample"], + }`), + "peak/other/Blueprints": []byte(` + mock_library { + name: "libother", + deps: ["libexample"], + }`), + }, + }, + { + // Verify that //vendor... cannot be used outside vendor apart from //vendor:__subpackages__ + name: `//vendor`, + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_library { + name: "libexample", + visibility: ["//vendor:__subpackages__"], + } + + mock_library { + name: "libsamepackage", + visibility: ["//vendor/apps/AcmeSettings"], + }`), + "vendor/Blueprints": []byte(` + mock_library { + name: "libvendorexample", + deps: ["libexample"], + visibility: ["//vendor/nested"], + }`), + "vendor/nested/Blueprints": []byte(` + mock_library { + name: "libvendornested", + deps: ["libexample", "libvendorexample"], + }`), + }, + expectedErrors: []string{ + `module "libsamepackage": visibility: "//vendor/apps/AcmeSettings"` + + ` is not allowed. Packages outside //vendor cannot make themselves visible to specific` + + ` targets within //vendor, they can only use //vendor:__subpackages__.`, + }, + }, + + // Defaults propagation tests + { + // Check that visibility is the union of the defaults modules. + name: "defaults union, basic", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_defaults { + name: "libexample_defaults", + visibility: ["//other"], + } + mock_library { + name: "libexample", + visibility: ["//top/nested"], + defaults: ["libexample_defaults"], + } + mock_library { + name: "libsamepackage", + deps: ["libexample"], + }`), + "top/nested/Blueprints": []byte(` + mock_library { + name: "libnested", + deps: ["libexample"], + }`), + "other/Blueprints": []byte(` + mock_library { + name: "libother", + deps: ["libexample"], + }`), + "outsider/Blueprints": []byte(` + mock_library { + name: "liboutsider", + deps: ["libexample"], + }`), + }, + expectedErrors: []string{ + `module "liboutsider" variant "android_common": depends on //top:libexample which is not` + + ` visible to this module`, + }, + }, + { + name: "defaults union, multiple defaults", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_defaults { + name: "libexample_defaults_1", + visibility: ["//other"], + } + mock_defaults { + name: "libexample_defaults_2", + visibility: ["//top/nested"], + } + mock_library { + name: "libexample", + defaults: ["libexample_defaults_1", "libexample_defaults_2"], + } + mock_library { + name: "libsamepackage", + deps: ["libexample"], + }`), + "top/nested/Blueprints": []byte(` + mock_library { + name: "libnested", + deps: ["libexample"], + }`), + "other/Blueprints": []byte(` + mock_library { + name: "libother", + deps: ["libexample"], + }`), + "outsider/Blueprints": []byte(` + mock_library { + name: "liboutsider", + deps: ["libexample"], + }`), + }, + expectedErrors: []string{ + `module "liboutsider" variant "android_common": depends on //top:libexample which is not` + + ` visible to this module`, + }, + }, + { + name: "//visibility:public mixed with other in defaults", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_defaults { + name: "libexample_defaults", + visibility: ["//visibility:public", "//namespace"], + } + mock_library { + name: "libexample", + defaults: ["libexample_defaults"], + }`), + }, + expectedErrors: []string{ + `module "libexample_defaults": visibility: cannot mix "//visibility:public"` + + ` with any other visibility rules`, + }, + }, + { + name: "//visibility:public overriding defaults", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_defaults { + name: "libexample_defaults", + visibility: ["//namespace"], + } + mock_library { + name: "libexample", + visibility: ["//visibility:public"], + defaults: ["libexample_defaults"], + }`), + "outsider/Blueprints": []byte(` + mock_library { + name: "liboutsider", + deps: ["libexample"], + }`), + }, + effectiveVisibility: map[qualifiedModuleName][]string{ + qualifiedModuleName{pkg: "top", name: "libexample"}: {"//visibility:public"}, + }, + }, + { + name: "//visibility:public mixed with other from different defaults 1", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_defaults { + name: "libexample_defaults_1", + visibility: ["//namespace"], + } + mock_defaults { + name: "libexample_defaults_2", + visibility: ["//visibility:public"], + } + mock_library { + name: "libexample", + defaults: ["libexample_defaults_1", "libexample_defaults_2"], + }`), + "outsider/Blueprints": []byte(` + mock_library { + name: "liboutsider", + deps: ["libexample"], + }`), + }, + }, + { + name: "//visibility:public mixed with other from different defaults 2", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_defaults { + name: "libexample_defaults_1", + visibility: ["//visibility:public"], + } + mock_defaults { + name: "libexample_defaults_2", + visibility: ["//namespace"], + } + mock_library { + name: "libexample", + defaults: ["libexample_defaults_1", "libexample_defaults_2"], + }`), + "outsider/Blueprints": []byte(` + mock_library { + name: "liboutsider", + deps: ["libexample"], + }`), + }, + }, + { + name: "//visibility:private in defaults", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_defaults { + name: "libexample_defaults", + visibility: ["//visibility:private"], + } + mock_library { + name: "libexample", + defaults: ["libexample_defaults"], + } + mock_library { + name: "libsamepackage", + deps: ["libexample"], + }`), + "top/nested/Blueprints": []byte(` + mock_library { + name: "libnested", + deps: ["libexample"], + }`), + "other/Blueprints": []byte(` + mock_library { + name: "libother", + deps: ["libexample"], + }`), + }, + expectedErrors: []string{ + `module "libnested" variant "android_common": depends on //top:libexample which is not` + + ` visible to this module`, + `module "libother" variant "android_common": depends on //top:libexample which is not` + + ` visible to this module`, + }, + }, + { + name: "//visibility:private mixed with other in defaults", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_defaults { + name: "libexample_defaults", + visibility: ["//visibility:private", "//namespace"], + } + mock_library { + name: "libexample", + defaults: ["libexample_defaults"], + }`), + }, + expectedErrors: []string{ + `module "libexample_defaults": visibility: cannot mix "//visibility:private"` + + ` with any other visibility rules`, + }, + }, + { + name: "//visibility:private overriding defaults", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_defaults { + name: "libexample_defaults", + visibility: ["//namespace"], + } + mock_library { + name: "libexample", + visibility: ["//visibility:private"], + defaults: ["libexample_defaults"], + }`), + }, + expectedErrors: []string{ + `module "libexample": visibility: cannot mix "//visibility:private"` + + ` with any other visibility rules`, + }, + }, + { + name: "//visibility:private in defaults overridden", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_defaults { + name: "libexample_defaults", + visibility: ["//visibility:private"], + } + mock_library { + name: "libexample", + visibility: ["//namespace"], + defaults: ["libexample_defaults"], + }`), + }, + expectedErrors: []string{ + `module "libexample": visibility: cannot mix "//visibility:private"` + + ` with any other visibility rules`, + }, + }, + { + name: "//visibility:private override //visibility:public", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_defaults { + name: "libexample_defaults", + visibility: ["//visibility:public"], + } + mock_library { + name: "libexample", + visibility: ["//visibility:private"], + defaults: ["libexample_defaults"], + }`), + }, + expectedErrors: []string{ + `module "libexample": visibility: cannot mix "//visibility:private" with any other visibility rules`, + }, + }, + { + name: "//visibility:public override //visibility:private", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_defaults { + name: "libexample_defaults", + visibility: ["//visibility:private"], + } + mock_library { + name: "libexample", + visibility: ["//visibility:public"], + defaults: ["libexample_defaults"], + }`), + }, + expectedErrors: []string{ + `module "libexample": visibility: cannot mix "//visibility:private" with any other visibility rules`, + }, + }, + { + name: "//visibility:override must be first in the list", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_library { + name: "libexample", + visibility: ["//other", "//visibility:override", "//namespace"], + }`), + }, + expectedErrors: []string{ + `module "libexample": visibility: "//visibility:override" may only be used at the start of the visibility rules`, + }, + }, + { + name: "//visibility:override discards //visibility:private", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_defaults { + name: "libexample_defaults", + visibility: ["//visibility:private"], + } + mock_library { + name: "libexample", + // Make this visibility to //other but not //visibility:private + visibility: ["//visibility:override", "//other"], + defaults: ["libexample_defaults"], + }`), + "other/Blueprints": []byte(` + mock_library { + name: "libother", + deps: ["libexample"], + }`), + }, + }, + { + name: "//visibility:override discards //visibility:public", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_defaults { + name: "libexample_defaults", + visibility: ["//visibility:public"], + } + mock_library { + name: "libexample", + // Make this visibility to //other but not //visibility:public + visibility: ["//visibility:override", "//other"], + defaults: ["libexample_defaults"], + }`), + "other/Blueprints": []byte(` + mock_library { + name: "libother", + deps: ["libexample"], + }`), + "namespace/Blueprints": []byte(` + mock_library { + name: "libnamespace", + deps: ["libexample"], + }`), + }, + expectedErrors: []string{ + `module "libnamespace" variant "android_common": depends on //top:libexample which is not visible to this module`, + }, + }, + { + name: "//visibility:override discards defaults supplied rules", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_defaults { + name: "libexample_defaults", + visibility: ["//namespace"], + } + mock_library { + name: "libexample", + // Make this visibility to //other but not //namespace + visibility: ["//visibility:override", "//other"], + defaults: ["libexample_defaults"], + }`), + "other/Blueprints": []byte(` + mock_library { + name: "libother", + deps: ["libexample"], + }`), + "namespace/Blueprints": []byte(` + mock_library { + name: "libnamespace", + deps: ["libexample"], + }`), + }, + expectedErrors: []string{ + `module "libnamespace" variant "android_common": depends on //top:libexample which is not visible to this module`, + }, + }, + { + name: "//visibility:override can override //visibility:public with //visibility:private", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_defaults { + name: "libexample_defaults", + visibility: ["//visibility:public"], + } + mock_library { + name: "libexample", + visibility: ["//visibility:override", "//visibility:private"], + defaults: ["libexample_defaults"], + }`), + "namespace/Blueprints": []byte(` + mock_library { + name: "libnamespace", + deps: ["libexample"], + }`), + }, + expectedErrors: []string{ + `module "libnamespace" variant "android_common": depends on //top:libexample which is not visible to this module`, + }, + }, + { + name: "//visibility:override can override //visibility:private with //visibility:public", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_defaults { + name: "libexample_defaults", + visibility: ["//visibility:private"], + } + mock_library { + name: "libexample", + visibility: ["//visibility:override", "//visibility:public"], + defaults: ["libexample_defaults"], + }`), + "namespace/Blueprints": []byte(` + mock_library { + name: "libnamespace", + deps: ["libexample"], + }`), + }, + }, + { + name: "//visibility:private mixed with itself", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_defaults { + name: "libexample_defaults_1", + visibility: ["//visibility:private"], + } + mock_defaults { + name: "libexample_defaults_2", + visibility: ["//visibility:private"], + } + mock_library { + name: "libexample", + visibility: ["//visibility:private"], + defaults: ["libexample_defaults_1", "libexample_defaults_2"], + }`), + "outsider/Blueprints": []byte(` + mock_library { + name: "liboutsider", + deps: ["libexample"], + }`), + }, + expectedErrors: []string{ + `module "liboutsider" variant "android_common": depends on //top:libexample which is not` + + ` visible to this module`, + }, + }, + + // Defaults module's defaults_visibility tests + { + name: "defaults_visibility invalid", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_defaults { + name: "top_defaults", + defaults_visibility: ["//visibility:invalid"], + }`), + }, + expectedErrors: []string{ + `defaults_visibility: unrecognized visibility rule "//visibility:invalid"`, + }, + }, + { + name: "defaults_visibility overrides package default", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + package { + default_visibility: ["//visibility:private"], + } + mock_defaults { + name: "top_defaults", + defaults_visibility: ["//visibility:public"], + }`), + "outsider/Blueprints": []byte(` + mock_library { + name: "liboutsider", + defaults: ["top_defaults"], + }`), + }, + }, + + // Package default_visibility tests + { + name: "package default_visibility property is checked", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + package { + default_visibility: ["//visibility:invalid"], + }`), + }, + expectedErrors: []string{`default_visibility: unrecognized visibility rule "//visibility:invalid"`}, + }, + { + // This test relies on the default visibility being legacy_public. + name: "package default_visibility property used when no visibility specified", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + package { + default_visibility: ["//visibility:private"], + } + + mock_library { + name: "libexample", + }`), + "outsider/Blueprints": []byte(` + mock_library { + name: "liboutsider", + deps: ["libexample"], + }`), + }, + expectedErrors: []string{ + `module "liboutsider" variant "android_common": depends on //top:libexample which is not` + + ` visible to this module`, + }, + }, + { + name: "package default_visibility public does not override visibility private", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + package { + default_visibility: ["//visibility:public"], + } + + mock_library { + name: "libexample", + visibility: ["//visibility:private"], + }`), + "outsider/Blueprints": []byte(` + mock_library { + name: "liboutsider", + deps: ["libexample"], + }`), + }, + expectedErrors: []string{ + `module "liboutsider" variant "android_common": depends on //top:libexample which is not` + + ` visible to this module`, + }, + }, + { + name: "package default_visibility private does not override visibility public", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + package { + default_visibility: ["//visibility:private"], + } + + mock_library { + name: "libexample", + visibility: ["//visibility:public"], + }`), + "outsider/Blueprints": []byte(` + mock_library { + name: "liboutsider", + deps: ["libexample"], + }`), + }, + }, + { + name: "package default_visibility :__subpackages__", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + package { + default_visibility: [":__subpackages__"], + } + + mock_library { + name: "libexample", + }`), + "top/nested/Blueprints": []byte(` + mock_library { + name: "libnested", + deps: ["libexample"], + }`), + "outsider/Blueprints": []byte(` + mock_library { + name: "liboutsider", + deps: ["libexample"], + }`), + }, + expectedErrors: []string{ + `module "liboutsider" variant "android_common": depends on //top:libexample which is not` + + ` visible to this module`, + }, + }, + { + name: "package default_visibility inherited to subpackages", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + package { + default_visibility: ["//outsider"], + } + + mock_library { + name: "libexample", + visibility: [":__subpackages__"], + }`), + "top/nested/Blueprints": []byte(` + mock_library { + name: "libnested", + deps: ["libexample"], + }`), + "outsider/Blueprints": []byte(` + mock_library { + name: "liboutsider", + deps: ["libexample", "libnested"], + }`), + }, + expectedErrors: []string{ + `module "liboutsider" variant "android_common": depends on //top:libexample which is not` + + ` visible to this module`, + }, + }, + { + name: "package default_visibility inherited to subpackages", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + package { + default_visibility: ["//visibility:private"], + }`), + "top/nested/Blueprints": []byte(` + package { + default_visibility: ["//outsider"], + } + + mock_library { + name: "libnested", + }`), + "top/other/Blueprints": []byte(` + mock_library { + name: "libother", + }`), + "outsider/Blueprints": []byte(` + mock_library { + name: "liboutsider", + deps: ["libother", "libnested"], + }`), + }, + expectedErrors: []string{ + `module "liboutsider" variant "android_common": depends on //top/other:libother which is` + + ` not visible to this module`, + }, + }, + { + name: "verify that prebuilt dependencies are ignored for visibility reasons (not preferred)", + fs: map[string][]byte{ + "prebuilts/Blueprints": []byte(` + prebuilt { + name: "module", + visibility: ["//top/other"], + }`), + "top/sources/source_file": nil, + "top/sources/Blueprints": []byte(` + source { + name: "module", + visibility: ["//top/other"], + }`), + "top/other/source_file": nil, + "top/other/Blueprints": []byte(` + source { + name: "other", + deps: [":module"], + }`), + }, + }, + { + name: "verify that prebuilt dependencies are ignored for visibility reasons (preferred)", + fs: map[string][]byte{ + "prebuilts/Blueprints": []byte(` + prebuilt { + name: "module", + visibility: ["//top/other"], + prefer: true, + }`), + "top/sources/source_file": nil, + "top/sources/Blueprints": []byte(` + source { + name: "module", + visibility: ["//top/other"], + }`), + "top/other/source_file": nil, + "top/other/Blueprints": []byte(` + source { + name: "other", + deps: [":module"], + }`), + }, + }, + { + name: "ensure visibility properties are checked for correctness", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_parent { + name: "parent", + visibility: ["//top/nested"], + child: { + name: "libchild", + visibility: ["top/other"], + }, + }`), + }, + expectedErrors: []string{ + `module "parent": child.visibility: invalid visibility pattern "top/other"`, + }, + }, + { + name: "invalid visibility added to child detected during gather phase", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_parent { + name: "parent", + visibility: ["//top/nested"], + child: { + name: "libchild", + invalid_visibility: ["top/other"], + }, + }`), + }, + expectedErrors: []string{ + // That this error is reported against the child not the parent shows it was + // not being detected in the parent which is correct as invalid_visibility is + // purposely not added to the list of visibility properties to check, and was + // in fact detected in the child in the gather phase. Contrast this error message + // with the preceding one. + `module "libchild" \(created by module "parent"\): visibility: invalid visibility pattern "top/other"`, + }, + }, + { + name: "automatic visibility inheritance enabled", + fs: map[string][]byte{ + "top/Blueprints": []byte(` + mock_parent { + name: "parent", + visibility: ["//top/nested"], + child: { + name: "libchild", + visibility: ["//top/other"], + }, + }`), + "top/nested/Blueprints": []byte(` + mock_library { + name: "libnested", + deps: ["libchild"], + }`), + "top/other/Blueprints": []byte(` + mock_library { + name: "libother", + deps: ["libchild"], + }`), + }, + }, +} + +func TestVisibility(t *testing.T) { + for _, test := range visibilityTests { + t.Run(test.name, func(t *testing.T) { + ctx, errs := testVisibility(buildDir, test.fs) + + CheckErrorsAgainstExpectations(t, errs, test.expectedErrors) + + if test.effectiveVisibility != nil { + checkEffectiveVisibility(t, ctx, test.effectiveVisibility) + } + }) + } +} + +func checkEffectiveVisibility(t *testing.T, ctx *TestContext, effectiveVisibility map[qualifiedModuleName][]string) { + for moduleName, expectedRules := range effectiveVisibility { + rule := effectiveVisibilityRules(ctx.config, moduleName) + stringRules := rule.Strings() + if !reflect.DeepEqual(expectedRules, stringRules) { + t.Errorf("effective rules mismatch: expected %q, found %q", expectedRules, stringRules) + } + } +} + +func testVisibility(buildDir string, fs map[string][]byte) (*TestContext, []error) { + + // Create a new config per test as visibility information is stored in the config. + config := TestArchConfig(buildDir, nil, "", fs) + + ctx := NewTestArchContext() + ctx.RegisterModuleType("mock_library", newMockLibraryModule) + ctx.RegisterModuleType("mock_parent", newMockParentFactory) + ctx.RegisterModuleType("mock_defaults", defaultsFactory) + + // Order of the following method calls is significant. + RegisterPackageBuildComponents(ctx) + registerTestPrebuiltBuildComponents(ctx) + ctx.PreArchMutators(RegisterVisibilityRuleChecker) + ctx.PreArchMutators(RegisterDefaultsPreArchMutators) + ctx.PreArchMutators(RegisterVisibilityRuleGatherer) + ctx.PostDepsMutators(RegisterVisibilityRuleEnforcer) + ctx.Register(config) + + _, errs := ctx.ParseBlueprintsFiles(".") + if len(errs) > 0 { + return ctx, errs + } + + _, errs = ctx.PrepareBuildActions(config) + return ctx, errs +} + +type mockLibraryProperties struct { + Deps []string +} + +type mockLibraryModule struct { + ModuleBase + DefaultableModuleBase + properties mockLibraryProperties +} + +func newMockLibraryModule() Module { + m := &mockLibraryModule{} + m.AddProperties(&m.properties) + InitAndroidArchModule(m, HostAndDeviceSupported, MultilibCommon) + InitDefaultableModule(m) + return m +} + +type dependencyTag struct { + blueprint.BaseDependencyTag + name string +} + +func (j *mockLibraryModule) DepsMutator(ctx BottomUpMutatorContext) { + ctx.AddVariationDependencies(nil, dependencyTag{name: "mockdeps"}, j.properties.Deps...) +} + +func (p *mockLibraryModule) GenerateAndroidBuildActions(ModuleContext) { +} + +type mockDefaults struct { + ModuleBase + DefaultsModuleBase +} + +func defaultsFactory() Module { + m := &mockDefaults{} + InitDefaultsModule(m) + return m +} + +type mockParentProperties struct { + Child struct { + Name *string + + // Visibility to pass to the child module. + Visibility []string + + // Purposely not validated visibility to pass to the child. + Invalid_visibility []string + } +} + +type mockParent struct { + ModuleBase + DefaultableModuleBase + properties mockParentProperties +} + +func (p *mockParent) GenerateAndroidBuildActions(ModuleContext) { +} + +func newMockParentFactory() Module { + m := &mockParent{} + m.AddProperties(&m.properties) + InitAndroidArchModule(m, HostAndDeviceSupported, MultilibCommon) + InitDefaultableModule(m) + AddVisibilityProperty(m, "child.visibility", &m.properties.Child.Visibility) + + m.SetDefaultableHook(func(ctx DefaultableHookContext) { + visibility := m.properties.Child.Visibility + visibility = append(visibility, m.properties.Child.Invalid_visibility...) + ctx.CreateModule(newMockLibraryModule, &struct { + Name *string + Visibility []string + }{m.properties.Child.Name, visibility}) + }) + return m +} diff --git a/android/vts_config.go b/android/vts_config.go index c44b3a32e..77fb9fed9 100644 --- a/android/vts_config.go +++ b/android/vts_config.go @@ -17,6 +17,7 @@ package android import ( "fmt" "io" + "strings" ) func init() { @@ -26,6 +27,8 @@ func init() { type vtsConfigProperties struct { // Override the default (AndroidTest.xml) test manifest file name. Test_config *string + // Additional test suites to add the test to. + Test_suites []string `android:"arch_variant"` } type VtsConfig struct { @@ -41,16 +44,18 @@ func (me *VtsConfig) GenerateAndroidBuildActions(ctx ModuleContext) { func (me *VtsConfig) AndroidMk() AndroidMkData { androidMkData := AndroidMkData{ Class: "FAKE", - Include: "$(BUILD_SYSTEM)/android_vts_host_config.mk", + Include: "$(BUILD_SYSTEM)/suite_host_config.mk", OutputFile: OptionalPathForPath(me.OutputFilePath), } - if me.properties.Test_config != nil { - androidMkData.Extra = []AndroidMkExtraFunc{ - func(w io.Writer, outputFile Path) { + androidMkData.Extra = []AndroidMkExtraFunc{ + func(w io.Writer, outputFile Path) { + if me.properties.Test_config != nil { fmt.Fprintf(w, "LOCAL_TEST_CONFIG := %s\n", *me.properties.Test_config) - }, - } + } + fmt.Fprintf(w, "LOCAL_COMPATIBILITY_SUITE := vts10 %s\n", + strings.Join(me.properties.Test_suites, " ")) + }, } return androidMkData } @@ -59,7 +64,7 @@ func InitVtsConfigModule(me *VtsConfig) { me.AddProperties(&me.properties) } -// vts_config generates a Vendor Test Suite (VTS) configuration file from the +// vts_config generates a Vendor Test Suite (VTS10) configuration file from the // <test_config> xml file and stores it in a subdirectory of $(HOST_OUT). func VtsConfigFactory() Module { module := &VtsConfig{} diff --git a/android/vts_config_test.go b/android/vts_config_test.go index 7d4c9b1cf..254fa92fb 100644 --- a/android/vts_config_test.go +++ b/android/vts_config_test.go @@ -15,27 +15,15 @@ package android import ( - "io/ioutil" - "os" "testing" ) func testVtsConfig(test *testing.T, bpFileContents string) *TestContext { - buildDir, err := ioutil.TempDir("", "soong_vts_config_test") - if err != nil { - test.Fatal(err) - } - - config := TestArchConfig(buildDir, nil) - defer func() { os.RemoveAll(buildDir) }() + config := TestArchConfig(buildDir, nil, bpFileContents, nil) ctx := NewTestArchContext() - ctx.RegisterModuleType("vts_config", ModuleFactoryAdaptor(VtsConfigFactory)) - ctx.Register() - mockFiles := map[string][]byte{ - "Android.bp": []byte(bpFileContents), - } - ctx.MockFileSystem(mockFiles) + ctx.RegisterModuleType("vts_config", VtsConfigFactory) + ctx.Register(config) _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) FailIfErrored(test, errs) _, errs = ctx.PrepareBuildActions(config) diff --git a/android/writedocs.go b/android/writedocs.go index 7262ad810..9e43e80a6 100644 --- a/android/writedocs.go +++ b/android/writedocs.go @@ -69,9 +69,5 @@ func (c *docsSingleton) GenerateBuildActions(ctx SingletonContext) { }) // Add a phony target for building the documentation - ctx.Build(pctx, BuildParams{ - Rule: blueprint.Phony, - Output: PathForPhony(ctx, "soong_docs"), - Input: docsFile, - }) + ctx.Phony("soong_docs", docsFile) } diff --git a/androidmk/Android.bp b/androidmk/Android.bp index 1d939b0c5..70fc1f75c 100644 --- a/androidmk/Android.bp +++ b/androidmk/Android.bp @@ -19,12 +19,23 @@ blueprint_go_binary { name: "androidmk", srcs: [ - "cmd/androidmk/android.go", - "cmd/androidmk/androidmk.go", - "cmd/androidmk/values.go", + "cmd/androidmk.go", + ], + deps: [ + "androidmk-lib", + ], +} + +bootstrap_go_package { + name: "androidmk-lib", + pkgPath: "android/soong/androidmk/androidmk", + srcs: [ + "androidmk/android.go", + "androidmk/androidmk.go", + "androidmk/values.go", ], testSrcs: [ - "cmd/androidmk/androidmk_test.go", + "androidmk/androidmk_test.go", ], deps: [ "androidmk-parser", diff --git a/androidmk/cmd/androidmk/android.go b/androidmk/androidmk/android.go index 52bcf9c99..88609849d 100644 --- a/androidmk/cmd/androidmk/android.go +++ b/androidmk/androidmk/android.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package androidmk import ( mkparser "android/soong/androidmk/parser" @@ -123,13 +123,16 @@ func init() { "LOCAL_SYSTEM_SHARED_LIBRARIES": "system_shared_libs", "LOCAL_ASFLAGS": "asflags", "LOCAL_CLANG_ASFLAGS": "clang_asflags", + "LOCAL_COMPATIBILITY_SUPPORT_FILES": "data", "LOCAL_CONLYFLAGS": "conlyflags", "LOCAL_CPPFLAGS": "cppflags", "LOCAL_REQUIRED_MODULES": "required", + "LOCAL_HOST_REQUIRED_MODULES": "host_required", + "LOCAL_TARGET_REQUIRED_MODULES": "target_required", "LOCAL_OVERRIDES_MODULES": "overrides", "LOCAL_LDLIBS": "host_ldlibs", "LOCAL_CLANG_CFLAGS": "clang_cflags", - "LOCAL_YACCFLAGS": "yaccflags", + "LOCAL_YACCFLAGS": "yacc.flags", "LOCAL_SANITIZE_RECOVER": "sanitize.recover", "LOCAL_LOGTAGS_FILES": "logtags", "LOCAL_EXPORT_HEADER_LIBRARY_HEADERS": "export_header_lib_headers", @@ -144,6 +147,7 @@ func init() { "LOCAL_RENDERSCRIPT_FLAGS": "renderscript.flags", "LOCAL_JAVA_RESOURCE_DIRS": "java_resource_dirs", + "LOCAL_JAVA_RESOURCE_FILES": "java_resources", "LOCAL_JAVACFLAGS": "javacflags", "LOCAL_ERROR_PRONE_FLAGS": "errorprone.javacflags", "LOCAL_DX_FLAGS": "dxflags", @@ -169,6 +173,8 @@ func init() { // Jacoco filters: "LOCAL_JACK_COVERAGE_INCLUDE_FILTER": "jacoco.include_filter", "LOCAL_JACK_COVERAGE_EXCLUDE_FILTER": "jacoco.exclude_filter", + + "LOCAL_FULL_LIBS_MANIFEST_FILES": "additional_manifests", }) addStandardProperties(bpparser.BoolType, @@ -181,7 +187,6 @@ func init() { "LOCAL_NO_CRT": "nocrt", "LOCAL_ALLOW_UNDEFINED_SYMBOLS": "allow_undefined_symbols", "LOCAL_RTTI_FLAG": "rtti", - "LOCAL_NO_STANDARD_LIBRARIES": "no_standard_libs", "LOCAL_PACK_MODULE_RELOCATIONS": "pack_relocations", "LOCAL_TIDY": "tidy", "LOCAL_USE_CLANG_LLD": "use_clang_lld", @@ -189,10 +194,11 @@ func init() { "LOCAL_VENDOR_MODULE": "vendor", "LOCAL_ODM_MODULE": "device_specific", "LOCAL_PRODUCT_MODULE": "product_specific", - "LOCAL_PRODUCT_SERVICES_MODULE": "product_services_specific", + "LOCAL_SYSTEM_EXT_MODULE": "system_ext_specific", "LOCAL_EXPORT_PACKAGE_RESOURCES": "export_package_resources", "LOCAL_PRIVILEGED_MODULE": "privileged", "LOCAL_AAPT_INCLUDE_ALL_RESOURCES": "aapt_include_all_resources", + "LOCAL_DONT_MERGE_MANIFESTS": "dont_merge_manifests", "LOCAL_USE_EMBEDDED_NATIVE_LIBS": "use_embedded_native_libs", "LOCAL_USE_EMBEDDED_DEX": "use_embedded_dex", @@ -331,15 +337,6 @@ func classifyLocalOrGlobalPath(value bpparser.Expression) (string, bpparser.Expr } } -func sortedMapKeys(inputMap map[string]string) (sortedKeys []string) { - keys := make([]string, 0, len(inputMap)) - for key := range inputMap { - keys = append(keys, key) - } - sort.Strings(keys) - return keys -} - // splitAndAssign splits a Make list into components and then // creates the corresponding variable assignments. func splitAndAssign(ctx variableAssignmentContext, splitFunc listSplitFunc, namesByClassification map[string]string) error { @@ -353,7 +350,13 @@ func splitAndAssign(ctx variableAssignmentContext, splitFunc listSplitFunc, name return err } - for _, nameClassification := range sortedMapKeys(namesByClassification) { + var classifications []string + for classification := range namesByClassification { + classifications = append(classifications, classification) + } + sort.Strings(classifications) + + for _, nameClassification := range classifications { name := namesByClassification[nameClassification] if component, ok := lists[nameClassification]; ok && !emptyList(component) { err = setVariable(ctx.file, ctx.append, ctx.prefix, name, component, true) @@ -525,7 +528,7 @@ func sanitize(sub string) func(ctx variableAssignmentContext) error { ctx.file.errorf(ctx.mkvalue, "unsupported sanitize expression") case *bpparser.String: switch v.Value { - case "never", "address", "coverage", "thread", "undefined", "cfi": + case "never", "address", "fuzzer", "thread", "undefined", "cfi": bpTrue := &bpparser.Bool{ Value: true, } @@ -608,8 +611,8 @@ func prebuiltModulePath(ctx variableAssignmentContext) error { return fmt.Errorf("Cannot handle appending to LOCAL_MODULE_PATH") } // Analyze value in order to set the correct values for the 'device_specific', - // 'product_specific', 'product_services_specific' 'vendor'/'soc_specific', - // 'product_services_specific' attribute. Two cases are allowed: + // 'product_specific', 'system_ext_specific' 'vendor'/'soc_specific', + // 'system_ext_specific' attribute. Two cases are allowed: // $(VAR)/<literal-value> // $(PRODUCT_OUT)/$(TARGET_COPY_OUT_VENDOR)/<literal-value> // The last case is equivalent to $(TARGET_OUT_VENDOR)/<literal-value> @@ -933,6 +936,7 @@ var prebuiltTypes = map[string]string{ "STATIC_LIBRARIES": "cc_prebuilt_library_static", "EXECUTABLES": "cc_prebuilt_binary", "JAVA_LIBRARIES": "java_import", + "APPS": "android_app_import", "ETC": "prebuilt_etc", } diff --git a/androidmk/cmd/androidmk/androidmk.go b/androidmk/androidmk/androidmk.go index d2a84d139..9d0c3ac9f 100644 --- a/androidmk/cmd/androidmk/androidmk.go +++ b/androidmk/androidmk/androidmk.go @@ -12,14 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package androidmk import ( "bytes" - "flag" "fmt" - "io/ioutil" - "os" "strings" "text/scanner" @@ -30,13 +27,6 @@ import ( bpparser "github.com/google/blueprint/parser" ) -var usage = func() { - fmt.Fprintf(os.Stderr, "usage: androidmk [flags] <inputFile>\n"+ - "\nandroidmk parses <inputFile> as an Android.mk file and attempts to output an analogous Android.bp file (to standard out)\n") - flag.PrintDefaults() - os.Exit(1) -} - // TODO: non-expanded variables with expressions type bpFile struct { @@ -118,31 +108,7 @@ type conditional struct { eq bool } -func main() { - flag.Usage = usage - flag.Parse() - if len(flag.Args()) != 1 { - usage() - } - filePathToRead := flag.Arg(0) - b, err := ioutil.ReadFile(filePathToRead) - if err != nil { - fmt.Println(err.Error()) - return - } - - output, errs := convertFile(os.Args[1], bytes.NewBuffer(b)) - if len(errs) > 0 { - for _, err := range errs { - fmt.Fprintln(os.Stderr, "ERROR: ", err) - } - os.Exit(1) - } - - fmt.Print(output) -} - -func convertFile(filename string, buffer *bytes.Buffer) (string, []error) { +func ConvertFile(filename string, buffer *bytes.Buffer) (string, []error) { p := mkparser.NewParser(filename, buffer) nodes, errs := p.Parse() diff --git a/androidmk/cmd/androidmk/androidmk_test.go b/androidmk/androidmk/androidmk_test.go index f2dc6ff21..7e1a72cd6 100644 --- a/androidmk/cmd/androidmk/androidmk_test.go +++ b/androidmk/androidmk/androidmk_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package androidmk import ( "bytes" @@ -577,6 +577,10 @@ include $(call all-makefiles-under,$(LOCAL_PATH)) include $(CLEAR_VARS) LOCAL_SRC_FILES := $(call all-java-files-under, src gen) include $(BUILD_STATIC_JAVA_LIBRARY) + + include $(CLEAR_VARS) + LOCAL_JAVA_RESOURCE_FILES := foo bar + include $(BUILD_STATIC_JAVA_LIBRARY) `, expected: ` java_library { @@ -604,6 +608,13 @@ include $(call all-makefiles-under,$(LOCAL_PATH)) "gen/**/*.java", ], } + + java_library { + java_resources: [ + "foo", + "bar", + ], + } `, }, { @@ -798,6 +809,7 @@ include $(CLEAR_VARS) LOCAL_PACKAGE_NAME := FooTest LOCAL_COMPATIBILITY_SUITE := cts LOCAL_CTS_TEST_PACKAGE := foo.bar +LOCAL_COMPATIBILITY_SUPPORT_FILES := file1 include $(BUILD_CTS_PACKAGE) `, expected: ` @@ -806,6 +818,7 @@ android_test { defaults: ["cts_defaults"], test_suites: ["cts"], + data: ["file1"], } `, }, @@ -868,7 +881,6 @@ prebuilt_etc { } `, }, - { desc: "prebuilt_etc_PRODUCT_OUT/system/etc", in: ` @@ -945,37 +957,37 @@ prebuilt_etc { `, }, { - desc: "prebuilt_etc_TARGET_OUT_PRODUCT_SERVICES/etc", + desc: "prebuilt_etc_TARGET_OUT_SYSTEM_EXT/etc", in: ` include $(CLEAR_VARS) LOCAL_MODULE := etc.test1 LOCAL_MODULE_CLASS := ETC -LOCAL_MODULE_PATH := $(TARGET_OUT_PRODUCT_SERVICES)/etc/foo/bar +LOCAL_MODULE_PATH := $(TARGET_OUT_SYSTEM_EXT)/etc/foo/bar include $(BUILD_PREBUILT) `, expected: ` prebuilt_etc { name: "etc.test1", sub_dir: "foo/bar", - product_services_specific: true, + system_ext_specific: true, } `, }, { - desc: "prebuilt_etc_TARGET_OUT_PRODUCT_SERVICES_ETC", + desc: "prebuilt_etc_TARGET_OUT_SYSTEM_EXT_ETC", in: ` include $(CLEAR_VARS) LOCAL_MODULE := etc.test1 LOCAL_MODULE_CLASS := ETC -LOCAL_MODULE_PATH := $(TARGET_OUT_PRODUCT_SERVICES_ETC)/foo/bar +LOCAL_MODULE_PATH := $(TARGET_OUT_SYSTEM_EXT_ETC)/foo/bar include $(BUILD_PREBUILT) `, expected: ` prebuilt_etc { name: "etc.test1", sub_dir: "foo/bar", - product_services_specific: true, + system_ext_specific: true, } @@ -1054,6 +1066,197 @@ prebuilt_etc { `, }, { + desc: "prebuilt_usr_share", + in: ` +include $(CLEAR_VARS) +LOCAL_MODULE := foo +LOCAL_MODULE_CLASS := ETC +LOCAL_MODULE_PATH := $(TARGET_OUT)/usr/share +LOCAL_SRC_FILES := foo.txt +include $(BUILD_PREBUILT) +`, + expected: ` +prebuilt_usr_share { + name: "foo", + + src: "foo.txt", +} +`, + }, + { + desc: "prebuilt_usr_share subdir_bar", + in: ` +include $(CLEAR_VARS) +LOCAL_MODULE := foo +LOCAL_MODULE_CLASS := ETC +LOCAL_MODULE_PATH := $(TARGET_OUT)/usr/share/bar +LOCAL_SRC_FILES := foo.txt +include $(BUILD_PREBUILT) +`, + expected: ` +prebuilt_usr_share { + name: "foo", + + src: "foo.txt", + sub_dir: "bar", +} +`, + }, + { + desc: "prebuilt_usr_share_host", + in: ` +include $(CLEAR_VARS) +LOCAL_MODULE := foo +LOCAL_MODULE_CLASS := ETC +LOCAL_MODULE_PATH := $(HOST_OUT)/usr/share +LOCAL_SRC_FILES := foo.txt +include $(BUILD_PREBUILT) +`, + expected: ` +prebuilt_usr_share_host { + name: "foo", + + src: "foo.txt", +} +`, + }, + { + desc: "prebuilt_font", + in: ` +include $(CLEAR_VARS) +LOCAL_MODULE := font.ttf +LOCAL_SRC_FILES := $(LOCAL_MODULE) +LOCAL_MODULE_CLASS := ETC +LOCAL_MODULE_TAGS := optional +LOCAL_MODULE_PATH := $(TARGET_OUT)/fonts +include $(BUILD_PREBUILT) +`, + expected: ` +prebuilt_font { + name: "font.ttf", + src: "font.ttf", + +} +`, + }, + { + desc: "prebuilt_font", + in: ` +include $(CLEAR_VARS) +LOCAL_MODULE := font.ttf +LOCAL_SRC_FILES := $(LOCAL_MODULE) +LOCAL_MODULE_CLASS := ETC +LOCAL_MODULE_TAGS := optional +LOCAL_MODULE_PATH := $(TARGET_OUT_PRODUCT)/fonts +include $(BUILD_PREBUILT) +`, + expected: ` +prebuilt_font { + name: "font.ttf", + src: "font.ttf", + product_specific: true, + +} +`, + }, + { + desc: "prebuilt_usr_share_host subdir_bar", + in: ` +include $(CLEAR_VARS) +LOCAL_MODULE := foo +LOCAL_MODULE_CLASS := ETC +LOCAL_MODULE_PATH := $(HOST_OUT)/usr/share/bar +LOCAL_SRC_FILES := foo.txt +include $(BUILD_PREBUILT) +`, + expected: ` +prebuilt_usr_share_host { + name: "foo", + + src: "foo.txt", + sub_dir: "bar", +} +`, + }, + { + desc: "prebuilt_firmware subdir_bar in $(TARGET_OUT_ETC)", + in: ` +include $(CLEAR_VARS) +LOCAL_MODULE := foo +LOCAL_MODULE_CLASS := ETC +LOCAL_MODULE_PATH := $(TARGET_OUT_ETC)/firmware/bar +LOCAL_SRC_FILES := foo.fw +include $(BUILD_PREBUILT) +`, + expected: ` +prebuilt_firmware { + name: "foo", + + src: "foo.fw", + sub_dir: "bar", +} +`, + }, + { + desc: "prebuilt_firmware subdir_bar in $(TARGET_OUT)", + in: ` +include $(CLEAR_VARS) +LOCAL_MODULE := foo +LOCAL_MODULE_CLASS := ETC +LOCAL_MODULE_PATH := $(TARGET_OUT)/etc/firmware/bar +LOCAL_SRC_FILES := foo.fw +include $(BUILD_PREBUILT) +`, + expected: ` +prebuilt_firmware { + name: "foo", + + src: "foo.fw", + sub_dir: "bar", +} +`, + }, + { + desc: "prebuilt_firmware subdir_bar in $(TARGET_OUT_VENDOR)", + in: ` +include $(CLEAR_VARS) +LOCAL_MODULE := foo +LOCAL_MODULE_CLASS := ETC +LOCAL_MODULE_PATH := $(TARGET_OUT_VENDOR)/firmware/bar +LOCAL_SRC_FILES := foo.fw +include $(BUILD_PREBUILT) +`, + expected: ` +prebuilt_firmware { + name: "foo", + + src: "foo.fw", + sub_dir: "bar", + proprietary: true, +} +`, + }, + { + desc: "prebuilt_firmware subdir_bar in $(TARGET_OUT)/vendor", + in: ` +include $(CLEAR_VARS) +LOCAL_MODULE := foo +LOCAL_MODULE_CLASS := ETC +LOCAL_MODULE_PATH := $(TARGET_OUT)/vendor/firmware/bar +LOCAL_SRC_FILES := foo.fw +include $(BUILD_PREBUILT) +`, + expected: ` +prebuilt_firmware { + name: "foo", + + src: "foo.fw", + sub_dir: "bar", + proprietary: true, +} +`, + }, + { desc: "vts_config", in: ` include $(CLEAR_VARS) @@ -1112,6 +1315,32 @@ android_app { } `, }, + { + desc: "android_app_import", + in: ` +include $(CLEAR_VARS) +LOCAL_MODULE := foo +LOCAL_SRC_FILES := foo.apk +LOCAL_PRIVILEGED_MODULE := true +LOCAL_MODULE_CLASS := APPS +LOCAL_MODULE_TAGS := optional +LOCAL_DEX_PREOPT := false +include $(BUILD_PREBUILT) +`, + expected: ` +android_app_import { + name: "foo", + + privileged: true, + + dex_preopt: { + enabled: false, + }, + apk: "foo.apk", + +} +`, + }, } func TestEndToEnd(t *testing.T) { @@ -1121,7 +1350,7 @@ func TestEndToEnd(t *testing.T) { t.Error(err) } - got, errs := convertFile(fmt.Sprintf("<testcase %d>", i), bytes.NewBufferString(test.in)) + got, errs := ConvertFile(fmt.Sprintf("<testcase %d>", i), bytes.NewBufferString(test.in)) if len(errs) > 0 { t.Errorf("Unexpected errors: %q", errs) continue diff --git a/androidmk/cmd/androidmk/values.go b/androidmk/androidmk/values.go index 90f2e74b3..6b18a651e 100644 --- a/androidmk/cmd/androidmk/values.go +++ b/androidmk/androidmk/values.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package androidmk import ( "fmt" diff --git a/androidmk/cmd/androidmk.go b/androidmk/cmd/androidmk.go new file mode 100644 index 000000000..00488eb9f --- /dev/null +++ b/androidmk/cmd/androidmk.go @@ -0,0 +1,56 @@ +// Copyright 2017 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "flag" + "fmt" + "io/ioutil" + "os" + + "android/soong/androidmk/androidmk" +) + +var usage = func() { + fmt.Fprintf(os.Stderr, "usage: androidmk [flags] <inputFile>\n"+ + "\nandroidmk parses <inputFile> as an Android.mk file and attempts to output an analogous Android.bp file (to standard out)\n") + flag.PrintDefaults() + os.Exit(1) +} + +func main() { + flag.Usage = usage + flag.Parse() + if len(flag.Args()) != 1 { + usage() + } + filePathToRead := flag.Arg(0) + b, err := ioutil.ReadFile(filePathToRead) + if err != nil { + fmt.Println(err.Error()) + return + } + + output, errs := androidmk.ConvertFile(os.Args[1], bytes.NewBuffer(b)) + if len(errs) > 0 { + for _, err := range errs { + fmt.Fprintln(os.Stderr, "ERROR: ", err) + } + os.Exit(1) + } + + fmt.Print(output) +} diff --git a/apex/Android.bp b/apex/Android.bp new file mode 100644 index 000000000..144f44197 --- /dev/null +++ b/apex/Android.bp @@ -0,0 +1,27 @@ +bootstrap_go_package { + name: "soong-apex", + pkgPath: "android/soong/apex", + deps: [ + "blueprint", + "soong", + "soong-android", + "soong-cc", + "soong-java", + "soong-python", + "soong-sh", + ], + srcs: [ + "androidmk.go", + "apex.go", + "apex_singleton.go", + "builder.go", + "key.go", + "prebuilt.go", + "vndk.go", + ], + testSrcs: [ + "apex_test.go", + "vndk_test.go", + ], + pluginFor: ["soong_build"], +} diff --git a/apex/TEST_MAPPING b/apex/TEST_MAPPING new file mode 100644 index 000000000..d0223d1f6 --- /dev/null +++ b/apex/TEST_MAPPING @@ -0,0 +1,7 @@ +{ + "imports": [ + { + "path": "system/apex/apexd" + } + ] +}
\ No newline at end of file diff --git a/apex/androidmk.go b/apex/androidmk.go new file mode 100644 index 000000000..e4cdef0bc --- /dev/null +++ b/apex/androidmk.go @@ -0,0 +1,350 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apex + +import ( + "fmt" + "io" + "path/filepath" + "strings" + + "android/soong/android" + "android/soong/cc" + "android/soong/java" + + "github.com/google/blueprint/proptools" +) + +func (a *apexBundle) AndroidMk() android.AndroidMkData { + if a.properties.HideFromMake { + return android.AndroidMkData{ + Disabled: true, + } + } + writers := []android.AndroidMkData{} + writers = append(writers, a.androidMkForType()) + return android.AndroidMkData{ + Custom: func(w io.Writer, name, prefix, moduleDir string, data android.AndroidMkData) { + for _, data := range writers { + data.Custom(w, name, prefix, moduleDir, data) + } + }} +} + +func (a *apexBundle) androidMkForFiles(w io.Writer, apexBundleName, apexName, moduleDir string) []string { + // apexBundleName comes from the 'name' property; apexName comes from 'apex_name' property. + // An apex is installed to /system/apex/<apexBundleName> and is activated at /apex/<apexName> + // In many cases, the two names are the same, but could be different in general. + + moduleNames := []string{} + apexType := a.properties.ApexType + // To avoid creating duplicate build rules, run this function only when primaryApexType is true + // to install symbol files in $(PRODUCT_OUT}/apex. + // And if apexType is flattened, run this function to install files in $(PRODUCT_OUT}/system/apex. + if !a.primaryApexType && apexType != flattenedApex { + return moduleNames + } + + // b/140136207. When there are overriding APEXes for a VNDK APEX, the symbols file for the overridden + // APEX and the overriding APEX will have the same installation paths at /apex/com.android.vndk.v<ver> + // as their apexName will be the same. To avoid the path conflicts, skip installing the symbol files + // for the overriding VNDK APEXes. + symbolFilesNotNeeded := a.vndkApex && len(a.overridableProperties.Overrides) > 0 + if symbolFilesNotNeeded && apexType != flattenedApex { + return moduleNames + } + + var postInstallCommands []string + for _, fi := range a.filesInfo { + if a.linkToSystemLib && fi.transitiveDep && fi.AvailableToPlatform() { + // TODO(jiyong): pathOnDevice should come from fi.module, not being calculated here + linkTarget := filepath.Join("/system", fi.Path()) + linkPath := filepath.Join(a.installDir.ToMakePath().String(), apexBundleName, fi.Path()) + mkdirCmd := "mkdir -p " + filepath.Dir(linkPath) + linkCmd := "ln -sfn " + linkTarget + " " + linkPath + postInstallCommands = append(postInstallCommands, mkdirCmd, linkCmd) + } + } + + for _, fi := range a.filesInfo { + if ccMod, ok := fi.module.(*cc.Module); ok && ccMod.Properties.HideFromMake { + continue + } + + linkToSystemLib := a.linkToSystemLib && fi.transitiveDep && fi.AvailableToPlatform() + + var moduleName string + if linkToSystemLib { + moduleName = fi.moduleName + } else { + moduleName = fi.moduleName + "." + apexBundleName + a.suffix + } + + if !android.InList(moduleName, moduleNames) { + moduleNames = append(moduleNames, moduleName) + } + + if linkToSystemLib { + // No need to copy the file since it's linked to the system file + continue + } + + fmt.Fprintln(w, "\ninclude $(CLEAR_VARS)") + if fi.moduleDir != "" { + fmt.Fprintln(w, "LOCAL_PATH :=", fi.moduleDir) + } else { + fmt.Fprintln(w, "LOCAL_PATH :=", moduleDir) + } + fmt.Fprintln(w, "LOCAL_MODULE :=", moduleName) + if fi.module != nil && fi.module.Owner() != "" { + fmt.Fprintln(w, "LOCAL_MODULE_OWNER :=", fi.module.Owner()) + } + // /apex/<apex_name>/{lib|framework|...} + pathWhenActivated := filepath.Join("$(PRODUCT_OUT)", "apex", apexName, fi.installDir) + var modulePath string + if apexType == flattenedApex { + // /system/apex/<name>/{lib|framework|...} + modulePath = filepath.Join(a.installDir.ToMakePath().String(), apexBundleName, fi.installDir) + fmt.Fprintln(w, "LOCAL_MODULE_PATH :=", modulePath) + if a.primaryApexType && !symbolFilesNotNeeded { + fmt.Fprintln(w, "LOCAL_SOONG_SYMBOL_PATH :=", pathWhenActivated) + } + if len(fi.symlinks) > 0 { + fmt.Fprintln(w, "LOCAL_MODULE_SYMLINKS :=", strings.Join(fi.symlinks, " ")) + } + + if fi.module != nil && fi.module.NoticeFile().Valid() { + fmt.Fprintln(w, "LOCAL_NOTICE_FILE :=", fi.module.NoticeFile().Path().String()) + } + } else { + modulePath = pathWhenActivated + fmt.Fprintln(w, "LOCAL_MODULE_PATH :=", pathWhenActivated) + + // For non-flattend APEXes, the merged notice file is attached to the APEX itself. + // We don't need to have notice file for the individual modules in it. Otherwise, + // we will have duplicated notice entries. + fmt.Fprintln(w, "LOCAL_NO_NOTICE_FILE := true") + } + fmt.Fprintln(w, "LOCAL_PREBUILT_MODULE_FILE :=", fi.builtFile.String()) + fmt.Fprintln(w, "LOCAL_MODULE_CLASS :=", fi.class.NameInMake()) + if fi.module != nil { + archStr := fi.module.Target().Arch.ArchType.String() + host := false + switch fi.module.Target().Os.Class { + case android.Host: + if fi.module.Target().Arch.ArchType != android.Common { + fmt.Fprintln(w, "LOCAL_MODULE_HOST_ARCH :=", archStr) + } + host = true + case android.HostCross: + if fi.module.Target().Arch.ArchType != android.Common { + fmt.Fprintln(w, "LOCAL_MODULE_HOST_CROSS_ARCH :=", archStr) + } + host = true + case android.Device: + if fi.module.Target().Arch.ArchType != android.Common { + fmt.Fprintln(w, "LOCAL_MODULE_TARGET_ARCH :=", archStr) + } + } + if host { + makeOs := fi.module.Target().Os.String() + if fi.module.Target().Os == android.Linux || fi.module.Target().Os == android.LinuxBionic { + makeOs = "linux" + } + fmt.Fprintln(w, "LOCAL_MODULE_HOST_OS :=", makeOs) + fmt.Fprintln(w, "LOCAL_IS_HOST_MODULE := true") + } + } + if fi.jacocoReportClassesFile != nil { + fmt.Fprintln(w, "LOCAL_SOONG_JACOCO_REPORT_CLASSES_JAR :=", fi.jacocoReportClassesFile.String()) + } + switch fi.class { + case javaSharedLib: + // soong_java_prebuilt.mk sets LOCAL_MODULE_SUFFIX := .jar Therefore + // we need to remove the suffix from LOCAL_MODULE_STEM, otherwise + // we will have foo.jar.jar + fmt.Fprintln(w, "LOCAL_MODULE_STEM :=", strings.TrimSuffix(fi.Stem(), ".jar")) + if javaModule, ok := fi.module.(java.ApexDependency); ok { + fmt.Fprintln(w, "LOCAL_SOONG_CLASSES_JAR :=", javaModule.ImplementationAndResourcesJars()[0].String()) + fmt.Fprintln(w, "LOCAL_SOONG_HEADER_JAR :=", javaModule.HeaderJars()[0].String()) + } else { + fmt.Fprintln(w, "LOCAL_SOONG_CLASSES_JAR :=", fi.builtFile.String()) + fmt.Fprintln(w, "LOCAL_SOONG_HEADER_JAR :=", fi.builtFile.String()) + } + fmt.Fprintln(w, "LOCAL_SOONG_DEX_JAR :=", fi.builtFile.String()) + fmt.Fprintln(w, "LOCAL_DEX_PREOPT := false") + fmt.Fprintln(w, "include $(BUILD_SYSTEM)/soong_java_prebuilt.mk") + case app: + fmt.Fprintln(w, "LOCAL_CERTIFICATE :=", fi.certificate.AndroidMkString()) + // soong_app_prebuilt.mk sets LOCAL_MODULE_SUFFIX := .apk Therefore + // we need to remove the suffix from LOCAL_MODULE_STEM, otherwise + // we will have foo.apk.apk + fmt.Fprintln(w, "LOCAL_MODULE_STEM :=", strings.TrimSuffix(fi.Stem(), ".apk")) + if app, ok := fi.module.(*java.AndroidApp); ok { + if jniCoverageOutputs := app.JniCoverageOutputs(); len(jniCoverageOutputs) > 0 { + fmt.Fprintln(w, "LOCAL_PREBUILT_COVERAGE_ARCHIVE :=", strings.Join(jniCoverageOutputs.Strings(), " ")) + } + if jniLibSymbols := app.JNISymbolsInstalls(modulePath); len(jniLibSymbols) > 0 { + fmt.Fprintln(w, "LOCAL_SOONG_JNI_LIBS_SYMBOLS :=", jniLibSymbols.String()) + } + } + fmt.Fprintln(w, "include $(BUILD_SYSTEM)/soong_app_prebuilt.mk") + case appSet: + as, ok := fi.module.(*java.AndroidAppSet) + if !ok { + panic(fmt.Sprintf("Expected %s to be AndroidAppSet", fi.module)) + } + fmt.Fprintln(w, "LOCAL_APK_SET_MASTER_FILE :=", as.MasterFile()) + fmt.Fprintln(w, "LOCAL_APKCERTS_FILE :=", as.APKCertsFile().String()) + fmt.Fprintln(w, "include $(BUILD_SYSTEM)/soong_android_app_set.mk") + case nativeSharedLib, nativeExecutable, nativeTest: + fmt.Fprintln(w, "LOCAL_MODULE_STEM :=", fi.Stem()) + if ccMod, ok := fi.module.(*cc.Module); ok { + if ccMod.UnstrippedOutputFile() != nil { + fmt.Fprintln(w, "LOCAL_SOONG_UNSTRIPPED_BINARY :=", ccMod.UnstrippedOutputFile().String()) + } + ccMod.AndroidMkWriteAdditionalDependenciesForSourceAbiDiff(w) + if ccMod.CoverageOutputFile().Valid() { + fmt.Fprintln(w, "LOCAL_PREBUILT_COVERAGE_ARCHIVE :=", ccMod.CoverageOutputFile().String()) + } + } + fmt.Fprintln(w, "include $(BUILD_SYSTEM)/soong_cc_prebuilt.mk") + default: + fmt.Fprintln(w, "LOCAL_MODULE_STEM :=", fi.Stem()) + if fi.builtFile == a.manifestPbOut && apexType == flattenedApex { + if a.primaryApexType { + // Make apex_manifest.pb module for this APEX to override all other + // modules in the APEXes being overridden by this APEX + var patterns []string + for _, o := range a.overridableProperties.Overrides { + patterns = append(patterns, "%."+o+a.suffix) + } + fmt.Fprintln(w, "LOCAL_OVERRIDES_MODULES :=", strings.Join(patterns, " ")) + + if len(a.compatSymlinks) > 0 { + // For flattened apexes, compat symlinks are attached to apex_manifest.json which is guaranteed for every apex + postInstallCommands = append(postInstallCommands, a.compatSymlinks...) + } + } + if len(postInstallCommands) > 0 { + fmt.Fprintln(w, "LOCAL_POST_INSTALL_CMD :=", strings.Join(postInstallCommands, " && ")) + } + } + fmt.Fprintln(w, "include $(BUILD_PREBUILT)") + } + + // m <module_name> will build <module_name>.<apex_name> as well. + if fi.moduleName != moduleName && a.primaryApexType { + fmt.Fprintln(w, ".PHONY: "+fi.moduleName) + fmt.Fprintln(w, fi.moduleName+": "+moduleName) + } + } + return moduleNames +} + +func (a *apexBundle) writeRequiredModules(w io.Writer) { + var required []string + var targetRequired []string + var hostRequired []string + for _, fi := range a.filesInfo { + required = append(required, fi.requiredModuleNames...) + targetRequired = append(targetRequired, fi.targetRequiredModuleNames...) + hostRequired = append(hostRequired, fi.hostRequiredModuleNames...) + } + + if len(required) > 0 { + fmt.Fprintln(w, "LOCAL_REQUIRED_MODULES +=", strings.Join(required, " ")) + } + if len(targetRequired) > 0 { + fmt.Fprintln(w, "LOCAL_TARGET_REQUIRED_MODULES +=", strings.Join(targetRequired, " ")) + } + if len(hostRequired) > 0 { + fmt.Fprintln(w, "LOCAL_HOST_REQUIRED_MODULES +=", strings.Join(hostRequired, " ")) + } +} + +func (a *apexBundle) androidMkForType() android.AndroidMkData { + return android.AndroidMkData{ + Custom: func(w io.Writer, name, prefix, moduleDir string, data android.AndroidMkData) { + moduleNames := []string{} + apexType := a.properties.ApexType + if a.installable() { + apexName := proptools.StringDefault(a.properties.Apex_name, name) + moduleNames = a.androidMkForFiles(w, name, apexName, moduleDir) + } + + if apexType == flattenedApex { + // Only image APEXes can be flattened. + fmt.Fprintln(w, "\ninclude $(CLEAR_VARS)") + fmt.Fprintln(w, "LOCAL_PATH :=", moduleDir) + fmt.Fprintln(w, "LOCAL_MODULE :=", name+a.suffix) + if len(moduleNames) > 0 { + fmt.Fprintln(w, "LOCAL_REQUIRED_MODULES :=", strings.Join(moduleNames, " ")) + } + a.writeRequiredModules(w) + fmt.Fprintln(w, "include $(BUILD_PHONY_PACKAGE)") + + } else { + fmt.Fprintln(w, "\ninclude $(CLEAR_VARS)") + fmt.Fprintln(w, "LOCAL_PATH :=", moduleDir) + fmt.Fprintln(w, "LOCAL_MODULE :=", name+a.suffix) + fmt.Fprintln(w, "LOCAL_MODULE_CLASS := ETC") // do we need a new class? + fmt.Fprintln(w, "LOCAL_PREBUILT_MODULE_FILE :=", a.outputFile.String()) + fmt.Fprintln(w, "LOCAL_MODULE_PATH :=", a.installDir.ToMakePath().String()) + fmt.Fprintln(w, "LOCAL_MODULE_STEM :=", name+apexType.suffix()) + fmt.Fprintln(w, "LOCAL_UNINSTALLABLE_MODULE :=", !a.installable()) + fmt.Fprintln(w, "LOCAL_OVERRIDES_MODULES :=", strings.Join(a.overridableProperties.Overrides, " ")) + if len(moduleNames) > 0 { + fmt.Fprintln(w, "LOCAL_REQUIRED_MODULES +=", strings.Join(moduleNames, " ")) + } + if len(a.requiredDeps) > 0 { + fmt.Fprintln(w, "LOCAL_REQUIRED_MODULES +=", strings.Join(a.requiredDeps, " ")) + } + a.writeRequiredModules(w) + var postInstallCommands []string + if a.prebuiltFileToDelete != "" { + postInstallCommands = append(postInstallCommands, "rm -rf "+ + filepath.Join(a.installDir.ToMakePath().String(), a.prebuiltFileToDelete)) + } + // For unflattened apexes, compat symlinks are attached to apex package itself as LOCAL_POST_INSTALL_CMD + postInstallCommands = append(postInstallCommands, a.compatSymlinks...) + if len(postInstallCommands) > 0 { + fmt.Fprintln(w, "LOCAL_POST_INSTALL_CMD :=", strings.Join(postInstallCommands, " && ")) + } + + if a.mergedNotices.Merged.Valid() { + fmt.Fprintln(w, "LOCAL_NOTICE_FILE :=", a.mergedNotices.Merged.Path().String()) + } + + fmt.Fprintln(w, "include $(BUILD_PREBUILT)") + + if apexType == imageApex { + fmt.Fprintln(w, "ALL_MODULES.$(LOCAL_MODULE).BUNDLE :=", a.bundleModuleFile.String()) + } + if len(a.lintReports) > 0 { + fmt.Fprintln(w, "ALL_MODULES.$(my_register_name).LINT_REPORTS :=", + strings.Join(a.lintReports.Strings(), " ")) + } + + if a.installedFilesFile != nil { + goal := "checkbuild" + distFile := name + "-installed-files.txt" + fmt.Fprintln(w, ".PHONY:", goal) + fmt.Fprintf(w, "$(call dist-for-goals,%s,%s:%s)\n", + goal, a.installedFilesFile.String(), distFile) + } + } + }} +} diff --git a/apex/apex.go b/apex/apex.go index 8ee8035b9..7da8e1cf2 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -16,198 +16,1001 @@ package apex import ( "fmt" - "io" + "path" "path/filepath" - "runtime" + "regexp" "sort" - "strconv" "strings" + "sync" + + "github.com/google/blueprint" + "github.com/google/blueprint/bootstrap" + "github.com/google/blueprint/proptools" "android/soong/android" "android/soong/cc" + prebuilt_etc "android/soong/etc" "android/soong/java" "android/soong/python" - - "github.com/google/blueprint" - "github.com/google/blueprint/bootstrap" - "github.com/google/blueprint/proptools" + "android/soong/sh" ) -var ( - pctx = android.NewPackageContext("android/apex") - - // Create a canned fs config file where all files and directories are - // by default set to (uid/gid/mode) = (1000/1000/0644) - // TODO(b/113082813) make this configurable using config.fs syntax - generateFsConfig = pctx.StaticRule("generateFsConfig", blueprint.RuleParams{ - Command: `echo '/ 1000 1000 0755' > ${out} && ` + - `echo '/apex_manifest.json 1000 1000 0644' >> ${out} && ` + - `echo ${ro_paths} | tr ' ' '\n' | awk '{print "/"$$1 " 1000 1000 0644"}' >> ${out} && ` + - `echo ${exec_paths} | tr ' ' '\n' | awk '{print "/"$$1 " 0 2000 0755"}' >> ${out}`, - Description: "fs_config ${out}", - }, "ro_paths", "exec_paths") - - // TODO(b/113233103): make sure that file_contexts is sane, i.e., validate - // against the binary policy using sefcontext_compiler -p <policy>. - - // TODO(b/114327326): automate the generation of file_contexts - apexRule = pctx.StaticRule("apexRule", blueprint.RuleParams{ - Command: `rm -rf ${image_dir} && mkdir -p ${image_dir} && ` + - `(${copy_commands}) && ` + - `APEXER_TOOL_PATH=${tool_path} ` + - `${apexer} --force --manifest ${manifest} ` + - `--file_contexts ${file_contexts} ` + - `--canned_fs_config ${canned_fs_config} ` + - `--payload_type image ` + - `--key ${key} ${opt_flags} ${image_dir} ${out} `, - CommandDeps: []string{"${apexer}", "${avbtool}", "${e2fsdroid}", "${merge_zips}", - "${mke2fs}", "${resize2fs}", "${sefcontext_compile}", - "${soong_zip}", "${zipalign}", "${aapt2}"}, - Description: "APEX ${image_dir} => ${out}", - }, "tool_path", "image_dir", "copy_commands", "manifest", "file_contexts", "canned_fs_config", "key", "opt_flags") - - zipApexRule = pctx.StaticRule("zipApexRule", blueprint.RuleParams{ - Command: `rm -rf ${image_dir} && mkdir -p ${image_dir} && ` + - `(${copy_commands}) && ` + - `APEXER_TOOL_PATH=${tool_path} ` + - `${apexer} --force --manifest ${manifest} ` + - `--payload_type zip ` + - `${image_dir} ${out} `, - CommandDeps: []string{"${apexer}", "${merge_zips}", "${soong_zip}", "${zipalign}", "${aapt2}"}, - Description: "ZipAPEX ${image_dir} => ${out}", - }, "tool_path", "image_dir", "copy_commands", "manifest") - - apexProtoConvertRule = pctx.AndroidStaticRule("apexProtoConvertRule", - blueprint.RuleParams{ - Command: `${aapt2} convert --output-format proto $in -o $out`, - CommandDeps: []string{"${aapt2}"}, - }) +const ( + imageApexSuffix = ".apex" + zipApexSuffix = ".zipapex" + flattenedSuffix = ".flattened" - apexBundleRule = pctx.StaticRule("apexBundleRule", blueprint.RuleParams{ - Command: `${zip2zip} -i $in -o $out ` + - `apex_payload.img:apex/${abi}.img ` + - `apex_manifest.json:root/apex_manifest.json ` + - `AndroidManifest.xml:manifest/AndroidManifest.xml ` + - `assets/NOTICE.html.gz:assets/NOTICE.html.gz`, - CommandDeps: []string{"${zip2zip}"}, - Description: "app bundle", - }, "abi") - - extractMatchingApex = pctx.StaticRule( - "extractMatchingApex", - blueprint.RuleParams{ - Command: `rm -rf "$out" && ` + - `${extract_apks} -o "${out}" -allow-prereleased=${allow-prereleased} ` + - `-sdk-version=${sdk-version} -abis=${abis} -screen-densities=all -extract-single ` + - `${in}`, - CommandDeps: []string{"${extract_apks}"}, - }, - "abis", "allow-prereleased", "sdk-version") + imageApexType = "image" + zipApexType = "zip" + flattenedApexType = "flattened" ) -var imageApexSuffix = ".apex" -var zipApexSuffix = ".zipapex" - -var imageApexType = "image" -var zipApexType = "zip" - type dependencyTag struct { blueprint.BaseDependencyTag name string + + // determines if the dependent will be part of the APEX payload + payload bool } var ( - sharedLibTag = dependencyTag{name: "sharedLib"} - executableTag = dependencyTag{name: "executable"} - javaLibTag = dependencyTag{name: "javaLib"} - prebuiltTag = dependencyTag{name: "prebuilt"} + sharedLibTag = dependencyTag{name: "sharedLib", payload: true} + executableTag = dependencyTag{name: "executable", payload: true} + javaLibTag = dependencyTag{name: "javaLib", payload: true} + prebuiltTag = dependencyTag{name: "prebuilt", payload: true} + testTag = dependencyTag{name: "test", payload: true} keyTag = dependencyTag{name: "key"} certificateTag = dependencyTag{name: "certificate"} + usesTag = dependencyTag{name: "uses"} + androidAppTag = dependencyTag{name: "androidApp", payload: true} + + apexAvailBaseline = makeApexAvailableBaseline() + + inverseApexAvailBaseline = invertApexBaseline(apexAvailBaseline) ) -func init() { - pctx.Import("android/soong/android") - pctx.Import("android/soong/java") - pctx.HostBinToolVariable("apexer", "apexer") - // ART minimal builds (using the master-art manifest) do not have the "frameworks/base" - // projects, and hence cannot built 'aapt2'. Use the SDK prebuilt instead. - hostBinToolVariableWithPrebuilt := func(name, prebuiltDir, tool string) { - pctx.VariableFunc(name, func(ctx android.PackageVarContext) string { - if !ctx.Config().FrameworksBaseDirExists(ctx) { - return filepath.Join(prebuiltDir, runtime.GOOS, "bin", tool) - } else { - return pctx.HostBinToolPath(ctx, tool).String() - } - }) +// Transform the map of apex -> modules to module -> apexes. +func invertApexBaseline(m map[string][]string) map[string][]string { + r := make(map[string][]string) + for apex, modules := range m { + for _, module := range modules { + r[module] = append(r[module], apex) + } + } + return r +} + +// Retrieve the baseline of apexes to which the supplied module belongs. +func BaselineApexAvailable(moduleName string) []string { + return inverseApexAvailBaseline[normalizeModuleName(moduleName)] +} + +// This is a map from apex to modules, which overrides the +// apex_available setting for that particular module to make +// it available for the apex regardless of its setting. +// TODO(b/147364041): remove this +func makeApexAvailableBaseline() map[string][]string { + // The "Module separator"s below are employed to minimize merge conflicts. + m := make(map[string][]string) + // + // Module separator + // + m["com.android.appsearch"] = []string{ + "icing-java-proto-lite", + "libprotobuf-java-lite", + } + // + // Module separator + // + m["com.android.bluetooth.updatable"] = []string{ + "android.hardware.audio.common@5.0", + "android.hardware.bluetooth.a2dp@1.0", + "android.hardware.bluetooth.audio@2.0", + "android.hardware.bluetooth@1.0", + "android.hardware.bluetooth@1.1", + "android.hardware.graphics.bufferqueue@1.0", + "android.hardware.graphics.bufferqueue@2.0", + "android.hardware.graphics.common@1.0", + "android.hardware.graphics.common@1.1", + "android.hardware.graphics.common@1.2", + "android.hardware.media@1.0", + "android.hidl.safe_union@1.0", + "android.hidl.token@1.0", + "android.hidl.token@1.0-utils", + "avrcp-target-service", + "avrcp_headers", + "bluetooth-protos-lite", + "bluetooth.mapsapi", + "com.android.vcard", + "dnsresolver_aidl_interface-V2-java", + "ipmemorystore-aidl-interfaces-V5-java", + "ipmemorystore-aidl-interfaces-java", + "internal_include_headers", + "lib-bt-packets", + "lib-bt-packets-avrcp", + "lib-bt-packets-base", + "libFraunhoferAAC", + "libaudio-a2dp-hw-utils", + "libaudio-hearing-aid-hw-utils", + "libbinder_headers", + "libbluetooth", + "libbluetooth-types", + "libbluetooth-types-header", + "libbluetooth_gd", + "libbluetooth_headers", + "libbluetooth_jni", + "libbt-audio-hal-interface", + "libbt-bta", + "libbt-common", + "libbt-hci", + "libbt-platform-protos-lite", + "libbt-protos-lite", + "libbt-sbc-decoder", + "libbt-sbc-encoder", + "libbt-stack", + "libbt-utils", + "libbtcore", + "libbtdevice", + "libbte", + "libbtif", + "libchrome", + "libevent", + "libfmq", + "libg722codec", + "libgui_headers", + "libmedia_headers", + "libmodpb64", + "libosi", + "libstagefright_foundation_headers", + "libstagefright_headers", + "libstatslog", + "libstatssocket", + "libtinyxml2", + "libudrv-uipc", + "libz", + "media_plugin_headers", + "net-utils-services-common", + "netd_aidl_interface-unstable-java", + "netd_event_listener_interface-java", + "netlink-client", + "networkstack-client", + "sap-api-java-static", + "services.net", + } + // + // Module separator + // + m["com.android.cellbroadcast"] = []string{"CellBroadcastApp", "CellBroadcastServiceModule"} + // + // Module separator + // + m["com.android.conscrypt"] = []string{ + "libnativehelper_header_only", + } + // + // Module separator + // + m["com.android.extservices"] = []string{ + "error_prone_annotations", + "ExtServices-core", + "ExtServices", + "libtextclassifier-java", + "libz_current", + "textclassifier-statsd", + "TextClassifierNotificationLibNoManifest", + "TextClassifierServiceLibNoManifest", + } + // + // Module separator + // + m["com.android.neuralnetworks"] = []string{ + "android.hardware.neuralnetworks@1.0", + "android.hardware.neuralnetworks@1.1", + "android.hardware.neuralnetworks@1.2", + "android.hardware.neuralnetworks@1.3", + "android.hidl.allocator@1.0", + "android.hidl.memory.token@1.0", + "android.hidl.memory@1.0", + "android.hidl.safe_union@1.0", + "libarect", + "libbuildversion", + "libmath", + "libprocpartition", + "libsync", + } + // + // Module separator + // + m["com.android.media"] = []string{ + "android.frameworks.bufferhub@1.0", + "android.hardware.cas.native@1.0", + "android.hardware.cas@1.0", + "android.hardware.configstore-utils", + "android.hardware.configstore@1.0", + "android.hardware.configstore@1.1", + "android.hardware.graphics.allocator@2.0", + "android.hardware.graphics.allocator@3.0", + "android.hardware.graphics.bufferqueue@1.0", + "android.hardware.graphics.bufferqueue@2.0", + "android.hardware.graphics.common@1.0", + "android.hardware.graphics.common@1.1", + "android.hardware.graphics.common@1.2", + "android.hardware.graphics.mapper@2.0", + "android.hardware.graphics.mapper@2.1", + "android.hardware.graphics.mapper@3.0", + "android.hardware.media.omx@1.0", + "android.hardware.media@1.0", + "android.hidl.allocator@1.0", + "android.hidl.memory.token@1.0", + "android.hidl.memory@1.0", + "android.hidl.token@1.0", + "android.hidl.token@1.0-utils", + "bionic_libc_platform_headers", + "exoplayer2-extractor", + "exoplayer2-extractor-annotation-stubs", + "gl_headers", + "jsr305", + "libEGL", + "libEGL_blobCache", + "libEGL_getProcAddress", + "libFLAC", + "libFLAC-config", + "libFLAC-headers", + "libGLESv2", + "libaacextractor", + "libamrextractor", + "libarect", + "libaudio_system_headers", + "libaudioclient", + "libaudioclient_headers", + "libaudiofoundation", + "libaudiofoundation_headers", + "libaudiomanager", + "libaudiopolicy", + "libaudioutils", + "libaudioutils_fixedfft", + "libbinder_headers", + "libbluetooth-types-header", + "libbufferhub", + "libbufferhub_headers", + "libbufferhubqueue", + "libc_malloc_debug_backtrace", + "libcamera_client", + "libcamera_metadata", + "libdexfile_external_headers", + "libdexfile_support", + "libdvr_headers", + "libexpat", + "libfifo", + "libflacextractor", + "libgrallocusage", + "libgraphicsenv", + "libgui", + "libgui_headers", + "libhardware_headers", + "libinput", + "liblzma", + "libmath", + "libmedia", + "libmedia_codeclist", + "libmedia_headers", + "libmedia_helper", + "libmedia_helper_headers", + "libmedia_midiiowrapper", + "libmedia_omx", + "libmediautils", + "libmidiextractor", + "libmkvextractor", + "libmp3extractor", + "libmp4extractor", + "libmpeg2extractor", + "libnativebase_headers", + "libnativebridge-headers", + "libnativebridge_lazy", + "libnativeloader-headers", + "libnativeloader_lazy", + "libnativewindow_headers", + "libnblog", + "liboggextractor", + "libpackagelistparser", + "libpdx", + "libpdx_default_transport", + "libpdx_headers", + "libpdx_uds", + "libprocinfo", + "libspeexresampler", + "libspeexresampler", + "libstagefright_esds", + "libstagefright_flacdec", + "libstagefright_flacdec", + "libstagefright_foundation", + "libstagefright_foundation_headers", + "libstagefright_foundation_without_imemory", + "libstagefright_headers", + "libstagefright_id3", + "libstagefright_metadatautils", + "libstagefright_mpeg2extractor", + "libstagefright_mpeg2support", + "libsync", + "libui", + "libui_headers", + "libunwindstack", + "libvibrator", + "libvorbisidec", + "libwavextractor", + "libwebm", + "media_ndk_headers", + "media_plugin_headers", + "updatable-media", + } + // + // Module separator + // + m["com.android.media.swcodec"] = []string{ + "android.frameworks.bufferhub@1.0", + "android.hardware.common-ndk_platform", + "android.hardware.configstore-utils", + "android.hardware.configstore@1.0", + "android.hardware.configstore@1.1", + "android.hardware.graphics.allocator@2.0", + "android.hardware.graphics.allocator@3.0", + "android.hardware.graphics.allocator@4.0", + "android.hardware.graphics.bufferqueue@1.0", + "android.hardware.graphics.bufferqueue@2.0", + "android.hardware.graphics.common-ndk_platform", + "android.hardware.graphics.common@1.0", + "android.hardware.graphics.common@1.1", + "android.hardware.graphics.common@1.2", + "android.hardware.graphics.mapper@2.0", + "android.hardware.graphics.mapper@2.1", + "android.hardware.graphics.mapper@3.0", + "android.hardware.graphics.mapper@4.0", + "android.hardware.media.bufferpool@2.0", + "android.hardware.media.c2@1.0", + "android.hardware.media.c2@1.1", + "android.hardware.media.omx@1.0", + "android.hardware.media@1.0", + "android.hardware.media@1.0", + "android.hidl.memory.token@1.0", + "android.hidl.memory@1.0", + "android.hidl.safe_union@1.0", + "android.hidl.token@1.0", + "android.hidl.token@1.0-utils", + "libEGL", + "libFLAC", + "libFLAC-config", + "libFLAC-headers", + "libFraunhoferAAC", + "libLibGuiProperties", + "libarect", + "libaudio_system_headers", + "libaudioutils", + "libaudioutils", + "libaudioutils_fixedfft", + "libavcdec", + "libavcenc", + "libavservices_minijail", + "libavservices_minijail", + "libbinder_headers", + "libbinderthreadstateutils", + "libbluetooth-types-header", + "libbufferhub_headers", + "libc_scudo", + "libcodec2", + "libcodec2_headers", + "libcodec2_hidl@1.0", + "libcodec2_hidl@1.1", + "libcodec2_internal", + "libcodec2_soft_aacdec", + "libcodec2_soft_aacenc", + "libcodec2_soft_amrnbdec", + "libcodec2_soft_amrnbenc", + "libcodec2_soft_amrwbdec", + "libcodec2_soft_amrwbenc", + "libcodec2_soft_av1dec_gav1", + "libcodec2_soft_avcdec", + "libcodec2_soft_avcenc", + "libcodec2_soft_common", + "libcodec2_soft_flacdec", + "libcodec2_soft_flacenc", + "libcodec2_soft_g711alawdec", + "libcodec2_soft_g711mlawdec", + "libcodec2_soft_gsmdec", + "libcodec2_soft_h263dec", + "libcodec2_soft_h263enc", + "libcodec2_soft_hevcdec", + "libcodec2_soft_hevcenc", + "libcodec2_soft_mp3dec", + "libcodec2_soft_mpeg2dec", + "libcodec2_soft_mpeg4dec", + "libcodec2_soft_mpeg4enc", + "libcodec2_soft_opusdec", + "libcodec2_soft_opusenc", + "libcodec2_soft_rawdec", + "libcodec2_soft_vorbisdec", + "libcodec2_soft_vp8dec", + "libcodec2_soft_vp8enc", + "libcodec2_soft_vp9dec", + "libcodec2_soft_vp9enc", + "libcodec2_vndk", + "libdexfile_support", + "libdvr_headers", + "libfmq", + "libfmq", + "libgav1", + "libgralloctypes", + "libgrallocusage", + "libgraphicsenv", + "libgsm", + "libgui_bufferqueue_static", + "libgui_headers", + "libhardware", + "libhardware_headers", + "libhevcdec", + "libhevcenc", + "libion", + "libjpeg", + "liblzma", + "libmath", + "libmedia_codecserviceregistrant", + "libmedia_headers", + "libmpeg2dec", + "libnativebase_headers", + "libnativebridge_lazy", + "libnativeloader_lazy", + "libnativewindow_headers", + "libpdx_headers", + "libscudo_wrapper", + "libsfplugin_ccodec_utils", + "libspeexresampler", + "libstagefright_amrnb_common", + "libstagefright_amrnbdec", + "libstagefright_amrnbenc", + "libstagefright_amrwbdec", + "libstagefright_amrwbenc", + "libstagefright_bufferpool@2.0.1", + "libstagefright_bufferqueue_helper", + "libstagefright_enc_common", + "libstagefright_flacdec", + "libstagefright_foundation", + "libstagefright_foundation_headers", + "libstagefright_headers", + "libstagefright_m4vh263dec", + "libstagefright_m4vh263enc", + "libstagefright_mp3dec", + "libsync", + "libui", + "libui_headers", + "libunwindstack", + "libvorbisidec", + "libvpx", + "libyuv", + "libyuv_static", + "media_ndk_headers", + "media_plugin_headers", + "mediaswcodec", + } + // + // Module separator + // + m["com.android.mediaprovider"] = []string{ + "MediaProvider", + "MediaProviderGoogle", + "fmtlib_ndk", + "libbase_ndk", + "libfuse", + "libfuse_jni", + "libnativehelper_header_only", + } + // + // Module separator + // + m["com.android.permission"] = []string{ + "car-ui-lib", + "iconloader", + "kotlin-annotations", + "kotlin-stdlib", + "kotlin-stdlib-jdk7", + "kotlin-stdlib-jdk8", + "kotlinx-coroutines-android", + "kotlinx-coroutines-android-nodeps", + "kotlinx-coroutines-core", + "kotlinx-coroutines-core-nodeps", + "permissioncontroller-statsd", + "GooglePermissionController", + "PermissionController", + "SettingsLibActionBarShadow", + "SettingsLibAppPreference", + "SettingsLibBarChartPreference", + "SettingsLibLayoutPreference", + "SettingsLibProgressBar", + "SettingsLibSearchWidget", + "SettingsLibSettingsTheme", + "SettingsLibRestrictedLockUtils", + "SettingsLibHelpUtils", + } + // + // Module separator + // + m["com.android.runtime"] = []string{ + "bionic_libc_platform_headers", + "libarm-optimized-routines-math", + "libc_aeabi", + "libc_bionic", + "libc_bionic_ndk", + "libc_bootstrap", + "libc_common", + "libc_common_shared", + "libc_common_static", + "libc_dns", + "libc_dynamic_dispatch", + "libc_fortify", + "libc_freebsd", + "libc_freebsd_large_stack", + "libc_gdtoa", + "libc_init_dynamic", + "libc_init_static", + "libc_jemalloc_wrapper", + "libc_netbsd", + "libc_nomalloc", + "libc_nopthread", + "libc_openbsd", + "libc_openbsd_large_stack", + "libc_openbsd_ndk", + "libc_pthread", + "libc_static_dispatch", + "libc_syscalls", + "libc_tzcode", + "libc_unwind_static", + "libdebuggerd", + "libdebuggerd_common_headers", + "libdebuggerd_handler_core", + "libdebuggerd_handler_fallback", + "libdexfile_external_headers", + "libdexfile_support", + "libdexfile_support_static", + "libdl_static", + "libjemalloc5", + "liblinker_main", + "liblinker_malloc", + "liblz4", + "liblzma", + "libprocinfo", + "libpropertyinfoparser", + "libscudo", + "libstdc++", + "libsystemproperties", + "libtombstoned_client_static", + "libunwindstack", + "libz", + "libziparchive", + } + // + // Module separator + // + m["com.android.tethering"] = []string{ + "android.hardware.tetheroffload.config-V1.0-java", + "android.hardware.tetheroffload.control-V1.0-java", + "android.hidl.base-V1.0-java", + "ipmemorystore-aidl-interfaces-java", + "libcgrouprc", + "libcgrouprc_format", + "libnativehelper_compat_libc++", + "libtetherutilsjni", + "libvndksupport", + "net-utils-framework-common", + "netd_aidl_interface-V3-java", + "netlink-client", + "networkstack-aidl-interfaces-java", + "tethering-aidl-interfaces-java", + "TetheringApiCurrentLib", + } + // + // Module separator + // + m["com.android.wifi"] = []string{ + "PlatformProperties", + "android.hardware.wifi-V1.0-java", + "android.hardware.wifi-V1.0-java-constants", + "android.hardware.wifi-V1.1-java", + "android.hardware.wifi-V1.2-java", + "android.hardware.wifi-V1.3-java", + "android.hardware.wifi-V1.4-java", + "android.hardware.wifi.hostapd-V1.0-java", + "android.hardware.wifi.hostapd-V1.1-java", + "android.hardware.wifi.hostapd-V1.2-java", + "android.hardware.wifi.supplicant-V1.0-java", + "android.hardware.wifi.supplicant-V1.1-java", + "android.hardware.wifi.supplicant-V1.2-java", + "android.hardware.wifi.supplicant-V1.3-java", + "android.hidl.base-V1.0-java", + "android.hidl.manager-V1.0-java", + "android.hidl.manager-V1.1-java", + "android.hidl.manager-V1.2-java", + "bouncycastle-unbundled", + "dnsresolver_aidl_interface-V2-java", + "error_prone_annotations", + "framework-wifi-pre-jarjar", + "framework-wifi-util-lib", + "ipmemorystore-aidl-interfaces-V3-java", + "ipmemorystore-aidl-interfaces-java", + "ksoap2", + "libnanohttpd", + "libwifi-jni", + "net-utils-services-common", + "netd_aidl_interface-V2-java", + "netd_aidl_interface-unstable-java", + "netd_event_listener_interface-java", + "netlink-client", + "networkstack-client", + "services.net", + "wifi-lite-protos", + "wifi-nano-protos", + "wifi-service-pre-jarjar", + "wifi-service-resources", + } + // + // Module separator + // + m["com.android.sdkext"] = []string{ + "fmtlib_ndk", + "libbase_ndk", + "libprotobuf-cpp-lite-ndk", + } + // + // Module separator + // + m["com.android.os.statsd"] = []string{ + "libstatssocket", } - hostBinToolVariableWithPrebuilt("aapt2", "prebuilts/sdk/tools", "aapt2") - pctx.HostBinToolVariable("avbtool", "avbtool") - pctx.HostBinToolVariable("e2fsdroid", "e2fsdroid") - pctx.HostBinToolVariable("merge_zips", "merge_zips") - pctx.HostBinToolVariable("mke2fs", "mke2fs") - pctx.HostBinToolVariable("resize2fs", "resize2fs") - pctx.HostBinToolVariable("sefcontext_compile", "sefcontext_compile") - pctx.HostBinToolVariable("soong_zip", "soong_zip") - pctx.HostBinToolVariable("zip2zip", "zip2zip") - pctx.HostBinToolVariable("zipalign", "zipalign") - pctx.HostBinToolVariable("extract_apks", "extract_apks") - - android.RegisterModuleType("apex", apexBundleFactory) + // + // Module separator + // + m[android.AvailableToAnyApex] = []string{ + // TODO(b/156996905) Set apex_available/min_sdk_version for androidx/extras support libraries + "androidx", + "androidx-constraintlayout_constraintlayout", + "androidx-constraintlayout_constraintlayout-nodeps", + "androidx-constraintlayout_constraintlayout-solver", + "androidx-constraintlayout_constraintlayout-solver-nodeps", + "com.google.android.material_material", + "com.google.android.material_material-nodeps", + + "libatomic", + "libclang_rt", + "libgcc_stripped", + "libprofile-clang-extras", + "libprofile-clang-extras_ndk", + "libprofile-extras", + "libprofile-extras_ndk", + "libunwind_llvm", + } + return m +} + +// DO NOT EDIT! These are the package prefixes that are exempted from being AOT'ed by ART. +// Adding code to the bootclasspath in new packages will cause issues on module update. +func qModulesPackages() map[string][]string { + return map[string][]string{ + "com.android.conscrypt": []string{ + "android.net.ssl", + "com.android.org.conscrypt", + }, + "com.android.media": []string{ + "android.media", + }, + } +} + +// DO NOT EDIT! These are the package prefixes that are exempted from being AOT'ed by ART. +// Adding code to the bootclasspath in new packages will cause issues on module update. +func rModulesPackages() map[string][]string { + return map[string][]string{ + "com.android.mediaprovider": []string{ + "android.provider", + }, + "com.android.permission": []string{ + "android.permission", + "android.app.role", + "com.android.permission", + "com.android.role", + }, + "com.android.sdkext": []string{ + "android.os.ext", + }, + "com.android.os.statsd": []string{ + "android.app", + "android.os", + "android.util", + "com.android.internal.statsd", + "com.android.server.stats", + }, + "com.android.wifi": []string{ + "com.android.server.wifi", + "com.android.wifi.x", + "android.hardware.wifi", + "android.net.wifi", + }, + "com.android.tethering": []string{ + "android.net", + }, + } +} + +func init() { + android.RegisterModuleType("apex", BundleFactory) android.RegisterModuleType("apex_test", testApexBundleFactory) + android.RegisterModuleType("apex_vndk", vndkApexBundleFactory) android.RegisterModuleType("apex_defaults", defaultsFactory) android.RegisterModuleType("prebuilt_apex", PrebuiltFactory) + android.RegisterModuleType("override_apex", overrideApexFactory) android.RegisterModuleType("apex_set", apexSetFactory) - android.PostDepsMutators(func(ctx android.RegisterMutatorsContext) { - ctx.TopDown("apex_deps", apexDepsMutator) - ctx.BottomUp("apex", apexMutator) + android.PreDepsMutators(RegisterPreDepsMutators) + android.PostDepsMutators(RegisterPostDepsMutators) + + android.RegisterMakeVarsProvider(pctx, func(ctx android.MakeVarsContext) { + apexFileContextsInfos := apexFileContextsInfos(ctx.Config()) + sort.Strings(*apexFileContextsInfos) + ctx.Strict("APEX_FILE_CONTEXTS_INFOS", strings.Join(*apexFileContextsInfos, " ")) }) + + android.AddNeverAllowRules(createApexPermittedPackagesRules(qModulesPackages())...) + android.AddNeverAllowRules(createApexPermittedPackagesRules(rModulesPackages())...) +} + +func createApexPermittedPackagesRules(modules_packages map[string][]string) []android.Rule { + rules := make([]android.Rule, 0, len(modules_packages)) + for module_name, module_packages := range modules_packages { + permitted_packages_rule := android.NeverAllow(). + BootclasspathJar(). + With("apex_available", module_name). + WithMatcher("permitted_packages", android.NotInList(module_packages)). + Because("jars that are part of the " + module_name + + " module may only allow these packages: " + strings.Join(module_packages, ",") + + ". Please jarjar or move code around.") + rules = append(rules, permitted_packages_rule) + } + return rules +} + +func RegisterPreDepsMutators(ctx android.RegisterMutatorsContext) { + ctx.TopDown("apex_vndk", apexVndkMutator).Parallel() + ctx.BottomUp("apex_vndk_deps", apexVndkDepsMutator).Parallel() +} + +func RegisterPostDepsMutators(ctx android.RegisterMutatorsContext) { + ctx.TopDown("apex_deps", apexDepsMutator) + ctx.BottomUp("apex", apexMutator).Parallel() + ctx.BottomUp("apex_flattened", apexFlattenedMutator).Parallel() + ctx.BottomUp("apex_uses", apexUsesMutator).Parallel() + ctx.BottomUp("mark_platform_availability", markPlatformAvailability).Parallel() } // Mark the direct and transitive dependencies of apex bundles so that they // can be built for the apex bundles. func apexDepsMutator(mctx android.TopDownMutatorContext) { - if a, ok := mctx.Module().(*apexBundle); ok { - apexBundleName := mctx.ModuleName() - mctx.WalkDeps(func(child, parent android.Module) bool { - depName := mctx.OtherModuleName(child) - // If the parent is apexBundle, this child is directly depended. - _, directDep := parent.(*apexBundle) - if a.installable() && !a.testApex { - // TODO(b/123892969): Workaround for not having any way to annotate test-apexs - // non-installable apex's cannot be installed and so should not prevent libraries from being - // installed to the system. - android.UpdateApexDependency(apexBundleName, depName, directDep) - } + if !mctx.Module().Enabled() { + return + } + var apexBundles []android.ApexInfo + var directDep bool + if a, ok := mctx.Module().(*apexBundle); ok && !a.vndkApex { + apexBundles = []android.ApexInfo{{ + ApexName: mctx.ModuleName(), + MinSdkVersion: a.minSdkVersion(mctx), + Updatable: a.Updatable(), + }} + directDep = true + } else if am, ok := mctx.Module().(android.ApexModule); ok { + apexBundles = am.ApexVariations() + directDep = false + } - if am, ok := child.(android.ApexModule); ok && am.CanHaveApexVariants() { - am.BuildForApex(apexBundleName) - return true - } else { - return false + if len(apexBundles) == 0 { + return + } + + cur := mctx.Module().(android.DepIsInSameApex) + + mctx.VisitDirectDeps(func(child android.Module) { + depName := mctx.OtherModuleName(child) + if am, ok := child.(android.ApexModule); ok && am.CanHaveApexVariants() && + (cur.DepIsInSameApex(mctx, child) || inAnySdk(child)) { + android.UpdateApexDependency(apexBundles, depName, directDep) + am.BuildForApexes(apexBundles) + } + }) +} + +// mark if a module cannot be available to platform. A module cannot be available +// to platform if 1) it is explicitly marked as not available (i.e. "//apex_available:platform" +// is absent) or 2) it depends on another module that isn't (or can't be) available to platform +func markPlatformAvailability(mctx android.BottomUpMutatorContext) { + // Host and recovery are not considered as platform + if mctx.Host() || mctx.Module().InstallInRecovery() { + return + } + + if am, ok := mctx.Module().(android.ApexModule); ok { + availableToPlatform := am.AvailableFor(android.AvailableToPlatform) + + // In a rare case when a lib is marked as available only to an apex + // but the apex doesn't exist. This can happen in a partial manifest branch + // like master-art. Currently, libstatssocket in the stats APEX is causing + // this problem. + // Include the lib in platform because the module SDK that ought to provide + // it doesn't exist, so it would otherwise be left out completely. + // TODO(b/154888298) remove this by adding those libraries in module SDKS and skipping + // this check for libraries provided by SDKs. + if !availableToPlatform && !android.InAnyApex(am.Name()) { + availableToPlatform = true + } + + // If any of the dep is not available to platform, this module is also considered + // as being not available to platform even if it has "//apex_available:platform" + mctx.VisitDirectDeps(func(child android.Module) { + if !am.DepIsInSameApex(mctx, child) { + // if the dependency crosses apex boundary, don't consider it + return + } + if dep, ok := child.(android.ApexModule); ok && dep.NotAvailableForPlatform() { + availableToPlatform = false + // TODO(b/154889534) trigger an error when 'am' has "//apex_available:platform" } }) + + // Exception 1: stub libraries and native bridge libraries are always available to platform + if cc, ok := mctx.Module().(*cc.Module); ok && + (cc.IsStubs() || cc.Target().NativeBridge == android.NativeBridgeEnabled) { + availableToPlatform = true + } + + // Exception 2: bootstrap bionic libraries are also always available to platform + if cc.InstallToBootstrap(mctx.ModuleName(), mctx.Config()) { + availableToPlatform = true + } + + if !availableToPlatform { + am.SetNotAvailableForPlatform() + } } } +// If a module in an APEX depends on a module from an SDK then it needs an APEX +// specific variant created for it. Refer to sdk.sdkDepsReplaceMutator. +func inAnySdk(module android.Module) bool { + if sa, ok := module.(android.SdkAware); ok { + return sa.IsInAnySdk() + } + + return false +} + // Create apex variations if a module is included in APEX(s). func apexMutator(mctx android.BottomUpMutatorContext) { + if !mctx.Module().Enabled() { + return + } if am, ok := mctx.Module().(android.ApexModule); ok && am.CanHaveApexVariants() { am.CreateApexVariations(mctx) - } else if _, ok := mctx.Module().(*apexBundle); ok { + } else if a, ok := mctx.Module().(*apexBundle); ok && !a.vndkApex { // apex bundle itself is mutated so that it and its modules have same // apex variant. apexBundleName := mctx.ModuleName() mctx.CreateVariations(apexBundleName) + } else if o, ok := mctx.Module().(*OverrideApex); ok { + apexBundleName := o.GetOverriddenModuleName() + if apexBundleName == "" { + mctx.ModuleErrorf("base property is not set") + return + } + mctx.CreateVariations(apexBundleName) + } + +} + +var ( + apexFileContextsInfosKey = android.NewOnceKey("apexFileContextsInfosKey") + apexFileContextsInfosMutex sync.Mutex +) + +func apexFileContextsInfos(config android.Config) *[]string { + return config.Once(apexFileContextsInfosKey, func() interface{} { + return &[]string{} + }).(*[]string) +} + +func addFlattenedFileContextsInfos(ctx android.BaseModuleContext, fileContextsInfo string) { + apexFileContextsInfosMutex.Lock() + defer apexFileContextsInfosMutex.Unlock() + apexFileContextsInfos := apexFileContextsInfos(ctx.Config()) + *apexFileContextsInfos = append(*apexFileContextsInfos, fileContextsInfo) +} + +func apexFlattenedMutator(mctx android.BottomUpMutatorContext) { + if !mctx.Module().Enabled() { + return + } + if ab, ok := mctx.Module().(*apexBundle); ok { + var variants []string + switch proptools.StringDefault(ab.properties.Payload_type, "image") { + case "image": + variants = append(variants, imageApexType, flattenedApexType) + case "zip": + variants = append(variants, zipApexType) + case "both": + variants = append(variants, imageApexType, zipApexType, flattenedApexType) + default: + mctx.PropertyErrorf("type", "%q is not one of \"image\", \"zip\", or \"both\".", *ab.properties.Payload_type) + return + } + + modules := mctx.CreateLocalVariations(variants...) + + for i, v := range variants { + switch v { + case imageApexType: + modules[i].(*apexBundle).properties.ApexType = imageApex + case zipApexType: + modules[i].(*apexBundle).properties.ApexType = zipApex + case flattenedApexType: + modules[i].(*apexBundle).properties.ApexType = flattenedApex + if !mctx.Config().FlattenApex() && ab.Platform() { + modules[i].(*apexBundle).MakeAsSystemExt() + } + } + } + } else if _, ok := mctx.Module().(*OverrideApex); ok { + mctx.CreateVariations(imageApexType, flattenedApexType) } } +func apexUsesMutator(mctx android.BottomUpMutatorContext) { + if ab, ok := mctx.Module().(*apexBundle); ok { + mctx.AddFarVariationDependencies(nil, usesTag, ab.properties.Uses...) + } +} + +var ( + useVendorAllowListKey = android.NewOnceKey("useVendorAllowList") +) + +// useVendorAllowList returns the list of APEXes which are allowed to use_vendor. +// When use_vendor is used, native modules are built with __ANDROID_VNDK__ and __ANDROID_APEX__, +// which may cause compatibility issues. (e.g. libbinder) +// Even though libbinder restricts its availability via 'apex_available' property and relies on +// yet another macro __ANDROID_APEX_<NAME>__, we restrict usage of "use_vendor:" from other APEX modules +// to avoid similar problems. +func useVendorAllowList(config android.Config) []string { + return config.Once(useVendorAllowListKey, func() interface{} { + return []string{ + // swcodec uses "vendor" variants for smaller size + "com.android.media.swcodec", + "test_com.android.media.swcodec", + } + }).([]string) +} + +// setUseVendorAllowListForTest overrides useVendorAllowList and must be +// called before the first call to useVendorAllowList() +func setUseVendorAllowListForTest(config android.Config, allowList []string) { + config.Once(useVendorAllowListKey, func() interface{} { + return allowList + }) +} + type apexNativeDependencies struct { // List of native libraries Native_shared_libs []string + // List of native executables Binaries []string + + // List of native tests + Tests []string } + type apexMultilibProperties struct { // Native dependencies whose compile_multilib is "first" First apexNativeDependencies @@ -234,20 +1037,20 @@ type apexBundleProperties struct { // If unspecified, a default one is automatically generated. AndroidManifest *string `android:"path"` - // Canonical name of the APEX bundle in the manifest file. - // If unspecified, defaults to the value of name + // Canonical name of the APEX bundle. Used to determine the path to the activated APEX on + // device (/apex/<apex_name>). + // If unspecified, defaults to the value of name. Apex_name *string // Determines the file contexts file for setting security context to each file in this APEX bundle. - // Specifically, when this is set to <value>, /system/sepolicy/apex/<value>_file_contexts file is - // used. - // Default: <name_of_this_module> - File_contexts *string + // For platform APEXes, this should points to a file under /system/sepolicy + // Default: /system/sepolicy/apex/<module_name>_file_contexts. + File_contexts *string `android:"path"` // List of native shared libs that are embedded inside this APEX bundle Native_shared_libs []string - // List of native executables that are embedded inside this APEX bundle + // List of executables that are embedded inside this APEX bundle Binaries []string // List of java libraries that are embedded inside this APEX bundle @@ -256,6 +1059,9 @@ type apexBundleProperties struct { // List of prebuilt files that are embedded inside this APEX bundle Prebuilts []string + // List of tests that are embedded inside this APEX bundle + Tests []string + // Name of the apex_key module that provides the private key to sign APEX Key *string @@ -285,6 +1091,42 @@ type apexBundleProperties struct { PreventInstall bool `blueprint:"mutated"` HideFromMake bool `blueprint:"mutated"` + + // Indicates this APEX provides C++ shared libaries to other APEXes. Default: false. + Provide_cpp_shared_libs *bool + + // List of providing APEXes' names so that this APEX can depend on provided shared libraries. + Uses []string + + // package format of this apex variant; could be non-flattened, flattened, or zip. + // imageApex, zipApex or flattened + ApexType apexPackaging `blueprint:"mutated"` + + // List of SDKs that are used to build this APEX. A reference to an SDK should be either + // `name#version` or `name` which is an alias for `name#current`. If left empty, `platform#current` + // is implied. This value affects all modules included in this APEX. In other words, they are + // also built with the SDKs specified here. + Uses_sdks []string + + // Whenever apex_payload.img of the APEX should include dm-verity hashtree. + // Should be only used in tests#. + Test_only_no_hashtree *bool + + // Whenever apex_payload.img of the APEX should not be dm-verity signed. + // Should be only used in tests#. + Test_only_unsigned_payload *bool + + IsCoverageVariant bool `blueprint:"mutated"` + + // Whether this APEX is considered updatable or not. When set to true, this will enforce additional + // rules for making sure that the APEX is truly updatable. + // - To be updatable, min_sdk_version should be set as well + // This will also disable the size optimizations like symlinking to the system libs. + // Default is false. + Updatable *bool + + // The minimum SDK version that this apex must be compatible with. + Min_sdk_version *string } type apexTargetBundleProperties struct { @@ -293,14 +1135,17 @@ type apexTargetBundleProperties struct { Android struct { Multilib apexMultilibProperties } + // Multilib properties only for host. Host struct { Multilib apexMultilibProperties } + // Multilib properties only for host linux_bionic. Linux_bionic struct { Multilib apexMultilibProperties } + // Multilib properties only for host linux_glibc. Linux_glibc struct { Multilib apexMultilibProperties @@ -308,52 +1153,45 @@ type apexTargetBundleProperties struct { } } -type apexFileClass int +type overridableProperties struct { + // List of APKs to package inside APEX + Apps []string -const ( - etc apexFileClass = iota - nativeSharedLib - nativeExecutable - shBinary - pyBinary - goBinary - javaSharedLib -) + // Names of modules to be overridden. Listed modules can only be other binaries + // (in Make or Soong). + // This does not completely prevent installation of the overridden binaries, but if both + // binaries would be installed by default (in PRODUCT_PACKAGES) the other binary will be removed + // from PRODUCT_PACKAGES. + Overrides []string + + // Logging Parent value + Logging_parent string + + // Apex Container Package Name. + // Override value for attribute package:name in AndroidManifest.xml + Package_name string + + // A txt file containing list of files that are allowed to be included in this APEX. + Allowed_files *string `android:"path"` +} type apexPackaging int const ( imageApex apexPackaging = iota zipApex - both + flattenedApex ) -func (a apexPackaging) image() bool { - switch a { - case imageApex, both: - return true - } - return false -} - -func (a apexPackaging) zip() bool { - switch a { - case zipApex, both: - return true - } - return false -} - +// The suffix for the output "file", not the module func (a apexPackaging) suffix() string { switch a { case imageApex: return imageApexSuffix case zipApex: return zipApexSuffix - case both: - panic(fmt.Errorf("must be either zip or image")) default: - panic(fmt.Errorf("unkonwn APEX type %d", a)) + panic(fmt.Errorf("unknown APEX type %d", a)) } } @@ -363,13 +1201,26 @@ func (a apexPackaging) name() string { return imageApexType case zipApex: return zipApexType - case both: - panic(fmt.Errorf("must be either zip or image")) default: - panic(fmt.Errorf("unkonwn APEX type %d", a)) + panic(fmt.Errorf("unknown APEX type %d", a)) } } +type apexFileClass int + +const ( + etc apexFileClass = iota + nativeSharedLib + nativeExecutable + shBinary + pyBinary + goBinary + javaSharedLib + nativeTest + app + appSet +) + func (class apexFileClass) NameInMake() string { switch class { case etc: @@ -380,32 +1231,119 @@ func (class apexFileClass) NameInMake() string { return "EXECUTABLES" case javaSharedLib: return "JAVA_LIBRARIES" + case nativeTest: + return "NATIVE_TESTS" + case app, appSet: + // b/142537672 Why isn't this APP? We want to have full control over + // the paths and file names of the apk file under the flattend APEX. + // If this is set to APP, then the paths and file names are modified + // by the Make build system. For example, it is installed to + // /system/apex/<apexname>/app/<Appname>/<apexname>.<Appname>/ instead of + // /system/apex/<apexname>/app/<Appname> because the build system automatically + // appends module name (which is <apexname>.<Appname> to the path. + return "ETC" default: - panic(fmt.Errorf("unkonwn class %d", class)) + panic(fmt.Errorf("unknown class %d", class)) } } +// apexFile represents a file in an APEX bundle type apexFile struct { builtFile android.Path + stem string moduleName string installDir string class apexFileClass module android.Module - symlinks []string + // list of symlinks that will be created in installDir that point to this apexFile + symlinks []string + transitiveDep bool + moduleDir string + + requiredModuleNames []string + targetRequiredModuleNames []string + hostRequiredModuleNames []string + + jacocoReportClassesFile android.Path // only for javalibs and apps + lintDepSets java.LintDepSets // only for javalibs and apps + certificate java.Certificate // only for apps + overriddenPackageName string // only for apps +} + +func newApexFile(ctx android.BaseModuleContext, builtFile android.Path, moduleName string, installDir string, class apexFileClass, module android.Module) apexFile { + ret := apexFile{ + builtFile: builtFile, + moduleName: moduleName, + installDir: installDir, + class: class, + module: module, + } + if module != nil { + ret.moduleDir = ctx.OtherModuleDir(module) + ret.requiredModuleNames = module.RequiredModuleNames() + ret.targetRequiredModuleNames = module.TargetRequiredModuleNames() + ret.hostRequiredModuleNames = module.HostRequiredModuleNames() + } + return ret +} + +func (af *apexFile) Ok() bool { + return af.builtFile != nil && af.builtFile.String() != "" +} + +func (af *apexFile) apexRelativePath(path string) string { + return filepath.Join(af.installDir, path) +} + +// Path() returns path of this apex file relative to the APEX root +func (af *apexFile) Path() string { + return af.apexRelativePath(af.Stem()) +} + +func (af *apexFile) Stem() string { + if af.stem != "" { + return af.stem + } + return af.builtFile.Base() +} + +// SymlinkPaths() returns paths of the symlinks (if any) relative to the APEX root +func (af *apexFile) SymlinkPaths() []string { + var ret []string + for _, symlink := range af.symlinks { + ret = append(ret, filepath.Join(af.installDir, symlink)) + } + return ret +} + +func (af *apexFile) AvailableToPlatform() bool { + if af.module == nil { + return false + } + if am, ok := af.module.(android.ApexModule); ok { + return am.AvailableFor(android.AvailableToPlatform) + } + return false } type apexBundle struct { android.ModuleBase android.DefaultableModuleBase + android.OverridableModuleBase + android.SdkBase - properties apexBundleProperties - targetProperties apexTargetBundleProperties + properties apexBundleProperties + targetProperties apexTargetBundleProperties + overridableProperties overridableProperties - apexTypes apexPackaging + // specific to apex_vndk modules + vndkProperties apexVndkProperties bundleModuleFile android.WritablePath - outputFiles map[apexPackaging]android.WritablePath - installDir android.OutputPath + outputFile android.WritablePath + installDir android.InstallPath + + prebuiltFileToDelete string public_key_file android.Path private_key_file android.Path @@ -413,34 +1351,69 @@ type apexBundle struct { container_certificate_file android.Path container_private_key_file android.Path + fileContexts android.Path + // list of files to be included in this apex filesInfo []apexFile - // list of module names that this APEX is depending on - externalDeps []string + // list of module names that should be installed along with this APEX + requiredDeps []string + + // list of module names that this APEX is including (to be shown via *-deps-info target) + android.ApexBundleDepsInfo + + testApex bool + vndkApex bool + artApex bool + primaryApexType bool + + manifestJsonOut android.WritablePath + manifestPbOut android.WritablePath + + // list of commands to create symlinks for backward compatibility. + // these commands will be attached as LOCAL_POST_INSTALL_CMD to + // apex package itself(for unflattened build) or apex_manifest(for flattened build) + // so that compat symlinks are always installed regardless of TARGET_FLATTEN_APEX setting. + compatSymlinks []string + + // Suffix of module name in Android.mk + // ".flattened", ".apex", ".zipapex", or "" + suffix string + + installedFilesFile android.WritablePath + + // Whether to create symlink to the system file instead of having a file + // inside the apex or not + linkToSystemLib bool - flattened bool + // Struct holding the merged notice file paths in different formats + mergedNotices android.NoticeOutputs - testApex bool + // Optional list of lint report zip files for apexes that contain java or app modules + lintReports android.Paths } func addDependenciesForNativeModules(ctx android.BottomUpMutatorContext, - native_shared_libs []string, binaries []string, arch string, imageVariation string) { + native_shared_libs []string, binaries []string, tests []string, + target android.Target, imageVariation string) { // Use *FarVariation* to be able to depend on modules having // conflicting variations with this module. This is required since // arch variant of an APEX bundle is 'common' but it is 'arm' or 'arm64' // for native shared libs. - ctx.AddFarVariationDependencies([]blueprint.Variation{ - {Mutator: "arch", Variation: arch}, + ctx.AddFarVariationDependencies(append(target.Variations(), []blueprint.Variation{ {Mutator: "image", Variation: imageVariation}, {Mutator: "link", Variation: "shared"}, {Mutator: "version", Variation: ""}, // "" is the non-stub variant - }, sharedLibTag, native_shared_libs...) + }...), sharedLibTag, native_shared_libs...) - ctx.AddFarVariationDependencies([]blueprint.Variation{ - {Mutator: "arch", Variation: arch}, + ctx.AddFarVariationDependencies(append(target.Variations(), + blueprint.Variation{Mutator: "image", Variation: imageVariation}), + executableTag, binaries...) + + ctx.AddFarVariationDependencies(append(target.Variations(), []blueprint.Variation{ {Mutator: "image", Variation: imageVariation}, - }, executableTag, binaries...) + {Mutator: "test_per_src", Variation: ""}, // "" is the all-tests variant + }...), testTag, tests...) } func (a *apexBundle) combineProperties(ctx android.BottomUpMutatorContext) { @@ -457,6 +1430,9 @@ func (a *apexBundle) combineProperties(ctx android.BottomUpMutatorContext) { } func (a *apexBundle) DepsMutator(ctx android.BottomUpMutatorContext) { + if proptools.Bool(a.properties.Use_vendor) && !android.InList(a.Name(), useVendorAllowList(ctx.Config())) { + ctx.PropertyErrorf("use_vendor", "not allowed to set use_vendor: true") + } targets := ctx.MultiTargets() config := ctx.DeviceConfig() @@ -472,37 +1448,41 @@ func (a *apexBundle) DepsMutator(ctx android.BottomUpMutatorContext) { for i, target := range targets { // When multilib.* is omitted for native_shared_libs, it implies // multilib.both. - ctx.AddFarVariationDependencies([]blueprint.Variation{ - {Mutator: "arch", Variation: target.String()}, + ctx.AddFarVariationDependencies(append(target.Variations(), []blueprint.Variation{ {Mutator: "image", Variation: a.getImageVariation(config)}, {Mutator: "link", Variation: "shared"}, - }, sharedLibTag, a.properties.Native_shared_libs...) + }...), sharedLibTag, a.properties.Native_shared_libs...) + + // When multilib.* is omitted for tests, it implies + // multilib.both. + ctx.AddFarVariationDependencies(append(target.Variations(), []blueprint.Variation{ + {Mutator: "image", Variation: a.getImageVariation(config)}, + {Mutator: "test_per_src", Variation: ""}, // "" is the all-tests variant + }...), testTag, a.properties.Tests...) // Add native modules targetting both ABIs addDependenciesForNativeModules(ctx, a.properties.Multilib.Both.Native_shared_libs, - a.properties.Multilib.Both.Binaries, target.String(), + a.properties.Multilib.Both.Binaries, + a.properties.Multilib.Both.Tests, + target, a.getImageVariation(config)) isPrimaryAbi := i == 0 if isPrimaryAbi { // When multilib.* is omitted for binaries, it implies // multilib.first. - ctx.AddFarVariationDependencies([]blueprint.Variation{ - {Mutator: "arch", Variation: target.String()}, - {Mutator: "image", Variation: a.getImageVariation(config)}, - }, executableTag, a.properties.Binaries...) + ctx.AddFarVariationDependencies(append(target.Variations(), + blueprint.Variation{Mutator: "image", Variation: a.getImageVariation(config)}), + executableTag, a.properties.Binaries...) // Add native modules targetting the first ABI addDependenciesForNativeModules(ctx, a.properties.Multilib.First.Native_shared_libs, - a.properties.Multilib.First.Binaries, target.String(), + a.properties.Multilib.First.Binaries, + a.properties.Multilib.First.Tests, + target, a.getImageVariation(config)) - - // When multilib.* is omitted for prebuilts, it implies multilib.first. - ctx.AddFarVariationDependencies([]blueprint.Variation{ - {Mutator: "arch", Variation: target.String()}, - }, prebuiltTag, a.properties.Prebuilts...) } switch target.Arch.ArchType.Multilib { @@ -510,24 +1490,32 @@ func (a *apexBundle) DepsMutator(ctx android.BottomUpMutatorContext) { // Add native modules targetting 32-bit ABI addDependenciesForNativeModules(ctx, a.properties.Multilib.Lib32.Native_shared_libs, - a.properties.Multilib.Lib32.Binaries, target.String(), + a.properties.Multilib.Lib32.Binaries, + a.properties.Multilib.Lib32.Tests, + target, a.getImageVariation(config)) addDependenciesForNativeModules(ctx, a.properties.Multilib.Prefer32.Native_shared_libs, - a.properties.Multilib.Prefer32.Binaries, target.String(), + a.properties.Multilib.Prefer32.Binaries, + a.properties.Multilib.Prefer32.Tests, + target, a.getImageVariation(config)) case "lib64": // Add native modules targetting 64-bit ABI addDependenciesForNativeModules(ctx, a.properties.Multilib.Lib64.Native_shared_libs, - a.properties.Multilib.Lib64.Binaries, target.String(), + a.properties.Multilib.Lib64.Binaries, + a.properties.Multilib.Lib64.Tests, + target, a.getImageVariation(config)) if !has32BitTarget { addDependenciesForNativeModules(ctx, a.properties.Multilib.Prefer32.Native_shared_libs, - a.properties.Multilib.Prefer32.Binaries, target.String(), + a.properties.Multilib.Prefer32.Binaries, + a.properties.Multilib.Prefer32.Tests, + target, a.getImageVariation(config)) } @@ -536,7 +1524,7 @@ func (a *apexBundle) DepsMutator(ctx android.BottomUpMutatorContext) { if sanitizer == "hwaddress" { addDependenciesForNativeModules(ctx, []string{"libclang_rt.hwasan-aarch64-android"}, - nil, target.String(), a.getImageVariation(config)) + nil, nil, target, a.getImageVariation(config)) break } } @@ -545,9 +1533,30 @@ func (a *apexBundle) DepsMutator(ctx android.BottomUpMutatorContext) { } + // For prebuilt_etc, use the first variant (64 on 64/32bit device, + // 32 on 32bit device) regardless of the TARGET_PREFER_* setting. + // b/144532908 + archForPrebuiltEtc := config.Arches()[0] + for _, arch := range config.Arches() { + // Prefer 64-bit arch if there is any + if arch.ArchType.Multilib == "lib64" { + archForPrebuiltEtc = arch + break + } + } ctx.AddFarVariationDependencies([]blueprint.Variation{ - {Mutator: "arch", Variation: "android_common"}, - }, javaLibTag, a.properties.Java_libs...) + {Mutator: "os", Variation: ctx.Os().String()}, + {Mutator: "arch", Variation: archForPrebuiltEtc.String()}, + }, prebuiltTag, a.properties.Prebuilts...) + + ctx.AddFarVariationDependencies(ctx.Config().AndroidCommonTarget.Variations(), + javaLibTag, a.properties.Java_libs...) + + // With EMMA_INSTRUMENT_FRAMEWORK=true the ART boot image includes jacoco library. + if a.artApex && ctx.Config().IsEnvTrue("EMMA_INSTRUMENT_FRAMEWORK") { + ctx.AddFarVariationDependencies(ctx.Config().AndroidCommonTarget.Variations(), + javaLibTag, "jacocoagent") + } if String(a.properties.Key) == "" { ctx.ModuleErrorf("key is missing") @@ -559,21 +1568,51 @@ func (a *apexBundle) DepsMutator(ctx android.BottomUpMutatorContext) { if cert != "" { ctx.AddDependency(ctx.Module(), certificateTag, cert) } + + // TODO(jiyong): ensure that all apexes are with non-empty uses_sdks + if len(a.properties.Uses_sdks) > 0 { + sdkRefs := []android.SdkRef{} + for _, str := range a.properties.Uses_sdks { + parsed := android.ParseSdkRef(ctx, str, "uses_sdks") + sdkRefs = append(sdkRefs, parsed) + } + a.BuildWithSdks(sdkRefs) + } } -func (a *apexBundle) getCertString(ctx android.BaseContext) string { - certificate, overridden := ctx.DeviceConfig().OverrideCertificateFor(ctx.ModuleName()) +func (a *apexBundle) OverridablePropertiesDepsMutator(ctx android.BottomUpMutatorContext) { + if a.overridableProperties.Allowed_files != nil { + android.ExtractSourceDeps(ctx, a.overridableProperties.Allowed_files) + } + ctx.AddFarVariationDependencies(ctx.Config().AndroidCommonTarget.Variations(), + androidAppTag, a.overridableProperties.Apps...) +} + +func (a *apexBundle) DepIsInSameApex(ctx android.BaseModuleContext, dep android.Module) bool { + // direct deps of an APEX bundle are all part of the APEX bundle + return true +} + +func (a *apexBundle) getCertString(ctx android.BaseModuleContext) string { + moduleName := ctx.ModuleName() + // VNDK APEXes share the same certificate. To avoid adding a new VNDK version to the OVERRIDE_* list, + // we check with the pseudo module name to see if its certificate is overridden. + if a.vndkApex { + moduleName = vndkApexName + } + certificate, overridden := ctx.DeviceConfig().OverrideCertificateFor(moduleName) if overridden { return ":" + certificate } return String(a.properties.Certificate) } -func (a *apexBundle) Srcs() android.Paths { - if file, ok := a.outputFiles[imageApex]; ok { - return android.Paths{file} - } else { - return nil +func (a *apexBundle) OutputFiles(tag string) (android.Paths, error) { + switch tag { + case "": + return android.Paths{a.outputFile}, nil + default: + return nil, fmt.Errorf("unsupported module reference tag %q", tag) } } @@ -581,11 +1620,22 @@ func (a *apexBundle) installable() bool { return !a.properties.PreventInstall && (a.properties.Installable == nil || proptools.Bool(a.properties.Installable)) } +func (a *apexBundle) testOnlyShouldSkipHashtreeGeneration() bool { + return proptools.Bool(a.properties.Test_only_no_hashtree) +} + +func (a *apexBundle) testOnlyShouldSkipPayloadSign() bool { + return proptools.Bool(a.properties.Test_only_unsigned_payload) +} + func (a *apexBundle) getImageVariation(config android.DeviceConfig) string { + if a.vndkApex { + return cc.VendorVariationPrefix + a.vndkVersion(config) + } if config.VndkVersion() != "" && proptools.Bool(a.properties.Use_vendor) { - return "vendor" + return cc.VendorVariationPrefix + config.PlatformVndkVersion() } else { - return "core" + return android.CoreVariation } } @@ -613,7 +1663,7 @@ func (a *apexBundle) IsSanitizerEnabled(ctx android.BaseModuleContext, sanitizer return android.InList(sanitizerName, globalSanitizerNames) } -func (a *apexBundle) IsNativeCoverageNeeded(ctx android.BaseContext) bool { +func (a *apexBundle) IsNativeCoverageNeeded(ctx android.BaseModuleContext) bool { return ctx.Device() && ctx.DeviceConfig().NativeCoverageEnabled() } @@ -625,200 +1675,533 @@ func (a *apexBundle) HideFromMake() { a.properties.HideFromMake = true } -func getCopyManifestForNativeLibrary(cc *cc.Module, handleSpecialLibs bool) (fileToCopy android.Path, dirInApex string) { +func (a *apexBundle) MarkAsCoverageVariant(coverage bool) { + a.properties.IsCoverageVariant = coverage +} + +// TODO(jiyong) move apexFileFor* close to the apexFile type definition +func apexFileForNativeLibrary(ctx android.BaseModuleContext, ccMod *cc.Module, handleSpecialLibs bool) apexFile { // Decide the APEX-local directory by the multilib of the library // In the future, we may query this to the module. - switch cc.Arch().ArchType.Multilib { + var dirInApex string + switch ccMod.Arch().ArchType.Multilib { case "lib32": dirInApex = "lib" case "lib64": dirInApex = "lib64" } - dirInApex = filepath.Join(dirInApex, cc.RelativeInstallPath()) - if !cc.Arch().Native { - dirInApex = filepath.Join(dirInApex, cc.Arch().ArchType.String()) - } - if handleSpecialLibs { - switch cc.Name() { - case "libc", "libm", "libdl": - // Special case for bionic libs. This is to prevent the bionic libs - // from being included in the search path /apex/com.android.apex/lib. - // This exclusion is required because bionic libs in the runtime APEX - // are available via the legacy paths /system/lib/libc.so, etc. By the - // init process, the bionic libs in the APEX are bind-mounted to the - // legacy paths and thus will be loaded into the default linker namespace. - // If the bionic libs are directly in /apex/com.android.apex/lib then - // the same libs will be again loaded to the runtime linker namespace, - // which will result double loading of bionic libs that isn't supported. - dirInApex = filepath.Join(dirInApex, "bionic") - } + dirInApex = filepath.Join(dirInApex, ccMod.RelativeInstallPath()) + if ccMod.Target().NativeBridge == android.NativeBridgeEnabled { + dirInApex = filepath.Join(dirInApex, ccMod.Target().NativeBridgeRelativePath) + } + if handleSpecialLibs && cc.InstallToBootstrap(ccMod.BaseModuleName(), ctx.Config()) { + // Special case for Bionic libs and other libs installed with them. This is + // to prevent those libs from being included in the search path + // /apex/com.android.runtime/${LIB}. This exclusion is required because + // those libs in the Runtime APEX are available via the legacy paths in + // /system/lib/. By the init process, the libs in the APEX are bind-mounted + // to the legacy paths and thus will be loaded into the default linker + // namespace (aka "platform" namespace). If the libs are directly in + // /apex/com.android.runtime/${LIB} then the same libs will be loaded again + // into the runtime linker namespace, which will result in double loading of + // them, which isn't supported. + dirInApex = filepath.Join(dirInApex, "bionic") } - fileToCopy = cc.OutputFile().Path() - return + fileToCopy := ccMod.OutputFile().Path() + return newApexFile(ctx, fileToCopy, ccMod.Name(), dirInApex, nativeSharedLib, ccMod) } -func getCopyManifestForExecutable(cc *cc.Module) (fileToCopy android.Path, dirInApex string) { - dirInApex = filepath.Join("bin", cc.RelativeInstallPath()) - fileToCopy = cc.OutputFile().Path() - return +func apexFileForExecutable(ctx android.BaseModuleContext, cc *cc.Module) apexFile { + dirInApex := filepath.Join("bin", cc.RelativeInstallPath()) + if cc.Target().NativeBridge == android.NativeBridgeEnabled { + dirInApex = filepath.Join(dirInApex, cc.Target().NativeBridgeRelativePath) + } + fileToCopy := cc.OutputFile().Path() + af := newApexFile(ctx, fileToCopy, cc.Name(), dirInApex, nativeExecutable, cc) + af.symlinks = cc.Symlinks() + return af } -func getCopyManifestForPyBinary(py *python.Module) (fileToCopy android.Path, dirInApex string) { - dirInApex = "bin" - fileToCopy = py.HostToolPath().Path() - return +func apexFileForPyBinary(ctx android.BaseModuleContext, py *python.Module) apexFile { + dirInApex := "bin" + fileToCopy := py.HostToolPath().Path() + return newApexFile(ctx, fileToCopy, py.Name(), dirInApex, pyBinary, py) } -func getCopyManifestForGoBinary(ctx android.ModuleContext, gb bootstrap.GoBinaryTool) (fileToCopy android.Path, dirInApex string) { - dirInApex = "bin" +func apexFileForGoBinary(ctx android.BaseModuleContext, depName string, gb bootstrap.GoBinaryTool) apexFile { + dirInApex := "bin" s, err := filepath.Rel(android.PathForOutput(ctx).String(), gb.InstallPath()) if err != nil { ctx.ModuleErrorf("Unable to use compiled binary at %s", gb.InstallPath()) - return + return apexFile{} } - fileToCopy = android.PathForOutput(ctx, s) - return + fileToCopy := android.PathForOutput(ctx, s) + // NB: Since go binaries are static we don't need the module for anything here, which is + // good since the go tool is a blueprint.Module not an android.Module like we would + // normally use. + return newApexFile(ctx, fileToCopy, depName, dirInApex, goBinary, nil) +} + +func apexFileForShBinary(ctx android.BaseModuleContext, sh *sh.ShBinary) apexFile { + dirInApex := filepath.Join("bin", sh.SubDir()) + fileToCopy := sh.OutputFile() + af := newApexFile(ctx, fileToCopy, sh.Name(), dirInApex, shBinary, sh) + af.symlinks = sh.Symlinks() + return af +} + +type javaDependency interface { + DexJar() android.Path + JacocoReportClassesFile() android.Path + LintDepSets() java.LintDepSets + + Stem() string +} + +var _ javaDependency = (*java.Library)(nil) +var _ javaDependency = (*java.SdkLibrary)(nil) +var _ javaDependency = (*java.DexImport)(nil) +var _ javaDependency = (*java.SdkLibraryImport)(nil) + +func apexFileForJavaLibrary(ctx android.BaseModuleContext, lib javaDependency, module android.Module) apexFile { + dirInApex := "javalib" + fileToCopy := lib.DexJar() + // Remove prebuilt_ if necessary so the source and prebuilt modules have the same name. + name := strings.TrimPrefix(module.Name(), "prebuilt_") + af := newApexFile(ctx, fileToCopy, name, dirInApex, javaSharedLib, module) + af.jacocoReportClassesFile = lib.JacocoReportClassesFile() + af.lintDepSets = lib.LintDepSets() + af.stem = lib.Stem() + ".jar" + return af +} + +func apexFileForPrebuiltEtc(ctx android.BaseModuleContext, prebuilt prebuilt_etc.PrebuiltEtcModule, depName string) apexFile { + dirInApex := filepath.Join("etc", prebuilt.SubDir()) + fileToCopy := prebuilt.OutputFile() + return newApexFile(ctx, fileToCopy, depName, dirInApex, etc, prebuilt) +} + +func apexFileForCompatConfig(ctx android.BaseModuleContext, config java.PlatformCompatConfigIntf, depName string) apexFile { + dirInApex := filepath.Join("etc", config.SubDir()) + fileToCopy := config.CompatConfig() + return newApexFile(ctx, fileToCopy, depName, dirInApex, etc, config) +} + +func apexFileForAndroidApp(ctx android.BaseModuleContext, aapp interface { + android.Module + Privileged() bool + InstallApkName() string + OutputFile() android.Path + JacocoReportClassesFile() android.Path + Certificate() java.Certificate +}) apexFile { + appDir := "app" + if aapp.Privileged() { + appDir = "priv-app" + } + dirInApex := filepath.Join(appDir, aapp.InstallApkName()) + fileToCopy := aapp.OutputFile() + af := newApexFile(ctx, fileToCopy, aapp.Name(), dirInApex, app, aapp) + af.jacocoReportClassesFile = aapp.JacocoReportClassesFile() + af.certificate = aapp.Certificate() + + if app, ok := aapp.(interface { + OverriddenManifestPackageName() string + }); ok { + af.overriddenPackageName = app.OverriddenManifestPackageName() + } + return af +} + +// Context "decorator", overriding the InstallBypassMake method to always reply `true`. +type flattenedApexContext struct { + android.ModuleContext +} + +func (c *flattenedApexContext) InstallBypassMake() bool { + return true +} + +// Function called while walking an APEX's payload dependencies. +// +// Return true if the `to` module should be visited, false otherwise. +type payloadDepsCallback func(ctx android.ModuleContext, from blueprint.Module, to android.ApexModule, externalDep bool) bool + +// Visit dependencies that contributes to the payload of this APEX +func (a *apexBundle) walkPayloadDeps(ctx android.ModuleContext, do payloadDepsCallback) { + ctx.WalkDeps(func(child, parent android.Module) bool { + am, ok := child.(android.ApexModule) + if !ok || !am.CanHaveApexVariants() { + return false + } + + // Check for the direct dependencies that contribute to the payload + if dt, ok := ctx.OtherModuleDependencyTag(child).(dependencyTag); ok { + if dt.payload { + return do(ctx, parent, am, false /* externalDep */) + } + // As soon as the dependency graph crosses the APEX boundary, don't go further. + return false + } + + // Check for the indirect dependencies if it is considered as part of the APEX + if am.ApexName() != "" { + return do(ctx, parent, am, false /* externalDep */) + } + + return do(ctx, parent, am, true /* externalDep */) + }) +} + +func (a *apexBundle) minSdkVersion(ctx android.BaseModuleContext) int { + ver := proptools.StringDefault(a.properties.Min_sdk_version, "current") + intVer, err := android.ApiStrToNum(ctx, ver) + if err != nil { + ctx.PropertyErrorf("min_sdk_version", "%s", err.Error()) + } + return intVer +} + +// A regexp for removing boilerplate from BaseDependencyTag from the string representation of +// a dependency tag. +var tagCleaner = regexp.MustCompile(`\QBaseDependencyTag:blueprint.BaseDependencyTag{}\E(, )?`) + +func PrettyPrintTag(tag blueprint.DependencyTag) string { + // Use tag's custom String() method if available. + if stringer, ok := tag.(fmt.Stringer); ok { + return stringer.String() + } + + // Otherwise, get a default string representation of the tag's struct. + tagString := fmt.Sprintf("%#v", tag) + + // Remove the boilerplate from BaseDependencyTag as it adds no value. + tagString = tagCleaner.ReplaceAllString(tagString, "") + return tagString } -func getCopyManifestForShBinary(sh *android.ShBinary) (fileToCopy android.Path, dirInApex string) { - dirInApex = filepath.Join("bin", sh.SubDir()) - fileToCopy = sh.OutputFile() - return +func (a *apexBundle) Updatable() bool { + return proptools.Bool(a.properties.Updatable) } -func getCopyManifestForJavaLibrary(java *java.Library) (fileToCopy android.Path, dirInApex string) { - dirInApex = "javalib" - fileToCopy = java.DexJarFile() - return +var _ android.ApexBundleDepsInfoIntf = (*apexBundle)(nil) + +// Ensures that the dependencies are marked as available for this APEX +func (a *apexBundle) checkApexAvailability(ctx android.ModuleContext) { + // Let's be practical. Availability for test, host, and the VNDK apex isn't important + if ctx.Host() || a.testApex || a.vndkApex { + return + } + + // Coverage build adds additional dependencies for the coverage-only runtime libraries. + // Requiring them and their transitive depencies with apex_available is not right + // because they just add noise. + if ctx.Config().IsEnvTrue("EMMA_INSTRUMENT") || a.IsNativeCoverageNeeded(ctx) { + return + } + + a.walkPayloadDeps(ctx, func(ctx android.ModuleContext, from blueprint.Module, to android.ApexModule, externalDep bool) bool { + if externalDep { + // As soon as the dependency graph crosses the APEX boundary, don't go further. + return false + } + + apexName := ctx.ModuleName() + fromName := ctx.OtherModuleName(from) + toName := ctx.OtherModuleName(to) + + // If `to` is not actually in the same APEX as `from` then it does not need apex_available and neither + // do any of its dependencies. + if am, ok := from.(android.DepIsInSameApex); ok && !am.DepIsInSameApex(ctx, to) { + // As soon as the dependency graph crosses the APEX boundary, don't go further. + return false + } + + if to.AvailableFor(apexName) || baselineApexAvailable(apexName, toName) { + return true + } + message := "" + tagPath := ctx.GetTagPath() + // Skip the first module as that will be added at the start of the error message by ctx.ModuleErrorf(). + walkPath := ctx.GetWalkPath()[1:] + for i, m := range walkPath { + message = fmt.Sprintf("%s\n via tag %s\n -> %s", message, PrettyPrintTag(tagPath[i]), m.String()) + } + ctx.ModuleErrorf("%q requires %q that is not available for the APEX. Dependency path:%s", fromName, toName, message) + // Visit this module's dependencies to check and report any issues with their availability. + return true + }) } -func getCopyManifestForPrebuiltEtc(prebuilt *android.PrebuiltEtc) (fileToCopy android.Path, dirInApex string) { - dirInApex = filepath.Join("etc", prebuilt.SubDir()) - fileToCopy = prebuilt.OutputFile() - return +func (a *apexBundle) checkUpdatable(ctx android.ModuleContext) { + if a.Updatable() { + if String(a.properties.Min_sdk_version) == "" { + ctx.PropertyErrorf("updatable", "updatable APEXes should set min_sdk_version as well") + } + + a.checkJavaStableSdkVersion(ctx) + } } func (a *apexBundle) GenerateAndroidBuildActions(ctx android.ModuleContext) { - filesInfo := []apexFile{} - - if a.properties.Payload_type == nil || *a.properties.Payload_type == "image" { - a.apexTypes = imageApex - } else if *a.properties.Payload_type == "zip" { - a.apexTypes = zipApex - } else if *a.properties.Payload_type == "both" { - a.apexTypes = both - } else { - ctx.PropertyErrorf("type", "%q is not one of \"image\", \"zip\", or \"both\".", *a.properties.Payload_type) + buildFlattenedAsDefault := ctx.Config().FlattenApex() && !ctx.Config().UnbundledBuild() + switch a.properties.ApexType { + case imageApex: + if buildFlattenedAsDefault { + a.suffix = imageApexSuffix + } else { + a.suffix = "" + a.primaryApexType = true + + if ctx.Config().InstallExtraFlattenedApexes() { + a.requiredDeps = append(a.requiredDeps, a.Name()+flattenedSuffix) + } + } + case zipApex: + if proptools.String(a.properties.Payload_type) == "zip" { + a.suffix = "" + a.primaryApexType = true + } else { + a.suffix = zipApexSuffix + } + case flattenedApex: + if buildFlattenedAsDefault { + a.suffix = "" + a.primaryApexType = true + } else { + a.suffix = flattenedSuffix + } + } + + if len(a.properties.Tests) > 0 && !a.testApex { + ctx.PropertyErrorf("tests", "property not allowed in apex module type") return } + a.checkApexAvailability(ctx) + a.checkUpdatable(ctx) + handleSpecialLibs := !android.Bool(a.properties.Ignore_system_library_special_case) + // native lib dependencies + var provideNativeLibs []string + var requireNativeLibs []string + + // Check if "uses" requirements are met with dependent apexBundles + var providedNativeSharedLibs []string + useVendor := proptools.Bool(a.properties.Use_vendor) + ctx.VisitDirectDepsBlueprint(func(m blueprint.Module) { + if ctx.OtherModuleDependencyTag(m) != usesTag { + return + } + otherName := ctx.OtherModuleName(m) + other, ok := m.(*apexBundle) + if !ok { + ctx.PropertyErrorf("uses", "%q is not a provider", otherName) + return + } + if proptools.Bool(other.properties.Use_vendor) != useVendor { + ctx.PropertyErrorf("use_vendor", "%q has different value of use_vendor", otherName) + return + } + if !proptools.Bool(other.properties.Provide_cpp_shared_libs) { + ctx.PropertyErrorf("uses", "%q does not provide native_shared_libs", otherName) + return + } + providedNativeSharedLibs = append(providedNativeSharedLibs, other.properties.Native_shared_libs...) + }) + + var filesInfo []apexFile + // TODO(jiyong) do this using walkPayloadDeps ctx.WalkDepsBlueprint(func(child, parent blueprint.Module) bool { - if _, ok := parent.(*apexBundle); ok { - // direct dependencies - depTag := ctx.OtherModuleDependencyTag(child) - depName := ctx.OtherModuleName(child) + depTag := ctx.OtherModuleDependencyTag(child) + if _, ok := depTag.(android.ExcludeFromApexContentsTag); ok { + return false + } + depName := ctx.OtherModuleName(child) + if _, isDirectDep := parent.(*apexBundle); isDirectDep { switch depTag { case sharedLibTag: - if cc, ok := child.(*cc.Module); ok { - fileToCopy, dirInApex := getCopyManifestForNativeLibrary(cc, handleSpecialLibs) - filesInfo = append(filesInfo, apexFile{fileToCopy, depName, dirInApex, nativeSharedLib, cc, nil}) - return true + if c, ok := child.(*cc.Module); ok { + fi := apexFileForNativeLibrary(ctx, c, handleSpecialLibs) + filesInfo = append(filesInfo, fi) + // bootstrap bionic libs are treated as provided by system + if c.HasStubsVariants() && !cc.InstallToBootstrap(c.BaseModuleName(), ctx.Config()) { + provideNativeLibs = append(provideNativeLibs, fi.Stem()) + } + return true // track transitive dependencies } else { ctx.PropertyErrorf("native_shared_libs", "%q is not a cc_library or cc_library_shared module", depName) } case executableTag: if cc, ok := child.(*cc.Module); ok { - if !cc.Arch().Native { - // There is only one 'bin' directory so we shouldn't bother copying in - // native-bridge'd binaries and only use main ones. - return true - } - fileToCopy, dirInApex := getCopyManifestForExecutable(cc) - filesInfo = append(filesInfo, apexFile{fileToCopy, depName, dirInApex, nativeExecutable, cc, cc.Symlinks()}) - return true - } else if sh, ok := child.(*android.ShBinary); ok { - fileToCopy, dirInApex := getCopyManifestForShBinary(sh) - filesInfo = append(filesInfo, apexFile{fileToCopy, depName, dirInApex, shBinary, sh, nil}) + filesInfo = append(filesInfo, apexFileForExecutable(ctx, cc)) + return true // track transitive dependencies + } else if sh, ok := child.(*sh.ShBinary); ok { + filesInfo = append(filesInfo, apexFileForShBinary(ctx, sh)) } else if py, ok := child.(*python.Module); ok && py.HostToolPath().Valid() { - fileToCopy, dirInApex := getCopyManifestForPyBinary(py) - filesInfo = append(filesInfo, apexFile{fileToCopy, depName, dirInApex, pyBinary, py, nil}) + filesInfo = append(filesInfo, apexFileForPyBinary(ctx, py)) } else if gb, ok := child.(bootstrap.GoBinaryTool); ok && a.Host() { - fileToCopy, dirInApex := getCopyManifestForGoBinary(ctx, gb) - // NB: Since go binaries are static we don't need the module for anything here, which is - // good since the go tool is a blueprint.Module not an android.Module like we would - // normally use. - filesInfo = append(filesInfo, apexFile{fileToCopy, depName, dirInApex, goBinary, nil, nil}) + filesInfo = append(filesInfo, apexFileForGoBinary(ctx, depName, gb)) } else { ctx.PropertyErrorf("binaries", "%q is neither cc_binary, (embedded) py_binary, (host) blueprint_go_binary, (host) bootstrap_go_binary, nor sh_binary", depName) } case javaLibTag: - if java, ok := child.(*java.Library); ok { - fileToCopy, dirInApex := getCopyManifestForJavaLibrary(java) - if fileToCopy == nil { + switch child.(type) { + case *java.Library, *java.SdkLibrary, *java.DexImport, *java.SdkLibraryImport: + af := apexFileForJavaLibrary(ctx, child.(javaDependency), child.(android.Module)) + if !af.Ok() { ctx.PropertyErrorf("java_libs", "%q is not configured to be compiled into dex", depName) - } else { - filesInfo = append(filesInfo, apexFile{fileToCopy, depName, dirInApex, javaSharedLib, java, nil}) + return false + } + filesInfo = append(filesInfo, af) + return true // track transitive dependencies + default: + ctx.PropertyErrorf("java_libs", "%q of type %q is not supported", depName, ctx.OtherModuleType(child)) + } + case androidAppTag: + if ap, ok := child.(*java.AndroidApp); ok { + filesInfo = append(filesInfo, apexFileForAndroidApp(ctx, ap)) + return true // track transitive dependencies + } else if ap, ok := child.(*java.AndroidAppImport); ok { + filesInfo = append(filesInfo, apexFileForAndroidApp(ctx, ap)) + } else if ap, ok := child.(*java.AndroidTestHelperApp); ok { + filesInfo = append(filesInfo, apexFileForAndroidApp(ctx, ap)) + } else if ap, ok := child.(*java.AndroidAppSet); ok { + appDir := "app" + if ap.Privileged() { + appDir = "priv-app" } - return true + af := newApexFile(ctx, ap.OutputFile(), ap.Name(), + filepath.Join(appDir, ap.BaseModuleName()), appSet, ap) + af.certificate = java.PresignedCertificate + filesInfo = append(filesInfo, af) } else { - ctx.PropertyErrorf("java_libs", "%q is not a java_library module", depName) + ctx.PropertyErrorf("apps", "%q is not an android_app module", depName) } case prebuiltTag: - if prebuilt, ok := child.(*android.PrebuiltEtc); ok { - fileToCopy, dirInApex := getCopyManifestForPrebuiltEtc(prebuilt) - filesInfo = append(filesInfo, apexFile{fileToCopy, depName, dirInApex, etc, prebuilt, nil}) - return true + if prebuilt, ok := child.(prebuilt_etc.PrebuiltEtcModule); ok { + filesInfo = append(filesInfo, apexFileForPrebuiltEtc(ctx, prebuilt, depName)) + } else if prebuilt, ok := child.(java.PlatformCompatConfigIntf); ok { + filesInfo = append(filesInfo, apexFileForCompatConfig(ctx, prebuilt, depName)) } else { - ctx.PropertyErrorf("prebuilts", "%q is not a prebuilt_etc module", depName) + ctx.PropertyErrorf("prebuilts", "%q is not a prebuilt_etc and not a platform_compat_config module", depName) + } + case testTag: + if ccTest, ok := child.(*cc.Module); ok { + if ccTest.IsTestPerSrcAllTestsVariation() { + // Multiple-output test module (where `test_per_src: true`). + // + // `ccTest` is the "" ("all tests") variation of a `test_per_src` module. + // We do not add this variation to `filesInfo`, as it has no output; + // however, we do add the other variations of this module as indirect + // dependencies (see below). + return true + } else { + // Single-output test module (where `test_per_src: false`). + af := apexFileForExecutable(ctx, ccTest) + af.class = nativeTest + filesInfo = append(filesInfo, af) + } + } else { + ctx.PropertyErrorf("tests", "%q is not a cc module", depName) } case keyTag: if key, ok := child.(*apexKey); ok { a.private_key_file = key.private_key_file a.public_key_file = key.public_key_file - return false } else { ctx.PropertyErrorf("key", "%q is not an apex_key module", depName) } + return false case certificateTag: if dep, ok := child.(*java.AndroidAppCertificate); ok { a.container_certificate_file = dep.Certificate.Pem a.container_private_key_file = dep.Certificate.Key - return false } else { ctx.ModuleErrorf("certificate dependency %q must be an android_app_certificate module", depName) } + case android.PrebuiltDepTag: + // If the prebuilt is force disabled, remember to delete the prebuilt file + // that might have been installed in the previous builds + if prebuilt, ok := child.(prebuilt); ok && prebuilt.isForceDisabled() { + a.prebuiltFileToDelete = prebuilt.InstallFilename() + } } - } else { + } else if !a.vndkApex { // indirect dependencies - if am, ok := child.(android.ApexModule); ok && am.CanHaveApexVariants() && am.IsInstallableToApex() { - if cc, ok := child.(*cc.Module); ok { - if !a.Host() && (cc.IsStubs() || cc.HasStubsVariants()) { - // If the dependency is a stubs lib, don't include it in this APEX, - // but make sure that the lib is installed on the device. - // In case no APEX is having the lib, the lib is installed to the system - // partition. - // - // Always include if we are a host-apex however since those won't have any - // system libraries. - if !android.DirectlyInAnyApex(ctx, cc.Name()) && !android.InList(cc.Name(), a.externalDeps) { - a.externalDeps = append(a.externalDeps, cc.Name()) + if am, ok := child.(android.ApexModule); ok { + // We cannot use a switch statement on `depTag` here as the checked + // tags used below are private (e.g. `cc.sharedDepTag`). + if cc.IsSharedDepTag(depTag) || cc.IsRuntimeDepTag(depTag) { + if cc, ok := child.(*cc.Module); ok { + if android.InList(cc.Name(), providedNativeSharedLibs) { + // If we're using a shared library which is provided from other APEX, + // don't include it in this APEX + return false } - // Don't track further - return false + af := apexFileForNativeLibrary(ctx, cc, handleSpecialLibs) + af.transitiveDep = true + if !a.Host() && !android.DirectlyInApex(ctx.ModuleName(), ctx.OtherModuleName(cc)) && (cc.IsStubs() || cc.HasStubsVariants()) { + // If the dependency is a stubs lib, don't include it in this APEX, + // but make sure that the lib is installed on the device. + // In case no APEX is having the lib, the lib is installed to the system + // partition. + // + // Always include if we are a host-apex however since those won't have any + // system libraries. + if !android.DirectlyInAnyApex(ctx, cc.Name()) && !android.InList(cc.Name(), a.requiredDeps) { + a.requiredDeps = append(a.requiredDeps, cc.Name()) + } + requireNativeLibs = append(requireNativeLibs, af.Stem()) + // Don't track further + return false + } + filesInfo = append(filesInfo, af) + return true // track transitive dependencies } - depName := ctx.OtherModuleName(child) - fileToCopy, dirInApex := getCopyManifestForNativeLibrary(cc, handleSpecialLibs) - filesInfo = append(filesInfo, apexFile{fileToCopy, depName, dirInApex, nativeSharedLib, cc, nil}) - return true + } else if cc.IsTestPerSrcDepTag(depTag) { + if cc, ok := child.(*cc.Module); ok { + af := apexFileForExecutable(ctx, cc) + // Handle modules created as `test_per_src` variations of a single test module: + // use the name of the generated test binary (`fileToCopy`) instead of the name + // of the original test module (`depName`, shared by all `test_per_src` + // variations of that module). + af.moduleName = filepath.Base(af.builtFile.String()) + // these are not considered transitive dep + af.transitiveDep = false + filesInfo = append(filesInfo, af) + return true // track transitive dependencies + } + } else if java.IsJniDepTag(depTag) { + // Because APK-in-APEX embeds jni_libs transitively, we don't need to track transitive deps + return false + } else if java.IsXmlPermissionsFileDepTag(depTag) { + if prebuilt, ok := child.(prebuilt_etc.PrebuiltEtcModule); ok { + filesInfo = append(filesInfo, apexFileForPrebuiltEtc(ctx, prebuilt, depName)) + } + } else if am.CanHaveApexVariants() && am.IsInstallableToApex() { + ctx.ModuleErrorf("unexpected tag %s for indirect dependency %q", PrettyPrintTag(depTag), depName) } } } return false }) - a.flattened = ctx.Config().FlattenApex() && !ctx.Config().UnbundledBuild() + // Specific to the ART apex: dexpreopt artifacts for libcore Java libraries. + // Build rules are generated by the dexpreopt singleton, and here we access build artifacts + // via the global boot image config. + if a.artApex { + for arch, files := range java.DexpreoptedArtApexJars(ctx) { + dirInApex := filepath.Join("javalib", arch.String()) + for _, f := range files { + localModule := "javalib_" + arch.String() + "_" + filepath.Base(f.String()) + af := newApexFile(ctx, f, localModule, dirInApex, etc, nil) + filesInfo = append(filesInfo, af) + } + } + } + if a.private_key_file == nil { ctx.PropertyErrorf("key", "private_key for %q could not be found", String(a.properties.Key)) return @@ -826,14 +2209,22 @@ func (a *apexBundle) GenerateAndroidBuildActions(ctx android.ModuleContext) { // remove duplicates in filesInfo removeDup := func(filesInfo []apexFile) []apexFile { - encountered := make(map[android.Path]bool) - result := []apexFile{} + encountered := make(map[string]apexFile) for _, f := range filesInfo { - if !encountered[f.builtFile] { - encountered[f.builtFile] = true - result = append(result, f) + dest := filepath.Join(f.installDir, f.builtFile.Base()) + if e, ok := encountered[dest]; !ok { + encountered[dest] = f + } else { + // If a module is directly included and also transitively depended on + // consider it as directly included. + e.transitiveDep = e.transitiveDep && f.transitiveDep + encountered[dest] = e } } + var result []apexFile + for _, v := range encountered { + result = append(result, v) + } return result } filesInfo = removeDup(filesInfo) @@ -843,467 +2234,147 @@ func (a *apexBundle) GenerateAndroidBuildActions(ctx android.ModuleContext) { return filesInfo[i].builtFile.String() < filesInfo[j].builtFile.String() }) - // prepend the name of this APEX to the module names. These names will be the names of - // modules that will be defined if the APEX is flattened. - for i := range filesInfo { - filesInfo[i].moduleName = ctx.ModuleName() + "." + filesInfo[i].moduleName - } - a.installDir = android.PathForModuleInstall(ctx, "apex") a.filesInfo = filesInfo - if a.apexTypes.zip() { - a.buildUnflattenedApex(ctx, zipApex) - } - if a.apexTypes.image() { - // Build rule for unflattened APEX is created even when ctx.Config().FlattenApex() - // is true. This is to support referencing APEX via ":<module_name" syntax - // in other modules. It is in AndroidMk where the selection of flattened - // or unflattened APEX is made. - a.buildUnflattenedApex(ctx, imageApex) - a.buildFlattenedApex(ctx) - } -} - -func (a *apexBundle) buildNoticeFile(ctx android.ModuleContext, apexFileName string) android.OptionalPath { - noticeFiles := []android.Path{} - for _, f := range a.filesInfo { - if f.module != nil { - notice := f.module.NoticeFile() - if notice.Valid() { - noticeFiles = append(noticeFiles, notice.Path()) + if a.properties.ApexType != zipApex { + if a.properties.File_contexts == nil { + a.fileContexts = android.PathForSource(ctx, "system/sepolicy/apex", ctx.ModuleName()+"-file_contexts") + } else { + a.fileContexts = android.PathForModuleSrc(ctx, *a.properties.File_contexts) + if a.Platform() { + if matched, err := path.Match("system/sepolicy/**/*", a.fileContexts.String()); err != nil || !matched { + ctx.PropertyErrorf("file_contexts", "should be under system/sepolicy, but %q", a.fileContexts) + } } } - } - // append the notice file specified in the apex module itself - if a.NoticeFile().Valid() { - noticeFiles = append(noticeFiles, a.NoticeFile().Path()) - } - - if len(noticeFiles) == 0 { - return android.OptionalPath{} - } - - return android.OptionalPathForPath( - android.BuildNoticeOutput(ctx, a.installDir, apexFileName, android.FirstUniquePaths(noticeFiles))) -} - -func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext, apexType apexPackaging) { - cert := String(a.properties.Certificate) - if cert != "" && android.SrcIsModule(cert) == "" { - defaultDir := ctx.Config().DefaultAppCertificateDir(ctx) - a.container_certificate_file = defaultDir.Join(ctx, cert+".x509.pem") - a.container_private_key_file = defaultDir.Join(ctx, cert+".pk8") - } else if cert == "" { - pem, key := ctx.Config().DefaultAppCertificate(ctx) - a.container_certificate_file = pem - a.container_private_key_file = key - } - - manifest := android.PathForModuleSrc(ctx, proptools.StringDefault(a.properties.Manifest, "apex_manifest.json")) - - var abis []string - for _, target := range ctx.MultiTargets() { - if len(target.Arch.Abi) > 0 { - abis = append(abis, target.Arch.Abi[0]) + if !android.ExistentPathForSource(ctx, a.fileContexts.String()).Valid() { + ctx.PropertyErrorf("file_contexts", "cannot find file_contexts file: %q", a.fileContexts) + return } } - - abis = android.FirstUniqueStrings(abis) - - suffix := apexType.suffix() - unsignedOutputFile := android.PathForModuleOut(ctx, ctx.ModuleName()+suffix+".unsigned") - - filesToCopy := []android.Path{} - for _, f := range a.filesInfo { - filesToCopy = append(filesToCopy, f.builtFile) + // Optimization. If we are building bundled APEX, for the files that are gathered due to the + // transitive dependencies, don't place them inside the APEX, but place a symlink pointing + // the same library in the system partition, thus effectively sharing the same libraries + // across the APEX boundary. For unbundled APEX, all the gathered files are actually placed + // in the APEX. + a.linkToSystemLib = !ctx.Config().UnbundledBuild() && + a.installable() && + !proptools.Bool(a.properties.Use_vendor) + + // We don't need the optimization for updatable APEXes, as it might give false signal + // to the system health when the APEXes are still bundled (b/149805758) + if a.Updatable() && a.properties.ApexType == imageApex { + a.linkToSystemLib = false } - copyCommands := []string{} - for i, src := range filesToCopy { - dest := filepath.Join(a.filesInfo[i].installDir, src.Base()) - dest_path := filepath.Join(android.PathForModuleOut(ctx, "image"+suffix).String(), dest) - copyCommands = append(copyCommands, "mkdir -p "+filepath.Dir(dest_path)) - copyCommands = append(copyCommands, "cp "+src.String()+" "+dest_path) - for _, sym := range a.filesInfo[i].symlinks { - symlinkDest := filepath.Join(filepath.Dir(dest_path), sym) - copyCommands = append(copyCommands, "ln -s "+filepath.Base(dest)+" "+symlinkDest) - } + // We also don't want the optimization for host APEXes, because it doesn't make sense. + if ctx.Host() { + a.linkToSystemLib = false } - implicitInputs := append(android.Paths(nil), filesToCopy...) - implicitInputs = append(implicitInputs, manifest) - - outHostBinDir := android.PathForOutput(ctx, "host", ctx.Config().PrebuiltOS(), "bin").String() - prebuiltSdkToolsBinDir := filepath.Join("prebuilts", "sdk", "tools", runtime.GOOS, "bin") - - if apexType.image() { - // files and dirs that will be created in APEX - var readOnlyPaths []string - var executablePaths []string // this also includes dirs - for _, f := range a.filesInfo { - pathInApex := filepath.Join(f.installDir, f.builtFile.Base()) - if f.installDir == "bin" { - executablePaths = append(executablePaths, pathInApex) - for _, s := range f.symlinks { - executablePaths = append(executablePaths, filepath.Join("bin", s)) - } - } else { - readOnlyPaths = append(readOnlyPaths, pathInApex) - } - dir := f.installDir - for !android.InList(dir, executablePaths) && dir != "" { - executablePaths = append(executablePaths, dir) - dir, _ = filepath.Split(dir) // move up to the parent - if len(dir) > 0 { - // remove trailing slash - dir = dir[:len(dir)-1] - } - } - } - sort.Strings(readOnlyPaths) - sort.Strings(executablePaths) - cannedFsConfig := android.PathForModuleOut(ctx, "canned_fs_config") - ctx.Build(pctx, android.BuildParams{ - Rule: generateFsConfig, - Output: cannedFsConfig, - Description: "generate fs config", - Args: map[string]string{ - "ro_paths": strings.Join(readOnlyPaths, " "), - "exec_paths": strings.Join(executablePaths, " "), - }, - }) - - fcName := proptools.StringDefault(a.properties.File_contexts, ctx.ModuleName()) - fileContextsPath := "system/sepolicy/apex/" + fcName + "-file_contexts" - fileContextsOptionalPath := android.ExistentPathForSource(ctx, fileContextsPath) - if !fileContextsOptionalPath.Valid() { - ctx.ModuleErrorf("Cannot find file_contexts file: %q", fileContextsPath) - return - } - fileContexts := fileContextsOptionalPath.Path() - - optFlags := []string{} - - // Additional implicit inputs. - implicitInputs = append(implicitInputs, cannedFsConfig, fileContexts, a.private_key_file, a.public_key_file) - optFlags = append(optFlags, "--pubkey "+a.public_key_file.String()) - - manifestPackageName, overridden := ctx.DeviceConfig().OverrideManifestPackageNameFor(ctx.ModuleName()) - if overridden { - optFlags = append(optFlags, "--override_apk_package_name "+manifestPackageName) - } - - if a.properties.AndroidManifest != nil { - androidManifestFile := android.PathForModuleSrc(ctx, proptools.String(a.properties.AndroidManifest)) - implicitInputs = append(implicitInputs, androidManifestFile) - optFlags = append(optFlags, "--android_manifest "+androidManifestFile.String()) - } - - targetSdkVersion := ctx.Config().DefaultAppTargetSdk() - if targetSdkVersion == ctx.Config().PlatformSdkCodename() && - ctx.Config().UnbundledBuild() && - !ctx.Config().UnbundledBuildUsePrebuiltSdks() && - ctx.Config().IsEnvTrue("UNBUNDLED_BUILD_TARGET_SDK_WITH_API_FINGERPRINT") { - apiFingerprint := java.ApiFingerprintPath(ctx) - targetSdkVersion += fmt.Sprintf(".$$(cat %s)", apiFingerprint.String()) - implicitInputs = append(implicitInputs, apiFingerprint) - } - optFlags = append(optFlags, "--target_sdk_version "+targetSdkVersion) - - noticeFile := a.buildNoticeFile(ctx, ctx.ModuleName()+suffix) - if noticeFile.Valid() { - // If there's a NOTICE file, embed it as an asset file in the APEX. - implicitInputs = append(implicitInputs, noticeFile.Path()) - optFlags = append(optFlags, "--assets_dir "+filepath.Dir(noticeFile.String())) - } - - ctx.Build(pctx, android.BuildParams{ - Rule: apexRule, - Implicits: implicitInputs, - Output: unsignedOutputFile, - Description: "apex (" + apexType.name() + ")", - Args: map[string]string{ - "tool_path": outHostBinDir + ":" + prebuiltSdkToolsBinDir, - "image_dir": android.PathForModuleOut(ctx, "image"+suffix).String(), - "copy_commands": strings.Join(copyCommands, " && "), - "manifest": manifest.String(), - "file_contexts": fileContexts.String(), - "canned_fs_config": cannedFsConfig.String(), - "key": a.private_key_file.String(), - "opt_flags": strings.Join(optFlags, " "), - }, - }) - apexProtoFile := android.PathForModuleOut(ctx, ctx.ModuleName()+".pb"+suffix) - bundleModuleFile := android.PathForModuleOut(ctx, ctx.ModuleName()+suffix+"-base.zip") - a.bundleModuleFile = bundleModuleFile + // prepare apex_manifest.json + a.buildManifest(ctx, provideNativeLibs, requireNativeLibs) - ctx.Build(pctx, android.BuildParams{ - Rule: apexProtoConvertRule, - Input: unsignedOutputFile, - Output: apexProtoFile, - Description: "apex proto convert", - }) - - ctx.Build(pctx, android.BuildParams{ - Rule: apexBundleRule, - Input: apexProtoFile, - Output: a.bundleModuleFile, - Description: "apex bundle module", - Args: map[string]string{ - "abi": strings.Join(abis, "."), - }, - }) + a.setCertificateAndPrivateKey(ctx) + if a.properties.ApexType == flattenedApex { + a.buildFlattenedApex(ctx) } else { - ctx.Build(pctx, android.BuildParams{ - Rule: zipApexRule, - Implicits: implicitInputs, - Output: unsignedOutputFile, - Description: "apex (" + apexType.name() + ")", - Args: map[string]string{ - "tool_path": outHostBinDir + ":" + prebuiltSdkToolsBinDir, - "image_dir": android.PathForModuleOut(ctx, "image"+suffix).String(), - "copy_commands": strings.Join(copyCommands, " && "), - "manifest": manifest.String(), - }, - }) + a.buildUnflattenedApex(ctx) } - a.outputFiles[apexType] = android.PathForModuleOut(ctx, ctx.ModuleName()+suffix) - ctx.Build(pctx, android.BuildParams{ - Rule: java.Signapk, - Description: "signapk", - Output: a.outputFiles[apexType], - Input: unsignedOutputFile, - Args: map[string]string{ - "certificates": a.container_certificate_file.String() + " " + a.container_private_key_file.String(), - "flags": "-a 4096", //alignment - }, - }) + a.compatSymlinks = makeCompatSymlinks(a.BaseModuleName(), ctx) - // Install to $OUT/soong/{target,host}/.../apex - if a.installable() && (!ctx.Config().FlattenApex() || apexType.zip()) { - ctx.InstallFile(a.installDir, ctx.ModuleName()+suffix, a.outputFiles[apexType]) - } -} + a.buildApexDependencyInfo(ctx) -func (a *apexBundle) buildFlattenedApex(ctx android.ModuleContext) { - if a.installable() { - // For flattened APEX, do nothing but make sure that apex_manifest.json and apex_pubkey are also copied along - // with other ordinary files. - manifest := android.PathForModuleSrc(ctx, proptools.StringDefault(a.properties.Manifest, "apex_manifest.json")) - - // rename to apex_manifest.json - copiedManifest := android.PathForModuleOut(ctx, "apex_manifest.json") - ctx.Build(pctx, android.BuildParams{ - Rule: android.Cp, - Input: manifest, - Output: copiedManifest, - }) - a.filesInfo = append(a.filesInfo, apexFile{copiedManifest, ctx.ModuleName() + ".apex_manifest.json", ".", etc, nil, nil}) - - // rename to apex_pubkey - copiedPubkey := android.PathForModuleOut(ctx, "apex_pubkey") - ctx.Build(pctx, android.BuildParams{ - Rule: android.Cp, - Input: a.public_key_file, - Output: copiedPubkey, - }) - a.filesInfo = append(a.filesInfo, apexFile{copiedPubkey, ctx.ModuleName() + ".apex_pubkey", ".", etc, nil, nil}) - - if ctx.Config().FlattenApex() { - for _, fi := range a.filesInfo { - dir := filepath.Join("apex", ctx.ModuleName(), fi.installDir) - target := ctx.InstallFile(android.PathForModuleInstall(ctx, dir), fi.builtFile.Base(), fi.builtFile) - for _, sym := range fi.symlinks { - ctx.InstallSymlink(android.PathForModuleInstall(ctx, dir), sym, target) - } - } - } - } + a.buildLintReports(ctx) } -func (a *apexBundle) AndroidMk() android.AndroidMkData { - if a.properties.HideFromMake { - return android.AndroidMkData{ - Disabled: true, - } - } - writers := []android.AndroidMkData{} - if a.apexTypes.image() { - writers = append(writers, a.androidMkForType(imageApex)) - } - if a.apexTypes.zip() { - writers = append(writers, a.androidMkForType(zipApex)) - } - return android.AndroidMkData{ - Custom: func(w io.Writer, name, prefix, moduleDir string, data android.AndroidMkData) { - for _, data := range writers { - data.Custom(w, name, prefix, moduleDir, data) - } - }} -} - -func (a *apexBundle) androidMkForFiles(w io.Writer, name, moduleDir string, apexType apexPackaging) []string { - moduleNames := []string{} - - for _, fi := range a.filesInfo { - if cc, ok := fi.module.(*cc.Module); ok && cc.Properties.HideFromMake { - continue - } - if !android.InList(fi.moduleName, moduleNames) { - moduleNames = append(moduleNames, fi.moduleName) - } - fmt.Fprintln(w, "\ninclude $(CLEAR_VARS)") - fmt.Fprintln(w, "LOCAL_PATH :=", moduleDir) - fmt.Fprintln(w, "LOCAL_MODULE :=", fi.moduleName) - // /apex/<name>/{lib|framework|...} - pathWhenActivated := filepath.Join("$(PRODUCT_OUT)", "apex", - proptools.StringDefault(a.properties.Apex_name, name), fi.installDir) - if a.flattened && apexType.image() { - // /system/apex/<name>/{lib|framework|...} - fmt.Fprintln(w, "LOCAL_MODULE_PATH :=", filepath.Join("$(OUT_DIR)", - a.installDir.RelPathString(), name, fi.installDir)) - fmt.Fprintln(w, "LOCAL_SOONG_SYMBOL_PATH :=", pathWhenActivated) - if len(fi.symlinks) > 0 { - fmt.Fprintln(w, "LOCAL_MODULE_SYMLINKS :=", strings.Join(fi.symlinks, " ")) - } - - if fi.module != nil && fi.module.NoticeFile().Valid() { - fmt.Fprintln(w, "LOCAL_NOTICE_FILE :=", fi.module.NoticeFile().Path().String()) - } - } else { - fmt.Fprintln(w, "LOCAL_MODULE_PATH :=", pathWhenActivated) - } - fmt.Fprintln(w, "LOCAL_PREBUILT_MODULE_FILE :=", fi.builtFile.String()) - fmt.Fprintln(w, "LOCAL_MODULE_CLASS :=", fi.class.NameInMake()) - if fi.module != nil { - archStr := fi.module.Target().Arch.ArchType.String() - host := false - switch fi.module.Target().Os.Class { - case android.Host: - if archStr != "common" { - fmt.Fprintln(w, "LOCAL_MODULE_HOST_ARCH :=", archStr) - } - host = true - case android.HostCross: - if archStr != "common" { - fmt.Fprintln(w, "LOCAL_MODULE_HOST_CROSS_ARCH :=", archStr) - } - host = true - case android.Device: - if archStr != "common" { - fmt.Fprintln(w, "LOCAL_MODULE_TARGET_ARCH :=", archStr) +// Enforce that Java deps of the apex are using stable SDKs to compile +func (a *apexBundle) checkJavaStableSdkVersion(ctx android.ModuleContext) { + // Visit direct deps only. As long as we guarantee top-level deps are using + // stable SDKs, java's checkLinkType guarantees correct usage for transitive deps + ctx.VisitDirectDepsBlueprint(func(module blueprint.Module) { + tag := ctx.OtherModuleDependencyTag(module) + switch tag { + case javaLibTag, androidAppTag: + if m, ok := module.(interface{ CheckStableSdkVersion() error }); ok { + if err := m.CheckStableSdkVersion(); err != nil { + ctx.ModuleErrorf("cannot depend on \"%v\": %v", ctx.OtherModuleName(module), err) } } - if host { - makeOs := fi.module.Target().Os.String() - if fi.module.Target().Os == android.Linux || fi.module.Target().Os == android.LinuxBionic { - makeOs = "linux" - } - fmt.Fprintln(w, "LOCAL_MODULE_HOST_OS :=", makeOs) - fmt.Fprintln(w, "LOCAL_IS_HOST_MODULE := true") - } } - if fi.class == javaSharedLib { - javaModule := fi.module.(*java.Library) - // soong_java_prebuilt.mk sets LOCAL_MODULE_SUFFIX := .jar Therefore - // we need to remove the suffix from LOCAL_MODULE_STEM, otherwise - // we will have foo.jar.jar - fmt.Fprintln(w, "LOCAL_MODULE_STEM :=", strings.TrimSuffix(fi.builtFile.Base(), ".jar")) - fmt.Fprintln(w, "LOCAL_SOONG_CLASSES_JAR :=", javaModule.ImplementationAndResourcesJars()[0].String()) - fmt.Fprintln(w, "LOCAL_SOONG_HEADER_JAR :=", javaModule.HeaderJars()[0].String()) - fmt.Fprintln(w, "LOCAL_SOONG_DEX_JAR :=", fi.builtFile.String()) - fmt.Fprintln(w, "LOCAL_DEX_PREOPT := false") - fmt.Fprintln(w, "include $(BUILD_SYSTEM)/soong_java_prebuilt.mk") - } else if fi.class == nativeSharedLib || fi.class == nativeExecutable { - fmt.Fprintln(w, "LOCAL_MODULE_STEM :=", fi.builtFile.Base()) - if cc, ok := fi.module.(*cc.Module); ok { - if cc.UnstrippedOutputFile() != nil { - fmt.Fprintln(w, "LOCAL_SOONG_UNSTRIPPED_BINARY :=", cc.UnstrippedOutputFile().String()) - } - if cc.CoverageOutputFile().Valid() { - fmt.Fprintln(w, "LOCAL_PREBUILT_COVERAGE_ARCHIVE :=", cc.CoverageOutputFile().String()) - } - } - fmt.Fprintln(w, "include $(BUILD_SYSTEM)/soong_cc_prebuilt.mk") - } else { - fmt.Fprintln(w, "LOCAL_MODULE_STEM :=", fi.builtFile.Base()) - fmt.Fprintln(w, "include $(BUILD_PREBUILT)") - } - } - return moduleNames + }) } -func (a *apexBundle) androidMkForType(apexType apexPackaging) android.AndroidMkData { - return android.AndroidMkData{ - Custom: func(w io.Writer, name, prefix, moduleDir string, data android.AndroidMkData) { - moduleNames := []string{} - if a.installable() { - moduleNames = a.androidMkForFiles(w, name, moduleDir, apexType) - } +func baselineApexAvailable(apex, moduleName string) bool { + key := apex + moduleName = normalizeModuleName(moduleName) - if a.flattened && apexType.image() { - // Only image APEXes can be flattened. - fmt.Fprintln(w, "\ninclude $(CLEAR_VARS)") - fmt.Fprintln(w, "LOCAL_PATH :=", moduleDir) - fmt.Fprintln(w, "LOCAL_MODULE :=", name) - if len(moduleNames) > 0 { - fmt.Fprintln(w, "LOCAL_REQUIRED_MODULES :=", strings.Join(moduleNames, " ")) - } - fmt.Fprintln(w, "include $(BUILD_PHONY_PACKAGE)") - } else { - // zip-apex is the less common type so have the name refer to the image-apex - // only and use {name}.zip if you want the zip-apex - if apexType == zipApex && a.apexTypes == both { - name = name + ".zip" - } - fmt.Fprintln(w, "\ninclude $(CLEAR_VARS)") - fmt.Fprintln(w, "LOCAL_PATH :=", moduleDir) - fmt.Fprintln(w, "LOCAL_MODULE :=", name) - fmt.Fprintln(w, "LOCAL_MODULE_CLASS := ETC") // do we need a new class? - fmt.Fprintln(w, "LOCAL_PREBUILT_MODULE_FILE :=", a.outputFiles[apexType].String()) - fmt.Fprintln(w, "LOCAL_MODULE_PATH :=", filepath.Join("$(OUT_DIR)", a.installDir.RelPathString())) - fmt.Fprintln(w, "LOCAL_MODULE_STEM :=", name+apexType.suffix()) - fmt.Fprintln(w, "LOCAL_UNINSTALLABLE_MODULE :=", !a.installable()) - if len(moduleNames) > 0 { - fmt.Fprintln(w, "LOCAL_REQUIRED_MODULES +=", strings.Join(moduleNames, " ")) - } - if len(a.externalDeps) > 0 { - fmt.Fprintln(w, "LOCAL_REQUIRED_MODULES +=", strings.Join(a.externalDeps, " ")) - } - fmt.Fprintln(w, "include $(BUILD_PREBUILT)") + if val, ok := apexAvailBaseline[key]; ok && android.InList(moduleName, val) { + return true + } - if apexType == imageApex { - fmt.Fprintln(w, "ALL_MODULES.$(LOCAL_MODULE).BUNDLE :=", a.bundleModuleFile.String()) - } - } - }} -} + key = android.AvailableToAnyApex + if val, ok := apexAvailBaseline[key]; ok && android.InList(moduleName, val) { + return true + } -func testApexBundleFactory() android.Module { - return ApexBundleFactory( /*testApex*/ true) + return false } -func apexBundleFactory() android.Module { - return ApexBundleFactory( /*testApex*/ false) +func normalizeModuleName(moduleName string) string { + // Prebuilt modules (e.g. java_import, etc.) have "prebuilt_" prefix added by the build + // system. Trim the prefix for the check since they are confusing + moduleName = strings.TrimPrefix(moduleName, "prebuilt_") + if strings.HasPrefix(moduleName, "libclang_rt.") { + // This module has many arch variants that depend on the product being built. + // We don't want to list them all + moduleName = "libclang_rt" + } + if strings.HasPrefix(moduleName, "androidx.") { + // TODO(b/156996905) Set apex_available/min_sdk_version for androidx support libraries + moduleName = "androidx" + } + return moduleName } -func ApexBundleFactory(testApex bool) android.Module { - module := &apexBundle{ - outputFiles: map[apexPackaging]android.WritablePath{}, - testApex: testApex, - } +func newApexBundle() *apexBundle { + module := &apexBundle{} module.AddProperties(&module.properties) module.AddProperties(&module.targetProperties) + module.AddProperties(&module.overridableProperties) module.Prefer32(func(ctx android.BaseModuleContext, base *android.ModuleBase, class android.OsClass) bool { return class == android.Device && ctx.Config().DevicePrefer32BitExecutables() }) android.InitAndroidMultiTargetsArchModule(module, android.HostAndDeviceSupported, android.MultilibCommon) android.InitDefaultableModule(module) + android.InitSdkAwareModule(module) + android.InitOverridableModule(module, &module.overridableProperties.Overrides) return module } +func ApexBundleFactory(testApex bool, artApex bool) android.Module { + bundle := newApexBundle() + bundle.testApex = testApex + bundle.artApex = artApex + return bundle +} + +// apex_test is an APEX for testing. The difference from the ordinary apex module type is that +// certain compatibility checks such as apex_available are not done for apex_test. +func testApexBundleFactory() android.Module { + bundle := newApexBundle() + bundle.testApex = true + return bundle +} + +// apex packages other modules into an APEX file which is a packaging format for system-level +// components like binaries, shared libraries, etc. +func BundleFactory() android.Module { + return newApexBundle() +} + // // Defaults // @@ -1312,9 +2383,6 @@ type Defaults struct { android.DefaultsModuleBase } -func (*Defaults) GenerateAndroidBuildActions(ctx android.ModuleContext) { -} - func defaultsFactory() android.Module { return DefaultsFactory() } @@ -1326,6 +2394,7 @@ func DefaultsFactory(props ...interface{}) android.Module { module.AddProperties( &apexBundleProperties{}, &apexTargetBundleProperties{}, + &overridableProperties{}, ) android.InitDefaultsModule(module) @@ -1333,265 +2402,24 @@ func DefaultsFactory(props ...interface{}) android.Module { } // -// Prebuilt APEX +// OverrideApex // -type Prebuilt struct { - android.ModuleBase - prebuilt android.Prebuilt - - properties PrebuiltProperties - - inputApex android.Path - installDir android.OutputPath - installFilename string - outputApex android.WritablePath -} - -type PrebuiltProperties struct { - // the path to the prebuilt .apex file to import. - Source string `blueprint:"mutated"` - ForceDisable bool `blueprint:"mutated"` - - Src *string - Arch struct { - Arm struct { - Src *string - } - Arm64 struct { - Src *string - } - X86 struct { - Src *string - } - X86_64 struct { - Src *string - } - } - - Installable *bool - // Optional name for the installed apex. If unspecified, name of the - // module is used as the file name - Filename *string -} - -func (p *Prebuilt) installable() bool { - return p.properties.Installable == nil || proptools.Bool(p.properties.Installable) -} - -func (p *Prebuilt) DepsMutator(ctx android.BottomUpMutatorContext) { - // If the device is configured to use flattened APEX, force disable the prebuilt because - // the prebuilt is a non-flattened one. - forceDisable := ctx.Config().FlattenApex() - - // Force disable the prebuilts when we are doing unbundled build. We do unbundled build - // to build the prebuilts themselves. - forceDisable = forceDisable || ctx.Config().UnbundledBuild() - - // Force disable the prebuilts when coverage is enabled. - forceDisable = forceDisable || ctx.DeviceConfig().NativeCoverageEnabled() - forceDisable = forceDisable || ctx.Config().IsEnvTrue("EMMA_INSTRUMENT") - - // b/137216042 don't use prebuilts when address sanitizer is on - forceDisable = forceDisable || android.InList("address", ctx.Config().SanitizeDevice()) || - android.InList("hwaddress", ctx.Config().SanitizeDevice()) - - if forceDisable && p.prebuilt.SourceExists() { - p.properties.ForceDisable = true - return - } - - // This is called before prebuilt_select and prebuilt_postdeps mutators - // The mutators requires that src to be set correctly for each arch so that - // arch variants are disabled when src is not provided for the arch. - if len(ctx.MultiTargets()) != 1 { - ctx.ModuleErrorf("compile_multilib shouldn't be \"both\" for prebuilt_apex") - return - } - var src string - switch ctx.MultiTargets()[0].Arch.ArchType { - case android.Arm: - src = String(p.properties.Arch.Arm.Src) - case android.Arm64: - src = String(p.properties.Arch.Arm64.Src) - case android.X86: - src = String(p.properties.Arch.X86.Src) - case android.X86_64: - src = String(p.properties.Arch.X86_64.Src) - default: - ctx.ModuleErrorf("prebuilt_apex does not support %q", ctx.MultiTargets()[0].Arch.String()) - return - } - if src == "" { - src = String(p.properties.Src) - } - p.properties.Source = src -} - -func (p *Prebuilt) Srcs() android.Paths { - return android.Paths{p.outputApex} -} - -func (p *Prebuilt) InstallFilename() string { - return proptools.StringDefault(p.properties.Filename, p.BaseModuleName()+imageApexSuffix) -} - -func (p *Prebuilt) GenerateAndroidBuildActions(ctx android.ModuleContext) { - if p.properties.ForceDisable { - return - } - - // TODO(jungjw): Check the key validity. - p.inputApex = p.Prebuilt().SingleSourcePath(ctx) - p.installDir = android.PathForModuleInstall(ctx, "apex") - p.installFilename = p.InstallFilename() - if !strings.HasSuffix(p.installFilename, imageApexSuffix) { - ctx.ModuleErrorf("filename should end in %s for prebuilt_apex", imageApexSuffix) - } - p.outputApex = android.PathForModuleOut(ctx, p.installFilename) - ctx.Build(pctx, android.BuildParams{ - Rule: android.Cp, - Input: p.inputApex, - Output: p.outputApex, - }) - if p.installable() { - ctx.InstallFile(p.installDir, p.installFilename, p.inputApex) - } -} - -func (p *Prebuilt) Prebuilt() *android.Prebuilt { - return &p.prebuilt -} - -func (p *Prebuilt) Name() string { - return p.prebuilt.Name(p.ModuleBase.Name()) -} - -func (p *Prebuilt) AndroidMk() android.AndroidMkData { - return android.AndroidMkData{ - Class: "ETC", - OutputFile: android.OptionalPathForPath(p.inputApex), - Include: "$(BUILD_PREBUILT)", - Extra: []android.AndroidMkExtraFunc{ - func(w io.Writer, outputFile android.Path) { - fmt.Fprintln(w, "LOCAL_MODULE_PATH :=", filepath.Join("$(OUT_DIR)", p.installDir.RelPathString())) - fmt.Fprintln(w, "LOCAL_MODULE_STEM :=", p.installFilename) - fmt.Fprintln(w, "LOCAL_UNINSTALLABLE_MODULE :=", !p.installable()) - }, - }, - } -} - -// prebuilt_apex imports an `.apex` file into the build graph as if it was built with apex. -func PrebuiltFactory() android.Module { - module := &Prebuilt{} - module.AddProperties(&module.properties) - android.InitSingleSourcePrebuiltModule(module, &module.properties.Source) - android.InitAndroidMultiTargetsArchModule(module, android.DeviceSupported, android.MultilibCommon) - return module -} - -type ApexSet struct { +type OverrideApex struct { android.ModuleBase - prebuilt android.Prebuilt - - properties ApexSetProperties - - installDir android.OutputPath - installFilename string - outputApex android.WritablePath + android.OverrideModuleBase } -type ApexSetProperties struct { - // the .apks file path that contains prebuilt apex files to be extracted. - Set string - - // whether the extracted apex file installable. - Installable *bool - - // optional name for the installed apex. If unspecified, name of the - // module is used as the file name - Filename *string - - // names of modules to be overridden. Listed modules can only be other binaries - // (in Make or Soong). - // This does not completely prevent installation of the overridden binaries, but if both - // binaries would be installed by default (in PRODUCT_PACKAGES) the other binary will be removed - // from PRODUCT_PACKAGES. - Overrides []string - - // apexes in this set use prerelease SDK version - Prerelease *bool +func (o *OverrideApex) GenerateAndroidBuildActions(ctx android.ModuleContext) { + // All the overrides happen in the base module. } -func (a *ApexSet) installable() bool { - return a.properties.Installable == nil || proptools.Bool(a.properties.Installable) -} +// override_apex is used to create an apex module based on another apex module +// by overriding some of its properties. +func overrideApexFactory() android.Module { + m := &OverrideApex{} + m.AddProperties(&overridableProperties{}) -func (a *ApexSet) InstallFilename() string { - return proptools.StringDefault(a.properties.Filename, a.BaseModuleName()+imageApexSuffix) -} - -func (a *ApexSet) Prebuilt() *android.Prebuilt { - return &a.prebuilt -} - -func (a *ApexSet) Name() string { - return a.prebuilt.Name(a.ModuleBase.Name()) -} - -func (a *ApexSet) Overrides() []string { - return a.properties.Overrides -} - -// prebuilt_apex imports an `.apex` file into the build graph as if it was built with apex. -func apexSetFactory() android.Module { - module := &ApexSet{} - module.AddProperties(&module.properties) - android.InitSingleSourcePrebuiltModule(module, &module.properties.Set) - android.InitAndroidMultiTargetsArchModule(module, android.DeviceSupported, android.MultilibCommon) - return module -} - -func (a *ApexSet) GenerateAndroidBuildActions(ctx android.ModuleContext) { - a.installFilename = a.InstallFilename() - if !strings.HasSuffix(a.installFilename, imageApexSuffix) { - ctx.ModuleErrorf("filename should end in %s for apex_set", imageApexSuffix) - } - - apexSet := a.prebuilt.SingleSourcePath(ctx) - a.outputApex = android.PathForModuleOut(ctx, a.installFilename) - ctx.Build(pctx, - android.BuildParams{ - Rule: extractMatchingApex, - Description: "Extract an apex from an apex set", - Inputs: android.Paths{apexSet}, - Output: a.outputApex, - Args: map[string]string{ - "abis": strings.Join(java.SupportedAbis(ctx), ","), - "allow-prereleased": strconv.FormatBool(proptools.Bool(a.properties.Prerelease)), - "sdk-version": ctx.Config().PlatformSdkVersion(), - }, - }) - a.installDir = android.PathForModuleInstall(ctx, "apex") - if a.installable() { - ctx.InstallFile(a.installDir, a.installFilename, a.outputApex) - } -} - -func (a *ApexSet) AndroidMk() android.AndroidMkData { - return android.AndroidMkData{ - Class: "ETC", - OutputFile: android.OptionalPathForPath(a.outputApex), - Include: "$(BUILD_PREBUILT)", - Extra: []android.AndroidMkExtraFunc{ - func(w io.Writer, outputFile android.Path) { - fmt.Fprintln(w, "LOCAL_MODULE_PATH := ", filepath.Join("$(OUT_DIR)", a.installDir.RelPathString())) - fmt.Fprintln(w, "LOCAL_MODULE_STEM :=", a.installFilename) - if !a.installable() { - fmt.Fprintln(w, "LOCAL_UNINSTALLABLE_MODULE := true") - } - fmt.Fprintln(w, "LOCAL_OVERRIDES_MODULES :=", strings.Join(a.properties.Overrides, " ")) - }, - }, - } + android.InitAndroidMultiTargetsArchModule(m, android.DeviceSupported, android.MultilibCommon) + android.InitOverrideModule(m) + return m } diff --git a/apex/apex_singleton.go b/apex/apex_singleton.go new file mode 100644 index 000000000..83a56a2b5 --- /dev/null +++ b/apex/apex_singleton.go @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package apex + +import ( + "github.com/google/blueprint" + + "android/soong/android" +) + +func init() { + android.RegisterSingletonType("apex_depsinfo_singleton", apexDepsInfoSingletonFactory) +} + +type apexDepsInfoSingleton struct { + // Output file with all flatlists from updatable modules' deps-info combined + updatableFlatListsPath android.OutputPath +} + +func apexDepsInfoSingletonFactory() android.Singleton { + return &apexDepsInfoSingleton{} +} + +var combineFilesRule = pctx.AndroidStaticRule("combineFilesRule", + blueprint.RuleParams{ + Command: "cat $out.rsp | xargs cat > $out", + Rspfile: "$out.rsp", + RspfileContent: "$in", + }, +) + +func (s *apexDepsInfoSingleton) GenerateBuildActions(ctx android.SingletonContext) { + updatableFlatLists := android.Paths{} + ctx.VisitAllModules(func(module android.Module) { + if binaryInfo, ok := module.(android.ApexBundleDepsInfoIntf); ok { + if path := binaryInfo.FlatListPath(); path != nil { + if binaryInfo.Updatable() { + updatableFlatLists = append(updatableFlatLists, path) + } + } + } + }) + + s.updatableFlatListsPath = android.PathForOutput(ctx, "apex", "depsinfo", "updatable-flatlists.txt") + ctx.Build(pctx, android.BuildParams{ + Rule: combineFilesRule, + Description: "Generate " + s.updatableFlatListsPath.String(), + Inputs: updatableFlatLists, + Output: s.updatableFlatListsPath, + }) +} diff --git a/apex/apex_test.go b/apex/apex_test.go index 406ade889..8803a5f97 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -17,6 +17,10 @@ package apex import ( "io/ioutil" "os" + "path" + "reflect" + "regexp" + "sort" "strings" "testing" @@ -24,203 +28,234 @@ import ( "android/soong/android" "android/soong/cc" + "android/soong/dexpreopt" + prebuilt_etc "android/soong/etc" "android/soong/java" + "android/soong/sh" ) var buildDir string -type testCustomizer func(fs map[string][]byte, config android.Config) - -func testApex(t *testing.T, bp string, handlers ...testCustomizer) *android.TestContext { - var config android.Config - config, buildDir = setup(t) - for _, handler := range handlers { - tempFS := map[string][]byte{} - handler(tempFS, config) +// names returns name list from white space separated string +func names(s string) (ns []string) { + for _, n := range strings.Split(s, " ") { + if len(n) > 0 { + ns = append(ns, n) + } } - defer teardown(buildDir) + return +} - ctx := android.NewTestArchContext() - ctx.RegisterModuleType("apex", android.ModuleFactoryAdaptor(apexBundleFactory)) - ctx.RegisterModuleType("apex_test", android.ModuleFactoryAdaptor(testApexBundleFactory)) - ctx.RegisterModuleType("apex_key", android.ModuleFactoryAdaptor(apexKeyFactory)) - ctx.RegisterModuleType("apex_defaults", android.ModuleFactoryAdaptor(defaultsFactory)) - ctx.RegisterModuleType("prebuilt_apex", android.ModuleFactoryAdaptor(PrebuiltFactory)) - ctx.RegisterModuleType("apex_set", android.ModuleFactoryAdaptor(apexSetFactory)) - ctx.PreArchMutators(android.RegisterDefaultsPreArchMutators) +func testApexError(t *testing.T, pattern, bp string, handlers ...testCustomizer) { + t.Helper() + ctx, config := testApexContext(t, bp, handlers...) + _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) + if len(errs) > 0 { + android.FailIfNoMatchingErrors(t, pattern, errs) + return + } + _, errs = ctx.PrepareBuildActions(config) + if len(errs) > 0 { + android.FailIfNoMatchingErrors(t, pattern, errs) + return + } - ctx.PostDepsMutators(func(ctx android.RegisterMutatorsContext) { - ctx.TopDown("apex_deps", apexDepsMutator) - ctx.BottomUp("apex", apexMutator) - ctx.TopDown("prebuilt_select", android.PrebuiltSelectModuleMutator).Parallel() - ctx.BottomUp("prebuilt_postdeps", android.PrebuiltPostDepsMutator).Parallel() - }) + t.Fatalf("missing expected error %q (0 errors are returned)", pattern) +} - ctx.RegisterModuleType("cc_library", android.ModuleFactoryAdaptor(cc.LibraryFactory)) - ctx.RegisterModuleType("cc_library_shared", android.ModuleFactoryAdaptor(cc.LibrarySharedFactory)) - ctx.RegisterModuleType("cc_library_headers", android.ModuleFactoryAdaptor(cc.LibraryHeaderFactory)) - ctx.RegisterModuleType("cc_binary", android.ModuleFactoryAdaptor(cc.BinaryFactory)) - ctx.RegisterModuleType("cc_object", android.ModuleFactoryAdaptor(cc.ObjectFactory)) - ctx.RegisterModuleType("llndk_library", android.ModuleFactoryAdaptor(cc.LlndkLibraryFactory)) - ctx.RegisterModuleType("toolchain_library", android.ModuleFactoryAdaptor(cc.ToolchainLibraryFactory)) - ctx.RegisterModuleType("prebuilt_etc", android.ModuleFactoryAdaptor(android.PrebuiltEtcFactory)) - ctx.RegisterModuleType("sh_binary", android.ModuleFactoryAdaptor(android.ShBinaryFactory)) - ctx.RegisterModuleType("android_app_certificate", android.ModuleFactoryAdaptor(java.AndroidAppCertificateFactory)) - ctx.RegisterModuleType("filegroup", android.ModuleFactoryAdaptor(android.FileGroupFactory)) - ctx.PreArchMutators(func(ctx android.RegisterMutatorsContext) { - ctx.BottomUp("prebuilts", android.PrebuiltMutator).Parallel() - }) - ctx.PreDepsMutators(func(ctx android.RegisterMutatorsContext) { - ctx.BottomUp("image", cc.ImageMutator).Parallel() - ctx.BottomUp("link", cc.LinkageMutator).Parallel() - ctx.BottomUp("vndk", cc.VndkMutator).Parallel() - ctx.BottomUp("version", cc.VersionMutator).Parallel() - ctx.BottomUp("begin", cc.BeginMutator).Parallel() - }) - ctx.RegisterSingletonType("apex_keys_text", android.SingletonFactoryAdaptor(apexKeysTextFactory)) +func testApex(t *testing.T, bp string, handlers ...testCustomizer) (*android.TestContext, android.Config) { + t.Helper() + ctx, config := testApexContext(t, bp, handlers...) + _, errs := ctx.ParseBlueprintsFiles(".") + android.FailIfErrored(t, errs) + _, errs = ctx.PrepareBuildActions(config) + android.FailIfErrored(t, errs) + return ctx, config +} - ctx.Register() +type testCustomizer func(fs map[string][]byte, config android.Config) - bp = bp + ` - toolchain_library { - name: "libcompiler_rt-extras", - src: "", - vendor_available: true, - recovery_available: true, +func withFiles(files map[string][]byte) testCustomizer { + return func(fs map[string][]byte, config android.Config) { + for k, v := range files { + fs[k] = v } + } +} - toolchain_library { - name: "libatomic", - src: "", - vendor_available: true, - recovery_available: true, +func withTargets(targets map[android.OsType][]android.Target) testCustomizer { + return func(fs map[string][]byte, config android.Config) { + for k, v := range targets { + config.Targets[k] = v } + } +} - toolchain_library { - name: "libgcc", - src: "", - vendor_available: true, - recovery_available: true, - } +func withManifestPackageNameOverrides(specs []string) testCustomizer { + return func(fs map[string][]byte, config android.Config) { + config.TestProductVariables.ManifestPackageNameOverrides = specs + } +} - toolchain_library { - name: "libgcc_stripped", - src: "", - vendor_available: true, - recovery_available: true, - } +func withBinder32bit(_ map[string][]byte, config android.Config) { + config.TestProductVariables.Binder32bit = proptools.BoolPtr(true) +} - toolchain_library { - name: "libclang_rt.builtins-aarch64-android", - src: "", - vendor_available: true, - recovery_available: true, - } +func withUnbundledBuild(_ map[string][]byte, config android.Config) { + config.TestProductVariables.Unbundled_build = proptools.BoolPtr(true) +} - toolchain_library { - name: "libclang_rt.builtins-arm-android", - src: "", - vendor_available: true, - recovery_available: true, - } +func testApexContext(_ *testing.T, bp string, handlers ...testCustomizer) (*android.TestContext, android.Config) { + android.ClearApexDependency() - cc_object { - name: "crtbegin_so", - stl: "none", - vendor_available: true, - recovery_available: true, + bp = bp + ` + filegroup { + name: "myapex-file_contexts", + srcs: [ + "system/sepolicy/apex/myapex-file_contexts", + ], } + ` - cc_object { - name: "crtend_so", - stl: "none", - vendor_available: true, - recovery_available: true, - } + bp = bp + cc.GatherRequiredDepsForTest(android.Android) + + bp = bp + java.GatherRequiredDepsForTest() + + fs := map[string][]byte{ + "a.java": nil, + "PrebuiltAppFoo.apk": nil, + "PrebuiltAppFooPriv.apk": nil, + "build/make/target/product/security": nil, + "apex_manifest.json": nil, + "AndroidManifest.xml": nil, + "system/sepolicy/apex/myapex-file_contexts": nil, + "system/sepolicy/apex/myapex.updatable-file_contexts": nil, + "system/sepolicy/apex/myapex2-file_contexts": nil, + "system/sepolicy/apex/otherapex-file_contexts": nil, + "system/sepolicy/apex/commonapex-file_contexts": nil, + "system/sepolicy/apex/com.android.vndk-file_contexts": nil, + "mylib.cpp": nil, + "mylib_common.cpp": nil, + "mytest.cpp": nil, + "mytest1.cpp": nil, + "mytest2.cpp": nil, + "mytest3.cpp": nil, + "myprebuilt": nil, + "my_include": nil, + "foo/bar/MyClass.java": nil, + "prebuilt.jar": nil, + "prebuilt.so": nil, + "vendor/foo/devkeys/test.x509.pem": nil, + "vendor/foo/devkeys/test.pk8": nil, + "testkey.x509.pem": nil, + "testkey.pk8": nil, + "testkey.override.x509.pem": nil, + "testkey.override.pk8": nil, + "vendor/foo/devkeys/testkey.avbpubkey": nil, + "vendor/foo/devkeys/testkey.pem": nil, + "NOTICE": nil, + "custom_notice": nil, + "custom_notice_for_static_lib": nil, + "testkey2.avbpubkey": nil, + "testkey2.pem": nil, + "myapex-arm64.apex": nil, + "myapex-arm.apex": nil, + "myapex.apks": nil, + "frameworks/base/api/current.txt": nil, + "framework/aidl/a.aidl": nil, + "build/make/core/proguard.flags": nil, + "build/make/core/proguard_basic_keeps.flags": nil, + "dummy.txt": nil, + "AppSet.apks": nil, + } - cc_object { - name: "crtbegin_static", - stl: "none", - } + cc.GatherRequiredFilesForTest(fs) - cc_object { - name: "crtend_android", - stl: "none", - } + for _, handler := range handlers { + // The fs now needs to be populated before creating the config, call handlers twice + // for now, once to get any fs changes, and later after the config was created to + // set product variables or targets. + tempConfig := android.TestArchConfig(buildDir, nil, bp, fs) + handler(fs, tempConfig) + } - llndk_library { - name: "libc", - symbol_file: "", - } + config := android.TestArchConfig(buildDir, nil, bp, fs) + config.TestProductVariables.DeviceVndkVersion = proptools.StringPtr("current") + config.TestProductVariables.DefaultAppCertificate = proptools.StringPtr("vendor/foo/devkeys/test") + config.TestProductVariables.CertificateOverrides = []string{"myapex_keytest:myapex.certificate.override"} + config.TestProductVariables.Platform_sdk_codename = proptools.StringPtr("Q") + config.TestProductVariables.Platform_sdk_final = proptools.BoolPtr(false) + config.TestProductVariables.Platform_vndk_version = proptools.StringPtr("VER") - llndk_library { - name: "libm", - symbol_file: "", - } + for _, handler := range handlers { + // The fs now needs to be populated before creating the config, call handlers twice + // for now, earlier to get any fs changes, and now after the config was created to + // set product variables or targets. + tempFS := map[string][]byte{} + handler(tempFS, config) + } - llndk_library { - name: "libdl", - symbol_file: "", - } - ` + ctx := android.NewTestArchContext() - ctx.MockFileSystem(map[string][]byte{ - "Android.bp": []byte(bp), - "build/target/product/security": nil, - "apex_manifest.json": nil, - "AndroidManifest.xml": nil, - "system/sepolicy/apex/myapex-file_contexts": nil, - "system/sepolicy/apex/myapex_keytest-file_contexts": nil, - "system/sepolicy/apex/otherapex-file_contexts": nil, - "mylib.cpp": nil, - "myprebuilt": nil, - "my_include": nil, - "vendor/foo/devkeys/test.x509.pem": nil, - "vendor/foo/devkeys/test.pk8": nil, - "testkey.x509.pem": nil, - "testkey.pk8": nil, - "testkey.override.x509.pem": nil, - "testkey.override.pk8": nil, - "vendor/foo/devkeys/testkey.avbpubkey": nil, - "vendor/foo/devkeys/testkey.pem": nil, - "NOTICE": nil, - "custom_notice": nil, - "testkey2.avbpubkey": nil, - "testkey2.pem": nil, - "myapex-arm64.apex": nil, - "myapex-arm.apex": nil, - "myapex.apks": nil, - "frameworks/base/api/current.txt": nil, - }) - _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) - android.FailIfErrored(t, errs) - _, errs = ctx.PrepareBuildActions(config) - android.FailIfErrored(t, errs) + // from android package + android.RegisterPackageBuildComponents(ctx) + ctx.PreArchMutators(android.RegisterVisibilityRuleChecker) + + ctx.RegisterModuleType("apex", BundleFactory) + ctx.RegisterModuleType("apex_test", testApexBundleFactory) + ctx.RegisterModuleType("apex_vndk", vndkApexBundleFactory) + ctx.RegisterModuleType("apex_key", ApexKeyFactory) + ctx.RegisterModuleType("apex_defaults", defaultsFactory) + ctx.RegisterModuleType("prebuilt_apex", PrebuiltFactory) + ctx.RegisterModuleType("override_apex", overrideApexFactory) + ctx.RegisterModuleType("apex_set", apexSetFactory) - return ctx + ctx.PreArchMutators(android.RegisterDefaultsPreArchMutators) + ctx.PostDepsMutators(android.RegisterOverridePostDepsMutators) + + cc.RegisterRequiredBuildComponentsForTest(ctx) + + // Register this after the prebuilt mutators have been registered (in + // cc.RegisterRequiredBuildComponentsForTest) to match what happens at runtime. + ctx.PreArchMutators(android.RegisterVisibilityRuleGatherer) + ctx.PostDepsMutators(android.RegisterVisibilityRuleEnforcer) + + ctx.RegisterModuleType("cc_test", cc.TestFactory) + ctx.RegisterModuleType("vndk_prebuilt_shared", cc.VndkPrebuiltSharedFactory) + ctx.RegisterModuleType("vndk_libraries_txt", cc.VndkLibrariesTxtFactory) + ctx.RegisterModuleType("prebuilt_etc", prebuilt_etc.PrebuiltEtcFactory) + ctx.RegisterModuleType("platform_compat_config", java.PlatformCompatConfigFactory) + ctx.RegisterModuleType("sh_binary", sh.ShBinaryFactory) + ctx.RegisterModuleType("filegroup", android.FileGroupFactory) + java.RegisterJavaBuildComponents(ctx) + java.RegisterSystemModulesBuildComponents(ctx) + java.RegisterAppBuildComponents(ctx) + java.RegisterSdkLibraryBuildComponents(ctx) + ctx.RegisterSingletonType("apex_keys_text", apexKeysTextFactory) + + ctx.PreDepsMutators(RegisterPreDepsMutators) + ctx.PostDepsMutators(RegisterPostDepsMutators) + + ctx.Register(config) + + return ctx, config } -func setup(t *testing.T) (config android.Config, buildDir string) { - buildDir, err := ioutil.TempDir("", "soong_apex_test") +func setUp() { + var err error + buildDir, err = ioutil.TempDir("", "soong_apex_test") if err != nil { - t.Fatal(err) + panic(err) } - - config = android.TestArchConfig(buildDir, nil) - config.TestProductVariables.DeviceVndkVersion = proptools.StringPtr("current") - config.TestProductVariables.DefaultAppCertificate = proptools.StringPtr("vendor/foo/devkeys/test") - config.TestProductVariables.CertificateOverrides = []string{"myapex_keytest:myapex.certificate.override"} - config.TestProductVariables.Platform_sdk_codename = proptools.StringPtr("Q") - config.TestProductVariables.Platform_sdk_final = proptools.BoolPtr(false) - return } -func teardown(buildDir string) { - os.RemoveAll(buildDir) +func tearDown() { + _ = os.RemoveAll(buildDir) } // ensure that 'result' contains 'expected' func ensureContains(t *testing.T, result string, expected string) { + t.Helper() if !strings.Contains(result, expected) { t.Errorf("%q is not found in %q", expected, result) } @@ -228,26 +263,47 @@ func ensureContains(t *testing.T, result string, expected string) { // ensures that 'result' does not contain 'notExpected' func ensureNotContains(t *testing.T, result string, notExpected string) { + t.Helper() if strings.Contains(result, notExpected) { t.Errorf("%q is found in %q", notExpected, result) } } +func ensureMatches(t *testing.T, result string, expectedRex string) { + ok, err := regexp.MatchString(expectedRex, result) + if err != nil { + t.Fatalf("regexp failure trying to match %s against `%s` expression: %s", result, expectedRex, err) + return + } + if !ok { + t.Errorf("%s does not match regular expession %s", result, expectedRex) + } +} + func ensureListContains(t *testing.T, result []string, expected string) { + t.Helper() if !android.InList(expected, result) { t.Errorf("%q is not found in %v", expected, result) } } func ensureListNotContains(t *testing.T, result []string, notExpected string) { + t.Helper() if android.InList(notExpected, result) { t.Errorf("%q is found in %v", notExpected, result) } } +func ensureListEmpty(t *testing.T, result []string) { + t.Helper() + if len(result) > 0 { + t.Errorf("%q is expected to be empty", result) + } +} + // Minimal test func TestBasicApex(t *testing.T) { - ctx := testApex(t, ` + ctx, config := testApex(t, ` apex_defaults { name: "myapex-defaults", manifest: ":myapex.manifest", @@ -258,7 +314,11 @@ func TestBasicApex(t *testing.T) { both: { binaries: ["foo",], } - } + }, + java_libs: [ + "myjar", + "myjar_dex", + ], } apex { @@ -288,6 +348,11 @@ func TestBasicApex(t *testing.T) { shared_libs: ["mylib2"], system_shared_libs: [], stl: "none", + // TODO: remove //apex_available:platform + apex_available: [ + "//apex_available:platform", + "myapex", + ], } cc_binary { @@ -307,23 +372,106 @@ func TestBasicApex(t *testing.T) { system_shared_libs: [], static_executable: true, stl: "none", + apex_available: [ "myapex" ], } - cc_library { + cc_library_shared { name: "mylib2", srcs: ["mylib.cpp"], system_shared_libs: [], stl: "none", notice: "custom_notice", + static_libs: ["libstatic"], + // TODO: remove //apex_available:platform + apex_available: [ + "//apex_available:platform", + "myapex", + ], + } + + cc_prebuilt_library_shared { + name: "mylib2", + srcs: ["prebuilt.so"], + // TODO: remove //apex_available:platform + apex_available: [ + "//apex_available:platform", + "myapex", + ], + } + + cc_library_static { + name: "libstatic", + srcs: ["mylib.cpp"], + system_shared_libs: [], + stl: "none", + notice: "custom_notice_for_static_lib", + // TODO: remove //apex_available:platform + apex_available: [ + "//apex_available:platform", + "myapex", + ], + } + + java_library { + name: "myjar", + srcs: ["foo/bar/MyClass.java"], + stem: "myjar_stem", + sdk_version: "none", + system_modules: "none", + static_libs: ["myotherjar"], + libs: ["mysharedjar"], + // TODO: remove //apex_available:platform + apex_available: [ + "//apex_available:platform", + "myapex", + ], + } + + dex_import { + name: "myjar_dex", + jars: ["prebuilt.jar"], + apex_available: [ + "//apex_available:platform", + "myapex", + ], + } + + java_library { + name: "myotherjar", + srcs: ["foo/bar/MyClass.java"], + sdk_version: "none", + system_modules: "none", + // TODO: remove //apex_available:platform + apex_available: [ + "//apex_available:platform", + "myapex", + ], + } + + java_library { + name: "mysharedjar", + srcs: ["foo/bar/MyClass.java"], + sdk_version: "none", + system_modules: "none", } `) - apexRule := ctx.ModuleForTests("myapex", "android_common_myapex").Rule("apexRule") + apexRule := ctx.ModuleForTests("myapex", "android_common_myapex_image").Rule("apexRule") + + // Make sure that Android.mk is created + ab := ctx.ModuleForTests("myapex", "android_common_myapex_image").Module().(*apexBundle) + data := android.AndroidMkDataForTest(t, config, "", ab) + var builder strings.Builder + data.Custom(&builder, ab.BaseModuleName(), "TARGET_", "", data) + + androidMk := builder.String() + ensureContains(t, androidMk, "LOCAL_MODULE := mylib.myapex\n") + ensureNotContains(t, androidMk, "LOCAL_MODULE := mylib.com.android.myapex\n") optFlags := apexRule.Args["opt_flags"] ensureContains(t, optFlags, "--pubkey vendor/foo/devkeys/testkey.avbpubkey") // Ensure that the NOTICE output is being packaged as an asset. - ensureContains(t, optFlags, "--assets_dir "+buildDir+"/.intermediates/myapex/android_common_myapex/NOTICE") + ensureContains(t, optFlags, "--assets_dir "+buildDir+"/.intermediates/myapex/android_common_myapex_image/NOTICE") copyCmds := apexRule.Args["copy_commands"] @@ -331,24 +479,38 @@ func TestBasicApex(t *testing.T) { ensureContains(t, apexRule.Output.String(), "myapex.apex.unsigned") // Ensure that apex variant is created for the direct dep - ensureListContains(t, ctx.ModuleVariantsForTests("mylib"), "android_arm64_armv8-a_core_shared_myapex") + ensureListContains(t, ctx.ModuleVariantsForTests("mylib"), "android_arm64_armv8-a_shared_myapex") + ensureListContains(t, ctx.ModuleVariantsForTests("myjar"), "android_common_myapex") + ensureListContains(t, ctx.ModuleVariantsForTests("myjar_dex"), "android_common_myapex") // Ensure that apex variant is created for the indirect dep - ensureListContains(t, ctx.ModuleVariantsForTests("mylib2"), "android_arm64_armv8-a_core_shared_myapex") + ensureListContains(t, ctx.ModuleVariantsForTests("mylib2"), "android_arm64_armv8-a_shared_myapex") + ensureListContains(t, ctx.ModuleVariantsForTests("myotherjar"), "android_common_myapex") // Ensure that both direct and indirect deps are copied into apex ensureContains(t, copyCmds, "image.apex/lib64/mylib.so") ensureContains(t, copyCmds, "image.apex/lib64/mylib2.so") - - // Ensure that the platform variant ends with _core_shared - ensureListContains(t, ctx.ModuleVariantsForTests("mylib"), "android_arm64_armv8-a_core_shared") - ensureListContains(t, ctx.ModuleVariantsForTests("mylib2"), "android_arm64_armv8-a_core_shared") + ensureContains(t, copyCmds, "image.apex/javalib/myjar_stem.jar") + ensureContains(t, copyCmds, "image.apex/javalib/myjar_dex.jar") + // .. but not for java libs + ensureNotContains(t, copyCmds, "image.apex/javalib/myotherjar.jar") + ensureNotContains(t, copyCmds, "image.apex/javalib/msharedjar.jar") + + // Ensure that the platform variant ends with _shared or _common + ensureListContains(t, ctx.ModuleVariantsForTests("mylib"), "android_arm64_armv8-a_shared") + ensureListContains(t, ctx.ModuleVariantsForTests("mylib2"), "android_arm64_armv8-a_shared") + ensureListContains(t, ctx.ModuleVariantsForTests("myjar"), "android_common") + ensureListContains(t, ctx.ModuleVariantsForTests("myotherjar"), "android_common") + ensureListContains(t, ctx.ModuleVariantsForTests("mysharedjar"), "android_common") + + // Ensure that dynamic dependency to java libs are not included + ensureListNotContains(t, ctx.ModuleVariantsForTests("mysharedjar"), "android_common_myapex") // Ensure that all symlinks are present. found_foo_link_64 := false found_foo := false for _, cmd := range strings.Split(copyCmds, " && ") { - if strings.HasPrefix(cmd, "ln -s foo64") { + if strings.HasPrefix(cmd, "ln -sfn foo64") { if strings.HasSuffix(cmd, "bin/foo") { found_foo = true } else if strings.HasSuffix(cmd, "bin/foo_link_64") { @@ -361,17 +523,111 @@ func TestBasicApex(t *testing.T) { t.Errorf("Could not find all expected symlinks! foo: %t, foo_link_64: %t. Command was %s", found_foo, found_foo_link_64, copyCmds) } - mergeNoticesRule := ctx.ModuleForTests("myapex", "android_common_myapex").Rule("mergeNoticesRule") + mergeNoticesRule := ctx.ModuleForTests("myapex", "android_common_myapex_image").Rule("mergeNoticesRule") noticeInputs := mergeNoticesRule.Inputs.Strings() - if len(noticeInputs) != 2 { - t.Errorf("number of input notice files: expected = 2, actual = %q", len(noticeInputs)) + if len(noticeInputs) != 3 { + t.Errorf("number of input notice files: expected = 3, actual = %q", len(noticeInputs)) } ensureListContains(t, noticeInputs, "NOTICE") ensureListContains(t, noticeInputs, "custom_notice") + ensureListContains(t, noticeInputs, "custom_notice_for_static_lib") + + fullDepsInfo := strings.Split(ctx.ModuleForTests("myapex", "android_common_myapex_image").Output("depsinfo/fulllist.txt").Args["content"], "\\n") + ensureListContains(t, fullDepsInfo, "myjar(minSdkVersion:(no version)) <- myapex") + ensureListContains(t, fullDepsInfo, "mylib(minSdkVersion:(no version)) <- myapex") + ensureListContains(t, fullDepsInfo, "mylib2(minSdkVersion:(no version)) <- mylib") + ensureListContains(t, fullDepsInfo, "myotherjar(minSdkVersion:(no version)) <- myjar") + ensureListContains(t, fullDepsInfo, "mysharedjar(minSdkVersion:(no version)) (external) <- myjar") + + flatDepsInfo := strings.Split(ctx.ModuleForTests("myapex", "android_common_myapex_image").Output("depsinfo/flatlist.txt").Args["content"], "\\n") + ensureListContains(t, flatDepsInfo, " myjar(minSdkVersion:(no version))") + ensureListContains(t, flatDepsInfo, " mylib(minSdkVersion:(no version))") + ensureListContains(t, flatDepsInfo, " mylib2(minSdkVersion:(no version))") + ensureListContains(t, flatDepsInfo, " myotherjar(minSdkVersion:(no version))") + ensureListContains(t, flatDepsInfo, " mysharedjar(minSdkVersion:(no version)) (external)") +} + +func TestDefaults(t *testing.T) { + ctx, _ := testApex(t, ` + apex_defaults { + name: "myapex-defaults", + key: "myapex.key", + prebuilts: ["myetc"], + native_shared_libs: ["mylib"], + java_libs: ["myjar"], + apps: ["AppFoo"], + } + + prebuilt_etc { + name: "myetc", + src: "myprebuilt", + } + + apex { + name: "myapex", + defaults: ["myapex-defaults"], + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + cc_library { + name: "mylib", + system_shared_libs: [], + stl: "none", + apex_available: [ "myapex" ], + } + + java_library { + name: "myjar", + srcs: ["foo/bar/MyClass.java"], + sdk_version: "none", + system_modules: "none", + apex_available: [ "myapex" ], + } + + android_app { + name: "AppFoo", + srcs: ["foo/bar/MyClass.java"], + sdk_version: "none", + system_modules: "none", + apex_available: [ "myapex" ], + } + `) + ensureExactContents(t, ctx, "myapex", "android_common_myapex_image", []string{ + "etc/myetc", + "javalib/myjar.jar", + "lib64/mylib.so", + "app/AppFoo/AppFoo.apk", + }) +} + +func TestApexManifest(t *testing.T) { + ctx, _ := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + `) + + module := ctx.ModuleForTests("myapex", "android_common_myapex_image") + args := module.Rule("apexRule").Args + if manifest := args["manifest"]; manifest != module.Output("apex_manifest.pb").Output.String() { + t.Error("manifest should be apex_manifest.pb, but " + manifest) + } } func TestBasicZipApex(t *testing.T) { - ctx := testApex(t, ` + ctx, _ := testApex(t, ` apex { name: "myapex", key: "myapex.key", @@ -391,6 +647,7 @@ func TestBasicZipApex(t *testing.T) { shared_libs: ["mylib2"], system_shared_libs: [], stl: "none", + apex_available: [ "myapex" ], } cc_library { @@ -398,20 +655,21 @@ func TestBasicZipApex(t *testing.T) { srcs: ["mylib.cpp"], system_shared_libs: [], stl: "none", + apex_available: [ "myapex" ], } `) - zipApexRule := ctx.ModuleForTests("myapex", "android_common_myapex").Rule("zipApexRule") + zipApexRule := ctx.ModuleForTests("myapex", "android_common_myapex_zip").Rule("zipApexRule") copyCmds := zipApexRule.Args["copy_commands"] // Ensure that main rule creates an output ensureContains(t, zipApexRule.Output.String(), "myapex.zipapex.unsigned") // Ensure that APEX variant is created for the direct dep - ensureListContains(t, ctx.ModuleVariantsForTests("mylib"), "android_arm64_armv8-a_core_shared_myapex") + ensureListContains(t, ctx.ModuleVariantsForTests("mylib"), "android_arm64_armv8-a_shared_myapex") // Ensure that APEX variant is created for the indirect dep - ensureListContains(t, ctx.ModuleVariantsForTests("mylib2"), "android_arm64_armv8-a_core_shared_myapex") + ensureListContains(t, ctx.ModuleVariantsForTests("mylib2"), "android_arm64_armv8-a_shared_myapex") // Ensure that both direct and indirect deps are copied into apex ensureContains(t, copyCmds, "image.zipapex/lib64/mylib.so") @@ -419,7 +677,7 @@ func TestBasicZipApex(t *testing.T) { } func TestApexWithStubs(t *testing.T) { - ctx := testApex(t, ` + ctx, _ := testApex(t, ` apex { name: "myapex", key: "myapex.key", @@ -438,6 +696,7 @@ func TestApexWithStubs(t *testing.T) { shared_libs: ["mylib2", "mylib3"], system_shared_libs: [], stl: "none", + apex_available: [ "myapex" ], } cc_library { @@ -460,6 +719,7 @@ func TestApexWithStubs(t *testing.T) { stubs: { versions: ["10", "11", "12"], }, + apex_available: [ "myapex" ], } cc_library { @@ -467,10 +727,11 @@ func TestApexWithStubs(t *testing.T) { srcs: ["mylib.cpp"], system_shared_libs: [], stl: "none", + apex_available: [ "myapex" ], } `) - apexRule := ctx.ModuleForTests("myapex", "android_common_myapex").Rule("apexRule") + apexRule := ctx.ModuleForTests("myapex", "android_common_myapex_image").Rule("apexRule") copyCmds := apexRule.Args["copy_commands"] // Ensure that direct non-stubs dep is always included @@ -482,36 +743,42 @@ func TestApexWithStubs(t *testing.T) { // Ensure that direct stubs dep is included ensureContains(t, copyCmds, "image.apex/lib64/mylib3.so") - mylibLdFlags := ctx.ModuleForTests("mylib", "android_arm64_armv8-a_core_shared_myapex").Rule("ld").Args["libFlags"] + mylibLdFlags := ctx.ModuleForTests("mylib", "android_arm64_armv8-a_shared_myapex").Rule("ld").Args["libFlags"] // Ensure that mylib is linking with the latest version of stubs for mylib2 - ensureContains(t, mylibLdFlags, "mylib2/android_arm64_armv8-a_core_shared_3_myapex/mylib2.so") + ensureContains(t, mylibLdFlags, "mylib2/android_arm64_armv8-a_shared_3/mylib2.so") // ... and not linking to the non-stub (impl) variant of mylib2 - ensureNotContains(t, mylibLdFlags, "mylib2/android_arm64_armv8-a_core_shared_myapex/mylib2.so") + ensureNotContains(t, mylibLdFlags, "mylib2/android_arm64_armv8-a_shared/mylib2.so") // Ensure that mylib is linking with the non-stub (impl) of mylib3 (because mylib3 is in the same apex) - ensureContains(t, mylibLdFlags, "mylib3/android_arm64_armv8-a_core_shared_myapex/mylib3.so") + ensureContains(t, mylibLdFlags, "mylib3/android_arm64_armv8-a_shared_myapex/mylib3.so") // .. and not linking to the stubs variant of mylib3 - ensureNotContains(t, mylibLdFlags, "mylib3/android_arm64_armv8-a_core_shared_12_myapex/mylib3.so") + ensureNotContains(t, mylibLdFlags, "mylib3/android_arm64_armv8-a_shared_12_myapex/mylib3.so") // Ensure that stubs libs are built without -include flags - mylib2Cflags := ctx.ModuleForTests("mylib2", "android_arm64_armv8-a_core_static_myapex").Rule("cc").Args["cFlags"] + mylib2Cflags := ctx.ModuleForTests("mylib2", "android_arm64_armv8-a_static").Rule("cc").Args["cFlags"] ensureNotContains(t, mylib2Cflags, "-include ") // Ensure that genstub is invoked with --apex - ensureContains(t, "--apex", ctx.ModuleForTests("mylib2", "android_arm64_armv8-a_core_static_3_myapex").Rule("genStubSrc").Args["flags"]) + ensureContains(t, "--apex", ctx.ModuleForTests("mylib2", "android_arm64_armv8-a_static_3").Rule("genStubSrc").Args["flags"]) + + ensureExactContents(t, ctx, "myapex", "android_common_myapex_image", []string{ + "lib64/mylib.so", + "lib64/mylib3.so", + "lib64/mylib4.so", + }) } func TestApexWithExplicitStubsDependency(t *testing.T) { - ctx := testApex(t, ` + ctx, _ := testApex(t, ` apex { - name: "myapex", - key: "myapex.key", + name: "myapex2", + key: "myapex2.key", native_shared_libs: ["mylib"], } apex_key { - name: "myapex.key", + name: "myapex2.key", public_key: "testkey.avbpubkey", private_key: "testkey.pem", } @@ -520,8 +787,10 @@ func TestApexWithExplicitStubsDependency(t *testing.T) { name: "mylib", srcs: ["mylib.cpp"], shared_libs: ["libfoo#10"], + static_libs: ["libbaz"], system_shared_libs: [], stl: "none", + apex_available: [ "myapex2" ], } cc_library { @@ -542,9 +811,17 @@ func TestApexWithExplicitStubsDependency(t *testing.T) { stl: "none", } + cc_library_static { + name: "libbaz", + srcs: ["mylib.cpp"], + system_shared_libs: [], + stl: "none", + apex_available: [ "myapex2" ], + } + `) - apexRule := ctx.ModuleForTests("myapex", "android_common_myapex").Rule("apexRule") + apexRule := ctx.ModuleForTests("myapex2", "android_common_myapex2_image").Rule("apexRule") copyCmds := apexRule.Args["copy_commands"] // Ensure that direct non-stubs dep is always included @@ -556,25 +833,43 @@ func TestApexWithExplicitStubsDependency(t *testing.T) { // Ensure that dependency of stubs is not included ensureNotContains(t, copyCmds, "image.apex/lib64/libbar.so") - mylibLdFlags := ctx.ModuleForTests("mylib", "android_arm64_armv8-a_core_shared_myapex").Rule("ld").Args["libFlags"] + mylibLdFlags := ctx.ModuleForTests("mylib", "android_arm64_armv8-a_shared_myapex2").Rule("ld").Args["libFlags"] // Ensure that mylib is linking with version 10 of libfoo - ensureContains(t, mylibLdFlags, "libfoo/android_arm64_armv8-a_core_shared_10_myapex/libfoo.so") + ensureContains(t, mylibLdFlags, "libfoo/android_arm64_armv8-a_shared_10/libfoo.so") // ... and not linking to the non-stub (impl) variant of libfoo - ensureNotContains(t, mylibLdFlags, "libfoo/android_arm64_armv8-a_core_shared_myapex/libfoo.so") + ensureNotContains(t, mylibLdFlags, "libfoo/android_arm64_armv8-a_shared/libfoo.so") - libFooStubsLdFlags := ctx.ModuleForTests("libfoo", "android_arm64_armv8-a_core_shared_10_myapex").Rule("ld").Args["libFlags"] + libFooStubsLdFlags := ctx.ModuleForTests("libfoo", "android_arm64_armv8-a_shared_10").Rule("ld").Args["libFlags"] // Ensure that libfoo stubs is not linking to libbar (since it is a stubs) ensureNotContains(t, libFooStubsLdFlags, "libbar.so") + + fullDepsInfo := strings.Split(ctx.ModuleForTests("myapex2", "android_common_myapex2_image").Output("depsinfo/fulllist.txt").Args["content"], "\\n") + ensureListContains(t, fullDepsInfo, "mylib(minSdkVersion:(no version)) <- myapex2") + ensureListContains(t, fullDepsInfo, "libbaz(minSdkVersion:(no version)) <- mylib") + ensureListContains(t, fullDepsInfo, "libfoo(minSdkVersion:(no version)) (external) <- mylib") + + flatDepsInfo := strings.Split(ctx.ModuleForTests("myapex2", "android_common_myapex2_image").Output("depsinfo/flatlist.txt").Args["content"], "\\n") + ensureListContains(t, flatDepsInfo, " mylib(minSdkVersion:(no version))") + ensureListContains(t, flatDepsInfo, " libbaz(minSdkVersion:(no version))") + ensureListContains(t, flatDepsInfo, " libfoo(minSdkVersion:(no version)) (external)") } -func TestApexWithSystemLibsStubs(t *testing.T) { - ctx := testApex(t, ` +func TestApexWithRuntimeLibsDependency(t *testing.T) { + /* + myapex + | + v (runtime_libs) + mylib ------+------> libfoo [provides stub] + | + `------> libbar + */ + ctx, _ := testApex(t, ` apex { name: "myapex", key: "myapex.key", - native_shared_libs: ["mylib", "mylib_shared", "libdl", "libm"], + native_shared_libs: ["mylib"], } apex_key { @@ -586,48 +881,163 @@ func TestApexWithSystemLibsStubs(t *testing.T) { cc_library { name: "mylib", srcs: ["mylib.cpp"], - shared_libs: ["libdl#27"], - stl: "none", - } - - cc_library_shared { - name: "mylib_shared", - srcs: ["mylib.cpp"], - shared_libs: ["libdl#27"], + runtime_libs: ["libfoo", "libbar"], + system_shared_libs: [], stl: "none", + apex_available: [ "myapex" ], } cc_library { - name: "libc", - no_libgcc: true, - nocrt: true, + name: "libfoo", + srcs: ["mylib.cpp"], system_shared_libs: [], stl: "none", stubs: { - versions: ["27", "28", "29"], + versions: ["10", "20", "30"], }, } cc_library { - name: "libm", - no_libgcc: true, - nocrt: true, + name: "libbar", + srcs: ["mylib.cpp"], system_shared_libs: [], stl: "none", - stubs: { - versions: ["27", "28", "29"], - }, + apex_available: [ "myapex" ], + } + + `) + + apexRule := ctx.ModuleForTests("myapex", "android_common_myapex_image").Rule("apexRule") + copyCmds := apexRule.Args["copy_commands"] + + // Ensure that direct non-stubs dep is always included + ensureContains(t, copyCmds, "image.apex/lib64/mylib.so") + + // Ensure that indirect stubs dep is not included + ensureNotContains(t, copyCmds, "image.apex/lib64/libfoo.so") + + // Ensure that runtime_libs dep in included + ensureContains(t, copyCmds, "image.apex/lib64/libbar.so") + + apexManifestRule := ctx.ModuleForTests("myapex", "android_common_myapex_image").Rule("apexManifestRule") + ensureListEmpty(t, names(apexManifestRule.Args["provideNativeLibs"])) + ensureListContains(t, names(apexManifestRule.Args["requireNativeLibs"]), "libfoo.so") + +} + +func TestApexDependsOnLLNDKTransitively(t *testing.T) { + testcases := []struct { + name string + minSdkVersion string + shouldLink string + shouldNotLink []string + }{ + { + name: "should link to the latest", + minSdkVersion: "current", + shouldLink: "30", + shouldNotLink: []string{"29"}, + }, + { + name: "should link to llndk#29", + minSdkVersion: "29", + shouldLink: "29", + shouldNotLink: []string{"30"}, + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + ctx, _ := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + use_vendor: true, + native_shared_libs: ["mylib"], + min_sdk_version: "`+tc.minSdkVersion+`", + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + cc_library { + name: "mylib", + srcs: ["mylib.cpp"], + vendor_available: true, + shared_libs: ["libbar"], + system_shared_libs: [], + stl: "none", + apex_available: [ "myapex" ], + } + + cc_library { + name: "libbar", + srcs: ["mylib.cpp"], + system_shared_libs: [], + stl: "none", + stubs: { versions: ["29","30"] }, + } + + llndk_library { + name: "libbar", + symbol_file: "", + } + `, func(fs map[string][]byte, config android.Config) { + setUseVendorAllowListForTest(config, []string{"myapex"}) + }, withUnbundledBuild) + + // Ensure that LLNDK dep is not included + ensureExactContents(t, ctx, "myapex", "android_common_myapex_image", []string{ + "lib64/mylib.so", + }) + + // Ensure that LLNDK dep is required + apexManifestRule := ctx.ModuleForTests("myapex", "android_common_myapex_image").Rule("apexManifestRule") + ensureListEmpty(t, names(apexManifestRule.Args["provideNativeLibs"])) + ensureListContains(t, names(apexManifestRule.Args["requireNativeLibs"]), "libbar.so") + + mylibLdFlags := ctx.ModuleForTests("mylib", "android_vendor.VER_arm64_armv8-a_shared_myapex").Rule("ld").Args["libFlags"] + ensureContains(t, mylibLdFlags, "libbar.llndk/android_vendor.VER_arm64_armv8-a_shared_"+tc.shouldLink+"/libbar.so") + for _, ver := range tc.shouldNotLink { + ensureNotContains(t, mylibLdFlags, "libbar.llndk/android_vendor.VER_arm64_armv8-a_shared_"+ver+"/libbar.so") + } + + mylibCFlags := ctx.ModuleForTests("mylib", "android_vendor.VER_arm64_armv8-a_static_myapex").Rule("cc").Args["cFlags"] + ensureContains(t, mylibCFlags, "__LIBBAR_API__="+tc.shouldLink) + }) + } +} + +func TestApexWithSystemLibsStubs(t *testing.T) { + ctx, _ := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + native_shared_libs: ["mylib", "mylib_shared", "libdl", "libm"], + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", } cc_library { - name: "libdl", - no_libgcc: true, - nocrt: true, - system_shared_libs: [], + name: "mylib", + srcs: ["mylib.cpp"], + shared_libs: ["libdl#27"], stl: "none", - stubs: { - versions: ["27", "28", "29"], - }, + apex_available: [ "myapex" ], + } + + cc_library_shared { + name: "mylib_shared", + srcs: ["mylib.cpp"], + shared_libs: ["libdl#27"], + stl: "none", + apex_available: [ "myapex" ], } cc_library { @@ -638,7 +1048,7 @@ func TestApexWithSystemLibsStubs(t *testing.T) { } `) - apexRule := ctx.ModuleForTests("myapex", "android_common_myapex").Rule("apexRule") + apexRule := ctx.ModuleForTests("myapex", "android_common_myapex_image").Rule("apexRule") copyCmds := apexRule.Args["copy_commands"] // Ensure that mylib, libm, libdl are included. @@ -649,49 +1059,507 @@ func TestApexWithSystemLibsStubs(t *testing.T) { // Ensure that libc is not included (since it has stubs and not listed in native_shared_libs) ensureNotContains(t, copyCmds, "image.apex/lib64/bionic/libc.so") - mylibLdFlags := ctx.ModuleForTests("mylib", "android_arm64_armv8-a_core_shared_myapex").Rule("ld").Args["libFlags"] - mylibCFlags := ctx.ModuleForTests("mylib", "android_arm64_armv8-a_core_static_myapex").Rule("cc").Args["cFlags"] - mylibSharedCFlags := ctx.ModuleForTests("mylib_shared", "android_arm64_armv8-a_core_shared_myapex").Rule("cc").Args["cFlags"] + mylibLdFlags := ctx.ModuleForTests("mylib", "android_arm64_armv8-a_shared_myapex").Rule("ld").Args["libFlags"] + mylibCFlags := ctx.ModuleForTests("mylib", "android_arm64_armv8-a_static_myapex").Rule("cc").Args["cFlags"] + mylibSharedCFlags := ctx.ModuleForTests("mylib_shared", "android_arm64_armv8-a_shared_myapex").Rule("cc").Args["cFlags"] // For dependency to libc // Ensure that mylib is linking with the latest version of stubs - ensureContains(t, mylibLdFlags, "libc/android_arm64_armv8-a_core_shared_29_myapex/libc.so") + ensureContains(t, mylibLdFlags, "libc/android_arm64_armv8-a_shared_29/libc.so") // ... and not linking to the non-stub (impl) variant - ensureNotContains(t, mylibLdFlags, "libc/android_arm64_armv8-a_core_shared_myapex/libc.so") + ensureNotContains(t, mylibLdFlags, "libc/android_arm64_armv8-a_shared/libc.so") // ... Cflags from stub is correctly exported to mylib ensureContains(t, mylibCFlags, "__LIBC_API__=29") ensureContains(t, mylibSharedCFlags, "__LIBC_API__=29") // For dependency to libm // Ensure that mylib is linking with the non-stub (impl) variant - ensureContains(t, mylibLdFlags, "libm/android_arm64_armv8-a_core_shared_myapex/libm.so") + ensureContains(t, mylibLdFlags, "libm/android_arm64_armv8-a_shared_myapex/libm.so") // ... and not linking to the stub variant - ensureNotContains(t, mylibLdFlags, "libm/android_arm64_armv8-a_core_shared_29_myapex/libm.so") + ensureNotContains(t, mylibLdFlags, "libm/android_arm64_armv8-a_shared_29/libm.so") // ... and is not compiling with the stub ensureNotContains(t, mylibCFlags, "__LIBM_API__=29") ensureNotContains(t, mylibSharedCFlags, "__LIBM_API__=29") // For dependency to libdl // Ensure that mylib is linking with the specified version of stubs - ensureContains(t, mylibLdFlags, "libdl/android_arm64_armv8-a_core_shared_27_myapex/libdl.so") + ensureContains(t, mylibLdFlags, "libdl/android_arm64_armv8-a_shared_27/libdl.so") // ... and not linking to the other versions of stubs - ensureNotContains(t, mylibLdFlags, "libdl/android_arm64_armv8-a_core_shared_28_myapex/libdl.so") - ensureNotContains(t, mylibLdFlags, "libdl/android_arm64_armv8-a_core_shared_29_myapex/libdl.so") + ensureNotContains(t, mylibLdFlags, "libdl/android_arm64_armv8-a_shared_28/libdl.so") + ensureNotContains(t, mylibLdFlags, "libdl/android_arm64_armv8-a_shared_29/libdl.so") // ... and not linking to the non-stub (impl) variant - ensureNotContains(t, mylibLdFlags, "libdl/android_arm64_armv8-a_core_shared_myapex/libdl.so") + ensureNotContains(t, mylibLdFlags, "libdl/android_arm64_armv8-a_shared_myapex/libdl.so") // ... Cflags from stub is correctly exported to mylib ensureContains(t, mylibCFlags, "__LIBDL_API__=27") ensureContains(t, mylibSharedCFlags, "__LIBDL_API__=27") // Ensure that libBootstrap is depending on the platform variant of bionic libs - libFlags := ctx.ModuleForTests("libBootstrap", "android_arm64_armv8-a_core_shared").Rule("ld").Args["libFlags"] - ensureContains(t, libFlags, "libc/android_arm64_armv8-a_core_shared/libc.so") - ensureContains(t, libFlags, "libm/android_arm64_armv8-a_core_shared/libm.so") - ensureContains(t, libFlags, "libdl/android_arm64_armv8-a_core_shared/libdl.so") + libFlags := ctx.ModuleForTests("libBootstrap", "android_arm64_armv8-a_shared").Rule("ld").Args["libFlags"] + ensureContains(t, libFlags, "libc/android_arm64_armv8-a_shared/libc.so") + ensureContains(t, libFlags, "libm/android_arm64_armv8-a_shared/libm.so") + ensureContains(t, libFlags, "libdl/android_arm64_armv8-a_shared/libdl.so") +} + +func TestApexUseStubsAccordingToMinSdkVersionInUnbundledBuild(t *testing.T) { + // there are three links between liba --> libz + // 1) myapex -> libx -> liba -> libz : this should be #2 link, but fallback to #1 + // 2) otherapex -> liby -> liba -> libz : this should be #3 link + // 3) (platform) -> liba -> libz : this should be non-stub link + ctx, _ := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + native_shared_libs: ["libx"], + min_sdk_version: "2", + } + + apex { + name: "otherapex", + key: "myapex.key", + native_shared_libs: ["liby"], + min_sdk_version: "3", + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + cc_library { + name: "libx", + shared_libs: ["liba"], + system_shared_libs: [], + stl: "none", + apex_available: [ "myapex" ], + } + + cc_library { + name: "liby", + shared_libs: ["liba"], + system_shared_libs: [], + stl: "none", + apex_available: [ "otherapex" ], + } + + cc_library { + name: "liba", + shared_libs: ["libz"], + system_shared_libs: [], + stl: "none", + apex_available: [ + "//apex_available:anyapex", + "//apex_available:platform", + ], + } + + cc_library { + name: "libz", + system_shared_libs: [], + stl: "none", + stubs: { + versions: ["1", "3"], + }, + } + `, withUnbundledBuild) + + expectLink := func(from, from_variant, to, to_variant string) { + ldArgs := ctx.ModuleForTests(from, "android_arm64_armv8-a_"+from_variant).Rule("ld").Args["libFlags"] + ensureContains(t, ldArgs, "android_arm64_armv8-a_"+to_variant+"/"+to+".so") + } + expectNoLink := func(from, from_variant, to, to_variant string) { + ldArgs := ctx.ModuleForTests(from, "android_arm64_armv8-a_"+from_variant).Rule("ld").Args["libFlags"] + ensureNotContains(t, ldArgs, "android_arm64_armv8-a_"+to_variant+"/"+to+".so") + } + // platform liba is linked to non-stub version + expectLink("liba", "shared", "libz", "shared") + // liba in myapex is linked to #1 + expectLink("liba", "shared_myapex", "libz", "shared_1") + expectNoLink("liba", "shared_myapex", "libz", "shared_3") + expectNoLink("liba", "shared_myapex", "libz", "shared") + // liba in otherapex is linked to #3 + expectLink("liba", "shared_otherapex", "libz", "shared_3") + expectNoLink("liba", "shared_otherapex", "libz", "shared_1") + expectNoLink("liba", "shared_otherapex", "libz", "shared") +} + +func TestApexMinSdkVersion_SupportsCodeNames(t *testing.T) { + ctx, _ := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + native_shared_libs: ["libx"], + min_sdk_version: "R", + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + cc_library { + name: "libx", + shared_libs: ["libz"], + system_shared_libs: [], + stl: "none", + apex_available: [ "myapex" ], + } + + cc_library { + name: "libz", + system_shared_libs: [], + stl: "none", + stubs: { + versions: ["29", "R"], + }, + } + `, func(fs map[string][]byte, config android.Config) { + config.TestProductVariables.Platform_version_active_codenames = []string{"R"} + }) + + expectLink := func(from, from_variant, to, to_variant string) { + ldArgs := ctx.ModuleForTests(from, "android_arm64_armv8-a_"+from_variant).Rule("ld").Args["libFlags"] + ensureContains(t, ldArgs, "android_arm64_armv8-a_"+to_variant+"/"+to+".so") + } + expectNoLink := func(from, from_variant, to, to_variant string) { + ldArgs := ctx.ModuleForTests(from, "android_arm64_armv8-a_"+from_variant).Rule("ld").Args["libFlags"] + ensureNotContains(t, ldArgs, "android_arm64_armv8-a_"+to_variant+"/"+to+".so") + } + // 9000 is quite a magic number. + // Finalized SDK codenames are mapped as P(28), Q(29), ... + // And, codenames which are not finalized yet(active_codenames + future_codenames) are numbered from 9000, 9001, ... + // to distinguish them from finalized and future_api(10000) + // In this test, "R" is assumed not finalized yet( listed in Platform_version_active_codenames) and translated into 9000 + // (refer android/api_levels.go) + expectLink("libx", "shared_myapex", "libz", "shared_9000") + expectNoLink("libx", "shared_myapex", "libz", "shared_29") + expectNoLink("libx", "shared_myapex", "libz", "shared") +} + +func TestApexMinSdkVersionDefaultsToLatest(t *testing.T) { + ctx, _ := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + native_shared_libs: ["libx"], + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + cc_library { + name: "libx", + shared_libs: ["libz"], + system_shared_libs: [], + stl: "none", + apex_available: [ "myapex" ], + } + + cc_library { + name: "libz", + system_shared_libs: [], + stl: "none", + stubs: { + versions: ["1", "2"], + }, + } + `) + + expectLink := func(from, from_variant, to, to_variant string) { + ldArgs := ctx.ModuleForTests(from, "android_arm64_armv8-a_"+from_variant).Rule("ld").Args["libFlags"] + ensureContains(t, ldArgs, "android_arm64_armv8-a_"+to_variant+"/"+to+".so") + } + expectNoLink := func(from, from_variant, to, to_variant string) { + ldArgs := ctx.ModuleForTests(from, "android_arm64_armv8-a_"+from_variant).Rule("ld").Args["libFlags"] + ensureNotContains(t, ldArgs, "android_arm64_armv8-a_"+to_variant+"/"+to+".so") + } + expectLink("libx", "shared_myapex", "libz", "shared_2") + expectNoLink("libx", "shared_myapex", "libz", "shared_1") + expectNoLink("libx", "shared_myapex", "libz", "shared") +} + +func TestPlatformUsesLatestStubsFromApexes(t *testing.T) { + ctx, _ := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + native_shared_libs: ["libx"], + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + cc_library { + name: "libx", + system_shared_libs: [], + stl: "none", + apex_available: [ "myapex" ], + stubs: { + versions: ["1", "2"], + }, + } + + cc_library { + name: "libz", + shared_libs: ["libx"], + system_shared_libs: [], + stl: "none", + } + `) + + expectLink := func(from, from_variant, to, to_variant string) { + ldArgs := ctx.ModuleForTests(from, "android_arm64_armv8-a_"+from_variant).Rule("ld").Args["libFlags"] + ensureContains(t, ldArgs, "android_arm64_armv8-a_"+to_variant+"/"+to+".so") + } + expectNoLink := func(from, from_variant, to, to_variant string) { + ldArgs := ctx.ModuleForTests(from, "android_arm64_armv8-a_"+from_variant).Rule("ld").Args["libFlags"] + ensureNotContains(t, ldArgs, "android_arm64_armv8-a_"+to_variant+"/"+to+".so") + } + expectLink("libz", "shared", "libx", "shared_2") + expectNoLink("libz", "shared", "libz", "shared_1") + expectNoLink("libz", "shared", "libz", "shared") +} + +func TestQApexesUseLatestStubsInBundledBuildsAndHWASAN(t *testing.T) { + ctx, _ := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + native_shared_libs: ["libx"], + min_sdk_version: "29", + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + cc_library { + name: "libx", + shared_libs: ["libbar"], + apex_available: [ "myapex" ], + } + + cc_library { + name: "libbar", + stubs: { + versions: ["29", "30"], + }, + } + `, func(fs map[string][]byte, config android.Config) { + config.TestProductVariables.SanitizeDevice = []string{"hwaddress"} + }) + expectLink := func(from, from_variant, to, to_variant string) { + ld := ctx.ModuleForTests(from, "android_arm64_armv8-a_"+from_variant).Rule("ld") + libFlags := ld.Args["libFlags"] + ensureContains(t, libFlags, "android_arm64_armv8-a_"+to_variant+"/"+to+".so") + } + expectLink("libx", "shared_hwasan_myapex", "libbar", "shared_30") +} + +func TestQTargetApexUsesStaticUnwinder(t *testing.T) { + ctx, _ := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + native_shared_libs: ["libx"], + min_sdk_version: "29", + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + cc_library { + name: "libx", + apex_available: [ "myapex" ], + } + `) + + // ensure apex variant of c++ is linked with static unwinder + cm := ctx.ModuleForTests("libc++", "android_arm64_armv8-a_shared_myapex").Module().(*cc.Module) + ensureListContains(t, cm.Properties.AndroidMkStaticLibs, "libgcc_stripped") + // note that platform variant is not. + cm = ctx.ModuleForTests("libc++", "android_arm64_armv8-a_shared").Module().(*cc.Module) + ensureListNotContains(t, cm.Properties.AndroidMkStaticLibs, "libgcc_stripped") +} + +func TestInvalidMinSdkVersion(t *testing.T) { + testApexError(t, `"libz" .*: not found a version\(<=29\)`, ` + apex { + name: "myapex", + key: "myapex.key", + native_shared_libs: ["libx"], + min_sdk_version: "29", + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + cc_library { + name: "libx", + shared_libs: ["libz"], + system_shared_libs: [], + stl: "none", + apex_available: [ "myapex" ], + } + + cc_library { + name: "libz", + system_shared_libs: [], + stl: "none", + stubs: { + versions: ["30"], + }, + } + `) + + testApexError(t, `"myapex" .*: min_sdk_version: SDK version should be .*`, ` + apex { + name: "myapex", + key: "myapex.key", + min_sdk_version: "abc", + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + `) +} + +func TestJavaStableSdkVersion(t *testing.T) { + testCases := []struct { + name string + expectedError string + bp string + }{ + { + name: "Non-updatable apex with non-stable dep", + bp: ` + apex { + name: "myapex", + java_libs: ["myjar"], + key: "myapex.key", + } + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + java_library { + name: "myjar", + srcs: ["foo/bar/MyClass.java"], + sdk_version: "core_platform", + apex_available: ["myapex"], + } + `, + }, + { + name: "Updatable apex with stable dep", + bp: ` + apex { + name: "myapex", + java_libs: ["myjar"], + key: "myapex.key", + updatable: true, + min_sdk_version: "29", + } + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + java_library { + name: "myjar", + srcs: ["foo/bar/MyClass.java"], + sdk_version: "current", + apex_available: ["myapex"], + } + `, + }, + { + name: "Updatable apex with non-stable dep", + expectedError: "cannot depend on \"myjar\"", + bp: ` + apex { + name: "myapex", + java_libs: ["myjar"], + key: "myapex.key", + updatable: true, + } + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + java_library { + name: "myjar", + srcs: ["foo/bar/MyClass.java"], + sdk_version: "core_platform", + apex_available: ["myapex"], + } + `, + }, + { + name: "Updatable apex with non-stable transitive dep", + expectedError: "compiles against Android API, but dependency \"transitive-jar\" is compiling against non-public Android API.", + bp: ` + apex { + name: "myapex", + java_libs: ["myjar"], + key: "myapex.key", + updatable: true, + } + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + java_library { + name: "myjar", + srcs: ["foo/bar/MyClass.java"], + sdk_version: "current", + apex_available: ["myapex"], + static_libs: ["transitive-jar"], + } + java_library { + name: "transitive-jar", + srcs: ["foo/bar/MyClass.java"], + sdk_version: "core_platform", + apex_available: ["myapex"], + } + `, + }, + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + if test.expectedError == "" { + testApex(t, test.bp) + } else { + testApexError(t, test.expectedError, test.bp) + } + }) + } } func TestFilesInSubDir(t *testing.T) { - ctx := testApex(t, ` + ctx, _ := testApex(t, ` apex { name: "myapex", key: "myapex.key", @@ -719,6 +1587,7 @@ func TestFilesInSubDir(t *testing.T) { relative_install_path: "foo/bar", system_shared_libs: [], stl: "none", + apex_available: [ "myapex" ], } cc_binary { @@ -728,10 +1597,11 @@ func TestFilesInSubDir(t *testing.T) { system_shared_libs: [], static_executable: true, stl: "none", + apex_available: [ "myapex" ], } `) - generateFsRule := ctx.ModuleForTests("myapex", "android_common_myapex").Rule("generateFsConfig") + generateFsRule := ctx.ModuleForTests("myapex", "android_common_myapex_image").Rule("generateFsConfig") dirs := strings.Split(generateFsRule.Args["exec_paths"], " ") // Ensure that the subdirectories are all listed @@ -751,7 +1621,7 @@ func TestFilesInSubDir(t *testing.T) { } func TestUseVendor(t *testing.T) { - ctx := testApex(t, ` + ctx, _ := testApex(t, ` apex { name: "myapex", key: "myapex.key", @@ -772,6 +1642,7 @@ func TestUseVendor(t *testing.T) { system_shared_libs: [], vendor_available: true, stl: "none", + apex_available: [ "myapex" ], } cc_library { @@ -780,11 +1651,14 @@ func TestUseVendor(t *testing.T) { system_shared_libs: [], vendor_available: true, stl: "none", + apex_available: [ "myapex" ], } - `) + `, func(fs map[string][]byte, config android.Config) { + setUseVendorAllowListForTest(config, []string{"myapex"}) + }) inputsList := []string{} - for _, i := range ctx.ModuleForTests("myapex", "android_common_myapex").Module().BuildParamsForTests() { + for _, i := range ctx.ModuleForTests("myapex", "android_common_myapex_image").Module().BuildParamsForTests() { for _, implicit := range i.Implicits { inputsList = append(inputsList, implicit.String()) } @@ -792,16 +1666,72 @@ func TestUseVendor(t *testing.T) { inputsString := strings.Join(inputsList, " ") // ensure that the apex includes vendor variants of the direct and indirect deps - ensureContains(t, inputsString, "android_arm64_armv8-a_vendor_shared_myapex/mylib.so") - ensureContains(t, inputsString, "android_arm64_armv8-a_vendor_shared_myapex/mylib2.so") + ensureContains(t, inputsString, "android_vendor.VER_arm64_armv8-a_shared_myapex/mylib.so") + ensureContains(t, inputsString, "android_vendor.VER_arm64_armv8-a_shared_myapex/mylib2.so") // ensure that the apex does not include core variants - ensureNotContains(t, inputsString, "android_arm64_armv8-a_core_shared_myapex/mylib.so") - ensureNotContains(t, inputsString, "android_arm64_armv8-a_core_shared_myapex/mylib2.so") + ensureNotContains(t, inputsString, "android_arm64_armv8-a_shared_myapex/mylib.so") + ensureNotContains(t, inputsString, "android_arm64_armv8-a_shared_myapex/mylib2.so") +} + +func TestUseVendorRestriction(t *testing.T) { + testApexError(t, `module "myapex" .*: use_vendor: not allowed`, ` + apex { + name: "myapex", + key: "myapex.key", + use_vendor: true, + } + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + `, func(fs map[string][]byte, config android.Config) { + setUseVendorAllowListForTest(config, []string{""}) + }) + // no error with allow list + testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + use_vendor: true, + } + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + `, func(fs map[string][]byte, config android.Config) { + setUseVendorAllowListForTest(config, []string{"myapex"}) + }) +} + +func TestUseVendorFailsIfNotVendorAvailable(t *testing.T) { + testApexError(t, `dependency "mylib" of "myapex" missing variant:\n.*image:vendor`, ` + apex { + name: "myapex", + key: "myapex.key", + native_shared_libs: ["mylib"], + use_vendor: true, + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + cc_library { + name: "mylib", + srcs: ["mylib.cpp"], + system_shared_libs: [], + stl: "none", + } + `) } func TestStaticLinking(t *testing.T) { - ctx := testApex(t, ` + ctx, _ := testApex(t, ` apex { name: "myapex", key: "myapex.key", @@ -822,6 +1752,10 @@ func TestStaticLinking(t *testing.T) { stubs: { versions: ["1", "2", "3"], }, + apex_available: [ + "//apex_available:platform", + "myapex", + ], } cc_binary { @@ -834,19 +1768,20 @@ func TestStaticLinking(t *testing.T) { } `) - ldFlags := ctx.ModuleForTests("not_in_apex", "android_arm64_armv8-a_core").Rule("ld").Args["libFlags"] + ldFlags := ctx.ModuleForTests("not_in_apex", "android_arm64_armv8-a").Rule("ld").Args["libFlags"] // Ensure that not_in_apex is linking with the static variant of mylib - ensureContains(t, ldFlags, "mylib/android_arm64_armv8-a_core_static/mylib.a") + ensureContains(t, ldFlags, "mylib/android_arm64_armv8-a_static/mylib.a") } func TestKeys(t *testing.T) { - ctx := testApex(t, ` + ctx, _ := testApex(t, ` apex { name: "myapex_keytest", key: "myapex.key", certificate: ":myapex.certificate", native_shared_libs: ["mylib"], + file_contexts: ":myapex-file_contexts", } cc_library { @@ -854,6 +1789,7 @@ func TestKeys(t *testing.T) { srcs: ["mylib.cpp"], system_shared_libs: [], stl: "none", + apex_available: [ "myapex_keytest" ], } apex_key { @@ -887,25 +1823,154 @@ func TestKeys(t *testing.T) { } // check the APK certs. It should be overridden to myapex.certificate.override - certs := ctx.ModuleForTests("myapex_keytest", "android_common_myapex_keytest").Rule("signapk").Args["certificates"] + certs := ctx.ModuleForTests("myapex_keytest", "android_common_myapex_keytest_image").Rule("signapk").Args["certificates"] if certs != "testkey.override.x509.pem testkey.override.pk8" { t.Errorf("cert and private key %q are not %q", certs, "testkey.override.509.pem testkey.override.pk8") } } +func TestCertificate(t *testing.T) { + t.Run("if unspecified, it defaults to DefaultAppCertificate", func(t *testing.T) { + ctx, _ := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + } + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + }`) + rule := ctx.ModuleForTests("myapex", "android_common_myapex_image").Rule("signapk") + expected := "vendor/foo/devkeys/test.x509.pem vendor/foo/devkeys/test.pk8" + if actual := rule.Args["certificates"]; actual != expected { + t.Errorf("certificates should be %q, not %q", expected, actual) + } + }) + t.Run("override when unspecified", func(t *testing.T) { + ctx, _ := testApex(t, ` + apex { + name: "myapex_keytest", + key: "myapex.key", + file_contexts: ":myapex-file_contexts", + } + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + android_app_certificate { + name: "myapex.certificate.override", + certificate: "testkey.override", + }`) + rule := ctx.ModuleForTests("myapex_keytest", "android_common_myapex_keytest_image").Rule("signapk") + expected := "testkey.override.x509.pem testkey.override.pk8" + if actual := rule.Args["certificates"]; actual != expected { + t.Errorf("certificates should be %q, not %q", expected, actual) + } + }) + t.Run("if specified as :module, it respects the prop", func(t *testing.T) { + ctx, _ := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + certificate: ":myapex.certificate", + } + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + android_app_certificate { + name: "myapex.certificate", + certificate: "testkey", + }`) + rule := ctx.ModuleForTests("myapex", "android_common_myapex_image").Rule("signapk") + expected := "testkey.x509.pem testkey.pk8" + if actual := rule.Args["certificates"]; actual != expected { + t.Errorf("certificates should be %q, not %q", expected, actual) + } + }) + t.Run("override when specifiec as <:module>", func(t *testing.T) { + ctx, _ := testApex(t, ` + apex { + name: "myapex_keytest", + key: "myapex.key", + file_contexts: ":myapex-file_contexts", + certificate: ":myapex.certificate", + } + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + android_app_certificate { + name: "myapex.certificate.override", + certificate: "testkey.override", + }`) + rule := ctx.ModuleForTests("myapex_keytest", "android_common_myapex_keytest_image").Rule("signapk") + expected := "testkey.override.x509.pem testkey.override.pk8" + if actual := rule.Args["certificates"]; actual != expected { + t.Errorf("certificates should be %q, not %q", expected, actual) + } + }) + t.Run("if specified as name, finds it from DefaultDevKeyDir", func(t *testing.T) { + ctx, _ := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + certificate: "testkey", + } + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + }`) + rule := ctx.ModuleForTests("myapex", "android_common_myapex_image").Rule("signapk") + expected := "vendor/foo/devkeys/testkey.x509.pem vendor/foo/devkeys/testkey.pk8" + if actual := rule.Args["certificates"]; actual != expected { + t.Errorf("certificates should be %q, not %q", expected, actual) + } + }) + t.Run("override when specified as <name>", func(t *testing.T) { + ctx, _ := testApex(t, ` + apex { + name: "myapex_keytest", + key: "myapex.key", + file_contexts: ":myapex-file_contexts", + certificate: "testkey", + } + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + android_app_certificate { + name: "myapex.certificate.override", + certificate: "testkey.override", + }`) + rule := ctx.ModuleForTests("myapex_keytest", "android_common_myapex_keytest_image").Rule("signapk") + expected := "testkey.override.x509.pem testkey.override.pk8" + if actual := rule.Args["certificates"]; actual != expected { + t.Errorf("certificates should be %q, not %q", expected, actual) + } + }) +} + func TestMacro(t *testing.T) { - ctx := testApex(t, ` + ctx, _ := testApex(t, ` apex { name: "myapex", key: "myapex.key", - native_shared_libs: ["mylib"], + native_shared_libs: ["mylib", "mylib2"], } apex { name: "otherapex", key: "myapex.key", - native_shared_libs: ["mylib"], + native_shared_libs: ["mylib", "mylib2"], + min_sdk_version: "29", } apex_key { @@ -919,27 +1984,69 @@ func TestMacro(t *testing.T) { srcs: ["mylib.cpp"], system_shared_libs: [], stl: "none", + apex_available: [ + "myapex", + "otherapex", + ], + recovery_available: true, + } + cc_library { + name: "mylib2", + srcs: ["mylib.cpp"], + system_shared_libs: [], + stl: "none", + apex_available: [ + "myapex", + "otherapex", + ], + use_apex_name_macro: true, } `) - // non-APEX variant does not have __ANDROID__APEX__ defined - mylibCFlags := ctx.ModuleForTests("mylib", "android_arm64_armv8-a_core_static").Rule("cc").Args["cFlags"] - ensureNotContains(t, mylibCFlags, "-D__ANDROID_APEX__=myapex") - ensureNotContains(t, mylibCFlags, "-D__ANDROID_APEX__=otherapex") - - // APEX variant has __ANDROID_APEX__=<apexname> defined - mylibCFlags = ctx.ModuleForTests("mylib", "android_arm64_armv8-a_core_static_myapex").Rule("cc").Args["cFlags"] - ensureContains(t, mylibCFlags, "-D__ANDROID_APEX__=myapex") - ensureNotContains(t, mylibCFlags, "-D__ANDROID_APEX__=otherapex") - - // APEX variant has __ANDROID_APEX__=<apexname> defined - mylibCFlags = ctx.ModuleForTests("mylib", "android_arm64_armv8-a_core_static_otherapex").Rule("cc").Args["cFlags"] - ensureNotContains(t, mylibCFlags, "-D__ANDROID_APEX__=myapex") - ensureContains(t, mylibCFlags, "-D__ANDROID_APEX__=otherapex") + // non-APEX variant does not have __ANDROID_APEX__ defined + mylibCFlags := ctx.ModuleForTests("mylib", "android_arm64_armv8-a_static").Rule("cc").Args["cFlags"] + ensureNotContains(t, mylibCFlags, "-D__ANDROID_APEX__") + ensureNotContains(t, mylibCFlags, "-D__ANDROID_SDK_VERSION__") + + // APEX variant has __ANDROID_APEX__ and __ANDROID_APEX_SDK__ defined + mylibCFlags = ctx.ModuleForTests("mylib", "android_arm64_armv8-a_static_myapex").Rule("cc").Args["cFlags"] + ensureContains(t, mylibCFlags, "-D__ANDROID_APEX__") + ensureContains(t, mylibCFlags, "-D__ANDROID_SDK_VERSION__=10000") + ensureNotContains(t, mylibCFlags, "-D__ANDROID_APEX_MYAPEX__") + + // APEX variant has __ANDROID_APEX__ and __ANDROID_APEX_SDK__ defined + mylibCFlags = ctx.ModuleForTests("mylib", "android_arm64_armv8-a_static_otherapex").Rule("cc").Args["cFlags"] + ensureContains(t, mylibCFlags, "-D__ANDROID_APEX__") + ensureContains(t, mylibCFlags, "-D__ANDROID_SDK_VERSION__=29") + ensureNotContains(t, mylibCFlags, "-D__ANDROID_APEX_OTHERAPEX__") + + // When cc_library sets use_apex_name_macro: true + // apex variants define additional macro to distinguish which apex variant it is built for + + // non-APEX variant does not have __ANDROID_APEX__ defined + mylibCFlags = ctx.ModuleForTests("mylib2", "android_arm64_armv8-a_static").Rule("cc").Args["cFlags"] + ensureNotContains(t, mylibCFlags, "-D__ANDROID_APEX__") + + // APEX variant has __ANDROID_APEX__ defined + mylibCFlags = ctx.ModuleForTests("mylib2", "android_arm64_armv8-a_static_myapex").Rule("cc").Args["cFlags"] + ensureContains(t, mylibCFlags, "-D__ANDROID_APEX__") + ensureContains(t, mylibCFlags, "-D__ANDROID_APEX_MYAPEX__") + ensureNotContains(t, mylibCFlags, "-D__ANDROID_APEX_OTHERAPEX__") + + // APEX variant has __ANDROID_APEX__ defined + mylibCFlags = ctx.ModuleForTests("mylib2", "android_arm64_armv8-a_static_otherapex").Rule("cc").Args["cFlags"] + ensureContains(t, mylibCFlags, "-D__ANDROID_APEX__") + ensureNotContains(t, mylibCFlags, "-D__ANDROID_APEX_MYAPEX__") + ensureContains(t, mylibCFlags, "-D__ANDROID_APEX_OTHERAPEX__") + + // recovery variant does not set __ANDROID_SDK_VERSION__ + mylibCFlags = ctx.ModuleForTests("mylib", "android_recovery_arm64_armv8-a_static").Rule("cc").Args["cFlags"] + ensureNotContains(t, mylibCFlags, "-D__ANDROID_APEX__") + ensureNotContains(t, mylibCFlags, "-D__ANDROID_SDK_VERSION__") } func TestHeaderLibsDependency(t *testing.T) { - ctx := testApex(t, ` + ctx, _ := testApex(t, ` apex { name: "myapex", key: "myapex.key", @@ -957,6 +2064,7 @@ func TestHeaderLibsDependency(t *testing.T) { export_include_dirs: ["my_include"], system_shared_libs: [], stl: "none", + apex_available: [ "myapex" ], } cc_library { @@ -969,6 +2077,7 @@ func TestHeaderLibsDependency(t *testing.T) { stubs: { versions: ["1", "2", "3"], }, + apex_available: [ "myapex" ], } cc_library { @@ -980,14 +2089,674 @@ func TestHeaderLibsDependency(t *testing.T) { } `) - cFlags := ctx.ModuleForTests("otherlib", "android_arm64_armv8-a_core_static").Rule("cc").Args["cFlags"] + cFlags := ctx.ModuleForTests("otherlib", "android_arm64_armv8-a_static").Rule("cc").Args["cFlags"] // Ensure that the include path of the header lib is exported to 'otherlib' ensureContains(t, cFlags, "-Imy_include") } +type fileInApex struct { + path string // path in apex + src string // src path + isLink bool +} + +func getFiles(t *testing.T, ctx *android.TestContext, moduleName, variant string) []fileInApex { + t.Helper() + apexRule := ctx.ModuleForTests(moduleName, variant).Rule("apexRule") + copyCmds := apexRule.Args["copy_commands"] + imageApexDir := "/image.apex/" + var ret []fileInApex + for _, cmd := range strings.Split(copyCmds, "&&") { + cmd = strings.TrimSpace(cmd) + if cmd == "" { + continue + } + terms := strings.Split(cmd, " ") + var dst, src string + var isLink bool + switch terms[0] { + case "mkdir": + case "cp": + if len(terms) != 3 && len(terms) != 4 { + t.Fatal("copyCmds contains invalid cp command", cmd) + } + dst = terms[len(terms)-1] + src = terms[len(terms)-2] + isLink = false + case "ln": + if len(terms) != 3 && len(terms) != 4 { + // ln LINK TARGET or ln -s LINK TARGET + t.Fatal("copyCmds contains invalid ln command", cmd) + } + dst = terms[len(terms)-1] + src = terms[len(terms)-2] + isLink = true + default: + t.Fatalf("copyCmds should contain mkdir/cp commands only: %q", cmd) + } + if dst != "" { + index := strings.Index(dst, imageApexDir) + if index == -1 { + t.Fatal("copyCmds should copy a file to image.apex/", cmd) + } + dstFile := dst[index+len(imageApexDir):] + ret = append(ret, fileInApex{path: dstFile, src: src, isLink: isLink}) + } + } + return ret +} + +func ensureExactContents(t *testing.T, ctx *android.TestContext, moduleName, variant string, files []string) { + t.Helper() + var failed bool + var surplus []string + filesMatched := make(map[string]bool) + for _, file := range getFiles(t, ctx, moduleName, variant) { + mactchFound := false + for _, expected := range files { + if matched, _ := path.Match(expected, file.path); matched { + filesMatched[expected] = true + mactchFound = true + break + } + } + if !mactchFound { + surplus = append(surplus, file.path) + } + } + + if len(surplus) > 0 { + sort.Strings(surplus) + t.Log("surplus files", surplus) + failed = true + } + + if len(files) > len(filesMatched) { + var missing []string + for _, expected := range files { + if !filesMatched[expected] { + missing = append(missing, expected) + } + } + sort.Strings(missing) + t.Log("missing files", missing) + failed = true + } + if failed { + t.Fail() + } +} + +func TestVndkApexCurrent(t *testing.T) { + ctx, _ := testApex(t, ` + apex_vndk { + name: "myapex", + key: "myapex.key", + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + cc_library { + name: "libvndk", + srcs: ["mylib.cpp"], + vendor_available: true, + vndk: { + enabled: true, + }, + system_shared_libs: [], + stl: "none", + apex_available: [ "myapex" ], + } + + cc_library { + name: "libvndksp", + srcs: ["mylib.cpp"], + vendor_available: true, + vndk: { + enabled: true, + support_system_process: true, + }, + system_shared_libs: [], + stl: "none", + apex_available: [ "myapex" ], + } + `+vndkLibrariesTxtFiles("current")) + + ensureExactContents(t, ctx, "myapex", "android_common_image", []string{ + "lib/libvndk.so", + "lib/libvndksp.so", + "lib/libc++.so", + "lib64/libvndk.so", + "lib64/libvndksp.so", + "lib64/libc++.so", + "etc/llndk.libraries.VER.txt", + "etc/vndkcore.libraries.VER.txt", + "etc/vndksp.libraries.VER.txt", + "etc/vndkprivate.libraries.VER.txt", + }) +} + +func TestVndkApexWithPrebuilt(t *testing.T) { + ctx, _ := testApex(t, ` + apex_vndk { + name: "myapex", + key: "myapex.key", + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + cc_prebuilt_library_shared { + name: "libvndk", + srcs: ["libvndk.so"], + vendor_available: true, + vndk: { + enabled: true, + }, + system_shared_libs: [], + stl: "none", + apex_available: [ "myapex" ], + } + + cc_prebuilt_library_shared { + name: "libvndk.arm", + srcs: ["libvndk.arm.so"], + vendor_available: true, + vndk: { + enabled: true, + }, + enabled: false, + arch: { + arm: { + enabled: true, + }, + }, + system_shared_libs: [], + stl: "none", + apex_available: [ "myapex" ], + } + `+vndkLibrariesTxtFiles("current"), + withFiles(map[string][]byte{ + "libvndk.so": nil, + "libvndk.arm.so": nil, + })) + + ensureExactContents(t, ctx, "myapex", "android_common_image", []string{ + "lib/libvndk.so", + "lib/libvndk.arm.so", + "lib64/libvndk.so", + "lib/libc++.so", + "lib64/libc++.so", + "etc/*", + }) +} + +func vndkLibrariesTxtFiles(vers ...string) (result string) { + for _, v := range vers { + if v == "current" { + for _, txt := range []string{"llndk", "vndkcore", "vndksp", "vndkprivate"} { + result += ` + vndk_libraries_txt { + name: "` + txt + `.libraries.txt", + } + ` + } + } else { + for _, txt := range []string{"llndk", "vndkcore", "vndksp", "vndkprivate"} { + result += ` + prebuilt_etc { + name: "` + txt + `.libraries.` + v + `.txt", + src: "dummy.txt", + } + ` + } + } + } + return +} + +func TestVndkApexVersion(t *testing.T) { + ctx, _ := testApex(t, ` + apex_vndk { + name: "myapex_v27", + key: "myapex.key", + file_contexts: ":myapex-file_contexts", + vndk_version: "27", + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + vndk_prebuilt_shared { + name: "libvndk27", + version: "27", + vendor_available: true, + vndk: { + enabled: true, + }, + target_arch: "arm64", + arch: { + arm: { + srcs: ["libvndk27_arm.so"], + }, + arm64: { + srcs: ["libvndk27_arm64.so"], + }, + }, + apex_available: [ "myapex_v27" ], + } + + vndk_prebuilt_shared { + name: "libvndk27", + version: "27", + vendor_available: true, + vndk: { + enabled: true, + }, + target_arch: "x86_64", + arch: { + x86: { + srcs: ["libvndk27_x86.so"], + }, + x86_64: { + srcs: ["libvndk27_x86_64.so"], + }, + }, + } + `+vndkLibrariesTxtFiles("27"), + withFiles(map[string][]byte{ + "libvndk27_arm.so": nil, + "libvndk27_arm64.so": nil, + "libvndk27_x86.so": nil, + "libvndk27_x86_64.so": nil, + })) + + ensureExactContents(t, ctx, "myapex_v27", "android_common_image", []string{ + "lib/libvndk27_arm.so", + "lib64/libvndk27_arm64.so", + "etc/*", + }) +} + +func TestVndkApexErrorWithDuplicateVersion(t *testing.T) { + testApexError(t, `module "myapex_v27.*" .*: vndk_version: 27 is already defined in "myapex_v27.*"`, ` + apex_vndk { + name: "myapex_v27", + key: "myapex.key", + file_contexts: ":myapex-file_contexts", + vndk_version: "27", + } + apex_vndk { + name: "myapex_v27_other", + key: "myapex.key", + file_contexts: ":myapex-file_contexts", + vndk_version: "27", + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + cc_library { + name: "libvndk", + srcs: ["mylib.cpp"], + vendor_available: true, + vndk: { + enabled: true, + }, + system_shared_libs: [], + stl: "none", + } + + vndk_prebuilt_shared { + name: "libvndk", + version: "27", + vendor_available: true, + vndk: { + enabled: true, + }, + srcs: ["libvndk.so"], + } + `, withFiles(map[string][]byte{ + "libvndk.so": nil, + })) +} + +func TestVndkApexNameRule(t *testing.T) { + ctx, _ := testApex(t, ` + apex_vndk { + name: "myapex", + key: "myapex.key", + file_contexts: ":myapex-file_contexts", + } + apex_vndk { + name: "myapex_v28", + key: "myapex.key", + file_contexts: ":myapex-file_contexts", + vndk_version: "28", + } + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + }`+vndkLibrariesTxtFiles("28", "current")) + + assertApexName := func(expected, moduleName string) { + bundle := ctx.ModuleForTests(moduleName, "android_common_image").Module().(*apexBundle) + actual := proptools.String(bundle.properties.Apex_name) + if !reflect.DeepEqual(actual, expected) { + t.Errorf("Got '%v', expected '%v'", actual, expected) + } + } + + assertApexName("com.android.vndk.vVER", "myapex") + assertApexName("com.android.vndk.v28", "myapex_v28") +} + +func TestVndkApexSkipsNativeBridgeSupportedModules(t *testing.T) { + ctx, _ := testApex(t, ` + apex_vndk { + name: "myapex", + key: "myapex.key", + file_contexts: ":myapex-file_contexts", + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + cc_library { + name: "libvndk", + srcs: ["mylib.cpp"], + vendor_available: true, + native_bridge_supported: true, + host_supported: true, + vndk: { + enabled: true, + }, + system_shared_libs: [], + stl: "none", + apex_available: [ "myapex" ], + } + `+vndkLibrariesTxtFiles("current"), + withTargets(map[android.OsType][]android.Target{ + android.Android: []android.Target{ + {Os: android.Android, Arch: android.Arch{ArchType: android.Arm64, ArchVariant: "armv8-a", Abi: []string{"arm64-v8a"}}, NativeBridge: android.NativeBridgeDisabled, NativeBridgeHostArchName: "", NativeBridgeRelativePath: ""}, + {Os: android.Android, Arch: android.Arch{ArchType: android.Arm, ArchVariant: "armv7-a-neon", Abi: []string{"armeabi-v7a"}}, NativeBridge: android.NativeBridgeDisabled, NativeBridgeHostArchName: "", NativeBridgeRelativePath: ""}, + {Os: android.Android, Arch: android.Arch{ArchType: android.X86_64, ArchVariant: "silvermont", Abi: []string{"arm64-v8a"}}, NativeBridge: android.NativeBridgeEnabled, NativeBridgeHostArchName: "arm64", NativeBridgeRelativePath: "x86_64"}, + {Os: android.Android, Arch: android.Arch{ArchType: android.X86, ArchVariant: "silvermont", Abi: []string{"armeabi-v7a"}}, NativeBridge: android.NativeBridgeEnabled, NativeBridgeHostArchName: "arm", NativeBridgeRelativePath: "x86"}, + }, + })) + + ensureExactContents(t, ctx, "myapex", "android_common_image", []string{ + "lib/libvndk.so", + "lib64/libvndk.so", + "lib/libc++.so", + "lib64/libc++.so", + "etc/*", + }) +} + +func TestVndkApexDoesntSupportNativeBridgeSupported(t *testing.T) { + testApexError(t, `module "myapex" .*: native_bridge_supported: .* doesn't support native bridge binary`, ` + apex_vndk { + name: "myapex", + key: "myapex.key", + file_contexts: ":myapex-file_contexts", + native_bridge_supported: true, + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + cc_library { + name: "libvndk", + srcs: ["mylib.cpp"], + vendor_available: true, + native_bridge_supported: true, + host_supported: true, + vndk: { + enabled: true, + }, + system_shared_libs: [], + stl: "none", + } + `) +} + +func TestVndkApexWithBinder32(t *testing.T) { + ctx, _ := testApex(t, ` + apex_vndk { + name: "myapex_v27", + key: "myapex.key", + file_contexts: ":myapex-file_contexts", + vndk_version: "27", + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + vndk_prebuilt_shared { + name: "libvndk27", + version: "27", + target_arch: "arm", + vendor_available: true, + vndk: { + enabled: true, + }, + arch: { + arm: { + srcs: ["libvndk27.so"], + } + }, + } + + vndk_prebuilt_shared { + name: "libvndk27", + version: "27", + target_arch: "arm", + binder32bit: true, + vendor_available: true, + vndk: { + enabled: true, + }, + arch: { + arm: { + srcs: ["libvndk27binder32.so"], + } + }, + apex_available: [ "myapex_v27" ], + } + `+vndkLibrariesTxtFiles("27"), + withFiles(map[string][]byte{ + "libvndk27.so": nil, + "libvndk27binder32.so": nil, + }), + withBinder32bit, + withTargets(map[android.OsType][]android.Target{ + android.Android: []android.Target{ + {Os: android.Android, Arch: android.Arch{ArchType: android.Arm, ArchVariant: "armv7-a-neon", Abi: []string{"armeabi-v7a"}}, NativeBridge: android.NativeBridgeDisabled, NativeBridgeHostArchName: "", NativeBridgeRelativePath: ""}, + }, + }), + ) + + ensureExactContents(t, ctx, "myapex_v27", "android_common_image", []string{ + "lib/libvndk27binder32.so", + "etc/*", + }) +} + +func TestDependenciesInApexManifest(t *testing.T) { + ctx, _ := testApex(t, ` + apex { + name: "myapex_nodep", + key: "myapex.key", + native_shared_libs: ["lib_nodep"], + compile_multilib: "both", + file_contexts: ":myapex-file_contexts", + } + + apex { + name: "myapex_dep", + key: "myapex.key", + native_shared_libs: ["lib_dep"], + compile_multilib: "both", + file_contexts: ":myapex-file_contexts", + } + + apex { + name: "myapex_provider", + key: "myapex.key", + native_shared_libs: ["libfoo"], + compile_multilib: "both", + file_contexts: ":myapex-file_contexts", + } + + apex { + name: "myapex_selfcontained", + key: "myapex.key", + native_shared_libs: ["lib_dep", "libfoo"], + compile_multilib: "both", + file_contexts: ":myapex-file_contexts", + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + cc_library { + name: "lib_nodep", + srcs: ["mylib.cpp"], + system_shared_libs: [], + stl: "none", + apex_available: [ "myapex_nodep" ], + } + + cc_library { + name: "lib_dep", + srcs: ["mylib.cpp"], + shared_libs: ["libfoo"], + system_shared_libs: [], + stl: "none", + apex_available: [ + "myapex_dep", + "myapex_provider", + "myapex_selfcontained", + ], + } + + cc_library { + name: "libfoo", + srcs: ["mytest.cpp"], + stubs: { + versions: ["1"], + }, + system_shared_libs: [], + stl: "none", + apex_available: [ + "myapex_provider", + "myapex_selfcontained", + ], + } + `) + + var apexManifestRule android.TestingBuildParams + var provideNativeLibs, requireNativeLibs []string + + apexManifestRule = ctx.ModuleForTests("myapex_nodep", "android_common_myapex_nodep_image").Rule("apexManifestRule") + provideNativeLibs = names(apexManifestRule.Args["provideNativeLibs"]) + requireNativeLibs = names(apexManifestRule.Args["requireNativeLibs"]) + ensureListEmpty(t, provideNativeLibs) + ensureListEmpty(t, requireNativeLibs) + + apexManifestRule = ctx.ModuleForTests("myapex_dep", "android_common_myapex_dep_image").Rule("apexManifestRule") + provideNativeLibs = names(apexManifestRule.Args["provideNativeLibs"]) + requireNativeLibs = names(apexManifestRule.Args["requireNativeLibs"]) + ensureListEmpty(t, provideNativeLibs) + ensureListContains(t, requireNativeLibs, "libfoo.so") + + apexManifestRule = ctx.ModuleForTests("myapex_provider", "android_common_myapex_provider_image").Rule("apexManifestRule") + provideNativeLibs = names(apexManifestRule.Args["provideNativeLibs"]) + requireNativeLibs = names(apexManifestRule.Args["requireNativeLibs"]) + ensureListContains(t, provideNativeLibs, "libfoo.so") + ensureListEmpty(t, requireNativeLibs) + + apexManifestRule = ctx.ModuleForTests("myapex_selfcontained", "android_common_myapex_selfcontained_image").Rule("apexManifestRule") + provideNativeLibs = names(apexManifestRule.Args["provideNativeLibs"]) + requireNativeLibs = names(apexManifestRule.Args["requireNativeLibs"]) + ensureListContains(t, provideNativeLibs, "libfoo.so") + ensureListEmpty(t, requireNativeLibs) +} + +func TestApexName(t *testing.T) { + ctx, config := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + apex_name: "com.android.myapex", + native_shared_libs: ["mylib"], + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + cc_library { + name: "mylib", + srcs: ["mylib.cpp"], + system_shared_libs: [], + stl: "none", + apex_available: [ + "//apex_available:platform", + "myapex", + ], + } + `) + + module := ctx.ModuleForTests("myapex", "android_common_myapex_image") + apexManifestRule := module.Rule("apexManifestRule") + ensureContains(t, apexManifestRule.Args["opt"], "-v name com.android.myapex") + apexRule := module.Rule("apexRule") + ensureContains(t, apexRule.Args["opt_flags"], "--do_not_check_keyname") + + apexBundle := ctx.ModuleForTests("myapex", "android_common_myapex_image").Module().(*apexBundle) + data := android.AndroidMkDataForTest(t, config, "", apexBundle) + name := apexBundle.BaseModuleName() + prefix := "TARGET_" + var builder strings.Builder + data.Custom(&builder, name, prefix, "", data) + androidMk := builder.String() + ensureContains(t, androidMk, "LOCAL_MODULE := mylib.myapex\n") + ensureNotContains(t, androidMk, "LOCAL_MODULE := mylib.com.android.myapex\n") +} + func TestNonTestApex(t *testing.T) { - ctx := testApex(t, ` + ctx, _ := testApex(t, ` apex { name: "myapex", key: "myapex.key", @@ -1005,10 +2774,14 @@ func TestNonTestApex(t *testing.T) { srcs: ["mylib.cpp"], system_shared_libs: [], stl: "none", + apex_available: [ + "//apex_available:platform", + "myapex", + ], } `) - module := ctx.ModuleForTests("myapex", "android_common_myapex") + module := ctx.ModuleForTests("myapex", "android_common_myapex_image") apexRule := module.Rule("apexRule") copyCmds := apexRule.Args["copy_commands"] @@ -1020,13 +2793,13 @@ func TestNonTestApex(t *testing.T) { ensureContains(t, apexRule.Output.String(), "myapex.apex.unsigned") // Ensure that apex variant is created for the direct dep - ensureListContains(t, ctx.ModuleVariantsForTests("mylib_common"), "android_arm64_armv8-a_core_shared_myapex") + ensureListContains(t, ctx.ModuleVariantsForTests("mylib_common"), "android_arm64_armv8-a_shared_myapex") // Ensure that both direct and indirect deps are copied into apex ensureContains(t, copyCmds, "image.apex/lib64/mylib_common.so") - // Ensure that the platform variant ends with _core_shared - ensureListContains(t, ctx.ModuleVariantsForTests("mylib_common"), "android_arm64_armv8-a_core_shared") + // Ensure that the platform variant ends with _shared + ensureListContains(t, ctx.ModuleVariantsForTests("mylib_common"), "android_arm64_armv8-a_shared") if !android.InAnyApex("mylib_common") { t.Log("Found mylib_common not in any apex!") @@ -1038,7 +2811,7 @@ func TestTestApex(t *testing.T) { if android.InAnyApex("mylib_common_test") { t.Fatal("mylib_common_test must not be used in any other tests since this checks that global state is not updated in an illegal way!") } - ctx := testApex(t, ` + ctx, _ := testApex(t, ` apex_test { name: "myapex", key: "myapex.key", @@ -1056,10 +2829,15 @@ func TestTestApex(t *testing.T) { srcs: ["mylib.cpp"], system_shared_libs: [], stl: "none", + // TODO: remove //apex_available:platform + apex_available: [ + "//apex_available:platform", + "myapex", + ], } `) - module := ctx.ModuleForTests("myapex", "android_common_myapex") + module := ctx.ModuleForTests("myapex", "android_common_myapex_image") apexRule := module.Rule("apexRule") copyCmds := apexRule.Args["copy_commands"] @@ -1071,22 +2849,17 @@ func TestTestApex(t *testing.T) { ensureContains(t, apexRule.Output.String(), "myapex.apex.unsigned") // Ensure that apex variant is created for the direct dep - ensureListContains(t, ctx.ModuleVariantsForTests("mylib_common_test"), "android_arm64_armv8-a_core_shared_myapex") + ensureListContains(t, ctx.ModuleVariantsForTests("mylib_common_test"), "android_arm64_armv8-a_shared_myapex") // Ensure that both direct and indirect deps are copied into apex ensureContains(t, copyCmds, "image.apex/lib64/mylib_common_test.so") - // Ensure that the platform variant ends with _core_shared - ensureListContains(t, ctx.ModuleVariantsForTests("mylib_common_test"), "android_arm64_armv8-a_core_shared") - - if android.InAnyApex("mylib_common_test") { - t.Log("Found mylib_common_test in some apex!") - t.Fail() - } + // Ensure that the platform variant ends with _shared + ensureListContains(t, ctx.ModuleVariantsForTests("mylib_common_test"), "android_arm64_armv8-a_shared") } func TestApexWithTarget(t *testing.T) { - ctx := testApex(t, ` + ctx, _ := testApex(t, ` apex { name: "myapex", key: "myapex.key", @@ -1124,6 +2897,11 @@ func TestApexWithTarget(t *testing.T) { srcs: ["mylib.cpp"], system_shared_libs: [], stl: "none", + // TODO: remove //apex_available:platform + apex_available: [ + "//apex_available:platform", + "myapex", + ], } cc_library { @@ -1132,6 +2910,11 @@ func TestApexWithTarget(t *testing.T) { system_shared_libs: [], stl: "none", compile_multilib: "first", + // TODO: remove //apex_available:platform + apex_available: [ + "//apex_available:platform", + "myapex", + ], } cc_library { @@ -1143,30 +2926,30 @@ func TestApexWithTarget(t *testing.T) { } `) - apexRule := ctx.ModuleForTests("myapex", "android_common_myapex").Rule("apexRule") + apexRule := ctx.ModuleForTests("myapex", "android_common_myapex_image").Rule("apexRule") copyCmds := apexRule.Args["copy_commands"] // Ensure that main rule creates an output ensureContains(t, apexRule.Output.String(), "myapex.apex.unsigned") // Ensure that apex variant is created for the direct dep - ensureListContains(t, ctx.ModuleVariantsForTests("mylib"), "android_arm64_armv8-a_core_shared_myapex") - ensureListContains(t, ctx.ModuleVariantsForTests("mylib_common"), "android_arm64_armv8-a_core_shared_myapex") - ensureListNotContains(t, ctx.ModuleVariantsForTests("mylib2"), "android_arm64_armv8-a_core_shared_myapex") + ensureListContains(t, ctx.ModuleVariantsForTests("mylib"), "android_arm64_armv8-a_shared_myapex") + ensureListContains(t, ctx.ModuleVariantsForTests("mylib_common"), "android_arm64_armv8-a_shared_myapex") + ensureListNotContains(t, ctx.ModuleVariantsForTests("mylib2"), "android_arm64_armv8-a_shared_myapex") // Ensure that both direct and indirect deps are copied into apex ensureContains(t, copyCmds, "image.apex/lib64/mylib.so") ensureContains(t, copyCmds, "image.apex/lib64/mylib_common.so") ensureNotContains(t, copyCmds, "image.apex/lib64/mylib2.so") - // Ensure that the platform variant ends with _core_shared - ensureListContains(t, ctx.ModuleVariantsForTests("mylib"), "android_arm64_armv8-a_core_shared") - ensureListContains(t, ctx.ModuleVariantsForTests("mylib_common"), "android_arm64_armv8-a_core_shared") - ensureListContains(t, ctx.ModuleVariantsForTests("mylib2"), "android_arm64_armv8-a_core_shared") + // Ensure that the platform variant ends with _shared + ensureListContains(t, ctx.ModuleVariantsForTests("mylib"), "android_arm64_armv8-a_shared") + ensureListContains(t, ctx.ModuleVariantsForTests("mylib_common"), "android_arm64_armv8-a_shared") + ensureListContains(t, ctx.ModuleVariantsForTests("mylib2"), "android_arm64_armv8-a_shared") } func TestApexWithShBinary(t *testing.T) { - ctx := testApex(t, ` + ctx, _ := testApex(t, ` apex { name: "myapex", key: "myapex.key", @@ -1187,46 +2970,164 @@ func TestApexWithShBinary(t *testing.T) { } `) - apexRule := ctx.ModuleForTests("myapex", "android_common_myapex").Rule("apexRule") + apexRule := ctx.ModuleForTests("myapex", "android_common_myapex_image").Rule("apexRule") copyCmds := apexRule.Args["copy_commands"] ensureContains(t, copyCmds, "image.apex/bin/script/myscript.sh") } -func TestApexInProductPartition(t *testing.T) { - ctx := testApex(t, ` - apex { - name: "myapex", - key: "myapex.key", - native_shared_libs: ["mylib"], - product_specific: true, - } +func TestApexInVariousPartition(t *testing.T) { + testcases := []struct { + propName, parition, flattenedPartition string + }{ + {"", "system", "system_ext"}, + {"product_specific: true", "product", "product"}, + {"soc_specific: true", "vendor", "vendor"}, + {"proprietary: true", "vendor", "vendor"}, + {"vendor: true", "vendor", "vendor"}, + {"system_ext_specific: true", "system_ext", "system_ext"}, + } + for _, tc := range testcases { + t.Run(tc.propName+":"+tc.parition, func(t *testing.T) { + ctx, _ := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + `+tc.propName+` + } - apex_key { - name: "myapex.key", - public_key: "testkey.avbpubkey", - private_key: "testkey.pem", - product_specific: true, - } + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + `) - cc_library { - name: "mylib", - srcs: ["mylib.cpp"], - system_shared_libs: [], - stl: "none", - } + apex := ctx.ModuleForTests("myapex", "android_common_myapex_image").Module().(*apexBundle) + expected := buildDir + "/target/product/test_device/" + tc.parition + "/apex" + actual := apex.installDir.String() + if actual != expected { + t.Errorf("wrong install path. expected %q. actual %q", expected, actual) + } + + flattened := ctx.ModuleForTests("myapex", "android_common_myapex_flattened").Module().(*apexBundle) + expected = buildDir + "/target/product/test_device/" + tc.flattenedPartition + "/apex" + actual = flattened.installDir.String() + if actual != expected { + t.Errorf("wrong install path. expected %q. actual %q", expected, actual) + } + }) + } +} + +func TestFileContexts(t *testing.T) { + ctx, _ := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + `) + module := ctx.ModuleForTests("myapex", "android_common_myapex_image") + apexRule := module.Rule("apexRule") + actual := apexRule.Args["file_contexts"] + expected := "system/sepolicy/apex/myapex-file_contexts" + if actual != expected { + t.Errorf("wrong file_contexts. expected %q. actual %q", expected, actual) + } + + testApexError(t, `"myapex" .*: file_contexts: should be under system/sepolicy`, ` + apex { + name: "myapex", + key: "myapex.key", + file_contexts: "my_own_file_contexts", + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + `, withFiles(map[string][]byte{ + "my_own_file_contexts": nil, + })) + + testApexError(t, `"myapex" .*: file_contexts: cannot find`, ` + apex { + name: "myapex", + key: "myapex.key", + product_specific: true, + file_contexts: "product_specific_file_contexts", + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } `) - apex := ctx.ModuleForTests("myapex", "android_common_myapex").Module().(*apexBundle) - expected := "target/product/test_device/product/apex" - actual := apex.installDir.RelPathString() + ctx, _ = testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + product_specific: true, + file_contexts: "product_specific_file_contexts", + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + `, withFiles(map[string][]byte{ + "product_specific_file_contexts": nil, + })) + module = ctx.ModuleForTests("myapex", "android_common_myapex_image") + apexRule = module.Rule("apexRule") + actual = apexRule.Args["file_contexts"] + expected = "product_specific_file_contexts" if actual != expected { - t.Errorf("wrong install path. expected %q. actual %q", expected, actual) + t.Errorf("wrong file_contexts. expected %q. actual %q", expected, actual) + } + + ctx, _ = testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + product_specific: true, + file_contexts: ":my-file-contexts", + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + filegroup { + name: "my-file-contexts", + srcs: ["product_specific_file_contexts"], + } + `, withFiles(map[string][]byte{ + "product_specific_file_contexts": nil, + })) + module = ctx.ModuleForTests("myapex", "android_common_myapex_image") + apexRule = module.Rule("apexRule") + actual = apexRule.Args["file_contexts"] + expected = "product_specific_file_contexts" + if actual != expected { + t.Errorf("wrong file_contexts. expected %q. actual %q", expected, actual) } } func TestApexKeyFromOtherModule(t *testing.T) { - ctx := testApex(t, ` + ctx, _ := testApex(t, ` apex_key { name: "myapex.key", public_key: ":my.avbpubkey", @@ -1259,7 +3160,7 @@ func TestApexKeyFromOtherModule(t *testing.T) { } func TestPrebuilt(t *testing.T) { - ctx := testApex(t, ` + ctx, _ := testApex(t, ` prebuilt_apex { name: "myapex", arch: { @@ -1282,7 +3183,7 @@ func TestPrebuilt(t *testing.T) { } func TestPrebuiltFilenameOverride(t *testing.T) { - ctx := testApex(t, ` + ctx, _ := testApex(t, ` prebuilt_apex { name: "myapex", src: "myapex-arm.apex", @@ -1298,13 +3199,1856 @@ func TestPrebuiltFilenameOverride(t *testing.T) { } } +func TestPrebuiltOverrides(t *testing.T) { + ctx, config := testApex(t, ` + prebuilt_apex { + name: "myapex.prebuilt", + src: "myapex-arm.apex", + overrides: [ + "myapex", + ], + } + `) + + p := ctx.ModuleForTests("myapex.prebuilt", "android_common").Module().(*Prebuilt) + + expected := []string{"myapex"} + actual := android.AndroidMkEntriesForTest(t, config, "", p)[0].EntryMap["LOCAL_OVERRIDES_MODULES"] + if !reflect.DeepEqual(actual, expected) { + t.Errorf("Incorrect LOCAL_OVERRIDES_MODULES value '%s', expected '%s'", actual, expected) + } +} + +func TestApexWithTests(t *testing.T) { + ctx, config := testApex(t, ` + apex_test { + name: "myapex", + key: "myapex.key", + tests: [ + "mytest", + "mytests", + ], + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + cc_test { + name: "mytest", + gtest: false, + srcs: ["mytest.cpp"], + relative_install_path: "test", + system_shared_libs: [], + static_executable: true, + stl: "none", + } + + cc_test { + name: "mytests", + gtest: false, + srcs: [ + "mytest1.cpp", + "mytest2.cpp", + "mytest3.cpp", + ], + test_per_src: true, + relative_install_path: "test", + system_shared_libs: [], + static_executable: true, + stl: "none", + } + `) + + apexRule := ctx.ModuleForTests("myapex", "android_common_myapex_image").Rule("apexRule") + copyCmds := apexRule.Args["copy_commands"] + + // Ensure that test dep is copied into apex. + ensureContains(t, copyCmds, "image.apex/bin/test/mytest") + + // Ensure that test deps built with `test_per_src` are copied into apex. + ensureContains(t, copyCmds, "image.apex/bin/test/mytest1") + ensureContains(t, copyCmds, "image.apex/bin/test/mytest2") + ensureContains(t, copyCmds, "image.apex/bin/test/mytest3") + + // Ensure the module is correctly translated. + apexBundle := ctx.ModuleForTests("myapex", "android_common_myapex_image").Module().(*apexBundle) + data := android.AndroidMkDataForTest(t, config, "", apexBundle) + name := apexBundle.BaseModuleName() + prefix := "TARGET_" + var builder strings.Builder + data.Custom(&builder, name, prefix, "", data) + androidMk := builder.String() + ensureContains(t, androidMk, "LOCAL_MODULE := mytest.myapex\n") + ensureContains(t, androidMk, "LOCAL_MODULE := mytest1.myapex\n") + ensureContains(t, androidMk, "LOCAL_MODULE := mytest2.myapex\n") + ensureContains(t, androidMk, "LOCAL_MODULE := mytest3.myapex\n") + ensureContains(t, androidMk, "LOCAL_MODULE := apex_manifest.pb.myapex\n") + ensureContains(t, androidMk, "LOCAL_MODULE := apex_pubkey.myapex\n") + ensureContains(t, androidMk, "LOCAL_MODULE := myapex\n") +} + +func TestInstallExtraFlattenedApexes(t *testing.T) { + ctx, config := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + } + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + `, func(fs map[string][]byte, config android.Config) { + config.TestProductVariables.InstallExtraFlattenedApexes = proptools.BoolPtr(true) + }) + ab := ctx.ModuleForTests("myapex", "android_common_myapex_image").Module().(*apexBundle) + ensureListContains(t, ab.requiredDeps, "myapex.flattened") + mk := android.AndroidMkDataForTest(t, config, "", ab) + var builder strings.Builder + mk.Custom(&builder, ab.Name(), "TARGET_", "", mk) + androidMk := builder.String() + ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES += myapex.flattened") +} + +func TestApexUsesOtherApex(t *testing.T) { + ctx, _ := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + native_shared_libs: ["mylib"], + uses: ["commonapex"], + } + + apex { + name: "commonapex", + key: "myapex.key", + native_shared_libs: ["libcommon"], + provide_cpp_shared_libs: true, + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + cc_library { + name: "mylib", + srcs: ["mylib.cpp"], + shared_libs: ["libcommon"], + system_shared_libs: [], + stl: "none", + apex_available: [ "myapex" ], + } + + cc_library { + name: "libcommon", + srcs: ["mylib_common.cpp"], + system_shared_libs: [], + stl: "none", + // TODO: remove //apex_available:platform + apex_available: [ + "//apex_available:platform", + "commonapex", + "myapex", + ], + } + `) + + module1 := ctx.ModuleForTests("myapex", "android_common_myapex_image") + apexRule1 := module1.Rule("apexRule") + copyCmds1 := apexRule1.Args["copy_commands"] + + module2 := ctx.ModuleForTests("commonapex", "android_common_commonapex_image") + apexRule2 := module2.Rule("apexRule") + copyCmds2 := apexRule2.Args["copy_commands"] + + ensureListContains(t, ctx.ModuleVariantsForTests("mylib"), "android_arm64_armv8-a_shared_myapex") + ensureListContains(t, ctx.ModuleVariantsForTests("libcommon"), "android_arm64_armv8-a_shared_commonapex") + ensureContains(t, copyCmds1, "image.apex/lib64/mylib.so") + ensureContains(t, copyCmds2, "image.apex/lib64/libcommon.so") + ensureNotContains(t, copyCmds1, "image.apex/lib64/libcommon.so") +} + +func TestApexUsesFailsIfNotProvided(t *testing.T) { + testApexError(t, `uses: "commonapex" does not provide native_shared_libs`, ` + apex { + name: "myapex", + key: "myapex.key", + uses: ["commonapex"], + } + + apex { + name: "commonapex", + key: "myapex.key", + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + `) + testApexError(t, `uses: "commonapex" is not a provider`, ` + apex { + name: "myapex", + key: "myapex.key", + uses: ["commonapex"], + } + + cc_library { + name: "commonapex", + system_shared_libs: [], + stl: "none", + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + `) +} + +func TestApexUsesFailsIfUseVenderMismatch(t *testing.T) { + testApexError(t, `use_vendor: "commonapex" has different value of use_vendor`, ` + apex { + name: "myapex", + key: "myapex.key", + use_vendor: true, + uses: ["commonapex"], + } + + apex { + name: "commonapex", + key: "myapex.key", + provide_cpp_shared_libs: true, + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + `, func(fs map[string][]byte, config android.Config) { + setUseVendorAllowListForTest(config, []string{"myapex"}) + }) +} + +func TestErrorsIfDepsAreNotEnabled(t *testing.T) { + testApexError(t, `module "myapex" .* depends on disabled module "libfoo"`, ` + apex { + name: "myapex", + key: "myapex.key", + native_shared_libs: ["libfoo"], + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + cc_library { + name: "libfoo", + stl: "none", + system_shared_libs: [], + enabled: false, + apex_available: ["myapex"], + } + `) + testApexError(t, `module "myapex" .* depends on disabled module "myjar"`, ` + apex { + name: "myapex", + key: "myapex.key", + java_libs: ["myjar"], + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + java_library { + name: "myjar", + srcs: ["foo/bar/MyClass.java"], + sdk_version: "none", + system_modules: "none", + enabled: false, + apex_available: ["myapex"], + } + `) +} + +func TestApexWithApps(t *testing.T) { + ctx, _ := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + apps: [ + "AppFoo", + "AppFooPriv", + ], + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + android_app { + name: "AppFoo", + srcs: ["foo/bar/MyClass.java"], + sdk_version: "current", + system_modules: "none", + jni_libs: ["libjni"], + stl: "none", + apex_available: [ "myapex" ], + } + + android_app { + name: "AppFooPriv", + srcs: ["foo/bar/MyClass.java"], + sdk_version: "current", + system_modules: "none", + privileged: true, + stl: "none", + apex_available: [ "myapex" ], + } + + cc_library_shared { + name: "libjni", + srcs: ["mylib.cpp"], + shared_libs: ["libfoo"], + stl: "none", + system_shared_libs: [], + apex_available: [ "myapex" ], + sdk_version: "current", + } + + cc_library_shared { + name: "libfoo", + stl: "none", + system_shared_libs: [], + apex_available: [ "myapex" ], + sdk_version: "current", + } + `) + + module := ctx.ModuleForTests("myapex", "android_common_myapex_image") + apexRule := module.Rule("apexRule") + copyCmds := apexRule.Args["copy_commands"] + + ensureContains(t, copyCmds, "image.apex/app/AppFoo/AppFoo.apk") + ensureContains(t, copyCmds, "image.apex/priv-app/AppFooPriv/AppFooPriv.apk") + + appZipRule := ctx.ModuleForTests("AppFoo", "android_common_myapex").Description("zip jni libs") + // JNI libraries are uncompressed + if args := appZipRule.Args["jarArgs"]; !strings.Contains(args, "-L 0") { + t.Errorf("jni libs are not uncompressed for AppFoo") + } + // JNI libraries including transitive deps are + for _, jni := range []string{"libjni", "libfoo"} { + jniOutput := ctx.ModuleForTests(jni, "android_arm64_armv8-a_sdk_shared_myapex").Module().(*cc.Module).OutputFile() + // ... embedded inside APK (jnilibs.zip) + ensureListContains(t, appZipRule.Implicits.Strings(), jniOutput.String()) + // ... and not directly inside the APEX + ensureNotContains(t, copyCmds, "image.apex/lib64/"+jni+".so") + } +} + +func TestApexWithAppImports(t *testing.T) { + ctx, _ := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + apps: [ + "AppFooPrebuilt", + "AppFooPrivPrebuilt", + ], + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + android_app_import { + name: "AppFooPrebuilt", + apk: "PrebuiltAppFoo.apk", + presigned: true, + dex_preopt: { + enabled: false, + }, + } + + android_app_import { + name: "AppFooPrivPrebuilt", + apk: "PrebuiltAppFooPriv.apk", + privileged: true, + presigned: true, + dex_preopt: { + enabled: false, + }, + filename: "AwesomePrebuiltAppFooPriv.apk", + } + `) + + module := ctx.ModuleForTests("myapex", "android_common_myapex_image") + apexRule := module.Rule("apexRule") + copyCmds := apexRule.Args["copy_commands"] + + ensureContains(t, copyCmds, "image.apex/app/AppFooPrebuilt/AppFooPrebuilt.apk") + ensureContains(t, copyCmds, "image.apex/priv-app/AppFooPrivPrebuilt/AwesomePrebuiltAppFooPriv.apk") +} + +func TestApexWithAppImportsPrefer(t *testing.T) { + ctx, _ := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + apps: [ + "AppFoo", + ], + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + android_app { + name: "AppFoo", + srcs: ["foo/bar/MyClass.java"], + sdk_version: "none", + system_modules: "none", + apex_available: [ "myapex" ], + } + + android_app_import { + name: "AppFoo", + apk: "AppFooPrebuilt.apk", + filename: "AppFooPrebuilt.apk", + presigned: true, + prefer: true, + } + `, withFiles(map[string][]byte{ + "AppFooPrebuilt.apk": nil, + })) + + ensureExactContents(t, ctx, "myapex", "android_common_myapex_image", []string{ + "app/AppFoo/AppFooPrebuilt.apk", + }) +} + +func TestApexWithTestHelperApp(t *testing.T) { + ctx, _ := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + apps: [ + "TesterHelpAppFoo", + ], + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + android_test_helper_app { + name: "TesterHelpAppFoo", + srcs: ["foo/bar/MyClass.java"], + apex_available: [ "myapex" ], + } + + `) + + module := ctx.ModuleForTests("myapex", "android_common_myapex_image") + apexRule := module.Rule("apexRule") + copyCmds := apexRule.Args["copy_commands"] + + ensureContains(t, copyCmds, "image.apex/app/TesterHelpAppFoo/TesterHelpAppFoo.apk") +} + +func TestApexPropertiesShouldBeDefaultable(t *testing.T) { + // libfoo's apex_available comes from cc_defaults + testApexError(t, `requires "libfoo" that is not available for the APEX`, ` + apex { + name: "myapex", + key: "myapex.key", + native_shared_libs: ["libfoo"], + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + apex { + name: "otherapex", + key: "myapex.key", + native_shared_libs: ["libfoo"], + } + + cc_defaults { + name: "libfoo-defaults", + apex_available: ["otherapex"], + } + + cc_library { + name: "libfoo", + defaults: ["libfoo-defaults"], + stl: "none", + system_shared_libs: [], + }`) +} + +func TestApexAvailable_DirectDep(t *testing.T) { + // libfoo is not available to myapex, but only to otherapex + testApexError(t, "requires \"libfoo\" that is not available for the APEX", ` + apex { + name: "myapex", + key: "myapex.key", + native_shared_libs: ["libfoo"], + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + apex { + name: "otherapex", + key: "otherapex.key", + native_shared_libs: ["libfoo"], + } + + apex_key { + name: "otherapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + cc_library { + name: "libfoo", + stl: "none", + system_shared_libs: [], + apex_available: ["otherapex"], + }`) +} + +func TestApexAvailable_IndirectDep(t *testing.T) { + // libbbaz is an indirect dep + testApexError(t, `requires "libbaz" that is not available for the APEX. Dependency path: +.*via tag apex\.dependencyTag.*"sharedLib".* +.*-> libfoo.*link:shared.* +.*via tag cc\.DependencyTag.*"shared".* +.*-> libbar.*link:shared.* +.*via tag cc\.DependencyTag.*"shared".* +.*-> libbaz.*link:shared.*`, ` + apex { + name: "myapex", + key: "myapex.key", + native_shared_libs: ["libfoo"], + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + cc_library { + name: "libfoo", + stl: "none", + shared_libs: ["libbar"], + system_shared_libs: [], + apex_available: ["myapex"], + } + + cc_library { + name: "libbar", + stl: "none", + shared_libs: ["libbaz"], + system_shared_libs: [], + apex_available: ["myapex"], + } + + cc_library { + name: "libbaz", + stl: "none", + system_shared_libs: [], + }`) +} + +func TestApexAvailable_InvalidApexName(t *testing.T) { + testApexError(t, "\"otherapex\" is not a valid module name", ` + apex { + name: "myapex", + key: "myapex.key", + native_shared_libs: ["libfoo"], + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + cc_library { + name: "libfoo", + stl: "none", + system_shared_libs: [], + apex_available: ["otherapex"], + }`) + + testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + native_shared_libs: ["libfoo", "libbar"], + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + cc_library { + name: "libfoo", + stl: "none", + system_shared_libs: [], + runtime_libs: ["libbaz"], + apex_available: ["myapex"], + } + + cc_library { + name: "libbar", + stl: "none", + system_shared_libs: [], + apex_available: ["//apex_available:anyapex"], + } + + cc_library { + name: "libbaz", + stl: "none", + system_shared_libs: [], + stubs: { + versions: ["10", "20", "30"], + }, + }`) +} + +func TestApexAvailable_CheckForPlatform(t *testing.T) { + ctx, _ := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + native_shared_libs: ["libbar", "libbaz"], + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + cc_library { + name: "libfoo", + stl: "none", + system_shared_libs: [], + shared_libs: ["libbar"], + apex_available: ["//apex_available:platform"], + } + + cc_library { + name: "libfoo2", + stl: "none", + system_shared_libs: [], + shared_libs: ["libbaz"], + apex_available: ["//apex_available:platform"], + } + + cc_library { + name: "libbar", + stl: "none", + system_shared_libs: [], + apex_available: ["myapex"], + } + + cc_library { + name: "libbaz", + stl: "none", + system_shared_libs: [], + apex_available: ["myapex"], + stubs: { + versions: ["1"], + }, + }`) + + // libfoo shouldn't be available to platform even though it has "//apex_available:platform", + // because it depends on libbar which isn't available to platform + libfoo := ctx.ModuleForTests("libfoo", "android_arm64_armv8-a_shared").Module().(*cc.Module) + if libfoo.NotAvailableForPlatform() != true { + t.Errorf("%q shouldn't be available to platform", libfoo.String()) + } + + // libfoo2 however can be available to platform because it depends on libbaz which provides + // stubs + libfoo2 := ctx.ModuleForTests("libfoo2", "android_arm64_armv8-a_shared").Module().(*cc.Module) + if libfoo2.NotAvailableForPlatform() == true { + t.Errorf("%q should be available to platform", libfoo2.String()) + } +} + +func TestApexAvailable_CreatedForApex(t *testing.T) { + ctx, _ := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + native_shared_libs: ["libfoo"], + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + cc_library { + name: "libfoo", + stl: "none", + system_shared_libs: [], + apex_available: ["myapex"], + static: { + apex_available: ["//apex_available:platform"], + }, + }`) + + libfooShared := ctx.ModuleForTests("libfoo", "android_arm64_armv8-a_shared").Module().(*cc.Module) + if libfooShared.NotAvailableForPlatform() != true { + t.Errorf("%q shouldn't be available to platform", libfooShared.String()) + } + libfooStatic := ctx.ModuleForTests("libfoo", "android_arm64_armv8-a_static").Module().(*cc.Module) + if libfooStatic.NotAvailableForPlatform() != false { + t.Errorf("%q should be available to platform", libfooStatic.String()) + } +} + +func TestOverrideApex(t *testing.T) { + ctx, config := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + apps: ["app"], + overrides: ["oldapex"], + } + + override_apex { + name: "override_myapex", + base: "myapex", + apps: ["override_app"], + overrides: ["unknownapex"], + logging_parent: "com.foo.bar", + package_name: "test.overridden.package", + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + android_app { + name: "app", + srcs: ["foo/bar/MyClass.java"], + package_name: "foo", + sdk_version: "none", + system_modules: "none", + apex_available: [ "myapex" ], + } + + override_android_app { + name: "override_app", + base: "app", + package_name: "bar", + } + `, withManifestPackageNameOverrides([]string{"myapex:com.android.myapex"})) + + originalVariant := ctx.ModuleForTests("myapex", "android_common_myapex_image").Module().(android.OverridableModule) + overriddenVariant := ctx.ModuleForTests("myapex", "android_common_override_myapex_myapex_image").Module().(android.OverridableModule) + if originalVariant.GetOverriddenBy() != "" { + t.Errorf("GetOverriddenBy should be empty, but was %q", originalVariant.GetOverriddenBy()) + } + if overriddenVariant.GetOverriddenBy() != "override_myapex" { + t.Errorf("GetOverriddenBy should be \"override_myapex\", but was %q", overriddenVariant.GetOverriddenBy()) + } + + module := ctx.ModuleForTests("myapex", "android_common_override_myapex_myapex_image") + apexRule := module.Rule("apexRule") + copyCmds := apexRule.Args["copy_commands"] + + ensureNotContains(t, copyCmds, "image.apex/app/app/app.apk") + ensureContains(t, copyCmds, "image.apex/app/override_app/override_app.apk") + + apexBundle := module.Module().(*apexBundle) + name := apexBundle.Name() + if name != "override_myapex" { + t.Errorf("name should be \"override_myapex\", but was %q", name) + } + + if apexBundle.overridableProperties.Logging_parent != "com.foo.bar" { + t.Errorf("override_myapex should have logging parent (com.foo.bar), but was %q.", apexBundle.overridableProperties.Logging_parent) + } + + optFlags := apexRule.Args["opt_flags"] + ensureContains(t, optFlags, "--override_apk_package_name test.overridden.package") + + data := android.AndroidMkDataForTest(t, config, "", apexBundle) + var builder strings.Builder + data.Custom(&builder, name, "TARGET_", "", data) + androidMk := builder.String() + ensureContains(t, androidMk, "LOCAL_MODULE := override_app.override_myapex") + ensureContains(t, androidMk, "LOCAL_MODULE := apex_manifest.pb.override_myapex") + ensureContains(t, androidMk, "LOCAL_MODULE_STEM := override_myapex.apex") + ensureContains(t, androidMk, "LOCAL_OVERRIDES_MODULES := unknownapex myapex") + ensureNotContains(t, androidMk, "LOCAL_MODULE := app.myapex") + ensureNotContains(t, androidMk, "LOCAL_MODULE := override_app.myapex") + ensureNotContains(t, androidMk, "LOCAL_MODULE := apex_manifest.pb.myapex") + ensureNotContains(t, androidMk, "LOCAL_MODULE_STEM := myapex.apex") +} + +func TestLegacyAndroid10Support(t *testing.T) { + ctx, _ := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + native_shared_libs: ["mylib"], + min_sdk_version: "29", + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + cc_library { + name: "mylib", + srcs: ["mylib.cpp"], + stl: "libc++", + system_shared_libs: [], + apex_available: [ "myapex" ], + } + `, withUnbundledBuild) + + module := ctx.ModuleForTests("myapex", "android_common_myapex_image") + args := module.Rule("apexRule").Args + ensureContains(t, args["opt_flags"], "--manifest_json "+module.Output("apex_manifest.json").Output.String()) + ensureNotContains(t, args["opt_flags"], "--no_hashtree") + + // The copies of the libraries in the apex should have one more dependency than + // the ones outside the apex, namely the unwinder. Ideally we should check + // the dependency names directly here but for some reason the names are blank in + // this test. + for _, lib := range []string{"libc++", "mylib"} { + apexImplicits := ctx.ModuleForTests(lib, "android_arm64_armv8-a_shared_myapex").Rule("ld").Implicits + nonApexImplicits := ctx.ModuleForTests(lib, "android_arm64_armv8-a_shared").Rule("ld").Implicits + if len(apexImplicits) != len(nonApexImplicits)+1 { + t.Errorf("%q missing unwinder dep", lib) + } + } +} + +var filesForSdkLibrary = map[string][]byte{ + "api/current.txt": nil, + "api/removed.txt": nil, + "api/system-current.txt": nil, + "api/system-removed.txt": nil, + "api/test-current.txt": nil, + "api/test-removed.txt": nil, + + // For java_sdk_library_import + "a.jar": nil, +} + +func TestJavaSDKLibrary(t *testing.T) { + ctx, _ := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + java_libs: ["foo"], + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + java_sdk_library { + name: "foo", + srcs: ["a.java"], + api_packages: ["foo"], + apex_available: [ "myapex" ], + } + `, withFiles(filesForSdkLibrary)) + + // java_sdk_library installs both impl jar and permission XML + ensureExactContents(t, ctx, "myapex", "android_common_myapex_image", []string{ + "javalib/foo.jar", + "etc/permissions/foo.xml", + }) + // Permission XML should point to the activated path of impl jar of java_sdk_library + sdkLibrary := ctx.ModuleForTests("foo.xml", "android_common_myapex").Rule("java_sdk_xml") + ensureContains(t, sdkLibrary.RuleParams.Command, `<library name=\"foo\" file=\"/apex/myapex/javalib/foo.jar\"`) +} + +func TestJavaSDKLibrary_WithinApex(t *testing.T) { + ctx, _ := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + java_libs: ["foo", "bar"], + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + java_sdk_library { + name: "foo", + srcs: ["a.java"], + api_packages: ["foo"], + apex_available: ["myapex"], + sdk_version: "none", + system_modules: "none", + } + + java_library { + name: "bar", + srcs: ["a.java"], + libs: ["foo"], + apex_available: ["myapex"], + sdk_version: "none", + system_modules: "none", + } + `, withFiles(filesForSdkLibrary)) + + // java_sdk_library installs both impl jar and permission XML + ensureExactContents(t, ctx, "myapex", "android_common_myapex_image", []string{ + "javalib/bar.jar", + "javalib/foo.jar", + "etc/permissions/foo.xml", + }) + + // The bar library should depend on the implementation jar. + barLibrary := ctx.ModuleForTests("bar", "android_common_myapex").Rule("javac") + if expected, actual := `^-classpath /[^:]*/turbine-combined/foo\.jar$`, barLibrary.Args["classpath"]; !regexp.MustCompile(expected).MatchString(actual) { + t.Errorf("expected %q, found %#q", expected, actual) + } +} + +func TestJavaSDKLibrary_CrossBoundary(t *testing.T) { + ctx, _ := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + java_libs: ["foo"], + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + java_sdk_library { + name: "foo", + srcs: ["a.java"], + api_packages: ["foo"], + apex_available: ["myapex"], + sdk_version: "none", + system_modules: "none", + } + + java_library { + name: "bar", + srcs: ["a.java"], + libs: ["foo"], + sdk_version: "none", + system_modules: "none", + } + `, withFiles(filesForSdkLibrary)) + + // java_sdk_library installs both impl jar and permission XML + ensureExactContents(t, ctx, "myapex", "android_common_myapex_image", []string{ + "javalib/foo.jar", + "etc/permissions/foo.xml", + }) + + // The bar library should depend on the stubs jar. + barLibrary := ctx.ModuleForTests("bar", "android_common").Rule("javac") + if expected, actual := `^-classpath /[^:]*/turbine-combined/foo\.stubs\.jar$`, barLibrary.Args["classpath"]; !regexp.MustCompile(expected).MatchString(actual) { + t.Errorf("expected %q, found %#q", expected, actual) + } +} + +func TestJavaSDKLibrary_ImportPreferred(t *testing.T) { + ctx, _ := testApex(t, ``, + withFiles(map[string][]byte{ + "apex/a.java": nil, + "apex/apex_manifest.json": nil, + "apex/Android.bp": []byte(` + package { + default_visibility: ["//visibility:private"], + } + + apex { + name: "myapex", + key: "myapex.key", + java_libs: ["foo", "bar"], + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + java_library { + name: "bar", + srcs: ["a.java"], + libs: ["foo"], + apex_available: ["myapex"], + sdk_version: "none", + system_modules: "none", + } +`), + "source/a.java": nil, + "source/api/current.txt": nil, + "source/api/removed.txt": nil, + "source/Android.bp": []byte(` + package { + default_visibility: ["//visibility:private"], + } + + java_sdk_library { + name: "foo", + visibility: ["//apex"], + srcs: ["a.java"], + api_packages: ["foo"], + apex_available: ["myapex"], + sdk_version: "none", + system_modules: "none", + public: { + enabled: true, + }, + } +`), + "prebuilt/a.jar": nil, + "prebuilt/Android.bp": []byte(` + package { + default_visibility: ["//visibility:private"], + } + + java_sdk_library_import { + name: "foo", + visibility: ["//apex", "//source"], + apex_available: ["myapex"], + prefer: true, + public: { + jars: ["a.jar"], + }, + } +`), + }), + ) + + // java_sdk_library installs both impl jar and permission XML + ensureExactContents(t, ctx, "myapex", "android_common_myapex_image", []string{ + "javalib/bar.jar", + "javalib/foo.jar", + "etc/permissions/foo.xml", + }) + + // The bar library should depend on the implementation jar. + barLibrary := ctx.ModuleForTests("bar", "android_common_myapex").Rule("javac") + if expected, actual := `^-classpath /[^:]*/turbine-combined/foo\.impl\.jar$`, barLibrary.Args["classpath"]; !regexp.MustCompile(expected).MatchString(actual) { + t.Errorf("expected %q, found %#q", expected, actual) + } +} + +func TestJavaSDKLibrary_ImportOnly(t *testing.T) { + testApexError(t, `java_libs: "foo" is not configured to be compiled into dex`, ` + apex { + name: "myapex", + key: "myapex.key", + java_libs: ["foo"], + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + java_sdk_library_import { + name: "foo", + apex_available: ["myapex"], + prefer: true, + public: { + jars: ["a.jar"], + }, + } + + `, withFiles(filesForSdkLibrary)) +} + +func TestCompatConfig(t *testing.T) { + ctx, _ := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + prebuilts: ["myjar-platform-compat-config"], + java_libs: ["myjar"], + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + platform_compat_config { + name: "myjar-platform-compat-config", + src: ":myjar", + } + + java_library { + name: "myjar", + srcs: ["foo/bar/MyClass.java"], + sdk_version: "none", + system_modules: "none", + apex_available: [ "myapex" ], + } + `) + ensureExactContents(t, ctx, "myapex", "android_common_myapex_image", []string{ + "etc/compatconfig/myjar-platform-compat-config.xml", + "javalib/myjar.jar", + }) +} + +func TestRejectNonInstallableJavaLibrary(t *testing.T) { + testApexError(t, `"myjar" is not configured to be compiled into dex`, ` + apex { + name: "myapex", + key: "myapex.key", + java_libs: ["myjar"], + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + java_library { + name: "myjar", + srcs: ["foo/bar/MyClass.java"], + sdk_version: "none", + system_modules: "none", + compile_dex: false, + apex_available: ["myapex"], + } + `) +} + +func TestCarryRequiredModuleNames(t *testing.T) { + ctx, config := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + native_shared_libs: ["mylib"], + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + cc_library { + name: "mylib", + srcs: ["mylib.cpp"], + system_shared_libs: [], + stl: "none", + required: ["a", "b"], + host_required: ["c", "d"], + target_required: ["e", "f"], + apex_available: [ "myapex" ], + } + `) + + apexBundle := ctx.ModuleForTests("myapex", "android_common_myapex_image").Module().(*apexBundle) + data := android.AndroidMkDataForTest(t, config, "", apexBundle) + name := apexBundle.BaseModuleName() + prefix := "TARGET_" + var builder strings.Builder + data.Custom(&builder, name, prefix, "", data) + androidMk := builder.String() + ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES += a b\n") + ensureContains(t, androidMk, "LOCAL_HOST_REQUIRED_MODULES += c d\n") + ensureContains(t, androidMk, "LOCAL_TARGET_REQUIRED_MODULES += e f\n") +} + +func TestSymlinksFromApexToSystem(t *testing.T) { + bp := ` + apex { + name: "myapex", + key: "myapex.key", + native_shared_libs: ["mylib"], + java_libs: ["myjar"], + } + + apex { + name: "myapex.updatable", + key: "myapex.key", + native_shared_libs: ["mylib"], + java_libs: ["myjar"], + updatable: true, + min_sdk_version: "current", + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + cc_library { + name: "mylib", + srcs: ["mylib.cpp"], + shared_libs: ["myotherlib"], + system_shared_libs: [], + stl: "none", + apex_available: [ + "myapex", + "myapex.updatable", + "//apex_available:platform", + ], + } + + cc_library { + name: "myotherlib", + srcs: ["mylib.cpp"], + system_shared_libs: [], + stl: "none", + apex_available: [ + "myapex", + "myapex.updatable", + "//apex_available:platform", + ], + } + + java_library { + name: "myjar", + srcs: ["foo/bar/MyClass.java"], + sdk_version: "none", + system_modules: "none", + libs: ["myotherjar"], + apex_available: [ + "myapex", + "myapex.updatable", + "//apex_available:platform", + ], + } + + java_library { + name: "myotherjar", + srcs: ["foo/bar/MyClass.java"], + sdk_version: "none", + system_modules: "none", + apex_available: [ + "myapex", + "myapex.updatable", + "//apex_available:platform", + ], + } + ` + + ensureRealfileExists := func(t *testing.T, files []fileInApex, file string) { + for _, f := range files { + if f.path == file { + if f.isLink { + t.Errorf("%q is not a real file", file) + } + return + } + } + t.Errorf("%q is not found", file) + } + + ensureSymlinkExists := func(t *testing.T, files []fileInApex, file string) { + for _, f := range files { + if f.path == file { + if !f.isLink { + t.Errorf("%q is not a symlink", file) + } + return + } + } + t.Errorf("%q is not found", file) + } + + // For unbundled build, symlink shouldn't exist regardless of whether an APEX + // is updatable or not + ctx, _ := testApex(t, bp, withUnbundledBuild) + files := getFiles(t, ctx, "myapex", "android_common_myapex_image") + ensureRealfileExists(t, files, "javalib/myjar.jar") + ensureRealfileExists(t, files, "lib64/mylib.so") + ensureRealfileExists(t, files, "lib64/myotherlib.so") + + files = getFiles(t, ctx, "myapex.updatable", "android_common_myapex.updatable_image") + ensureRealfileExists(t, files, "javalib/myjar.jar") + ensureRealfileExists(t, files, "lib64/mylib.so") + ensureRealfileExists(t, files, "lib64/myotherlib.so") + + // For bundled build, symlink to the system for the non-updatable APEXes only + ctx, _ = testApex(t, bp) + files = getFiles(t, ctx, "myapex", "android_common_myapex_image") + ensureRealfileExists(t, files, "javalib/myjar.jar") + ensureRealfileExists(t, files, "lib64/mylib.so") + ensureSymlinkExists(t, files, "lib64/myotherlib.so") // this is symlink + + files = getFiles(t, ctx, "myapex.updatable", "android_common_myapex.updatable_image") + ensureRealfileExists(t, files, "javalib/myjar.jar") + ensureRealfileExists(t, files, "lib64/mylib.so") + ensureRealfileExists(t, files, "lib64/myotherlib.so") // this is a real file +} + +func TestApexMutatorsDontRunIfDisabled(t *testing.T) { + ctx, _ := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + } + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + `, func(fs map[string][]byte, config android.Config) { + delete(config.Targets, android.Android) + config.AndroidCommonTarget = android.Target{} + }) + + if expected, got := []string{""}, ctx.ModuleVariantsForTests("myapex"); !reflect.DeepEqual(expected, got) { + t.Errorf("Expected variants: %v, but got: %v", expected, got) + } +} + +func TestAppBundle(t *testing.T) { + ctx, _ := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + apps: ["AppFoo"], + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + android_app { + name: "AppFoo", + srcs: ["foo/bar/MyClass.java"], + sdk_version: "none", + system_modules: "none", + apex_available: [ "myapex" ], + } + `, withManifestPackageNameOverrides([]string{"AppFoo:com.android.foo"})) + + bundleConfigRule := ctx.ModuleForTests("myapex", "android_common_myapex_image").Description("Bundle Config") + content := bundleConfigRule.Args["content"] + + ensureContains(t, content, `"compression":{"uncompressed_glob":["apex_payload.img","apex_manifest.*"]}`) + ensureContains(t, content, `"apex_config":{"apex_embedded_apk_config":[{"package_name":"com.android.foo","path":"app/AppFoo/AppFoo.apk"}]}`) +} + +func TestAppSetBundle(t *testing.T) { + ctx, _ := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + apps: ["AppSet"], + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + android_app_set { + name: "AppSet", + set: "AppSet.apks", + }`) + mod := ctx.ModuleForTests("myapex", "android_common_myapex_image") + bundleConfigRule := mod.Description("Bundle Config") + content := bundleConfigRule.Args["content"] + ensureContains(t, content, `"compression":{"uncompressed_glob":["apex_payload.img","apex_manifest.*"]}`) + s := mod.Rule("apexRule").Args["copy_commands"] + copyCmds := regexp.MustCompile(" *&& *").Split(s, -1) + if len(copyCmds) != 3 { + t.Fatalf("Expected 3 commands, got %d in:\n%s", len(copyCmds), s) + } + ensureMatches(t, copyCmds[0], "^rm -rf .*/app/AppSet$") + ensureMatches(t, copyCmds[1], "^mkdir -p .*/app/AppSet$") + ensureMatches(t, copyCmds[2], "^unzip .*-d .*/app/AppSet .*/AppSet.zip$") +} + +func testNoUpdatableJarsInBootImage(t *testing.T, errmsg, bp string, transformDexpreoptConfig func(*dexpreopt.GlobalConfig)) { + t.Helper() + + bp = bp + ` + filegroup { + name: "some-updatable-apex-file_contexts", + srcs: [ + "system/sepolicy/apex/some-updatable-apex-file_contexts", + ], + } + + filegroup { + name: "some-non-updatable-apex-file_contexts", + srcs: [ + "system/sepolicy/apex/some-non-updatable-apex-file_contexts", + ], + } + ` + bp += cc.GatherRequiredDepsForTest(android.Android) + bp += java.GatherRequiredDepsForTest() + bp += dexpreopt.BpToolModulesForTest() + + fs := map[string][]byte{ + "a.java": nil, + "a.jar": nil, + "build/make/target/product/security": nil, + "apex_manifest.json": nil, + "AndroidManifest.xml": nil, + "system/sepolicy/apex/some-updatable-apex-file_contexts": nil, + "system/sepolicy/apex/some-non-updatable-apex-file_contexts": nil, + "system/sepolicy/apex/com.android.art.something-file_contexts": nil, + "framework/aidl/a.aidl": nil, + } + cc.GatherRequiredFilesForTest(fs) + + ctx := android.NewTestArchContext() + ctx.RegisterModuleType("apex", BundleFactory) + ctx.RegisterModuleType("apex_key", ApexKeyFactory) + ctx.RegisterModuleType("filegroup", android.FileGroupFactory) + ctx.PreArchMutators(android.RegisterDefaultsPreArchMutators) + cc.RegisterRequiredBuildComponentsForTest(ctx) + java.RegisterJavaBuildComponents(ctx) + java.RegisterSystemModulesBuildComponents(ctx) + java.RegisterAppBuildComponents(ctx) + java.RegisterDexpreoptBootJarsComponents(ctx) + ctx.PostDepsMutators(android.RegisterOverridePostDepsMutators) + ctx.PreDepsMutators(RegisterPreDepsMutators) + ctx.PostDepsMutators(RegisterPostDepsMutators) + + config := android.TestArchConfig(buildDir, nil, bp, fs) + ctx.Register(config) + + _ = dexpreopt.GlobalSoongConfigForTests(config) + dexpreopt.RegisterToolModulesForTest(ctx) + pathCtx := android.PathContextForTesting(config) + dexpreoptConfig := dexpreopt.GlobalConfigForTests(pathCtx) + transformDexpreoptConfig(dexpreoptConfig) + dexpreopt.SetTestGlobalConfig(config, dexpreoptConfig) + + _, errs := ctx.ParseBlueprintsFiles("Android.bp") + android.FailIfErrored(t, errs) + + _, errs = ctx.PrepareBuildActions(config) + if errmsg == "" { + android.FailIfErrored(t, errs) + } else if len(errs) > 0 { + android.FailIfNoMatchingErrors(t, errmsg, errs) + return + } else { + t.Fatalf("missing expected error %q (0 errors are returned)", errmsg) + } +} + +func TestUpdatable_should_set_min_sdk_version(t *testing.T) { + testApexError(t, `"myapex" .*: updatable: updatable APEXes should set min_sdk_version`, ` + apex { + name: "myapex", + key: "myapex.key", + updatable: true, + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + `) +} + +func TestNoUpdatableJarsInBootImage(t *testing.T) { + bp := ` + java_library { + name: "some-updatable-apex-lib", + srcs: ["a.java"], + sdk_version: "current", + apex_available: [ + "some-updatable-apex", + ], + } + + java_library { + name: "some-non-updatable-apex-lib", + srcs: ["a.java"], + apex_available: [ + "some-non-updatable-apex", + ], + } + + java_library { + name: "some-platform-lib", + srcs: ["a.java"], + sdk_version: "current", + installable: true, + } + + java_library { + name: "some-art-lib", + srcs: ["a.java"], + sdk_version: "current", + apex_available: [ + "com.android.art.something", + ], + hostdex: true, + } + + apex { + name: "some-updatable-apex", + key: "some-updatable-apex.key", + java_libs: ["some-updatable-apex-lib"], + updatable: true, + min_sdk_version: "current", + } + + apex { + name: "some-non-updatable-apex", + key: "some-non-updatable-apex.key", + java_libs: ["some-non-updatable-apex-lib"], + } + + apex_key { + name: "some-updatable-apex.key", + } + + apex_key { + name: "some-non-updatable-apex.key", + } + + apex { + name: "com.android.art.something", + key: "com.android.art.something.key", + java_libs: ["some-art-lib"], + updatable: true, + min_sdk_version: "current", + } + + apex_key { + name: "com.android.art.something.key", + } + ` + + var error string + var transform func(*dexpreopt.GlobalConfig) + + // updatable jar from ART apex in the ART boot image => ok + transform = func(config *dexpreopt.GlobalConfig) { + config.ArtApexJars = []string{"some-art-lib"} + } + testNoUpdatableJarsInBootImage(t, "", bp, transform) + + // updatable jar from ART apex in the framework boot image => error + error = "module 'some-art-lib' from updatable apex 'com.android.art.something' is not allowed in the framework boot image" + transform = func(config *dexpreopt.GlobalConfig) { + config.BootJars = []string{"some-art-lib"} + } + testNoUpdatableJarsInBootImage(t, error, bp, transform) + + // updatable jar from some other apex in the ART boot image => error + error = "module 'some-updatable-apex-lib' from updatable apex 'some-updatable-apex' is not allowed in the ART boot image" + transform = func(config *dexpreopt.GlobalConfig) { + config.ArtApexJars = []string{"some-updatable-apex-lib"} + } + testNoUpdatableJarsInBootImage(t, error, bp, transform) + + // non-updatable jar from some other apex in the ART boot image => error + error = "module 'some-non-updatable-apex-lib' is not allowed in the ART boot image" + transform = func(config *dexpreopt.GlobalConfig) { + config.ArtApexJars = []string{"some-non-updatable-apex-lib"} + } + testNoUpdatableJarsInBootImage(t, error, bp, transform) + + // updatable jar from some other apex in the framework boot image => error + error = "module 'some-updatable-apex-lib' from updatable apex 'some-updatable-apex' is not allowed in the framework boot image" + transform = func(config *dexpreopt.GlobalConfig) { + config.BootJars = []string{"some-updatable-apex-lib"} + } + testNoUpdatableJarsInBootImage(t, error, bp, transform) + + // non-updatable jar from some other apex in the framework boot image => ok + transform = func(config *dexpreopt.GlobalConfig) { + config.BootJars = []string{"some-non-updatable-apex-lib"} + } + testNoUpdatableJarsInBootImage(t, "", bp, transform) + + // nonexistent jar in the ART boot image => error + error = "failed to find a dex jar path for module 'nonexistent'" + transform = func(config *dexpreopt.GlobalConfig) { + config.ArtApexJars = []string{"nonexistent"} + } + testNoUpdatableJarsInBootImage(t, error, bp, transform) + + // nonexistent jar in the framework boot image => error + error = "failed to find a dex jar path for module 'nonexistent'" + transform = func(config *dexpreopt.GlobalConfig) { + config.BootJars = []string{"nonexistent"} + } + testNoUpdatableJarsInBootImage(t, error, bp, transform) + + // platform jar in the ART boot image => error + error = "module 'some-platform-lib' is not allowed in the ART boot image" + transform = func(config *dexpreopt.GlobalConfig) { + config.ArtApexJars = []string{"some-platform-lib"} + } + testNoUpdatableJarsInBootImage(t, error, bp, transform) + + // platform jar in the framework boot image => ok + transform = func(config *dexpreopt.GlobalConfig) { + config.BootJars = []string{"some-platform-lib"} + } + testNoUpdatableJarsInBootImage(t, "", bp, transform) +} + +func testApexPermittedPackagesRules(t *testing.T, errmsg, bp string, apexBootJars []string, rules []android.Rule) { + t.Helper() + android.ClearApexDependency() + bp += ` + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + }` + fs := map[string][]byte{ + "lib1/src/A.java": nil, + "lib2/src/B.java": nil, + "system/sepolicy/apex/myapex-file_contexts": nil, + } + + ctx := android.NewTestArchContext() + ctx.RegisterModuleType("apex", BundleFactory) + ctx.RegisterModuleType("apex_key", ApexKeyFactory) + ctx.PreArchMutators(android.RegisterDefaultsPreArchMutators) + cc.RegisterRequiredBuildComponentsForTest(ctx) + java.RegisterJavaBuildComponents(ctx) + java.RegisterSystemModulesBuildComponents(ctx) + java.RegisterDexpreoptBootJarsComponents(ctx) + ctx.PostDepsMutators(android.RegisterOverridePostDepsMutators) + ctx.PreDepsMutators(RegisterPreDepsMutators) + ctx.PostDepsMutators(RegisterPostDepsMutators) + ctx.PostDepsMutators(android.RegisterNeverallowMutator) + + config := android.TestArchConfig(buildDir, nil, bp, fs) + android.SetTestNeverallowRules(config, rules) + updatableBootJars := make([]string, 0, len(apexBootJars)) + for _, apexBootJar := range apexBootJars { + updatableBootJars = append(updatableBootJars, "myapex:"+apexBootJar) + } + config.TestProductVariables.UpdatableBootJars = updatableBootJars + + ctx.Register(config) + + _, errs := ctx.ParseBlueprintsFiles("Android.bp") + android.FailIfErrored(t, errs) + + _, errs = ctx.PrepareBuildActions(config) + if errmsg == "" { + android.FailIfErrored(t, errs) + } else if len(errs) > 0 { + android.FailIfNoMatchingErrors(t, errmsg, errs) + return + } else { + t.Fatalf("missing expected error %q (0 errors are returned)", errmsg) + } +} + +func TestApexPermittedPackagesRules(t *testing.T) { + testcases := []struct { + name string + expectedError string + bp string + bootJars []string + modulesPackages map[string][]string + }{ + + { + name: "Non-Bootclasspath apex jar not satisfying allowed module packages.", + expectedError: "", + bp: ` + java_library { + name: "bcp_lib1", + srcs: ["lib1/src/*.java"], + permitted_packages: ["foo.bar"], + apex_available: ["myapex"], + sdk_version: "none", + system_modules: "none", + } + java_library { + name: "nonbcp_lib2", + srcs: ["lib2/src/*.java"], + apex_available: ["myapex"], + permitted_packages: ["a.b"], + sdk_version: "none", + system_modules: "none", + } + apex { + name: "myapex", + key: "myapex.key", + java_libs: ["bcp_lib1", "nonbcp_lib2"], + }`, + bootJars: []string{"bcp_lib1"}, + modulesPackages: map[string][]string{ + "myapex": []string{ + "foo.bar", + }, + }, + }, + { + name: "Bootclasspath apex jar not satisfying allowed module packages.", + expectedError: `module "bcp_lib2" .* which is restricted because jars that are part of the myapex module may only allow these packages: foo.bar. Please jarjar or move code around.`, + bp: ` + java_library { + name: "bcp_lib1", + srcs: ["lib1/src/*.java"], + apex_available: ["myapex"], + permitted_packages: ["foo.bar"], + sdk_version: "none", + system_modules: "none", + } + java_library { + name: "bcp_lib2", + srcs: ["lib2/src/*.java"], + apex_available: ["myapex"], + permitted_packages: ["foo.bar", "bar.baz"], + sdk_version: "none", + system_modules: "none", + } + apex { + name: "myapex", + key: "myapex.key", + java_libs: ["bcp_lib1", "bcp_lib2"], + } + `, + bootJars: []string{"bcp_lib1", "bcp_lib2"}, + modulesPackages: map[string][]string{ + "myapex": []string{ + "foo.bar", + }, + }, + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + rules := createApexPermittedPackagesRules(tc.modulesPackages) + testApexPermittedPackagesRules(t, tc.expectedError, tc.bp, tc.bootJars, rules) + }) + } +} + +func TestTestFor(t *testing.T) { + ctx, _ := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + native_shared_libs: ["mylib", "myprivlib"], + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + cc_library { + name: "mylib", + srcs: ["mylib.cpp"], + system_shared_libs: [], + stl: "none", + stubs: { + versions: ["1"], + }, + apex_available: ["myapex"], + } + + cc_library { + name: "myprivlib", + srcs: ["mylib.cpp"], + system_shared_libs: [], + stl: "none", + apex_available: ["myapex"], + } + + + cc_test { + name: "mytest", + gtest: false, + srcs: ["mylib.cpp"], + system_shared_libs: [], + stl: "none", + shared_libs: ["mylib", "myprivlib"], + test_for: ["myapex"] + } + `) + + // the test 'mytest' is a test for the apex, therefore is linked to the + // actual implementation of mylib instead of its stub. + ldFlags := ctx.ModuleForTests("mytest", "android_arm64_armv8-a").Rule("ld").Args["libFlags"] + ensureContains(t, ldFlags, "mylib/android_arm64_armv8-a_shared/mylib.so") + ensureNotContains(t, ldFlags, "mylib/android_arm64_armv8-a_shared_1/mylib.so") +} + // TODO(jungjw): Move this to proptools func intPtr(i int) *int { return &i } func TestApexSet(t *testing.T) { - ctx := testApex(t, ` + ctx, config := testApex(t, ` apex_set { name: "myapex", set: "myapex.apks", @@ -1313,8 +5057,10 @@ func TestApexSet(t *testing.T) { } `, func(fs map[string][]byte, config android.Config) { config.TestProductVariables.Platform_sdk_version = intPtr(30) - config.TestProductVariables.DeviceArch = proptools.StringPtr("arm") - config.TestProductVariables.DeviceSecondaryArch = proptools.StringPtr("arm64") + config.Targets[android.Android] = []android.Target{ + {Os: android.Android, Arch: android.Arch{ArchType: android.Arm, ArchVariant: "armv7-a-neon", Abi: []string{"armeabi-v7a"}}}, + {Os: android.Android, Arch: android.Arch{ArchType: android.Arm64, ArchVariant: "armv8-a", Abi: []string{"arm64-v8a"}}}, + } }) m := ctx.ModuleForTests("myapex", "android_common") @@ -1331,10 +5077,28 @@ func TestApexSet(t *testing.T) { if actual != expected { t.Errorf("Unexpected abis parameter - expected %q vs actual %q", expected, actual) } + + a := m.Module().(*ApexSet) + expectedOverrides := []string{"foo"} + actualOverrides := android.AndroidMkEntriesForTest(t, config, "", a)[0].EntryMap["LOCAL_OVERRIDES_MODULES"] + if !reflect.DeepEqual(actualOverrides, expectedOverrides) { + t.Errorf("Incorrect LOCAL_OVERRIDES_MODULES - expected %q vs actual %q", expectedOverrides, actualOverrides) + } } func TestApexKeysTxt(t *testing.T) { - ctx := testApex(t, ` + ctx, _ := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + prebuilt_apex { name: "myapex", prefer: true, @@ -1354,14 +5118,76 @@ func TestApexKeysTxt(t *testing.T) { filename: "myapex_set.apex", overrides: ["myapex"], } - `, func(fs map[string][]byte, config android.Config) { - config.TestProductVariables.Platform_sdk_version = intPtr(30) - config.TestProductVariables.DeviceArch = proptools.StringPtr("arm") - config.TestProductVariables.DeviceSecondaryArch = proptools.StringPtr("arm64") - }) + `) apexKeysText := ctx.SingletonForTests("apex_keys_text") content := apexKeysText.MaybeDescription("apexkeys.txt").BuildParams.Args["content"] - ensureContains(t, content, `name="myapex_set.apex" public_key="PRESIGNED" private_key="PRESIGNED" container_certificate="PRESIGNED" container_private_key="PRESIGNED"`) - ensureContains(t, content, `name="myapex.apex" public_key="PRESIGNED" private_key="PRESIGNED" container_certificate="PRESIGNED" container_private_key="PRESIGNED"`) + ensureContains(t, content, `name="myapex_set.apex" public_key="PRESIGNED" private_key="PRESIGNED" container_certificate="PRESIGNED" container_private_key="PRESIGNED" partition="system"`) + ensureContains(t, content, `name="myapex.apex" public_key="PRESIGNED" private_key="PRESIGNED" container_certificate="PRESIGNED" container_private_key="PRESIGNED" partition="system"`) +} + +func TestAllowedFiles(t *testing.T) { + ctx, _ := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + apps: ["app"], + allowed_files: "allowed.txt", + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + android_app { + name: "app", + srcs: ["foo/bar/MyClass.java"], + package_name: "foo", + sdk_version: "none", + system_modules: "none", + apex_available: [ "myapex" ], + } + `, withFiles(map[string][]byte{ + "sub/Android.bp": []byte(` + override_apex { + name: "override_myapex", + base: "myapex", + apps: ["override_app"], + allowed_files: ":allowed", + } + // Overridable "path" property should be referenced indirectly + filegroup { + name: "allowed", + srcs: ["allowed.txt"], + } + override_android_app { + name: "override_app", + base: "app", + package_name: "bar", + } + `), + })) + + rule := ctx.ModuleForTests("myapex", "android_common_myapex_image").Rule("diffApexContentRule") + if expected, actual := "allowed.txt", rule.Args["allowed_files_file"]; expected != actual { + t.Errorf("allowed_files_file: expected %q but got %q", expected, actual) + } + + rule2 := ctx.ModuleForTests("myapex", "android_common_override_myapex_myapex_image").Rule("diffApexContentRule") + if expected, actual := "sub/allowed.txt", rule2.Args["allowed_files_file"]; expected != actual { + t.Errorf("allowed_files_file: expected %q but got %q", expected, actual) + } +} + +func TestMain(m *testing.M) { + run := func() int { + setUp() + defer tearDown() + + return m.Run() + } + + os.Exit(run()) } diff --git a/apex/builder.go b/apex/builder.go new file mode 100644 index 000000000..8573dc0d7 --- /dev/null +++ b/apex/builder.go @@ -0,0 +1,763 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apex + +import ( + "encoding/json" + "fmt" + "path/filepath" + "runtime" + "sort" + "strconv" + "strings" + + "android/soong/android" + "android/soong/java" + + "github.com/google/blueprint" + "github.com/google/blueprint/proptools" +) + +var ( + pctx = android.NewPackageContext("android/apex") +) + +func init() { + pctx.Import("android/soong/android") + pctx.Import("android/soong/java") + pctx.HostBinToolVariable("apexer", "apexer") + // ART minimal builds (using the master-art manifest) do not have the "frameworks/base" + // projects, and hence cannot built 'aapt2'. Use the SDK prebuilt instead. + hostBinToolVariableWithPrebuilt := func(name, prebuiltDir, tool string) { + pctx.VariableFunc(name, func(ctx android.PackageVarContext) string { + if !ctx.Config().FrameworksBaseDirExists(ctx) { + return filepath.Join(prebuiltDir, runtime.GOOS, "bin", tool) + } else { + return ctx.Config().HostToolPath(ctx, tool).String() + } + }) + } + hostBinToolVariableWithPrebuilt("aapt2", "prebuilts/sdk/tools", "aapt2") + pctx.HostBinToolVariable("avbtool", "avbtool") + pctx.HostBinToolVariable("e2fsdroid", "e2fsdroid") + pctx.HostBinToolVariable("merge_zips", "merge_zips") + pctx.HostBinToolVariable("mke2fs", "mke2fs") + pctx.HostBinToolVariable("resize2fs", "resize2fs") + pctx.HostBinToolVariable("sefcontext_compile", "sefcontext_compile") + pctx.HostBinToolVariable("soong_zip", "soong_zip") + pctx.HostBinToolVariable("zip2zip", "zip2zip") + pctx.HostBinToolVariable("zipalign", "zipalign") + pctx.HostBinToolVariable("jsonmodify", "jsonmodify") + pctx.HostBinToolVariable("conv_apex_manifest", "conv_apex_manifest") + pctx.HostBinToolVariable("extract_apks", "extract_apks") +} + +var ( + // Create a canned fs config file where all files and directories are + // by default set to (uid/gid/mode) = (1000/1000/0644) + // TODO(b/113082813) make this configurable using config.fs syntax + generateFsConfig = pctx.StaticRule("generateFsConfig", blueprint.RuleParams{ + Command: `( echo '/ 1000 1000 0755' ` + + `&& for i in ${ro_paths}; do echo "/$$i 1000 1000 0644"; done ` + + `&& for i in ${exec_paths}; do echo "/$$i 0 2000 0755"; done ` + + `&& ( tr ' ' '\n' <${out}.apklist | for i in ${apk_paths}; do read apk; echo "/$$i 0 2000 0755"; zipinfo -1 $$apk | sed "s:\(.*\):/$$i/\1 1000 1000 0644:"; done ) ) > ${out}`, + Description: "fs_config ${out}", + Rspfile: "$out.apklist", + RspfileContent: "$in", + }, "ro_paths", "exec_paths", "apk_paths") + + apexManifestRule = pctx.StaticRule("apexManifestRule", blueprint.RuleParams{ + Command: `rm -f $out && ${jsonmodify} $in ` + + `-a provideNativeLibs ${provideNativeLibs} ` + + `-a requireNativeLibs ${requireNativeLibs} ` + + `${opt} ` + + `-o $out`, + CommandDeps: []string{"${jsonmodify}"}, + Description: "prepare ${out}", + }, "provideNativeLibs", "requireNativeLibs", "opt") + + stripApexManifestRule = pctx.StaticRule("stripApexManifestRule", blueprint.RuleParams{ + Command: `rm -f $out && ${conv_apex_manifest} strip $in -o $out`, + CommandDeps: []string{"${conv_apex_manifest}"}, + Description: "strip ${in}=>${out}", + }) + + pbApexManifestRule = pctx.StaticRule("pbApexManifestRule", blueprint.RuleParams{ + Command: `rm -f $out && ${conv_apex_manifest} proto $in -o $out`, + CommandDeps: []string{"${conv_apex_manifest}"}, + Description: "convert ${in}=>${out}", + }) + + // TODO(b/113233103): make sure that file_contexts is sane, i.e., validate + // against the binary policy using sefcontext_compiler -p <policy>. + + // TODO(b/114327326): automate the generation of file_contexts + apexRule = pctx.StaticRule("apexRule", blueprint.RuleParams{ + Command: `rm -rf ${image_dir} && mkdir -p ${image_dir} && ` + + `(. ${out}.copy_commands) && ` + + `APEXER_TOOL_PATH=${tool_path} ` + + `${apexer} --force --manifest ${manifest} ` + + `--file_contexts ${file_contexts} ` + + `--canned_fs_config ${canned_fs_config} ` + + `--include_build_info ` + + `--payload_type image ` + + `--key ${key} ${opt_flags} ${image_dir} ${out} `, + CommandDeps: []string{"${apexer}", "${avbtool}", "${e2fsdroid}", "${merge_zips}", + "${mke2fs}", "${resize2fs}", "${sefcontext_compile}", + "${soong_zip}", "${zipalign}", "${aapt2}", "prebuilts/sdk/current/public/android.jar"}, + Rspfile: "${out}.copy_commands", + RspfileContent: "${copy_commands}", + Description: "APEX ${image_dir} => ${out}", + }, "tool_path", "image_dir", "copy_commands", "file_contexts", "canned_fs_config", "key", "opt_flags", "manifest") + + zipApexRule = pctx.StaticRule("zipApexRule", blueprint.RuleParams{ + Command: `rm -rf ${image_dir} && mkdir -p ${image_dir} && ` + + `(. ${out}.copy_commands) && ` + + `APEXER_TOOL_PATH=${tool_path} ` + + `${apexer} --force --manifest ${manifest} ` + + `--payload_type zip ` + + `${image_dir} ${out} `, + CommandDeps: []string{"${apexer}", "${merge_zips}", "${soong_zip}", "${zipalign}", "${aapt2}"}, + Rspfile: "${out}.copy_commands", + RspfileContent: "${copy_commands}", + Description: "ZipAPEX ${image_dir} => ${out}", + }, "tool_path", "image_dir", "copy_commands", "manifest") + + apexProtoConvertRule = pctx.AndroidStaticRule("apexProtoConvertRule", + blueprint.RuleParams{ + Command: `${aapt2} convert --output-format proto $in -o $out`, + CommandDeps: []string{"${aapt2}"}, + }) + + apexBundleRule = pctx.StaticRule("apexBundleRule", blueprint.RuleParams{ + Command: `${zip2zip} -i $in -o $out.base ` + + `apex_payload.img:apex/${abi}.img ` + + `apex_build_info.pb:apex/${abi}.build_info.pb ` + + `apex_manifest.json:root/apex_manifest.json ` + + `apex_manifest.pb:root/apex_manifest.pb ` + + `AndroidManifest.xml:manifest/AndroidManifest.xml ` + + `assets/NOTICE.html.gz:assets/NOTICE.html.gz &&` + + `${soong_zip} -o $out.config -C $$(dirname ${config}) -f ${config} && ` + + `${merge_zips} $out $out.base $out.config`, + CommandDeps: []string{"${zip2zip}", "${soong_zip}", "${merge_zips}"}, + Description: "app bundle", + }, "abi", "config") + + emitApexContentRule = pctx.StaticRule("emitApexContentRule", blueprint.RuleParams{ + Command: `rm -f ${out} && touch ${out} && (. ${out}.emit_commands)`, + Rspfile: "${out}.emit_commands", + RspfileContent: "${emit_commands}", + Description: "Emit APEX image content", + }, "emit_commands") + + diffApexContentRule = pctx.StaticRule("diffApexContentRule", blueprint.RuleParams{ + Command: `diff --unchanged-group-format='' \` + + `--changed-group-format='%<' \` + + `${image_content_file} ${allowed_files_file} || (` + + `echo -e "New unexpected files were added to ${apex_module_name}." ` + + ` "To fix the build run following command:" && ` + + `echo "system/apex/tools/update_allowed_list.sh ${allowed_files_file} ${image_content_file}" && ` + + `exit 1); touch ${out}`, + Description: "Diff ${image_content_file} and ${allowed_files_file}", + }, "image_content_file", "allowed_files_file", "apex_module_name") +) + +func (a *apexBundle) buildManifest(ctx android.ModuleContext, provideNativeLibs, requireNativeLibs []string) { + manifestSrc := android.PathForModuleSrc(ctx, proptools.StringDefault(a.properties.Manifest, "apex_manifest.json")) + + manifestJsonFullOut := android.PathForModuleOut(ctx, "apex_manifest_full.json") + + // put dependency({provide|require}NativeLibs) in apex_manifest.json + provideNativeLibs = android.SortedUniqueStrings(provideNativeLibs) + requireNativeLibs = android.SortedUniqueStrings(android.RemoveListFromList(requireNativeLibs, provideNativeLibs)) + + // apex name can be overridden + optCommands := []string{} + if a.properties.Apex_name != nil { + optCommands = append(optCommands, "-v name "+*a.properties.Apex_name) + } + + ctx.Build(pctx, android.BuildParams{ + Rule: apexManifestRule, + Input: manifestSrc, + Output: manifestJsonFullOut, + Args: map[string]string{ + "provideNativeLibs": strings.Join(provideNativeLibs, " "), + "requireNativeLibs": strings.Join(requireNativeLibs, " "), + "opt": strings.Join(optCommands, " "), + }, + }) + + if a.minSdkVersion(ctx) == android.SdkVersion_Android10 { + // b/143654022 Q apexd can't understand newly added keys in apex_manifest.json + // prepare stripped-down version so that APEX modules built from R+ can be installed to Q + a.manifestJsonOut = android.PathForModuleOut(ctx, "apex_manifest.json") + ctx.Build(pctx, android.BuildParams{ + Rule: stripApexManifestRule, + Input: manifestJsonFullOut, + Output: a.manifestJsonOut, + }) + } + + // from R+, protobuf binary format (.pb) is the standard format for apex_manifest + a.manifestPbOut = android.PathForModuleOut(ctx, "apex_manifest.pb") + ctx.Build(pctx, android.BuildParams{ + Rule: pbApexManifestRule, + Input: manifestJsonFullOut, + Output: a.manifestPbOut, + }) +} + +func (a *apexBundle) buildNoticeFiles(ctx android.ModuleContext, apexFileName string) android.NoticeOutputs { + var noticeFiles android.Paths + + a.walkPayloadDeps(ctx, func(ctx android.ModuleContext, from blueprint.Module, to android.ApexModule, externalDep bool) bool { + if externalDep { + // As soon as the dependency graph crosses the APEX boundary, don't go further. + return false + } + + notice := to.NoticeFile() + if notice.Valid() { + noticeFiles = append(noticeFiles, notice.Path()) + } + + return true + }) + + if len(noticeFiles) == 0 { + return android.NoticeOutputs{} + } + + return android.BuildNoticeOutput(ctx, a.installDir, apexFileName, android.SortedUniquePaths(noticeFiles)) +} + +func (a *apexBundle) buildInstalledFilesFile(ctx android.ModuleContext, builtApex android.Path, imageDir android.Path) android.OutputPath { + output := android.PathForModuleOut(ctx, "installed-files.txt") + rule := android.NewRuleBuilder() + rule.Command(). + Implicit(builtApex). + Text("(cd " + imageDir.String() + " ; "). + Text("find . \\( -type f -o -type l \\) -printf \"%s %p\\n\") "). + Text(" | sort -nr > "). + Output(output) + rule.Build(pctx, ctx, "installed-files."+a.Name(), "Installed files") + return output.OutputPath +} + +func (a *apexBundle) buildBundleConfig(ctx android.ModuleContext) android.OutputPath { + output := android.PathForModuleOut(ctx, "bundle_config.json") + + type ApkConfig struct { + Package_name string `json:"package_name"` + Apk_path string `json:"path"` + } + config := struct { + Compression struct { + Uncompressed_glob []string `json:"uncompressed_glob"` + } `json:"compression"` + Apex_config struct { + Apex_embedded_apk_config []ApkConfig `json:"apex_embedded_apk_config,omitempty"` + } `json:"apex_config,omitempty"` + }{} + + config.Compression.Uncompressed_glob = []string{ + "apex_payload.img", + "apex_manifest.*", + } + + // collect the manifest names and paths of android apps + // if their manifest names are overridden + for _, fi := range a.filesInfo { + if fi.class != app && fi.class != appSet { + continue + } + packageName := fi.overriddenPackageName + if packageName != "" { + config.Apex_config.Apex_embedded_apk_config = append( + config.Apex_config.Apex_embedded_apk_config, + ApkConfig{ + Package_name: packageName, + Apk_path: fi.Path(), + }) + } + } + + j, err := json.Marshal(config) + if err != nil { + panic(fmt.Errorf("error while marshalling to %q: %#v", output, err)) + } + + ctx.Build(pctx, android.BuildParams{ + Rule: android.WriteFile, + Output: output, + Description: "Bundle Config " + output.String(), + Args: map[string]string{ + "content": string(j), + }, + }) + + return output.OutputPath +} + +func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) { + var abis []string + for _, target := range ctx.MultiTargets() { + if len(target.Arch.Abi) > 0 { + abis = append(abis, target.Arch.Abi[0]) + } + } + + abis = android.FirstUniqueStrings(abis) + + apexType := a.properties.ApexType + suffix := apexType.suffix() + var implicitInputs []android.Path + unsignedOutputFile := android.PathForModuleOut(ctx, a.Name()+suffix+".unsigned") + + // TODO(jiyong): construct the copy rules using RuleBuilder + var copyCommands []string + for _, fi := range a.filesInfo { + destPath := android.PathForModuleOut(ctx, "image"+suffix, fi.Path()).String() + destPathDir := filepath.Dir(destPath) + if fi.class == appSet { + copyCommands = append(copyCommands, "rm -rf "+destPathDir) + } + copyCommands = append(copyCommands, "mkdir -p "+destPathDir) + if a.linkToSystemLib && fi.transitiveDep && fi.AvailableToPlatform() { + // TODO(jiyong): pathOnDevice should come from fi.module, not being calculated here + pathOnDevice := filepath.Join("/system", fi.Path()) + copyCommands = append(copyCommands, "ln -sfn "+pathOnDevice+" "+destPath) + } else { + if fi.class == appSet { + copyCommands = append(copyCommands, + fmt.Sprintf("unzip -qDD -d %s %s", destPathDir, fi.builtFile.String())) + } else { + copyCommands = append(copyCommands, "cp -f "+fi.builtFile.String()+" "+destPath) + } + implicitInputs = append(implicitInputs, fi.builtFile) + } + // create additional symlinks pointing the file inside the APEX + for _, symlinkPath := range fi.SymlinkPaths() { + symlinkDest := android.PathForModuleOut(ctx, "image"+suffix, symlinkPath).String() + copyCommands = append(copyCommands, "ln -sfn "+filepath.Base(destPath)+" "+symlinkDest) + } + } + + // TODO(jiyong): use RuleBuilder + var emitCommands []string + imageContentFile := android.PathForModuleOut(ctx, "content.txt") + emitCommands = append(emitCommands, "echo ./apex_manifest.pb >> "+imageContentFile.String()) + if a.minSdkVersion(ctx) == android.SdkVersion_Android10 { + emitCommands = append(emitCommands, "echo ./apex_manifest.json >> "+imageContentFile.String()) + } + for _, fi := range a.filesInfo { + emitCommands = append(emitCommands, "echo './"+fi.Path()+"' >> "+imageContentFile.String()) + } + emitCommands = append(emitCommands, "sort -o "+imageContentFile.String()+" "+imageContentFile.String()) + implicitInputs = append(implicitInputs, a.manifestPbOut) + + if a.overridableProperties.Allowed_files != nil { + ctx.Build(pctx, android.BuildParams{ + Rule: emitApexContentRule, + Implicits: implicitInputs, + Output: imageContentFile, + Description: "emit apex image content", + Args: map[string]string{ + "emit_commands": strings.Join(emitCommands, " && "), + }, + }) + implicitInputs = append(implicitInputs, imageContentFile) + allowedFilesFile := android.PathForModuleSrc(ctx, proptools.String(a.overridableProperties.Allowed_files)) + + phonyOutput := android.PathForModuleOut(ctx, a.Name()+"-diff-phony-output") + ctx.Build(pctx, android.BuildParams{ + Rule: diffApexContentRule, + Implicits: implicitInputs, + Output: phonyOutput, + Description: "diff apex image content", + Args: map[string]string{ + "allowed_files_file": allowedFilesFile.String(), + "image_content_file": imageContentFile.String(), + "apex_module_name": a.Name(), + }, + }) + + implicitInputs = append(implicitInputs, phonyOutput) + } + + outHostBinDir := android.PathForOutput(ctx, "host", ctx.Config().PrebuiltOS(), "bin").String() + prebuiltSdkToolsBinDir := filepath.Join("prebuilts", "sdk", "tools", runtime.GOOS, "bin") + + imageDir := android.PathForModuleOut(ctx, "image"+suffix) + if apexType == imageApex { + // files and dirs that will be created in APEX + var readOnlyPaths = []string{"apex_manifest.json", "apex_manifest.pb"} + var executablePaths []string // this also includes dirs + var extractedAppSetPaths android.Paths + var extractedAppSetDirs []string + for _, f := range a.filesInfo { + pathInApex := f.Path() + if f.installDir == "bin" || strings.HasPrefix(f.installDir, "bin/") { + executablePaths = append(executablePaths, pathInApex) + for _, s := range f.symlinks { + executablePaths = append(executablePaths, filepath.Join(f.installDir, s)) + } + } else if f.class == appSet { + extractedAppSetPaths = append(extractedAppSetPaths, f.builtFile) + extractedAppSetDirs = append(extractedAppSetDirs, f.installDir) + } else { + readOnlyPaths = append(readOnlyPaths, pathInApex) + } + dir := f.installDir + for !android.InList(dir, executablePaths) && dir != "" { + executablePaths = append(executablePaths, dir) + dir, _ = filepath.Split(dir) // move up to the parent + if len(dir) > 0 { + // remove trailing slash + dir = dir[:len(dir)-1] + } + } + } + sort.Strings(readOnlyPaths) + sort.Strings(executablePaths) + cannedFsConfig := android.PathForModuleOut(ctx, "canned_fs_config") + ctx.Build(pctx, android.BuildParams{ + Rule: generateFsConfig, + Output: cannedFsConfig, + Description: "generate fs config", + Inputs: extractedAppSetPaths, + Args: map[string]string{ + "ro_paths": strings.Join(readOnlyPaths, " "), + "exec_paths": strings.Join(executablePaths, " "), + "apk_paths": strings.Join(extractedAppSetDirs, " "), + }, + }) + + optFlags := []string{} + + // Additional implicit inputs. + implicitInputs = append(implicitInputs, cannedFsConfig, a.fileContexts, a.private_key_file, a.public_key_file) + optFlags = append(optFlags, "--pubkey "+a.public_key_file.String()) + + manifestPackageName := a.getOverrideManifestPackageName(ctx) + if manifestPackageName != "" { + optFlags = append(optFlags, "--override_apk_package_name "+manifestPackageName) + } + + if a.properties.AndroidManifest != nil { + androidManifestFile := android.PathForModuleSrc(ctx, proptools.String(a.properties.AndroidManifest)) + implicitInputs = append(implicitInputs, androidManifestFile) + optFlags = append(optFlags, "--android_manifest "+androidManifestFile.String()) + } + + targetSdkVersion := ctx.Config().DefaultAppTargetSdk() + // TODO(b/157078772): propagate min_sdk_version to apexer. + minSdkVersion := ctx.Config().DefaultAppTargetSdk() + + if a.minSdkVersion(ctx) == android.SdkVersion_Android10 { + minSdkVersion = strconv.Itoa(a.minSdkVersion(ctx)) + } + + if java.UseApiFingerprint(ctx) { + targetSdkVersion = ctx.Config().PlatformSdkCodename() + fmt.Sprintf(".$$(cat %s)", java.ApiFingerprintPath(ctx).String()) + implicitInputs = append(implicitInputs, java.ApiFingerprintPath(ctx)) + } + if java.UseApiFingerprint(ctx) { + minSdkVersion = ctx.Config().PlatformSdkCodename() + fmt.Sprintf(".$$(cat %s)", java.ApiFingerprintPath(ctx).String()) + implicitInputs = append(implicitInputs, java.ApiFingerprintPath(ctx)) + } + optFlags = append(optFlags, "--target_sdk_version "+targetSdkVersion) + optFlags = append(optFlags, "--min_sdk_version "+minSdkVersion) + + if a.overridableProperties.Logging_parent != "" { + optFlags = append(optFlags, "--logging_parent ", a.overridableProperties.Logging_parent) + } + + a.mergedNotices = a.buildNoticeFiles(ctx, a.Name()+suffix) + if a.mergedNotices.HtmlGzOutput.Valid() { + // If there's a NOTICE file, embed it as an asset file in the APEX. + implicitInputs = append(implicitInputs, a.mergedNotices.HtmlGzOutput.Path()) + optFlags = append(optFlags, "--assets_dir "+filepath.Dir(a.mergedNotices.HtmlGzOutput.String())) + } + + if ctx.ModuleDir() != "system/apex/apexd/apexd_testdata" && ctx.ModuleDir() != "system/apex/shim/build" && a.testOnlyShouldSkipHashtreeGeneration() { + ctx.PropertyErrorf("test_only_no_hashtree", "not available") + return + } + if a.minSdkVersion(ctx) > android.SdkVersion_Android10 || a.testOnlyShouldSkipHashtreeGeneration() { + // Apexes which are supposed to be installed in builtin dirs(/system, etc) + // don't need hashtree for activation. Therefore, by removing hashtree from + // apex bundle (filesystem image in it, to be specific), we can save storage. + optFlags = append(optFlags, "--no_hashtree") + } + + if a.testOnlyShouldSkipPayloadSign() { + optFlags = append(optFlags, "--unsigned_payload") + } + + if a.properties.Apex_name != nil { + // If apex_name is set, apexer can skip checking if key name matches with apex name. + // Note that apex_manifest is also mended. + optFlags = append(optFlags, "--do_not_check_keyname") + } + + if a.minSdkVersion(ctx) == android.SdkVersion_Android10 { + implicitInputs = append(implicitInputs, a.manifestJsonOut) + optFlags = append(optFlags, "--manifest_json "+a.manifestJsonOut.String()) + } + + ctx.Build(pctx, android.BuildParams{ + Rule: apexRule, + Implicits: implicitInputs, + Output: unsignedOutputFile, + Description: "apex (" + apexType.name() + ")", + Args: map[string]string{ + "tool_path": outHostBinDir + ":" + prebuiltSdkToolsBinDir, + "image_dir": imageDir.String(), + "copy_commands": strings.Join(copyCommands, " && "), + "manifest": a.manifestPbOut.String(), + "file_contexts": a.fileContexts.String(), + "canned_fs_config": cannedFsConfig.String(), + "key": a.private_key_file.String(), + "opt_flags": strings.Join(optFlags, " "), + }, + }) + + apexProtoFile := android.PathForModuleOut(ctx, a.Name()+".pb"+suffix) + bundleModuleFile := android.PathForModuleOut(ctx, a.Name()+suffix+"-base.zip") + a.bundleModuleFile = bundleModuleFile + + ctx.Build(pctx, android.BuildParams{ + Rule: apexProtoConvertRule, + Input: unsignedOutputFile, + Output: apexProtoFile, + Description: "apex proto convert", + }) + + bundleConfig := a.buildBundleConfig(ctx) + + ctx.Build(pctx, android.BuildParams{ + Rule: apexBundleRule, + Input: apexProtoFile, + Implicit: bundleConfig, + Output: a.bundleModuleFile, + Description: "apex bundle module", + Args: map[string]string{ + "abi": strings.Join(abis, "."), + "config": bundleConfig.String(), + }, + }) + } else { + ctx.Build(pctx, android.BuildParams{ + Rule: zipApexRule, + Implicits: implicitInputs, + Output: unsignedOutputFile, + Description: "apex (" + apexType.name() + ")", + Args: map[string]string{ + "tool_path": outHostBinDir + ":" + prebuiltSdkToolsBinDir, + "image_dir": imageDir.String(), + "copy_commands": strings.Join(copyCommands, " && "), + "manifest": a.manifestPbOut.String(), + }, + }) + } + + a.outputFile = android.PathForModuleOut(ctx, a.Name()+suffix) + rule := java.Signapk + args := map[string]string{ + "certificates": a.container_certificate_file.String() + " " + a.container_private_key_file.String(), + "flags": "-a 4096", //alignment + } + implicits := android.Paths{ + a.container_certificate_file, + a.container_private_key_file, + } + if ctx.Config().IsEnvTrue("RBE_SIGNAPK") { + rule = java.SignapkRE + args["implicits"] = strings.Join(implicits.Strings(), ",") + args["outCommaList"] = a.outputFile.String() + } + ctx.Build(pctx, android.BuildParams{ + Rule: rule, + Description: "signapk", + Output: a.outputFile, + Input: unsignedOutputFile, + Implicits: implicits, + Args: args, + }) + + // Install to $OUT/soong/{target,host}/.../apex + if a.installable() { + ctx.InstallFile(a.installDir, a.Name()+suffix, a.outputFile) + } + a.buildFilesInfo(ctx) + + // installed-files.txt is dist'ed + a.installedFilesFile = a.buildInstalledFilesFile(ctx, a.outputFile, imageDir) +} + +func (a *apexBundle) buildFlattenedApex(ctx android.ModuleContext) { + // Temporarily wrap the original `ctx` into a `flattenedApexContext` to have it + // reply true to `InstallBypassMake()` (thus making the call + // `android.PathForModuleInstall` below use `android.pathForInstallInMakeDir` + // instead of `android.PathForOutput`) to return the correct path to the flattened + // APEX (as its contents is installed by Make, not Soong). + factx := flattenedApexContext{ctx} + apexBundleName := a.Name() + a.outputFile = android.PathForModuleInstall(&factx, "apex", apexBundleName) + + if a.installable() { + installPath := android.PathForModuleInstall(ctx, "apex", apexBundleName) + devicePath := android.InstallPathToOnDevicePath(ctx, installPath) + addFlattenedFileContextsInfos(ctx, apexBundleName+":"+devicePath+":"+a.fileContexts.String()) + } + a.buildFilesInfo(ctx) +} + +func (a *apexBundle) setCertificateAndPrivateKey(ctx android.ModuleContext) { + if a.container_certificate_file == nil { + cert := String(a.properties.Certificate) + if cert == "" { + pem, key := ctx.Config().DefaultAppCertificate(ctx) + a.container_certificate_file = pem + a.container_private_key_file = key + } else { + defaultDir := ctx.Config().DefaultAppCertificateDir(ctx) + a.container_certificate_file = defaultDir.Join(ctx, cert+".x509.pem") + a.container_private_key_file = defaultDir.Join(ctx, cert+".pk8") + } + } +} + +func (a *apexBundle) buildFilesInfo(ctx android.ModuleContext) { + if a.installable() { + // For flattened APEX, do nothing but make sure that APEX manifest and apex_pubkey are also copied along + // with other ordinary files. + a.filesInfo = append(a.filesInfo, newApexFile(ctx, a.manifestPbOut, "apex_manifest.pb", ".", etc, nil)) + + // rename to apex_pubkey + copiedPubkey := android.PathForModuleOut(ctx, "apex_pubkey") + ctx.Build(pctx, android.BuildParams{ + Rule: android.Cp, + Input: a.public_key_file, + Output: copiedPubkey, + }) + a.filesInfo = append(a.filesInfo, newApexFile(ctx, copiedPubkey, "apex_pubkey", ".", etc, nil)) + + if a.properties.ApexType == flattenedApex { + apexBundleName := a.Name() + for _, fi := range a.filesInfo { + dir := filepath.Join("apex", apexBundleName, fi.installDir) + target := ctx.InstallFile(android.PathForModuleInstall(ctx, dir), fi.Stem(), fi.builtFile) + for _, sym := range fi.symlinks { + ctx.InstallSymlink(android.PathForModuleInstall(ctx, dir), sym, target) + } + } + } + } +} + +func (a *apexBundle) getOverrideManifestPackageName(ctx android.ModuleContext) string { + // For VNDK APEXes, check "com.android.vndk" in PRODUCT_MANIFEST_PACKAGE_NAME_OVERRIDES + // to see if it should be overridden because their <apex name> is dynamically generated + // according to its VNDK version. + if a.vndkApex { + overrideName, overridden := ctx.DeviceConfig().OverrideManifestPackageNameFor(vndkApexName) + if overridden { + return strings.Replace(*a.properties.Apex_name, vndkApexName, overrideName, 1) + } + return "" + } + if a.overridableProperties.Package_name != "" { + return a.overridableProperties.Package_name + } + manifestPackageName, overridden := ctx.DeviceConfig().OverrideManifestPackageNameFor(ctx.ModuleName()) + if overridden { + return manifestPackageName + } + return "" +} + +func (a *apexBundle) buildApexDependencyInfo(ctx android.ModuleContext) { + if !a.primaryApexType { + return + } + + if a.properties.IsCoverageVariant { + // Otherwise, we will have duplicated rules for coverage and + // non-coverage variants of the same APEX + return + } + + if ctx.Host() { + // No need to generate dependency info for host variant + return + } + + depInfos := android.DepNameToDepInfoMap{} + a.walkPayloadDeps(ctx, func(ctx android.ModuleContext, from blueprint.Module, to android.ApexModule, externalDep bool) bool { + if from.Name() == to.Name() { + // This can happen for cc.reuseObjTag. We are not interested in tracking this. + // As soon as the dependency graph crosses the APEX boundary, don't go further. + return !externalDep + } + + if info, exists := depInfos[to.Name()]; exists { + if !android.InList(from.Name(), info.From) { + info.From = append(info.From, from.Name()) + } + info.IsExternal = info.IsExternal && externalDep + depInfos[to.Name()] = info + } else { + toMinSdkVersion := "(no version)" + if m, ok := to.(interface{ MinSdkVersion() string }); ok { + if v := m.MinSdkVersion(); v != "" { + toMinSdkVersion = v + } + } + + depInfos[to.Name()] = android.ApexModuleDepInfo{ + To: to.Name(), + From: []string{from.Name()}, + IsExternal: externalDep, + MinSdkVersion: toMinSdkVersion, + } + } + + // As soon as the dependency graph crosses the APEX boundary, don't go further. + return !externalDep + }) + + a.ApexBundleDepsInfo.BuildDepsInfoLists(ctx, proptools.String(a.properties.Min_sdk_version), depInfos) + + ctx.Build(pctx, android.BuildParams{ + Rule: android.Phony, + Output: android.PathForPhony(ctx, a.Name()+"-deps-info"), + Inputs: []android.Path{ + a.ApexBundleDepsInfo.FullListPath(), + a.ApexBundleDepsInfo.FlatListPath(), + }, + }) +} + +func (a *apexBundle) buildLintReports(ctx android.ModuleContext) { + depSetsBuilder := java.NewLintDepSetBuilder() + for _, fi := range a.filesInfo { + depSetsBuilder.Transitive(fi.lintDepSets) + } + + a.lintReports = java.BuildModuleLintReportZips(ctx, depSetsBuilder.Build()) +} diff --git a/apex/key.go b/apex/key.go index ec33763dd..d2d5786b3 100644 --- a/apex/key.go +++ b/apex/key.go @@ -27,7 +27,7 @@ import ( var String = proptools.String func init() { - android.RegisterModuleType("apex_key", apexKeyFactory) + android.RegisterModuleType("apex_key", ApexKeyFactory) android.RegisterSingletonType("apex_keys_text", apexKeysTextFactory) } @@ -53,11 +53,10 @@ type apexKeyProperties struct { Installable *bool } -func apexKeyFactory() android.Module { +func ApexKeyFactory() android.Module { module := &apexKey{} module.AddProperties(&module.properties) - // This module is device-only - android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibCommon) + android.InitAndroidArchModule(module, android.HostAndDeviceDefault, android.MultilibCommon) return module } @@ -117,11 +116,11 @@ func (s *apexKeysText) GenerateBuildActions(ctx android.SingletonContext) { partition string } toString := func(e apexKeyEntry) string { - format := "name=%q public_key=%q private_key=%q container_certificate=%q container_private_key=%q\\n" + format := "name=%q public_key=%q private_key=%q container_certificate=%q container_private_key=%q partition=%q\\n" if e.presigned { - return fmt.Sprintf(format, e.name, "PRESIGNED", "PRESIGNED", "PRESIGNED", "PRESIGNED") + return fmt.Sprintf(format, e.name, "PRESIGNED", "PRESIGNED", "PRESIGNED", "PRESIGNED", e.partition) } else { - return fmt.Sprintf(format, e.name, e.public_key, e.private_key, e.container_certificate, e.container_private_key) + return fmt.Sprintf(format, e.name, e.public_key, e.private_key, e.container_certificate, e.container_private_key, e.partition) } } diff --git a/apex/prebuilt.go b/apex/prebuilt.go new file mode 100644 index 000000000..37457e921 --- /dev/null +++ b/apex/prebuilt.go @@ -0,0 +1,390 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apex + +import ( + "fmt" + "strconv" + "strings" + + "android/soong/android" + "android/soong/java" + + "github.com/google/blueprint" + + "github.com/google/blueprint/proptools" +) + +var ( + extractMatchingApex = pctx.StaticRule( + "extractMatchingApex", + blueprint.RuleParams{ + Command: `rm -rf "$out" && ` + + `${extract_apks} -o "${out}" -allow-prereleased=${allow-prereleased} ` + + `-sdk-version=${sdk-version} -abis=${abis} -screen-densities=all -extract-single ` + + `${in}`, + CommandDeps: []string{"${extract_apks}"}, + }, + "abis", "allow-prereleased", "sdk-version") +) + +type prebuilt interface { + isForceDisabled() bool + InstallFilename() string +} + +type prebuiltCommon struct { + prebuilt android.Prebuilt + properties prebuiltCommonProperties +} + +type prebuiltCommonProperties struct { + ForceDisable bool `blueprint:"mutated"` +} + +func (p *prebuiltCommon) Prebuilt() *android.Prebuilt { + return &p.prebuilt +} + +func (p *prebuiltCommon) isForceDisabled() bool { + return p.properties.ForceDisable +} + +func (p *prebuiltCommon) checkForceDisable(ctx android.ModuleContext) bool { + // If the device is configured to use flattened APEX, force disable the prebuilt because + // the prebuilt is a non-flattened one. + forceDisable := ctx.Config().FlattenApex() + + // Force disable the prebuilts when we are doing unbundled build. We do unbundled build + // to build the prebuilts themselves. + forceDisable = forceDisable || ctx.Config().UnbundledBuild() + + // Force disable the prebuilts when coverage is enabled. + forceDisable = forceDisable || ctx.DeviceConfig().NativeCoverageEnabled() + forceDisable = forceDisable || ctx.Config().IsEnvTrue("EMMA_INSTRUMENT") + + // b/137216042 don't use prebuilts when address sanitizer is on + forceDisable = forceDisable || android.InList("address", ctx.Config().SanitizeDevice()) || + android.InList("hwaddress", ctx.Config().SanitizeDevice()) + + if forceDisable && p.prebuilt.SourceExists() { + p.properties.ForceDisable = true + return true + } + return false +} + +type Prebuilt struct { + android.ModuleBase + prebuiltCommon + + properties PrebuiltProperties + + inputApex android.Path + installDir android.InstallPath + installFilename string + outputApex android.WritablePath + + // list of commands to create symlinks for backward compatibility. + // these commands will be attached as LOCAL_POST_INSTALL_CMD + compatSymlinks []string +} + +type PrebuiltProperties struct { + // the path to the prebuilt .apex file to import. + Source string `blueprint:"mutated"` + + Src *string + Arch struct { + Arm struct { + Src *string + } + Arm64 struct { + Src *string + } + X86 struct { + Src *string + } + X86_64 struct { + Src *string + } + } + + Installable *bool + // Optional name for the installed apex. If unspecified, name of the + // module is used as the file name + Filename *string + + // Names of modules to be overridden. Listed modules can only be other binaries + // (in Make or Soong). + // This does not completely prevent installation of the overridden binaries, but if both + // binaries would be installed by default (in PRODUCT_PACKAGES) the other binary will be removed + // from PRODUCT_PACKAGES. + Overrides []string +} + +func (p *Prebuilt) installable() bool { + return p.properties.Installable == nil || proptools.Bool(p.properties.Installable) +} + +func (p *Prebuilt) OutputFiles(tag string) (android.Paths, error) { + switch tag { + case "": + return android.Paths{p.outputApex}, nil + default: + return nil, fmt.Errorf("unsupported module reference tag %q", tag) + } +} + +func (p *Prebuilt) InstallFilename() string { + return proptools.StringDefault(p.properties.Filename, p.BaseModuleName()+imageApexSuffix) +} + +func (p *Prebuilt) Name() string { + return p.prebuiltCommon.prebuilt.Name(p.ModuleBase.Name()) +} + +// prebuilt_apex imports an `.apex` file into the build graph as if it was built with apex. +func PrebuiltFactory() android.Module { + module := &Prebuilt{} + module.AddProperties(&module.properties) + android.InitSingleSourcePrebuiltModule(module, &module.properties, "Source") + android.InitAndroidMultiTargetsArchModule(module, android.DeviceSupported, android.MultilibCommon) + return module +} + +func (p *Prebuilt) DepsMutator(ctx android.BottomUpMutatorContext) { + // This is called before prebuilt_select and prebuilt_postdeps mutators + // The mutators requires that src to be set correctly for each arch so that + // arch variants are disabled when src is not provided for the arch. + if len(ctx.MultiTargets()) != 1 { + ctx.ModuleErrorf("compile_multilib shouldn't be \"both\" for prebuilt_apex") + return + } + var src string + switch ctx.MultiTargets()[0].Arch.ArchType { + case android.Arm: + src = String(p.properties.Arch.Arm.Src) + case android.Arm64: + src = String(p.properties.Arch.Arm64.Src) + case android.X86: + src = String(p.properties.Arch.X86.Src) + case android.X86_64: + src = String(p.properties.Arch.X86_64.Src) + default: + ctx.ModuleErrorf("prebuilt_apex does not support %q", ctx.MultiTargets()[0].Arch.String()) + return + } + if src == "" { + src = String(p.properties.Src) + } + p.properties.Source = src +} + +func (p *Prebuilt) GenerateAndroidBuildActions(ctx android.ModuleContext) { + // TODO(jungjw): Check the key validity. + p.inputApex = p.Prebuilt().SingleSourcePath(ctx) + p.installDir = android.PathForModuleInstall(ctx, "apex") + p.installFilename = p.InstallFilename() + if !strings.HasSuffix(p.installFilename, imageApexSuffix) { + ctx.ModuleErrorf("filename should end in %s for prebuilt_apex", imageApexSuffix) + } + p.outputApex = android.PathForModuleOut(ctx, p.installFilename) + ctx.Build(pctx, android.BuildParams{ + Rule: android.Cp, + Input: p.inputApex, + Output: p.outputApex, + }) + + if p.prebuiltCommon.checkForceDisable(ctx) { + p.SkipInstall() + return + } + + if p.installable() { + ctx.InstallFile(p.installDir, p.installFilename, p.inputApex) + } + + // in case that prebuilt_apex replaces source apex (using prefer: prop) + p.compatSymlinks = makeCompatSymlinks(p.BaseModuleName(), ctx) + // or that prebuilt_apex overrides other apexes (using overrides: prop) + for _, overridden := range p.properties.Overrides { + p.compatSymlinks = append(p.compatSymlinks, makeCompatSymlinks(overridden, ctx)...) + } +} + +func (p *Prebuilt) AndroidMkEntries() []android.AndroidMkEntries { + return []android.AndroidMkEntries{android.AndroidMkEntries{ + Class: "ETC", + OutputFile: android.OptionalPathForPath(p.inputApex), + Include: "$(BUILD_PREBUILT)", + ExtraEntries: []android.AndroidMkExtraEntriesFunc{ + func(entries *android.AndroidMkEntries) { + entries.SetString("LOCAL_MODULE_PATH", p.installDir.ToMakePath().String()) + entries.SetString("LOCAL_MODULE_STEM", p.installFilename) + entries.SetBoolIfTrue("LOCAL_UNINSTALLABLE_MODULE", !p.installable()) + entries.AddStrings("LOCAL_OVERRIDES_MODULES", p.properties.Overrides...) + if len(p.compatSymlinks) > 0 { + entries.SetString("LOCAL_POST_INSTALL_CMD", strings.Join(p.compatSymlinks, " && ")) + } + }, + }, + }} +} + +type ApexSet struct { + android.ModuleBase + prebuiltCommon + + properties ApexSetProperties + + installDir android.InstallPath + installFilename string + outputApex android.WritablePath + + // list of commands to create symlinks for backward compatibility. + // these commands will be attached as LOCAL_POST_INSTALL_CMD + compatSymlinks []string + + hostRequired []string + postInstallCommands []string +} + +type ApexSetProperties struct { + // the .apks file path that contains prebuilt apex files to be extracted. + Set *string + + // whether the extracted apex file installable. + Installable *bool + + // optional name for the installed apex. If unspecified, name of the + // module is used as the file name + Filename *string + + // names of modules to be overridden. Listed modules can only be other binaries + // (in Make or Soong). + // This does not completely prevent installation of the overridden binaries, but if both + // binaries would be installed by default (in PRODUCT_PACKAGES) the other binary will be removed + // from PRODUCT_PACKAGES. + Overrides []string + + // apexes in this set use prerelease SDK version + Prerelease *bool +} + +func (a *ApexSet) installable() bool { + return a.properties.Installable == nil || proptools.Bool(a.properties.Installable) +} + +func (a *ApexSet) InstallFilename() string { + return proptools.StringDefault(a.properties.Filename, a.BaseModuleName()+imageApexSuffix) +} + +func (a *ApexSet) Name() string { + return a.prebuiltCommon.prebuilt.Name(a.ModuleBase.Name()) +} + +func (a *ApexSet) Overrides() []string { + return a.properties.Overrides +} + +// prebuilt_apex imports an `.apex` file into the build graph as if it was built with apex. +func apexSetFactory() android.Module { + module := &ApexSet{} + module.AddProperties(&module.properties) + android.InitSingleSourcePrebuiltModule(module, &module.properties, "Set") + android.InitAndroidMultiTargetsArchModule(module, android.DeviceSupported, android.MultilibCommon) + return module +} + +func (a *ApexSet) GenerateAndroidBuildActions(ctx android.ModuleContext) { + a.installFilename = a.InstallFilename() + if !strings.HasSuffix(a.installFilename, imageApexSuffix) { + ctx.ModuleErrorf("filename should end in %s for apex_set", imageApexSuffix) + } + + apexSet := a.prebuiltCommon.prebuilt.SingleSourcePath(ctx) + a.outputApex = android.PathForModuleOut(ctx, a.installFilename) + ctx.Build(pctx, + android.BuildParams{ + Rule: extractMatchingApex, + Description: "Extract an apex from an apex set", + Inputs: android.Paths{apexSet}, + Output: a.outputApex, + Args: map[string]string{ + "abis": strings.Join(java.SupportedAbis(ctx), ","), + "allow-prereleased": strconv.FormatBool(proptools.Bool(a.properties.Prerelease)), + "sdk-version": ctx.Config().PlatformSdkVersion(), + }, + }) + + if a.prebuiltCommon.checkForceDisable(ctx) { + a.SkipInstall() + return + } + + a.installDir = android.PathForModuleInstall(ctx, "apex") + if a.installable() { + ctx.InstallFile(a.installDir, a.installFilename, a.outputApex) + } + + // in case that apex_set replaces source apex (using prefer: prop) + a.compatSymlinks = makeCompatSymlinks(a.BaseModuleName(), ctx) + // or that apex_set overrides other apexes (using overrides: prop) + for _, overridden := range a.properties.Overrides { + a.compatSymlinks = append(a.compatSymlinks, makeCompatSymlinks(overridden, ctx)...) + } + + if ctx.Config().InstallExtraFlattenedApexes() { + // flattened apex should be in /system_ext/apex + flattenedApexDir := android.PathForModuleInstall(&systemExtContext{ctx}, "apex", a.BaseModuleName()) + a.postInstallCommands = append(a.postInstallCommands, + fmt.Sprintf("$(HOST_OUT_EXECUTABLES)/deapexer --debugfs_path $(HOST_OUT_EXECUTABLES)/debugfs extract %s %s", + a.outputApex.String(), + flattenedApexDir.ToMakePath().String(), + )) + a.hostRequired = []string{"deapexer", "debugfs"} + } +} + +type systemExtContext struct { + android.ModuleContext +} + +func (*systemExtContext) SystemExtSpecific() bool { + return true +} + +func (a *ApexSet) AndroidMkEntries() []android.AndroidMkEntries { + return []android.AndroidMkEntries{android.AndroidMkEntries{ + Class: "ETC", + OutputFile: android.OptionalPathForPath(a.outputApex), + Include: "$(BUILD_PREBUILT)", + Host_required: a.hostRequired, + ExtraEntries: []android.AndroidMkExtraEntriesFunc{ + func(entries *android.AndroidMkEntries) { + entries.SetString("LOCAL_MODULE_PATH", a.installDir.ToMakePath().String()) + entries.SetString("LOCAL_MODULE_STEM", a.installFilename) + entries.SetBoolIfTrue("LOCAL_UNINSTALLABLE_MODULE", !a.installable()) + entries.AddStrings("LOCAL_OVERRIDES_MODULES", a.properties.Overrides...) + postInstallCommands := append([]string{}, a.postInstallCommands...) + postInstallCommands = append(postInstallCommands, a.compatSymlinks...) + if len(postInstallCommands) > 0 { + entries.SetString("LOCAL_POST_INSTALL_CMD", strings.Join(postInstallCommands, " && ")) + } + }, + }, + }} +} diff --git a/apex/vndk.go b/apex/vndk.go new file mode 100644 index 000000000..5cc0e2a43 --- /dev/null +++ b/apex/vndk.go @@ -0,0 +1,166 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apex + +import ( + "path/filepath" + "strconv" + "strings" + "sync" + + "android/soong/android" + "android/soong/cc" + + "github.com/google/blueprint/proptools" +) + +const ( + vndkApexName = "com.android.vndk" + vndkApexNamePrefix = vndkApexName + ".v" +) + +// apex_vndk creates a special variant of apex modules which contains only VNDK libraries. +// If `vndk_version` is specified, the VNDK libraries of the specified VNDK version are gathered automatically. +// If not specified, then the "current" versions are gathered. +func vndkApexBundleFactory() android.Module { + bundle := newApexBundle() + bundle.vndkApex = true + bundle.AddProperties(&bundle.vndkProperties) + android.AddLoadHook(bundle, func(ctx android.LoadHookContext) { + ctx.AppendProperties(&struct { + Compile_multilib *string + }{ + proptools.StringPtr("both"), + }) + }) + return bundle +} + +func (a *apexBundle) vndkVersion(config android.DeviceConfig) string { + vndkVersion := proptools.StringDefault(a.vndkProperties.Vndk_version, "current") + if vndkVersion == "current" { + vndkVersion = config.PlatformVndkVersion() + } + return vndkVersion +} + +type apexVndkProperties struct { + // Indicates VNDK version of which this VNDK APEX bundles VNDK libs. Default is Platform VNDK Version. + Vndk_version *string +} + +var ( + vndkApexListKey = android.NewOnceKey("vndkApexList") + vndkApexListMutex sync.Mutex +) + +func vndkApexList(config android.Config) map[string]string { + return config.Once(vndkApexListKey, func() interface{} { + return map[string]string{} + }).(map[string]string) +} + +func apexVndkMutator(mctx android.TopDownMutatorContext) { + if ab, ok := mctx.Module().(*apexBundle); ok && ab.vndkApex { + if ab.IsNativeBridgeSupported() { + mctx.PropertyErrorf("native_bridge_supported", "%q doesn't support native bridge binary.", mctx.ModuleType()) + } + + vndkVersion := ab.vndkVersion(mctx.DeviceConfig()) + // Ensure VNDK APEX mount point is formatted as com.android.vndk.v### + ab.properties.Apex_name = proptools.StringPtr(vndkApexNamePrefix + vndkVersion) + + // vndk_version should be unique + vndkApexListMutex.Lock() + defer vndkApexListMutex.Unlock() + vndkApexList := vndkApexList(mctx.Config()) + if other, ok := vndkApexList[vndkVersion]; ok { + mctx.PropertyErrorf("vndk_version", "%v is already defined in %q", vndkVersion, other) + } + vndkApexList[vndkVersion] = mctx.ModuleName() + } +} + +func apexVndkDepsMutator(mctx android.BottomUpMutatorContext) { + if m, ok := mctx.Module().(*cc.Module); ok && cc.IsForVndkApex(mctx, m) { + vndkVersion := m.VndkVersion() + // For VNDK-Lite device, we gather core-variants of VNDK-Sp libraries, which doesn't have VNDK version defined + if vndkVersion == "" { + vndkVersion = mctx.DeviceConfig().PlatformVndkVersion() + } + vndkApexList := vndkApexList(mctx.Config()) + if vndkApex, ok := vndkApexList[vndkVersion]; ok { + mctx.AddReverseDependency(mctx.Module(), sharedLibTag, vndkApex) + } + } else if a, ok := mctx.Module().(*apexBundle); ok && a.vndkApex { + vndkVersion := proptools.StringDefault(a.vndkProperties.Vndk_version, "current") + mctx.AddDependency(mctx.Module(), prebuiltTag, cc.VndkLibrariesTxtModules(vndkVersion)...) + } +} + +// name is module.BaseModuleName() which is used as LOCAL_MODULE_NAME and also LOCAL_OVERRIDES_* +func makeCompatSymlinks(name string, ctx android.ModuleContext) (symlinks []string) { + // small helper to add symlink commands + addSymlink := func(target, dir, linkName string) { + link := filepath.Join(dir, linkName) + symlinks = append(symlinks, "mkdir -p "+dir+" && rm -rf "+link+" && ln -sf "+target+" "+link) + } + + // TODO(b/142911355): [VNDK APEX] Fix hard-coded references to /system/lib/vndk + // When all hard-coded references are fixed, remove symbolic links + // Note that we should keep following symlinks for older VNDKs (<=29) + // Since prebuilt vndk libs still depend on system/lib/vndk path + if strings.HasPrefix(name, vndkApexNamePrefix) { + vndkVersion := strings.TrimPrefix(name, vndkApexNamePrefix) + if numVer, err := strconv.Atoi(vndkVersion); err != nil { + ctx.ModuleErrorf("apex_vndk should be named as %v<ver:number>: %s", vndkApexNamePrefix, name) + return + } else if numVer > android.SdkVersion_Android10 { + return + } + // the name of vndk apex is formatted "com.android.vndk.v" + version + apexName := vndkApexNamePrefix + vndkVersion + if ctx.Config().Android64() { + addSymlink("/apex/"+apexName+"/lib64", "$(TARGET_OUT)/lib64", "vndk-sp-"+vndkVersion) + addSymlink("/apex/"+apexName+"/lib64", "$(TARGET_OUT)/lib64", "vndk-"+vndkVersion) + } + if !ctx.Config().Android64() || ctx.DeviceConfig().DeviceSecondaryArch() != "" { + addSymlink("/apex/"+apexName+"/lib", "$(TARGET_OUT)/lib", "vndk-sp-"+vndkVersion) + addSymlink("/apex/"+apexName+"/lib", "$(TARGET_OUT)/lib", "vndk-"+vndkVersion) + } + return + } + + // http://b/121248172 - create a link from /system/usr/icu to + // /apex/com.android.i18n/etc/icu so that apps can find the ICU .dat file. + // A symlink can't overwrite a directory and the /system/usr/icu directory once + // existed so the required structure must be created whatever we find. + if name == "com.android.i18n" { + addSymlink("/apex/com.android.i18n/etc/icu", "$(TARGET_OUT)/usr", "icu") + return + } + + // TODO(b/124106384): Clean up compat symlinks for ART binaries. + if strings.HasPrefix(name, "com.android.art.") { + addSymlink("/apex/com.android.art/bin/dalvikvm", "$(TARGET_OUT)/bin", "dalvikvm") + dex2oat := "dex2oat32" + if ctx.Config().Android64() { + dex2oat = "dex2oat64" + } + addSymlink("/apex/com.android.art/bin/"+dex2oat, "$(TARGET_OUT)/bin", "dex2oat") + return + } + return +} diff --git a/apex/vndk_test.go b/apex/vndk_test.go new file mode 100644 index 000000000..8557faee1 --- /dev/null +++ b/apex/vndk_test.go @@ -0,0 +1,168 @@ +package apex + +import ( + "testing" + + "github.com/google/blueprint/proptools" + + "android/soong/android" +) + +func TestVndkApexForVndkLite(t *testing.T) { + ctx, _ := testApex(t, ` + apex_vndk { + name: "myapex", + key: "myapex.key", + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + cc_library { + name: "libvndk", + srcs: ["mylib.cpp"], + vendor_available: true, + vndk: { + enabled: true, + }, + system_shared_libs: [], + stl: "none", + apex_available: [ "myapex" ], + } + + cc_library { + name: "libvndksp", + srcs: ["mylib.cpp"], + vendor_available: true, + vndk: { + enabled: true, + support_system_process: true, + }, + system_shared_libs: [], + stl: "none", + apex_available: [ "myapex" ], + } + `+vndkLibrariesTxtFiles("current"), func(fs map[string][]byte, config android.Config) { + config.TestProductVariables.DeviceVndkVersion = proptools.StringPtr("") + }) + // VNDK-Lite contains only core variants of VNDK-Sp libraries + ensureExactContents(t, ctx, "myapex", "android_common_image", []string{ + "lib/libvndksp.so", + "lib/libc++.so", + "lib64/libvndksp.so", + "lib64/libc++.so", + "etc/llndk.libraries.VER.txt", + "etc/vndkcore.libraries.VER.txt", + "etc/vndksp.libraries.VER.txt", + "etc/vndkprivate.libraries.VER.txt", + }) +} + +func TestVndkApexUsesVendorVariant(t *testing.T) { + bp := ` + apex_vndk { + name: "myapex", + key: "mykey", + } + apex_key { + name: "mykey", + } + cc_library { + name: "libfoo", + vendor_available: true, + vndk: { + enabled: true, + }, + system_shared_libs: [], + stl: "none", + notice: "custom_notice", + } + ` + vndkLibrariesTxtFiles("current") + + ensureFileSrc := func(t *testing.T, files []fileInApex, path, src string) { + t.Helper() + for _, f := range files { + if f.path == path { + ensureContains(t, f.src, src) + return + } + } + t.Fail() + } + + t.Run("VNDK lib doesn't have an apex variant", func(t *testing.T) { + ctx, _ := testApex(t, bp) + + // libfoo doesn't have apex variants + for _, variant := range ctx.ModuleVariantsForTests("libfoo") { + ensureNotContains(t, variant, "_myapex") + } + + // VNDK APEX doesn't create apex variant + files := getFiles(t, ctx, "myapex", "android_common_image") + ensureFileSrc(t, files, "lib/libfoo.so", "libfoo/android_vendor.VER_arm_armv7-a-neon_shared/libfoo.so") + }) + + t.Run("VNDK APEX gathers only vendor variants even if product variants are available", func(t *testing.T) { + ctx, _ := testApex(t, bp, func(fs map[string][]byte, config android.Config) { + // Now product variant is available + config.TestProductVariables.ProductVndkVersion = proptools.StringPtr("current") + }) + + files := getFiles(t, ctx, "myapex", "android_common_image") + ensureFileSrc(t, files, "lib/libfoo.so", "libfoo/android_vendor.VER_arm_armv7-a-neon_shared/libfoo.so") + }) + + t.Run("VNDK APEX supports coverage variants", func(t *testing.T) { + ctx, _ := testApex(t, bp+` + cc_library { + name: "libprofile-extras", + vendor_available: true, + recovery_available: true, + native_coverage: false, + system_shared_libs: [], + stl: "none", + notice: "custom_notice", + } + cc_library { + name: "libprofile-clang-extras", + vendor_available: true, + recovery_available: true, + native_coverage: false, + system_shared_libs: [], + stl: "none", + notice: "custom_notice", + } + cc_library { + name: "libprofile-extras_ndk", + vendor_available: true, + native_coverage: false, + system_shared_libs: [], + stl: "none", + notice: "custom_notice", + sdk_version: "current", + } + cc_library { + name: "libprofile-clang-extras_ndk", + vendor_available: true, + native_coverage: false, + system_shared_libs: [], + stl: "none", + notice: "custom_notice", + sdk_version: "current", + } + `, func(fs map[string][]byte, config android.Config) { + config.TestProductVariables.GcovCoverage = proptools.BoolPtr(true) + config.TestProductVariables.Native_coverage = proptools.BoolPtr(true) + }) + + files := getFiles(t, ctx, "myapex", "android_common_image") + ensureFileSrc(t, files, "lib/libfoo.so", "libfoo/android_vendor.VER_arm_armv7-a-neon_shared/libfoo.so") + + files = getFiles(t, ctx, "myapex", "android_common_cov_image") + ensureFileSrc(t, files, "lib/libfoo.so", "libfoo/android_vendor.VER_arm_armv7-a-neon_shared_cov/libfoo.so") + }) +} diff --git a/bpf/Android.bp b/bpf/Android.bp index 7bd4d4431..882cd8afe 100644 --- a/bpf/Android.bp +++ b/bpf/Android.bp @@ -21,10 +21,14 @@ bootstrap_go_package { "blueprint", "blueprint-proptools", "soong-android", + "soong-cc", "soong-cc-config", ], srcs: [ "bpf.go", ], + testSrcs: [ + "bpf_test.go", + ], pluginFor: ["soong_build"], } diff --git a/bpf/bpf.go b/bpf/bpf.go index 0047636f0..59d1502ff 100644 --- a/bpf/bpf.go +++ b/bpf/bpf.go @@ -33,7 +33,7 @@ func init() { var ( pctx = android.NewPackageContext("android/soong/bpf") - cc = pctx.AndroidRemoteStaticRule("cc", android.RemoteRuleSupports{Goma: true}, + ccRule = pctx.AndroidRemoteStaticRule("ccRule", android.RemoteRuleSupports{Goma: true}, blueprint.RuleParams{ Depfile: "${out}.d", Deps: blueprint.DepsGCC, @@ -66,6 +66,8 @@ func (bpf *bpf) GenerateAndroidBuildActions(ctx android.ModuleContext) { // The architecture doesn't matter here, but asm/types.h is included by linux/types.h. "-isystem bionic/libc/kernel/uapi/asm-arm64", "-isystem bionic/libc/kernel/android/uapi", + // TODO(b/149785767): only give access to specific file with AID_* constants + "-I system/core/libcutils/include", "-I system/bpf/progs/include", "-I " + ctx.ModuleDir(), } @@ -82,7 +84,7 @@ func (bpf *bpf) GenerateAndroidBuildActions(ctx android.ModuleContext) { obj := android.ObjPathWithExt(ctx, "", src, "o") ctx.Build(pctx, android.BuildParams{ - Rule: cc, + Rule: ccRule, Input: src, Output: obj, Args: map[string]string{ @@ -122,13 +124,18 @@ func (bpf *bpf) AndroidMk() android.AndroidMkData { } } -// Implements SourceFileProducer interface so that the obj output can be used in the data property +// Implements OutputFileFileProducer interface so that the obj output can be used in the data property // of other modules. -func (bpf *bpf) Srcs() android.Paths { - return bpf.objs +func (bpf *bpf) OutputFiles(tag string) (android.Paths, error) { + switch tag { + case "": + return bpf.objs, nil + default: + return nil, fmt.Errorf("unsupported module reference tag %q", tag) + } } -var _ android.SourceFileProducer = (*bpf)(nil) +var _ android.OutputFileProducer = (*bpf)(nil) func bpfFactory() android.Module { module := &bpf{} diff --git a/bpf/bpf_test.go b/bpf/bpf_test.go index 1d53e4138..eeca05771 100644 --- a/bpf/bpf_test.go +++ b/bpf/bpf_test.go @@ -20,7 +20,7 @@ import ( "testing" "android/soong/android" - cc2 "android/soong/cc" + "android/soong/cc" ) var buildDir string @@ -48,122 +48,24 @@ func TestMain(m *testing.M) { os.Exit(run()) } -func testContext(bp string) *android.TestContext { - ctx := android.NewTestArchContext() - ctx.RegisterModuleType("bpf", android.ModuleFactoryAdaptor(bpfFactory)) - ctx.RegisterModuleType("cc_test", android.ModuleFactoryAdaptor(cc2.TestFactory)) - ctx.RegisterModuleType("cc_library", android.ModuleFactoryAdaptor(cc2.LibraryFactory)) - ctx.RegisterModuleType("cc_library_static", android.ModuleFactoryAdaptor(cc2.LibraryStaticFactory)) - ctx.RegisterModuleType("cc_object", android.ModuleFactoryAdaptor(cc2.ObjectFactory)) - ctx.RegisterModuleType("toolchain_library", android.ModuleFactoryAdaptor(cc2.ToolchainLibraryFactory)) - ctx.PreDepsMutators(func(ctx android.RegisterMutatorsContext) { - ctx.BottomUp("link", cc2.LinkageMutator).Parallel() - }) - ctx.Register() - - // Add some modules that are required by the compiler and/or linker - bp = bp + ` - toolchain_library { - name: "libatomic", - vendor_available: true, - recovery_available: true, - src: "", - } - - toolchain_library { - name: "libclang_rt.builtins-arm-android", - vendor_available: true, - recovery_available: true, - src: "", - } - - toolchain_library { - name: "libclang_rt.builtins-aarch64-android", - vendor_available: true, - recovery_available: true, - src: "", - } - - toolchain_library { - name: "libgcc", - vendor_available: true, - recovery_available: true, - src: "", - } - - cc_library { - name: "libc", - no_libgcc: true, - nocrt: true, - system_shared_libs: [], - recovery_available: true, - } - - cc_library { - name: "libm", - no_libgcc: true, - nocrt: true, - system_shared_libs: [], - recovery_available: true, - } - - cc_library { - name: "libdl", - no_libgcc: true, - nocrt: true, - system_shared_libs: [], - recovery_available: true, - } - - cc_library { - name: "libgtest", - host_supported: true, - vendor_available: true, - } - - cc_library { - name: "libgtest_main", - host_supported: true, - vendor_available: true, - } - - cc_object { - name: "crtbegin_dynamic", - recovery_available: true, - vendor_available: true, - } - - cc_object { - name: "crtend_android", - recovery_available: true, - vendor_available: true, - } - - cc_object { - name: "crtbegin_so", - recovery_available: true, - vendor_available: true, - } - - cc_object { - name: "crtend_so", - recovery_available: true, - vendor_available: true, - } - ` +func testConfig(buildDir string, env map[string]string, bp string) android.Config { mockFS := map[string][]byte{ - "Android.bp": []byte(bp), "bpf.c": nil, "BpfTest.cpp": nil, } - ctx.MockFileSystem(mockFS) + return cc.TestConfig(buildDir, android.Android, env, bp, mockFS) +} + +func testContext(config android.Config) *android.TestContext { + ctx := cc.CreateTestContext() + ctx.RegisterModuleType("bpf", bpfFactory) + ctx.Register(config) return ctx } func TestBpfDataDependency(t *testing.T) { - config := android.TestArchConfig(buildDir, nil) bp := ` bpf { name: "bpf.o", @@ -174,10 +76,13 @@ func TestBpfDataDependency(t *testing.T) { name: "vts_test_binary_bpf_module", srcs: ["BpfTest.cpp"], data: [":bpf.o"], + gtest: false, } ` - ctx := testContext(bp) + config := testConfig(buildDir, nil, bp) + ctx := testContext(config) + _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) if errs == nil { _, errs = ctx.PrepareBuildActions(config) diff --git a/bpfix/Android.bp b/bpfix/Android.bp index 90a453d76..e29157849 100644 --- a/bpfix/Android.bp +++ b/bpfix/Android.bp @@ -19,7 +19,18 @@ blueprint_go_binary { name: "bpfix", srcs: [ - "cmd/bpfix.go", + "cmd/main.go", + ], + deps: [ + "bpfix-cmd", + ], +} + +bootstrap_go_package { + name: "bpfix-cmd", + pkgPath: "android/soong/bpfix/cmd_lib", + srcs: [ + "cmd_lib/bpfix.go", ], deps: [ "bpfix-lib", diff --git a/bpfix/bpfix/bpfix.go b/bpfix/bpfix/bpfix.go index 706c0ec25..a1c5de122 100644 --- a/bpfix/bpfix/bpfix.go +++ b/bpfix/bpfix/bpfix.go @@ -45,62 +45,88 @@ func Reformat(input string) (string, error) { // A FixRequest specifies the details of which fixes to apply to an individual file // A FixRequest doesn't specify whether to do a dry run or where to write the results; that's in cmd/bpfix.go type FixRequest struct { - steps []fixStep + steps []FixStep +} +type FixStepsExtension struct { + Name string + Steps []FixStep } -type fixStep struct { - name string - fix func(f *Fixer) error +type FixStep struct { + Name string + Fix func(f *Fixer) error } -var fixSteps = []fixStep{ +var fixStepsExtensions = []*FixStepsExtension(nil) + +func RegisterFixStepExtension(extension *FixStepsExtension) { + fixStepsExtensions = append(fixStepsExtensions, extension) +} + +var fixSteps = []FixStep{ { - name: "simplifyKnownRedundantVariables", - fix: runPatchListMod(simplifyKnownPropertiesDuplicatingEachOther), + Name: "simplifyKnownRedundantVariables", + Fix: runPatchListMod(simplifyKnownPropertiesDuplicatingEachOther), }, { - name: "rewriteIncorrectAndroidmkPrebuilts", - fix: rewriteIncorrectAndroidmkPrebuilts, + Name: "rewriteIncorrectAndroidmkPrebuilts", + Fix: rewriteIncorrectAndroidmkPrebuilts, }, { - name: "rewriteCtsModuleTypes", - fix: rewriteCtsModuleTypes, + Name: "rewriteCtsModuleTypes", + Fix: rewriteCtsModuleTypes, }, { - name: "rewriteIncorrectAndroidmkAndroidLibraries", - fix: rewriteIncorrectAndroidmkAndroidLibraries, + Name: "rewriteIncorrectAndroidmkAndroidLibraries", + Fix: rewriteIncorrectAndroidmkAndroidLibraries, }, { - name: "rewriteTestModuleTypes", - fix: rewriteTestModuleTypes, + Name: "rewriteTestModuleTypes", + Fix: rewriteTestModuleTypes, }, { - name: "rewriteAndroidmkJavaLibs", - fix: rewriteAndroidmkJavaLibs, + Name: "rewriteAndroidmkJavaLibs", + Fix: rewriteAndroidmkJavaLibs, }, { - name: "rewriteJavaStaticLibs", - fix: rewriteJavaStaticLibs, + Name: "rewriteJavaStaticLibs", + Fix: rewriteJavaStaticLibs, }, { - name: "rewritePrebuiltEtc", - fix: rewriteAndroidmkPrebuiltEtc, + Name: "rewritePrebuiltEtc", + Fix: rewriteAndroidmkPrebuiltEtc, }, { - name: "mergeMatchingModuleProperties", - fix: runPatchListMod(mergeMatchingModuleProperties), + Name: "mergeMatchingModuleProperties", + Fix: runPatchListMod(mergeMatchingModuleProperties), }, { - name: "reorderCommonProperties", - fix: runPatchListMod(reorderCommonProperties), + Name: "reorderCommonProperties", + Fix: runPatchListMod(reorderCommonProperties), }, { - name: "removeTags", - fix: runPatchListMod(removeTags), + Name: "removeTags", + Fix: runPatchListMod(removeTags), }, { - name: "rewriteAndroidTest", - fix: rewriteAndroidTest, + Name: "rewriteAndroidTest", + Fix: rewriteAndroidTest, + }, + { + Name: "rewriteAndroidAppImport", + Fix: rewriteAndroidAppImport, + }, + { + Name: "removeEmptyLibDependencies", + Fix: removeEmptyLibDependencies, + }, + { + Name: "removeHidlInterfaceTypes", + Fix: removeHidlInterfaceTypes, + }, + { + Name: "removeSoongConfigBoolVariable", + Fix: removeSoongConfigBoolVariable, }, } @@ -109,15 +135,38 @@ func NewFixRequest() FixRequest { } func (r FixRequest) AddAll() (result FixRequest) { - result.steps = append([]fixStep(nil), r.steps...) + result.steps = append([]FixStep(nil), r.steps...) + result.steps = append(result.steps, fixSteps...) + for _, extension := range fixStepsExtensions { + result.steps = append(result.steps, extension.Steps...) + } + return result +} + +func (r FixRequest) AddBase() (result FixRequest) { + result.steps = append([]FixStep(nil), r.steps...) result.steps = append(result.steps, fixSteps...) return result } +func (r FixRequest) AddMatchingExtensions(pattern string) (result FixRequest) { + result.steps = append([]FixStep(nil), r.steps...) + for _, extension := range fixStepsExtensions { + if match, _ := filepath.Match(pattern, extension.Name); match { + result.steps = append(result.steps, extension.Steps...) + } + } + return result +} + type Fixer struct { tree *parser.File } +func (f Fixer) Tree() *parser.File { + return f.tree +} + func NewFixer(tree *parser.File) *Fixer { fixer := &Fixer{tree} @@ -194,7 +243,7 @@ func parse(name string, r io.Reader) (*parser.File, error) { func (f *Fixer) fixTreeOnce(config FixRequest) error { for _, fix := range config.steps { - err := fix.fix(f) + err := fix.Fix(f) if err != nil { return err } @@ -431,14 +480,6 @@ func getStringProperty(prop *parser.Property, fieldName string) string { return "" } -// Create sub_dir: attribute for the given path -func makePrebuiltEtcDestination(mod *parser.Module, path string) { - mod.Properties = append(mod.Properties, &parser.Property{ - Name: "sub_dir", - Value: &parser.String{Value: path}, - }) -} - // Set the value of the given attribute to the error message func indicateAttributeError(mod *parser.Module, attributeName string, format string, a ...interface{}) error { msg := fmt.Sprintf(format, a...) @@ -464,25 +505,63 @@ func resolveLocalModule(mod *parser.Module, val parser.Expression) parser.Expres return val } -// A prefix to strip before setting 'filename' attribute and an array of boolean attributes to set. -type filenamePrefixToFlags struct { +// etcPrebuiltModuleUpdate contains information on updating certain parts of a defined module such as: +// * changing the module type from prebuilt_etc to a different one +// * stripping the prefix of the install path based on the module type +// * appending additional boolean properties to the prebuilt module +type etcPrebuiltModuleUpdate struct { + // The prefix of the install path defined in local_module_path. The prefix is removed from local_module_path + // before setting the 'filename' attribute. prefix string - flags []string + + // There is only one prebuilt module type in makefiles. In Soong, there are multiple versions of + // prebuilts based on local_module_path. By default, it is "prebuilt_etc" if modType is blank. An + // example is if the local_module_path contains $(TARGET_OUT)/usr/share, the module type is + // considered as prebuilt_usr_share. + modType string + + // Additional boolean attributes to be added in the prebuilt module. Each added boolean attribute + // has a value of true. + flags []string } -var localModulePathRewrite = map[string][]filenamePrefixToFlags{ - "HOST_OUT": {{prefix: "/etc"}}, - "PRODUCT_OUT": {{prefix: "/system/etc"}, {prefix: "/vendor/etc", flags: []string{"proprietary"}}}, - "TARGET_OUT": {{prefix: "/etc"}}, - "TARGET_OUT_ETC": {{prefix: ""}}, - "TARGET_OUT_PRODUCT": {{prefix: "/etc", flags: []string{"product_specific"}}}, - "TARGET_OUT_PRODUCT_ETC": {{prefix: "", flags: []string{"product_specific"}}}, - "TARGET_OUT_ODM": {{prefix: "/etc", flags: []string{"device_specific"}}}, - "TARGET_OUT_PRODUCT_SERVICES": {{prefix: "/etc", flags: []string{"product_services_specific"}}}, - "TARGET_OUT_PRODUCT_SERVICES_ETC": {{prefix: "", flags: []string{"product_services_specific"}}}, - "TARGET_OUT_VENDOR": {{prefix: "/etc", flags: []string{"proprietary"}}}, - "TARGET_OUT_VENDOR_ETC": {{prefix: "", flags: []string{"proprietary"}}}, - "TARGET_RECOVERY_ROOT_OUT": {{prefix: "/system/etc", flags: []string{"recovery"}}}, +func (f etcPrebuiltModuleUpdate) update(m *parser.Module, path string) bool { + updated := false + if path == f.prefix { + updated = true + } else if trimmedPath := strings.TrimPrefix(path, f.prefix+"/"); trimmedPath != path { + m.Properties = append(m.Properties, &parser.Property{ + Name: "sub_dir", + Value: &parser.String{Value: trimmedPath}, + }) + updated = true + } + if updated { + for _, flag := range f.flags { + m.Properties = append(m.Properties, &parser.Property{Name: flag, Value: &parser.Bool{Value: true, Token: "true"}}) + } + if f.modType != "" { + m.Type = f.modType + } + } + return updated +} + +var localModuleUpdate = map[string][]etcPrebuiltModuleUpdate{ + "HOST_OUT": {{prefix: "/etc", modType: "prebuilt_etc_host"}, {prefix: "/usr/share", modType: "prebuilt_usr_share_host"}}, + "PRODUCT_OUT": {{prefix: "/system/etc"}, {prefix: "/vendor/etc", flags: []string{"proprietary"}}}, + "TARGET_OUT": {{prefix: "/usr/share", modType: "prebuilt_usr_share"}, {prefix: "/fonts", modType: "prebuilt_font"}, + {prefix: "/etc/firmware", modType: "prebuilt_firmware"}, {prefix: "/vendor/firmware", modType: "prebuilt_firmware", flags: []string{"proprietary"}}, + {prefix: "/etc"}}, + "TARGET_OUT_ETC": {{prefix: "/firmware", modType: "prebuilt_firmware"}, {prefix: ""}}, + "TARGET_OUT_PRODUCT": {{prefix: "/etc", flags: []string{"product_specific"}}, {prefix: "/fonts", modType: "prebuilt_font", flags: []string{"product_specific"}}}, + "TARGET_OUT_PRODUCT_ETC": {{prefix: "", flags: []string{"product_specific"}}}, + "TARGET_OUT_ODM": {{prefix: "/etc", flags: []string{"device_specific"}}}, + "TARGET_OUT_SYSTEM_EXT": {{prefix: "/etc", flags: []string{"system_ext_specific"}}}, + "TARGET_OUT_SYSTEM_EXT_ETC": {{prefix: "", flags: []string{"system_ext_specific"}}}, + "TARGET_OUT_VENDOR": {{prefix: "/etc", flags: []string{"proprietary"}}, {prefix: "/firmware", modType: "prebuilt_firmware", flags: []string{"proprietary"}}}, + "TARGET_OUT_VENDOR_ETC": {{prefix: "", flags: []string{"proprietary"}}}, + "TARGET_RECOVERY_ROOT_OUT": {{prefix: "/system/etc", flags: []string{"recovery"}}}, } // rewriteAndroidPrebuiltEtc fixes prebuilt_etc rule @@ -497,27 +576,8 @@ func rewriteAndroidmkPrebuiltEtc(f *Fixer) error { continue } - // The rewriter converts LOCAL_SRC_FILES to `srcs` attribute. Convert - // it to 'src' attribute (which is where the file is installed). If the - // value 'srcs' is a list, we can convert it only if it contains a single - // element. - if srcs, ok := mod.GetProperty("srcs"); ok { - if srcList, ok := srcs.Value.(*parser.List); ok { - removeProperty(mod, "srcs") - if len(srcList.Values) == 1 { - mod.Properties = append(mod.Properties, - &parser.Property{Name: "src", NamePos: srcs.NamePos, ColonPos: srcs.ColonPos, Value: resolveLocalModule(mod, srcList.Values[0])}) - } else if len(srcList.Values) > 1 { - indicateAttributeError(mod, "src", "LOCAL_SRC_FILES should contain at most one item") - } - } else if _, ok = srcs.Value.(*parser.Variable); ok { - removeProperty(mod, "srcs") - mod.Properties = append(mod.Properties, - &parser.Property{Name: "src", NamePos: srcs.NamePos, ColonPos: srcs.ColonPos, Value: resolveLocalModule(mod, srcs.Value)}) - } else { - renameProperty(mod, "srcs", "src") - } - } + // 'srcs' --> 'src' conversion + convertToSingleSource(mod, "src") // The rewriter converts LOCAL_MODULE_PATH attribute into a struct attribute // 'local_module_path'. Analyze its contents and create the correct sub_dir:, @@ -526,37 +586,23 @@ func rewriteAndroidmkPrebuiltEtc(f *Fixer) error { if prop_local_module_path, ok := mod.GetProperty(local_module_path); ok { removeProperty(mod, local_module_path) prefixVariableName := getStringProperty(prop_local_module_path, "var") - path := getStringProperty(prop_local_module_path, "fixed") - if prefixRewrites, ok := localModulePathRewrite[prefixVariableName]; ok { - rewritten := false - for _, prefixRewrite := range prefixRewrites { - if path == prefixRewrite.prefix { - rewritten = true - } else if trimmedPath := strings.TrimPrefix(path, prefixRewrite.prefix+"/"); trimmedPath != path { - makePrebuiltEtcDestination(mod, trimmedPath) - rewritten = true - } - if rewritten { - for _, flag := range prefixRewrite.flags { - mod.Properties = append(mod.Properties, &parser.Property{Name: flag, Value: &parser.Bool{Value: true, Token: "true"}}) - } - break - } + if moduleUpdates, ok := localModuleUpdate[prefixVariableName]; ok { + path := getStringProperty(prop_local_module_path, "fixed") + updated := false + for i := 0; i < len(moduleUpdates) && !updated; i++ { + updated = moduleUpdates[i].update(mod, path) } - if !rewritten { + if !updated { expectedPrefices := "" sep := "" - for _, prefixRewrite := range prefixRewrites { + for _, moduleUpdate := range moduleUpdates { expectedPrefices += sep sep = ", " - expectedPrefices += prefixRewrite.prefix + expectedPrefices += moduleUpdate.prefix } return indicateAttributeError(mod, "filename", "LOCAL_MODULE_PATH value under $(%s) should start with %s", prefixVariableName, expectedPrefices) } - if prefixVariableName == "HOST_OUT" { - mod.Type = "prebuilt_etc_host" - } } else { return indicateAttributeError(mod, "filename", "Cannot handle $(%s) for the prebuilt_etc", prefixVariableName) } @@ -589,6 +635,190 @@ func rewriteAndroidTest(f *Fixer) error { return nil } +func rewriteAndroidAppImport(f *Fixer) error { + for _, def := range f.tree.Defs { + mod, ok := def.(*parser.Module) + if !(ok && mod.Type == "android_app_import") { + continue + } + // 'srcs' --> 'apk' conversion + convertToSingleSource(mod, "apk") + // Handle special certificate value, "PRESIGNED". + if cert, ok := mod.GetProperty("certificate"); ok { + if certStr, ok := cert.Value.(*parser.String); ok { + if certStr.Value == "PRESIGNED" { + removeProperty(mod, "certificate") + prop := &parser.Property{ + Name: "presigned", + Value: &parser.Bool{ + Value: true, + }, + } + mod.Properties = append(mod.Properties, prop) + } + } + } + } + return nil +} + +// Removes library dependencies which are empty (and restricted from usage in Soong) +func removeEmptyLibDependencies(f *Fixer) error { + emptyLibraries := []string{ + "libhidltransport", + "libhwbinder", + } + relevantFields := []string{ + "export_shared_lib_headers", + "export_static_lib_headers", + "static_libs", + "whole_static_libs", + "shared_libs", + } + for _, def := range f.tree.Defs { + mod, ok := def.(*parser.Module) + if !ok { + continue + } + for _, field := range relevantFields { + listValue, ok := getLiteralListProperty(mod, field) + if !ok { + continue + } + newValues := []parser.Expression{} + for _, v := range listValue.Values { + stringValue, ok := v.(*parser.String) + if !ok { + return fmt.Errorf("Expecting string for %s.%s fields", mod.Type, field) + } + if inList(stringValue.Value, emptyLibraries) { + continue + } + newValues = append(newValues, stringValue) + } + if len(newValues) == 0 && len(listValue.Values) != 0 { + removeProperty(mod, field) + } else { + listValue.Values = newValues + } + } + } + return nil +} + +// Removes hidl_interface 'types' which are no longer needed +func removeHidlInterfaceTypes(f *Fixer) error { + for _, def := range f.tree.Defs { + mod, ok := def.(*parser.Module) + if !(ok && mod.Type == "hidl_interface") { + continue + } + removeProperty(mod, "types") + } + return nil +} + +func removeSoongConfigBoolVariable(f *Fixer) error { + found := map[string]bool{} + newDefs := make([]parser.Definition, 0, len(f.tree.Defs)) + for _, def := range f.tree.Defs { + if mod, ok := def.(*parser.Module); ok && mod.Type == "soong_config_bool_variable" { + if name, ok := getLiteralStringPropertyValue(mod, "name"); ok { + found[name] = true + } else { + return fmt.Errorf("Found soong_config_bool_variable without a name") + } + } else { + newDefs = append(newDefs, def) + } + } + f.tree.Defs = newDefs + + if len(found) == 0 { + return nil + } + + return runPatchListMod(func(mod *parser.Module, buf []byte, patchList *parser.PatchList) error { + if mod.Type != "soong_config_module_type" { + return nil + } + + variables, ok := getLiteralListProperty(mod, "variables") + if !ok { + return nil + } + + boolValues := strings.Builder{} + empty := true + for _, item := range variables.Values { + nameValue, ok := item.(*parser.String) + if !ok { + empty = false + continue + } + if found[nameValue.Value] { + patchList.Add(item.Pos().Offset, item.End().Offset+2, "") + + boolValues.WriteString(`"`) + boolValues.WriteString(nameValue.Value) + boolValues.WriteString(`",`) + } else { + empty = false + } + } + if empty { + *patchList = parser.PatchList{} + + prop, _ := mod.GetProperty("variables") + patchList.Add(prop.Pos().Offset, prop.End().Offset+2, "") + } + if boolValues.Len() == 0 { + return nil + } + + bool_variables, ok := getLiteralListProperty(mod, "bool_variables") + if ok { + patchList.Add(bool_variables.RBracePos.Offset, bool_variables.RBracePos.Offset, ","+boolValues.String()) + } else { + patchList.Add(variables.RBracePos.Offset+2, variables.RBracePos.Offset+2, + fmt.Sprintf(`bool_variables: [%s],`, boolValues.String())) + } + + return nil + })(f) + + return nil +} + +// Converts the default source list property, 'srcs', to a single source property with a given name. +// "LOCAL_MODULE" reference is also resolved during the conversion process. +func convertToSingleSource(mod *parser.Module, srcPropertyName string) { + if srcs, ok := mod.GetProperty("srcs"); ok { + if srcList, ok := srcs.Value.(*parser.List); ok { + removeProperty(mod, "srcs") + if len(srcList.Values) == 1 { + mod.Properties = append(mod.Properties, + &parser.Property{ + Name: srcPropertyName, + NamePos: srcs.NamePos, + ColonPos: srcs.ColonPos, + Value: resolveLocalModule(mod, srcList.Values[0])}) + } else if len(srcList.Values) > 1 { + indicateAttributeError(mod, srcPropertyName, "LOCAL_SRC_FILES should contain at most one item") + } + } else if _, ok = srcs.Value.(*parser.Variable); ok { + removeProperty(mod, "srcs") + mod.Properties = append(mod.Properties, + &parser.Property{Name: srcPropertyName, + NamePos: srcs.NamePos, + ColonPos: srcs.ColonPos, + Value: resolveLocalModule(mod, srcs.Value)}) + } else { + renameProperty(mod, "srcs", "apk") + } + } +} + func runPatchListMod(modFunc func(mod *parser.Module, buf []byte, patchlist *parser.PatchList) error) func(*Fixer) error { return func(f *Fixer) error { // Make sure all the offsets are accurate @@ -994,3 +1224,12 @@ func removeProperty(mod *parser.Module, propertyName string) { } mod.Properties = newList } + +func inList(s string, list []string) bool { + for _, v := range list { + if s == v { + return true + } + } + return false +} diff --git a/bpfix/bpfix/bpfix_test.go b/bpfix/bpfix/bpfix_test.go index 459cd36b8..64a7b93b2 100644 --- a/bpfix/bpfix/bpfix_test.go +++ b/bpfix/bpfix/bpfix_test.go @@ -784,3 +784,201 @@ func TestRewriteAndroidTest(t *testing.T) { }) } } + +func TestRewriteAndroidAppImport(t *testing.T) { + tests := []struct { + name string + in string + out string + }{ + { + name: "android_app_import apk", + in: ` + android_app_import { + name: "foo", + srcs: ["package.apk"], + } + `, + out: ` + android_app_import { + name: "foo", + apk: "package.apk", + } + `, + }, + { + name: "android_app_import presigned", + in: ` + android_app_import { + name: "foo", + apk: "package.apk", + certificate: "PRESIGNED", + } + `, + out: ` + android_app_import { + name: "foo", + apk: "package.apk", + presigned: true, + + } + `, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + runPass(t, test.in, test.out, func(fixer *Fixer) error { + return rewriteAndroidAppImport(fixer) + }) + }) + } +} + +func TestRemoveEmptyLibDependencies(t *testing.T) { + tests := []struct { + name string + in string + out string + }{ + { + name: "remove sole shared lib", + in: ` + cc_library { + name: "foo", + shared_libs: ["libhwbinder"], + } + `, + out: ` + cc_library { + name: "foo", + + } + `, + }, + { + name: "remove a shared lib", + in: ` + cc_library { + name: "foo", + shared_libs: [ + "libhwbinder", + "libfoo", + "libhidltransport", + ], + } + `, + out: ` + cc_library { + name: "foo", + shared_libs: [ + + "libfoo", + + ], + } + `, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + runPass(t, test.in, test.out, func(fixer *Fixer) error { + return removeEmptyLibDependencies(fixer) + }) + }) + } +} + +func TestRemoveHidlInterfaceTypes(t *testing.T) { + tests := []struct { + name string + in string + out string + }{ + { + name: "remove types", + in: ` + hidl_interface { + name: "foo@1.0", + types: ["ParcelFooBar"], + } + `, + out: ` + hidl_interface { + name: "foo@1.0", + + } + `, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + runPass(t, test.in, test.out, func(fixer *Fixer) error { + return removeHidlInterfaceTypes(fixer) + }) + }) + } +} + +func TestRemoveSoongConfigBoolVariable(t *testing.T) { + tests := []struct { + name string + in string + out string + }{ + { + name: "remove bool", + in: ` + soong_config_module_type { + name: "foo", + variables: ["bar", "baz"], + } + + soong_config_bool_variable { + name: "bar", + } + + soong_config_string_variable { + name: "baz", + } + `, + out: ` + soong_config_module_type { + name: "foo", + variables: [ + "baz" + ], + bool_variables: ["bar"], + } + + soong_config_string_variable { + name: "baz", + } + `, + }, + { + name: "existing bool_variables", + in: ` + soong_config_module_type { + name: "foo", + variables: ["baz"], + bool_variables: ["bar"], + } + + soong_config_bool_variable { + name: "baz", + } + `, + out: ` + soong_config_module_type { + name: "foo", + bool_variables: ["bar", "baz"], + } + `, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + runPass(t, test.in, test.out, removeSoongConfigBoolVariable) + }) + } +} diff --git a/bpfix/cmd/main.go b/bpfix/cmd/main.go new file mode 100644 index 000000000..ad6814493 --- /dev/null +++ b/bpfix/cmd/main.go @@ -0,0 +1,23 @@ +// Copyright 2017 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file provides a wrapper to the bpfix command-line library + +package main + +import "android/soong/bpfix/cmd_lib" + +func main() { + cmd_lib.Run() +} diff --git a/bpfix/cmd/bpfix.go b/bpfix/cmd_lib/bpfix.go index ccdae1656..f90f65be6 100644 --- a/bpfix/cmd/bpfix.go +++ b/bpfix/cmd_lib/bpfix.go @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -// This file provides a command-line interface to bpfix +// This file provides a bpfix command-line library // TODO(jeffrygaston) should this file be consolidated with bpfmt.go? -package main +package cmd_lib import ( "bytes" @@ -128,7 +128,7 @@ func walkDir(path string, fixRequest bpfix.FixRequest) { filepath.Walk(path, makeFileVisitor(fixRequest)) } -func main() { +func Run() { flag.Parse() fixRequest := bpfix.NewFixRequest().AddAll() diff --git a/build_kzip.bash b/build_kzip.bash new file mode 100755 index 000000000..008030fef --- /dev/null +++ b/build_kzip.bash @@ -0,0 +1,42 @@ +#! /bin/bash -uv +# +# Build kzip files (source files for the indexing pipeline) for the given configuration, +# merge them and place the resulting all.kzip into $DIST_DIR. +# It is assumed that the current directory is the top of the source tree. +# The following environment variables affect the result: +# BUILD_NUMBER build number, used to generate unique ID (will use UUID if not set) +# DIST_DIR where the resulting all.kzip will be placed +# KYTHE_KZIP_ENCODING proto or json (proto is default) +# OUT_DIR output directory (out if not specified}) +# TARGET_BUILD_VARIANT variant, e.g., `userdebug` +# TARGET_PRODUCT target device name, e.g., 'aosp_blueline' +# XREF_CORPUS source code repository URI, e.g., 'android.googlesource.com/platform/superproject' + +: ${BUILD_NUMBER:=$(uuidgen)} +: ${KYTHE_KZIP_ENCODING:=proto} +export KYTHE_KZIP_ENCODING + +# The extraction might fail for some source files, so run with -k and then check that +# sufficiently many files were generated. +declare -r out="${OUT_DIR:-out}" +# Build extraction files for C++ and Java. Build `merge_zips` which we use later. +build/soong/soong_ui.bash --build-mode --all-modules --dir=$PWD -k merge_zips xref_cxx xref_java +#Build extraction file for Go files in build/soong directory. +declare -r abspath_out=$(realpath "${out}") +declare -r go_extractor=$(realpath prebuilts/build-tools/linux-x86/bin/go_extractor) +declare -r go_root=$(realpath prebuilts/go/linux-x86) +for dir in blueprint soong; do + (cd "build/$dir"; + "$go_extractor" --goroot="$go_root" --rules=vnames.go.json --canonicalize_package_corpus \ + --output "${abspath_out}/soong/build_${dir}.go.kzip" ./... + ) +done + +declare -r kzip_count=$(find "$out" -name '*.kzip' | wc -l) +(($kzip_count>100000)) || { printf "Too few kzip files were generated: %d\n" $kzip_count; exit 1; } + +# Pack +# TODO(asmundak): this should be done by soong. +declare -r allkzip="$BUILD_NUMBER.kzip" +"$out/soong/host/linux-x86/bin/merge_zips" "$DIST_DIR/$allkzip" @<(find "$out" -name '*.kzip') + diff --git a/cc/Android.bp b/cc/Android.bp new file mode 100644 index 000000000..9ece05f6c --- /dev/null +++ b/cc/Android.bp @@ -0,0 +1,87 @@ +bootstrap_go_package { + name: "soong-cc", + pkgPath: "android/soong/cc", + deps: [ + "blueprint", + "blueprint-pathtools", + "soong", + "soong-android", + "soong-cc-config", + "soong-etc", + "soong-genrule", + "soong-tradefed", + ], + srcs: [ + "androidmk.go", + "builder.go", + "cc.go", + "ccdeps.go", + "check.go", + "coverage.go", + "gen.go", + "linkable.go", + "lto.go", + "makevars.go", + "pgo.go", + "prebuilt.go", + "proto.go", + "rs.go", + "sanitize.go", + "sabi.go", + "sdk.go", + "snapshot_utils.go", + "stl.go", + "strip.go", + "sysprop.go", + "tidy.go", + "util.go", + "vendor_snapshot.go", + "vndk.go", + "vndk_prebuilt.go", + + "cflag_artifacts.go", + "cmakelists.go", + "compdb.go", + "compiler.go", + "installer.go", + "linker.go", + + "binary.go", + "binary_sdk_member.go", + "fuzz.go", + "library.go", + "library_headers.go", + "library_sdk_member.go", + "object.go", + "test.go", + "toolchain_library.go", + + "ndk_prebuilt.go", + "ndk_headers.go", + "ndk_library.go", + "ndk_sysroot.go", + + "llndk_library.go", + + "kernel_headers.go", + + "genrule.go", + + "vendor_public_library.go", + + "testing.go", + ], + testSrcs: [ + "cc_test.go", + "compiler_test.go", + "gen_test.go", + "genrule_test.go", + "library_headers_test.go", + "library_test.go", + "object_test.go", + "prebuilt_test.go", + "proto_test.go", + "test_data_test.go", + ], + pluginFor: ["soong_build"], +} diff --git a/cc/androidmk.go b/cc/androidmk.go index 02806f982..fede601b4 100644 --- a/cc/androidmk.go +++ b/cc/androidmk.go @@ -24,99 +24,129 @@ import ( ) var ( - vendorSuffix = ".vendor" - recoverySuffix = ".recovery" + nativeBridgeSuffix = ".native_bridge" + productSuffix = ".product" + vendorSuffix = ".vendor" + ramdiskSuffix = ".ramdisk" + recoverySuffix = ".recovery" + sdkSuffix = ".sdk" ) type AndroidMkContext interface { Name() string Target() android.Target - subAndroidMk(*android.AndroidMkData, interface{}) + subAndroidMk(*android.AndroidMkEntries, interface{}) Arch() android.Arch Os() android.OsType Host() bool - useVndk() bool + UseVndk() bool + VndkVersion() string static() bool - inRecovery() bool + InRamdisk() bool + InRecovery() bool } type subAndroidMkProvider interface { - AndroidMk(AndroidMkContext, *android.AndroidMkData) + AndroidMkEntries(AndroidMkContext, *android.AndroidMkEntries) } -func (c *Module) subAndroidMk(data *android.AndroidMkData, obj interface{}) { +func (c *Module) subAndroidMk(entries *android.AndroidMkEntries, obj interface{}) { if c.subAndroidMkOnce == nil { c.subAndroidMkOnce = make(map[subAndroidMkProvider]bool) } if androidmk, ok := obj.(subAndroidMkProvider); ok { if !c.subAndroidMkOnce[androidmk] { c.subAndroidMkOnce[androidmk] = true - androidmk.AndroidMk(c, data) + androidmk.AndroidMkEntries(c, entries) } } } -func (c *Module) AndroidMk() android.AndroidMkData { +func (c *Module) AndroidMkEntries() []android.AndroidMkEntries { if c.Properties.HideFromMake || !c.IsForPlatform() { - return android.AndroidMkData{ + return []android.AndroidMkEntries{{ Disabled: true, - } + }} } - ret := android.AndroidMkData{ + entries := android.AndroidMkEntries{ OutputFile: c.outputFile, // TODO(jiyong): add the APEXes providing shared libs to the required modules // Currently, adding c.Properties.ApexesProvidingSharedLibs is causing multiple - // runtime APEXes (com.android.runtime.debug|release) to be installed. And this + // ART APEXes (com.android.art.debug|release) to be installed. And this // is breaking some older devices (like marlin) where system.img is small. Required: c.Properties.AndroidMkRuntimeLibs, Include: "$(BUILD_SYSTEM)/soong_cc_prebuilt.mk", - Extra: []android.AndroidMkExtraFunc{ - func(w io.Writer, outputFile android.Path) { + ExtraEntries: []android.AndroidMkExtraEntriesFunc{ + func(entries *android.AndroidMkEntries) { if len(c.Properties.Logtags) > 0 { - fmt.Fprintln(w, "LOCAL_LOGTAGS_FILES :=", strings.Join(c.Properties.Logtags, " ")) + entries.AddStrings("LOCAL_LOGTAGS_FILES", c.Properties.Logtags...) } if len(c.Properties.AndroidMkSharedLibs) > 0 { - fmt.Fprintln(w, "LOCAL_SHARED_LIBRARIES := "+strings.Join(c.Properties.AndroidMkSharedLibs, " ")) + entries.AddStrings("LOCAL_SHARED_LIBRARIES", c.Properties.AndroidMkSharedLibs...) } if len(c.Properties.AndroidMkStaticLibs) > 0 { - fmt.Fprintln(w, "LOCAL_STATIC_LIBRARIES := "+strings.Join(c.Properties.AndroidMkStaticLibs, " ")) + entries.AddStrings("LOCAL_STATIC_LIBRARIES", c.Properties.AndroidMkStaticLibs...) } if len(c.Properties.AndroidMkWholeStaticLibs) > 0 { - fmt.Fprintln(w, "LOCAL_WHOLE_STATIC_LIBRARIES := "+strings.Join(c.Properties.AndroidMkWholeStaticLibs, " ")) + entries.AddStrings("LOCAL_WHOLE_STATIC_LIBRARIES", c.Properties.AndroidMkWholeStaticLibs...) + } + entries.SetString("LOCAL_SOONG_LINK_TYPE", c.makeLinkType) + if c.UseVndk() { + entries.SetBool("LOCAL_USE_VNDK", true) + if c.IsVndk() && !c.static() { + entries.SetString("LOCAL_SOONG_VNDK_VERSION", c.VndkVersion()) + // VNDK libraries available to vendor are not installed because + // they are packaged in VNDK APEX and installed by APEX packages (apex/apex.go) + if !c.isVndkExt() { + entries.SetBool("LOCAL_UNINSTALLABLE_MODULE", true) + } + } + } + if c.Properties.IsSdkVariant && c.Properties.SdkAndPlatformVariantVisibleToMake { + // Make the SDK variant uninstallable so that there are not two rules to install + // to the same location. + entries.SetBool("LOCAL_UNINSTALLABLE_MODULE", true) + // Add the unsuffixed name to SOONG_SDK_VARIANT_MODULES so that Make can rewrite + // dependencies to the .sdk suffix when building a module that uses the SDK. + entries.SetString("SOONG_SDK_VARIANT_MODULES", + "$(SOONG_SDK_VARIANT_MODULES) $(patsubst %.sdk,%,$(LOCAL_MODULE))") } - fmt.Fprintln(w, "LOCAL_SOONG_LINK_TYPE :=", c.getMakeLinkType()) - if c.useVndk() { - fmt.Fprintln(w, "LOCAL_USE_VNDK := true") + }, + }, + ExtraFooters: []android.AndroidMkExtraFootersFunc{ + func(w io.Writer, name, prefix, moduleDir string, entries *android.AndroidMkEntries) { + if c.Properties.IsSdkVariant && c.Properties.SdkAndPlatformVariantVisibleToMake && + c.CcLibraryInterface() && c.Shared() { + // Using the SDK variant as a JNI library needs a copy of the .so that + // is not named .sdk.so so that it can be packaged into the APK with + // the right name. + fmt.Fprintln(w, "$(eval $(call copy-one-file,", + "$(LOCAL_BUILT_MODULE),", + "$(patsubst %.sdk.so,%.so,$(LOCAL_BUILT_MODULE))))") } }, }, } for _, feature := range c.features { - c.subAndroidMk(&ret, feature) + c.subAndroidMk(&entries, feature) } - c.subAndroidMk(&ret, c.compiler) - c.subAndroidMk(&ret, c.linker) + c.subAndroidMk(&entries, c.compiler) + c.subAndroidMk(&entries, c.linker) if c.sanitize != nil { - c.subAndroidMk(&ret, c.sanitize) - } - c.subAndroidMk(&ret, c.installer) - - if c.useVndk() && c.hasVendorVariant() { - // .vendor suffix is added only when we will have two variants: core and vendor. - // The suffix is not added for vendor-only module. - ret.SubName += vendorSuffix - } else if c.inRecovery() && !c.onlyInRecovery() { - ret.SubName += recoverySuffix + c.subAndroidMk(&entries, c.sanitize) } + c.subAndroidMk(&entries, c.installer) + + entries.SubName += c.Properties.SubName - return ret + return []android.AndroidMkEntries{entries} } -func androidMkWriteTestData(data android.Paths, ctx AndroidMkContext, ret *android.AndroidMkData) { +func androidMkWriteTestData(data android.Paths, ctx AndroidMkContext, entries *android.AndroidMkEntries) { var testFiles []string for _, d := range data { rel := d.Rel() @@ -128,288 +158,458 @@ func androidMkWriteTestData(data android.Paths, ctx AndroidMkContext, ret *andro testFiles = append(testFiles, path+":"+rel) } if len(testFiles) > 0 { - ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) { - fmt.Fprintln(w, "LOCAL_TEST_DATA := "+strings.Join(testFiles, " ")) + entries.ExtraEntries = append(entries.ExtraEntries, func(entries *android.AndroidMkEntries) { + entries.AddStrings("LOCAL_TEST_DATA", testFiles...) }) } } -func (library *libraryDecorator) androidMkWriteExportedFlags(w io.Writer) { +func makeOverrideModuleNames(ctx AndroidMkContext, overrides []string) []string { + if ctx.Target().NativeBridge == android.NativeBridgeEnabled { + var result []string + for _, override := range overrides { + result = append(result, override+nativeBridgeSuffix) + } + return result + } + + return overrides +} + +func (library *libraryDecorator) androidMkWriteExportedFlags(entries *android.AndroidMkEntries) { exportedFlags := library.exportedFlags() + for _, dir := range library.exportedDirs() { + exportedFlags = append(exportedFlags, "-I"+dir.String()) + } + for _, dir := range library.exportedSystemDirs() { + exportedFlags = append(exportedFlags, "-isystem "+dir.String()) + } if len(exportedFlags) > 0 { - fmt.Fprintln(w, "LOCAL_EXPORT_CFLAGS :=", strings.Join(exportedFlags, " ")) + entries.AddStrings("LOCAL_EXPORT_CFLAGS", exportedFlags...) + } + exportedDeps := library.exportedDeps() + if len(exportedDeps) > 0 { + entries.AddStrings("LOCAL_EXPORT_C_INCLUDE_DEPS", exportedDeps.Strings()...) } - exportedFlagsDeps := library.exportedFlagsDeps() - if len(exportedFlagsDeps) > 0 { - fmt.Fprintln(w, "LOCAL_EXPORT_C_INCLUDE_DEPS :=", strings.Join(exportedFlagsDeps.Strings(), " ")) +} + +func (library *libraryDecorator) androidMkEntriesWriteAdditionalDependenciesForSourceAbiDiff(entries *android.AndroidMkEntries) { + if library.sAbiOutputFile.Valid() { + entries.SetString("LOCAL_ADDITIONAL_DEPENDENCIES", + "$(LOCAL_ADDITIONAL_DEPENDENCIES) "+library.sAbiOutputFile.String()) + if library.sAbiDiff.Valid() && !library.static() { + entries.SetString("LOCAL_ADDITIONAL_DEPENDENCIES", + "$(LOCAL_ADDITIONAL_DEPENDENCIES) "+library.sAbiDiff.String()) + entries.SetString("HEADER_ABI_DIFFS", + "$(HEADER_ABI_DIFFS) "+library.sAbiDiff.String()) + } + } +} + +// TODO(ccross): remove this once apex/androidmk.go is converted to AndroidMkEntries +func (library *libraryDecorator) androidMkWriteAdditionalDependenciesForSourceAbiDiff(w io.Writer) { + if library.sAbiOutputFile.Valid() { + fmt.Fprintln(w, "LOCAL_ADDITIONAL_DEPENDENCIES := $(LOCAL_ADDITIONAL_DEPENDENCIES) ", + library.sAbiOutputFile.String()) + if library.sAbiDiff.Valid() && !library.static() { + fmt.Fprintln(w, "LOCAL_ADDITIONAL_DEPENDENCIES := $(LOCAL_ADDITIONAL_DEPENDENCIES) ", + library.sAbiDiff.String()) + fmt.Fprintln(w, "HEADER_ABI_DIFFS := $(HEADER_ABI_DIFFS) ", + library.sAbiDiff.String()) + } } } -func (library *libraryDecorator) AndroidMk(ctx AndroidMkContext, ret *android.AndroidMkData) { +func (library *libraryDecorator) AndroidMkEntries(ctx AndroidMkContext, entries *android.AndroidMkEntries) { if library.static() { - ret.Class = "STATIC_LIBRARIES" + entries.Class = "STATIC_LIBRARIES" } else if library.shared() { - ret.Class = "SHARED_LIBRARIES" - ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) { - fmt.Fprintln(w, "LOCAL_SOONG_TOC :=", library.toc().String()) + entries.Class = "SHARED_LIBRARIES" + entries.ExtraEntries = append(entries.ExtraEntries, func(entries *android.AndroidMkEntries) { + entries.SetString("LOCAL_SOONG_TOC", library.toc().String()) if !library.buildStubs() { - fmt.Fprintln(w, "LOCAL_SOONG_UNSTRIPPED_BINARY :=", library.unstrippedOutputFile.String()) + entries.SetString("LOCAL_SOONG_UNSTRIPPED_BINARY", library.unstrippedOutputFile.String()) } if len(library.Properties.Overrides) > 0 { - fmt.Fprintln(w, "LOCAL_OVERRIDES_MODULES := "+strings.Join(library.Properties.Overrides, " ")) + entries.SetString("LOCAL_OVERRIDES_MODULES", strings.Join(makeOverrideModuleNames(ctx, library.Properties.Overrides), " ")) } if len(library.post_install_cmds) > 0 { - fmt.Fprintln(w, "LOCAL_POST_INSTALL_CMD := "+strings.Join(library.post_install_cmds, "&& ")) + entries.SetString("LOCAL_POST_INSTALL_CMD", strings.Join(library.post_install_cmds, "&& ")) } }) } else if library.header() { - ret.Class = "HEADER_LIBRARIES" + entries.Class = "HEADER_LIBRARIES" } - ret.DistFile = library.distFile - ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) { - library.androidMkWriteExportedFlags(w) - fmt.Fprintln(w, "LOCAL_ADDITIONAL_DEPENDENCIES := ") - if library.sAbiOutputFile.Valid() { - fmt.Fprintln(w, "LOCAL_ADDITIONAL_DEPENDENCIES += ", library.sAbiOutputFile.String()) - if library.sAbiDiff.Valid() && !library.static() { - fmt.Fprintln(w, "LOCAL_ADDITIONAL_DEPENDENCIES += ", library.sAbiDiff.String()) - fmt.Fprintln(w, "HEADER_ABI_DIFFS += ", library.sAbiDiff.String()) - } - } + entries.DistFile = library.distFile + entries.ExtraEntries = append(entries.ExtraEntries, func(entries *android.AndroidMkEntries) { + library.androidMkWriteExportedFlags(entries) + library.androidMkEntriesWriteAdditionalDependenciesForSourceAbiDiff(entries) - _, _, ext := splitFileExt(outputFile.Base()) + _, _, ext := android.SplitFileExt(entries.OutputFile.Path().Base()) - fmt.Fprintln(w, "LOCAL_BUILT_MODULE_STEM := $(LOCAL_MODULE)"+ext) + entries.SetString("LOCAL_BUILT_MODULE_STEM", "$(LOCAL_MODULE)"+ext) if library.coverageOutputFile.Valid() { - fmt.Fprintln(w, "LOCAL_PREBUILT_COVERAGE_ARCHIVE :=", library.coverageOutputFile.String()) + entries.SetString("LOCAL_PREBUILT_COVERAGE_ARCHIVE", library.coverageOutputFile.String()) } if library.useCoreVariant { - fmt.Fprintln(w, "LOCAL_UNINSTALLABLE_MODULE := true") - fmt.Fprintln(w, "LOCAL_NO_NOTICE_FILE := true") - fmt.Fprintln(w, "LOCAL_VNDK_DEPEND_ON_CORE_VARIANT := true") + entries.SetBool("LOCAL_UNINSTALLABLE_MODULE", true) + entries.SetBool("LOCAL_NO_NOTICE_FILE", true) + entries.SetBool("LOCAL_VNDK_DEPEND_ON_CORE_VARIANT", true) + } + if library.checkSameCoreVariant { + entries.SetBool("LOCAL_CHECK_SAME_VNDK_VARIANTS", true) } }) if library.shared() && !library.buildStubs() { - ctx.subAndroidMk(ret, library.baseInstaller) + ctx.subAndroidMk(entries, library.baseInstaller) } else { - ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) { - fmt.Fprintln(w, "LOCAL_UNINSTALLABLE_MODULE := true") + if library.buildStubs() { + entries.SubName = "." + library.stubsVersion() + } + entries.ExtraEntries = append(entries.ExtraEntries, func(entries *android.AndroidMkEntries) { + // Note library.skipInstall() has a special case to get here for static + // libraries that otherwise would have skipped installation and hence not + // have executed AndroidMkEntries at all. The reason is to ensure they get + // a NOTICE file make target which other libraries might depend on. + entries.SetBool("LOCAL_UNINSTALLABLE_MODULE", true) if library.buildStubs() { - fmt.Fprintln(w, "LOCAL_NO_NOTICE_FILE := true") + entries.SetBool("LOCAL_NO_NOTICE_FILE", true) } }) } if len(library.Properties.Stubs.Versions) > 0 && - android.DirectlyInAnyApex(ctx, ctx.Name()) && !ctx.inRecovery() && !ctx.useVndk() && + android.DirectlyInAnyApex(ctx, ctx.Name()) && !ctx.InRamdisk() && !ctx.InRecovery() && !ctx.UseVndk() && !ctx.static() { + if library.buildStubs() && library.isLatestStubVersion() { + // reference the latest version via its name without suffix when it is provided by apex + entries.SubName = "" + } if !library.buildStubs() { - ret.SubName = ".bootstrap" + entries.SubName = ".bootstrap" } } } -func (object *objectLinker) AndroidMk(ctx AndroidMkContext, ret *android.AndroidMkData) { - ret.Custom = func(w io.Writer, name, prefix, moduleDir string, data android.AndroidMkData) { - out := ret.OutputFile.Path() - varname := fmt.Sprintf("SOONG_%sOBJECT_%s%s", prefix, name, data.SubName) +func (object *objectLinker) AndroidMkEntries(ctx AndroidMkContext, entries *android.AndroidMkEntries) { + entries.Class = "STATIC_LIBRARIES" + entries.ExtraFooters = append(entries.ExtraFooters, + func(w io.Writer, name, prefix, moduleDir string, entries *android.AndroidMkEntries) { + out := entries.OutputFile.Path() + varname := fmt.Sprintf("SOONG_%sOBJECT_%s%s", prefix, name, entries.SubName) - fmt.Fprintf(w, "\n%s := %s\n", varname, out.String()) - fmt.Fprintln(w, ".KATI_READONLY: "+varname) - } + fmt.Fprintf(w, "\n%s := %s\n", varname, out.String()) + fmt.Fprintln(w, ".KATI_READONLY: "+varname) + }) } -func (binary *binaryDecorator) AndroidMk(ctx AndroidMkContext, ret *android.AndroidMkData) { - ctx.subAndroidMk(ret, binary.baseInstaller) +func (binary *binaryDecorator) AndroidMkEntries(ctx AndroidMkContext, entries *android.AndroidMkEntries) { + ctx.subAndroidMk(entries, binary.baseInstaller) - ret.Class = "EXECUTABLES" - ret.DistFile = binary.distFile - ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) { - fmt.Fprintln(w, "LOCAL_SOONG_UNSTRIPPED_BINARY :=", binary.unstrippedOutputFile.String()) + entries.Class = "EXECUTABLES" + entries.DistFile = binary.distFile + entries.ExtraEntries = append(entries.ExtraEntries, func(entries *android.AndroidMkEntries) { + entries.SetString("LOCAL_SOONG_UNSTRIPPED_BINARY", binary.unstrippedOutputFile.String()) if len(binary.symlinks) > 0 { - fmt.Fprintln(w, "LOCAL_MODULE_SYMLINKS := "+strings.Join(binary.symlinks, " ")) + entries.AddStrings("LOCAL_MODULE_SYMLINKS", binary.symlinks...) } if binary.coverageOutputFile.Valid() { - fmt.Fprintln(w, "LOCAL_PREBUILT_COVERAGE_ARCHIVE :=", binary.coverageOutputFile.String()) + entries.SetString("LOCAL_PREBUILT_COVERAGE_ARCHIVE", binary.coverageOutputFile.String()) } if len(binary.Properties.Overrides) > 0 { - fmt.Fprintln(w, "LOCAL_OVERRIDES_MODULES := "+strings.Join(binary.Properties.Overrides, " ")) + entries.SetString("LOCAL_OVERRIDES_MODULES", strings.Join(makeOverrideModuleNames(ctx, binary.Properties.Overrides), " ")) } if len(binary.post_install_cmds) > 0 { - fmt.Fprintln(w, "LOCAL_POST_INSTALL_CMD := "+strings.Join(binary.post_install_cmds, "&& ")) + entries.SetString("LOCAL_POST_INSTALL_CMD", strings.Join(binary.post_install_cmds, "&& ")) } }) } -func (benchmark *benchmarkDecorator) AndroidMk(ctx AndroidMkContext, ret *android.AndroidMkData) { - ctx.subAndroidMk(ret, benchmark.binaryDecorator) - ret.Class = "NATIVE_TESTS" - ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) { +func (benchmark *benchmarkDecorator) AndroidMkEntries(ctx AndroidMkContext, entries *android.AndroidMkEntries) { + ctx.subAndroidMk(entries, benchmark.binaryDecorator) + entries.Class = "NATIVE_TESTS" + entries.ExtraEntries = append(entries.ExtraEntries, func(entries *android.AndroidMkEntries) { if len(benchmark.Properties.Test_suites) > 0 { - fmt.Fprintln(w, "LOCAL_COMPATIBILITY_SUITE :=", + entries.SetString("LOCAL_COMPATIBILITY_SUITE", strings.Join(benchmark.Properties.Test_suites, " ")) } if benchmark.testConfig != nil { - fmt.Fprintln(w, "LOCAL_FULL_TEST_CONFIG :=", benchmark.testConfig.String()) + entries.SetString("LOCAL_FULL_TEST_CONFIG", benchmark.testConfig.String()) + } + entries.SetBool("LOCAL_NATIVE_BENCHMARK", true) + if !BoolDefault(benchmark.Properties.Auto_gen_config, true) { + entries.SetBool("LOCAL_DISABLE_AUTO_GENERATE_TEST_CONFIG", true) } - fmt.Fprintln(w, "LOCAL_NATIVE_BENCHMARK := true") }) - androidMkWriteTestData(benchmark.data, ctx, ret) + androidMkWriteTestData(benchmark.data, ctx, entries) } -func (test *testBinary) AndroidMk(ctx AndroidMkContext, ret *android.AndroidMkData) { - ctx.subAndroidMk(ret, test.binaryDecorator) - ret.Class = "NATIVE_TESTS" +func (test *testBinary) AndroidMkEntries(ctx AndroidMkContext, entries *android.AndroidMkEntries) { + ctx.subAndroidMk(entries, test.binaryDecorator) + entries.Class = "NATIVE_TESTS" if Bool(test.Properties.Test_per_src) { - ret.SubName = "_" + String(test.binaryDecorator.Properties.Stem) + entries.SubName = "_" + String(test.binaryDecorator.Properties.Stem) } - - ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) { + entries.ExtraEntries = append(entries.ExtraEntries, func(entries *android.AndroidMkEntries) { if len(test.Properties.Test_suites) > 0 { - fmt.Fprintln(w, "LOCAL_COMPATIBILITY_SUITE :=", + entries.SetString("LOCAL_COMPATIBILITY_SUITE", strings.Join(test.Properties.Test_suites, " ")) } if test.testConfig != nil { - fmt.Fprintln(w, "LOCAL_FULL_TEST_CONFIG :=", test.testConfig.String()) + entries.SetString("LOCAL_FULL_TEST_CONFIG", test.testConfig.String()) + } + if !BoolDefault(test.Properties.Auto_gen_config, true) { + entries.SetBool("LOCAL_DISABLE_AUTO_GENERATE_TEST_CONFIG", true) } }) - androidMkWriteTestData(test.data, ctx, ret) + androidMkWriteTestData(test.data, ctx, entries) +} + +func (fuzz *fuzzBinary) AndroidMkEntries(ctx AndroidMkContext, entries *android.AndroidMkEntries) { + ctx.subAndroidMk(entries, fuzz.binaryDecorator) + + var fuzzFiles []string + for _, d := range fuzz.corpus { + fuzzFiles = append(fuzzFiles, + filepath.Dir(fuzz.corpusIntermediateDir.String())+":corpus/"+d.Base()) + } + + for _, d := range fuzz.data { + fuzzFiles = append(fuzzFiles, + filepath.Dir(fuzz.dataIntermediateDir.String())+":data/"+d.Rel()) + } + + if fuzz.dictionary != nil { + fuzzFiles = append(fuzzFiles, + filepath.Dir(fuzz.dictionary.String())+":"+fuzz.dictionary.Base()) + } + + if fuzz.config != nil { + fuzzFiles = append(fuzzFiles, + filepath.Dir(fuzz.config.String())+":config.json") + } + + entries.ExtraEntries = append(entries.ExtraEntries, func(entries *android.AndroidMkEntries) { + entries.SetBool("LOCAL_IS_FUZZ_TARGET", true) + if len(fuzzFiles) > 0 { + entries.AddStrings("LOCAL_TEST_DATA", fuzzFiles...) + } + if fuzz.installedSharedDeps != nil { + entries.AddStrings("LOCAL_FUZZ_INSTALLED_SHARED_DEPS", fuzz.installedSharedDeps...) + } + }) } -func (test *testLibrary) AndroidMk(ctx AndroidMkContext, ret *android.AndroidMkData) { - ctx.subAndroidMk(ret, test.libraryDecorator) +func (test *testLibrary) AndroidMkEntries(ctx AndroidMkContext, entries *android.AndroidMkEntries) { + ctx.subAndroidMk(entries, test.libraryDecorator) } -func (library *toolchainLibraryDecorator) AndroidMk(ctx AndroidMkContext, ret *android.AndroidMkData) { - ret.Class = "STATIC_LIBRARIES" - ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) { - _, suffix, _ := splitFileExt(outputFile.Base()) - fmt.Fprintln(w, "LOCAL_MODULE_SUFFIX := "+suffix) +func (library *toolchainLibraryDecorator) AndroidMkEntries(ctx AndroidMkContext, entries *android.AndroidMkEntries) { + entries.Class = "STATIC_LIBRARIES" + entries.ExtraEntries = append(entries.ExtraEntries, func(entries *android.AndroidMkEntries) { + _, suffix, _ := android.SplitFileExt(entries.OutputFile.Path().Base()) + entries.SetString("LOCAL_MODULE_SUFFIX", suffix) }) } -func (installer *baseInstaller) AndroidMk(ctx AndroidMkContext, ret *android.AndroidMkData) { +func (installer *baseInstaller) AndroidMkEntries(ctx AndroidMkContext, entries *android.AndroidMkEntries) { + if installer.path == (android.InstallPath{}) { + return + } // Soong installation is only supported for host modules. Have Make // installation trigger Soong installation. if ctx.Target().Os.Class == android.Host { - ret.OutputFile = android.OptionalPathForPath(installer.path) + entries.OutputFile = android.OptionalPathForPath(installer.path) } - ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) { - path := installer.path.RelPathString() - dir, file := filepath.Split(path) - stem, suffix, _ := splitFileExt(file) - fmt.Fprintln(w, "LOCAL_MODULE_SUFFIX := "+suffix) - fmt.Fprintln(w, "LOCAL_MODULE_PATH := $(OUT_DIR)/"+filepath.Clean(dir)) - fmt.Fprintln(w, "LOCAL_MODULE_STEM := "+stem) + entries.ExtraEntries = append(entries.ExtraEntries, func(entries *android.AndroidMkEntries) { + path, file := filepath.Split(installer.path.ToMakePath().String()) + stem, suffix, _ := android.SplitFileExt(file) + entries.SetString("LOCAL_MODULE_SUFFIX", suffix) + entries.SetString("LOCAL_MODULE_PATH", path) + entries.SetString("LOCAL_MODULE_STEM", stem) }) } -func (c *stubDecorator) AndroidMk(ctx AndroidMkContext, ret *android.AndroidMkData) { - ret.SubName = ndkLibrarySuffix + "." + c.properties.ApiLevel - ret.Class = "SHARED_LIBRARIES" +func (c *stubDecorator) AndroidMkEntries(ctx AndroidMkContext, entries *android.AndroidMkEntries) { + entries.SubName = ndkLibrarySuffix + "." + c.properties.ApiLevel + entries.Class = "SHARED_LIBRARIES" - ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) { + entries.ExtraEntries = append(entries.ExtraEntries, func(entries *android.AndroidMkEntries) { path, file := filepath.Split(c.installPath.String()) - stem, suffix, _ := splitFileExt(file) - fmt.Fprintln(w, "LOCAL_MODULE_SUFFIX := "+suffix) - fmt.Fprintln(w, "LOCAL_MODULE_PATH := "+path) - fmt.Fprintln(w, "LOCAL_MODULE_STEM := "+stem) - fmt.Fprintln(w, "LOCAL_NO_NOTICE_FILE := true") + stem, suffix, _ := android.SplitFileExt(file) + entries.SetString("LOCAL_MODULE_SUFFIX", suffix) + entries.SetString("LOCAL_MODULE_PATH", path) + entries.SetString("LOCAL_MODULE_STEM", stem) + entries.SetBool("LOCAL_NO_NOTICE_FILE", true) + }) +} + +func (c *llndkStubDecorator) AndroidMkEntries(ctx AndroidMkContext, entries *android.AndroidMkEntries) { + entries.Class = "SHARED_LIBRARIES" + + entries.ExtraEntries = append(entries.ExtraEntries, func(entries *android.AndroidMkEntries) { + c.libraryDecorator.androidMkWriteExportedFlags(entries) + _, _, ext := android.SplitFileExt(entries.OutputFile.Path().Base()) + + entries.SetString("LOCAL_BUILT_MODULE_STEM", "$(LOCAL_MODULE)"+ext) + entries.SetBool("LOCAL_UNINSTALLABLE_MODULE", true) + entries.SetBool("LOCAL_NO_NOTICE_FILE", true) + entries.SetString("LOCAL_SOONG_TOC", c.toc().String()) }) } -func (c *llndkStubDecorator) AndroidMk(ctx AndroidMkContext, ret *android.AndroidMkData) { - ret.Class = "SHARED_LIBRARIES" - ret.SubName = vendorSuffix +func (c *vndkPrebuiltLibraryDecorator) AndroidMkEntries(ctx AndroidMkContext, entries *android.AndroidMkEntries) { + entries.Class = "SHARED_LIBRARIES" + + entries.SubName = c.androidMkSuffix - ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) { - c.libraryDecorator.androidMkWriteExportedFlags(w) - _, _, ext := splitFileExt(outputFile.Base()) + entries.ExtraEntries = append(entries.ExtraEntries, func(entries *android.AndroidMkEntries) { + c.libraryDecorator.androidMkWriteExportedFlags(entries) - fmt.Fprintln(w, "LOCAL_BUILT_MODULE_STEM := $(LOCAL_MODULE)"+ext) - fmt.Fprintln(w, "LOCAL_UNINSTALLABLE_MODULE := true") - fmt.Fprintln(w, "LOCAL_NO_NOTICE_FILE := true") - fmt.Fprintln(w, "LOCAL_SOONG_TOC :=", c.toc().String()) + path, file := filepath.Split(c.path.ToMakePath().String()) + stem, suffix, ext := android.SplitFileExt(file) + entries.SetString("LOCAL_BUILT_MODULE_STEM", "$(LOCAL_MODULE)"+ext) + entries.SetString("LOCAL_MODULE_SUFFIX", suffix) + entries.SetString("LOCAL_MODULE_PATH", path) + entries.SetString("LOCAL_MODULE_STEM", stem) + if c.tocFile.Valid() { + entries.SetString("LOCAL_SOONG_TOC", c.tocFile.String()) + } }) } -func (c *vndkPrebuiltLibraryDecorator) AndroidMk(ctx AndroidMkContext, ret *android.AndroidMkData) { - ret.Class = "SHARED_LIBRARIES" +func (c *vendorSnapshotLibraryDecorator) AndroidMkEntries(ctx AndroidMkContext, entries *android.AndroidMkEntries) { + // Each vendor snapshot is exported to androidMk only when BOARD_VNDK_VERSION != current + // and the version of the prebuilt is same as BOARD_VNDK_VERSION. + if c.shared() { + entries.Class = "SHARED_LIBRARIES" + } else if c.static() { + entries.Class = "STATIC_LIBRARIES" + } else if c.header() { + entries.Class = "HEADER_LIBRARIES" + } + + if c.androidMkVendorSuffix { + entries.SubName = vendorSuffix + } else { + entries.SubName = "" + } + + entries.ExtraEntries = append(entries.ExtraEntries, func(entries *android.AndroidMkEntries) { + c.libraryDecorator.androidMkWriteExportedFlags(entries) + + if c.shared() || c.static() { + path, file := filepath.Split(c.path.ToMakePath().String()) + stem, suffix, ext := android.SplitFileExt(file) + entries.SetString("LOCAL_BUILT_MODULE_STEM", "$(LOCAL_MODULE)"+ext) + entries.SetString("LOCAL_MODULE_SUFFIX", suffix) + entries.SetString("LOCAL_MODULE_STEM", stem) + if c.shared() { + entries.SetString("LOCAL_MODULE_PATH", path) + } + if c.tocFile.Valid() { + entries.SetString("LOCAL_SOONG_TOC", c.tocFile.String()) + } + } + + if !c.shared() { // static or header + entries.SetBool("LOCAL_UNINSTALLABLE_MODULE", true) + } + }) +} - ret.SubName = c.NameSuffix() +func (c *vendorSnapshotBinaryDecorator) AndroidMkEntries(ctx AndroidMkContext, entries *android.AndroidMkEntries) { + entries.Class = "EXECUTABLES" - ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) { - c.libraryDecorator.androidMkWriteExportedFlags(w) + if c.androidMkVendorSuffix { + entries.SubName = vendorSuffix + } else { + entries.SubName = "" + } - path := c.path.RelPathString() - dir, file := filepath.Split(path) - stem, suffix, ext := splitFileExt(file) - fmt.Fprintln(w, "LOCAL_BUILT_MODULE_STEM := $(LOCAL_MODULE)"+ext) - fmt.Fprintln(w, "LOCAL_MODULE_SUFFIX := "+suffix) - fmt.Fprintln(w, "LOCAL_MODULE_PATH := $(OUT_DIR)/"+filepath.Clean(dir)) - fmt.Fprintln(w, "LOCAL_MODULE_STEM := "+stem) + entries.ExtraEntries = append(entries.ExtraEntries, func(entries *android.AndroidMkEntries) { + entries.AddStrings("LOCAL_MODULE_SYMLINKS", c.Properties.Symlinks...) }) } -func (c *ndkPrebuiltStlLinker) AndroidMk(ctx AndroidMkContext, ret *android.AndroidMkData) { - ret.Class = "SHARED_LIBRARIES" +func (c *vendorSnapshotObjectLinker) AndroidMkEntries(ctx AndroidMkContext, entries *android.AndroidMkEntries) { + entries.Class = "STATIC_LIBRARIES" + + if c.androidMkVendorSuffix { + entries.SubName = vendorSuffix + } else { + entries.SubName = "" + } + + entries.ExtraFooters = append(entries.ExtraFooters, + func(w io.Writer, name, prefix, moduleDir string, entries *android.AndroidMkEntries) { + out := entries.OutputFile.Path() + varname := fmt.Sprintf("SOONG_%sOBJECT_%s%s", prefix, name, entries.SubName) + + fmt.Fprintf(w, "\n%s := %s\n", varname, out.String()) + fmt.Fprintln(w, ".KATI_READONLY: "+varname) + }) +} + +func (c *ndkPrebuiltStlLinker) AndroidMkEntries(ctx AndroidMkContext, entries *android.AndroidMkEntries) { + entries.Class = "SHARED_LIBRARIES" } -func (c *vendorPublicLibraryStubDecorator) AndroidMk(ctx AndroidMkContext, ret *android.AndroidMkData) { - ret.Class = "SHARED_LIBRARIES" - ret.SubName = vendorPublicLibrarySuffix +func (c *vendorPublicLibraryStubDecorator) AndroidMkEntries(ctx AndroidMkContext, entries *android.AndroidMkEntries) { + entries.Class = "SHARED_LIBRARIES" + entries.SubName = vendorPublicLibrarySuffix - ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) { - c.libraryDecorator.androidMkWriteExportedFlags(w) - _, _, ext := splitFileExt(outputFile.Base()) + entries.ExtraEntries = append(entries.ExtraEntries, func(entries *android.AndroidMkEntries) { + c.libraryDecorator.androidMkWriteExportedFlags(entries) + _, _, ext := android.SplitFileExt(entries.OutputFile.Path().Base()) - fmt.Fprintln(w, "LOCAL_BUILT_MODULE_STEM := $(LOCAL_MODULE)"+ext) - fmt.Fprintln(w, "LOCAL_UNINSTALLABLE_MODULE := true") - fmt.Fprintln(w, "LOCAL_NO_NOTICE_FILE := true") + entries.SetString("LOCAL_BUILT_MODULE_STEM", "$(LOCAL_MODULE)"+ext) + entries.SetBool("LOCAL_UNINSTALLABLE_MODULE", true) + entries.SetBool("LOCAL_NO_NOTICE_FILE", true) }) } -func (p *prebuiltLinker) AndroidMk(ctx AndroidMkContext, ret *android.AndroidMkData) { - ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) { +func (p *prebuiltLinker) AndroidMkEntries(ctx AndroidMkContext, entries *android.AndroidMkEntries) { + entries.ExtraEntries = append(entries.ExtraEntries, func(entries *android.AndroidMkEntries) { if p.properties.Check_elf_files != nil { - fmt.Fprintln(w, "LOCAL_CHECK_ELF_FILES :=", *p.properties.Check_elf_files) + entries.SetBool("LOCAL_CHECK_ELF_FILES", *p.properties.Check_elf_files) } else { // soong_cc_prebuilt.mk does not include check_elf_file.mk by default // because cc_library_shared and cc_binary use soong_cc_prebuilt.mk as well. // In order to turn on prebuilt ABI checker, set `LOCAL_CHECK_ELF_FILES` to // true if `p.properties.Check_elf_files` is not specified. - fmt.Fprintln(w, "LOCAL_CHECK_ELF_FILES := true") + entries.SetBool("LOCAL_CHECK_ELF_FILES", true) } }) } -func (p *prebuiltLibraryLinker) AndroidMk(ctx AndroidMkContext, ret *android.AndroidMkData) { - ctx.subAndroidMk(ret, p.libraryDecorator) +func (p *prebuiltLibraryLinker) AndroidMkEntries(ctx AndroidMkContext, entries *android.AndroidMkEntries) { + ctx.subAndroidMk(entries, p.libraryDecorator) if p.shared() { - ctx.subAndroidMk(ret, &p.prebuiltLinker) - androidMkWriteAllowUndefinedSymbols(p.baseLinker, ret) + ctx.subAndroidMk(entries, &p.prebuiltLinker) + androidMkWriteAllowUndefinedSymbols(p.baseLinker, entries) } } -func (p *prebuiltBinaryLinker) AndroidMk(ctx AndroidMkContext, ret *android.AndroidMkData) { - ctx.subAndroidMk(ret, p.binaryDecorator) - ctx.subAndroidMk(ret, &p.prebuiltLinker) - androidMkWriteAllowUndefinedSymbols(p.baseLinker, ret) +func (p *prebuiltBinaryLinker) AndroidMkEntries(ctx AndroidMkContext, entries *android.AndroidMkEntries) { + ctx.subAndroidMk(entries, p.binaryDecorator) + ctx.subAndroidMk(entries, &p.prebuiltLinker) + androidMkWriteAllowUndefinedSymbols(p.baseLinker, entries) } -func androidMkWriteAllowUndefinedSymbols(linker *baseLinker, ret *android.AndroidMkData) { - ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) { - allow := linker.Properties.Allow_undefined_symbols - if allow != nil { - fmt.Fprintln(w, "LOCAL_ALLOW_UNDEFINED_SYMBOLS :=", *allow) - } - }) +func androidMkWriteAllowUndefinedSymbols(linker *baseLinker, entries *android.AndroidMkEntries) { + allow := linker.Properties.Allow_undefined_symbols + if allow != nil { + entries.ExtraEntries = append(entries.ExtraEntries, func(entries *android.AndroidMkEntries) { + entries.SetBool("LOCAL_ALLOW_UNDEFINED_SYMBOLS", *allow) + }) + } } diff --git a/cc/binary.go b/cc/binary.go index 2a6ceb821..251b7f0c4 100644 --- a/cc/binary.go +++ b/cc/binary.go @@ -50,11 +50,18 @@ type BinaryLinkerProperties struct { // binaries would be installed by default (in PRODUCT_PACKAGES) the other binary will be removed // from PRODUCT_PACKAGES. Overrides []string + + // Inject boringssl hash into the shared library. This is only intended for use by external/boringssl. + Inject_bssl_hash *bool `android:"arch_variant"` } func init() { - android.RegisterModuleType("cc_binary", BinaryFactory) - android.RegisterModuleType("cc_binary_host", binaryHostFactory) + RegisterBinaryBuildComponents(android.InitRegistrationContext) +} + +func RegisterBinaryBuildComponents(ctx android.RegistrationContext) { + ctx.RegisterModuleType("cc_binary", BinaryFactory) + ctx.RegisterModuleType("cc_binary_host", binaryHostFactory) } // cc_binary produces a binary that is runnable on a device. @@ -106,13 +113,17 @@ func (binary *binaryDecorator) linkerProps() []interface{} { } -func (binary *binaryDecorator) getStem(ctx BaseModuleContext) string { +func (binary *binaryDecorator) getStemWithoutSuffix(ctx BaseModuleContext) string { stem := ctx.baseModuleName() if String(binary.Properties.Stem) != "" { stem = String(binary.Properties.Stem) } - return stem + String(binary.Properties.Suffix) + return stem +} + +func (binary *binaryDecorator) getStem(ctx BaseModuleContext) string { + return binary.getStemWithoutSuffix(ctx) + String(binary.Properties.Suffix) } func (binary *binaryDecorator) linkerDeps(ctx DepsContext, deps Deps) Deps { @@ -151,7 +162,7 @@ func (binary *binaryDecorator) linkerDeps(ctx DepsContext, deps Deps) Deps { if binary.static() { if ctx.selectedStl() == "libc++_static" { - deps.StaticLibs = append(deps.StaticLibs, "libm", "libc", "libdl") + deps.StaticLibs = append(deps.StaticLibs, "libm", "libc") } // static libraries libcompiler_rt, libc and libc_nomalloc need to be linked with // --start-group/--end-group along with libgcc. If they are in deps.StaticLibs, @@ -189,6 +200,11 @@ func NewBinary(hod android.HostOrDeviceSupported) (*Module, *binaryDecorator) { module.compiler = NewBaseCompiler() module.linker = binary module.installer = binary + + // Allow module to be added as member of an sdk/module_exports. + module.sdkMemberTypes = []android.SdkMemberType{ + ccBinarySdkMemberType, + } return module, binary } @@ -215,12 +231,16 @@ func (binary *binaryDecorator) staticBinary() bool { return binary.static() } +func (binary *binaryDecorator) binary() bool { + return true +} + func (binary *binaryDecorator) linkerFlags(ctx ModuleContext, flags Flags) Flags { flags = binary.baseLinker.linkerFlags(ctx, flags) if ctx.Host() && !ctx.Windows() && !binary.static() { if !ctx.Config().IsEnvTrue("DISABLE_HOST_PIE") { - flags.LdFlags = append(flags.LdFlags, "-pie") + flags.Global.LdFlags = append(flags.Global.LdFlags, "-pie") } } @@ -228,7 +248,7 @@ func (binary *binaryDecorator) linkerFlags(ctx ModuleContext, flags Flags) Flags // all code is position independent, and then those warnings get promoted to // errors. if !ctx.Windows() { - flags.CFlags = append(flags.CFlags, "-fPIE") + flags.Global.CFlags = append(flags.Global.CFlags, "-fPIE") } if ctx.toolchain().Bionic() { @@ -237,11 +257,11 @@ func (binary *binaryDecorator) linkerFlags(ctx ModuleContext, flags Flags) Flags // However, bionic/linker uses -shared to overwrite. // Linker for x86 targets does not allow coexistance of -static and -shared, // so we add -static only if -shared is not used. - if !inList("-shared", flags.LdFlags) { - flags.LdFlags = append(flags.LdFlags, "-static") + if !inList("-shared", flags.Local.LdFlags) { + flags.Global.LdFlags = append(flags.Global.LdFlags, "-static") } - flags.LdFlags = append(flags.LdFlags, + flags.Global.LdFlags = append(flags.Global.LdFlags, "-nostdlib", "-Bstatic", "-Wl,--gc-sections", @@ -253,7 +273,7 @@ func (binary *binaryDecorator) linkerFlags(ctx ModuleContext, flags Flags) Flags } else { switch ctx.Os() { case android.Android: - if ctx.bootstrap() && !ctx.inRecovery() { + if ctx.bootstrap() && !ctx.inRecovery() && !ctx.inRamdisk() { flags.DynamicLinker = "/system/bin/bootstrap/linker" } else { flags.DynamicLinker = "/system/bin/linker" @@ -271,14 +291,14 @@ func (binary *binaryDecorator) linkerFlags(ctx ModuleContext, flags Flags) Flags if ctx.Os() == android.LinuxBionic { // Use the dlwrap entry point, but keep _start around so // that it can be used by host_bionic_inject - flags.LdFlags = append(flags.LdFlags, + flags.Global.LdFlags = append(flags.Global.LdFlags, "-Wl,--entry=__dlwrap__start", "-Wl,--undefined=_start", ) } } - flags.LdFlags = append(flags.LdFlags, + flags.Global.LdFlags = append(flags.Global.LdFlags, "-pie", "-nostdlib", "-Bdynamic", @@ -288,10 +308,10 @@ func (binary *binaryDecorator) linkerFlags(ctx ModuleContext, flags Flags) Flags } } else { if binary.static() { - flags.LdFlags = append(flags.LdFlags, "-static") + flags.Global.LdFlags = append(flags.Global.LdFlags, "-static") } if ctx.Darwin() { - flags.LdFlags = append(flags.LdFlags, "-Wl,-headerpad_max_install_names") + flags.Global.LdFlags = append(flags.Global.LdFlags, "-Wl,-headerpad_max_install_names") } } @@ -308,14 +328,14 @@ func (binary *binaryDecorator) link(ctx ModuleContext, var linkerDeps android.Paths if deps.LinkerFlagsFile.Valid() { - flags.LdFlags = append(flags.LdFlags, "$$(cat "+deps.LinkerFlagsFile.String()+")") + flags.Local.LdFlags = append(flags.Local.LdFlags, "$$(cat "+deps.LinkerFlagsFile.String()+")") linkerDeps = append(linkerDeps, deps.LinkerFlagsFile.Path()) } if flags.DynamicLinker != "" { - flags.LdFlags = append(flags.LdFlags, "-Wl,-dynamic-linker,"+flags.DynamicLinker) + flags.Local.LdFlags = append(flags.Local.LdFlags, "-Wl,-dynamic-linker,"+flags.DynamicLinker) } else if ctx.toolchain().Bionic() && !binary.static() { - flags.LdFlags = append(flags.LdFlags, "-Wl,--no-dynamic-linker") + flags.Local.LdFlags = append(flags.Local.LdFlags, "-Wl,--no-dynamic-linker") } builderFlags := flagsToBuilderFlags(flags) @@ -326,7 +346,7 @@ func (binary *binaryDecorator) link(ctx ModuleContext, } strippedOutputFile := outputFile outputFile = android.PathForModuleOut(ctx, "unstripped", fileName) - binary.stripper.strip(ctx, outputFile, strippedOutputFile, builderFlags) + binary.stripper.stripExecutableOrSharedLib(ctx, outputFile, strippedOutputFile, builderFlags) } binary.unstrippedOutputFile = outputFile @@ -338,6 +358,8 @@ func (binary *binaryDecorator) link(ctx ModuleContext, flagsToBuilderFlags(flags), afterPrefixSymbols) } + outputFile = maybeInjectBoringSSLHash(ctx, outputFile, binary.Properties.Inject_bssl_hash, fileName) + if Bool(binary.baseLinker.Properties.Use_version_lib) { if ctx.Host() { versionedOutputFile := outputFile @@ -350,7 +372,7 @@ func (binary *binaryDecorator) link(ctx ModuleContext, if binary.stripper.needsStrip(ctx) { out := android.PathForModuleOut(ctx, "versioned-stripped", fileName) binary.distFile = android.OptionalPathForPath(out) - binary.stripper.strip(ctx, versionedOutputFile, out, builderFlags) + binary.stripper.stripExecutableOrSharedLib(ctx, versionedOutputFile, out, builderFlags) } binary.injectVersionSymbol(ctx, outputFile, versionedOutputFile) @@ -384,7 +406,7 @@ func (binary *binaryDecorator) link(ctx ModuleContext, TransformObjToDynamicBinary(ctx, objs.objFiles, sharedLibs, deps.StaticLibs, deps.LateStaticLibs, deps.WholeStaticLibs, linkerDeps, deps.CrtBegin, deps.CrtEnd, true, - builderFlags, outputFile) + builderFlags, outputFile, nil) objs.coverageFiles = append(objs.coverageFiles, deps.StaticLibObjs.coverageFiles...) objs.coverageFiles = append(objs.coverageFiles, deps.WholeStaticLibObjs.coverageFiles...) @@ -398,11 +420,11 @@ func (binary *binaryDecorator) link(ctx ModuleContext, } if Bool(binary.Properties.Symlink_preferred_arch) { - if String(binary.Properties.Stem) == "" && String(binary.Properties.Suffix) == "" { - ctx.PropertyErrorf("symlink_preferred_arch", "must also specify stem or suffix") + if String(binary.Properties.Suffix) == "" { + ctx.PropertyErrorf("symlink_preferred_arch", "must also specify suffix") } if ctx.TargetPrimary() { - binary.symlinks = append(binary.symlinks, ctx.baseModuleName()) + binary.symlinks = append(binary.symlinks, binary.getStemWithoutSuffix(ctx)) } } @@ -444,7 +466,8 @@ func (binary *binaryDecorator) install(ctx ModuleContext, file android.Path) { // Bionic binaries (e.g. linker) is installed to the bootstrap subdirectory. // The original path becomes a symlink to the corresponding file in the // runtime APEX. - if installToBootstrap(ctx.baseModuleName(), ctx.Config()) && ctx.Arch().Native && ctx.apexName() == "" && !ctx.inRecovery() { + translatedArch := ctx.Target().NativeBridge == android.NativeBridgeEnabled + if InstallToBootstrap(ctx.baseModuleName(), ctx.Config()) && !translatedArch && ctx.apexName() == "" && !ctx.inRamdisk() && !ctx.inRecovery() { if ctx.Device() && isBionic(ctx.baseModuleName()) { binary.installSymlinkToRuntimeApex(ctx, file) } diff --git a/cc/binary_sdk_member.go b/cc/binary_sdk_member.go new file mode 100644 index 000000000..88ac51349 --- /dev/null +++ b/cc/binary_sdk_member.go @@ -0,0 +1,146 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cc + +import ( + "path/filepath" + + "android/soong/android" + + "github.com/google/blueprint" +) + +func init() { + android.RegisterSdkMemberType(ccBinarySdkMemberType) +} + +var ccBinarySdkMemberType = &binarySdkMemberType{ + SdkMemberTypeBase: android.SdkMemberTypeBase{ + PropertyName: "native_binaries", + }, +} + +type binarySdkMemberType struct { + android.SdkMemberTypeBase +} + +func (mt *binarySdkMemberType) AddDependencies(mctx android.BottomUpMutatorContext, dependencyTag blueprint.DependencyTag, names []string) { + targets := mctx.MultiTargets() + for _, lib := range names { + for _, target := range targets { + name, version := StubsLibNameAndVersion(lib) + if version == "" { + version = LatestStubsVersionFor(mctx.Config(), name) + } + mctx.AddFarVariationDependencies(append(target.Variations(), []blueprint.Variation{ + {Mutator: "version", Variation: version}, + }...), dependencyTag, name) + } + } +} + +func (mt *binarySdkMemberType) IsInstance(module android.Module) bool { + // Check the module to see if it can be used with this module type. + if m, ok := module.(*Module); ok { + for _, allowableMemberType := range m.sdkMemberTypes { + if allowableMemberType == mt { + return true + } + } + } + + return false +} + +func (mt *binarySdkMemberType) AddPrebuiltModule(ctx android.SdkMemberContext, member android.SdkMember) android.BpModule { + return ctx.SnapshotBuilder().AddPrebuiltModule(member, "cc_prebuilt_binary") +} + +func (mt *binarySdkMemberType) CreateVariantPropertiesStruct() android.SdkMemberProperties { + return &nativeBinaryInfoProperties{} +} + +const ( + nativeBinaryDir = "bin" +) + +// path to the native binary. Relative to <sdk_root>/<api_dir> +func nativeBinaryPathFor(lib nativeBinaryInfoProperties) string { + return filepath.Join(lib.OsPrefix(), lib.archType, + nativeBinaryDir, lib.outputFile.Base()) +} + +// nativeBinaryInfoProperties represents properties of a native binary +// +// The exported (capitalized) fields will be examined and may be changed during common value extraction. +// The unexported fields will be left untouched. +type nativeBinaryInfoProperties struct { + android.SdkMemberPropertiesBase + + // archType is not exported as if set (to a non default value) it is always arch specific. + // This is "" for common properties. + archType string + + // outputFile is not exported as it is always arch specific. + outputFile android.Path + + // The set of shared libraries + // + // This field is exported as its contents may not be arch specific. + SharedLibs []string + + // The set of system shared libraries + // + // This field is exported as its contents may not be arch specific. + SystemSharedLibs []string +} + +func (p *nativeBinaryInfoProperties) PopulateFromVariant(ctx android.SdkMemberContext, variant android.Module) { + ccModule := variant.(*Module) + + p.archType = ccModule.Target().Arch.ArchType.String() + p.outputFile = getRequiredMemberOutputFile(ctx, ccModule) + + if ccModule.linker != nil { + specifiedDeps := specifiedDeps{} + specifiedDeps = ccModule.linker.linkerSpecifiedDeps(specifiedDeps) + + p.SharedLibs = specifiedDeps.sharedLibs + p.SystemSharedLibs = specifiedDeps.systemSharedLibs + } +} + +func (p *nativeBinaryInfoProperties) AddToPropertySet(ctx android.SdkMemberContext, propertySet android.BpPropertySet) { + if p.Compile_multilib != "" { + propertySet.AddProperty("compile_multilib", p.Compile_multilib) + } + + builder := ctx.SnapshotBuilder() + if p.outputFile != nil { + propertySet.AddProperty("srcs", []string{nativeBinaryPathFor(*p)}) + + builder.CopyToSnapshot(p.outputFile, nativeBinaryPathFor(*p)) + } + + if len(p.SharedLibs) > 0 { + propertySet.AddPropertyWithTag("shared_libs", p.SharedLibs, builder.SdkMemberReferencePropertyTag(false)) + } + + // SystemSharedLibs needs to be propagated if it's a list, even if it's empty, + // so check for non-nil instead of nonzero length. + if p.SystemSharedLibs != nil { + propertySet.AddPropertyWithTag("system_shared_libs", p.SystemSharedLibs, builder.SdkMemberReferencePropertyTag(false)) + } +} diff --git a/cc/builder.go b/cc/builder.go index 98c58dd84..5deb1299e 100644 --- a/cc/builder.go +++ b/cc/builder.go @@ -20,13 +20,12 @@ package cc import ( "fmt" - "os" "path/filepath" "runtime" - "strconv" "strings" "github.com/google/blueprint" + "github.com/google/blueprint/pathtools" "android/soong/android" "android/soong/cc/config" @@ -45,14 +44,6 @@ var ( } ) -func absSrcDir() string { - dir, err := os.Getwd() - if err != nil { - panic(fmt.Errorf("failed to get working directory: %s", err)) - } - return dir -} - var ( pctx = android.NewPackageContext("android/soong/cc") @@ -75,7 +66,7 @@ var ( ld, ldRE = remoteexec.StaticRules(pctx, "ld", blueprint.RuleParams{ Command: "$reTemplate$ldCmd ${crtBegin} @${out}.rsp " + - "${libFlags} ${crtEnd} -o ${out} ${ldFlags}", + "${libFlags} ${crtEnd} -o ${out} ${ldFlags} ${extraLibFlags}", CommandDeps: []string{"$ldCmd"}, Rspfile: "${out}.rsp", RspfileContent: "${in}", @@ -85,27 +76,27 @@ var ( &remoteexec.REParams{ Labels: map[string]string{"type": "link", "tool": "clang"}, ExecStrategy: "${config.RECXXLinksExecStrategy}", - Inputs: []string{"${out}.rsp"}, + Inputs: []string{"${out}.rsp", "$implicitInputs"}, RSPFile: "${out}.rsp", - OutputFiles: []string{"${out}"}, + OutputFiles: []string{"${out}", "$implicitOutputs"}, ToolchainInputs: []string{"$ldCmd"}, Platform: map[string]string{remoteexec.PoolKey: "${config.RECXXLinksPool}"}, - }, []string{"ldCmd", "crtBegin", "libFlags", "crtEnd", "ldFlags"}, nil) + }, []string{"ldCmd", "crtBegin", "libFlags", "crtEnd", "ldFlags", "extraLibFlags"}, []string{"implicitInputs", "implicitOutputs"}) partialLd, partialLdRE = remoteexec.StaticRules(pctx, "partialLd", blueprint.RuleParams{ // Without -no-pie, clang 7.0 adds -pie to link Android files, // but -r and -pie cannot be used together. - Command: "$reTemplate$ldCmd -nostdlib -no-pie -Wl,-r ${in} -o ${out} ${ldFlags}", + Command: "$reTemplate$ldCmd -fuse-ld=lld -nostdlib -no-pie -Wl,-r ${in} -o ${out} ${ldFlags}", CommandDeps: []string{"$ldCmd"}, }, &remoteexec.REParams{ Labels: map[string]string{"type": "link", "tool": "clang"}, ExecStrategy: "${config.RECXXLinksExecStrategy}", - Inputs: []string{"$inCommaList"}, - OutputFiles: []string{"${out}"}, + Inputs: []string{"$inCommaList", "$implicitInputs"}, + OutputFiles: []string{"${out}", "$implicitOutputs"}, ToolchainInputs: []string{"$ldCmd"}, Platform: map[string]string{remoteexec.PoolKey: "${config.RECXXLinksPool}"}, - }, []string{"ldCmd", "ldFlags"}, []string{"inCommaList"}) + }, []string{"ldCmd", "ldFlags"}, []string{"implicitInputs", "inCommaList", "implicitOutputs"}) ar = pctx.AndroidStaticRule("ar", blueprint.RuleParams{ @@ -116,20 +107,6 @@ var ( }, "arCmd", "arFlags") - darwinAr = pctx.AndroidStaticRule("darwinAr", - blueprint.RuleParams{ - Command: "rm -f ${out} && ${config.MacArPath} $arFlags $out $in", - CommandDeps: []string{"${config.MacArPath}"}, - }, - "arFlags") - - darwinAppendAr = pctx.AndroidStaticRule("darwinAppendAr", - blueprint.RuleParams{ - Command: "cp -f ${inAr} ${out}.tmp && ${config.MacArPath} $arFlags ${out}.tmp $in && mv ${out}.tmp ${out}", - CommandDeps: []string{"${config.MacArPath}", "${inAr}"}, - }, - "arFlags", "inAr") - darwinStrip = pctx.AndroidStaticRule("darwinStrip", blueprint.RuleParams{ Command: "${config.MacStripPath} -u -r -o $out $in", @@ -168,6 +145,17 @@ var ( }, "args", "crossCompile") + _ = pctx.SourcePathVariable("archiveRepackPath", "build/soong/scripts/archive_repack.sh") + + archiveRepack = pctx.AndroidStaticRule("archiveRepack", + blueprint.RuleParams{ + Depfile: "${out}.d", + Deps: blueprint.DepsGCC, + Command: "CLANG_BIN=${config.ClangBin} $archiveRepackPath -i ${in} -o ${out} -d ${out}.d ${objects}", + CommandDeps: []string{"$archiveRepackPath"}, + }, + "objects") + emptyFile = pctx.AndroidStaticRule("emptyFile", blueprint.RuleParams{ Command: "rm -f $out && touch $out", @@ -187,8 +175,8 @@ var ( clangTidy = pctx.AndroidStaticRule("clangTidy", blueprint.RuleParams{ - Command: "rm -f $out && CLANG_TIDY=${config.ClangBin}/clang-tidy ${config.ClangTidyShellPath} $tidyFlags $in -- $cFlags && touch $out", - CommandDeps: []string{"${config.ClangBin}/clang-tidy", "${config.ClangTidyShellPath}"}, + Command: "rm -f $out && ${config.ClangBin}/clang-tidy $tidyFlags $in -- $cFlags && touch $out", + CommandDeps: []string{"${config.ClangBin}/clang-tidy"}, }, "cFlags", "tidyFlags") @@ -205,7 +193,7 @@ var ( windres = pctx.AndroidStaticRule("windres", blueprint.RuleParams{ - Command: "$windresCmd $flags -I$$(dirname $in) -i $in -o $out", + Command: "$windresCmd $flags -I$$(dirname $in) -i $in -o $out --preprocessor \"${config.ClangBin}/clang -E -xc-header -DRC_INVOKED\"", CommandDeps: []string{"$windresCmd"}, }, "windresCmd", "flags") @@ -222,20 +210,28 @@ var ( ExecStrategy: "${config.REAbiDumperExecStrategy}", Platform: map[string]string{ remoteexec.PoolKey: "${config.RECXXPool}", - "InputRootAbsolutePath": absSrcDir(), + "InputRootAbsolutePath": android.AbsSrcDirForExistingUseCases(), }, }, []string{"cFlags", "exportDirs"}, nil) _ = pctx.SourcePathVariable("sAbiLinker", "prebuilts/clang-tools/${config.HostPrebuiltTag}/bin/header-abi-linker") + _ = pctx.SourcePathVariable("sAbiLinkerLibs", "prebuilts/clang-tools/${config.HostPrebuiltTag}/lib64") - sAbiLink = pctx.AndroidStaticRule("sAbiLink", + sAbiLink, sAbiLinkRE = remoteexec.StaticRules(pctx, "sAbiLink", blueprint.RuleParams{ - Command: "$sAbiLinker -o ${out} $symbolFilter -arch $arch $exportedHeaderFlags @${out}.rsp ", + Command: "$reTemplate$sAbiLinker -o ${out} $symbolFilter -arch $arch $exportedHeaderFlags @${out}.rsp ", CommandDeps: []string{"$sAbiLinker"}, Rspfile: "${out}.rsp", RspfileContent: "${in}", - }, - "symbolFilter", "arch", "exportedHeaderFlags") + }, &remoteexec.REParams{ + Labels: map[string]string{"type": "tool", "name": "abi-linker"}, + ExecStrategy: "${config.REAbiLinkerExecStrategy}", + Inputs: []string{"$sAbiLinkerLibs", "${out}.rsp", "$implicitInputs"}, + RSPFile: "${out}.rsp", + OutputFiles: []string{"$out"}, + ToolchainInputs: []string{"$sAbiLinker"}, + Platform: map[string]string{remoteexec.PoolKey: "${config.RECXXPool}"}, + }, []string{"symbolFilter", "arch", "exportedHeaderFlags"}, []string{"implicitInputs"}) _ = pctx.SourcePathVariable("sAbiDiffer", "prebuilts/clang-tools/${config.HostPrebuiltTag}/bin/header-abi-diff") @@ -265,6 +261,22 @@ var ( Rspfile: "$out.rsp", RspfileContent: "$in", }) + + _ = pctx.SourcePathVariable("cxxExtractor", + "prebuilts/clang-tools/${config.HostPrebuiltTag}/bin/cxx_extractor") + _ = pctx.SourcePathVariable("kytheVnames", "build/soong/vnames.json") + _ = pctx.VariableFunc("kytheCorpus", + func(ctx android.PackageVarContext) string { return ctx.Config().XrefCorpusName() }) + _ = pctx.VariableFunc("kytheCuEncoding", + func(ctx android.PackageVarContext) string { return ctx.Config().XrefCuEncoding() }) + kytheExtract = pctx.StaticRule("kythe", + blueprint.RuleParams{ + Command: `rm -f $out && ` + + `KYTHE_CORPUS=${kytheCorpus} KYTHE_OUTPUT_FILE=$out KYTHE_VNAMES=$kytheVnames KYTHE_KZIP_ENCODING=${kytheCuEncoding} ` + + `$cxxExtractor $cFlags $in `, + CommandDeps: []string{"$cxxExtractor", "$kytheVnames"}, + }, + "cFlags") ) func init() { @@ -283,40 +295,56 @@ func init() { } type builderFlags struct { - globalFlags string - arFlags string - asFlags string - cFlags string - toolingCFlags string // A separate set of cFlags for clang LibTooling tools - toolingCppFlags string // A separate set of cppFlags for clang LibTooling tools - conlyFlags string - cppFlags string - ldFlags string - libFlags string - yaccFlags string - tidyFlags string - sAbiFlags string - yasmFlags string - aidlFlags string - rsFlags string - toolchain config.Toolchain - tidy bool - coverage bool - sAbiDump bool + globalCommonFlags string + globalAsFlags string + globalYasmFlags string + globalCFlags string + globalToolingCFlags string // A separate set of cFlags for clang LibTooling tools + globalToolingCppFlags string // A separate set of cppFlags for clang LibTooling tools + globalConlyFlags string + globalCppFlags string + globalLdFlags string + + localCommonFlags string + localAsFlags string + localYasmFlags string + localCFlags string + localToolingCFlags string // A separate set of cFlags for clang LibTooling tools + localToolingCppFlags string // A separate set of cppFlags for clang LibTooling tools + localConlyFlags string + localCppFlags string + localLdFlags string + + libFlags string + extraLibFlags string + tidyFlags string + sAbiFlags string + aidlFlags string + rsFlags string + toolchain config.Toolchain + tidy bool + gcovCoverage bool + sAbiDump bool + emitXrefs bool + + assemblerWithCpp bool systemIncludeFlags string groupStaticLibs bool - stripKeepSymbols bool - stripKeepSymbolsList string - stripKeepMiniDebugInfo bool - stripAddGnuDebuglink bool - stripUseGnuStrip bool + stripKeepSymbols bool + stripKeepSymbolsList string + stripKeepSymbolsAndDebugFrame bool + stripKeepMiniDebugInfo bool + stripAddGnuDebuglink bool + stripUseGnuStrip bool proto android.ProtoFlags protoC bool protoOptionsFile bool + + yacc *YaccProperties } type Objects struct { @@ -324,6 +352,7 @@ type Objects struct { tidyFiles android.Paths coverageFiles android.Paths sAbiDumpFiles android.Paths + kytheFiles android.Paths } func (a Objects) Copy() Objects { @@ -332,6 +361,7 @@ func (a Objects) Copy() Objects { tidyFiles: append(android.Paths{}, a.tidyFiles...), coverageFiles: append(android.Paths{}, a.coverageFiles...), sAbiDumpFiles: append(android.Paths{}, a.sAbiDumpFiles...), + kytheFiles: append(android.Paths{}, a.kytheFiles...), } } @@ -341,6 +371,7 @@ func (a Objects) Append(b Objects) Objects { tidyFiles: append(a.tidyFiles, b.tidyFiles...), coverageFiles: append(a.coverageFiles, b.coverageFiles...), sAbiDumpFiles: append(a.sAbiDumpFiles, b.sAbiDumpFiles...), + kytheFiles: append(a.kytheFiles, b.kytheFiles...), } } @@ -354,43 +385,53 @@ func TransformSourceToObj(ctx android.ModuleContext, subdir string, srcFiles and tidyFiles = make(android.Paths, 0, len(srcFiles)) } var coverageFiles android.Paths - if flags.coverage { + if flags.gcovCoverage { coverageFiles = make(android.Paths, 0, len(srcFiles)) } - - commonFlags := strings.Join([]string{ - flags.globalFlags, - flags.systemIncludeFlags, - }, " ") - - toolingCflags := strings.Join([]string{ - commonFlags, - flags.toolingCFlags, - flags.conlyFlags, - }, " ") - - cflags := strings.Join([]string{ - commonFlags, - flags.cFlags, - flags.conlyFlags, - }, " ") - - toolingCppflags := strings.Join([]string{ - commonFlags, - flags.toolingCFlags, - flags.toolingCppFlags, - }, " ") - - cppflags := strings.Join([]string{ - commonFlags, - flags.cFlags, - flags.cppFlags, - }, " ") - - asflags := strings.Join([]string{ - commonFlags, - flags.asFlags, - }, " ") + var kytheFiles android.Paths + if flags.emitXrefs { + kytheFiles = make(android.Paths, 0, len(srcFiles)) + } + + // Produce fully expanded flags for use by C tools, C compiles, C++ tools, C++ compiles, and asm compiles + // respectively. + toolingCflags := flags.globalCommonFlags + " " + + flags.globalToolingCFlags + " " + + flags.globalConlyFlags + " " + + flags.localCommonFlags + " " + + flags.localToolingCFlags + " " + + flags.localConlyFlags + " " + + flags.systemIncludeFlags + + cflags := flags.globalCommonFlags + " " + + flags.globalCFlags + " " + + flags.globalConlyFlags + " " + + flags.localCommonFlags + " " + + flags.localCFlags + " " + + flags.localConlyFlags + " " + + flags.systemIncludeFlags + + toolingCppflags := flags.globalCommonFlags + " " + + flags.globalToolingCFlags + " " + + flags.globalToolingCppFlags + " " + + flags.localCommonFlags + " " + + flags.localToolingCFlags + " " + + flags.localToolingCppFlags + " " + + flags.systemIncludeFlags + + cppflags := flags.globalCommonFlags + " " + + flags.globalCFlags + " " + + flags.globalCppFlags + " " + + flags.localCommonFlags + " " + + flags.localCFlags + " " + + flags.localCppFlags + " " + + flags.systemIncludeFlags + + asflags := flags.globalCommonFlags + " " + + flags.globalAsFlags + " " + + flags.localCommonFlags + " " + + flags.localAsFlags + " " + + flags.systemIncludeFlags var sAbiDumpFiles android.Paths if flags.sAbiDump { @@ -417,7 +458,7 @@ func TransformSourceToObj(ctx android.ModuleContext, subdir string, srcFiles and Implicits: cFlagsDeps, OrderOnly: pathDeps, Args: map[string]string{ - "asFlags": flags.yasmFlags, + "asFlags": flags.globalYasmFlags + " " + flags.localYasmFlags, }, }) continue @@ -435,34 +476,42 @@ func TransformSourceToObj(ctx android.ModuleContext, subdir string, srcFiles and }, }) continue + case ".o": + objFiles[i] = srcFile + continue } - var moduleCflags string - var moduleToolingCflags string + var moduleFlags string + var moduleToolingFlags string + var ccCmd string tidy := flags.tidy - coverage := flags.coverage + coverage := flags.gcovCoverage dump := flags.sAbiDump rule := cc + emitXref := flags.emitXrefs switch srcFile.Ext() { case ".s": - rule = ccNoDeps + if !flags.assemblerWithCpp { + rule = ccNoDeps + } fallthrough case ".S": ccCmd = "clang" - moduleCflags = asflags + moduleFlags = asflags tidy = false coverage = false dump = false + emitXref = false case ".c": ccCmd = "clang" - moduleCflags = cflags - moduleToolingCflags = toolingCflags - case ".cpp", ".cc", ".mm": + moduleFlags = cflags + moduleToolingFlags = toolingCflags + case ".cpp", ".cc", ".cxx", ".mm": ccCmd = "clang++" - moduleCflags = cppflags - moduleToolingCflags = toolingCppflags + moduleFlags = cppflags + moduleToolingFlags = toolingCppflags default: ctx.ModuleErrorf("File %s has unknown extension", srcFile) continue @@ -488,11 +537,27 @@ func TransformSourceToObj(ctx android.ModuleContext, subdir string, srcFiles and Implicits: cFlagsDeps, OrderOnly: pathDeps, Args: map[string]string{ - "cFlags": moduleCflags, + "cFlags": moduleFlags, "ccCmd": ccCmd, }, }) + if emitXref { + kytheFile := android.ObjPathWithExt(ctx, subdir, srcFile, "kzip") + ctx.Build(pctx, android.BuildParams{ + Rule: kytheExtract, + Description: "Xref C++ extractor " + srcFile.Rel(), + Output: kytheFile, + Input: srcFile, + Implicits: cFlagsDeps, + OrderOnly: pathDeps, + Args: map[string]string{ + "cFlags": moduleFlags, + }, + }) + kytheFiles = append(kytheFiles, kytheFile) + } + if tidy { tidyFile := android.ObjPathWithExt(ctx, subdir, srcFile, "tidy") tidyFiles = append(tidyFiles, tidyFile) @@ -504,9 +569,11 @@ func TransformSourceToObj(ctx android.ModuleContext, subdir string, srcFiles and Input: srcFile, // We must depend on objFile, since clang-tidy doesn't // support exporting dependencies. - Implicit: objFile, + Implicit: objFile, + Implicits: cFlagsDeps, + OrderOnly: pathDeps, Args: map[string]string{ - "cFlags": moduleToolingCflags, + "cFlags": moduleToolingFlags, "tidyFlags": flags.tidyFlags, }, }) @@ -526,8 +593,10 @@ func TransformSourceToObj(ctx android.ModuleContext, subdir string, srcFiles and Output: sAbiDumpFile, Input: srcFile, Implicit: objFile, + Implicits: cFlagsDeps, + OrderOnly: pathDeps, Args: map[string]string{ - "cFlags": moduleToolingCflags, + "cFlags": moduleToolingFlags, "exportDirs": flags.sAbiFlags, }, }) @@ -540,6 +609,7 @@ func TransformSourceToObj(ctx android.ModuleContext, subdir string, srcFiles and tidyFiles: tidyFiles, coverageFiles: coverageFiles, sAbiDumpFiles: sAbiDumpFiles, + kytheFiles: kytheFiles, } } @@ -547,19 +617,11 @@ func TransformSourceToObj(ctx android.ModuleContext, subdir string, srcFiles and func TransformObjToStaticLib(ctx android.ModuleContext, objFiles android.Paths, flags builderFlags, outputFile android.ModuleOutPath, deps android.Paths) { - if ctx.Darwin() { - transformDarwinObjToStaticLib(ctx, objFiles, flags, outputFile, deps) - return - } - arCmd := "${config.ClangBin}/llvm-ar" - arFlags := "crsD" + arFlags := "crsPD" if !ctx.Darwin() { arFlags += " -format=gnu" } - if flags.arFlags != "" { - arFlags += " " + flags.arFlags - } ctx.Build(pctx, android.BuildParams{ Rule: ar, @@ -574,87 +636,11 @@ func TransformObjToStaticLib(ctx android.ModuleContext, objFiles android.Paths, }) } -// Generate a rule for compiling multiple .o files to a static library (.a) on -// darwin. The darwin ar tool doesn't support @file for list files, and has a -// very small command line length limit, so we have to split the ar into multiple -// steps, each appending to the previous one. -func transformDarwinObjToStaticLib(ctx android.ModuleContext, objFiles android.Paths, - flags builderFlags, outputFile android.ModuleOutPath, deps android.Paths) { - - arFlags := "cqs" - - if len(objFiles) == 0 { - dummy := android.PathForModuleOut(ctx, "dummy"+objectExtension) - dummyAr := android.PathForModuleOut(ctx, "dummy"+staticLibraryExtension) - - ctx.Build(pctx, android.BuildParams{ - Rule: emptyFile, - Description: "empty object file", - Output: dummy, - Implicits: deps, - }) - - ctx.Build(pctx, android.BuildParams{ - Rule: darwinAr, - Description: "empty static archive", - Output: dummyAr, - Input: dummy, - Args: map[string]string{ - "arFlags": arFlags, - }, - }) - - ctx.Build(pctx, android.BuildParams{ - Rule: darwinAppendAr, - Description: "static link " + outputFile.Base(), - Output: outputFile, - Input: dummy, - Args: map[string]string{ - "arFlags": "d", - "inAr": dummyAr.String(), - }, - }) - - return - } - - // ARG_MAX on darwin is 262144, use half that to be safe - objFilesLists, err := splitListForSize(objFiles, 131072) - if err != nil { - ctx.ModuleErrorf("%s", err.Error()) - } - - var in, out android.WritablePath - for i, l := range objFilesLists { - in = out - out = outputFile - if i != len(objFilesLists)-1 { - out = android.PathForModuleOut(ctx, outputFile.Base()+strconv.Itoa(i)) - } - - build := android.BuildParams{ - Rule: darwinAr, - Description: "static link " + out.Base(), - Output: out, - Inputs: l, - Implicits: deps, - Args: map[string]string{ - "arFlags": arFlags, - }, - } - if i != 0 { - build.Rule = darwinAppendAr - build.Args["inAr"] = in.String() - } - ctx.Build(pctx, build) - } -} - // Generate a rule for compiling multiple .o files, plus static libraries, whole static libraries, // and shared libraries, to a shared library (.so) or dynamic executable func TransformObjToDynamicBinary(ctx android.ModuleContext, objFiles, sharedLibs, staticLibs, lateStaticLibs, wholeStaticLibs, deps android.Paths, - crtBegin, crtEnd android.OptionalPath, groupLate bool, flags builderFlags, outputFile android.WritablePath) { + crtBegin, crtEnd android.OptionalPath, groupLate bool, flags builderFlags, outputFile android.WritablePath, implicitOutputs android.WritablePaths) { ldCmd := "${config.ClangBin}/clang++" @@ -691,7 +677,11 @@ func TransformObjToDynamicBinary(ctx android.ModuleContext, } for _, lib := range sharedLibs { - libFlagsList = append(libFlagsList, lib.String()) + libFile := lib.String() + if ctx.Windows() { + libFile = pathtools.ReplaceExtension(libFile, "lib") + } + libFlagsList = append(libFlagsList, libFile) } deps = append(deps, staticLibs...) @@ -702,23 +692,28 @@ func TransformObjToDynamicBinary(ctx android.ModuleContext, } rule := ld + args := map[string]string{ + "ldCmd": ldCmd, + "crtBegin": crtBegin.String(), + "libFlags": strings.Join(libFlagsList, " "), + "extraLibFlags": flags.extraLibFlags, + "ldFlags": flags.globalLdFlags + " " + flags.localLdFlags, + "crtEnd": crtEnd.String(), + } if ctx.Config().IsEnvTrue("RBE_CXX_LINKS") { rule = ldRE + args["implicitOutputs"] = strings.Join(implicitOutputs.Strings(), ",") + args["implicitInputs"] = strings.Join(deps.Strings(), ",") } ctx.Build(pctx, android.BuildParams{ Rule: rule, Description: "link " + outputFile.Base(), Output: outputFile, + ImplicitOutputs: implicitOutputs, Inputs: objFiles, Implicits: deps, - Args: map[string]string{ - "ldCmd": ldCmd, - "crtBegin": crtBegin.String(), - "libFlags": strings.Join(libFlagsList, " "), - "ldFlags": flags.ldFlags, - "crtEnd": crtEnd.String(), - }, + Args: args, }) } @@ -729,9 +724,6 @@ func TransformDumpToLinkedDump(ctx android.ModuleContext, sAbiDumps android.Path excludedSymbolVersions, excludedSymbolTags []string) android.OptionalPath { outputFile := android.PathForModuleOut(ctx, baseName+".lsdump") - sabiLock.Lock() - lsdumpPaths = append(lsdumpPaths, outputFile.String()) - sabiLock.Unlock() implicits := android.Paths{soFile} symbolFilterStr := "-so " + soFile.String() @@ -746,17 +738,30 @@ func TransformDumpToLinkedDump(ctx android.ModuleContext, sAbiDumps android.Path for _, tag := range excludedSymbolTags { symbolFilterStr += " --exclude-symbol-tag " + tag } + rule := sAbiLink + args := map[string]string{ + "symbolFilter": symbolFilterStr, + "arch": ctx.Arch().ArchType.Name, + "exportedHeaderFlags": exportedHeaderFlags, + } + if ctx.Config().IsEnvTrue("RBE_ABI_LINKER") { + rule = sAbiLinkRE + rbeImplicits := implicits.Strings() + for _, p := range strings.Split(exportedHeaderFlags, " ") { + if len(p) > 2 { + // Exclude the -I prefix. + rbeImplicits = append(rbeImplicits, p[2:]) + } + } + args["implicitInputs"] = strings.Join(rbeImplicits, ",") + } ctx.Build(pctx, android.BuildParams{ - Rule: sAbiLink, + Rule: rule, Description: "header-abi-linker " + outputFile.Base(), Output: outputFile, Inputs: sAbiDumps, Implicits: implicits, - Args: map[string]string{ - "symbolFilter": symbolFilterStr, - "arch": ctx.Arch().ArchType.Name, - "exportedHeaderFlags": exportedHeaderFlags, - }, + Args: args, }) return android.OptionalPathForPath(outputFile) } @@ -773,7 +778,7 @@ func UnzipRefDump(ctx android.ModuleContext, zippedRefDump android.Path, baseNam } func SourceAbiDiff(ctx android.ModuleContext, inputDump android.Path, referenceDump android.Path, - baseName, exportedHeaderFlags string, isLlndk, isVndkExt bool) android.OptionalPath { + baseName, exportedHeaderFlags string, isLlndk, isNdk, isVndkExt bool) android.OptionalPath { outputFile := android.PathForModuleOut(ctx, baseName+".abidiff") libName := strings.TrimSuffix(baseName, filepath.Ext(baseName)) @@ -783,9 +788,14 @@ func SourceAbiDiff(ctx android.ModuleContext, inputDump android.Path, referenceD if exportedHeaderFlags == "" { localAbiCheckAllowFlags = append(localAbiCheckAllowFlags, "-advice-only") } - if isLlndk { - localAbiCheckAllowFlags = append(localAbiCheckAllowFlags, "-consider-opaque-types-different") + if isLlndk || isNdk { createReferenceDumpFlags = "--llndk" + if isLlndk { + // TODO(b/130324828): "-consider-opaque-types-different" should apply to + // both LLNDK and NDK shared libs. However, a known issue in header-abi-diff + // breaks libaaudio. Remove the if-guard after the issue is fixed. + localAbiCheckAllowFlags = append(localAbiCheckAllowFlags, "-consider-opaque-types-different") + } } if isVndkExt { localAbiCheckAllowFlags = append(localAbiCheckAllowFlags, "-allow-extensions") @@ -839,24 +849,26 @@ func TransformSharedObjectToToc(ctx android.ModuleContext, inputFile android.Pat // Generate a rule for compiling multiple .o files to a .o using ld partial linking func TransformObjsToObj(ctx android.ModuleContext, objFiles android.Paths, - flags builderFlags, outputFile android.WritablePath) { + flags builderFlags, outputFile android.WritablePath, deps android.Paths) { ldCmd := "${config.ClangBin}/clang++" rule := partialLd args := map[string]string{ "ldCmd": ldCmd, - "ldFlags": flags.ldFlags, + "ldFlags": flags.globalLdFlags + " " + flags.localLdFlags, } if ctx.Config().IsEnvTrue("RBE_CXX_LINKS") { rule = partialLdRE args["inCommaList"] = strings.Join(objFiles.Strings(), ",") + args["implicitInputs"] = strings.Join(deps.Strings(), ",") } ctx.Build(pctx, android.BuildParams{ Rule: rule, Description: "link " + outputFile.Base(), Output: outputFile, Inputs: objFiles, + Implicits: deps, Args: args, }) } @@ -896,6 +908,9 @@ func TransformStrip(ctx android.ModuleContext, inputFile android.Path, if flags.stripKeepSymbolsList != "" { args += " -k" + flags.stripKeepSymbolsList } + if flags.stripKeepSymbolsAndDebugFrame { + args += " --keep-symbols-and-debug-frame" + } if flags.stripUseGnuStrip { args += " --use-gnu-strip" } @@ -942,6 +957,20 @@ func TransformCoverageFilesToZip(ctx android.ModuleContext, return android.OptionalPath{} } +func TransformArchiveRepack(ctx android.ModuleContext, inputFile android.Path, + outputFile android.WritablePath, objects []string) { + + ctx.Build(pctx, android.BuildParams{ + Rule: archiveRepack, + Description: "Repack archive " + outputFile.Base(), + Output: outputFile, + Input: inputFile, + Args: map[string]string{ + "objects": strings.Join(objects, " "), + }, + }) +} + func gccCmd(toolchain config.Toolchain, cmd string) string { return filepath.Join(toolchain.GccRoot(), "bin", toolchain.GccTriple()+"-"+cmd) } @@ -19,6 +19,8 @@ package cc // is handled in builder.go import ( + "fmt" + "io" "strconv" "strings" @@ -31,26 +33,37 @@ import ( ) func init() { - android.RegisterModuleType("cc_defaults", defaultsFactory) + RegisterCCBuildComponents(android.InitRegistrationContext) - android.PreDepsMutators(func(ctx android.RegisterMutatorsContext) { - ctx.BottomUp("image", ImageMutator).Parallel() - ctx.BottomUp("link", LinkageMutator).Parallel() + pctx.Import("android/soong/cc/config") +} + +func RegisterCCBuildComponents(ctx android.RegistrationContext) { + ctx.RegisterModuleType("cc_defaults", defaultsFactory) + + ctx.PreDepsMutators(func(ctx android.RegisterMutatorsContext) { + ctx.BottomUp("sdk", sdkMutator).Parallel() ctx.BottomUp("vndk", VndkMutator).Parallel() - ctx.BottomUp("ndk_api", ndkApiMutator).Parallel() - ctx.BottomUp("test_per_src", testPerSrcMutator).Parallel() + ctx.BottomUp("link", LinkageMutator).Parallel() + ctx.BottomUp("ndk_api", NdkApiMutator).Parallel() + ctx.BottomUp("test_per_src", TestPerSrcMutator).Parallel() ctx.BottomUp("version", VersionMutator).Parallel() ctx.BottomUp("begin", BeginMutator).Parallel() - ctx.BottomUp("sysprop", SyspropMutator).Parallel() + ctx.BottomUp("sysprop_cc", SyspropMutator).Parallel() + ctx.BottomUp("vendor_snapshot", VendorSnapshotMutator).Parallel() + ctx.BottomUp("vendor_snapshot_source", VendorSnapshotSourceMutator).Parallel() }) - android.PostDepsMutators(func(ctx android.RegisterMutatorsContext) { + ctx.PostDepsMutators(func(ctx android.RegisterMutatorsContext) { ctx.TopDown("asan_deps", sanitizerDepsMutator(asan)) ctx.BottomUp("asan", sanitizerMutator(asan)).Parallel() ctx.TopDown("hwasan_deps", sanitizerDepsMutator(hwasan)) ctx.BottomUp("hwasan", sanitizerMutator(hwasan)).Parallel() + ctx.TopDown("fuzzer_deps", sanitizerDepsMutator(fuzzer)) + ctx.BottomUp("fuzzer", sanitizerMutator(fuzzer)).Parallel() + // cfi mutator shouldn't run before sanitizers that return true for // incompatibleWithCfi() ctx.TopDown("cfi_deps", sanitizerDepsMutator(cfi)) @@ -62,7 +75,7 @@ func init() { ctx.TopDown("tsan_deps", sanitizerDepsMutator(tsan)) ctx.BottomUp("tsan", sanitizerMutator(tsan)).Parallel() - ctx.TopDown("sanitize_runtime_deps", sanitizerRuntimeDepsMutator) + ctx.TopDown("sanitize_runtime_deps", sanitizerRuntimeDepsMutator).Parallel() ctx.BottomUp("sanitize_runtime", sanitizerRuntimeMutator).Parallel() ctx.BottomUp("coverage", coverageMutator).Parallel() @@ -74,7 +87,7 @@ func init() { ctx.TopDown("double_loadable", checkDoubleLoadableLibraries).Parallel() }) - pctx.Import("android/soong/cc/config") + android.RegisterSingletonType("kythe_extract_all", kytheExtractAllFactory) } type Deps struct { @@ -83,12 +96,15 @@ type Deps struct { HeaderLibs []string RuntimeLibs []string + StaticUnwinderIfLegacy bool + ReexportSharedLibHeaders, ReexportStaticLibHeaders, ReexportHeaderLibHeaders []string ObjFiles []string GeneratedSources []string GeneratedHeaders []string + GeneratedDeps []string ReexportGeneratedHeaders []string @@ -115,9 +131,16 @@ type PathDeps struct { // Paths to generated source files GeneratedSources android.Paths GeneratedHeaders android.Paths + GeneratedDeps android.Paths - Flags, ReexportedFlags []string - ReexportedFlagsDeps android.Paths + Flags []string + IncludeDirs android.Paths + SystemIncludeDirs android.Paths + ReexportedDirs android.Paths + ReexportedSystemDirs android.Paths + ReexportedFlags []string + ReexportedGeneratedHeaders android.Paths + ReexportedDeps android.Paths // Paths to crt*.o files CrtBegin, CrtEnd android.OptionalPath @@ -129,32 +152,41 @@ type PathDeps struct { DynamicLinker android.OptionalPath } -type Flags struct { - GlobalFlags []string // Flags that apply to C, C++, and assembly source files - ArFlags []string // Flags that apply to ar +// LocalOrGlobalFlags contains flags that need to have values set globally by the build system or locally by the module +// tracked separately, in order to maintain the required ordering (most of the global flags need to go first on the +// command line so they can be overridden by the local module flags). +type LocalOrGlobalFlags struct { + CommonFlags []string // Flags that apply to C, C++, and assembly source files AsFlags []string // Flags that apply to assembly source files + YasmFlags []string // Flags that apply to yasm assembly source files CFlags []string // Flags that apply to C and C++ source files ToolingCFlags []string // Flags that apply to C and C++ source files parsed by clang LibTooling tools ConlyFlags []string // Flags that apply to C source files CppFlags []string // Flags that apply to C++ source files ToolingCppFlags []string // Flags that apply to C++ source files parsed by clang LibTooling tools - YaccFlags []string // Flags that apply to Yacc source files - aidlFlags []string // Flags that apply to aidl source files - rsFlags []string // Flags that apply to renderscript source files LdFlags []string // Flags that apply to linker command lines - libFlags []string // Flags to add libraries early to the link order - TidyFlags []string // Flags that apply to clang-tidy - SAbiFlags []string // Flags that apply to header-abi-dumper - YasmFlags []string // Flags that apply to yasm assembly source files +} + +type Flags struct { + Local LocalOrGlobalFlags + Global LocalOrGlobalFlags + + aidlFlags []string // Flags that apply to aidl source files + rsFlags []string // Flags that apply to renderscript source files + libFlags []string // Flags to add libraries early to the link order + extraLibFlags []string // Flags to add libraries late in the link order after LdFlags + TidyFlags []string // Flags that apply to clang-tidy + SAbiFlags []string // Flags that apply to header-abi-dumper // Global include flags that apply to C, C++, and assembly source files - // These must be after any module include flags, which will be in GlobalFlags. + // These must be after any module include flags, which will be in CommonFlags. SystemIncludeFlags []string - Toolchain config.Toolchain - Tidy bool - Coverage bool - SAbiDump bool + Toolchain config.Toolchain + Tidy bool + GcovCoverage bool + SAbiDump bool + EmitXrefs bool // If true, generate Ninja rules to generate emitXrefs input files for Kythe RequiredInstructionSet string DynamicLinker string @@ -162,19 +194,14 @@ type Flags struct { CFlagsDeps android.Paths // Files depended on by compiler flags LdFlagsDeps android.Paths // Files depended on by linker flags - GroupStaticLibs bool + AssemblerWithCpp bool + GroupStaticLibs bool proto android.ProtoFlags protoC bool // Whether to use C instead of C++ protoOptionsFile bool // Whether to look for a .options file next to the .proto -} -type ObjectLinkerProperties struct { - // names of other cc_object modules to link into this module using partial linking - Objs []string `android:"arch_variant"` - - // if set, add an extra objcopy --prefix-symbols= step - Prefix_symbols *string + Yacc *YaccProperties } // Properties used to compile all C or C++ modules @@ -182,9 +209,16 @@ type BaseProperties struct { // Deprecated. true is the default, false is invalid. Clang *bool `android:"arch_variant"` - // Minimum sdk version supported when compiling against the ndk + // Minimum sdk version supported when compiling against the ndk. Setting this property causes + // two variants to be built, one for the platform and one for apps. Sdk_version *string + // Minimum sdk version that the artifact should support when it runs as part of mainline modules(APEX). + Min_sdk_version *string + + // If true, always create an sdk variant and don't create a platform variant. + Sdk_variant_only *bool + AndroidMkSharedLibs []string `blueprint:"mutated"` AndroidMkStaticLibs []string `blueprint:"mutated"` AndroidMkRuntimeLibs []string `blueprint:"mutated"` @@ -193,39 +227,72 @@ type BaseProperties struct { PreventInstall bool `blueprint:"mutated"` ApexesProvidingSharedLibs []string `blueprint:"mutated"` - UseVndk bool `blueprint:"mutated"` + ImageVariationPrefix string `blueprint:"mutated"` + VndkVersion string `blueprint:"mutated"` + SubName string `blueprint:"mutated"` // *.logtags files, to combine together in order to generate the /system/etc/event-log-tags // file Logtags []string + // Make this module available when building for ramdisk + Ramdisk_available *bool + // Make this module available when building for recovery Recovery_available *bool - InRecovery bool `blueprint:"mutated"` + // Set by imageMutator + CoreVariantNeeded bool `blueprint:"mutated"` + RamdiskVariantNeeded bool `blueprint:"mutated"` + RecoveryVariantNeeded bool `blueprint:"mutated"` + ExtraVariants []string `blueprint:"mutated"` // Allows this module to use non-APEX version of libraries. Useful // for building binaries that are started before APEXes are activated. Bootstrap *bool + + // Even if DeviceConfig().VndkUseCoreVariant() is set, this module must use vendor variant. + // see soong/cc/config/vndk.go + MustUseVendorVariant bool `blueprint:"mutated"` + + // Used by vendor snapshot to record dependencies from snapshot modules. + SnapshotSharedLibs []string `blueprint:"mutated"` + SnapshotRuntimeLibs []string `blueprint:"mutated"` + + Installable *bool + + // Set by factories of module types that can only be referenced from variants compiled against + // the SDK. + AlwaysSdk bool `blueprint:"mutated"` + + // Variant is an SDK variant created by sdkMutator + IsSdkVariant bool `blueprint:"mutated"` + // Set when both SDK and platform variants are exported to Make to trigger renaming the SDK + // variant to have a ".sdk" suffix. + SdkAndPlatformVariantVisibleToMake bool `blueprint:"mutated"` } type VendorProperties struct { // whether this module should be allowed to be directly depended by other // modules with `vendor: true`, `proprietary: true`, or `vendor_available:true`. - // If set to true, two variants will be built separately, one like - // normal, and the other limited to the set of libraries and headers - // that are exposed to /vendor modules. + // In addition, this module should be allowed to be directly depended by + // product modules with `product_specific: true`. + // If set to true, three variants will be built separately, one like + // normal, another limited to the set of libraries and headers + // that are exposed to /vendor modules, and the other to /product modules. // - // The vendor variant may be used with a different (newer) /system, + // The vendor and product variants may be used with a different (newer) /system, // so it shouldn't have any unversioned runtime dependencies, or // make assumptions about the system that may not be true in the // future. // - // If set to false, this module becomes inaccessible from /vendor modules. + // If set to false, this module becomes inaccessible from /vendor or /product + // modules. // // Default value is true when vndk: {enabled: true} or vendor: true. // // Nothing happens if BOARD_VNDK_VERSION isn't set in the BoardConfig.mk + // If PRODUCT_PRODUCT_VNDK_VERSION isn't set, product variant will not be used. Vendor_available *bool // whether this module is capable of being loaded with other instance @@ -242,26 +309,34 @@ type ModuleContextIntf interface { static() bool staticBinary() bool header() bool + binary() bool + object() bool toolchain() config.Toolchain + canUseSdk() bool useSdk() bool sdkVersion() string useVndk() bool isNdk() bool - isLlndk() bool - isLlndkPublic() bool - isVndkPrivate() bool + isLlndk(config android.Config) bool + isLlndkPublic(config android.Config) bool + isVndkPrivate(config android.Config) bool isVndk() bool isVndkSp() bool isVndkExt() bool + inProduct() bool + inVendor() bool + inRamdisk() bool inRecovery() bool - shouldCreateVndkSourceAbiDump() bool + shouldCreateSourceAbiDump() bool selectedStl() string baseModuleName() string getVndkExtendsModuleName() string isPgoCompile() bool isNDKStubLibrary() bool useClangLld(actx ModuleContext) bool + isForPlatform() bool apexName() string + apexSdkVersion() int hasStubsVariants() bool isStubs() bool bootstrap() bool @@ -275,7 +350,7 @@ type ModuleContext interface { } type BaseModuleContext interface { - android.BaseContext + android.BaseModuleContext ModuleContextIntf } @@ -315,55 +390,73 @@ type linker interface { nativeCoverage() bool coverageOutputFilePath() android.OptionalPath + + // Get the deps that have been explicitly specified in the properties. + // Only updates the + linkerSpecifiedDeps(specifiedDeps specifiedDeps) specifiedDeps +} + +type specifiedDeps struct { + sharedLibs []string + systemSharedLibs []string // Note nil and [] are semantically distinct. } type installer interface { installerProps() []interface{} install(ctx ModuleContext, path android.Path) + everInstallable() bool inData() bool inSanitizerDir() bool hostToolPath() android.OptionalPath relativeInstallPath() string + skipInstall(mod *Module) } -type dependencyTag struct { - blueprint.BaseDependencyTag - name string - library bool - - reexportFlags bool - - explicitlyVersioned bool +type xref interface { + XrefCcFiles() android.Paths } var ( - sharedDepTag = dependencyTag{name: "shared", library: true} - sharedExportDepTag = dependencyTag{name: "shared", library: true, reexportFlags: true} - earlySharedDepTag = dependencyTag{name: "early_shared", library: true} - lateSharedDepTag = dependencyTag{name: "late shared", library: true} - staticDepTag = dependencyTag{name: "static", library: true} - staticExportDepTag = dependencyTag{name: "static", library: true, reexportFlags: true} - lateStaticDepTag = dependencyTag{name: "late static", library: true} - wholeStaticDepTag = dependencyTag{name: "whole static", library: true, reexportFlags: true} - headerDepTag = dependencyTag{name: "header", library: true} - headerExportDepTag = dependencyTag{name: "header", library: true, reexportFlags: true} - genSourceDepTag = dependencyTag{name: "gen source"} - genHeaderDepTag = dependencyTag{name: "gen header"} - genHeaderExportDepTag = dependencyTag{name: "gen header", reexportFlags: true} - objDepTag = dependencyTag{name: "obj"} - crtBeginDepTag = dependencyTag{name: "crtbegin"} - crtEndDepTag = dependencyTag{name: "crtend"} - linkerFlagsDepTag = dependencyTag{name: "linker flags file"} - dynamicLinkerDepTag = dependencyTag{name: "dynamic linker"} - reuseObjTag = dependencyTag{name: "reuse objects"} - staticVariantTag = dependencyTag{name: "static variant"} - ndkStubDepTag = dependencyTag{name: "ndk stub", library: true} - ndkLateStubDepTag = dependencyTag{name: "ndk late stub", library: true} - vndkExtDepTag = dependencyTag{name: "vndk extends", library: true} - runtimeDepTag = dependencyTag{name: "runtime lib"} - coverageDepTag = dependencyTag{name: "coverage"} + sharedExportDepTag = DependencyTag{Name: "shared", Library: true, Shared: true, ReexportFlags: true} + earlySharedDepTag = DependencyTag{Name: "early_shared", Library: true, Shared: true} + lateSharedDepTag = DependencyTag{Name: "late shared", Library: true, Shared: true} + staticExportDepTag = DependencyTag{Name: "static", Library: true, ReexportFlags: true} + lateStaticDepTag = DependencyTag{Name: "late static", Library: true} + staticUnwinderDepTag = DependencyTag{Name: "static unwinder", Library: true} + wholeStaticDepTag = DependencyTag{Name: "whole static", Library: true, ReexportFlags: true} + headerDepTag = DependencyTag{Name: "header", Library: true} + headerExportDepTag = DependencyTag{Name: "header", Library: true, ReexportFlags: true} + genSourceDepTag = DependencyTag{Name: "gen source"} + genHeaderDepTag = DependencyTag{Name: "gen header"} + genHeaderExportDepTag = DependencyTag{Name: "gen header", ReexportFlags: true} + objDepTag = DependencyTag{Name: "obj"} + linkerFlagsDepTag = DependencyTag{Name: "linker flags file"} + dynamicLinkerDepTag = DependencyTag{Name: "dynamic linker"} + reuseObjTag = DependencyTag{Name: "reuse objects"} + staticVariantTag = DependencyTag{Name: "static variant"} + ndkStubDepTag = DependencyTag{Name: "ndk stub", Library: true} + ndkLateStubDepTag = DependencyTag{Name: "ndk late stub", Library: true} + vndkExtDepTag = DependencyTag{Name: "vndk extends", Library: true} + runtimeDepTag = DependencyTag{Name: "runtime lib"} + coverageDepTag = DependencyTag{Name: "coverage"} + testPerSrcDepTag = DependencyTag{Name: "test_per_src"} ) +func IsSharedDepTag(depTag blueprint.DependencyTag) bool { + ccDepTag, ok := depTag.(DependencyTag) + return ok && ccDepTag.Shared +} + +func IsRuntimeDepTag(depTag blueprint.DependencyTag) bool { + ccDepTag, ok := depTag.(DependencyTag) + return ok && ccDepTag == runtimeDepTag +} + +func IsTestPerSrcDepTag(depTag blueprint.DependencyTag) bool { + ccDepTag, ok := depTag.(DependencyTag) + return ok && ccDepTag == testPerSrcDepTag +} + // Module contains the properties and members used by all C/C++ module types, and implements // the blueprint.Module interface. It delegates to compiler, linker, and installer interfaces // to construct the output file. Behavior can be customized with a Customizer interface @@ -371,6 +464,7 @@ type Module struct { android.ModuleBase android.DefaultableModuleBase android.ApexModuleBase + android.SdkBase Properties BaseProperties VendorProperties VendorProperties @@ -379,6 +473,9 @@ type Module struct { hod android.HostOrDeviceSupported multilib android.Multilib + // Allowable SdkMemberTypes of this module type. + sdkMemberTypes []android.SdkMemberType + // delegates, initialize before calling Init features []feature compiler compiler @@ -391,9 +488,6 @@ type Module struct { vndkdep *vndkdep lto *lto pgo *pgo - xom *xom - - androidMkSharedLibDeps []string outputFile android.OptionalPath @@ -410,13 +504,250 @@ type Module struct { depsInLinkOrder android.Paths // only non-nil when this is a shared library that reuses the objects of a static library - staticVariant *Module + staticVariant LinkableInterface + + makeLinkType string + // Kythe (source file indexer) paths for this compilation module + kytheFiles android.Paths + + // For apex variants, this is set as apex.min_sdk_version + apexSdkVersion int +} + +func (c *Module) Toc() android.OptionalPath { + if c.linker != nil { + if library, ok := c.linker.(libraryInterface); ok { + return library.toc() + } + } + panic(fmt.Errorf("Toc() called on non-library module: %q", c.BaseModuleName())) +} + +func (c *Module) ApiLevel() string { + if c.linker != nil { + if stub, ok := c.linker.(*stubDecorator); ok { + return stub.properties.ApiLevel + } + } + panic(fmt.Errorf("ApiLevel() called on non-stub library module: %q", c.BaseModuleName())) +} + +func (c *Module) Static() bool { + if c.linker != nil { + if library, ok := c.linker.(libraryInterface); ok { + return library.static() + } + } + panic(fmt.Errorf("Static() called on non-library module: %q", c.BaseModuleName())) +} + +func (c *Module) Shared() bool { + if c.linker != nil { + if library, ok := c.linker.(libraryInterface); ok { + return library.shared() + } + } + panic(fmt.Errorf("Shared() called on non-library module: %q", c.BaseModuleName())) +} + +func (c *Module) SelectedStl() string { + if c.stl != nil { + return c.stl.Properties.SelectedStl + } + return "" +} + +func (c *Module) ToolchainLibrary() bool { + if _, ok := c.linker.(*toolchainLibraryDecorator); ok { + return true + } + return false +} + +func (c *Module) NdkPrebuiltStl() bool { + if _, ok := c.linker.(*ndkPrebuiltStlLinker); ok { + return true + } + return false +} + +func (c *Module) StubDecorator() bool { + if _, ok := c.linker.(*stubDecorator); ok { + return true + } + return false +} + +func (c *Module) SdkVersion() string { + return String(c.Properties.Sdk_version) +} + +func (c *Module) MinSdkVersion() string { + return String(c.Properties.Min_sdk_version) +} + +func (c *Module) AlwaysSdk() bool { + return c.Properties.AlwaysSdk || Bool(c.Properties.Sdk_variant_only) +} + +func (c *Module) IncludeDirs() android.Paths { + if c.linker != nil { + if library, ok := c.linker.(exportedFlagsProducer); ok { + return library.exportedDirs() + } + } + panic(fmt.Errorf("IncludeDirs called on non-exportedFlagsProducer module: %q", c.BaseModuleName())) +} + +func (c *Module) HasStaticVariant() bool { + if c.staticVariant != nil { + return true + } + return false +} + +func (c *Module) GetStaticVariant() LinkableInterface { + return c.staticVariant +} + +func (c *Module) SetDepsInLinkOrder(depsInLinkOrder []android.Path) { + c.depsInLinkOrder = depsInLinkOrder +} + +func (c *Module) GetDepsInLinkOrder() []android.Path { + return c.depsInLinkOrder +} + +func (c *Module) StubsVersions() []string { + if c.linker != nil { + if library, ok := c.linker.(*libraryDecorator); ok { + return library.Properties.Stubs.Versions + } + } + panic(fmt.Errorf("StubsVersions called on non-library module: %q", c.BaseModuleName())) +} + +func (c *Module) CcLibrary() bool { + if c.linker != nil { + if _, ok := c.linker.(*libraryDecorator); ok { + return true + } + } + return false +} + +func (c *Module) CcLibraryInterface() bool { + if _, ok := c.linker.(libraryInterface); ok { + return true + } + return false +} + +func (c *Module) NonCcVariants() bool { + return false +} + +func (c *Module) SetBuildStubs() { + if c.linker != nil { + if library, ok := c.linker.(*libraryDecorator); ok { + library.MutatedProperties.BuildStubs = true + c.Properties.HideFromMake = true + c.sanitize = nil + c.stl = nil + c.Properties.PreventInstall = true + return + } + if _, ok := c.linker.(*llndkStubDecorator); ok { + c.Properties.HideFromMake = true + return + } + } + panic(fmt.Errorf("SetBuildStubs called on non-library module: %q", c.BaseModuleName())) +} + +func (c *Module) BuildStubs() bool { + if c.linker != nil { + if library, ok := c.linker.(*libraryDecorator); ok { + return library.buildStubs() + } + } + panic(fmt.Errorf("BuildStubs called on non-library module: %q", c.BaseModuleName())) +} + +func (c *Module) SetStubsVersions(version string) { + if c.linker != nil { + if library, ok := c.linker.(*libraryDecorator); ok { + library.MutatedProperties.StubsVersion = version + return + } + if llndk, ok := c.linker.(*llndkStubDecorator); ok { + llndk.libraryDecorator.MutatedProperties.StubsVersion = version + return + } + } + panic(fmt.Errorf("SetStubsVersions called on non-library module: %q", c.BaseModuleName())) +} + +func (c *Module) StubsVersion() string { + if c.linker != nil { + if library, ok := c.linker.(*libraryDecorator); ok { + return library.MutatedProperties.StubsVersion + } + if llndk, ok := c.linker.(*llndkStubDecorator); ok { + return llndk.libraryDecorator.MutatedProperties.StubsVersion + } + } + panic(fmt.Errorf("StubsVersion called on non-library module: %q", c.BaseModuleName())) +} + +func (c *Module) SetStatic() { + if c.linker != nil { + if library, ok := c.linker.(libraryInterface); ok { + library.setStatic() + return + } + } + panic(fmt.Errorf("SetStatic called on non-library module: %q", c.BaseModuleName())) +} + +func (c *Module) SetShared() { + if c.linker != nil { + if library, ok := c.linker.(libraryInterface); ok { + library.setShared() + return + } + } + panic(fmt.Errorf("SetShared called on non-library module: %q", c.BaseModuleName())) +} + +func (c *Module) BuildStaticVariant() bool { + if c.linker != nil { + if library, ok := c.linker.(libraryInterface); ok { + return library.buildStatic() + } + } + panic(fmt.Errorf("BuildStaticVariant called on non-library module: %q", c.BaseModuleName())) +} + +func (c *Module) BuildSharedVariant() bool { + if c.linker != nil { + if library, ok := c.linker.(libraryInterface); ok { + return library.buildShared() + } + } + panic(fmt.Errorf("BuildSharedVariant called on non-library module: %q", c.BaseModuleName())) +} + +func (c *Module) Module() android.Module { + return c } func (c *Module) OutputFile() android.OptionalPath { return c.outputFile } +var _ LinkableInterface = (*Module)(nil) + func (c *Module) UnstrippedOutputFile() android.Path { if c.linker != nil { return c.linker.unstrippedOutputFilePath() @@ -438,6 +769,10 @@ func (c *Module) RelativeInstallPath() string { return "" } +func (c *Module) VndkVersion() string { + return c.Properties.VndkVersion +} + func (c *Module) Init() android.Module { c.AddProperties(&c.Properties, &c.VendorProperties) if c.compiler != nil { @@ -470,9 +805,6 @@ func (c *Module) Init() android.Module { if c.pgo != nil { c.AddProperties(c.pgo.props()...) } - if c.xom != nil { - c.AddProperties(c.xom.props()...) - } for _, feature := range c.features { c.AddProperties(feature.props()...) } @@ -489,10 +821,9 @@ func (c *Module) Init() android.Module { } }) android.InitAndroidArchModule(c, c.hod, c.multilib) - - android.InitDefaultableModule(c) - android.InitApexModule(c) + android.InitSdkAwareModule(c) + android.InitDefaultableModule(c) return c } @@ -508,34 +839,52 @@ func (c *Module) isDependencyRoot() bool { return false } -func (c *Module) useVndk() bool { - return c.Properties.UseVndk +// Returns true if the module is using VNDK libraries instead of the libraries in /system/lib or /system/lib64. +// "product" and "vendor" variant modules return true for this function. +// When BOARD_VNDK_VERSION is set, vendor variants of "vendor_available: true", "vendor: true", +// "soc_specific: true" and more vendor installed modules are included here. +// When PRODUCT_PRODUCT_VNDK_VERSION is set, product variants of "vendor_available: true" or +// "product_specific: true" modules are included here. +func (c *Module) UseVndk() bool { + return c.Properties.VndkVersion != "" +} + +func (c *Module) canUseSdk() bool { + return c.Os() == android.Android && !c.UseVndk() && !c.InRamdisk() && !c.InRecovery() +} + +func (c *Module) UseSdk() bool { + if c.canUseSdk() { + return String(c.Properties.Sdk_version) != "" + } + return false } func (c *Module) isCoverageVariant() bool { return c.coverage.Properties.IsCoverageVariant } -func (c *Module) isNdk() bool { +func (c *Module) IsNdk() bool { return inList(c.Name(), ndkMigratedLibs) } -func (c *Module) isLlndk() bool { +func (c *Module) isLlndk(config android.Config) bool { // Returns true for both LLNDK (public) and LLNDK-private libs. - return inList(c.Name(), llndkLibraries) + return isLlndkLibrary(c.BaseModuleName(), config) } -func (c *Module) isLlndkPublic() bool { +func (c *Module) isLlndkPublic(config android.Config) bool { // Returns true only for LLNDK (public) libs. - return c.isLlndk() && !c.isVndkPrivate() + name := c.BaseModuleName() + return isLlndkLibrary(name, config) && !isVndkPrivateLibrary(name, config) } -func (c *Module) isVndkPrivate() bool { +func (c *Module) isVndkPrivate(config android.Config) bool { // Returns true for LLNDK-private, VNDK-SP-private, and VNDK-core-private. - return inList(c.Name(), vndkPrivateLibraries) + return isVndkPrivateLibrary(c.BaseModuleName(), config) } -func (c *Module) isVndk() bool { +func (c *Module) IsVndk() bool { if vndkdep := c.vndkdep; vndkdep != nil { return vndkdep.isVndk() } @@ -570,8 +919,8 @@ func (c *Module) isVndkExt() bool { return false } -func (c *Module) mustUseVendorVariant() bool { - return c.isVndkSp() || inList(c.Name(), config.VndkMustUseVendorVariantList) +func (c *Module) MustUseVendorVariant() bool { + return c.isVndkSp() || c.Properties.MustUseVendorVariant } func (c *Module) getVndkExtendsModuleName() string { @@ -581,17 +930,45 @@ func (c *Module) getVndkExtendsModuleName() string { return "" } -// Returns true only when this module is configured to have core and vendor +// Returns true only when this module is configured to have core, product and vendor // variants. -func (c *Module) hasVendorVariant() bool { - return c.isVndk() || Bool(c.VendorProperties.Vendor_available) +func (c *Module) HasVendorVariant() bool { + return c.IsVndk() || Bool(c.VendorProperties.Vendor_available) +} + +const ( + // VendorVariationPrefix is the variant prefix used for /vendor code that compiles + // against the VNDK. + VendorVariationPrefix = "vendor." + + // ProductVariationPrefix is the variant prefix used for /product code that compiles + // against the VNDK. + ProductVariationPrefix = "product." +) + +// Returns true if the module is "product" variant. Usually these modules are installed in /product +func (c *Module) inProduct() bool { + return c.Properties.ImageVariationPrefix == ProductVariationPrefix +} + +// Returns true if the module is "vendor" variant. Usually these modules are installed in /vendor +func (c *Module) inVendor() bool { + return c.Properties.ImageVariationPrefix == VendorVariationPrefix } -func (c *Module) inRecovery() bool { - return c.Properties.InRecovery || c.ModuleBase.InstallInRecovery() +func (c *Module) InRamdisk() bool { + return c.ModuleBase.InRamdisk() || c.ModuleBase.InstallInRamdisk() } -func (c *Module) onlyInRecovery() bool { +func (c *Module) InRecovery() bool { + return c.ModuleBase.InRecovery() || c.ModuleBase.InstallInRecovery() +} + +func (c *Module) OnlyInRamdisk() bool { + return c.ModuleBase.InstallInRamdisk() +} + +func (c *Module) OnlyInRecovery() bool { return c.ModuleBase.InstallInRecovery() } @@ -619,26 +996,76 @@ func (c *Module) bootstrap() bool { } func (c *Module) nativeCoverage() bool { + // Bug: http://b/137883967 - native-bridge modules do not currently work with coverage + if c.Target().NativeBridge == android.NativeBridgeEnabled { + return false + } return c.linker != nil && c.linker.nativeCoverage() } +func (c *Module) isSnapshotPrebuilt() bool { + if p, ok := c.linker.(interface{ isSnapshotPrebuilt() bool }); ok { + return p.isSnapshotPrebuilt() + } + return false +} + +func (c *Module) ExportedIncludeDirs() android.Paths { + if flagsProducer, ok := c.linker.(exportedFlagsProducer); ok { + return flagsProducer.exportedDirs() + } + return nil +} + +func (c *Module) ExportedSystemIncludeDirs() android.Paths { + if flagsProducer, ok := c.linker.(exportedFlagsProducer); ok { + return flagsProducer.exportedSystemDirs() + } + return nil +} + +func (c *Module) ExportedFlags() []string { + if flagsProducer, ok := c.linker.(exportedFlagsProducer); ok { + return flagsProducer.exportedFlags() + } + return nil +} + +func (c *Module) ExportedDeps() android.Paths { + if flagsProducer, ok := c.linker.(exportedFlagsProducer); ok { + return flagsProducer.exportedDeps() + } + return nil +} + +func (c *Module) ExportedGeneratedHeaders() android.Paths { + if flagsProducer, ok := c.linker.(exportedFlagsProducer); ok { + return flagsProducer.exportedGeneratedHeaders() + } + return nil +} + func isBionic(name string) bool { switch name { - case "libc", "libm", "libdl", "linker": + case "libc", "libm", "libdl", "libdl_android", "linker": return true } return false } -func installToBootstrap(name string, config android.Config) bool { +func InstallToBootstrap(name string, config android.Config) bool { if name == "libclang_rt.hwasan-aarch64-android" { return inList("hwaddress", config.SanitizeDevice()) } return isBionic(name) } +func (c *Module) XrefCcFiles() android.Paths { + return c.kytheFiles +} + type baseModuleContext struct { - android.BaseContext + android.BaseModuleContext moduleContextImpl } @@ -652,9 +1079,14 @@ type moduleContext struct { moduleContextImpl } +func (ctx *moduleContext) ProductSpecific() bool { + return ctx.ModuleContext.ProductSpecific() || + (ctx.mod.HasVendorVariant() && ctx.mod.inProduct() && !ctx.mod.IsVndk()) +} + func (ctx *moduleContext) SocSpecific() bool { return ctx.ModuleContext.SocSpecific() || - (ctx.mod.hasVendorVariant() && ctx.mod.useVndk() && !ctx.mod.isVndk()) + (ctx.mod.HasVendorVariant() && ctx.mod.inVendor() && !ctx.mod.IsVndk()) } type moduleContextImpl struct { @@ -678,25 +1110,30 @@ func (ctx *moduleContextImpl) header() bool { return ctx.mod.header() } +func (ctx *moduleContextImpl) binary() bool { + return ctx.mod.binary() +} + +func (ctx *moduleContextImpl) object() bool { + return ctx.mod.object() +} + +func (ctx *moduleContextImpl) canUseSdk() bool { + return ctx.mod.canUseSdk() +} + func (ctx *moduleContextImpl) useSdk() bool { - if ctx.ctx.Device() && !ctx.useVndk() && !ctx.inRecovery() && !ctx.ctx.Fuchsia() { - return String(ctx.mod.Properties.Sdk_version) != "" - } - return false + return ctx.mod.UseSdk() } func (ctx *moduleContextImpl) sdkVersion() string { if ctx.ctx.Device() { if ctx.useVndk() { - vndk_ver := ctx.ctx.DeviceConfig().VndkVersion() - if vndk_ver == "current" { - platform_vndk_ver := ctx.ctx.DeviceConfig().PlatformVndkVersion() - if inList(platform_vndk_ver, ctx.ctx.Config().PlatformVersionCombinedCodenames()) { - return "current" - } - return platform_vndk_ver + vndkVer := ctx.mod.VndkVersion() + if inList(vndkVer, ctx.ctx.Config().PlatformVersionActiveCodenames()) { + return "current" } - return vndk_ver + return vndkVer } return String(ctx.mod.Properties.Sdk_version) } @@ -704,27 +1141,27 @@ func (ctx *moduleContextImpl) sdkVersion() string { } func (ctx *moduleContextImpl) useVndk() bool { - return ctx.mod.useVndk() + return ctx.mod.UseVndk() } func (ctx *moduleContextImpl) isNdk() bool { - return ctx.mod.isNdk() + return ctx.mod.IsNdk() } -func (ctx *moduleContextImpl) isLlndk() bool { - return ctx.mod.isLlndk() +func (ctx *moduleContextImpl) isLlndk(config android.Config) bool { + return ctx.mod.isLlndk(config) } -func (ctx *moduleContextImpl) isLlndkPublic() bool { - return ctx.mod.isLlndkPublic() +func (ctx *moduleContextImpl) isLlndkPublic(config android.Config) bool { + return ctx.mod.isLlndkPublic(config) } -func (ctx *moduleContextImpl) isVndkPrivate() bool { - return ctx.mod.isVndkPrivate() +func (ctx *moduleContextImpl) isVndkPrivate(config android.Config) bool { + return ctx.mod.isVndkPrivate(config) } func (ctx *moduleContextImpl) isVndk() bool { - return ctx.mod.isVndk() + return ctx.mod.IsVndk() } func (ctx *moduleContextImpl) isPgoCompile() bool { @@ -744,15 +1181,27 @@ func (ctx *moduleContextImpl) isVndkExt() bool { } func (ctx *moduleContextImpl) mustUseVendorVariant() bool { - return ctx.mod.mustUseVendorVariant() + return ctx.mod.MustUseVendorVariant() +} + +func (ctx *moduleContextImpl) inProduct() bool { + return ctx.mod.inProduct() +} + +func (ctx *moduleContextImpl) inVendor() bool { + return ctx.mod.inVendor() +} + +func (ctx *moduleContextImpl) inRamdisk() bool { + return ctx.mod.InRamdisk() } func (ctx *moduleContextImpl) inRecovery() bool { - return ctx.mod.inRecovery() + return ctx.mod.InRecovery() } // Check whether ABI dumps should be created for this module. -func (ctx *moduleContextImpl) shouldCreateVndkSourceAbiDump() bool { +func (ctx *moduleContextImpl) shouldCreateSourceAbiDump() bool { if ctx.ctx.Config().IsEnvTrue("SKIP_ABI_CHECKS") { return false } @@ -770,22 +1219,11 @@ func (ctx *moduleContextImpl) shouldCreateVndkSourceAbiDump() bool { // Host modules do not need ABI dumps. return false } - if !ctx.mod.IsForPlatform() { - // APEX variants do not need ABI dumps. + if ctx.isStubs() || ctx.isNDKStubLibrary() { + // Stubs do not need ABI dumps. return false } - if ctx.isNdk() { - return true - } - if ctx.isLlndkPublic() { - return true - } - if ctx.useVndk() && ctx.isVndk() && !ctx.isVndkPrivate() { - // Return true if this is VNDK-core, VNDK-SP, or VNDK-Ext and this is not - // VNDK-private. - return true - } - return false + return true } func (ctx *moduleContextImpl) selectedStl() string { @@ -807,10 +1245,18 @@ func (ctx *moduleContextImpl) getVndkExtendsModuleName() string { return ctx.mod.getVndkExtendsModuleName() } +func (ctx *moduleContextImpl) isForPlatform() bool { + return ctx.mod.IsForPlatform() +} + func (ctx *moduleContextImpl) apexName() string { return ctx.mod.ApexName() } +func (ctx *moduleContextImpl) apexSdkVersion() int { + return ctx.mod.apexSdkVersion +} + func (ctx *moduleContextImpl) hasStubsVariants() bool { return ctx.mod.HasStubsVariants() } @@ -846,7 +1292,6 @@ func newModule(hod android.HostOrDeviceSupported, multilib android.Multilib) *Mo module.vndkdep = &vndkdep{} module.lto = <o{} module.pgo = &pgo{} - module.xom = &xom{} return module } @@ -907,30 +1352,100 @@ func orderDeps(directStaticDeps []android.Path, directSharedDeps []android.Path, return orderedAllDeps, orderedDeclaredDeps } -func orderStaticModuleDeps(module *Module, staticDeps []*Module, sharedDeps []*Module) (results []android.Path) { +func orderStaticModuleDeps(module LinkableInterface, staticDeps []LinkableInterface, sharedDeps []LinkableInterface) (results []android.Path) { // convert Module to Path + var depsInLinkOrder []android.Path allTransitiveDeps := make(map[android.Path][]android.Path, len(staticDeps)) staticDepFiles := []android.Path{} for _, dep := range staticDeps { - allTransitiveDeps[dep.outputFile.Path()] = dep.depsInLinkOrder - staticDepFiles = append(staticDepFiles, dep.outputFile.Path()) + // The OutputFile may not be valid for a variant not present, and the AllowMissingDependencies flag is set. + if dep.OutputFile().Valid() { + allTransitiveDeps[dep.OutputFile().Path()] = dep.GetDepsInLinkOrder() + staticDepFiles = append(staticDepFiles, dep.OutputFile().Path()) + } } sharedDepFiles := []android.Path{} for _, sharedDep := range sharedDeps { - staticAnalogue := sharedDep.staticVariant - if staticAnalogue != nil { - allTransitiveDeps[staticAnalogue.outputFile.Path()] = staticAnalogue.depsInLinkOrder - sharedDepFiles = append(sharedDepFiles, staticAnalogue.outputFile.Path()) + if sharedDep.HasStaticVariant() { + staticAnalogue := sharedDep.GetStaticVariant() + allTransitiveDeps[staticAnalogue.OutputFile().Path()] = staticAnalogue.GetDepsInLinkOrder() + sharedDepFiles = append(sharedDepFiles, staticAnalogue.OutputFile().Path()) } } // reorder the dependencies based on transitive dependencies - module.depsInLinkOrder, results = orderDeps(staticDepFiles, sharedDepFiles, allTransitiveDeps) + depsInLinkOrder, results = orderDeps(staticDepFiles, sharedDepFiles, allTransitiveDeps) + module.SetDepsInLinkOrder(depsInLinkOrder) return results } +func (c *Module) IsTestPerSrcAllTestsVariation() bool { + test, ok := c.linker.(testPerSrc) + return ok && test.isAllTestsVariation() +} + +func (c *Module) getNameSuffixWithVndkVersion(ctx android.ModuleContext) string { + // Returns the name suffix for product and vendor variants. If the VNDK version is not + // "current", it will append the VNDK version to the name suffix. + var vndkVersion string + var nameSuffix string + if c.inProduct() { + vndkVersion = ctx.DeviceConfig().ProductVndkVersion() + nameSuffix = productSuffix + } else { + vndkVersion = ctx.DeviceConfig().VndkVersion() + nameSuffix = vendorSuffix + } + if vndkVersion == "current" { + vndkVersion = ctx.DeviceConfig().PlatformVndkVersion() + } + if c.Properties.VndkVersion != vndkVersion { + // add version suffix only if the module is using different vndk version than the + // version in product or vendor partition. + nameSuffix += "." + c.Properties.VndkVersion + } + return nameSuffix +} + func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { + // Handle the case of a test module split by `test_per_src` mutator. + // + // The `test_per_src` mutator adds an extra variation named "", depending on all the other + // `test_per_src` variations of the test module. Set `outputFile` to an empty path for this + // module and return early, as this module does not produce an output file per se. + if c.IsTestPerSrcAllTestsVariation() { + c.outputFile = android.OptionalPath{} + return + } + + c.makeLinkType = c.getMakeLinkType(actx) + + c.Properties.SubName = "" + + if c.Target().NativeBridge == android.NativeBridgeEnabled { + c.Properties.SubName += nativeBridgeSuffix + } + + _, llndk := c.linker.(*llndkStubDecorator) + _, llndkHeader := c.linker.(*llndkHeadersDecorator) + if llndk || llndkHeader || (c.UseVndk() && c.HasVendorVariant()) { + // .vendor.{version} suffix is added for vendor variant or .product.{version} suffix is + // added for product variant only when we have vendor and product variants with core + // variant. The suffix is not added for vendor-only or product-only module. + c.Properties.SubName += c.getNameSuffixWithVndkVersion(actx) + } else if _, ok := c.linker.(*vndkPrebuiltLibraryDecorator); ok { + // .vendor suffix is added for backward compatibility with VNDK snapshot whose names with + // such suffixes are already hard-coded in prebuilts/vndk/.../Android.bp. + c.Properties.SubName += vendorSuffix + } else if c.InRamdisk() && !c.OnlyInRamdisk() { + c.Properties.SubName += ramdiskSuffix + } else if c.InRecovery() && !c.OnlyInRecovery() { + c.Properties.SubName += recoverySuffix + } else if c.Properties.IsSdkVariant && c.Properties.SdkAndPlatformVariantVisibleToMake { + c.Properties.SubName += sdkSuffix + } + ctx := &moduleContext{ ModuleContext: actx, moduleContextImpl: moduleContextImpl{ @@ -950,6 +1465,7 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { flags := Flags{ Toolchain: c.toolchain(ctx), + EmitXrefs: ctx.Config().EmitXrefRules(), } if c.compiler != nil { flags = c.compiler.compilerFlags(ctx, flags, deps) @@ -972,9 +1488,6 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { if c.pgo != nil { flags = c.pgo.flags(ctx, flags) } - if c.xom != nil { - flags = c.xom.flags(ctx, flags) - } for _, feature := range c.features { flags = feature.flags(ctx, flags) } @@ -982,24 +1495,35 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { return } - flags.CFlags, _ = filterList(flags.CFlags, config.IllegalFlags) - flags.CppFlags, _ = filterList(flags.CppFlags, config.IllegalFlags) - flags.ConlyFlags, _ = filterList(flags.ConlyFlags, config.IllegalFlags) + flags.Local.CFlags, _ = filterList(flags.Local.CFlags, config.IllegalFlags) + flags.Local.CppFlags, _ = filterList(flags.Local.CppFlags, config.IllegalFlags) + flags.Local.ConlyFlags, _ = filterList(flags.Local.ConlyFlags, config.IllegalFlags) + + flags.Local.CommonFlags = append(flags.Local.CommonFlags, deps.Flags...) + + for _, dir := range deps.IncludeDirs { + flags.Local.CommonFlags = append(flags.Local.CommonFlags, "-I"+dir.String()) + } + for _, dir := range deps.SystemIncludeDirs { + flags.Local.CommonFlags = append(flags.Local.CommonFlags, "-isystem "+dir.String()) + } - flags.GlobalFlags = append(flags.GlobalFlags, deps.Flags...) c.flags = flags // We need access to all the flags seen by a source file. if c.sabi != nil { flags = c.sabi.flags(ctx, flags) } + + flags.AssemblerWithCpp = inList("-xassembler-with-cpp", flags.Local.AsFlags) + // Optimization to reduce size of build.ninja // Replace the long list of flags for each file with a module-local variable - ctx.Variable(pctx, "cflags", strings.Join(flags.CFlags, " ")) - ctx.Variable(pctx, "cppflags", strings.Join(flags.CppFlags, " ")) - ctx.Variable(pctx, "asflags", strings.Join(flags.AsFlags, " ")) - flags.CFlags = []string{"$cflags"} - flags.CppFlags = []string{"$cppflags"} - flags.AsFlags = []string{"$asflags"} + ctx.Variable(pctx, "cflags", strings.Join(flags.Local.CFlags, " ")) + ctx.Variable(pctx, "cppflags", strings.Join(flags.Local.CppFlags, " ")) + ctx.Variable(pctx, "asflags", strings.Join(flags.Local.AsFlags, " ")) + flags.Local.CFlags = []string{"$cflags"} + flags.Local.CppFlags = []string{"$cppflags"} + flags.Local.AsFlags = []string{"$asflags"} var objs Objects if c.compiler != nil { @@ -1007,6 +1531,7 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { if ctx.Failed() { return } + c.kytheFiles = objs.kytheFiles } if c.linker != nil { @@ -1023,23 +1548,37 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { // (unless it is explicitly referenced via .bootstrap suffix or the // module is marked with 'bootstrap: true'). if c.HasStubsVariants() && - android.DirectlyInAnyApex(ctx, ctx.baseModuleName()) && - !c.inRecovery() && !c.useVndk() && !c.static() && !c.isCoverageVariant() && + android.DirectlyInAnyApex(ctx, ctx.baseModuleName()) && !c.InRamdisk() && + !c.InRecovery() && !c.UseVndk() && !c.static() && !c.isCoverageVariant() && c.IsStubs() { c.Properties.HideFromMake = false // unhide // Note: this is still non-installable } + + // glob exported headers for snapshot, if BOARD_VNDK_VERSION is current. + if i, ok := c.linker.(snapshotLibraryInterface); ok && ctx.DeviceConfig().VndkVersion() == "current" { + if isSnapshotAware(ctx, c) { + i.collectHeadersForSnapshot(ctx) + } + } } - if c.installer != nil && !c.Properties.PreventInstall && c.IsForPlatform() && c.outputFile.Valid() { + if c.installable() { c.installer.install(ctx, c.outputFile.Path()) if ctx.Failed() { return } + } else if !proptools.BoolDefault(c.Properties.Installable, true) { + // If the module has been specifically configure to not be installed then + // skip the installation as otherwise it will break when running inside make + // as the output path to install will not be specified. Not all uninstallable + // modules can skip installation as some are needed for resolving make side + // dependencies. + c.SkipInstall() } } -func (c *Module) toolchain(ctx android.BaseContext) config.Toolchain { +func (c *Module) toolchain(ctx android.BaseModuleContext) config.Toolchain { if c.cachedToolchain == nil { c.cachedToolchain = config.FindToolchain(ctx.Os(), ctx.Arch()) } @@ -1160,7 +1699,7 @@ func (c *Module) deps(ctx DepsContext) Deps { func (c *Module) beginMutator(actx android.BottomUpMutatorContext) { ctx := &baseModuleContext{ - BaseContext: actx, + BaseModuleContext: actx, moduleContextImpl: moduleContextImpl{ mod: c, }, @@ -1171,7 +1710,7 @@ func (c *Module) beginMutator(actx android.BottomUpMutatorContext) { } // Split name#version into name and version -func stubsLibNameAndVersion(name string) (string, string) { +func StubsLibNameAndVersion(name string) (string, string) { if sharp := strings.LastIndex(name, "#"); sharp != -1 && sharp != len(name)-1 { version := name[sharp+1:] libname := name[:sharp] @@ -1181,6 +1720,10 @@ func stubsLibNameAndVersion(name string) (string, string) { } func (c *Module) DepsMutator(actx android.BottomUpMutatorContext) { + if !c.Enabled() { + return + } + ctx := &depsContext{ BottomUpMutatorContext: actx, moduleContextImpl: moduleContextImpl{ @@ -1196,7 +1739,7 @@ func (c *Module) DepsMutator(actx android.BottomUpMutatorContext) { if ctx.Os() == android.Android { version := ctx.sdkVersion() - // rewriteNdkLibs takes a list of names of shared libraries and scans it for three types + // rewriteLibs takes a list of names of shared libraries and scans it for three types // of names: // // 1. Name of an NDK library that refers to a prebuilt module. @@ -1210,21 +1753,42 @@ func (c *Module) DepsMutator(actx android.BottomUpMutatorContext) { // // The caller can then know to add the variantLibs dependencies differently from the // nonvariantLibs - rewriteNdkLibs := func(list []string) (nonvariantLibs []string, variantLibs []string) { + + vendorPublicLibraries := vendorPublicLibraries(actx.Config()) + vendorSnapshotSharedLibs := vendorSnapshotSharedLibs(actx.Config()) + + rewriteVendorLibs := func(lib string) string { + if isLlndkLibrary(lib, ctx.Config()) { + return lib + llndkLibrarySuffix + } + + // only modules with BOARD_VNDK_VERSION uses snapshot. + if c.VndkVersion() != actx.DeviceConfig().VndkVersion() { + return lib + } + + if snapshot, ok := vendorSnapshotSharedLibs.get(lib, actx.Arch().ArchType); ok { + return snapshot + } + + return lib + } + + rewriteLibs := func(list []string) (nonvariantLibs []string, variantLibs []string) { variantLibs = []string{} nonvariantLibs = []string{} for _, entry := range list { // strip #version suffix out - name, _ := stubsLibNameAndVersion(entry) + name, _ := StubsLibNameAndVersion(entry) if ctx.useSdk() && inList(name, ndkPrebuiltSharedLibraries) { if !inList(name, ndkMigratedLibs) { nonvariantLibs = append(nonvariantLibs, name+".ndk."+version) } else { variantLibs = append(variantLibs, name+ndkLibrarySuffix) } - } else if ctx.useVndk() && inList(name, llndkLibraries) { - nonvariantLibs = append(nonvariantLibs, name+llndkLibrarySuffix) - } else if (ctx.Platform() || ctx.ProductSpecific()) && inList(name, vendorPublicLibraries) { + } else if ctx.useVndk() { + nonvariantLibs = append(nonvariantLibs, rewriteVendorLibs(entry)) + } else if (ctx.Platform() || ctx.ProductSpecific()) && inList(name, *vendorPublicLibraries) { vendorPublicLib := name + vendorPublicLibrarySuffix if actx.OtherModuleExists(vendorPublicLib) { nonvariantLibs = append(nonvariantLibs, vendorPublicLib) @@ -1242,9 +1806,14 @@ func (c *Module) DepsMutator(actx android.BottomUpMutatorContext) { return nonvariantLibs, variantLibs } - deps.SharedLibs, variantNdkLibs = rewriteNdkLibs(deps.SharedLibs) - deps.LateSharedLibs, variantLateNdkLibs = rewriteNdkLibs(deps.LateSharedLibs) - deps.ReexportSharedLibHeaders, _ = rewriteNdkLibs(deps.ReexportSharedLibHeaders) + deps.SharedLibs, variantNdkLibs = rewriteLibs(deps.SharedLibs) + deps.LateSharedLibs, variantLateNdkLibs = rewriteLibs(deps.LateSharedLibs) + deps.ReexportSharedLibHeaders, _ = rewriteLibs(deps.ReexportSharedLibHeaders) + if ctx.useVndk() { + for idx, lib := range deps.RuntimeLibs { + deps.RuntimeLibs[idx] = rewriteVendorLibs(lib) + } + } } buildStubs := false @@ -1256,16 +1825,31 @@ func (c *Module) DepsMutator(actx android.BottomUpMutatorContext) { } } + rewriteSnapshotLibs := func(lib string, snapshotMap *snapshotMap) string { + // only modules with BOARD_VNDK_VERSION uses snapshot. + if c.VndkVersion() != actx.DeviceConfig().VndkVersion() { + return lib + } + + if snapshot, ok := snapshotMap.get(lib, actx.Arch().ArchType); ok { + return snapshot + } + + return lib + } + + vendorSnapshotHeaderLibs := vendorSnapshotHeaderLibs(actx.Config()) for _, lib := range deps.HeaderLibs { depTag := headerDepTag if inList(lib, deps.ReexportHeaderLibHeaders) { depTag = headerExportDepTag } + + lib = rewriteSnapshotLibs(lib, vendorSnapshotHeaderLibs) + if buildStubs { - actx.AddFarVariationDependencies([]blueprint.Variation{ - {Mutator: "arch", Variation: ctx.Target().String()}, - {Mutator: "image", Variation: c.imageVariation()}, - }, depTag, lib) + actx.AddFarVariationDependencies(append(ctx.Target().Variations(), c.ImageVariation()), + depTag, lib) } else { actx.AddVariationDependencies(nil, depTag, lib) } @@ -1278,19 +1862,23 @@ func (c *Module) DepsMutator(actx android.BottomUpMutatorContext) { } syspropImplLibraries := syspropImplLibraries(actx.Config()) + vendorSnapshotStaticLibs := vendorSnapshotStaticLibs(actx.Config()) for _, lib := range deps.WholeStaticLibs { depTag := wholeStaticDepTag if impl, ok := syspropImplLibraries[lib]; ok { lib = impl } + + lib = rewriteSnapshotLibs(lib, vendorSnapshotStaticLibs) + actx.AddVariationDependencies([]blueprint.Variation{ {Mutator: "link", Variation: "static"}, }, depTag, lib) } for _, lib := range deps.StaticLibs { - depTag := staticDepTag + depTag := StaticDepTag if inList(lib, deps.ReexportStaticLibHeaders) { depTag = staticExportDepTag } @@ -1299,36 +1887,49 @@ func (c *Module) DepsMutator(actx android.BottomUpMutatorContext) { lib = impl } + lib = rewriteSnapshotLibs(lib, vendorSnapshotStaticLibs) + actx.AddVariationDependencies([]blueprint.Variation{ {Mutator: "link", Variation: "static"}, }, depTag, lib) } - actx.AddVariationDependencies([]blueprint.Variation{ - {Mutator: "link", Variation: "static"}, - }, lateStaticDepTag, deps.LateStaticLibs...) + // staticUnwinderDep is treated as staticDep for Q apexes + // so that native libraries/binaries are linked with static unwinder + // because Q libc doesn't have unwinder APIs + if deps.StaticUnwinderIfLegacy { + actx.AddVariationDependencies([]blueprint.Variation{ + {Mutator: "link", Variation: "static"}, + }, staticUnwinderDepTag, rewriteSnapshotLibs(staticUnwinder(actx), vendorSnapshotStaticLibs)) + } - addSharedLibDependencies := func(depTag dependencyTag, name string, version string) { + for _, lib := range deps.LateStaticLibs { + actx.AddVariationDependencies([]blueprint.Variation{ + {Mutator: "link", Variation: "static"}, + }, lateStaticDepTag, rewriteSnapshotLibs(lib, vendorSnapshotStaticLibs)) + } + + addSharedLibDependencies := func(depTag DependencyTag, name string, version string) { var variations []blueprint.Variation variations = append(variations, blueprint.Variation{Mutator: "link", Variation: "shared"}) - versionVariantAvail := !ctx.useVndk() && !c.inRecovery() - if version != "" && versionVariantAvail { + if version != "" && VersionVariantAvailable(c) { // Version is explicitly specified. i.e. libFoo#30 variations = append(variations, blueprint.Variation{Mutator: "version", Variation: version}) - depTag.explicitlyVersioned = true + depTag.ExplicitlyVersioned = true } actx.AddVariationDependencies(variations, depTag, name) - // If the version is not specified, add dependency to the latest stubs library. + // If the version is not specified, add dependency to all stubs libraries. // The stubs library will be used when the depending module is built for APEX and // the dependent module is not in the same APEX. - latestVersion := latestStubsVersionFor(actx.Config(), name) - if version == "" && latestVersion != "" && versionVariantAvail { - actx.AddVariationDependencies([]blueprint.Variation{ - {Mutator: "link", Variation: "shared"}, - {Mutator: "version", Variation: latestVersion}, - }, depTag, name) - // Note that depTag.explicitlyVersioned is false in this case. + if version == "" && VersionVariantAvailable(c) { + for _, ver := range stubsVersionsFor(actx.Config())[name] { + // Note that depTag.ExplicitlyVersioned is false in this case. + actx.AddVariationDependencies([]blueprint.Variation{ + {Mutator: "link", Variation: "shared"}, + {Mutator: "version", Variation: ver}, + }, depTag, name) + } } } @@ -1336,7 +1937,10 @@ func (c *Module) DepsMutator(actx android.BottomUpMutatorContext) { var sharedLibNames []string for _, lib := range deps.SharedLibs { - depTag := sharedDepTag + depTag := SharedDepTag + if c.static() { + depTag = SharedFromStaticDepTag + } if inList(lib, deps.ReexportSharedLibHeaders) { depTag = sharedExportDepTag } @@ -1345,7 +1949,7 @@ func (c *Module) DepsMutator(actx android.BottomUpMutatorContext) { lib = impl } - name, version := stubsLibNameAndVersion(lib) + name, version := StubsLibNameAndVersion(lib) sharedLibNames = append(sharedLibNames, name) addSharedLibDependencies(depTag, name, version) @@ -1377,11 +1981,13 @@ func (c *Module) DepsMutator(actx android.BottomUpMutatorContext) { actx.AddVariationDependencies(nil, objDepTag, deps.ObjFiles...) + vendorSnapshotObjects := vendorSnapshotObjects(actx.Config()) + if deps.CrtBegin != "" { - actx.AddVariationDependencies(nil, crtBeginDepTag, deps.CrtBegin) + actx.AddVariationDependencies(nil, CrtBeginDepTag, rewriteSnapshotLibs(deps.CrtBegin, vendorSnapshotObjects)) } if deps.CrtEnd != "" { - actx.AddVariationDependencies(nil, crtEndDepTag, deps.CrtEnd) + actx.AddVariationDependencies(nil, CrtEndDepTag, rewriteSnapshotLibs(deps.CrtEnd, vendorSnapshotObjects)) } if deps.LinkerFlagsFile != "" { actx.AddDependency(c, linkerFlagsDepTag, deps.LinkerFlagsFile) @@ -1402,12 +2008,8 @@ func (c *Module) DepsMutator(actx android.BottomUpMutatorContext) { if vndkdep := c.vndkdep; vndkdep != nil { if vndkdep.isVndkExt() { - baseModuleMode := vendorMode - if actx.DeviceConfig().VndkVersion() == "" { - baseModuleMode = coreMode - } actx.AddVariationDependencies([]blueprint.Variation{ - {Mutator: "image", Variation: baseModuleMode}, + c.ImageVariation(), {Mutator: "link", Variation: "shared"}, }, vndkExtDepTag, vndkdep.getVndkExtendsModuleName()) } @@ -1422,52 +2024,62 @@ func BeginMutator(ctx android.BottomUpMutatorContext) { // Whether a module can link to another module, taking into // account NDK linking. -func checkLinkType(ctx android.ModuleContext, from *Module, to *Module, tag dependencyTag) { - if from.Target().Os != android.Android { +func checkLinkType(ctx android.ModuleContext, from LinkableInterface, to LinkableInterface, tag DependencyTag) { + if from.Module().Target().Os != android.Android { // Host code is not restricted return } - if from.Properties.UseVndk { + + // VNDK is cc.Module supported only for now. + if ccFrom, ok := from.(*Module); ok && from.UseVndk() { // Though vendor code is limited by the vendor mutator, // each vendor-available module needs to check // link-type for VNDK. - if from.vndkdep != nil { - from.vndkdep.vndkCheckLinkType(ctx, to, tag) + if ccTo, ok := to.(*Module); ok { + if ccFrom.vndkdep != nil { + ccFrom.vndkdep.vndkCheckLinkType(ctx, ccTo, tag) + } + } else { + ctx.ModuleErrorf("Attempting to link VNDK cc.Module with unsupported module type") } return } - if String(from.Properties.Sdk_version) == "" { + if from.SdkVersion() == "" { // Platform code can link to anything return } - if from.inRecovery() { + if from.InRamdisk() { + // Ramdisk code is not NDK + return + } + if from.InRecovery() { // Recovery code is not NDK return } - if _, ok := to.linker.(*toolchainLibraryDecorator); ok { + if to.ToolchainLibrary() { // These are always allowed return } - if _, ok := to.linker.(*ndkPrebuiltStlLinker); ok { + if to.NdkPrebuiltStl() { // These are allowed, but they don't set sdk_version return } - if _, ok := to.linker.(*stubDecorator); ok { + if to.StubDecorator() { // These aren't real libraries, but are the stub shared libraries that are included in // the NDK. return } - if strings.HasPrefix(ctx.ModuleName(), "libclang_rt.") && to.Name() == "libc++" { + if strings.HasPrefix(ctx.ModuleName(), "libclang_rt.") && to.Module().Name() == "libc++" { // Bug: http://b/121358700 - Allow libclang_rt.* shared libraries (with sdk_version) // to link to libc++ (non-NDK and without sdk_version). return } - if String(to.Properties.Sdk_version) == "" { + if to.SdkVersion() == "" { // NDK code linking to platform code is never okay. ctx.ModuleErrorf("depends on non-NDK-built library %q", - ctx.OtherModuleName(to)) + ctx.OtherModuleName(to.Module())) return } @@ -1477,36 +2089,36 @@ func checkLinkType(ctx android.ModuleContext, from *Module, to *Module, tag depe // APIs. // Current can link against anything. - if String(from.Properties.Sdk_version) != "current" { + if from.SdkVersion() != "current" { // Otherwise we need to check. - if String(to.Properties.Sdk_version) == "current" { + if to.SdkVersion() == "current" { // Current can't be linked against by anything else. ctx.ModuleErrorf("links %q built against newer API version %q", - ctx.OtherModuleName(to), "current") + ctx.OtherModuleName(to.Module()), "current") } else { - fromApi, err := strconv.Atoi(String(from.Properties.Sdk_version)) + fromApi, err := strconv.Atoi(from.SdkVersion()) if err != nil { ctx.PropertyErrorf("sdk_version", "Invalid sdk_version value (must be int or current): %q", - String(from.Properties.Sdk_version)) + from.SdkVersion()) } - toApi, err := strconv.Atoi(String(to.Properties.Sdk_version)) + toApi, err := strconv.Atoi(to.SdkVersion()) if err != nil { ctx.PropertyErrorf("sdk_version", "Invalid sdk_version value (must be int or current): %q", - String(to.Properties.Sdk_version)) + to.SdkVersion()) } if toApi > fromApi { ctx.ModuleErrorf("links %q built against newer API version %q", - ctx.OtherModuleName(to), String(to.Properties.Sdk_version)) + ctx.OtherModuleName(to.Module()), to.SdkVersion()) } } } // Also check that the two STL choices are compatible. - fromStl := from.stl.Properties.SelectedStl - toStl := to.stl.Properties.SelectedStl + fromStl := from.SelectedStl() + toStl := to.SelectedStl() if fromStl == "" || toStl == "" { // Libraries that don't use the STL are unrestricted. } else if fromStl == "ndk_system" || toStl == "ndk_system" { @@ -1515,8 +2127,8 @@ func checkLinkType(ctx android.ModuleContext, from *Module, to *Module, tag depe // using either libc++ or nothing. } else if getNdkStlFamily(from) != getNdkStlFamily(to) { ctx.ModuleErrorf("uses %q and depends on %q which uses incompatible %q", - from.stl.Properties.SelectedStl, ctx.OtherModuleName(to), - to.stl.Properties.SelectedStl) + from.SelectedStl(), ctx.OtherModuleName(to.Module()), + to.SelectedStl()) } } @@ -1537,11 +2149,11 @@ func checkDoubleLoadableLibraries(ctx android.TopDownMutatorContext) { } // if target lib has no vendor variant, keep checking dependency graph - if !to.hasVendorVariant() { + if !to.HasVendorVariant() { return true } - if to.isVndkSp() || inList(child.Name(), llndkLibraries) || Bool(to.VendorProperties.Double_loadable) { + if to.isVndkSp() || to.isLlndk(ctx.Config()) || Bool(to.VendorProperties.Double_loadable) { return false } @@ -1556,7 +2168,7 @@ func checkDoubleLoadableLibraries(ctx android.TopDownMutatorContext) { } if module, ok := ctx.Module().(*Module); ok { if lib, ok := module.linker.(*libraryDecorator); ok && lib.shared() { - if inList(ctx.ModuleName(), llndkLibraries) || Bool(module.VendorProperties.Double_loadable) { + if module.isLlndk(ctx.Config()) || Bool(module.VendorProperties.Double_loadable) { ctx.WalkDeps(check) } } @@ -1567,15 +2179,26 @@ func checkDoubleLoadableLibraries(ctx android.TopDownMutatorContext) { func (c *Module) depsToPaths(ctx android.ModuleContext) PathDeps { var depPaths PathDeps - directStaticDeps := []*Module{} - directSharedDeps := []*Module{} + directStaticDeps := []LinkableInterface{} + directSharedDeps := []LinkableInterface{} + + vendorPublicLibraries := vendorPublicLibraries(ctx.Config()) + + reexportExporter := func(exporter exportedFlagsProducer) { + depPaths.ReexportedDirs = append(depPaths.ReexportedDirs, exporter.exportedDirs()...) + depPaths.ReexportedSystemDirs = append(depPaths.ReexportedSystemDirs, exporter.exportedSystemDirs()...) + depPaths.ReexportedFlags = append(depPaths.ReexportedFlags, exporter.exportedFlags()...) + depPaths.ReexportedDeps = append(depPaths.ReexportedDeps, exporter.exportedDeps()...) + depPaths.ReexportedGeneratedHeaders = append(depPaths.ReexportedGeneratedHeaders, exporter.exportedGeneratedHeaders()...) + } ctx.VisitDirectDeps(func(dep android.Module) { depName := ctx.OtherModuleName(dep) depTag := ctx.OtherModuleDependencyTag(dep) - ccDep, _ := dep.(*Module) - if ccDep == nil { + ccDep, ok := dep.(LinkableInterface) + if !ok { + // handling for a few module types that aren't cc Module but that are also supported switch depTag { case genSourceDepTag: @@ -1590,15 +2213,18 @@ func (c *Module) depsToPaths(ctx android.ModuleContext) PathDeps { case genHeaderDepTag, genHeaderExportDepTag: if genRule, ok := dep.(genrule.SourceFileGenerator); ok { depPaths.GeneratedHeaders = append(depPaths.GeneratedHeaders, + genRule.GeneratedSourceFiles()...) + depPaths.GeneratedDeps = append(depPaths.GeneratedDeps, genRule.GeneratedDeps()...) - flags := includeDirsToFlags(genRule.GeneratedHeaderDirs()) - depPaths.Flags = append(depPaths.Flags, flags) + dirs := genRule.GeneratedHeaderDirs() + depPaths.IncludeDirs = append(depPaths.IncludeDirs, dirs...) if depTag == genHeaderExportDepTag { - depPaths.ReexportedFlags = append(depPaths.ReexportedFlags, flags) - depPaths.ReexportedFlagsDeps = append(depPaths.ReexportedFlagsDeps, - genRule.GeneratedDeps()...) + depPaths.ReexportedDirs = append(depPaths.ReexportedDirs, dirs...) + depPaths.ReexportedGeneratedHeaders = append(depPaths.ReexportedGeneratedHeaders, + genRule.GeneratedSourceFiles()...) + depPaths.ReexportedDeps = append(depPaths.ReexportedDeps, genRule.GeneratedDeps()...) // Add these re-exported flags to help header-abi-dumper to infer the abi exported by a library. - c.sabi.Properties.ReexportedIncludeFlags = append(c.sabi.Properties.ReexportedIncludeFlags, flags) + c.sabi.Properties.ReexportedIncludes = append(c.sabi.Properties.ReexportedIncludes, dirs.Strings()...) } } else { @@ -1622,54 +2248,81 @@ func (c *Module) depsToPaths(ctx android.ModuleContext) PathDeps { if depTag == android.ProtoPluginDepTag { return } + if depTag == llndkImplDep { + return + } if dep.Target().Os != ctx.Os() { ctx.ModuleErrorf("OS mismatch between %q and %q", ctx.ModuleName(), depName) return } if dep.Target().Arch.ArchType != ctx.Arch().ArchType { - ctx.ModuleErrorf("Arch mismatch between %q and %q", ctx.ModuleName(), depName) + ctx.ModuleErrorf("Arch mismatch between %q(%v) and %q(%v)", + ctx.ModuleName(), ctx.Arch().ArchType, depName, dep.Target().Arch.ArchType) return } // re-exporting flags if depTag == reuseObjTag { - if l, ok := ccDep.compiler.(libraryInterface); ok { + // reusing objects only make sense for cc.Modules. + if ccReuseDep, ok := ccDep.(*Module); ok && ccDep.CcLibraryInterface() { c.staticVariant = ccDep - objs, flags, deps := l.reuseObjs() + objs, exporter := ccReuseDep.compiler.(libraryInterface).reuseObjs() depPaths.Objs = depPaths.Objs.Append(objs) - depPaths.ReexportedFlags = append(depPaths.ReexportedFlags, flags...) - depPaths.ReexportedFlagsDeps = append(depPaths.ReexportedFlagsDeps, deps...) + reexportExporter(exporter) return } } if depTag == staticVariantTag { - if _, ok := ccDep.compiler.(libraryInterface); ok { + // staticVariants are a cc.Module specific concept. + if _, ok := ccDep.(*Module); ok && ccDep.CcLibraryInterface() { c.staticVariant = ccDep return } } - // Extract explicitlyVersioned field from the depTag and reset it inside the struct. - // Otherwise, sharedDepTag and lateSharedDepTag with explicitlyVersioned set to true - // won't be matched to sharedDepTag and lateSharedDepTag. + // For the dependency from platform to apex, use the latest stubs + c.apexSdkVersion = android.FutureApiLevel + if !c.IsForPlatform() { + c.apexSdkVersion = c.ApexProperties.Info.MinSdkVersion + } + + if android.InList("hwaddress", ctx.Config().SanitizeDevice()) { + // In hwasan build, we override apexSdkVersion to the FutureApiLevel(10000) + // so that even Q(29/Android10) apexes could use the dynamic unwinder by linking the newer stubs(e.g libc(R+)). + // (b/144430859) + c.apexSdkVersion = android.FutureApiLevel + } + + if depTag == staticUnwinderDepTag { + // Use static unwinder for legacy (min_sdk_version = 29) apexes (b/144430859) + if c.apexSdkVersion <= android.SdkVersion_Android10 { + depTag = StaticDepTag + } else { + return + } + } + + // Extract ExplicitlyVersioned field from the depTag and reset it inside the struct. + // Otherwise, SharedDepTag and lateSharedDepTag with ExplicitlyVersioned set to true + // won't be matched to SharedDepTag and lateSharedDepTag. explicitlyVersioned := false - if t, ok := depTag.(dependencyTag); ok { - explicitlyVersioned = t.explicitlyVersioned - t.explicitlyVersioned = false + if t, ok := depTag.(DependencyTag); ok { + explicitlyVersioned = t.ExplicitlyVersioned + t.ExplicitlyVersioned = false depTag = t } - if t, ok := depTag.(dependencyTag); ok && t.library { + if t, ok := depTag.(DependencyTag); ok && t.Library { depIsStatic := false switch depTag { - case staticDepTag, staticExportDepTag, lateStaticDepTag, wholeStaticDepTag: + case StaticDepTag, staticExportDepTag, lateStaticDepTag, wholeStaticDepTag: depIsStatic = true } - if dependentLibrary, ok := ccDep.linker.(*libraryDecorator); ok && !depIsStatic { - depIsStubs := dependentLibrary.buildStubs() - depHasStubs := ccDep.HasStubsVariants() + if ccDep.CcLibrary() && !depIsStatic { + depIsStubs := ccDep.BuildStubs() + depHasStubs := VersionVariantAvailable(c) && ccDep.HasStubsVariants() depInSameApex := android.DirectlyInApex(c.ApexName(), depName) depInPlatform := !android.DirectlyInAnyApex(ctx, depName) @@ -1685,99 +2338,154 @@ func (c *Module) depsToPaths(ctx android.ModuleContext) PathDeps { // If not building for APEX, use stubs only when it is from // an APEX (and not from platform) useThisDep = (depInPlatform != depIsStubs) - if c.inRecovery() || c.bootstrap() { - // However, for recovery or bootstrap modules, + if c.bootstrap() { + // However, for host, ramdisk, recovery or bootstrap modules, // always link to non-stub variant useThisDep = !depIsStubs } + for _, testFor := range c.TestFor() { + // Another exception: if this module is bundled with an APEX, then + // it is linked with the non-stub variant of a module in the APEX + // as if this is part of the APEX. + if android.DirectlyInApex(testFor, depName) { + useThisDep = !depIsStubs + break + } + } } else { // If building for APEX, use stubs only when it is not from // the same APEX useThisDep = (depInSameApex != depIsStubs) } + // when to use (unspecified) stubs, check min_sdk_version and choose the right one + if useThisDep && depIsStubs && !explicitlyVersioned { + versionToUse, err := c.ChooseSdkVersion(ccDep.StubsVersions(), c.apexSdkVersion) + if err != nil { + ctx.OtherModuleErrorf(dep, err.Error()) + return + } + if versionToUse != ccDep.StubsVersion() { + useThisDep = false + } + } + if !useThisDep { return // stop processing this dep } } - - if i, ok := ccDep.linker.(exportedFlagsProducer); ok { - flags := i.exportedFlags() - deps := i.exportedFlagsDeps() - depPaths.Flags = append(depPaths.Flags, flags...) - depPaths.GeneratedHeaders = append(depPaths.GeneratedHeaders, deps...) - - if t.reexportFlags { - depPaths.ReexportedFlags = append(depPaths.ReexportedFlags, flags...) - depPaths.ReexportedFlagsDeps = append(depPaths.ReexportedFlagsDeps, deps...) - // Add these re-exported flags to help header-abi-dumper to infer the abi exported by a library. - // Re-exported shared library headers must be included as well since they can help us with type information - // about template instantiations (instantiated from their headers). - c.sabi.Properties.ReexportedIncludeFlags = append(c.sabi.Properties.ReexportedIncludeFlags, flags...) + if c.UseVndk() { + if m, ok := ccDep.(*Module); ok && m.IsStubs() { // LLNDK + // by default, use current version of LLNDK + versionToUse := "" + versions := stubsVersionsFor(ctx.Config())[depName] + if c.ApexName() != "" && len(versions) > 0 { + // if this is for use_vendor apex && dep has stubsVersions + // apply the same rule of apex sdk enforcement to choose right version + var err error + versionToUse, err = c.ChooseSdkVersion(versions, c.apexSdkVersion) + if err != nil { + ctx.OtherModuleErrorf(dep, err.Error()) + return + } + } + if versionToUse != ccDep.StubsVersion() { + return + } } } + depPaths.IncludeDirs = append(depPaths.IncludeDirs, ccDep.IncludeDirs()...) + + // Exporting flags only makes sense for cc.Modules + if _, ok := ccDep.(*Module); ok { + if i, ok := ccDep.(*Module).linker.(exportedFlagsProducer); ok { + depPaths.SystemIncludeDirs = append(depPaths.SystemIncludeDirs, i.exportedSystemDirs()...) + depPaths.GeneratedHeaders = append(depPaths.GeneratedHeaders, i.exportedGeneratedHeaders()...) + depPaths.GeneratedDeps = append(depPaths.GeneratedDeps, i.exportedDeps()...) + depPaths.Flags = append(depPaths.Flags, i.exportedFlags()...) + + if t.ReexportFlags { + reexportExporter(i) + // Add these re-exported flags to help header-abi-dumper to infer the abi exported by a library. + // Re-exported shared library headers must be included as well since they can help us with type information + // about template instantiations (instantiated from their headers). + // -isystem headers are not included since for bionic libraries, abi-filtering is taken care of by version + // scripts. + c.sabi.Properties.ReexportedIncludes = append( + c.sabi.Properties.ReexportedIncludes, i.exportedDirs().Strings()...) + } + } + } checkLinkType(ctx, c, ccDep, t) } var ptr *android.Paths var depPtr *android.Paths - linkFile := ccDep.outputFile + linkFile := ccDep.OutputFile() depFile := android.OptionalPath{} switch depTag { - case ndkStubDepTag, sharedDepTag, sharedExportDepTag: + case ndkStubDepTag, SharedDepTag, SharedFromStaticDepTag, sharedExportDepTag: ptr = &depPaths.SharedLibs depPtr = &depPaths.SharedLibsDeps - depFile = ccDep.linker.(libraryInterface).toc() + depFile = ccDep.Toc() directSharedDeps = append(directSharedDeps, ccDep) + case earlySharedDepTag: ptr = &depPaths.EarlySharedLibs depPtr = &depPaths.EarlySharedLibsDeps - depFile = ccDep.linker.(libraryInterface).toc() + depFile = ccDep.Toc() directSharedDeps = append(directSharedDeps, ccDep) case lateSharedDepTag, ndkLateStubDepTag: ptr = &depPaths.LateSharedLibs depPtr = &depPaths.LateSharedLibsDeps - depFile = ccDep.linker.(libraryInterface).toc() - case staticDepTag, staticExportDepTag: + depFile = ccDep.Toc() + case StaticDepTag, staticExportDepTag: ptr = nil directStaticDeps = append(directStaticDeps, ccDep) case lateStaticDepTag: ptr = &depPaths.LateStaticLibs case wholeStaticDepTag: ptr = &depPaths.WholeStaticLibs - staticLib, ok := ccDep.linker.(libraryInterface) - if !ok || !staticLib.static() { + if !ccDep.CcLibraryInterface() || !ccDep.Static() { ctx.ModuleErrorf("module %q not a static library", depName) return } - if missingDeps := staticLib.getWholeStaticMissingDeps(); missingDeps != nil { - postfix := " (required by " + ctx.OtherModuleName(dep) + ")" - for i := range missingDeps { - missingDeps[i] += postfix + // Because the static library objects are included, this only makes sense + // in the context of proper cc.Modules. + if ccWholeStaticLib, ok := ccDep.(*Module); ok { + staticLib := ccWholeStaticLib.linker.(libraryInterface) + if missingDeps := staticLib.getWholeStaticMissingDeps(); missingDeps != nil { + postfix := " (required by " + ctx.OtherModuleName(dep) + ")" + for i := range missingDeps { + missingDeps[i] += postfix + } + ctx.AddMissingDependencies(missingDeps) } - ctx.AddMissingDependencies(missingDeps) + depPaths.WholeStaticLibObjs = depPaths.WholeStaticLibObjs.Append(staticLib.objs()) + } else { + ctx.ModuleErrorf( + "non-cc.Modules cannot be included as whole static libraries.", depName) + return } - depPaths.WholeStaticLibObjs = depPaths.WholeStaticLibObjs.Append(staticLib.objs()) case headerDepTag: // Nothing case objDepTag: depPaths.Objs.objFiles = append(depPaths.Objs.objFiles, linkFile.Path()) - case crtBeginDepTag: + case CrtBeginDepTag: depPaths.CrtBegin = linkFile - case crtEndDepTag: + case CrtEndDepTag: depPaths.CrtEnd = linkFile case dynamicLinkerDepTag: depPaths.DynamicLinker = linkFile } switch depTag { - case staticDepTag, staticExportDepTag, lateStaticDepTag: - staticLib, ok := ccDep.linker.(libraryInterface) - if !ok || !staticLib.static() { + case StaticDepTag, staticExportDepTag, lateStaticDepTag: + if !ccDep.CcLibraryInterface() || !ccDep.Static() { ctx.ModuleErrorf("module %q not a static library", depName) return } @@ -1785,16 +2493,23 @@ func (c *Module) depsToPaths(ctx android.ModuleContext) PathDeps { // When combining coverage files for shared libraries and executables, coverage files // in static libraries act as if they were whole static libraries. The same goes for // source based Abi dump files. - depPaths.StaticLibObjs.coverageFiles = append(depPaths.StaticLibObjs.coverageFiles, - staticLib.objs().coverageFiles...) - depPaths.StaticLibObjs.sAbiDumpFiles = append(depPaths.StaticLibObjs.sAbiDumpFiles, - staticLib.objs().sAbiDumpFiles...) - + // This should only be done for cc.Modules + if c, ok := ccDep.(*Module); ok { + staticLib := c.linker.(libraryInterface) + depPaths.StaticLibObjs.coverageFiles = append(depPaths.StaticLibObjs.coverageFiles, + staticLib.objs().coverageFiles...) + depPaths.StaticLibObjs.sAbiDumpFiles = append(depPaths.StaticLibObjs.sAbiDumpFiles, + staticLib.objs().sAbiDumpFiles...) + } } if ptr != nil { if !linkFile.Valid() { - ctx.ModuleErrorf("module %q missing output file", depName) + if !ctx.Config().AllowMissingDependencies() { + ctx.ModuleErrorf("module %q missing output file", depName) + } else { + ctx.AddMissingDependencies([]string{depName}) + } return } *ptr = append(*ptr, linkFile.Path()) @@ -1808,26 +2523,54 @@ func (c *Module) depsToPaths(ctx android.ModuleContext) PathDeps { *depPtr = append(*depPtr, dep.Path()) } - makeLibName := func(depName string) string { + vendorSuffixModules := vendorSuffixModules(ctx.Config()) + + baseLibName := func(depName string) string { libName := strings.TrimSuffix(depName, llndkLibrarySuffix) libName = strings.TrimSuffix(libName, vendorPublicLibrarySuffix) libName = strings.TrimPrefix(libName, "prebuilt_") - isLLndk := inList(libName, llndkLibraries) - isVendorPublicLib := inList(libName, vendorPublicLibraries) - bothVendorAndCoreVariantsExist := ccDep.hasVendorVariant() || isLLndk + return libName + } + + makeLibName := func(depName string) string { + libName := baseLibName(depName) + isLLndk := isLlndkLibrary(libName, ctx.Config()) + isVendorPublicLib := inList(libName, *vendorPublicLibraries) + bothVendorAndCoreVariantsExist := ccDep.HasVendorVariant() || isLLndk + + if c, ok := ccDep.(*Module); ok { + // Use base module name for snapshots when exporting to Makefile. + if c.isSnapshotPrebuilt() { + baseName := c.BaseModuleName() + + if c.IsVndk() { + return baseName + ".vendor" + } - if ctx.DeviceConfig().VndkUseCoreVariant() && ccDep.isVndk() && !ccDep.mustUseVendorVariant() { + if vendorSuffixModules[baseName] { + return baseName + ".vendor" + } else { + return baseName + } + } + } + + if ctx.DeviceConfig().VndkUseCoreVariant() && ccDep.IsVndk() && !ccDep.MustUseVendorVariant() && !c.InRamdisk() && !c.InRecovery() { // The vendor module is a no-vendor-variant VNDK library. Depend on the // core module instead. return libName - } else if c.useVndk() && bothVendorAndCoreVariantsExist { + } else if c.UseVndk() && bothVendorAndCoreVariantsExist { // The vendor module in Make will have been renamed to not conflict with the core // module, so update the dependency name here accordingly. - return libName + vendorSuffix + return libName + c.getNameSuffixWithVndkVersion(ctx) } else if (ctx.Platform() || ctx.ProductSpecific()) && isVendorPublicLib { return libName + vendorPublicLibrarySuffix - } else if ccDep.inRecovery() && !ccDep.onlyInRecovery() { + } else if ccDep.InRamdisk() && !ccDep.OnlyInRamdisk() { + return libName + ramdiskSuffix + } else if ccDep.InRecovery() && !ccDep.OnlyInRecovery() { return libName + recoverySuffix + } else if ccDep.Module().Target().NativeBridge == android.NativeBridgeEnabled { + return libName + nativeBridgeSuffix } else { return libName } @@ -1835,9 +2578,9 @@ func (c *Module) depsToPaths(ctx android.ModuleContext) PathDeps { // Export the shared libs to Make. switch depTag { - case sharedDepTag, sharedExportDepTag, lateSharedDepTag, earlySharedDepTag: - if dependentLibrary, ok := ccDep.linker.(*libraryDecorator); ok { - if dependentLibrary.buildStubs() && android.InAnyApex(depName) { + case SharedDepTag, sharedExportDepTag, lateSharedDepTag, earlySharedDepTag: + if ccDep.CcLibrary() { + if ccDep.BuildStubs() && android.InAnyApex(depName) { // Add the dependency to the APEX(es) providing the library so that // m <module> can trigger building the APEXes as well. for _, an := range android.GetApexesForModule(depName) { @@ -1851,17 +2594,20 @@ func (c *Module) depsToPaths(ctx android.ModuleContext) PathDeps { // they merely serve as Make dependencies and do not affect this lib itself. c.Properties.AndroidMkSharedLibs = append( c.Properties.AndroidMkSharedLibs, makeLibName(depName)) + // Record baseLibName for snapshots. + c.Properties.SnapshotSharedLibs = append(c.Properties.SnapshotSharedLibs, baseLibName(depName)) case ndkStubDepTag, ndkLateStubDepTag: - ndkStub := ccDep.linker.(*stubDecorator) c.Properties.AndroidMkSharedLibs = append( c.Properties.AndroidMkSharedLibs, - depName+"."+ndkStub.properties.ApiLevel) - case staticDepTag, staticExportDepTag, lateStaticDepTag: + depName+"."+ccDep.ApiLevel()) + case StaticDepTag, staticExportDepTag, lateStaticDepTag: c.Properties.AndroidMkStaticLibs = append( c.Properties.AndroidMkStaticLibs, makeLibName(depName)) case runtimeDepTag: c.Properties.AndroidMkRuntimeLibs = append( c.Properties.AndroidMkRuntimeLibs, makeLibName(depName)) + // Record baseLibName for snapshots. + c.Properties.SnapshotRuntimeLibs = append(c.Properties.SnapshotRuntimeLibs, baseLibName(depName)) case wholeStaticDepTag: c.Properties.AndroidMkWholeStaticLibs = append( c.Properties.AndroidMkWholeStaticLibs, makeLibName(depName)) @@ -1873,12 +2619,18 @@ func (c *Module) depsToPaths(ctx android.ModuleContext) PathDeps { // Dedup exported flags from dependencies depPaths.Flags = android.FirstUniqueStrings(depPaths.Flags) + depPaths.IncludeDirs = android.FirstUniquePaths(depPaths.IncludeDirs) + depPaths.SystemIncludeDirs = android.FirstUniquePaths(depPaths.SystemIncludeDirs) depPaths.GeneratedHeaders = android.FirstUniquePaths(depPaths.GeneratedHeaders) + depPaths.GeneratedDeps = android.FirstUniquePaths(depPaths.GeneratedDeps) + depPaths.ReexportedDirs = android.FirstUniquePaths(depPaths.ReexportedDirs) + depPaths.ReexportedSystemDirs = android.FirstUniquePaths(depPaths.ReexportedSystemDirs) depPaths.ReexportedFlags = android.FirstUniqueStrings(depPaths.ReexportedFlags) - depPaths.ReexportedFlagsDeps = android.FirstUniquePaths(depPaths.ReexportedFlagsDeps) + depPaths.ReexportedDeps = android.FirstUniquePaths(depPaths.ReexportedDeps) + depPaths.ReexportedGeneratedHeaders = android.FirstUniquePaths(depPaths.ReexportedGeneratedHeaders) if c.sabi != nil { - c.sabi.Properties.ReexportedIncludeFlags = android.FirstUniqueStrings(c.sabi.Properties.ReexportedIncludeFlags) + c.sabi.Properties.ReexportedIncludes = android.FirstUniqueStrings(c.sabi.Properties.ReexportedIncludes) } return depPaths @@ -1901,8 +2653,20 @@ func (c *Module) InstallInSanitizerDir() bool { return c.installer.inSanitizerDir() } +func (c *Module) InstallInRamdisk() bool { + return c.InRamdisk() +} + func (c *Module) InstallInRecovery() bool { - return c.inRecovery() + return c.InRecovery() +} + +func (c *Module) SkipInstall() { + if c.installer == nil { + c.ModuleBase.SkipInstall() + return + } + c.installer.skipInstall(c) } func (c *Module) HostToolPath() android.OptionalPath { @@ -1916,11 +2680,16 @@ func (c *Module) IntermPathForModuleOut() android.OptionalPath { return c.outputFile } -func (c *Module) Srcs() android.Paths { - if c.outputFile.Valid() { - return android.Paths{c.outputFile.Path()} +func (c *Module) OutputFiles(tag string) (android.Paths, error) { + switch tag { + case "": + if c.outputFile.Valid() { + return android.Paths{c.outputFile.Path()}, nil + } + return android.Paths{}, nil + default: + return nil, fmt.Errorf("unsupported module reference tag %q", tag) } - return android.Paths{} } func (c *Module) static() bool { @@ -1950,25 +2719,52 @@ func (c *Module) header() bool { return false } -func (c *Module) getMakeLinkType() string { - if c.useVndk() { - if inList(c.Name(), vndkCoreLibraries) || inList(c.Name(), vndkSpLibraries) || inList(c.Name(), llndkLibraries) { - if inList(c.Name(), vndkPrivateLibraries) { - return "native:vndk_private" - } else { +func (c *Module) binary() bool { + if b, ok := c.linker.(interface { + binary() bool + }); ok { + return b.binary() + } + return false +} + +func (c *Module) object() bool { + if o, ok := c.linker.(interface { + object() bool + }); ok { + return o.object() + } + return false +} + +func (c *Module) getMakeLinkType(actx android.ModuleContext) string { + if c.UseVndk() { + if lib, ok := c.linker.(*llndkStubDecorator); ok { + if Bool(lib.Properties.Vendor_available) { return "native:vndk" } - } else { - return "native:vendor" + return "native:vndk_private" } - } else if c.inRecovery() { + if c.IsVndk() && !c.isVndkExt() { + if Bool(c.VendorProperties.Vendor_available) { + return "native:vndk" + } + return "native:vndk_private" + } + if c.inProduct() { + return "native:product" + } + return "native:vendor" + } else if c.InRamdisk() { + return "native:ramdisk" + } else if c.InRecovery() { return "native:recovery" } else if c.Target().Os == android.Android && String(c.Properties.Sdk_version) != "" { return "native:ndk:none:none" // TODO(b/114741097): use the correct ndk stl once build errors have been fixed //family, link := getNdkStlFamilyAndLinkType(c) //return fmt.Sprintf("native:ndk:%s:%s", family, link) - } else if inList(c.Name(), vndkUsingCoreVariantLibraries) { + } else if actx.DeviceConfig().VndkUseCoreVariant() && !c.MustUseVendorVariant() { return "native:platform_vndk" } else { return "native:platform" @@ -1976,28 +2772,102 @@ func (c *Module) getMakeLinkType() string { } // Overrides ApexModule.IsInstallabeToApex() -// Only shared libraries are installable to APEX. +// Only shared/runtime libraries and "test_per_src" tests are installable to APEX. func (c *Module) IsInstallableToApex() bool { if shared, ok := c.linker.(interface { shared() bool }); ok { - return shared.shared() + // Stub libs and prebuilt libs in a versioned SDK are not + // installable to APEX even though they are shared libs. + return shared.shared() && !c.IsStubs() && c.ContainingSdk().Unversioned() + } else if _, ok := c.linker.(testPerSrc); ok { + return true } return false } -func (c *Module) imageVariation() string { - variation := "core" - if c.useVndk() { - variation = "vendor" - } else if c.inRecovery() { - variation = "recovery" +func (c *Module) AvailableFor(what string) bool { + if linker, ok := c.linker.(interface { + availableFor(string) bool + }); ok { + return c.ApexModuleBase.AvailableFor(what) || linker.availableFor(what) + } else { + return c.ApexModuleBase.AvailableFor(what) + } +} + +func (c *Module) TestFor() []string { + if test, ok := c.linker.(interface { + testFor() []string + }); ok { + return test.testFor() + } else { + return c.ApexModuleBase.TestFor() } - return variation } -func (c *Module) IDEInfo(dpInfo *android.IdeInfo) { - dpInfo.Srcs = append(dpInfo.Srcs, c.Srcs().Strings()...) +// Return true if the module is ever installable. +func (c *Module) EverInstallable() bool { + return c.installer != nil && + // Check to see whether the module is actually ever installable. + c.installer.everInstallable() +} + +func (c *Module) installable() bool { + ret := c.EverInstallable() && + // Check to see whether the module has been configured to not be installed. + proptools.BoolDefault(c.Properties.Installable, true) && + !c.Properties.PreventInstall && c.outputFile.Valid() + + // The platform variant doesn't need further condition. Apex variants however might not + // be installable because it will likely to be included in the APEX and won't appear + // in the system partition. + if c.IsForPlatform() { + return ret + } + + // Special case for modules that are configured to be installed to /data, which includes + // test modules. For these modules, both APEX and non-APEX variants are considered as + // installable. This is because even the APEX variants won't be included in the APEX, but + // will anyway be installed to /data/*. + // See b/146995717 + if c.InstallInData() { + return ret + } + + return false +} + +func (c *Module) AndroidMkWriteAdditionalDependenciesForSourceAbiDiff(w io.Writer) { + if c.linker != nil { + if library, ok := c.linker.(*libraryDecorator); ok { + library.androidMkWriteAdditionalDependenciesForSourceAbiDiff(w) + } + } +} + +func (c *Module) DepIsInSameApex(ctx android.BaseModuleContext, dep android.Module) bool { + if depTag, ok := ctx.OtherModuleDependencyTag(dep).(DependencyTag); ok { + if cc, ok := dep.(*Module); ok { + if cc.HasStubsVariants() { + if depTag.Shared && depTag.Library { + // dynamic dep to a stubs lib crosses APEX boundary + return false + } + if IsRuntimeDepTag(depTag) { + // runtime dep to a stubs lib also crosses APEX boundary + return false + } + } + if depTag.FromStatic { + // shared_lib dependency from a static lib is considered as crossing + // the APEX boundary because the dependency doesn't actually is + // linked; the dependency is used only during the compilation phase. + return false + } + } + } + return true } // @@ -2009,9 +2879,6 @@ type Defaults struct { android.ApexModuleBase } -func (*Defaults) GenerateAndroidBuildActions(ctx android.ModuleContext) { -} - // cc_defaults provides a set of properties that can be inherited by other cc // modules. A module can use the properties from a cc_defaults using // `defaults: ["<:default_module_name>"]`. Properties of both modules are @@ -2030,11 +2897,15 @@ func DefaultsFactory(props ...interface{}) android.Module { &VendorProperties{}, &BaseCompilerProperties{}, &BaseLinkerProperties{}, + &ObjectLinkerProperties{}, &LibraryProperties{}, + &StaticProperties{}, + &SharedProperties{}, &FlagExporterProperties{}, &BinaryLinkerProperties{}, &TestProperties{}, &TestBinaryProperties{}, + &FuzzProperties{}, &StlProperties{}, &SanitizeProperties{}, &StripProperties{}, @@ -2045,28 +2916,14 @@ func DefaultsFactory(props ...interface{}) android.Module { &VndkProperties{}, <OProperties{}, &PgoProperties{}, - &XomProperties{}, &android.ProtoProperties{}, ) android.InitDefaultsModule(module) - android.InitApexModule(module) return module } -const ( - // coreMode is the variant used for framework-private libraries, or - // SDK libraries. (which framework-private libraries can use) - coreMode = "core" - - // vendorMode is the variant used for /vendor code that compiles - // against the VNDK. - vendorMode = "vendor" - - recoveryMode = "recovery" -) - func squashVendorSrcs(m *Module) { if lib, ok := m.compiler.(*libraryDecorator); ok { lib.baseCompiler.Properties.Srcs = append(lib.baseCompiler.Properties.Srcs, @@ -2087,63 +2944,9 @@ func squashRecoverySrcs(m *Module) { } } -func ImageMutator(mctx android.BottomUpMutatorContext) { - if mctx.Os() != android.Android { - return - } - - if g, ok := mctx.Module().(*genrule.Module); ok { - if props, ok := g.Extra.(*GenruleExtraProperties); ok { - var coreVariantNeeded bool = false - var vendorVariantNeeded bool = false - var recoveryVariantNeeded bool = false - if mctx.DeviceConfig().VndkVersion() == "" { - coreVariantNeeded = true - } else if Bool(props.Vendor_available) { - coreVariantNeeded = true - vendorVariantNeeded = true - } else if mctx.SocSpecific() || mctx.DeviceSpecific() { - vendorVariantNeeded = true - } else { - coreVariantNeeded = true - } - if Bool(props.Recovery_available) { - recoveryVariantNeeded = true - } - - if recoveryVariantNeeded { - primaryArch := mctx.Config().DevicePrimaryArchType() - moduleArch := g.Target().Arch.ArchType - if moduleArch != primaryArch { - recoveryVariantNeeded = false - } - } - - var variants []string - if coreVariantNeeded { - variants = append(variants, coreMode) - } - if vendorVariantNeeded { - variants = append(variants, vendorMode) - } - if recoveryVariantNeeded { - variants = append(variants, recoveryMode) - } - mod := mctx.CreateVariations(variants...) - for i, v := range variants { - if v == recoveryMode { - m := mod[i].(*genrule.Module) - m.Extra.(*GenruleExtraProperties).InRecovery = true - } - } - } - } - - m, ok := mctx.Module().(*Module) - if !ok { - return - } +var _ android.ImageInterface = (*Module)(nil) +func (m *Module) ImageMutatorBegin(mctx android.BaseModuleContext) { // Sanity check vendorSpecific := mctx.SocSpecific() || mctx.DeviceSpecific() productSpecific := mctx.ProductSpecific() @@ -2151,77 +2954,124 @@ func ImageMutator(mctx android.BottomUpMutatorContext) { if m.VendorProperties.Vendor_available != nil && vendorSpecific { mctx.PropertyErrorf("vendor_available", "doesn't make sense at the same time as `vendor: true`, `proprietary: true`, or `device_specific:true`") - return } if vndkdep := m.vndkdep; vndkdep != nil { if vndkdep.isVndk() { - if productSpecific { - mctx.PropertyErrorf("product_specific", - "product_specific must not be true when `vndk: {enabled: true}`") - return - } - if vendorSpecific { + if vendorSpecific || productSpecific { if !vndkdep.isVndkExt() { mctx.PropertyErrorf("vndk", "must set `extends: \"...\"` to vndk extension") - return + } else if m.VendorProperties.Vendor_available != nil { + mctx.PropertyErrorf("vendor_available", + "must not set at the same time as `vndk: {extends: \"...\"}`") } } else { if vndkdep.isVndkExt() { mctx.PropertyErrorf("vndk", - "must set `vendor: true` to set `extends: %q`", + "must set `vendor: true` or `product_specific: true` to set `extends: %q`", m.getVndkExtendsModuleName()) - return } if m.VendorProperties.Vendor_available == nil { mctx.PropertyErrorf("vndk", "vendor_available must be set to either true or false when `vndk: {enabled: true}`") - return } } } else { if vndkdep.isVndkSp() { mctx.PropertyErrorf("vndk", "must set `enabled: true` to set `support_system_process: true`") - return } if vndkdep.isVndkExt() { mctx.PropertyErrorf("vndk", "must set `enabled: true` to set `extends: %q`", m.getVndkExtendsModuleName()) - return } } } var coreVariantNeeded bool = false - var vendorVariantNeeded bool = false + var ramdiskVariantNeeded bool = false var recoveryVariantNeeded bool = false - if mctx.DeviceConfig().VndkVersion() == "" { + var vendorVariants []string + var productVariants []string + + platformVndkVersion := mctx.DeviceConfig().PlatformVndkVersion() + boardVndkVersion := mctx.DeviceConfig().VndkVersion() + productVndkVersion := mctx.DeviceConfig().ProductVndkVersion() + if boardVndkVersion == "current" { + boardVndkVersion = platformVndkVersion + } + if productVndkVersion == "current" { + productVndkVersion = platformVndkVersion + } + + if boardVndkVersion == "" { // If the device isn't compiling against the VNDK, we always // use the core mode. coreVariantNeeded = true } else if _, ok := m.linker.(*llndkStubDecorator); ok { - // LL-NDK stubs only exist in the vendor variant, since the - // real libraries will be used in the core variant. - vendorVariantNeeded = true + // LL-NDK stubs only exist in the vendor and product variants, + // since the real libraries will be used in the core variant. + vendorVariants = append(vendorVariants, + platformVndkVersion, + boardVndkVersion, + ) + productVariants = append(productVariants, + platformVndkVersion, + productVndkVersion, + ) } else if _, ok := m.linker.(*llndkHeadersDecorator); ok { // ... and LL-NDK headers as well - vendorVariantNeeded = true - } else if _, ok := m.linker.(*vndkPrebuiltLibraryDecorator); ok { + vendorVariants = append(vendorVariants, + platformVndkVersion, + boardVndkVersion, + ) + productVariants = append(productVariants, + platformVndkVersion, + productVndkVersion, + ) + } else if m.isSnapshotPrebuilt() { // Make vendor variants only for the versions in BOARD_VNDK_VERSION and // PRODUCT_EXTRA_VNDK_VERSIONS. - vendorVariantNeeded = true - } else if m.hasVendorVariant() && !vendorSpecific { - // This will be available in both /system and /vendor - // or a /system directory that is available to vendor. + if snapshot, ok := m.linker.(interface { + version() string + }); ok { + vendorVariants = append(vendorVariants, snapshot.version()) + } else { + mctx.ModuleErrorf("version is unknown for snapshot prebuilt") + } + } else if m.HasVendorVariant() && !m.isVndkExt() { + // This will be available in /system, /vendor and /product + // or a /system directory that is available to vendor and product. coreVariantNeeded = true - vendorVariantNeeded = true + + // We assume that modules under proprietary paths are compatible for + // BOARD_VNDK_VERSION. The other modules are regarded as AOSP, or + // PLATFORM_VNDK_VERSION. + if isVendorProprietaryPath(mctx.ModuleDir()) { + vendorVariants = append(vendorVariants, boardVndkVersion) + } else { + vendorVariants = append(vendorVariants, platformVndkVersion) + } + + // vendor_available modules are also available to /product. + productVariants = append(productVariants, platformVndkVersion) + // VNDK is always PLATFORM_VNDK_VERSION + if !m.IsVndk() { + productVariants = append(productVariants, productVndkVersion) + } } else if vendorSpecific && String(m.Properties.Sdk_version) == "" { // This will be available in /vendor (or /odm) only - vendorVariantNeeded = true + // We assume that modules under proprietary paths are compatible for + // BOARD_VNDK_VERSION. The other modules are regarded as AOSP, or + // PLATFORM_VNDK_VERSION. + if isVendorProprietaryPath(mctx.ModuleDir()) { + vendorVariants = append(vendorVariants, boardVndkVersion) + } else { + vendorVariants = append(vendorVariants, platformVndkVersion) + } } else { // This is either in /system (or similar: /data), or is a // modules built with the NDK. Modules built with the NDK @@ -2229,6 +3079,28 @@ func ImageMutator(mctx android.BottomUpMutatorContext) { coreVariantNeeded = true } + if boardVndkVersion != "" && productVndkVersion != "" { + if coreVariantNeeded && productSpecific && String(m.Properties.Sdk_version) == "" { + // The module has "product_specific: true" that does not create core variant. + coreVariantNeeded = false + productVariants = append(productVariants, productVndkVersion) + } + } else { + // Unless PRODUCT_PRODUCT_VNDK_VERSION is set, product partition has no + // restriction to use system libs. + // No product variants defined in this case. + productVariants = []string{} + } + + if Bool(m.Properties.Ramdisk_available) { + ramdiskVariantNeeded = true + } + + if m.ModuleBase.InstallInRamdisk() { + ramdiskVariantNeeded = true + coreVariantNeeded = false + } + if Bool(m.Properties.Recovery_available) { recoveryVariantNeeded = true } @@ -2238,36 +3110,58 @@ func ImageMutator(mctx android.BottomUpMutatorContext) { coreVariantNeeded = false } - if recoveryVariantNeeded { - primaryArch := mctx.Config().DevicePrimaryArchType() - moduleArch := m.Target().Arch.ArchType - if moduleArch != primaryArch { - recoveryVariantNeeded = false - } + for _, variant := range android.FirstUniqueStrings(vendorVariants) { + m.Properties.ExtraVariants = append(m.Properties.ExtraVariants, VendorVariationPrefix+variant) } - var variants []string - if coreVariantNeeded { - variants = append(variants, coreMode) - } - if vendorVariantNeeded { - variants = append(variants, vendorMode) + for _, variant := range android.FirstUniqueStrings(productVariants) { + m.Properties.ExtraVariants = append(m.Properties.ExtraVariants, ProductVariationPrefix+variant) } - if recoveryVariantNeeded { - variants = append(variants, recoveryMode) - } - mod := mctx.CreateVariations(variants...) - for i, v := range variants { - if v == vendorMode { - m := mod[i].(*Module) - m.Properties.UseVndk = true - squashVendorSrcs(m) - } else if v == recoveryMode { - m := mod[i].(*Module) - m.Properties.InRecovery = true - m.MakeAsPlatform() - squashRecoverySrcs(m) + + m.Properties.RamdiskVariantNeeded = ramdiskVariantNeeded + m.Properties.RecoveryVariantNeeded = recoveryVariantNeeded + m.Properties.CoreVariantNeeded = coreVariantNeeded +} + +func (c *Module) CoreVariantNeeded(ctx android.BaseModuleContext) bool { + return c.Properties.CoreVariantNeeded +} + +func (c *Module) RamdiskVariantNeeded(ctx android.BaseModuleContext) bool { + return c.Properties.RamdiskVariantNeeded +} + +func (c *Module) RecoveryVariantNeeded(ctx android.BaseModuleContext) bool { + return c.Properties.RecoveryVariantNeeded +} + +func (c *Module) ExtraImageVariations(ctx android.BaseModuleContext) []string { + return c.Properties.ExtraVariants +} + +func (c *Module) SetImageVariation(ctx android.BaseModuleContext, variant string, module android.Module) { + m := module.(*Module) + if variant == android.RamdiskVariation { + m.MakeAsPlatform() + } else if variant == android.RecoveryVariation { + m.MakeAsPlatform() + squashRecoverySrcs(m) + } else if strings.HasPrefix(variant, VendorVariationPrefix) { + m.Properties.ImageVariationPrefix = VendorVariationPrefix + m.Properties.VndkVersion = strings.TrimPrefix(variant, VendorVariationPrefix) + squashVendorSrcs(m) + + // Makefile shouldn't know vendor modules other than BOARD_VNDK_VERSION. + // Hide other vendor variants to avoid collision. + vndkVersion := ctx.DeviceConfig().VndkVersion() + if vndkVersion != "current" && vndkVersion != "" && vndkVersion != m.Properties.VndkVersion { + m.Properties.HideFromMake = true + m.SkipInstall() } + } else if strings.HasPrefix(variant, ProductVariationPrefix) { + m.Properties.ImageVariationPrefix = ProductVariationPrefix + m.Properties.VndkVersion = strings.TrimPrefix(variant, ProductVariationPrefix) + squashVendorSrcs(m) } } @@ -2278,6 +3172,26 @@ func getCurrentNdkPrebuiltVersion(ctx DepsContext) string { return ctx.Config().PlatformSdkVersion() } +func kytheExtractAllFactory() android.Singleton { + return &kytheExtractAllSingleton{} +} + +type kytheExtractAllSingleton struct { +} + +func (ks *kytheExtractAllSingleton) GenerateBuildActions(ctx android.SingletonContext) { + var xrefTargets android.Paths + ctx.VisitAllModules(func(module android.Module) { + if ccModule, ok := module.(xref); ok { + xrefTargets = append(xrefTargets, ccModule.XrefCcFiles()...) + } + }) + // TODO(asmundak): Perhaps emit a rule to output a warning if there were no xrefTargets + if len(xrefTargets) > 0 { + ctx.Phony("xref_cxx", xrefTargets...) + } +} + var Bool = proptools.Bool var BoolDefault = proptools.BoolDefault var BoolPtr = proptools.BoolPtr diff --git a/cc/cc_test.go b/cc/cc_test.go index 678f90d20..f73e02172 100644 --- a/cc/cc_test.go +++ b/cc/cc_test.go @@ -15,15 +15,16 @@ package cc import ( - "android/soong/android" - "fmt" "io/ioutil" "os" + "path/filepath" "reflect" "sort" "strings" "testing" + + "android/soong/android" ) var buildDir string @@ -51,64 +52,10 @@ func TestMain(m *testing.M) { os.Exit(run()) } -func createTestContext(t *testing.T, config android.Config, bp string, fs map[string][]byte, - os android.OsType) *android.TestContext { - - ctx := android.NewTestArchContext() - ctx.RegisterModuleType("cc_binary", android.ModuleFactoryAdaptor(BinaryFactory)) - ctx.RegisterModuleType("cc_binary_host", android.ModuleFactoryAdaptor(binaryHostFactory)) - ctx.RegisterModuleType("cc_library", android.ModuleFactoryAdaptor(LibraryFactory)) - ctx.RegisterModuleType("cc_library_shared", android.ModuleFactoryAdaptor(LibrarySharedFactory)) - ctx.RegisterModuleType("cc_library_static", android.ModuleFactoryAdaptor(LibraryStaticFactory)) - ctx.RegisterModuleType("cc_library_headers", android.ModuleFactoryAdaptor(LibraryHeaderFactory)) - ctx.RegisterModuleType("toolchain_library", android.ModuleFactoryAdaptor(ToolchainLibraryFactory)) - ctx.RegisterModuleType("llndk_library", android.ModuleFactoryAdaptor(LlndkLibraryFactory)) - ctx.RegisterModuleType("llndk_headers", android.ModuleFactoryAdaptor(llndkHeadersFactory)) - ctx.RegisterModuleType("vendor_public_library", android.ModuleFactoryAdaptor(vendorPublicLibraryFactory)) - ctx.RegisterModuleType("cc_object", android.ModuleFactoryAdaptor(ObjectFactory)) - ctx.RegisterModuleType("filegroup", android.ModuleFactoryAdaptor(android.FileGroupFactory)) - ctx.PreDepsMutators(func(ctx android.RegisterMutatorsContext) { - ctx.BottomUp("image", ImageMutator).Parallel() - ctx.BottomUp("link", LinkageMutator).Parallel() - ctx.BottomUp("vndk", VndkMutator).Parallel() - ctx.BottomUp("version", VersionMutator).Parallel() - ctx.BottomUp("begin", BeginMutator).Parallel() - }) - ctx.PostDepsMutators(func(ctx android.RegisterMutatorsContext) { - ctx.TopDown("double_loadable", checkDoubleLoadableLibraries).Parallel() - }) - - // add some modules that are required by the compiler and/or linker - bp = bp + GatherRequiredDepsForTest(os) - - mockFS := map[string][]byte{ - "Android.bp": []byte(bp), - "foo.c": nil, - "bar.c": nil, - "a.proto": nil, - "b.aidl": nil, - "my_include": nil, - "foo.map.txt": nil, - "liba.so": nil, - } - - for k, v := range fs { - mockFS[k] = v - } - - ctx.MockFileSystem(mockFS) - - return ctx -} - -func testCcWithConfig(t *testing.T, bp string, config android.Config) *android.TestContext { - return testCcWithConfigForOs(t, bp, config, android.Android) -} - -func testCcWithConfigForOs(t *testing.T, bp string, config android.Config, os android.OsType) *android.TestContext { +func testCcWithConfig(t *testing.T, config android.Config) *android.TestContext { t.Helper() - ctx := createTestContext(t, config, bp, nil, os) - ctx.Register() + ctx := CreateTestContext() + ctx.Register(config) _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) android.FailIfErrored(t, errs) @@ -120,29 +67,26 @@ func testCcWithConfigForOs(t *testing.T, bp string, config android.Config, os an func testCc(t *testing.T, bp string) *android.TestContext { t.Helper() - config := android.TestArchConfig(buildDir, nil) + config := TestConfig(buildDir, android.Android, nil, bp, nil) config.TestProductVariables.DeviceVndkVersion = StringPtr("current") config.TestProductVariables.Platform_vndk_version = StringPtr("VER") - return testCcWithConfig(t, bp, config) + return testCcWithConfig(t, config) } func testCcNoVndk(t *testing.T, bp string) *android.TestContext { t.Helper() - config := android.TestArchConfig(buildDir, nil) + config := TestConfig(buildDir, android.Android, nil, bp, nil) config.TestProductVariables.Platform_vndk_version = StringPtr("VER") - return testCcWithConfig(t, bp, config) + return testCcWithConfig(t, config) } -func testCcError(t *testing.T, pattern string, bp string) { +func testCcErrorWithConfig(t *testing.T, pattern string, config android.Config) { t.Helper() - config := android.TestArchConfig(buildDir, nil) - config.TestProductVariables.DeviceVndkVersion = StringPtr("current") - config.TestProductVariables.Platform_vndk_version = StringPtr("VER") - ctx := createTestContext(t, config, bp, nil, android.Android) - ctx.Register() + ctx := CreateTestContext() + ctx.Register(config) _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) if len(errs) > 0 { @@ -159,10 +103,28 @@ func testCcError(t *testing.T, pattern string, bp string) { t.Fatalf("missing expected error %q (0 errors are returned)", pattern) } +func testCcError(t *testing.T, pattern string, bp string) { + config := TestConfig(buildDir, android.Android, nil, bp, nil) + config.TestProductVariables.DeviceVndkVersion = StringPtr("current") + config.TestProductVariables.Platform_vndk_version = StringPtr("VER") + testCcErrorWithConfig(t, pattern, config) + return +} + +func testCcErrorProductVndk(t *testing.T, pattern string, bp string) { + config := TestConfig(buildDir, android.Android, nil, bp, nil) + config.TestProductVariables.DeviceVndkVersion = StringPtr("current") + config.TestProductVariables.ProductVndkVersion = StringPtr("current") + config.TestProductVariables.Platform_vndk_version = StringPtr("VER") + testCcErrorWithConfig(t, pattern, config) + return +} + const ( - coreVariant = "android_arm64_armv8-a_core_shared" - vendorVariant = "android_arm64_armv8-a_vendor_shared" - recoveryVariant = "android_arm64_armv8-a_recovery_shared" + coreVariant = "android_arm64_armv8-a_shared" + vendorVariant = "android_vendor.VER_arm64_armv8-a_shared" + productVariant = "android_product.VER_arm64_armv8-a_shared" + recoveryVariant = "android_recovery_arm64_armv8-a_shared" ) func TestFuchsiaDeps(t *testing.T) { @@ -179,8 +141,8 @@ func TestFuchsiaDeps(t *testing.T) { }, }` - config := android.TestArchConfigFuchsia(buildDir, nil) - ctx := testCcWithConfigForOs(t, bp, config, android.Fuchsia) + config := TestConfig(buildDir, android.Fuchsia, nil, bp, nil) + ctx := testCcWithConfig(t, config) rt := false fb := false @@ -216,8 +178,8 @@ func TestFuchsiaTargetDecl(t *testing.T) { }, }` - config := android.TestArchConfigFuchsia(buildDir, nil) - ctx := testCcWithConfigForOs(t, bp, config, android.Fuchsia) + config := TestConfig(buildDir, android.Fuchsia, nil, bp, nil) + ctx := testCcWithConfig(t, config) ld := ctx.ModuleForTests("libTest", "fuchsia_arm64_shared").Rule("ld") var objs []string for _, o := range ld.Inputs { @@ -233,7 +195,7 @@ func TestVendorSrc(t *testing.T) { cc_library { name: "libTest", srcs: ["foo.c"], - no_libgcc: true, + no_libcrt: true, nocrt: true, system_shared_libs: [], vendor_available: true, @@ -256,13 +218,13 @@ func TestVendorSrc(t *testing.T) { } func checkVndkModule(t *testing.T, ctx *android.TestContext, name, subDir string, - isVndkSp bool, extends string) { + isVndkSp bool, extends string, variant string) { t.Helper() - mod := ctx.ModuleForTests(name, vendorVariant).Module().(*Module) - if !mod.hasVendorVariant() { - t.Errorf("%q must have vendor variant", name) + mod := ctx.ModuleForTests(name, variant).Module().(*Module) + if !mod.HasVendorVariant() { + t.Errorf("%q must have variant %q", name, variant) } // Check library properties. @@ -278,8 +240,8 @@ func checkVndkModule(t *testing.T, ctx *android.TestContext, name, subDir string if mod.vndkdep == nil { t.Fatalf("%q must have `vndkdep`", name) } - if !mod.isVndk() { - t.Errorf("%q isVndk() must equal to true", name) + if !mod.IsVndk() { + t.Errorf("%q IsVndk() must equal to true", name) } if mod.isVndkSp() != isVndkSp { t.Errorf("%q isVndkSp() must equal to %t", name, isVndkSp) @@ -296,8 +258,54 @@ func checkVndkModule(t *testing.T, ctx *android.TestContext, name, subDir string } } +func checkSnapshot(t *testing.T, ctx *android.TestContext, singleton android.TestingSingleton, moduleName, snapshotFilename, subDir, variant string) { + mod, ok := ctx.ModuleForTests(moduleName, variant).Module().(android.OutputFileProducer) + if !ok { + t.Errorf("%q must have output\n", moduleName) + return + } + outputFiles, err := mod.OutputFiles("") + if err != nil || len(outputFiles) != 1 { + t.Errorf("%q must have single output\n", moduleName) + return + } + snapshotPath := filepath.Join(subDir, snapshotFilename) + + out := singleton.Output(snapshotPath) + if out.Input.String() != outputFiles[0].String() { + t.Errorf("The input of snapshot %q must be %q, but %q", moduleName, out.Input.String(), outputFiles[0]) + } +} + +func checkWriteFileOutput(t *testing.T, params android.TestingBuildParams, expected []string) { + t.Helper() + assertString(t, params.Rule.String(), android.WriteFile.String()) + actual := strings.FieldsFunc(strings.ReplaceAll(params.Args["content"], "\\n", "\n"), func(r rune) bool { return r == '\n' }) + assertArrayString(t, actual, expected) +} + +func checkVndkOutput(t *testing.T, ctx *android.TestContext, output string, expected []string) { + t.Helper() + vndkSnapshot := ctx.SingletonForTests("vndk-snapshot") + checkWriteFileOutput(t, vndkSnapshot.Output(output), expected) +} + +func checkVndkLibrariesOutput(t *testing.T, ctx *android.TestContext, module string, expected []string) { + t.Helper() + vndkLibraries := ctx.ModuleForTests(module, "") + + var output string + if module != "vndkcorevariant.libraries.txt" { + output = insertVndkVersion(module, "VER") + } else { + output = module + } + + checkWriteFileOutput(t, vndkLibraries.Output(output), expected) +} + func TestVndk(t *testing.T) { - ctx := testCc(t, ` + bp := ` cc_library { name: "libvndk", vendor_available: true, @@ -314,6 +322,7 @@ func TestVndk(t *testing.T) { enabled: true, }, nocrt: true, + stem: "libvndk-private", } cc_library { @@ -324,6 +333,7 @@ func TestVndk(t *testing.T) { support_system_process: true, }, nocrt: true, + suffix: "-x", } cc_library { @@ -334,13 +344,209 @@ func TestVndk(t *testing.T) { support_system_process: true, }, nocrt: true, + target: { + vendor: { + suffix: "-x", + }, + }, + } + vndk_libraries_txt { + name: "llndk.libraries.txt", + } + vndk_libraries_txt { + name: "vndkcore.libraries.txt", + } + vndk_libraries_txt { + name: "vndksp.libraries.txt", + } + vndk_libraries_txt { + name: "vndkprivate.libraries.txt", + } + vndk_libraries_txt { + name: "vndkcorevariant.libraries.txt", + } + ` + + config := TestConfig(buildDir, android.Android, nil, bp, nil) + config.TestProductVariables.DeviceVndkVersion = StringPtr("current") + config.TestProductVariables.Platform_vndk_version = StringPtr("VER") + + ctx := testCcWithConfig(t, config) + + checkVndkModule(t, ctx, "libvndk", "vndk-VER", false, "", vendorVariant) + checkVndkModule(t, ctx, "libvndk_private", "vndk-VER", false, "", vendorVariant) + checkVndkModule(t, ctx, "libvndk_sp", "vndk-sp-VER", true, "", vendorVariant) + checkVndkModule(t, ctx, "libvndk_sp_private", "vndk-sp-VER", true, "", vendorVariant) + + // Check VNDK snapshot output. + + snapshotDir := "vndk-snapshot" + snapshotVariantPath := filepath.Join(buildDir, snapshotDir, "arm64") + + vndkLibPath := filepath.Join(snapshotVariantPath, fmt.Sprintf("arch-%s-%s", + "arm64", "armv8-a")) + vndkLib2ndPath := filepath.Join(snapshotVariantPath, fmt.Sprintf("arch-%s-%s", + "arm", "armv7-a-neon")) + + vndkCoreLibPath := filepath.Join(vndkLibPath, "shared", "vndk-core") + vndkSpLibPath := filepath.Join(vndkLibPath, "shared", "vndk-sp") + vndkCoreLib2ndPath := filepath.Join(vndkLib2ndPath, "shared", "vndk-core") + vndkSpLib2ndPath := filepath.Join(vndkLib2ndPath, "shared", "vndk-sp") + + variant := "android_vendor.VER_arm64_armv8-a_shared" + variant2nd := "android_vendor.VER_arm_armv7-a-neon_shared" + + snapshotSingleton := ctx.SingletonForTests("vndk-snapshot") + + checkSnapshot(t, ctx, snapshotSingleton, "libvndk", "libvndk.so", vndkCoreLibPath, variant) + checkSnapshot(t, ctx, snapshotSingleton, "libvndk", "libvndk.so", vndkCoreLib2ndPath, variant2nd) + checkSnapshot(t, ctx, snapshotSingleton, "libvndk_sp", "libvndk_sp-x.so", vndkSpLibPath, variant) + checkSnapshot(t, ctx, snapshotSingleton, "libvndk_sp", "libvndk_sp-x.so", vndkSpLib2ndPath, variant2nd) + + snapshotConfigsPath := filepath.Join(snapshotVariantPath, "configs") + checkSnapshot(t, ctx, snapshotSingleton, "llndk.libraries.txt", "llndk.libraries.txt", snapshotConfigsPath, "") + checkSnapshot(t, ctx, snapshotSingleton, "vndkcore.libraries.txt", "vndkcore.libraries.txt", snapshotConfigsPath, "") + checkSnapshot(t, ctx, snapshotSingleton, "vndksp.libraries.txt", "vndksp.libraries.txt", snapshotConfigsPath, "") + checkSnapshot(t, ctx, snapshotSingleton, "vndkprivate.libraries.txt", "vndkprivate.libraries.txt", snapshotConfigsPath, "") + + checkVndkOutput(t, ctx, "vndk/vndk.libraries.txt", []string{ + "LLNDK: libc.so", + "LLNDK: libdl.so", + "LLNDK: libft2.so", + "LLNDK: libm.so", + "VNDK-SP: libc++.so", + "VNDK-SP: libvndk_sp-x.so", + "VNDK-SP: libvndk_sp_private-x.so", + "VNDK-core: libvndk-private.so", + "VNDK-core: libvndk.so", + "VNDK-private: libft2.so", + "VNDK-private: libvndk-private.so", + "VNDK-private: libvndk_sp_private-x.so", + }) + checkVndkLibrariesOutput(t, ctx, "llndk.libraries.txt", []string{"libc.so", "libdl.so", "libft2.so", "libm.so"}) + checkVndkLibrariesOutput(t, ctx, "vndkcore.libraries.txt", []string{"libvndk-private.so", "libvndk.so"}) + checkVndkLibrariesOutput(t, ctx, "vndkprivate.libraries.txt", []string{"libft2.so", "libvndk-private.so", "libvndk_sp_private-x.so"}) + checkVndkLibrariesOutput(t, ctx, "vndksp.libraries.txt", []string{"libc++.so", "libvndk_sp-x.so", "libvndk_sp_private-x.so"}) + checkVndkLibrariesOutput(t, ctx, "vndkcorevariant.libraries.txt", nil) +} + +func TestVndkWithHostSupported(t *testing.T) { + ctx := testCc(t, ` + cc_library { + name: "libvndk_host_supported", + vendor_available: true, + vndk: { + enabled: true, + }, + host_supported: true, + } + + cc_library { + name: "libvndk_host_supported_but_disabled_on_device", + vendor_available: true, + vndk: { + enabled: true, + }, + host_supported: true, + enabled: false, + target: { + host: { + enabled: true, + } + } + } + + vndk_libraries_txt { + name: "vndkcore.libraries.txt", + } + `) + + checkVndkLibrariesOutput(t, ctx, "vndkcore.libraries.txt", []string{"libvndk_host_supported.so"}) +} + +func TestVndkLibrariesTxtAndroidMk(t *testing.T) { + bp := ` + vndk_libraries_txt { + name: "llndk.libraries.txt", + }` + config := TestConfig(buildDir, android.Android, nil, bp, nil) + config.TestProductVariables.DeviceVndkVersion = StringPtr("current") + config.TestProductVariables.Platform_vndk_version = StringPtr("VER") + ctx := testCcWithConfig(t, config) + + module := ctx.ModuleForTests("llndk.libraries.txt", "") + entries := android.AndroidMkEntriesForTest(t, config, "", module.Module())[0] + assertArrayString(t, entries.EntryMap["LOCAL_MODULE_STEM"], []string{"llndk.libraries.VER.txt"}) +} + +func TestVndkUsingCoreVariant(t *testing.T) { + bp := ` + cc_library { + name: "libvndk", + vendor_available: true, + vndk: { + enabled: true, + }, + nocrt: true, + } + + cc_library { + name: "libvndk_sp", + vendor_available: true, + vndk: { + enabled: true, + support_system_process: true, + }, + nocrt: true, + } + + cc_library { + name: "libvndk2", + vendor_available: false, + vndk: { + enabled: true, + }, + nocrt: true, + } + + vndk_libraries_txt { + name: "vndkcorevariant.libraries.txt", + } + ` + + config := TestConfig(buildDir, android.Android, nil, bp, nil) + config.TestProductVariables.DeviceVndkVersion = StringPtr("current") + config.TestProductVariables.Platform_vndk_version = StringPtr("VER") + config.TestProductVariables.VndkUseCoreVariant = BoolPtr(true) + + setVndkMustUseVendorVariantListForTest(config, []string{"libvndk"}) + + ctx := testCcWithConfig(t, config) + + checkVndkLibrariesOutput(t, ctx, "vndkcorevariant.libraries.txt", []string{"libc++.so", "libvndk2.so", "libvndk_sp.so"}) +} + +func TestVndkWhenVndkVersionIsNotSet(t *testing.T) { + ctx := testCcNoVndk(t, ` + cc_library { + name: "libvndk", + vendor_available: true, + vndk: { + enabled: true, + }, + nocrt: true, } `) - checkVndkModule(t, ctx, "libvndk", "vndk-VER", false, "") - checkVndkModule(t, ctx, "libvndk_private", "vndk-VER", false, "") - checkVndkModule(t, ctx, "libvndk_sp", "vndk-sp-VER", true, "") - checkVndkModule(t, ctx, "libvndk_sp_private", "vndk-sp-VER", true, "") + checkVndkOutput(t, ctx, "vndk/vndk.libraries.txt", []string{ + "LLNDK: libc.so", + "LLNDK: libdl.so", + "LLNDK: libft2.so", + "LLNDK: libm.so", + "VNDK-SP: libc++.so", + "VNDK-core: libvndk.so", + "VNDK-private: libft2.so", + }) } func TestVndkDepError(t *testing.T) { @@ -627,6 +833,131 @@ func TestDoubleLoadbleDep(t *testing.T) { `) } +func TestVendorSnapshot(t *testing.T) { + bp := ` + cc_library { + name: "libvndk", + vendor_available: true, + vndk: { + enabled: true, + }, + nocrt: true, + } + + cc_library { + name: "libvendor", + vendor: true, + nocrt: true, + } + + cc_library { + name: "libvendor_available", + vendor_available: true, + nocrt: true, + } + + cc_library_headers { + name: "libvendor_headers", + vendor_available: true, + nocrt: true, + } + + cc_binary { + name: "vendor_bin", + vendor: true, + nocrt: true, + } + + cc_binary { + name: "vendor_available_bin", + vendor_available: true, + nocrt: true, + } + + toolchain_library { + name: "libb", + vendor_available: true, + src: "libb.a", + } + + cc_object { + name: "obj", + vendor_available: true, + } +` + config := TestConfig(buildDir, android.Android, nil, bp, nil) + config.TestProductVariables.DeviceVndkVersion = StringPtr("current") + config.TestProductVariables.Platform_vndk_version = StringPtr("VER") + ctx := testCcWithConfig(t, config) + + // Check Vendor snapshot output. + + snapshotDir := "vendor-snapshot" + snapshotVariantPath := filepath.Join(buildDir, snapshotDir, "arm64") + snapshotSingleton := ctx.SingletonForTests("vendor-snapshot") + + var jsonFiles []string + + for _, arch := range [][]string{ + []string{"arm64", "armv8-a"}, + []string{"arm", "armv7-a-neon"}, + } { + archType := arch[0] + archVariant := arch[1] + archDir := fmt.Sprintf("arch-%s-%s", archType, archVariant) + + // For shared libraries, only non-VNDK vendor_available modules are captured + sharedVariant := fmt.Sprintf("android_vendor.VER_%s_%s_shared", archType, archVariant) + sharedDir := filepath.Join(snapshotVariantPath, archDir, "shared") + checkSnapshot(t, ctx, snapshotSingleton, "libvendor", "libvendor.so", sharedDir, sharedVariant) + checkSnapshot(t, ctx, snapshotSingleton, "libvendor_available", "libvendor_available.so", sharedDir, sharedVariant) + jsonFiles = append(jsonFiles, + filepath.Join(sharedDir, "libvendor.so.json"), + filepath.Join(sharedDir, "libvendor_available.so.json")) + + // For static libraries, all vendor:true and vendor_available modules (including VNDK) are captured. + staticVariant := fmt.Sprintf("android_vendor.VER_%s_%s_static", archType, archVariant) + staticDir := filepath.Join(snapshotVariantPath, archDir, "static") + checkSnapshot(t, ctx, snapshotSingleton, "libb", "libb.a", staticDir, staticVariant) + checkSnapshot(t, ctx, snapshotSingleton, "libvndk", "libvndk.a", staticDir, staticVariant) + checkSnapshot(t, ctx, snapshotSingleton, "libvendor", "libvendor.a", staticDir, staticVariant) + checkSnapshot(t, ctx, snapshotSingleton, "libvendor_available", "libvendor_available.a", staticDir, staticVariant) + jsonFiles = append(jsonFiles, + filepath.Join(staticDir, "libb.a.json"), + filepath.Join(staticDir, "libvndk.a.json"), + filepath.Join(staticDir, "libvendor.a.json"), + filepath.Join(staticDir, "libvendor_available.a.json")) + + // For binary executables, all vendor:true and vendor_available modules are captured. + if archType == "arm64" { + binaryVariant := fmt.Sprintf("android_vendor.VER_%s_%s", archType, archVariant) + binaryDir := filepath.Join(snapshotVariantPath, archDir, "binary") + checkSnapshot(t, ctx, snapshotSingleton, "vendor_bin", "vendor_bin", binaryDir, binaryVariant) + checkSnapshot(t, ctx, snapshotSingleton, "vendor_available_bin", "vendor_available_bin", binaryDir, binaryVariant) + jsonFiles = append(jsonFiles, + filepath.Join(binaryDir, "vendor_bin.json"), + filepath.Join(binaryDir, "vendor_available_bin.json")) + } + + // For header libraries, all vendor:true and vendor_available modules are captured. + headerDir := filepath.Join(snapshotVariantPath, archDir, "header") + jsonFiles = append(jsonFiles, filepath.Join(headerDir, "libvendor_headers.json")) + + // For object modules, all vendor:true and vendor_available modules are captured. + objectVariant := fmt.Sprintf("android_vendor.VER_%s_%s", archType, archVariant) + objectDir := filepath.Join(snapshotVariantPath, archDir, "object") + checkSnapshot(t, ctx, snapshotSingleton, "obj", "obj.o", objectDir, objectVariant) + jsonFiles = append(jsonFiles, filepath.Join(objectDir, "obj.o.json")) + } + + for _, jsonFile := range jsonFiles { + // verify all json files exist + if snapshotSingleton.MaybeOutput(jsonFile).Rule == nil { + t.Errorf("%q expected but not found", jsonFile) + } + } +} + func TestDoubleLoadableDepError(t *testing.T) { // Check whether an error is emitted when a LLNDK depends on a non-double_loadable VNDK lib. testCcError(t, "module \".*\" variant \".*\": link.* \".*\" which is not LL-NDK, VNDK-SP, .*double_loadable", ` @@ -653,7 +984,7 @@ func TestDoubleLoadableDepError(t *testing.T) { testCcError(t, "module \".*\" variant \".*\": link.* \".*\" which is not LL-NDK, VNDK-SP, .*double_loadable", ` cc_library { name: "libllndk", - no_libgcc: true, + no_libcrt: true, shared_libs: ["libnondoubleloadable"], } @@ -747,30 +1078,28 @@ func TestDoubleLoadableDepError(t *testing.T) { `) } -func TestVndkMustNotBeProductSpecific(t *testing.T) { - // Check whether an error is emitted when a vndk lib has 'product_specific: true'. - testCcError(t, "product_specific must not be true when `vndk: {enabled: true}`", ` +func TestVndkExt(t *testing.T) { + // This test checks the VNDK-Ext properties. + bp := ` cc_library { name: "libvndk", - product_specific: true, // Cause error vendor_available: true, vndk: { enabled: true, }, nocrt: true, } - `) -} - -func TestVndkExt(t *testing.T) { - // This test checks the VNDK-Ext properties. - ctx := testCc(t, ` cc_library { - name: "libvndk", + name: "libvndk2", vendor_available: true, vndk: { enabled: true, }, + target: { + vendor: { + suffix: "-suffix", + }, + }, nocrt: true, } @@ -783,9 +1112,52 @@ func TestVndkExt(t *testing.T) { }, nocrt: true, } - `) - checkVndkModule(t, ctx, "libvndk_ext", "vndk", false, "libvndk") + cc_library { + name: "libvndk2_ext", + vendor: true, + vndk: { + enabled: true, + extends: "libvndk2", + }, + nocrt: true, + } + + cc_library { + name: "libvndk_ext_product", + product_specific: true, + vndk: { + enabled: true, + extends: "libvndk", + }, + nocrt: true, + } + + cc_library { + name: "libvndk2_ext_product", + product_specific: true, + vndk: { + enabled: true, + extends: "libvndk2", + }, + nocrt: true, + } + ` + config := TestConfig(buildDir, android.Android, nil, bp, nil) + config.TestProductVariables.DeviceVndkVersion = StringPtr("current") + config.TestProductVariables.ProductVndkVersion = StringPtr("current") + config.TestProductVariables.Platform_vndk_version = StringPtr("VER") + + ctx := testCcWithConfig(t, config) + + checkVndkModule(t, ctx, "libvndk_ext", "vndk", false, "libvndk", vendorVariant) + checkVndkModule(t, ctx, "libvndk_ext_product", "vndk", false, "libvndk", productVariant) + + mod_vendor := ctx.ModuleForTests("libvndk2_ext", vendorVariant).Module().(*Module) + assertString(t, mod_vendor.outputFile.Path().Base(), "libvndk2-suffix.so") + + mod_product := ctx.ModuleForTests("libvndk2_ext_product", productVariant).Module().(*Module) + assertString(t, mod_product.outputFile.Path().Base(), "libvndk2-suffix.so") } func TestVndkExtWithoutBoardVndkVersion(t *testing.T) { @@ -818,9 +1190,39 @@ func TestVndkExtWithoutBoardVndkVersion(t *testing.T) { } } +func TestVndkExtWithoutProductVndkVersion(t *testing.T) { + // This test checks the VNDK-Ext properties when PRODUCT_PRODUCT_VNDK_VERSION is not set. + ctx := testCc(t, ` + cc_library { + name: "libvndk", + vendor_available: true, + vndk: { + enabled: true, + }, + nocrt: true, + } + + cc_library { + name: "libvndk_ext_product", + product_specific: true, + vndk: { + enabled: true, + extends: "libvndk", + }, + nocrt: true, + } + `) + + // Ensures that the core variant of "libvndk_ext_product" can be found. + mod := ctx.ModuleForTests("libvndk_ext_product", coreVariant).Module().(*Module) + if extends := mod.getVndkExtendsModuleName(); extends != "libvndk" { + t.Errorf("\"libvndk_ext_product\" must extend from \"libvndk\" but get %q", extends) + } +} + func TestVndkExtError(t *testing.T) { // This test ensures an error is emitted in ill-formed vndk-ext definition. - testCcError(t, "must set `vendor: true` to set `extends: \".*\"`", ` + testCcError(t, "must set `vendor: true` or `product_specific: true` to set `extends: \".*\"`", ` cc_library { name: "libvndk", vendor_available: true, @@ -859,6 +1261,48 @@ func TestVndkExtError(t *testing.T) { nocrt: true, } `) + + testCcErrorProductVndk(t, "must set `extends: \"\\.\\.\\.\"` to vndk extension", ` + cc_library { + name: "libvndk", + vendor_available: true, + vndk: { + enabled: true, + }, + nocrt: true, + } + + cc_library { + name: "libvndk_ext_product", + product_specific: true, + vndk: { + enabled: true, + }, + nocrt: true, + } + `) + + testCcErrorProductVndk(t, "must not set at the same time as `vndk: {extends: \"\\.\\.\\.\"}`", ` + cc_library { + name: "libvndk", + vendor_available: true, + vndk: { + enabled: true, + }, + nocrt: true, + } + + cc_library { + name: "libvndk_ext_product", + product_specific: true, + vendor_available: true, + vndk: { + enabled: true, + extends: "libvndk", + }, + nocrt: true, + } + `) } func TestVndkExtInconsistentSupportSystemProcessError(t *testing.T) { @@ -931,6 +1375,27 @@ func TestVndkExtVendorAvailableFalseError(t *testing.T) { nocrt: true, } `) + + testCcErrorProductVndk(t, "`extends` refers module \".*\" which does not have `vendor_available: true`", ` + cc_library { + name: "libvndk", + vendor_available: false, + vndk: { + enabled: true, + }, + nocrt: true, + } + + cc_library { + name: "libvndk_ext_product", + product_specific: true, + vndk: { + enabled: true, + extends: "libvndk", + }, + nocrt: true, + } + `) } func TestVendorModuleUseVndkExt(t *testing.T) { @@ -956,7 +1421,6 @@ func TestVendorModuleUseVndkExt(t *testing.T) { } cc_library { - name: "libvndk_sp", vendor_available: true, vndk: { @@ -1048,6 +1512,71 @@ func TestVndkExtUseVendorLib(t *testing.T) { `) } +func TestProductVndkExtDependency(t *testing.T) { + bp := ` + cc_library { + name: "libvndk", + vendor_available: true, + vndk: { + enabled: true, + }, + nocrt: true, + } + + cc_library { + name: "libvndk_ext_product", + product_specific: true, + vndk: { + enabled: true, + extends: "libvndk", + }, + shared_libs: ["libproduct_for_vndklibs"], + nocrt: true, + } + + cc_library { + name: "libvndk_sp", + vendor_available: true, + vndk: { + enabled: true, + support_system_process: true, + }, + nocrt: true, + } + + cc_library { + name: "libvndk_sp_ext_product", + product_specific: true, + vndk: { + enabled: true, + extends: "libvndk_sp", + support_system_process: true, + }, + shared_libs: ["libproduct_for_vndklibs"], + nocrt: true, + } + + cc_library { + name: "libproduct", + product_specific: true, + shared_libs: ["libvndk_ext_product", "libvndk_sp_ext_product"], + nocrt: true, + } + + cc_library { + name: "libproduct_for_vndklibs", + product_specific: true, + nocrt: true, + } + ` + config := TestConfig(buildDir, android.Android, nil, bp, nil) + config.TestProductVariables.DeviceVndkVersion = StringPtr("current") + config.TestProductVariables.ProductVndkVersion = StringPtr("current") + config.TestProductVariables.Platform_vndk_version = StringPtr("VER") + + testCcWithConfig(t, config) +} + func TestVndkSpExtUseVndkError(t *testing.T) { // This test ensures an error is emitted if a VNDK-SP-Ext library depends on a VNDK // library. @@ -1269,6 +1798,268 @@ func TestVndkUseVndkExtError(t *testing.T) { `) } +func TestEnforceProductVndkVersion(t *testing.T) { + bp := ` + cc_library { + name: "libllndk", + } + llndk_library { + name: "libllndk", + symbol_file: "", + } + cc_library { + name: "libvndk", + vendor_available: true, + vndk: { + enabled: true, + }, + nocrt: true, + } + cc_library { + name: "libvndk_sp", + vendor_available: true, + vndk: { + enabled: true, + support_system_process: true, + }, + nocrt: true, + } + cc_library { + name: "libva", + vendor_available: true, + nocrt: true, + } + cc_library { + name: "libproduct_va", + product_specific: true, + vendor_available: true, + nocrt: true, + } + cc_library { + name: "libprod", + product_specific: true, + shared_libs: [ + "libllndk", + "libvndk", + "libvndk_sp", + "libva", + "libproduct_va", + ], + nocrt: true, + } + cc_library { + name: "libvendor", + vendor: true, + shared_libs: [ + "libllndk", + "libvndk", + "libvndk_sp", + "libva", + "libproduct_va", + ], + nocrt: true, + } + ` + + config := TestConfig(buildDir, android.Android, nil, bp, nil) + config.TestProductVariables.DeviceVndkVersion = StringPtr("current") + config.TestProductVariables.ProductVndkVersion = StringPtr("current") + config.TestProductVariables.Platform_vndk_version = StringPtr("VER") + + ctx := testCcWithConfig(t, config) + + checkVndkModule(t, ctx, "libvndk", "vndk-VER", false, "", productVariant) + checkVndkModule(t, ctx, "libvndk_sp", "vndk-sp-VER", true, "", productVariant) +} + +func TestEnforceProductVndkVersionErrors(t *testing.T) { + testCcErrorProductVndk(t, "dependency \".*\" of \".*\" missing variant:\n.*image:product.VER", ` + cc_library { + name: "libprod", + product_specific: true, + shared_libs: [ + "libvendor", + ], + nocrt: true, + } + cc_library { + name: "libvendor", + vendor: true, + nocrt: true, + } + `) + testCcErrorProductVndk(t, "dependency \".*\" of \".*\" missing variant:\n.*image:product.VER", ` + cc_library { + name: "libprod", + product_specific: true, + shared_libs: [ + "libsystem", + ], + nocrt: true, + } + cc_library { + name: "libsystem", + nocrt: true, + } + `) + testCcErrorProductVndk(t, "Vendor module that is not VNDK should not link to \".*\" which is marked as `vendor_available: false`", ` + cc_library { + name: "libprod", + product_specific: true, + shared_libs: [ + "libvndk_private", + ], + nocrt: true, + } + cc_library { + name: "libvndk_private", + vendor_available: false, + vndk: { + enabled: true, + }, + nocrt: true, + } + `) + testCcErrorProductVndk(t, "dependency \".*\" of \".*\" missing variant:\n.*image:product.VER", ` + cc_library { + name: "libprod", + product_specific: true, + shared_libs: [ + "libsystem_ext", + ], + nocrt: true, + } + cc_library { + name: "libsystem_ext", + system_ext_specific: true, + nocrt: true, + } + `) + testCcErrorProductVndk(t, "dependency \".*\" of \".*\" missing variant:\n.*image:", ` + cc_library { + name: "libsystem", + shared_libs: [ + "libproduct_va", + ], + nocrt: true, + } + cc_library { + name: "libproduct_va", + product_specific: true, + vendor_available: true, + nocrt: true, + } + `) +} + +func TestMakeLinkType(t *testing.T) { + bp := ` + cc_library { + name: "libvndk", + vendor_available: true, + vndk: { + enabled: true, + }, + } + cc_library { + name: "libvndksp", + vendor_available: true, + vndk: { + enabled: true, + support_system_process: true, + }, + } + cc_library { + name: "libvndkprivate", + vendor_available: false, + vndk: { + enabled: true, + }, + } + cc_library { + name: "libvendor", + vendor: true, + } + cc_library { + name: "libvndkext", + vendor: true, + vndk: { + enabled: true, + extends: "libvndk", + }, + } + vndk_prebuilt_shared { + name: "prevndk", + version: "27", + target_arch: "arm", + binder32bit: true, + vendor_available: true, + vndk: { + enabled: true, + }, + arch: { + arm: { + srcs: ["liba.so"], + }, + }, + } + cc_library { + name: "libllndk", + } + llndk_library { + name: "libllndk", + symbol_file: "", + } + cc_library { + name: "libllndkprivate", + } + llndk_library { + name: "libllndkprivate", + vendor_available: false, + symbol_file: "", + }` + + config := TestConfig(buildDir, android.Android, nil, bp, nil) + config.TestProductVariables.DeviceVndkVersion = StringPtr("current") + config.TestProductVariables.Platform_vndk_version = StringPtr("VER") + // native:vndk + ctx := testCcWithConfig(t, config) + + assertMapKeys(t, vndkCoreLibraries(config), + []string{"libvndk", "libvndkprivate"}) + assertMapKeys(t, vndkSpLibraries(config), + []string{"libc++", "libvndksp"}) + assertMapKeys(t, llndkLibraries(config), + []string{"libc", "libdl", "libft2", "libllndk", "libllndkprivate", "libm"}) + assertMapKeys(t, vndkPrivateLibraries(config), + []string{"libft2", "libllndkprivate", "libvndkprivate"}) + + vendorVariant27 := "android_vendor.27_arm64_armv8-a_shared" + + tests := []struct { + variant string + name string + expected string + }{ + {vendorVariant, "libvndk", "native:vndk"}, + {vendorVariant, "libvndksp", "native:vndk"}, + {vendorVariant, "libvndkprivate", "native:vndk_private"}, + {vendorVariant, "libvendor", "native:vendor"}, + {vendorVariant, "libvndkext", "native:vendor"}, + {vendorVariant, "libllndk.llndk", "native:vndk"}, + {vendorVariant27, "prevndk.vndk.27.arm.binder32", "native:vndk"}, + {coreVariant, "libvndk", "native:platform"}, + {coreVariant, "libvndkprivate", "native:platform"}, + {coreVariant, "libllndk", "native:platform"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + module := ctx.ModuleForTests(test.name, test.variant).Module().(*Module) + assertString(t, module.makeLinkType, test.expected) + }) + } +} + var ( str11 = "01234567891" str10 = str11[:10] @@ -1624,7 +2415,7 @@ func TestStaticLibDepReordering(t *testing.T) { `) - variant := "android_arm64_armv8-a_core_static" + variant := "android_arm64_armv8-a_static" moduleA := ctx.ModuleForTests("a", variant).Module().(*Module) actual := moduleA.depsInLinkOrder expected := getOutputPaths(ctx, variant, []string{"c", "b", "d"}) @@ -1658,7 +2449,7 @@ func TestStaticLibDepReorderingWithShared(t *testing.T) { `) - variant := "android_arm64_armv8-a_core_static" + variant := "android_arm64_armv8-a_static" moduleA := ctx.ModuleForTests("a", variant).Module().(*Module) actual := moduleA.depsInLinkOrder expected := getOutputPaths(ctx, variant, []string{"c", "b"}) @@ -1673,6 +2464,45 @@ func TestStaticLibDepReorderingWithShared(t *testing.T) { } } +func checkEquals(t *testing.T, message string, expected, actual interface{}) { + if !reflect.DeepEqual(actual, expected) { + t.Errorf(message+ + "\nactual: %v"+ + "\nexpected: %v", + actual, + expected, + ) + } +} + +func TestLlndkLibrary(t *testing.T) { + ctx := testCc(t, ` + cc_library { + name: "libllndk", + stubs: { versions: ["1", "2"] }, + } + llndk_library { + name: "libllndk", + } + `) + actual := ctx.ModuleVariantsForTests("libllndk.llndk") + expected := []string{ + "android_vendor.VER_arm64_armv8-a_shared", + "android_vendor.VER_arm64_armv8-a_shared_1", + "android_vendor.VER_arm64_armv8-a_shared_2", + "android_vendor.VER_arm_armv7-a-neon_shared", + "android_vendor.VER_arm_armv7-a-neon_shared_1", + "android_vendor.VER_arm_armv7-a-neon_shared_2", + } + checkEquals(t, "variants for llndk stubs", expected, actual) + + params := ctx.ModuleForTests("libllndk.llndk", "android_vendor.VER_arm_armv7-a-neon_shared").Description("generate stub") + checkEquals(t, "use VNDK version for default stubs", "current", params.Args["apiLevel"]) + + params = ctx.ModuleForTests("libllndk.llndk", "android_vendor.VER_arm_armv7-a-neon_shared_1").Description("generate stub") + checkEquals(t, "override apiLevel for versioned stubs", "1", params.Args["apiLevel"]) +} + func TestLlndkHeaders(t *testing.T) { ctx := testCc(t, ` llndk_headers { @@ -1688,13 +2518,13 @@ func TestLlndkHeaders(t *testing.T) { shared_libs: ["libllndk"], vendor: true, srcs: ["foo.c"], - no_libgcc: true, + no_libcrt: true, nocrt: true, } `) // _static variant is used since _shared reuses *.o from the static variant - cc := ctx.ModuleForTests("libvendor", "android_arm_armv7-a-neon_vendor_static").Rule("cc") + cc := ctx.ModuleForTests("libvendor", "android_vendor.VER_arm_armv7-a-neon_static").Rule("cc") cflags := cc.Args["cFlags"] if !strings.Contains(cflags, "-Imy_include") { t.Errorf("cflags for libvendor must contain -Imy_include, but was %#v.", cflags) @@ -1717,7 +2547,7 @@ const runtimeLibAndroidBp = ` cc_library { name: "libvendor_available1", vendor_available: true, - no_libgcc : true, + no_libcrt : true, nocrt : true, system_shared_libs : [], } @@ -1725,7 +2555,7 @@ const runtimeLibAndroidBp = ` name: "libvendor_available2", vendor_available: true, runtime_libs: ["libvendor_available1"], - no_libgcc : true, + no_libcrt : true, nocrt : true, system_shared_libs : [], } @@ -1738,21 +2568,21 @@ const runtimeLibAndroidBp = ` exclude_runtime_libs: ["libvendor_available1"], } }, - no_libgcc : true, + no_libcrt : true, nocrt : true, system_shared_libs : [], } cc_library { name: "libcore", runtime_libs: ["libvendor_available1"], - no_libgcc : true, + no_libcrt : true, nocrt : true, system_shared_libs : [], } cc_library { name: "libvendor1", vendor: true, - no_libgcc : true, + no_libcrt : true, nocrt : true, system_shared_libs : [], } @@ -1760,7 +2590,7 @@ const runtimeLibAndroidBp = ` name: "libvendor2", vendor: true, runtime_libs: ["libvendor_available1", "libvendor1"], - no_libgcc : true, + no_libcrt : true, nocrt : true, system_shared_libs : [], } @@ -1770,7 +2600,7 @@ func TestRuntimeLibs(t *testing.T) { ctx := testCc(t, runtimeLibAndroidBp) // runtime_libs for core variants use the module names without suffixes. - variant := "android_arm64_armv8-a_core_shared" + variant := "android_arm64_armv8-a_shared" module := ctx.ModuleForTests("libvendor_available2", variant).Module().(*Module) checkRuntimeLibs(t, []string{"libvendor_available1"}, module) @@ -1780,7 +2610,7 @@ func TestRuntimeLibs(t *testing.T) { // runtime_libs for vendor variants have '.vendor' suffixes if the modules have both core // and vendor variants. - variant = "android_arm64_armv8-a_vendor_shared" + variant = "android_vendor.VER_arm64_armv8-a_shared" module = ctx.ModuleForTests("libvendor_available2", variant).Module().(*Module) checkRuntimeLibs(t, []string{"libvendor_available1.vendor"}, module) @@ -1792,11 +2622,11 @@ func TestRuntimeLibs(t *testing.T) { func TestExcludeRuntimeLibs(t *testing.T) { ctx := testCc(t, runtimeLibAndroidBp) - variant := "android_arm64_armv8-a_core_shared" + variant := "android_arm64_armv8-a_shared" module := ctx.ModuleForTests("libvendor_available3", variant).Module().(*Module) checkRuntimeLibs(t, []string{"libvendor_available1"}, module) - variant = "android_arm64_armv8-a_vendor_shared" + variant = "android_vendor.VER_arm64_armv8-a_shared" module = ctx.ModuleForTests("libvendor_available3", variant).Module().(*Module) checkRuntimeLibs(t, nil, module) } @@ -1806,7 +2636,7 @@ func TestRuntimeLibsNoVndk(t *testing.T) { // If DeviceVndkVersion is not defined, then runtime_libs are copied as-is. - variant := "android_arm64_armv8-a_core_shared" + variant := "android_arm64_armv8-a_shared" module := ctx.ModuleForTests("libvendor_available2", variant).Module().(*Module) checkRuntimeLibs(t, []string{"libvendor_available1"}, module) @@ -1816,6 +2646,7 @@ func TestRuntimeLibsNoVndk(t *testing.T) { } func checkStaticLibs(t *testing.T, expected []string, module *Module) { + t.Helper() actual := module.Properties.AndroidMkStaticLibs if !reflect.DeepEqual(actual, expected) { t.Errorf("incorrect static_libs"+ @@ -1841,15 +2672,15 @@ func TestStaticLibDepExport(t *testing.T) { ctx := testCc(t, staticLibAndroidBp) // Check the shared version of lib2. - variant := "android_arm64_armv8-a_core_shared" + variant := "android_arm64_armv8-a_shared" module := ctx.ModuleForTests("lib2", variant).Module().(*Module) - checkStaticLibs(t, []string{"lib1", "libclang_rt.builtins-aarch64-android", "libatomic", "libgcc_stripped"}, module) + checkStaticLibs(t, []string{"lib1", "libc++demangle", "libclang_rt.builtins-aarch64-android", "libatomic"}, module) // Check the static version of lib2. - variant = "android_arm64_armv8-a_core_static" + variant = "android_arm64_armv8-a_static" module = ctx.ModuleForTests("lib2", variant).Module().(*Module) // libc++_static is linked additionally. - checkStaticLibs(t, []string{"lib1", "libc++_static", "libclang_rt.builtins-aarch64-android", "libatomic", "libgcc_stripped"}, module) + checkStaticLibs(t, []string{"lib1", "libc++_static", "libc++demangle", "libclang_rt.builtins-aarch64-android", "libatomic"}, module) } var compilerFlagsTestCases = []struct { @@ -1952,7 +2783,7 @@ func TestVendorPublicLibraries(t *testing.T) { name: "libvendorpublic", srcs: ["foo.c"], vendor: true, - no_libgcc: true, + no_libcrt: true, nocrt: true, } @@ -1961,7 +2792,7 @@ func TestVendorPublicLibraries(t *testing.T) { shared_libs: ["libvendorpublic"], vendor: false, srcs: ["foo.c"], - no_libgcc: true, + no_libcrt: true, nocrt: true, } cc_library { @@ -1969,33 +2800,34 @@ func TestVendorPublicLibraries(t *testing.T) { shared_libs: ["libvendorpublic"], vendor: true, srcs: ["foo.c"], - no_libgcc: true, + no_libcrt: true, nocrt: true, } `) - variant := "android_arm64_armv8-a_core_shared" + coreVariant := "android_arm64_armv8-a_shared" + vendorVariant := "android_vendor.VER_arm64_armv8-a_shared" // test if header search paths are correctly added // _static variant is used since _shared reuses *.o from the static variant - cc := ctx.ModuleForTests("libsystem", strings.Replace(variant, "_shared", "_static", 1)).Rule("cc") + cc := ctx.ModuleForTests("libsystem", strings.Replace(coreVariant, "_shared", "_static", 1)).Rule("cc") cflags := cc.Args["cFlags"] if !strings.Contains(cflags, "-Imy_include") { t.Errorf("cflags for libsystem must contain -Imy_include, but was %#v.", cflags) } // test if libsystem is linked to the stub - ld := ctx.ModuleForTests("libsystem", variant).Rule("ld") + ld := ctx.ModuleForTests("libsystem", coreVariant).Rule("ld") libflags := ld.Args["libFlags"] - stubPaths := getOutputPaths(ctx, variant, []string{"libvendorpublic" + vendorPublicLibrarySuffix}) + stubPaths := getOutputPaths(ctx, coreVariant, []string{"libvendorpublic" + vendorPublicLibrarySuffix}) if !strings.Contains(libflags, stubPaths[0].String()) { t.Errorf("libflags for libsystem must contain %#v, but was %#v", stubPaths[0], libflags) } // test if libvendor is linked to the real shared lib - ld = ctx.ModuleForTests("libvendor", strings.Replace(variant, "_core", "_vendor", 1)).Rule("ld") + ld = ctx.ModuleForTests("libvendor", vendorVariant).Rule("ld") libflags = ld.Args["libFlags"] - stubPaths = getOutputPaths(ctx, strings.Replace(variant, "_core", "_vendor", 1), []string{"libvendorpublic"}) + stubPaths = getOutputPaths(ctx, vendorVariant, []string{"libvendorpublic"}) if !strings.Contains(libflags, stubPaths[0].String()) { t.Errorf("libflags for libvendor must contain %#v, but was %#v", stubPaths[0], libflags) } @@ -2021,7 +2853,7 @@ func TestRecovery(t *testing.T) { `) variants := ctx.ModuleVariantsForTests("librecovery") - const arm64 = "android_arm64_armv8-a_recovery_shared" + const arm64 = "android_recovery_arm64_armv8-a_shared" if len(variants) != 1 || !android.InList(arm64, variants) { t.Errorf("variants of librecovery must be \"%s\" only, but was %#v", arm64, variants) } @@ -2056,14 +2888,14 @@ func TestVersionedStubs(t *testing.T) { variants := ctx.ModuleVariantsForTests("libFoo") expectedVariants := []string{ - "android_arm64_armv8-a_core_shared", - "android_arm64_armv8-a_core_shared_1", - "android_arm64_armv8-a_core_shared_2", - "android_arm64_armv8-a_core_shared_3", - "android_arm_armv7-a-neon_core_shared", - "android_arm_armv7-a-neon_core_shared_1", - "android_arm_armv7-a-neon_core_shared_2", - "android_arm_armv7-a-neon_core_shared_3", + "android_arm64_armv8-a_shared", + "android_arm64_armv8-a_shared_1", + "android_arm64_armv8-a_shared_2", + "android_arm64_armv8-a_shared_3", + "android_arm_armv7-a-neon_shared", + "android_arm_armv7-a-neon_shared_1", + "android_arm_armv7-a-neon_shared_2", + "android_arm_armv7-a-neon_shared_3", } variantsMismatch := false if len(variants) != len(expectedVariants) { @@ -2086,14 +2918,14 @@ func TestVersionedStubs(t *testing.T) { } } - libBarLinkRule := ctx.ModuleForTests("libBar", "android_arm64_armv8-a_core_shared").Rule("ld") + libBarLinkRule := ctx.ModuleForTests("libBar", "android_arm64_armv8-a_shared").Rule("ld") libFlags := libBarLinkRule.Args["libFlags"] - libFoo1StubPath := "libFoo/android_arm64_armv8-a_core_shared_1/libFoo.so" + libFoo1StubPath := "libFoo/android_arm64_armv8-a_shared_1/libFoo.so" if !strings.Contains(libFlags, libFoo1StubPath) { t.Errorf("%q is not found in %q", libFoo1StubPath, libFlags) } - libBarCompileRule := ctx.ModuleForTests("libBar", "android_arm64_armv8-a_core_shared").Rule("cc") + libBarCompileRule := ctx.ModuleForTests("libBar", "android_arm64_armv8-a_shared").Rule("cc") cFlags := libBarCompileRule.Args["cFlags"] libFoo1VersioningMacro := "-D__LIBFOO_API__=1" if !strings.Contains(cFlags, libFoo1VersioningMacro) { @@ -2101,18 +2933,30 @@ func TestVersionedStubs(t *testing.T) { } } +func TestVersioningMacro(t *testing.T) { + for _, tc := range []struct{ moduleName, expected string }{ + {"libc", "__LIBC_API__"}, + {"libfoo", "__LIBFOO_API__"}, + {"libfoo@1", "__LIBFOO_1_API__"}, + {"libfoo-v1", "__LIBFOO_V1_API__"}, + {"libfoo.v1", "__LIBFOO_V1_API__"}, + } { + checkEquals(t, tc.moduleName, tc.expected, versioningMacroName(tc.moduleName)) + } +} + func TestStaticExecutable(t *testing.T) { ctx := testCc(t, ` cc_binary { name: "static_test", - srcs: ["foo.c"], + srcs: ["foo.c", "baz.o"], static_executable: true, }`) - variant := "android_arm64_armv8-a_core" + variant := "android_arm64_armv8-a" binModuleRule := ctx.ModuleForTests("static_test", variant).Rule("ld") libFlags := binModuleRule.Args["libFlags"] - systemStaticLibs := []string{"libc.a", "libm.a", "libdl.a"} + systemStaticLibs := []string{"libc.a", "libm.a"} for _, lib := range systemStaticLibs { if !strings.Contains(libFlags, lib) { t.Errorf("Static lib %q was not found in %q", lib, libFlags) @@ -2131,20 +2975,20 @@ func TestStaticDepsOrderWithStubs(t *testing.T) { cc_binary { name: "mybin", srcs: ["foo.c"], - static_libs: ["libB"], + static_libs: ["libfooB"], static_executable: true, stl: "none", } cc_library { - name: "libB", + name: "libfooB", srcs: ["foo.c"], - shared_libs: ["libC"], + shared_libs: ["libfooC"], stl: "none", } cc_library { - name: "libC", + name: "libfooC", srcs: ["foo.c"], stl: "none", stubs: { @@ -2152,9 +2996,9 @@ func TestStaticDepsOrderWithStubs(t *testing.T) { }, }`) - mybin := ctx.ModuleForTests("mybin", "android_arm64_armv8-a_core").Module().(*Module) + mybin := ctx.ModuleForTests("mybin", "android_arm64_armv8-a").Module().(*Module) actual := mybin.depsInLinkOrder - expected := getOutputPaths(ctx, "android_arm64_armv8-a_core_static", []string{"libB", "libC"}) + expected := getOutputPaths(ctx, "android_arm64_armv8-a_static", []string{"libfooB", "libfooC"}) if !reflect.DeepEqual(actual, expected) { t.Errorf("staticDeps orderings were not propagated correctly"+ @@ -2165,3 +3009,167 @@ func TestStaticDepsOrderWithStubs(t *testing.T) { ) } } + +func TestErrorsIfAModuleDependsOnDisabled(t *testing.T) { + testCcError(t, `module "libA" .* depends on disabled module "libB"`, ` + cc_library { + name: "libA", + srcs: ["foo.c"], + shared_libs: ["libB"], + stl: "none", + } + + cc_library { + name: "libB", + srcs: ["foo.c"], + enabled: false, + stl: "none", + } + `) +} + +// Simple smoke test for the cc_fuzz target that ensures the rule compiles +// correctly. +func TestFuzzTarget(t *testing.T) { + ctx := testCc(t, ` + cc_fuzz { + name: "fuzz_smoke_test", + srcs: ["foo.c"], + }`) + + variant := "android_arm64_armv8-a_fuzzer" + ctx.ModuleForTests("fuzz_smoke_test", variant).Rule("cc") +} + +func TestAidl(t *testing.T) { +} + +func assertString(t *testing.T, got, expected string) { + t.Helper() + if got != expected { + t.Errorf("expected %q got %q", expected, got) + } +} + +func assertArrayString(t *testing.T, got, expected []string) { + t.Helper() + if len(got) != len(expected) { + t.Errorf("expected %d (%q) got (%d) %q", len(expected), expected, len(got), got) + return + } + for i := range got { + if got[i] != expected[i] { + t.Errorf("expected %d-th %q (%q) got %q (%q)", + i, expected[i], expected, got[i], got) + return + } + } +} + +func assertMapKeys(t *testing.T, m map[string]string, expected []string) { + t.Helper() + assertArrayString(t, android.SortedStringKeys(m), expected) +} + +func TestDefaults(t *testing.T) { + ctx := testCc(t, ` + cc_defaults { + name: "defaults", + srcs: ["foo.c"], + static: { + srcs: ["bar.c"], + }, + shared: { + srcs: ["baz.c"], + }, + } + + cc_library_static { + name: "libstatic", + defaults: ["defaults"], + } + + cc_library_shared { + name: "libshared", + defaults: ["defaults"], + } + + cc_library { + name: "libboth", + defaults: ["defaults"], + } + + cc_binary { + name: "binary", + defaults: ["defaults"], + }`) + + pathsToBase := func(paths android.Paths) []string { + var ret []string + for _, p := range paths { + ret = append(ret, p.Base()) + } + return ret + } + + shared := ctx.ModuleForTests("libshared", "android_arm64_armv8-a_shared").Rule("ld") + if g, w := pathsToBase(shared.Inputs), []string{"foo.o", "baz.o"}; !reflect.DeepEqual(w, g) { + t.Errorf("libshared ld rule wanted %q, got %q", w, g) + } + bothShared := ctx.ModuleForTests("libboth", "android_arm64_armv8-a_shared").Rule("ld") + if g, w := pathsToBase(bothShared.Inputs), []string{"foo.o", "baz.o"}; !reflect.DeepEqual(w, g) { + t.Errorf("libboth ld rule wanted %q, got %q", w, g) + } + binary := ctx.ModuleForTests("binary", "android_arm64_armv8-a").Rule("ld") + if g, w := pathsToBase(binary.Inputs), []string{"foo.o"}; !reflect.DeepEqual(w, g) { + t.Errorf("binary ld rule wanted %q, got %q", w, g) + } + + static := ctx.ModuleForTests("libstatic", "android_arm64_armv8-a_static").Rule("ar") + if g, w := pathsToBase(static.Inputs), []string{"foo.o", "bar.o"}; !reflect.DeepEqual(w, g) { + t.Errorf("libstatic ar rule wanted %q, got %q", w, g) + } + bothStatic := ctx.ModuleForTests("libboth", "android_arm64_armv8-a_static").Rule("ar") + if g, w := pathsToBase(bothStatic.Inputs), []string{"foo.o", "bar.o"}; !reflect.DeepEqual(w, g) { + t.Errorf("libboth ar rule wanted %q, got %q", w, g) + } +} + +func TestProductVariableDefaults(t *testing.T) { + bp := ` + cc_defaults { + name: "libfoo_defaults", + srcs: ["foo.c"], + cppflags: ["-DFOO"], + product_variables: { + debuggable: { + cppflags: ["-DBAR"], + }, + }, + } + + cc_library { + name: "libfoo", + defaults: ["libfoo_defaults"], + } + ` + + config := TestConfig(buildDir, android.Android, nil, bp, nil) + config.TestProductVariables.Debuggable = BoolPtr(true) + + ctx := CreateTestContext() + ctx.PreDepsMutators(func(ctx android.RegisterMutatorsContext) { + ctx.BottomUp("variable", android.VariableMutator).Parallel() + }) + ctx.Register(config) + + _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) + android.FailIfErrored(t, errs) + _, errs = ctx.PrepareBuildActions(config) + android.FailIfErrored(t, errs) + + libfoo := ctx.ModuleForTests("libfoo", "android_arm64_armv8-a_static").Module().(*Module) + if !android.InList("-DBAR", libfoo.flags.Local.CppFlags) { + t.Errorf("expected -DBAR in cppflags, got %q", libfoo.flags.Local.CppFlags) + } +} diff --git a/cc/ccdeps.go b/cc/ccdeps.go new file mode 100644 index 000000000..b96d8b007 --- /dev/null +++ b/cc/ccdeps.go @@ -0,0 +1,266 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cc + +import ( + "encoding/json" + "fmt" + "path" + "sort" + "strings" + + "android/soong/android" +) + +// This singleton collects cc modules' source and flags into to a json file. +// It does so for generating CMakeLists.txt project files needed data when +// either make, mm, mma, mmm or mmma is called. +// The info file is generated in $OUT/module_bp_cc_depend.json. + +func init() { + android.RegisterSingletonType("ccdeps_generator", ccDepsGeneratorSingleton) +} + +func ccDepsGeneratorSingleton() android.Singleton { + return &ccdepsGeneratorSingleton{} +} + +type ccdepsGeneratorSingleton struct { + outputPath android.Path +} + +var _ android.SingletonMakeVarsProvider = (*ccdepsGeneratorSingleton)(nil) + +const ( + // Environment variables used to control the behavior of this singleton. + envVariableCollectCCDeps = "SOONG_COLLECT_CC_DEPS" + ccdepsJsonFileName = "module_bp_cc_deps.json" + cClang = "clang" + cppClang = "clang++" +) + +type ccIdeInfo struct { + Path []string `json:"path,omitempty"` + Srcs []string `json:"srcs,omitempty"` + Global_Common_Flags ccParameters `json:"global_common_flags,omitempty"` + Local_Common_Flags ccParameters `json:"local_common_flags,omitempty"` + Global_C_flags ccParameters `json:"global_c_flags,omitempty"` + Local_C_flags ccParameters `json:"local_c_flags,omitempty"` + Global_C_only_flags ccParameters `json:"global_c_only_flags,omitempty"` + Local_C_only_flags ccParameters `json:"local_c_only_flags,omitempty"` + Global_Cpp_flags ccParameters `json:"global_cpp_flags,omitempty"` + Local_Cpp_flags ccParameters `json:"local_cpp_flags,omitempty"` + System_include_flags ccParameters `json:"system_include_flags,omitempty"` + Module_name string `json:"module_name,omitempty"` +} + +type ccParameters struct { + HeaderSearchPath []string `json:"header_search_path,omitempty"` + SystemHeaderSearchPath []string `json:"system_search_path,omitempty"` + FlagParameters []string `json:"flag,omitempty"` + SysRoot string `json:"system_root,omitempty"` + RelativeFilePathFlags map[string]string `json:"relative_file_path,omitempty"` +} + +type ccMapIdeInfos map[string]ccIdeInfo + +type ccDeps struct { + C_clang string `json:"clang,omitempty"` + Cpp_clang string `json:"clang++,omitempty"` + Modules ccMapIdeInfos `json:"modules,omitempty"` +} + +func (c *ccdepsGeneratorSingleton) GenerateBuildActions(ctx android.SingletonContext) { + if !ctx.Config().IsEnvTrue(envVariableCollectCCDeps) { + return + } + + moduleDeps := ccDeps{} + moduleInfos := map[string]ccIdeInfo{} + + // Track which projects have already had CMakeLists.txt generated to keep the first + // variant for each project. + seenProjects := map[string]bool{} + + pathToCC, _ := evalVariable(ctx, "${config.ClangBin}/") + moduleDeps.C_clang = fmt.Sprintf("%s%s", buildCMakePath(pathToCC), cClang) + moduleDeps.Cpp_clang = fmt.Sprintf("%s%s", buildCMakePath(pathToCC), cppClang) + + ctx.VisitAllModules(func(module android.Module) { + if ccModule, ok := module.(*Module); ok { + if compiledModule, ok := ccModule.compiler.(CompiledInterface); ok { + generateCLionProjectData(ctx, compiledModule, ccModule, seenProjects, moduleInfos) + } + } + }) + + moduleDeps.Modules = moduleInfos + + ccfpath := android.PathForOutput(ctx, ccdepsJsonFileName) + err := createJsonFile(moduleDeps, ccfpath) + if err != nil { + ctx.Errorf(err.Error()) + } + c.outputPath = ccfpath + + // This is necessary to satisfy the dangling rules check as this file is written by Soong rather than a rule. + ctx.Build(pctx, android.BuildParams{ + Rule: android.Touch, + Output: ccfpath, + }) +} + +func (c *ccdepsGeneratorSingleton) MakeVars(ctx android.MakeVarsContext) { + if c.outputPath == nil { + return + } + + ctx.DistForGoal("general-tests", c.outputPath) +} + +func parseCompilerCCParameters(ctx android.SingletonContext, params []string) ccParameters { + compilerParams := ccParameters{} + + cparams := []string{} + for _, param := range params { + param, _ = evalVariable(ctx, param) + cparams = append(cparams, param) + } + + // Soong does not guarantee that each flag will be in an individual string. e.g: The + // input received could be: + // params = {"-isystem", "path/to/system"} + // or it could be + // params = {"-isystem path/to/system"} + // To normalize the input, we split all strings with the "space" character and consolidate + // all tokens into a flattened parameters list + cparams = normalizeParameters(cparams) + + for i := 0; i < len(cparams); i++ { + param := cparams[i] + if param == "" { + continue + } + + switch categorizeParameter(param) { + case headerSearchPath: + compilerParams.HeaderSearchPath = + append(compilerParams.HeaderSearchPath, strings.TrimPrefix(param, "-I")) + case systemHeaderSearchPath: + if i < len(cparams)-1 { + compilerParams.SystemHeaderSearchPath = append(compilerParams.SystemHeaderSearchPath, cparams[i+1]) + } + i = i + 1 + case flag: + c := cleanupParameter(param) + compilerParams.FlagParameters = append(compilerParams.FlagParameters, c) + case systemRoot: + if i < len(cparams)-1 { + compilerParams.SysRoot = cparams[i+1] + } + i = i + 1 + case relativeFilePathFlag: + flagComponents := strings.Split(param, "=") + if len(flagComponents) == 2 { + if compilerParams.RelativeFilePathFlags == nil { + compilerParams.RelativeFilePathFlags = map[string]string{} + } + compilerParams.RelativeFilePathFlags[flagComponents[0]] = flagComponents[1] + } + } + } + return compilerParams +} + +func generateCLionProjectData(ctx android.SingletonContext, compiledModule CompiledInterface, + ccModule *Module, seenProjects map[string]bool, moduleInfos map[string]ccIdeInfo) { + srcs := compiledModule.Srcs() + if len(srcs) == 0 { + return + } + + // Only keep the DeviceArch variant module. + if ctx.DeviceConfig().DeviceArch() != ccModule.ModuleBase.Arch().ArchType.Name { + return + } + + clionProjectLocation := getCMakeListsForModule(ccModule, ctx) + if seenProjects[clionProjectLocation] { + return + } + + seenProjects[clionProjectLocation] = true + + name := ccModule.ModuleBase.Name() + dpInfo := moduleInfos[name] + + dpInfo.Path = append(dpInfo.Path, path.Dir(ctx.BlueprintFile(ccModule))) + dpInfo.Srcs = append(dpInfo.Srcs, srcs.Strings()...) + dpInfo.Path = android.FirstUniqueStrings(dpInfo.Path) + dpInfo.Srcs = android.FirstUniqueStrings(dpInfo.Srcs) + + dpInfo.Global_Common_Flags = parseCompilerCCParameters(ctx, ccModule.flags.Global.CommonFlags) + dpInfo.Local_Common_Flags = parseCompilerCCParameters(ctx, ccModule.flags.Local.CommonFlags) + dpInfo.Global_C_flags = parseCompilerCCParameters(ctx, ccModule.flags.Global.CFlags) + dpInfo.Local_C_flags = parseCompilerCCParameters(ctx, ccModule.flags.Local.CFlags) + dpInfo.Global_C_only_flags = parseCompilerCCParameters(ctx, ccModule.flags.Global.ConlyFlags) + dpInfo.Local_C_only_flags = parseCompilerCCParameters(ctx, ccModule.flags.Local.ConlyFlags) + dpInfo.Global_Cpp_flags = parseCompilerCCParameters(ctx, ccModule.flags.Global.CppFlags) + dpInfo.Local_Cpp_flags = parseCompilerCCParameters(ctx, ccModule.flags.Local.CppFlags) + dpInfo.System_include_flags = parseCompilerCCParameters(ctx, ccModule.flags.SystemIncludeFlags) + + dpInfo.Module_name = name + + moduleInfos[name] = dpInfo +} + +type Deal struct { + Name string + ideInfo ccIdeInfo +} + +type Deals []Deal + +// Ensure it satisfies sort.Interface +func (d Deals) Len() int { return len(d) } +func (d Deals) Less(i, j int) bool { return d[i].Name < d[j].Name } +func (d Deals) Swap(i, j int) { d[i], d[j] = d[j], d[i] } + +func sortMap(moduleInfos map[string]ccIdeInfo) map[string]ccIdeInfo { + var deals Deals + for k, v := range moduleInfos { + deals = append(deals, Deal{k, v}) + } + + sort.Sort(deals) + + m := map[string]ccIdeInfo{} + for _, d := range deals { + m[d.Name] = d.ideInfo + } + return m +} + +func createJsonFile(moduleDeps ccDeps, ccfpath android.WritablePath) error { + buf, err := json.MarshalIndent(moduleDeps, "", "\t") + if err != nil { + return fmt.Errorf("JSON marshal of cc deps failed: %s", err) + } + err = android.WriteFileToOutputDir(ccfpath, buf, 0666) + if err != nil { + return fmt.Errorf("Writing cc deps to %s failed: %s", ccfpath.String(), err) + } + return nil +} diff --git a/cc/cflag_artifacts.go b/cc/cflag_artifacts.go new file mode 100644 index 000000000..855ff25b3 --- /dev/null +++ b/cc/cflag_artifacts.go @@ -0,0 +1,183 @@ +package cc + +import ( + "fmt" + "sort" + "strings" + + "github.com/google/blueprint/proptools" + + "android/soong/android" +) + +func init() { + android.RegisterSingletonType("cflag_artifacts_text", cflagArtifactsTextFactory) +} + +var ( + TrackedCFlags = []string{ + "-Wall", + "-Werror", + "-Wextra", + "-Wthread-safety", + "-O3", + } + + TrackedCFlagsDir = []string{ + "device/google/", + "vendor/google/", + } +) + +const FileBP = 50 + +// Stores output files. +type cflagArtifactsText struct { + interOutputs map[string]android.WritablePaths + outputs android.WritablePaths +} + +// allowedDir verifies if the directory/project is part of the TrackedCFlagsDir +// filter. +func allowedDir(subdir string) bool { + subdir += "/" + return android.HasAnyPrefix(subdir, TrackedCFlagsDir) +} + +func (s *cflagArtifactsText) genFlagFilename(flag string) string { + return fmt.Sprintf("module_cflags%s.txt", flag) +} + +// incrementFile is used to generate an output path object with the passed in flag +// and part number. +// e.g. FLAG + part # -> out/soong/cflags/module_cflags-FLAG.txt.0 +func (s *cflagArtifactsText) incrementFile(ctx android.SingletonContext, + flag string, part int) (string, android.OutputPath) { + + filename := fmt.Sprintf("%s.%d", s.genFlagFilename(flag), part) + filepath := android.PathForOutput(ctx, "cflags", filename) + s.interOutputs[flag] = append(s.interOutputs[flag], filepath) + return filename, filepath +} + +// GenCFlagArtifactParts is used to generate the build rules which produce the +// intermediary files for each desired C Flag artifact +// e.g. module_cflags-FLAG.txt.0, module_cflags-FLAG.txt.1, ... +func (s *cflagArtifactsText) GenCFlagArtifactParts(ctx android.SingletonContext, + flag string, using bool, modules []string, part int) int { + + cleanedName := strings.Replace(flag, "=", "_", -1) + filename, filepath := s.incrementFile(ctx, cleanedName, part) + rule := android.NewRuleBuilder() + rule.Command().Textf("rm -f %s", filepath.String()) + + if using { + rule.Command(). + Textf("echo '# Modules using %s'", flag). + FlagWithOutput(">> ", filepath) + } else { + rule.Command(). + Textf("echo '# Modules not using %s'", flag). + FlagWithOutput(">> ", filepath) + } + + length := len(modules) + + if length == 0 { + rule.Build(pctx, ctx, filename, "gen "+filename) + part++ + } + + // Following loop splits the module list for each tracked C Flag into + // chunks of length FileBP (file breakpoint) and generates a partial artifact + // (intermediary file) build rule for each split. + moduleShards := android.ShardStrings(modules, FileBP) + for index, shard := range moduleShards { + rule.Command(). + Textf("for m in %s; do echo $m", + strings.Join(proptools.ShellEscapeList(shard), " ")). + FlagWithOutput(">> ", filepath). + Text("; done") + rule.Build(pctx, ctx, filename, "gen "+filename) + + if index+1 != len(moduleShards) { + filename, filepath = s.incrementFile(ctx, cleanedName, part+index+1) + rule = android.NewRuleBuilder() + rule.Command().Textf("rm -f %s", filepath.String()) + } + } + + return part + len(moduleShards) +} + +// GenCFlagArtifacts is used to generate build rules which combine the +// intermediary files of a specific tracked flag into a single C Flag artifact +// for each tracked flag. +// e.g. module_cflags-FLAG.txt.0 + module_cflags-FLAG.txt.1 = module_cflags-FLAG.txt +func (s *cflagArtifactsText) GenCFlagArtifacts(ctx android.SingletonContext) { + // Scans through s.interOutputs and creates a build rule for each tracked C + // Flag that concatenates the associated intermediary file into a single + // artifact. + for _, flag := range TrackedCFlags { + // Generate build rule to combine related intermediary files into a + // C Flag artifact + rule := android.NewRuleBuilder() + filename := s.genFlagFilename(flag) + outputpath := android.PathForOutput(ctx, "cflags", filename) + rule.Command(). + Text("cat"). + Inputs(s.interOutputs[flag].Paths()). + FlagWithOutput("> ", outputpath) + rule.Build(pctx, ctx, filename, "gen "+filename) + s.outputs = append(s.outputs, outputpath) + } +} + +func (s *cflagArtifactsText) GenerateBuildActions(ctx android.SingletonContext) { + modulesWithCFlag := make(map[string][]string) + + // Scan through all modules, selecting the ones that are part of the filter, + // and then storing into a map which tracks whether or not tracked C flag is + // used or not. + ctx.VisitAllModules(func(module android.Module) { + if ccModule, ok := module.(*Module); ok { + if allowedDir(ctx.ModuleDir(ccModule)) { + cflags := ccModule.flags.Local.CFlags + cppflags := ccModule.flags.Local.CppFlags + module := fmt.Sprintf("%s:%s (%s)", + ctx.BlueprintFile(ccModule), + ctx.ModuleName(ccModule), + ctx.ModuleSubDir(ccModule)) + for _, flag := range TrackedCFlags { + if inList(flag, cflags) || inList(flag, cppflags) { + modulesWithCFlag[flag] = append(modulesWithCFlag[flag], module) + } else { + modulesWithCFlag["!"+flag] = append(modulesWithCFlag["!"+flag], module) + } + } + } + } + }) + + // Traversing map and setting up rules to produce intermediary files which + // contain parts of each expected C Flag artifact. + for _, flag := range TrackedCFlags { + sort.Strings(modulesWithCFlag[flag]) + part := s.GenCFlagArtifactParts(ctx, flag, true, modulesWithCFlag[flag], 0) + sort.Strings(modulesWithCFlag["!"+flag]) + s.GenCFlagArtifactParts(ctx, flag, false, modulesWithCFlag["!"+flag], part) + } + + // Combine intermediary files into a single C Flag artifact. + s.GenCFlagArtifacts(ctx) +} + +func cflagArtifactsTextFactory() android.Singleton { + return &cflagArtifactsText{ + interOutputs: make(map[string]android.WritablePaths), + } +} + +func (s *cflagArtifactsText) MakeVars(ctx android.MakeVarsContext) { + ctx.Strict("SOONG_MODULES_CFLAG_ARTIFACTS", strings.Join(s.outputs.Strings(), " ")) +} diff --git a/cc/check.go b/cc/check.go index 4e9e16080..46328e915 100644 --- a/cc/check.go +++ b/cc/check.go @@ -38,6 +38,11 @@ func CheckBadCompilerFlags(ctx BaseModuleContext, prop string, flags []string) { ctx.PropertyErrorf(prop, "Illegal flag `%s`", flag) } else if flag == "--coverage" { ctx.PropertyErrorf(prop, "Bad flag: `%s`, use native_coverage instead", flag) + } else if flag == "-Weverything" { + if !ctx.Config().IsEnvTrue("ANDROID_TEMPORARILY_ALLOW_WEVERYTHING") { + ctx.PropertyErrorf(prop, "-Weverything is not allowed in Android.bp files. "+ + "Build with `m ANDROID_TEMPORARILY_ALLOW_WEVERYTHING=true` to experiment locally with -Weverything.") + } } else if strings.Contains(flag, " ") { args := strings.Split(flag, " ") if args[0] == "-include" { diff --git a/cc/cmakelists.go b/cc/cmakelists.go index 7b4f89b54..f7d9081db 100644 --- a/cc/cmakelists.go +++ b/cc/cmakelists.go @@ -76,7 +76,7 @@ func (c *cmakelistsGeneratorSingleton) GenerateBuildActions(ctx android.Singleto // Link all handmade CMakeLists.txt aggregate from // BASE/development/ide/clion to // BASE/out/development/ide/clion. - dir := filepath.Join(getAndroidSrcRootDirectory(ctx), cLionAggregateProjectsDirectory) + dir := filepath.Join(android.AbsSrcDirForExistingUseCases(), cLionAggregateProjectsDirectory) filepath.Walk(dir, linkAggregateCMakeListsFiles) return @@ -147,7 +147,7 @@ func generateCLionProject(compiledModule CompiledInterface, ctx android.Singleto f.WriteString("# Tools > CMake > Change Project Root \n\n") f.WriteString(fmt.Sprintf("cmake_minimum_required(VERSION %s)\n", minimumCMakeVersionSupported)) f.WriteString(fmt.Sprintf("project(%s)\n", ccModule.ModuleBase.Name())) - f.WriteString(fmt.Sprintf("set(ANDROID_ROOT %s)\n\n", getAndroidSrcRootDirectory(ctx))) + f.WriteString(fmt.Sprintf("set(ANDROID_ROOT %s)\n\n", android.AbsSrcDirForExistingUseCases())) pathToCC, _ := evalVariable(ctx, "${config.ClangBin}/") f.WriteString(fmt.Sprintf("set(CMAKE_C_COMPILER \"%s%s\")\n", buildCMakePath(pathToCC), "clang")) @@ -162,25 +162,41 @@ func generateCLionProject(compiledModule CompiledInterface, ctx android.Singleto f.WriteString(")\n") // Add all header search path and compiler parameters (-D, -W, -f, -XXXX) - f.WriteString("\n# GLOBAL FLAGS:\n") - globalParameters := parseCompilerParameters(ccModule.flags.GlobalFlags, ctx, f) - translateToCMake(globalParameters, f, true, true) + f.WriteString("\n# GLOBAL ALL FLAGS:\n") + globalAllParameters := parseCompilerParameters(ccModule.flags.Global.CommonFlags, ctx, f) + translateToCMake(globalAllParameters, f, true, true) - f.WriteString("\n# CFLAGS:\n") - cParameters := parseCompilerParameters(ccModule.flags.CFlags, ctx, f) - translateToCMake(cParameters, f, true, true) + f.WriteString("\n# LOCAL ALL FLAGS:\n") + localAllParameters := parseCompilerParameters(ccModule.flags.Local.CommonFlags, ctx, f) + translateToCMake(localAllParameters, f, true, true) - f.WriteString("\n# C ONLY FLAGS:\n") - cOnlyParameters := parseCompilerParameters(ccModule.flags.ConlyFlags, ctx, f) - translateToCMake(cOnlyParameters, f, true, false) + f.WriteString("\n# GLOBAL CFLAGS:\n") + globalCParameters := parseCompilerParameters(ccModule.flags.Global.CFlags, ctx, f) + translateToCMake(globalCParameters, f, true, true) - f.WriteString("\n# CPP FLAGS:\n") - cppParameters := parseCompilerParameters(ccModule.flags.CppFlags, ctx, f) - translateToCMake(cppParameters, f, false, true) + f.WriteString("\n# LOCAL CFLAGS:\n") + localCParameters := parseCompilerParameters(ccModule.flags.Local.CFlags, ctx, f) + translateToCMake(localCParameters, f, true, true) - f.WriteString("\n# SYSTEM INCLUDE FLAGS:\n") - includeParameters := parseCompilerParameters(ccModule.flags.SystemIncludeFlags, ctx, f) - translateToCMake(includeParameters, f, true, true) + f.WriteString("\n# GLOBAL C ONLY FLAGS:\n") + globalConlyParameters := parseCompilerParameters(ccModule.flags.Global.ConlyFlags, ctx, f) + translateToCMake(globalConlyParameters, f, true, false) + + f.WriteString("\n# LOCAL C ONLY FLAGS:\n") + localConlyParameters := parseCompilerParameters(ccModule.flags.Local.ConlyFlags, ctx, f) + translateToCMake(localConlyParameters, f, true, false) + + f.WriteString("\n# GLOBAL CPP FLAGS:\n") + globalCppParameters := parseCompilerParameters(ccModule.flags.Global.CppFlags, ctx, f) + translateToCMake(globalCppParameters, f, false, true) + + f.WriteString("\n# LOCAL CPP FLAGS:\n") + localCppParameters := parseCompilerParameters(ccModule.flags.Local.CppFlags, ctx, f) + translateToCMake(localCppParameters, f, false, true) + + f.WriteString("\n# GLOBAL SYSTEM INCLUDE FLAGS:\n") + globalIncludeParameters := parseCompilerParameters(ccModule.flags.SystemIncludeFlags, ctx, f) + translateToCMake(globalIncludeParameters, f, true, true) // Add project executable. f.WriteString(fmt.Sprintf("\nadd_executable(%s ${SOURCE_FILES})\n", @@ -306,6 +322,20 @@ func categorizeParameter(parameter string) parameterType { return flag } +// Flattens a list of strings potentially containing space characters into a list of string containing no +// spaces. +func normalizeParameters(params []string) []string { + var flatParams []string + for _, s := range params { + s = strings.Trim(s, " ") + if len(s) == 0 { + continue + } + flatParams = append(flatParams, strings.Split(s, " ")...) + } + return flatParams +} + func parseCompilerParameters(params []string, ctx android.SingletonContext, f *os.File) compilerParameters { var compilerParameters = makeCompilerParameters() @@ -313,6 +343,15 @@ func parseCompilerParameters(params []string, ctx android.SingletonContext, f *o f.WriteString(fmt.Sprintf("# Raw param [%d] = '%s'\n", i, str)) } + // Soong does not guarantee that each flag will be in an individual string. e.g: The + // input received could be: + // params = {"-isystem", "path/to/system"} + // or it could be + // params = {"-isystem path/to/system"} + // To normalize the input, we split all strings with the "space" character and consolidate + // all tokens into a flattened parameters list + params = normalizeParameters(params) + for i := 0; i < len(params); i++ { param := params[i] if param == "" { @@ -426,7 +465,7 @@ func evalVariable(ctx android.SingletonContext, str string) (string, error) { } func getCMakeListsForModule(module *Module, ctx android.SingletonContext) string { - return filepath.Join(getAndroidSrcRootDirectory(ctx), + return filepath.Join(android.AbsSrcDirForExistingUseCases(), cLionOutputProjectsDirectory, path.Dir(ctx.BlueprintFile(module)), module.ModuleBase.Name()+"-"+ @@ -434,8 +473,3 @@ func getCMakeListsForModule(module *Module, ctx android.SingletonContext) string module.ModuleBase.Os().Name, cMakeListsFilename) } - -func getAndroidSrcRootDirectory(ctx android.SingletonContext) string { - srcPath, _ := filepath.Abs(android.PathForSource(ctx).String()) - return srcPath -} diff --git a/cc/compdb.go b/cc/compdb.go index 110265174..ea124438e 100644 --- a/cc/compdb.go +++ b/cc/compdb.go @@ -27,7 +27,7 @@ import ( // This singleton generates a compile_commands.json file. It does so for each // blueprint Android.bp resulting in a cc.Module when either make, mm, mma, mmm // or mmma is called. It will only create a single compile_commands.json file -// at out/development/ide/compdb/compile_commands.json. It will also symlink it +// at ${OUT_DIR}/soong/development/ide/compdb/compile_commands.json. It will also symlink it // to ${SOONG_LINK_COMPDB_TO} if set. In general this should be created by running // make SOONG_GEN_COMPDB=1 nothing to get all targets. @@ -43,7 +43,7 @@ type compdbGeneratorSingleton struct{} const ( compdbFilename = "compile_commands.json" - compdbOutputProjectsDirectory = "out/development/ide/compdb" + compdbOutputProjectsDirectory = "development/ide/compdb" // Environment variables used to modify behavior of this singleton. envVariableGenerateCompdb = "SOONG_GEN_COMPDB" @@ -78,12 +78,12 @@ func (c *compdbGeneratorSingleton) GenerateBuildActions(ctx android.SingletonCon }) // Create the output file. - dir := filepath.Join(getCompdbAndroidSrcRootDirectory(ctx), compdbOutputProjectsDirectory) - os.MkdirAll(dir, 0777) - compDBFile := filepath.Join(dir, compdbFilename) - f, err := os.Create(compdbFilename) + dir := android.PathForOutput(ctx, compdbOutputProjectsDirectory) + os.MkdirAll(filepath.Join(android.AbsSrcDirForExistingUseCases(), dir.String()), 0777) + compDBFile := dir.Join(ctx, compdbFilename) + f, err := os.Create(filepath.Join(android.AbsSrcDirForExistingUseCases(), compDBFile.String())) if err != nil { - log.Fatalf("Could not create file %s: %s", filepath.Join(dir, compdbFilename), err) + log.Fatalf("Could not create file %s: %s", compDBFile, err) } defer f.Close() @@ -103,10 +103,10 @@ func (c *compdbGeneratorSingleton) GenerateBuildActions(ctx android.SingletonCon } f.Write(dat) - finalLinkPath := filepath.Join(ctx.Config().Getenv(envVariableCompdbLink), compdbFilename) - if finalLinkPath != "" { + if finalLinkDir := ctx.Config().Getenv(envVariableCompdbLink); finalLinkDir != "" { + finalLinkPath := filepath.Join(finalLinkDir, compdbFilename) os.Remove(finalLinkPath) - if err := os.Symlink(compDBFile, finalLinkPath); err != nil { + if err := os.Symlink(compDBFile.String(), finalLinkPath); err != nil { log.Fatalf("Unable to symlink %s to %s: %s", compDBFile, finalLinkPath, err) } } @@ -141,7 +141,7 @@ func getArguments(src android.Path, ctx android.SingletonContext, ccModule *Modu isAsm = false isCpp = false clangPath = ccPath - case ".cpp", ".cc", ".mm": + case ".cpp", ".cc", ".cxx", ".mm": isAsm = false isCpp = true clangPath = cxxPath @@ -152,12 +152,16 @@ func getArguments(src android.Path, ctx android.SingletonContext, ccModule *Modu clangPath = ccPath } args = append(args, clangPath) - args = append(args, expandAllVars(ctx, ccModule.flags.GlobalFlags)...) - args = append(args, expandAllVars(ctx, ccModule.flags.CFlags)...) + args = append(args, expandAllVars(ctx, ccModule.flags.Global.CommonFlags)...) + args = append(args, expandAllVars(ctx, ccModule.flags.Local.CommonFlags)...) + args = append(args, expandAllVars(ctx, ccModule.flags.Global.CFlags)...) + args = append(args, expandAllVars(ctx, ccModule.flags.Local.CFlags)...) if isCpp { - args = append(args, expandAllVars(ctx, ccModule.flags.CppFlags)...) + args = append(args, expandAllVars(ctx, ccModule.flags.Global.CppFlags)...) + args = append(args, expandAllVars(ctx, ccModule.flags.Local.CppFlags)...) } else if !isAsm { - args = append(args, expandAllVars(ctx, ccModule.flags.ConlyFlags)...) + args = append(args, expandAllVars(ctx, ccModule.flags.Global.ConlyFlags)...) + args = append(args, expandAllVars(ctx, ccModule.flags.Local.ConlyFlags)...) } args = append(args, expandAllVars(ctx, ccModule.flags.SystemIncludeFlags)...) args = append(args, src.String()) @@ -170,18 +174,17 @@ func generateCompdbProject(compiledModule CompiledInterface, ctx android.Singlet return } - rootDir := getCompdbAndroidSrcRootDirectory(ctx) - pathToCC, err := ctx.Eval(pctx, rootDir+"/${config.ClangBin}/") + pathToCC, err := ctx.Eval(pctx, "${config.ClangBin}") ccPath := "/bin/false" cxxPath := "/bin/false" if err == nil { - ccPath = pathToCC + "clang" - cxxPath = pathToCC + "clang++" + ccPath = filepath.Join(pathToCC, "clang") + cxxPath = filepath.Join(pathToCC, "clang++") } for _, src := range srcs { if _, ok := builds[src.String()]; !ok { builds[src.String()] = compDbEntry{ - Directory: rootDir, + Directory: android.AbsSrcDirForExistingUseCases(), Arguments: getArguments(src, ctx, ccModule, ccPath, cxxPath), File: src.String(), } @@ -196,8 +199,3 @@ func evalAndSplitVariable(ctx android.SingletonContext, str string) ([]string, e } return []string{""}, err } - -func getCompdbAndroidSrcRootDirectory(ctx android.SingletonContext) string { - srcPath, _ := filepath.Abs(android.PathForSource(ctx).String()) - return srcPath -} diff --git a/cc/compiler.go b/cc/compiler.go index f9af4d829..f6c358775 100644 --- a/cc/compiler.go +++ b/cc/compiler.go @@ -17,6 +17,8 @@ package cc import ( "fmt" "path/filepath" + "regexp" + "strconv" "strings" "github.com/google/blueprint/proptools" @@ -25,6 +27,10 @@ import ( "android/soong/cc/config" ) +var ( + allowedManualInterfacePaths = []string{"vendor/", "hardware/"} +) + // This file contains the basic C/C++/assembly to .o compliation steps type BaseCompilerProperties struct { @@ -57,9 +63,6 @@ type BaseCompilerProperties struct { // compiling with clang Clang_asflags []string `android:"arch_variant"` - // list of module-specific flags that will be used for .y and .yy compiles - Yaccflags []string - // the instruction set architecture to use to compile the C/C++ // module. Instruction_set *string `android:"arch_variant"` @@ -103,6 +106,8 @@ type BaseCompilerProperties struct { // if set to false, use -std=c++* instead of -std=gnu++* Gnu_extensions *bool + Yacc *YaccProperties + Aidl struct { // list of directories that will be added to the aidl include paths. Include_dirs []string @@ -171,6 +176,9 @@ type BaseCompilerProperties struct { // Build and link with OpenMP Openmp *bool `android:"arch_variant"` + + // Adds __ANDROID_APEX_<APEX_MODULE_NAME>__ macro defined for apex variants in addition to __ANDROID_APEX__ + Use_apex_name_macro *bool } func NewBaseCompiler() *baseCompiler { @@ -226,11 +234,6 @@ func (compiler *baseCompiler) compilerDeps(ctx DepsContext, deps Deps) Deps { deps = protoDeps(ctx, deps, &compiler.Proto, Bool(compiler.Properties.Proto.Static)) } - if compiler.hasSrcExt(".sysprop") { - deps.HeaderLibs = append(deps.HeaderLibs, "libbase_headers") - deps.SharedLibs = append(deps.SharedLibs, "liblog") - } - if Bool(compiler.Properties.Openmp) { deps.StaticLibs = append(deps.StaticLibs, "libomp") } @@ -241,12 +244,7 @@ func (compiler *baseCompiler) compilerDeps(ctx DepsContext, deps Deps) Deps { // Return true if the module is in the WarningAllowedProjects. func warningsAreAllowed(subdir string) bool { subdir += "/" - for _, prefix := range config.WarningAllowedProjects { - if strings.HasPrefix(subdir, prefix) { - return true - } - } - return false + return android.HasAnyPrefix(subdir, config.WarningAllowedProjects) } func addToModuleList(ctx ModuleContext, key android.OnceKey, module string) { @@ -257,6 +255,7 @@ func addToModuleList(ctx ModuleContext, key android.OnceKey, module string) { // per-target values, module type values, and per-module Blueprints properties func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps PathDeps) Flags { tc := ctx.toolchain() + modulePath := android.PathForModuleSrc(ctx).String() compiler.srcsBeforeGen = android.PathsForModuleSrcExcludes(ctx, compiler.Properties.Srcs, compiler.Properties.Exclude_srcs) compiler.srcsBeforeGen = append(compiler.srcsBeforeGen, deps.GeneratedSources...) @@ -270,31 +269,32 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps esc := proptools.NinjaAndShellEscapeList - flags.CFlags = append(flags.CFlags, esc(compiler.Properties.Cflags)...) - flags.CppFlags = append(flags.CppFlags, esc(compiler.Properties.Cppflags)...) - flags.ConlyFlags = append(flags.ConlyFlags, esc(compiler.Properties.Conlyflags)...) - flags.AsFlags = append(flags.AsFlags, esc(compiler.Properties.Asflags)...) - flags.YasmFlags = append(flags.YasmFlags, esc(compiler.Properties.Asflags)...) - flags.YaccFlags = append(flags.YaccFlags, esc(compiler.Properties.Yaccflags)...) + flags.Local.CFlags = append(flags.Local.CFlags, esc(compiler.Properties.Cflags)...) + flags.Local.CppFlags = append(flags.Local.CppFlags, esc(compiler.Properties.Cppflags)...) + flags.Local.ConlyFlags = append(flags.Local.ConlyFlags, esc(compiler.Properties.Conlyflags)...) + flags.Local.AsFlags = append(flags.Local.AsFlags, esc(compiler.Properties.Asflags)...) + flags.Local.YasmFlags = append(flags.Local.YasmFlags, esc(compiler.Properties.Asflags)...) + + flags.Yacc = compiler.Properties.Yacc // Include dir cflags localIncludeDirs := android.PathsForModuleSrc(ctx, compiler.Properties.Local_include_dirs) if len(localIncludeDirs) > 0 { f := includeDirsToFlags(localIncludeDirs) - flags.GlobalFlags = append(flags.GlobalFlags, f) - flags.YasmFlags = append(flags.YasmFlags, f) + flags.Local.CommonFlags = append(flags.Local.CommonFlags, f) + flags.Local.YasmFlags = append(flags.Local.YasmFlags, f) } rootIncludeDirs := android.PathsForSource(ctx, compiler.Properties.Include_dirs) if len(rootIncludeDirs) > 0 { f := includeDirsToFlags(rootIncludeDirs) - flags.GlobalFlags = append(flags.GlobalFlags, f) - flags.YasmFlags = append(flags.YasmFlags, f) + flags.Local.CommonFlags = append(flags.Local.CommonFlags, f) + flags.Local.YasmFlags = append(flags.Local.YasmFlags, f) } if compiler.Properties.Include_build_directory == nil || *compiler.Properties.Include_build_directory { - flags.GlobalFlags = append(flags.GlobalFlags, "-I"+android.PathForModuleSrc(ctx).String()) - flags.YasmFlags = append(flags.YasmFlags, "-I"+android.PathForModuleSrc(ctx).String()) + flags.Local.CommonFlags = append(flags.Local.CommonFlags, "-I"+modulePath) + flags.Local.YasmFlags = append(flags.Local.YasmFlags, "-I"+modulePath) } if !(ctx.useSdk() || ctx.useVndk()) || ctx.Host() { @@ -313,35 +313,28 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps flags.SystemIncludeFlags = append(flags.SystemIncludeFlags, "-isystem "+getCurrentIncludePath(ctx).String(), "-isystem "+getCurrentIncludePath(ctx).Join(ctx, config.NDKTriple(tc)).String()) - - // TODO: Migrate to API suffixed triple? - // Traditionally this has come from android/api-level.h, but with the - // libc headers unified it must be set by the build system since we - // don't have per-API level copies of that header now. - version := ctx.sdkVersion() - if version == "current" { - version = "__ANDROID_API_FUTURE__" - } - flags.GlobalFlags = append(flags.GlobalFlags, - "-D__ANDROID_API__="+version) } if ctx.useVndk() { - // sdkVersion() returns VNDK version for vendor modules. - version := ctx.sdkVersion() - if version == "current" { - version = "__ANDROID_API_FUTURE__" - } - flags.GlobalFlags = append(flags.GlobalFlags, - "-D__ANDROID_API__="+version, "-D__ANDROID_VNDK__") + flags.Global.CommonFlags = append(flags.Global.CommonFlags, "-D__ANDROID_VNDK__") } if ctx.inRecovery() { - flags.GlobalFlags = append(flags.GlobalFlags, "-D__ANDROID_RECOVERY__") + flags.Global.CommonFlags = append(flags.Global.CommonFlags, "-D__ANDROID_RECOVERY__") } if ctx.apexName() != "" { - flags.GlobalFlags = append(flags.GlobalFlags, "-D__ANDROID_APEX__="+ctx.apexName()) + flags.Global.CommonFlags = append(flags.Global.CommonFlags, "-D__ANDROID_APEX__") + if Bool(compiler.Properties.Use_apex_name_macro) { + flags.Global.CommonFlags = append(flags.Global.CommonFlags, "-D__ANDROID_APEX_"+makeDefineString(ctx.apexName())+"__") + } + if ctx.Device() { + flags.Global.CommonFlags = append(flags.Global.CommonFlags, "-D__ANDROID_SDK_VERSION__="+strconv.Itoa(ctx.apexSdkVersion())) + } + } + + if ctx.Target().NativeBridge == android.NativeBridgeEnabled { + flags.Global.CommonFlags = append(flags.Global.CommonFlags, "-D__ANDROID_NATIVE_BRIDGE__") } instructionSet := String(compiler.Properties.Instruction_set) @@ -356,60 +349,69 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps CheckBadCompilerFlags(ctx, "release.cflags", compiler.Properties.Release.Cflags) // TODO: debug - flags.CFlags = append(flags.CFlags, esc(compiler.Properties.Release.Cflags)...) + flags.Local.CFlags = append(flags.Local.CFlags, esc(compiler.Properties.Release.Cflags)...) CheckBadCompilerFlags(ctx, "clang_cflags", compiler.Properties.Clang_cflags) CheckBadCompilerFlags(ctx, "clang_asflags", compiler.Properties.Clang_asflags) - flags.CFlags = config.ClangFilterUnknownCflags(flags.CFlags) - flags.CFlags = append(flags.CFlags, esc(compiler.Properties.Clang_cflags)...) - flags.AsFlags = append(flags.AsFlags, esc(compiler.Properties.Clang_asflags)...) - flags.CppFlags = config.ClangFilterUnknownCflags(flags.CppFlags) - flags.ConlyFlags = config.ClangFilterUnknownCflags(flags.ConlyFlags) - flags.LdFlags = config.ClangFilterUnknownCflags(flags.LdFlags) + flags.Local.CFlags = config.ClangFilterUnknownCflags(flags.Local.CFlags) + flags.Local.CFlags = append(flags.Local.CFlags, esc(compiler.Properties.Clang_cflags)...) + flags.Local.AsFlags = append(flags.Local.AsFlags, esc(compiler.Properties.Clang_asflags)...) + flags.Local.CppFlags = config.ClangFilterUnknownCflags(flags.Local.CppFlags) + flags.Local.ConlyFlags = config.ClangFilterUnknownCflags(flags.Local.ConlyFlags) + flags.Local.LdFlags = config.ClangFilterUnknownCflags(flags.Local.LdFlags) target := "-target " + tc.ClangTriple() + if ctx.Os().Class == android.Device { + version := ctx.sdkVersion() + if version == "" || version == "current" { + target += strconv.Itoa(android.FutureApiLevel) + } else { + target += version + } + } + gccPrefix := "-B" + config.ToolPath(tc) - flags.CFlags = append(flags.CFlags, target, gccPrefix) - flags.AsFlags = append(flags.AsFlags, target, gccPrefix) - flags.LdFlags = append(flags.LdFlags, target, gccPrefix) + flags.Global.CFlags = append(flags.Global.CFlags, target, gccPrefix) + flags.Global.AsFlags = append(flags.Global.AsFlags, target, gccPrefix) + flags.Global.LdFlags = append(flags.Global.LdFlags, target, gccPrefix) hod := "Host" if ctx.Os().Class == android.Device { hod = "Device" } - flags.GlobalFlags = append(flags.GlobalFlags, instructionSetFlags) - flags.ConlyFlags = append([]string{"${config.CommonGlobalConlyflags}"}, flags.ConlyFlags...) - flags.CppFlags = append([]string{fmt.Sprintf("${config.%sGlobalCppflags}", hod)}, flags.CppFlags...) + flags.Global.CommonFlags = append(flags.Global.CommonFlags, instructionSetFlags) + flags.Global.ConlyFlags = append([]string{"${config.CommonGlobalConlyflags}"}, flags.Global.ConlyFlags...) + flags.Global.CppFlags = append([]string{fmt.Sprintf("${config.%sGlobalCppflags}", hod)}, flags.Global.CppFlags...) - flags.AsFlags = append(flags.AsFlags, tc.ClangAsflags()) - flags.CppFlags = append([]string{"${config.CommonClangGlobalCppflags}"}, flags.CppFlags...) - flags.GlobalFlags = append(flags.GlobalFlags, + flags.Global.AsFlags = append(flags.Global.AsFlags, tc.ClangAsflags()) + flags.Global.CppFlags = append([]string{"${config.CommonClangGlobalCppflags}"}, flags.Global.CppFlags...) + flags.Global.CommonFlags = append(flags.Global.CommonFlags, tc.ClangCflags(), "${config.CommonClangGlobalCflags}", fmt.Sprintf("${config.%sClangGlobalCflags}", hod)) - if strings.HasPrefix(android.PathForModuleSrc(ctx).String(), "external/") { - flags.GlobalFlags = append([]string{"${config.ClangExternalCflags}"}, flags.GlobalFlags...) + if isThirdParty(modulePath) { + flags.Global.CommonFlags = append([]string{"${config.ClangExternalCflags}"}, flags.Global.CommonFlags...) } if tc.Bionic() { if Bool(compiler.Properties.Rtti) { - flags.CppFlags = append(flags.CppFlags, "-frtti") + flags.Local.CppFlags = append(flags.Local.CppFlags, "-frtti") } else { - flags.CppFlags = append(flags.CppFlags, "-fno-rtti") + flags.Local.CppFlags = append(flags.Local.CppFlags, "-fno-rtti") } } - flags.AsFlags = append(flags.AsFlags, "-D__ASSEMBLY__") + flags.Global.AsFlags = append(flags.Global.AsFlags, "-D__ASSEMBLY__") - flags.CppFlags = append(flags.CppFlags, tc.ClangCppflags()) + flags.Global.CppFlags = append(flags.Global.CppFlags, tc.ClangCppflags()) - flags.YasmFlags = append(flags.YasmFlags, tc.YasmFlags()) + flags.Global.YasmFlags = append(flags.Global.YasmFlags, tc.YasmFlags()) - flags.GlobalFlags = append(flags.GlobalFlags, tc.ToolchainClangCflags()) + flags.Global.CommonFlags = append(flags.Global.CommonFlags, tc.ToolchainClangCflags()) cStd := config.CStdVersion if String(compiler.Properties.C_std) == "experimental" { @@ -431,15 +433,15 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps cppStd = gnuToCReplacer.Replace(cppStd) } - flags.ConlyFlags = append([]string{"-std=" + cStd}, flags.ConlyFlags...) - flags.CppFlags = append([]string{"-std=" + cppStd}, flags.CppFlags...) + flags.Local.ConlyFlags = append([]string{"-std=" + cStd}, flags.Local.ConlyFlags...) + flags.Local.CppFlags = append([]string{"-std=" + cppStd}, flags.Local.CppFlags...) if ctx.useVndk() { - flags.CFlags = append(flags.CFlags, esc(compiler.Properties.Target.Vendor.Cflags)...) + flags.Local.CFlags = append(flags.Local.CFlags, esc(compiler.Properties.Target.Vendor.Cflags)...) } if ctx.inRecovery() { - flags.CFlags = append(flags.CFlags, esc(compiler.Properties.Target.Recovery.Cflags)...) + flags.Local.CFlags = append(flags.Local.CFlags, esc(compiler.Properties.Target.Recovery.Cflags)...) } // We can enforce some rules more strictly in the code we own. strict @@ -448,14 +450,14 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps // vendor/device specific things), we could extend this to be a ternary // value. strict := true - if strings.HasPrefix(android.PathForModuleSrc(ctx).String(), "external/") { + if strings.HasPrefix(modulePath, "external/") { strict = false } // Can be used to make some annotations stricter for code we can fix // (such as when we mark functions as deprecated). if strict { - flags.CFlags = append(flags.CFlags, "-DANDROID_STRICT") + flags.Global.CFlags = append(flags.Global.CFlags, "-DANDROID_STRICT") } if compiler.hasSrcExt(".proto") { @@ -463,12 +465,12 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps } if compiler.hasSrcExt(".y") || compiler.hasSrcExt(".yy") { - flags.GlobalFlags = append(flags.GlobalFlags, + flags.Local.CommonFlags = append(flags.Local.CommonFlags, "-I"+android.PathForModuleGen(ctx, "yacc", ctx.ModuleDir()).String()) } if compiler.hasSrcExt(".mc") { - flags.GlobalFlags = append(flags.GlobalFlags, + flags.Local.CommonFlags = append(flags.Local.CommonFlags, "-I"+android.PathForModuleGen(ctx, "windmc", ctx.ModuleDir()).String()) } @@ -486,35 +488,41 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps flags.aidlFlags = append(flags.aidlFlags, "-t") } - flags.GlobalFlags = append(flags.GlobalFlags, + flags.Local.CommonFlags = append(flags.Local.CommonFlags, "-I"+android.PathForModuleGen(ctx, "aidl").String()) } - if compiler.hasSrcExt(".rs") || compiler.hasSrcExt(".fs") { + if compiler.hasSrcExt(".rscript") || compiler.hasSrcExt(".fs") { flags = rsFlags(ctx, flags, &compiler.Properties) } if compiler.hasSrcExt(".sysprop") { - flags.GlobalFlags = append(flags.GlobalFlags, + flags.Local.CommonFlags = append(flags.Local.CommonFlags, "-I"+android.PathForModuleGen(ctx, "sysprop", "include").String()) } if len(compiler.Properties.Srcs) > 0 { module := ctx.ModuleDir() + "/Android.bp:" + ctx.ModuleName() - if inList("-Wno-error", flags.CFlags) || inList("-Wno-error", flags.CppFlags) { + if inList("-Wno-error", flags.Local.CFlags) || inList("-Wno-error", flags.Local.CppFlags) { addToModuleList(ctx, modulesUsingWnoErrorKey, module) - } else if !inList("-Werror", flags.CFlags) && !inList("-Werror", flags.CppFlags) { + } else if !inList("-Werror", flags.Local.CFlags) && !inList("-Werror", flags.Local.CppFlags) { if warningsAreAllowed(ctx.ModuleDir()) { addToModuleList(ctx, modulesAddedWallKey, module) - flags.CFlags = append([]string{"-Wall"}, flags.CFlags...) + flags.Local.CFlags = append([]string{"-Wall"}, flags.Local.CFlags...) } else { - flags.CFlags = append([]string{"-Wall", "-Werror"}, flags.CFlags...) + flags.Local.CFlags = append([]string{"-Wall", "-Werror"}, flags.Local.CFlags...) } } } if Bool(compiler.Properties.Openmp) { - flags.CFlags = append(flags.CFlags, "-fopenmp") + flags.Local.CFlags = append(flags.Local.CFlags, "-fopenmp") + } + + // Exclude directories from manual binder interface allowed list. + //TODO(b/145621474): Move this check into IInterface.h when clang-tidy no longer uses absolute paths. + if android.HasAnyPrefix(ctx.ModuleDir(), allowedManualInterfacePaths) { + flags.Local.CFlags = append(flags.Local.CFlags, "-DDO_NOT_CHECK_MANUAL_BINDER_INTERFACES") } return flags @@ -540,6 +548,12 @@ func (compiler *baseCompiler) hasSrcExt(ext string) bool { return false } +// makeDefineString transforms a name of an APEX module into a value to be used as value for C define +// For example, com.android.foo => COM_ANDROID_FOO +func makeDefineString(name string) string { + return strings.ReplaceAll(strings.ToUpper(name), ".", "_") +} + var gnuToCReplacer = strings.NewReplacer("gnu", "c") func ndkPathDeps(ctx ModuleContext) android.Paths { @@ -552,7 +566,7 @@ func ndkPathDeps(ctx ModuleContext) android.Paths { } func (compiler *baseCompiler) compile(ctx ModuleContext, flags Flags, deps PathDeps) Objects { - pathDeps := deps.GeneratedHeaders + pathDeps := deps.GeneratedDeps pathDeps = append(pathDeps, ndkPathDeps(ctx)...) buildFlags := flagsToBuilderFlags(flags) @@ -584,3 +598,24 @@ func compileObjs(ctx android.ModuleContext, flags builderFlags, return TransformSourceToObj(ctx, subdir, srcFiles, flags, pathDeps, cFlagsDeps) } + +var thirdPartyDirPrefixExceptions = []*regexp.Regexp{ + regexp.MustCompile("^vendor/[^/]*google[^/]*/"), + regexp.MustCompile("^hardware/google/"), + regexp.MustCompile("^hardware/interfaces/"), + regexp.MustCompile("^hardware/libhardware[^/]*/"), + regexp.MustCompile("^hardware/ril/"), +} + +func isThirdParty(path string) bool { + thirdPartyDirPrefixes := []string{"external/", "vendor/", "hardware/"} + + if android.HasAnyPrefix(path, thirdPartyDirPrefixes) { + for _, prefix := range thirdPartyDirPrefixExceptions { + if prefix.MatchString(path) { + return false + } + } + } + return true +} diff --git a/cc/compiler_test.go b/cc/compiler_test.go new file mode 100644 index 000000000..c301388ae --- /dev/null +++ b/cc/compiler_test.go @@ -0,0 +1,43 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cc + +import ( + "testing" +) + +func TestIsThirdParty(t *testing.T) { + shouldFail := []string{ + "external/foo/", + "vendor/bar/", + "hardware/underwater_jaguar/", + } + shouldPass := []string{ + "vendor/google/cts/", + "hardware/google/pixel", + "hardware/interfaces/camera", + "hardware/ril/supa_ril", + } + for _, path := range shouldFail { + if !isThirdParty(path) { + t.Errorf("Expected %s to be considered third party", path) + } + } + for _, path := range shouldPass { + if isThirdParty(path) { + t.Errorf("Expected %s to *not* be considered third party", path) + } + } +} diff --git a/cc/config/Android.bp b/cc/config/Android.bp new file mode 100644 index 000000000..7edb0c988 --- /dev/null +++ b/cc/config/Android.bp @@ -0,0 +1,32 @@ +bootstrap_go_package { + name: "soong-cc-config", + pkgPath: "android/soong/cc/config", + deps: [ + "soong-android", + "soong-remoteexec", + ], + srcs: [ + "clang.go", + "global.go", + "tidy.go", + "toolchain.go", + "vndk.go", + + "arm_device.go", + "arm64_device.go", + "arm64_fuchsia_device.go", + "mips_device.go", + "mips64_device.go", + "x86_device.go", + "x86_64_device.go", + "x86_64_fuchsia_device.go", + + "x86_darwin_host.go", + "x86_linux_host.go", + "x86_linux_bionic_host.go", + "x86_windows_host.go", + ], + testSrcs: [ + "tidy_test.go", + ], +} diff --git a/cc/config/OWNERS b/cc/config/OWNERS new file mode 100644 index 000000000..b2f54e5bb --- /dev/null +++ b/cc/config/OWNERS @@ -0,0 +1 @@ +per-file vndk.go = smoreland@google.com, victoryang@google.com diff --git a/cc/config/arm64_device.go b/cc/config/arm64_device.go index 1ca1656d0..19aedd99d 100644 --- a/cc/config/arm64_device.go +++ b/cc/config/arm64_device.go @@ -39,6 +39,7 @@ var ( arm64Ldflags = []string{ "-Wl,-m,aarch64_elf64_le_vec", "-Wl,--hash-style=gnu", + "-Wl,-z,separate-code", "-fuse-ld=gold", "-Wl,--icf=safe", } diff --git a/cc/config/arm_device.go b/cc/config/arm_device.go index cd7c4107e..d37e48620 100644 --- a/cc/config/arm_device.go +++ b/cc/config/arm_device.go @@ -236,6 +236,7 @@ var ( "cortex-a72": "${config.ArmClangCortexA53Cflags}", "cortex-a73": "${config.ArmClangCortexA53Cflags}", "cortex-a75": "${config.ArmClangCortexA55Cflags}", + "cortex-a76": "${config.ArmClangCortexA55Cflags}", "krait": "${config.ArmClangKraitCflags}", "kryo": "${config.ArmClangKryoCflags}", "kryo385": "${config.ArmClangCortexA53Cflags}", diff --git a/cc/config/clang.go b/cc/config/clang.go index a87d56956..2e0b24183 100644 --- a/cc/config/clang.go +++ b/cc/config/clang.go @@ -48,6 +48,8 @@ var ClangUnknownCflags = sorted([]string{ "-Wunused-but-set-parameter", "-Wunused-but-set-variable", "-fdiagnostics-color", + // http://b/153759688 + "-fuse-init-array", // arm + arm64 + mips + mips64 "-fgcse-after-reload", @@ -101,21 +103,17 @@ func init() { // not emit the table by default on Android since NDK still uses GNU binutils. "-faddrsig", - // -Wimplicit-fallthrough is not enabled by -Wall. - "-Wimplicit-fallthrough", - // Help catch common 32/64-bit errors. "-Werror=int-conversion", + // Enable the new pass manager. + "-fexperimental-new-pass-manager", + // Disable overly aggressive warning for macros defined with a leading underscore // This happens in AndroidConfig.h, which is included nearly everywhere. // TODO: can we remove this now? "-Wno-reserved-id-macro", - // Disable overly aggressive warning for format strings. - // Bug: 20148343 - "-Wno-format-pedantic", - // Workaround for ccache with clang. // See http://petereisentraut.blogspot.com/2011/05/ccache-and-clang.html. "-Wno-unused-command-line-argument", @@ -124,9 +122,6 @@ func init() { // color codes if it is not running in a terminal. "-fcolor-diagnostics", - // http://b/68236239 Allow 0/NULL instead of using nullptr everywhere. - "-Wno-zero-as-null-pointer-constant", - // Warnings from clang-7.0 "-Wno-sign-compare", @@ -136,13 +131,18 @@ func init() { // Disable -Winconsistent-missing-override until we can clean up the existing // codebase for it. "-Wno-inconsistent-missing-override", + + // Warnings from clang-10 + // Nested and array designated initialization is nice to have. + "-Wno-c99-designator", }, " ")) pctx.StaticVariable("ClangExtraCppflags", strings.Join([]string{ + // -Wimplicit-fallthrough is not enabled by -Wall. + "-Wimplicit-fallthrough", + // Enable clang's thread-safety annotations in libcxx. - // Turn off -Wthread-safety-negative, to avoid breaking projects that use -Weverything. "-D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS", - "-Wno-thread-safety-negative", // libc++'s math.h has an #include_next outside of system_headers. "-Wno-gnu-include-next", @@ -164,19 +164,29 @@ func init() { // new warnings are fixed. "-Wno-tautological-constant-compare", "-Wno-tautological-type-limit-compare", - "-Wno-tautological-unsigned-enum-zero-compare", - "-Wno-tautological-unsigned-zero-compare", - - // Disable c++98-specific warning since Android is not concerned with C++98 - // compatibility. - "-Wno-c++98-compat-extra-semi", - - // Disable this warning because we don't care about behavior with older compilers. - "-Wno-return-std-move-in-c++11", + // http://b/145210666 + "-Wno-reorder-init-list", + // http://b/145211066 + "-Wno-implicit-int-float-conversion", + // New warnings to be fixed after clang-r377782. + "-Wno-int-in-bool-context", // http://b/148287349 + "-Wno-sizeof-array-div", // http://b/148815709 + "-Wno-tautological-overlap-compare", // http://b/148815696 + // New warnings to be fixed after clang-r383902. + "-Wno-deprecated-copy", // http://b/153746672 + "-Wno-range-loop-construct", // http://b/153747076 + "-Wno-misleading-indentation", // http://b/153746954 + "-Wno-zero-as-null-pointer-constant", // http://b/68236239 + "-Wno-deprecated-anon-enum-enum-conversion", // http://b/153746485 + "-Wno-deprecated-enum-enum-conversion", // http://b/153746563 + "-Wno-string-compare", // http://b/153764102 + "-Wno-enum-enum-conversion", // http://b/154138986 + "-Wno-enum-float-conversion", // http://b/154255917 + "-Wno-pessimizing-move", // http://b/154270751 }, " ")) - // Extra cflags for projects under external/ directory to disable warnings that are infeasible - // to fix in all the external projects and their upstream repos. + // Extra cflags for external third-party projects to disable warnings that + // are infeasible to fix in all the external projects and their upstream repos. pctx.StaticVariable("ClangExtraExternalCflags", strings.Join([]string{ "-Wno-enum-compare", "-Wno-enum-compare-switch", @@ -188,6 +198,13 @@ func init() { // Bug: http://b/29823425 Disable -Wnull-dereference until the // new instances detected by this warning are fixed. "-Wno-null-dereference", + + // http://b/145211477 + "-Wno-pointer-compare", + // http://b/145211022 + "-Wno-xor-used-as-pow", + // http://b/145211022 + "-Wno-final-dtor-non-final-class", }, " ")) } diff --git a/cc/config/global.go b/cc/config/global.go index 926b6684e..57d8db96b 100644 --- a/cc/config/global.go +++ b/cc/config/global.go @@ -47,6 +47,10 @@ var ( "-g", "-fno-strict-aliasing", + + "-Werror=date-time", + "-Werror=pragma-pack", + "-Werror=pragma-pack-suspicious-include", } commonGlobalConlyflags = []string{} @@ -68,7 +72,6 @@ var ( "-Werror=non-virtual-dtor", "-Werror=address", "-Werror=sequence-point", - "-Werror=date-time", "-Werror=format-security", } @@ -86,6 +89,7 @@ var ( "-Wl,--no-undefined-version", "-Wl,--exclude-libs,libgcc.a", "-Wl,--exclude-libs,libgcc_stripped.a", + "-Wl,--exclude-libs,libunwind_llvm.a", } deviceGlobalLldflags = append(ClangFilterUnknownLldflags(deviceGlobalLdflags), @@ -108,6 +112,7 @@ var ( noOverrideGlobalCflags = []string{ "-Werror=int-to-pointer-cast", "-Werror=pointer-to-int-cast", + "-Werror=fortify-source", } IllegalFlags = []string{ @@ -123,8 +128,8 @@ var ( // prebuilts/clang default settings. ClangDefaultBase = "prebuilts/clang/host" - ClangDefaultVersion = "clang-r353983c1" - ClangDefaultShortVersion = "9.0.3" + ClangDefaultVersion = "clang-r383902b" + ClangDefaultShortVersion = "11.0.2" // Directories with warnings from Android.bp files. WarningAllowedProjects = []string{ @@ -151,8 +156,27 @@ func init() { pctx.StaticVariable("HostGlobalLdflags", strings.Join(hostGlobalLdflags, " ")) pctx.StaticVariable("HostGlobalLldflags", strings.Join(hostGlobalLldflags, " ")) - pctx.StaticVariable("CommonClangGlobalCflags", - strings.Join(append(ClangFilterUnknownCflags(commonGlobalCflags), "${ClangExtraCflags}"), " ")) + pctx.VariableFunc("CommonClangGlobalCflags", func(ctx android.PackageVarContext) string { + flags := ClangFilterUnknownCflags(commonGlobalCflags) + flags = append(flags, "${ClangExtraCflags}") + + // http://b/131390872 + // Automatically initialize any uninitialized stack variables. + // Prefer zero-init if multiple options are set. + if ctx.Config().IsEnvTrue("AUTO_ZERO_INITIALIZE") { + flags = append(flags, "-ftrivial-auto-var-init=zero -enable-trivial-auto-var-init-zero-knowing-it-will-be-removed-from-clang") + } else if ctx.Config().IsEnvTrue("AUTO_PATTERN_INITIALIZE") { + flags = append(flags, "-ftrivial-auto-var-init=pattern") + } else if ctx.Config().IsEnvTrue("AUTO_UNINITIALIZE") { + flags = append(flags, "-ftrivial-auto-var-init=uninitialized") + } else { + // Default to zero initialization. + flags = append(flags, "-ftrivial-auto-var-init=zero -enable-trivial-auto-var-init-zero-knowing-it-will-be-removed-from-clang") + } + + return strings.Join(flags, " ") + }) + pctx.VariableFunc("DeviceClangGlobalCflags", func(ctx android.PackageVarContext) string { if ctx.Config().Fuchsia() { return strings.Join(ClangFilterUnknownCflags(deviceGlobalCflags), " ") @@ -203,7 +227,6 @@ func init() { }) pctx.StaticVariable("ClangPath", "${ClangBase}/${HostPrebuiltTag}/${ClangVersion}") pctx.StaticVariable("ClangBin", "${ClangPath}/bin") - pctx.StaticVariable("ClangTidyShellPath", "build/soong/scripts/clang-tidy.sh") pctx.VariableFunc("ClangShortVersion", func(ctx android.PackageVarContext) string { if override := ctx.Config().Getenv("LLVM_RELEASE_VERSION"); override != "" { @@ -238,6 +261,7 @@ func init() { pctx.VariableFunc("RECXXLinksPool", remoteexec.EnvOverrideFunc("RBE_CXX_LINKS_POOL", remoteexec.DefaultPool)) pctx.VariableFunc("RECXXLinksExecStrategy", remoteexec.EnvOverrideFunc("RBE_CXX_LINKS_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) pctx.VariableFunc("REAbiDumperExecStrategy", remoteexec.EnvOverrideFunc("RBE_ABI_DUMPER_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) + pctx.VariableFunc("REAbiLinkerExecStrategy", remoteexec.EnvOverrideFunc("RBE_ABI_LINKER_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) } var HostPrebuiltTag = pctx.VariableConfigMethod("HostPrebuiltTag", android.Config.PrebuiltOS) diff --git a/cc/config/toolchain.go b/cc/config/toolchain.go index d5e9d0169..db9092db0 100644 --- a/cc/config/toolchain.go +++ b/cc/config/toolchain.go @@ -181,6 +181,9 @@ func LibclangRuntimeLibrary(t Toolchain, library string) string { if arch == "" { return "" } + if !t.Bionic() { + return "libclang_rt." + library + "-" + arch + } return "libclang_rt." + library + "-" + arch + "-android" } @@ -224,6 +227,10 @@ func ScudoMinimalRuntimeLibrary(t Toolchain) string { return LibclangRuntimeLibrary(t, "scudo_minimal") } +func LibFuzzerRuntimeLibrary(t Toolchain) string { + return LibclangRuntimeLibrary(t, "fuzzer") +} + func ToolPath(t Toolchain) string { if p := t.ToolPath(); p != "" { return p diff --git a/cc/config/vndk.go b/cc/config/vndk.go index 542f73759..6f2e80741 100644 --- a/cc/config/vndk.go +++ b/cc/config/vndk.go @@ -18,140 +18,33 @@ package config // For these libraries, the vendor variants must be installed even if the device // has VndkUseCoreVariant set. var VndkMustUseVendorVariantList = []string{ - "android.frameworks.sensorservice@1.0", - "android.hardware.atrace@1.0", - "android.hardware.audio.common@5.0", - "android.hardware.audio.effect@2.0", - "android.hardware.audio.effect@4.0", - "android.hardware.audio.effect@5.0", - "android.hardware.audio@2.0", - "android.hardware.audio@4.0", - "android.hardware.audio@5.0", - "android.hardware.automotive.evs@1.0", - "android.hardware.automotive.vehicle@2.0", - "android.hardware.bluetooth.audio@2.0", - "android.hardware.boot@1.0", - "android.hardware.broadcastradio@1.0", - "android.hardware.broadcastradio@1.1", - "android.hardware.broadcastradio@2.0", - "android.hardware.camera.device@1.0", - "android.hardware.camera.device@3.2", - "android.hardware.camera.device@3.3", - "android.hardware.camera.device@3.4", - "android.hardware.camera.provider@2.4", - "android.hardware.cas.native@1.0", - "android.hardware.cas@1.0", - "android.hardware.configstore@1.0", - "android.hardware.configstore@1.1", - "android.hardware.contexthub@1.0", - "android.hardware.drm@1.0", - "android.hardware.drm@1.1", - "android.hardware.fastboot@1.0", - "android.hardware.gatekeeper@1.0", - "android.hardware.gnss@1.0", - "android.hardware.graphics.allocator@2.0", - "android.hardware.graphics.bufferqueue@1.0", - "android.hardware.graphics.composer@2.1", - "android.hardware.graphics.composer@2.2", - "android.hardware.health@1.0", - "android.hardware.health@2.0", - "android.hardware.ir@1.0", - "android.hardware.keymaster@3.0", - "android.hardware.keymaster@4.0", - "android.hardware.light@2.0", - "android.hardware.media.bufferpool@1.0", - "android.hardware.media.omx@1.0", - "android.hardware.memtrack@1.0", - "android.hardware.neuralnetworks@1.0", - "android.hardware.neuralnetworks@1.1", - "android.hardware.neuralnetworks@1.2", - "android.hardware.nfc@1.1", + "android.hardware.automotive.occupant_awareness-ndk_platform", + "android.hardware.light-ndk_platform", + "android.hardware.identity-ndk_platform", "android.hardware.nfc@1.2", - "android.hardware.oemlock@1.0", - "android.hardware.power.stats@1.0", - "android.hardware.power@1.0", - "android.hardware.power@1.1", - "android.hardware.radio@1.4", - "android.hardware.secure_element@1.0", - "android.hardware.sensors@1.0", - "android.hardware.soundtrigger@2.0", - "android.hardware.soundtrigger@2.0-core", - "android.hardware.soundtrigger@2.1", - "android.hardware.tetheroffload.config@1.0", - "android.hardware.tetheroffload.control@1.0", - "android.hardware.thermal@1.0", - "android.hardware.tv.cec@1.0", - "android.hardware.tv.input@1.0", - "android.hardware.vibrator@1.0", - "android.hardware.vibrator@1.1", - "android.hardware.vibrator@1.2", - "android.hardware.weaver@1.0", - "android.hardware.wifi.hostapd@1.0", - "android.hardware.wifi.offload@1.0", - "android.hardware.wifi.supplicant@1.0", - "android.hardware.wifi.supplicant@1.1", - "android.hardware.wifi@1.0", - "android.hardware.wifi@1.1", - "android.hardware.wifi@1.2", - "android.hardwareundtrigger@2.0", - "android.hardwareundtrigger@2.0-core", - "android.hardwareundtrigger@2.1", - "android.hidl.allocator@1.0", - "android.hidl.token@1.0", - "android.hidl.token@1.0-utils", - "android.system.net.netd@1.0", - "android.system.wifi.keystore@1.0", - "libaudioroute", - "libaudioutils", + "android.hardware.power-ndk_platform", + "android.hardware.rebootescrow-ndk_platform", + "android.hardware.vibrator-ndk_platform", "libbinder", - "libcamera_metadata", "libcrypto", - "libdiskconfig", - "libdumpstateutil", "libexpat", - "libfmq", + "libgatekeeper", "libgui", "libhidlcache", - "libmedia_helper", + "libkeymaster_messages", + "libkeymaster_portable", "libmedia_omx", - "libmemtrack", - "libnetutils", "libpuresoftkeymasterdevice", - "libradio_metadata", "libselinux", "libsoftkeymasterdevice", "libsqlite", "libssl", + "libstagefright_bufferpool@2.0", "libstagefright_bufferqueue_helper", - "libstagefright_flacdec", "libstagefright_foundation", "libstagefright_omx", "libstagefright_omx_utils", - "libstagefright_soft_aacdec", - "libstagefright_soft_aacenc", - "libstagefright_soft_amrdec", - "libstagefright_soft_amrnbenc", - "libstagefright_soft_amrwbenc", - "libstagefright_soft_avcdec", - "libstagefright_soft_avcenc", - "libstagefright_soft_flacdec", - "libstagefright_soft_flacenc", - "libstagefright_soft_g711dec", - "libstagefright_soft_gsmdec", - "libstagefright_soft_hevcdec", - "libstagefright_soft_mp3dec", - "libstagefright_soft_mpeg2dec", - "libstagefright_soft_mpeg4dec", - "libstagefright_soft_mpeg4enc", - "libstagefright_soft_opusdec", - "libstagefright_soft_rawdec", - "libstagefright_soft_vorbisdec", - "libstagefright_soft_vpxdec", - "libstagefright_soft_vpxenc", "libstagefright_xmlparser", - "libsysutils", "libui", - "libvorbisidec", "libxml2", - "libziparchive", } diff --git a/cc/config/x86_64_device.go b/cc/config/x86_64_device.go index 0f0420f76..bcfae5d4d 100644 --- a/cc/config/x86_64_device.go +++ b/cc/config/x86_64_device.go @@ -63,14 +63,20 @@ var ( } x86_64ArchFeatureCflags = map[string][]string{ - "ssse3": []string{"-DUSE_SSSE3", "-mssse3"}, + "ssse3": []string{"-mssse3"}, "sse4": []string{"-msse4"}, "sse4_1": []string{"-msse4.1"}, "sse4_2": []string{"-msse4.2"}, + + // Not all cases there is performance gain by enabling -mavx -mavx2 + // flags so these flags are not enabled by default. + // if there is performance gain in individual library components, + // the compiler flags can be set in corresponding bp files. + // "avx": []string{"-mavx"}, + // "avx2": []string{"-mavx2"}, + // "avx512": []string{"-mavx512"} + "popcnt": []string{"-mpopcnt"}, - "avx": []string{"-mavx"}, - "avx2": []string{"-mavx2"}, - "avx512": []string{"-mavx512"}, "aes_ni": []string{"-maes"}, } ) diff --git a/cc/config/x86_64_fuchsia_device.go b/cc/config/x86_64_fuchsia_device.go index 79af00c5c..0f2013b54 100644 --- a/cc/config/x86_64_fuchsia_device.go +++ b/cc/config/x86_64_fuchsia_device.go @@ -92,7 +92,7 @@ func (t *toolchainFuchsiaX8664) YasmFlags() string { } func (t *toolchainFuchsiaX8664) ToolchainClangCflags() string { - return "-DUSE_SSSE3 -mssse3" + return "-mssse3" } var toolchainFuchsiaSingleton Toolchain = &toolchainFuchsiaX8664{} diff --git a/cc/config/x86_darwin_host.go b/cc/config/x86_darwin_host.go index 1026370dc..8eb79e34e 100644 --- a/cc/config/x86_darwin_host.go +++ b/cc/config/x86_darwin_host.go @@ -15,9 +15,11 @@ package config import ( + "fmt" "os/exec" "path/filepath" "strings" + "sync" "android/soong/android" ) @@ -63,6 +65,7 @@ var ( "10.12", "10.13", "10.14", + "10.15", } darwinAvailableLibraries = append( @@ -88,28 +91,20 @@ const ( ) func init() { - pctx.VariableFunc("macSdkPath", func(ctx android.PackageVarContext) string { - xcodeselect := ctx.Config().HostSystemTool("xcode-select") - bytes, err := exec.Command(xcodeselect, "--print-path").Output() - if err != nil { - ctx.Errorf("xcode-select failed with: %q", err.Error()) - } - return strings.TrimSpace(string(bytes)) - }) pctx.VariableFunc("macSdkRoot", func(ctx android.PackageVarContext) string { - return xcrunSdk(ctx, "--show-sdk-path") + return getMacTools(ctx).sdkRoot }) - pctx.StaticVariable("macMinVersion", "10.8") + pctx.StaticVariable("macMinVersion", "10.10") pctx.VariableFunc("MacArPath", func(ctx android.PackageVarContext) string { - return xcrun(ctx, "--find", "ar") + return getMacTools(ctx).arPath }) pctx.VariableFunc("MacStripPath", func(ctx android.PackageVarContext) string { - return xcrun(ctx, "--find", "strip") + return getMacTools(ctx).stripPath }) pctx.VariableFunc("MacToolPath", func(ctx android.PackageVarContext) string { - return filepath.Dir(xcrun(ctx, "--find", "ld")) + return getMacTools(ctx).toolPath }) pctx.StaticVariable("DarwinGccVersion", darwinGccVersion) @@ -125,38 +120,66 @@ func init() { pctx.StaticVariable("DarwinYasmFlags", "-f macho -m amd64") } -func xcrun(ctx android.PackageVarContext, args ...string) string { - xcrun := ctx.Config().HostSystemTool("xcrun") - bytes, err := exec.Command(xcrun, args...).Output() - if err != nil { - ctx.Errorf("xcrun failed with: %q", err.Error()) - } - return strings.TrimSpace(string(bytes)) +type macPlatformTools struct { + once sync.Once + err error + + sdkRoot string + arPath string + stripPath string + toolPath string } -func xcrunSdk(ctx android.PackageVarContext, arg string) string { - xcrun := ctx.Config().HostSystemTool("xcrun") - if selected := ctx.Config().Getenv("MAC_SDK_VERSION"); selected != "" { - if !inList(selected, darwinSupportedSdkVersions) { - ctx.Errorf("MAC_SDK_VERSION %s isn't supported: %q", selected, darwinSupportedSdkVersions) - return "" - } +var macTools = &macPlatformTools{} - bytes, err := exec.Command(xcrun, "--sdk", "macosx"+selected, arg).Output() - if err != nil { - ctx.Errorf("MAC_SDK_VERSION %s is not installed", selected) - } - return strings.TrimSpace(string(bytes)) - } +func getMacTools(ctx android.PackageVarContext) *macPlatformTools { + macTools.once.Do(func() { + xcrunTool := ctx.Config().HostSystemTool("xcrun") + + xcrun := func(args ...string) string { + if macTools.err != nil { + return "" + } + + bytes, err := exec.Command(xcrunTool, args...).Output() + if err != nil { + macTools.err = fmt.Errorf("xcrun %q failed with: %q", args, err) + return "" + } - for _, sdk := range darwinSupportedSdkVersions { - bytes, err := exec.Command(xcrun, "--sdk", "macosx"+sdk, arg).Output() - if err == nil { return strings.TrimSpace(string(bytes)) } + + xcrunSdk := func(arg string) string { + if selected := ctx.Config().Getenv("MAC_SDK_VERSION"); selected != "" { + if !inList(selected, darwinSupportedSdkVersions) { + macTools.err = fmt.Errorf("MAC_SDK_VERSION %s isn't supported: %q", selected, darwinSupportedSdkVersions) + return "" + } + + return xcrun("--sdk", "macosx"+selected, arg) + } + + for _, sdk := range darwinSupportedSdkVersions { + bytes, err := exec.Command(xcrunTool, "--sdk", "macosx"+sdk, arg).Output() + if err == nil { + return strings.TrimSpace(string(bytes)) + } + } + macTools.err = fmt.Errorf("Could not find a supported mac sdk: %q", darwinSupportedSdkVersions) + return "" + } + + macTools.sdkRoot = xcrunSdk("--show-sdk-path") + + macTools.arPath = xcrun("--find", "ar") + macTools.stripPath = xcrun("--find", "strip") + macTools.toolPath = filepath.Dir(xcrun("--find", "ld")) + }) + if macTools.err != nil { + ctx.Errorf("%q", macTools.err) } - ctx.Errorf("Could not find a supported mac sdk: %q", darwinSupportedSdkVersions) - return "" + return macTools } type toolchainDarwin struct { diff --git a/cc/config/x86_device.go b/cc/config/x86_device.go index 500014e9f..64392dc86 100644 --- a/cc/config/x86_device.go +++ b/cc/config/x86_device.go @@ -82,12 +82,19 @@ var ( } x86ArchFeatureCflags = map[string][]string{ - "ssse3": []string{"-DUSE_SSSE3", "-mssse3"}, + "ssse3": []string{"-mssse3"}, "sse4": []string{"-msse4"}, "sse4_1": []string{"-msse4.1"}, "sse4_2": []string{"-msse4.2"}, - "avx": []string{"-mavx"}, - "avx2": []string{"-mavx2"}, + + // Not all cases there is performance gain by enabling -mavx -mavx2 + // flags so these flags are not enabled by default. + // if there is performance gain in individual library components, + // the compiler flags can be set in corresponding bp files. + // "avx": []string{"-mavx"}, + // "avx2": []string{"-mavx2"}, + // "avx512": []string{"-mavx512"} + "aes_ni": []string{"-maes"}, } ) diff --git a/cc/config/x86_linux_host.go b/cc/config/x86_linux_host.go index f072f348a..13b55112a 100644 --- a/cc/config/x86_linux_host.go +++ b/cc/config/x86_linux_host.go @@ -233,6 +233,14 @@ func (t *toolchainLinuxX8664) YasmFlags() string { return "${config.LinuxX8664YasmFlags}" } +func (toolchainLinuxX86) LibclangRuntimeLibraryArch() string { + return "i386" +} + +func (toolchainLinuxX8664) LibclangRuntimeLibraryArch() string { + return "x86_64" +} + func (t *toolchainLinux) AvailableLibraries() []string { return linuxAvailableLibraries } diff --git a/cc/config/x86_windows_host.go b/cc/config/x86_windows_host.go index 0f500b6fb..cd0a50837 100644 --- a/cc/config/x86_windows_host.go +++ b/cc/config/x86_windows_host.go @@ -58,8 +58,11 @@ var ( "-Wl,--dynamicbase", "-Wl,--nxcompat", } + windowsLldflags = []string{ + "-Wl,--Xlink=-Brepro", // Enable deterministic build + } windowsClangLdflags = append(ClangFilterUnknownCflags(windowsLdflags), []string{}...) - windowsClangLldflags = ClangFilterUnknownLldflags(windowsClangLdflags) + windowsClangLldflags = append(ClangFilterUnknownLldflags(windowsClangLdflags), windowsLldflags...) windowsX86Cflags = []string{ "-m32", @@ -73,6 +76,7 @@ var ( "-m32", "-Wl,--large-address-aware", "-L${WindowsGccRoot}/${WindowsGccTriple}/lib32", + "-static-libgcc", } windowsX86ClangLdflags = append(ClangFilterUnknownCflags(windowsX86Ldflags), []string{ "-B${WindowsGccRoot}/${WindowsGccTriple}/bin", @@ -86,6 +90,7 @@ var ( "-m64", "-L${WindowsGccRoot}/${WindowsGccTriple}/lib64", "-Wl,--high-entropy-va", + "-static-libgcc", } windowsX8664ClangLdflags = append(ClangFilterUnknownCflags(windowsX8664Ldflags), []string{ "-B${WindowsGccRoot}/${WindowsGccTriple}/bin", diff --git a/cc/coverage.go b/cc/coverage.go index 2e81a9e4c..4431757d5 100644 --- a/cc/coverage.go +++ b/cc/coverage.go @@ -43,7 +43,7 @@ func (cov *coverage) props() []interface{} { return []interface{}{&cov.Properties} } -func getProfileLibraryName(ctx ModuleContextIntf) string { +func getGcovProfileLibraryName(ctx ModuleContextIntf) string { // This function should only ever be called for a cc.Module, so the // following statement should always succeed. if ctx.useSdk() { @@ -53,28 +53,47 @@ func getProfileLibraryName(ctx ModuleContextIntf) string { } } +func getClangProfileLibraryName(ctx ModuleContextIntf) string { + if ctx.useSdk() { + return "libprofile-clang-extras_ndk" + } else { + return "libprofile-clang-extras" + } +} + func (cov *coverage) deps(ctx DepsContext, deps Deps) Deps { if cov.Properties.NeedCoverageVariant { ctx.AddVariationDependencies([]blueprint.Variation{ {Mutator: "link", Variation: "static"}, - }, coverageDepTag, getProfileLibraryName(ctx)) + }, coverageDepTag, getGcovProfileLibraryName(ctx)) + ctx.AddVariationDependencies([]blueprint.Variation{ + {Mutator: "link", Variation: "static"}, + }, coverageDepTag, getClangProfileLibraryName(ctx)) } return deps } func (cov *coverage) flags(ctx ModuleContext, flags Flags, deps PathDeps) (Flags, PathDeps) { - if !ctx.DeviceConfig().NativeCoverageEnabled() { + clangCoverage := ctx.DeviceConfig().ClangCoverageEnabled() + gcovCoverage := ctx.DeviceConfig().GcovCoverageEnabled() + + if !gcovCoverage && !clangCoverage { return flags, deps } if cov.Properties.CoverageEnabled { - flags.Coverage = true - flags.GlobalFlags = append(flags.GlobalFlags, "--coverage", "-O0") cov.linkCoverage = true - // Override -Wframe-larger-than and non-default optimization - // flags that the module may use. - flags.CFlags = append(flags.CFlags, "-Wno-frame-larger-than=", "-O0") + if gcovCoverage { + flags.GcovCoverage = true + flags.Local.CommonFlags = append(flags.Local.CommonFlags, "--coverage", "-O0") + + // Override -Wframe-larger-than and non-default optimization + // flags that the module may use. + flags.Local.CFlags = append(flags.Local.CFlags, "-Wno-frame-larger-than=", "-O0") + } else if clangCoverage { + flags.Local.CommonFlags = append(flags.Local.CommonFlags, "-fprofile-instr-generate", "-fcoverage-mapping") + } } // Even if we don't have coverage enabled, if any of our object files were compiled @@ -112,12 +131,19 @@ func (cov *coverage) flags(ctx ModuleContext, flags Flags, deps PathDeps) (Flags } if cov.linkCoverage { - flags.LdFlags = append(flags.LdFlags, "--coverage") + if gcovCoverage { + flags.Local.LdFlags = append(flags.Local.LdFlags, "--coverage") + + coverage := ctx.GetDirectDepWithTag(getGcovProfileLibraryName(ctx), coverageDepTag).(*Module) + deps.WholeStaticLibs = append(deps.WholeStaticLibs, coverage.OutputFile().Path()) - coverage := ctx.GetDirectDepWithTag(getProfileLibraryName(ctx), coverageDepTag).(*Module) - deps.WholeStaticLibs = append(deps.WholeStaticLibs, coverage.OutputFile().Path()) + flags.Local.LdFlags = append(flags.Local.LdFlags, "-Wl,--wrap,getenv") + } else if clangCoverage { + flags.Local.LdFlags = append(flags.Local.LdFlags, "-fprofile-instr-generate") - flags.LdFlags = append(flags.LdFlags, "-Wl,--wrap,getenv") + coverage := ctx.GetDirectDepWithTag(getClangProfileLibraryName(ctx), coverageDepTag).(*Module) + deps.WholeStaticLibs = append(deps.WholeStaticLibs, coverage.OutputFile().Path()) + } } return flags, deps @@ -149,7 +175,7 @@ func (cov *coverage) begin(ctx BaseModuleContext) { if needCoverageVariant { // Coverage variant is actually built with coverage if enabled for its module path - needCoverageBuild = ctx.DeviceConfig().CoverageEnabledForPath(ctx.ModuleDir()) + needCoverageBuild = ctx.DeviceConfig().NativeCoverageEnabledForPath(ctx.ModuleDir()) } } @@ -163,6 +189,7 @@ type Coverage interface { IsNativeCoverageNeeded(ctx android.BaseModuleContext) bool PreventInstall() HideFromMake() + MarkAsCoverageVariant(bool) } func coverageMutator(mctx android.BottomUpMutatorContext) { @@ -191,6 +218,7 @@ func coverageMutator(mctx android.BottomUpMutatorContext) { // module which are split into "" and "cov" variants. e.g. when cc_test refers // to an APEX via 'data' property. m := mctx.CreateVariations("", "cov") + m[0].(Coverage).MarkAsCoverageVariant(true) m[0].(Coverage).PreventInstall() m[0].(Coverage).HideFromMake() } diff --git a/cc/fuzz.go b/cc/fuzz.go new file mode 100644 index 000000000..ee24300c4 --- /dev/null +++ b/cc/fuzz.go @@ -0,0 +1,516 @@ +// Copyright 2016 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cc + +import ( + "encoding/json" + "path/filepath" + "sort" + "strings" + + "android/soong/android" + "android/soong/cc/config" +) + +type FuzzConfig struct { + // Email address of people to CC on bugs or contact about this fuzz target. + Cc []string `json:"cc,omitempty"` + // Specify whether to enable continuous fuzzing on devices. Defaults to true. + Fuzz_on_haiku_device *bool `json:"fuzz_on_haiku_device,omitempty"` + // Specify whether to enable continuous fuzzing on host. Defaults to true. + Fuzz_on_haiku_host *bool `json:"fuzz_on_haiku_host,omitempty"` + // Component in Google's bug tracking system that bugs should be filed to. + Componentid *int64 `json:"componentid,omitempty"` + // Hotlists in Google's bug tracking system that bugs should be marked with. + Hotlists []string `json:"hotlists,omitempty"` +} + +func (f *FuzzConfig) String() string { + b, err := json.Marshal(f) + if err != nil { + panic(err) + } + + return string(b) +} + +type FuzzProperties struct { + // Optional list of seed files to be installed to the fuzz target's output + // directory. + Corpus []string `android:"path"` + // Optional list of data files to be installed to the fuzz target's output + // directory. Directory structure relative to the module is preserved. + Data []string `android:"path"` + // Optional dictionary to be installed to the fuzz target's output directory. + Dictionary *string `android:"path"` + // Config for running the target on fuzzing infrastructure. + Fuzz_config *FuzzConfig +} + +func init() { + android.RegisterModuleType("cc_fuzz", FuzzFactory) + android.RegisterSingletonType("cc_fuzz_packaging", fuzzPackagingFactory) +} + +// cc_fuzz creates a host/device fuzzer binary. Host binaries can be found at +// $ANDROID_HOST_OUT/fuzz/, and device binaries can be found at /data/fuzz on +// your device, or $ANDROID_PRODUCT_OUT/data/fuzz in your build tree. +func FuzzFactory() android.Module { + module := NewFuzz(android.HostAndDeviceSupported) + return module.Init() +} + +func NewFuzzInstaller() *baseInstaller { + return NewBaseInstaller("fuzz", "fuzz", InstallInData) +} + +type fuzzBinary struct { + *binaryDecorator + *baseCompiler + + Properties FuzzProperties + dictionary android.Path + corpus android.Paths + corpusIntermediateDir android.Path + config android.Path + data android.Paths + dataIntermediateDir android.Path + installedSharedDeps []string +} + +func (fuzz *fuzzBinary) linkerProps() []interface{} { + props := fuzz.binaryDecorator.linkerProps() + props = append(props, &fuzz.Properties) + return props +} + +func (fuzz *fuzzBinary) linkerInit(ctx BaseModuleContext) { + fuzz.binaryDecorator.linkerInit(ctx) +} + +func (fuzz *fuzzBinary) linkerDeps(ctx DepsContext, deps Deps) Deps { + deps.StaticLibs = append(deps.StaticLibs, + config.LibFuzzerRuntimeLibrary(ctx.toolchain())) + deps = fuzz.binaryDecorator.linkerDeps(ctx, deps) + return deps +} + +func (fuzz *fuzzBinary) linkerFlags(ctx ModuleContext, flags Flags) Flags { + flags = fuzz.binaryDecorator.linkerFlags(ctx, flags) + // RunPaths on devices isn't instantiated by the base linker. `../lib` for + // installed fuzz targets (both host and device), and `./lib` for fuzz + // target packages. + flags.Local.LdFlags = append(flags.Local.LdFlags, `-Wl,-rpath,\$$ORIGIN/../lib`) + flags.Local.LdFlags = append(flags.Local.LdFlags, `-Wl,-rpath,\$$ORIGIN/lib`) + return flags +} + +// This function performs a breadth-first search over the provided module's +// dependencies using `visitDirectDeps` to enumerate all shared library +// dependencies. We require breadth-first expansion, as otherwise we may +// incorrectly use the core libraries (sanitizer runtimes, libc, libdl, etc.) +// from a dependency. This may cause issues when dependencies have explicit +// sanitizer tags, as we may get a dependency on an unsanitized libc, etc. +func collectAllSharedDependencies(ctx android.SingletonContext, module android.Module) android.Paths { + var fringe []android.Module + + seen := make(map[android.Module]bool) + + // Enumerate the first level of dependencies, as we discard all non-library + // modules in the BFS loop below. + ctx.VisitDirectDeps(module, func(dep android.Module) { + if isValidSharedDependency(dep) { + fringe = append(fringe, dep) + } + }) + + var sharedLibraries android.Paths + + for i := 0; i < len(fringe); i++ { + module := fringe[i] + if seen[module] { + continue + } + seen[module] = true + + ccModule := module.(*Module) + sharedLibraries = append(sharedLibraries, ccModule.UnstrippedOutputFile()) + ctx.VisitDirectDeps(module, func(dep android.Module) { + if isValidSharedDependency(dep) && !seen[dep] { + fringe = append(fringe, dep) + } + }) + } + + return sharedLibraries +} + +// This function takes a module and determines if it is a unique shared library +// that should be installed in the fuzz target output directories. This function +// returns true, unless: +// - The module is not a shared library, or +// - The module is a header, stub, or vendor-linked library. +func isValidSharedDependency(dependency android.Module) bool { + // TODO(b/144090547): We should be parsing these modules using + // ModuleDependencyTag instead of the current brute-force checking. + + if linkable, ok := dependency.(LinkableInterface); !ok || // Discard non-linkables. + !linkable.CcLibraryInterface() || !linkable.Shared() || // Discard static libs. + linkable.UseVndk() || // Discard vendor linked libraries. + // Discard stubs libs (only CCLibrary variants). Prebuilt libraries should not + // be excluded on the basis of they're not CCLibrary()'s. + (linkable.CcLibrary() && linkable.BuildStubs()) { + return false + } + + // We discarded module stubs libraries above, but the LLNDK prebuilts stubs + // libraries must be handled differently - by looking for the stubDecorator. + // Discard LLNDK prebuilts stubs as well. + if ccLibrary, isCcLibrary := dependency.(*Module); isCcLibrary { + if _, isLLndkStubLibrary := ccLibrary.linker.(*stubDecorator); isLLndkStubLibrary { + return false + } + } + + return true +} + +func sharedLibraryInstallLocation( + libraryPath android.Path, isHost bool, archString string) string { + installLocation := "$(PRODUCT_OUT)/data" + if isHost { + installLocation = "$(HOST_OUT)" + } + installLocation = filepath.Join( + installLocation, "fuzz", archString, "lib", libraryPath.Base()) + return installLocation +} + +func (fuzz *fuzzBinary) install(ctx ModuleContext, file android.Path) { + fuzz.binaryDecorator.baseInstaller.dir = filepath.Join( + "fuzz", ctx.Target().Arch.ArchType.String(), ctx.ModuleName()) + fuzz.binaryDecorator.baseInstaller.dir64 = filepath.Join( + "fuzz", ctx.Target().Arch.ArchType.String(), ctx.ModuleName()) + fuzz.binaryDecorator.baseInstaller.install(ctx, file) + + fuzz.corpus = android.PathsForModuleSrc(ctx, fuzz.Properties.Corpus) + builder := android.NewRuleBuilder() + intermediateDir := android.PathForModuleOut(ctx, "corpus") + for _, entry := range fuzz.corpus { + builder.Command().Text("cp"). + Input(entry). + Output(intermediateDir.Join(ctx, entry.Base())) + } + builder.Build(pctx, ctx, "copy_corpus", "copy corpus") + fuzz.corpusIntermediateDir = intermediateDir + + fuzz.data = android.PathsForModuleSrc(ctx, fuzz.Properties.Data) + builder = android.NewRuleBuilder() + intermediateDir = android.PathForModuleOut(ctx, "data") + for _, entry := range fuzz.data { + builder.Command().Text("cp"). + Input(entry). + Output(intermediateDir.Join(ctx, entry.Rel())) + } + builder.Build(pctx, ctx, "copy_data", "copy data") + fuzz.dataIntermediateDir = intermediateDir + + if fuzz.Properties.Dictionary != nil { + fuzz.dictionary = android.PathForModuleSrc(ctx, *fuzz.Properties.Dictionary) + if fuzz.dictionary.Ext() != ".dict" { + ctx.PropertyErrorf("dictionary", + "Fuzzer dictionary %q does not have '.dict' extension", + fuzz.dictionary.String()) + } + } + + if fuzz.Properties.Fuzz_config != nil { + configPath := android.PathForModuleOut(ctx, "config").Join(ctx, "config.json") + ctx.Build(pctx, android.BuildParams{ + Rule: android.WriteFile, + Description: "fuzzer infrastructure configuration", + Output: configPath, + Args: map[string]string{ + "content": fuzz.Properties.Fuzz_config.String(), + }, + }) + fuzz.config = configPath + } + + // Grab the list of required shared libraries. + seen := make(map[android.Module]bool) + var sharedLibraries android.Paths + ctx.WalkDeps(func(child, parent android.Module) bool { + if seen[child] { + return false + } + seen[child] = true + + if isValidSharedDependency(child) { + sharedLibraries = append(sharedLibraries, child.(*Module).UnstrippedOutputFile()) + return true + } + return false + }) + + for _, lib := range sharedLibraries { + fuzz.installedSharedDeps = append(fuzz.installedSharedDeps, + sharedLibraryInstallLocation( + lib, ctx.Host(), ctx.Arch().ArchType.String())) + } +} + +func NewFuzz(hod android.HostOrDeviceSupported) *Module { + module, binary := NewBinary(hod) + + binary.baseInstaller = NewFuzzInstaller() + module.sanitize.SetSanitizer(fuzzer, true) + + fuzz := &fuzzBinary{ + binaryDecorator: binary, + baseCompiler: NewBaseCompiler(), + } + module.compiler = fuzz + module.linker = fuzz + module.installer = fuzz + + // The fuzzer runtime is not present for darwin host modules, disable cc_fuzz modules when targeting darwin. + android.AddLoadHook(module, func(ctx android.LoadHookContext) { + disableDarwinAndLinuxBionic := struct { + Target struct { + Darwin struct { + Enabled *bool + } + Linux_bionic struct { + Enabled *bool + } + } + }{} + disableDarwinAndLinuxBionic.Target.Darwin.Enabled = BoolPtr(false) + disableDarwinAndLinuxBionic.Target.Linux_bionic.Enabled = BoolPtr(false) + ctx.AppendProperties(&disableDarwinAndLinuxBionic) + }) + + return module +} + +// Responsible for generating GNU Make rules that package fuzz targets into +// their architecture & target/host specific zip file. +type fuzzPackager struct { + packages android.Paths + sharedLibInstallStrings []string + fuzzTargets map[string]bool +} + +func fuzzPackagingFactory() android.Singleton { + return &fuzzPackager{} +} + +type fileToZip struct { + SourceFilePath android.Path + DestinationPathPrefix string +} + +type archOs struct { + hostOrTarget string + arch string + dir string +} + +func (s *fuzzPackager) GenerateBuildActions(ctx android.SingletonContext) { + // Map between each architecture + host/device combination, and the files that + // need to be packaged (in the tuple of {source file, destination folder in + // archive}). + archDirs := make(map[archOs][]fileToZip) + + // Map tracking whether each shared library has an install rule to avoid duplicate install rules from + // multiple fuzzers that depend on the same shared library. + sharedLibraryInstalled := make(map[string]bool) + + // List of individual fuzz targets, so that 'make fuzz' also installs the targets + // to the correct output directories as well. + s.fuzzTargets = make(map[string]bool) + + ctx.VisitAllModules(func(module android.Module) { + // Discard non-fuzz targets. + ccModule, ok := module.(*Module) + if !ok { + return + } + + fuzzModule, ok := ccModule.compiler.(*fuzzBinary) + if !ok { + return + } + + // Discard vendor-NDK-linked + ramdisk + recovery modules, they're duplicates of + // fuzz targets we're going to package anyway. + if !ccModule.Enabled() || ccModule.Properties.PreventInstall || + ccModule.UseVndk() || ccModule.InRamdisk() || ccModule.InRecovery() { + return + } + + // Discard modules that are in an unavailable namespace. + if !ccModule.ExportedToMake() { + return + } + + s.fuzzTargets[module.Name()] = true + + hostOrTargetString := "target" + if ccModule.Host() { + hostOrTargetString = "host" + } + + archString := ccModule.Arch().ArchType.String() + archDir := android.PathForIntermediates(ctx, "fuzz", hostOrTargetString, archString) + archOs := archOs{hostOrTarget: hostOrTargetString, arch: archString, dir: archDir.String()} + + // Grab the list of required shared libraries. + sharedLibraries := collectAllSharedDependencies(ctx, module) + + var files []fileToZip + builder := android.NewRuleBuilder() + + // Package the corpora into a zipfile. + if fuzzModule.corpus != nil { + corpusZip := archDir.Join(ctx, module.Name()+"_seed_corpus.zip") + command := builder.Command().BuiltTool(ctx, "soong_zip"). + Flag("-j"). + FlagWithOutput("-o ", corpusZip) + command.FlagWithRspFileInputList("-l ", fuzzModule.corpus) + files = append(files, fileToZip{corpusZip, ""}) + } + + // Package the data into a zipfile. + if fuzzModule.data != nil { + dataZip := archDir.Join(ctx, module.Name()+"_data.zip") + command := builder.Command().BuiltTool(ctx, "soong_zip"). + FlagWithOutput("-o ", dataZip) + for _, f := range fuzzModule.data { + intermediateDir := strings.TrimSuffix(f.String(), f.Rel()) + command.FlagWithArg("-C ", intermediateDir) + command.FlagWithInput("-f ", f) + } + files = append(files, fileToZip{dataZip, ""}) + } + + // Find and mark all the transiently-dependent shared libraries for + // packaging. + for _, library := range sharedLibraries { + files = append(files, fileToZip{library, "lib"}) + + // For each architecture-specific shared library dependency, we need to + // install it to the output directory. Setup the install destination here, + // which will be used by $(copy-many-files) in the Make backend. + installDestination := sharedLibraryInstallLocation( + library, ccModule.Host(), archString) + if sharedLibraryInstalled[installDestination] { + continue + } + sharedLibraryInstalled[installDestination] = true + // Escape all the variables, as the install destination here will be called + // via. $(eval) in Make. + installDestination = strings.ReplaceAll( + installDestination, "$", "$$") + s.sharedLibInstallStrings = append(s.sharedLibInstallStrings, + library.String()+":"+installDestination) + } + + // The executable. + files = append(files, fileToZip{ccModule.UnstrippedOutputFile(), ""}) + + // The dictionary. + if fuzzModule.dictionary != nil { + files = append(files, fileToZip{fuzzModule.dictionary, ""}) + } + + // Additional fuzz config. + if fuzzModule.config != nil { + files = append(files, fileToZip{fuzzModule.config, ""}) + } + + fuzzZip := archDir.Join(ctx, module.Name()+".zip") + command := builder.Command().BuiltTool(ctx, "soong_zip"). + Flag("-j"). + FlagWithOutput("-o ", fuzzZip) + for _, file := range files { + if file.DestinationPathPrefix != "" { + command.FlagWithArg("-P ", file.DestinationPathPrefix) + } else { + command.Flag("-P ''") + } + command.FlagWithInput("-f ", file.SourceFilePath) + } + + builder.Build(pctx, ctx, "create-"+fuzzZip.String(), + "Package "+module.Name()+" for "+archString+"-"+hostOrTargetString) + + archDirs[archOs] = append(archDirs[archOs], fileToZip{fuzzZip, ""}) + }) + + var archOsList []archOs + for archOs := range archDirs { + archOsList = append(archOsList, archOs) + } + sort.Slice(archOsList, func(i, j int) bool { return archOsList[i].dir < archOsList[j].dir }) + + for _, archOs := range archOsList { + filesToZip := archDirs[archOs] + arch := archOs.arch + hostOrTarget := archOs.hostOrTarget + builder := android.NewRuleBuilder() + outputFile := android.PathForOutput(ctx, "fuzz-"+hostOrTarget+"-"+arch+".zip") + s.packages = append(s.packages, outputFile) + + command := builder.Command().BuiltTool(ctx, "soong_zip"). + Flag("-j"). + FlagWithOutput("-o ", outputFile). + Flag("-L 0") // No need to try and re-compress the zipfiles. + + for _, fileToZip := range filesToZip { + if fileToZip.DestinationPathPrefix != "" { + command.FlagWithArg("-P ", fileToZip.DestinationPathPrefix) + } else { + command.Flag("-P ''") + } + command.FlagWithInput("-f ", fileToZip.SourceFilePath) + } + + builder.Build(pctx, ctx, "create-fuzz-package-"+arch+"-"+hostOrTarget, + "Create fuzz target packages for "+arch+"-"+hostOrTarget) + } +} + +func (s *fuzzPackager) MakeVars(ctx android.MakeVarsContext) { + packages := s.packages.Strings() + sort.Strings(packages) + sort.Strings(s.sharedLibInstallStrings) + // TODO(mitchp): Migrate this to use MakeVarsContext::DistForGoal() when it's + // ready to handle phony targets created in Soong. In the meantime, this + // exports the phony 'fuzz' target and dependencies on packages to + // core/main.mk so that we can use dist-for-goals. + ctx.Strict("SOONG_FUZZ_PACKAGING_ARCH_MODULES", strings.Join(packages, " ")) + ctx.Strict("FUZZ_TARGET_SHARED_DEPS_INSTALL_PAIRS", + strings.Join(s.sharedLibInstallStrings, " ")) + + // Preallocate the slice of fuzz targets to minimise memory allocations. + fuzzTargets := make([]string, 0, len(s.fuzzTargets)) + for target, _ := range s.fuzzTargets { + fuzzTargets = append(fuzzTargets, target) + } + sort.Strings(fuzzTargets) + ctx.Strict("ALL_FUZZ_TARGETS", strings.Join(fuzzTargets, " ")) +} @@ -16,6 +16,7 @@ package cc import ( "path/filepath" + "strings" "github.com/google/blueprint" @@ -24,43 +25,26 @@ import ( func init() { pctx.SourcePathVariable("lexCmd", "prebuilts/build-tools/${config.HostPrebuiltTag}/bin/flex") - pctx.SourcePathVariable("yaccCmd", "prebuilts/build-tools/${config.HostPrebuiltTag}/bin/bison") - pctx.SourcePathVariable("yaccDataDir", "prebuilts/build-tools/common/bison") + pctx.SourcePathVariable("m4Cmd", "prebuilts/build-tools/${config.HostPrebuiltTag}/bin/m4") pctx.HostBinToolVariable("aidlCmd", "aidl-cpp") pctx.HostBinToolVariable("syspropCmd", "sysprop_cpp") } var ( - yacc = pctx.AndroidStaticRule("yacc", - blueprint.RuleParams{ - Command: "BISON_PKGDATADIR=$yaccDataDir $yaccCmd -d $yaccFlags --defines=$hFile -o $out $in", - CommandDeps: []string{"$yaccCmd"}, - }, - "yaccFlags", "hFile") - lex = pctx.AndroidStaticRule("lex", blueprint.RuleParams{ - Command: "$lexCmd -o$out $in", - CommandDeps: []string{"$lexCmd"}, + Command: "M4=$m4Cmd $lexCmd -o$out $in", + CommandDeps: []string{"$lexCmd", "$m4Cmd"}, }) - aidl = pctx.AndroidStaticRule("aidl", - blueprint.RuleParams{ - Command: "$aidlCmd -d${out}.d --ninja $aidlFlags $in $outDir $out", - CommandDeps: []string{"$aidlCmd"}, - Depfile: "${out}.d", - Deps: blueprint.DepsGCC, - }, - "aidlFlags", "outDir") - sysprop = pctx.AndroidStaticRule("sysprop", blueprint.RuleParams{ - Command: "$syspropCmd --header-dir=$headerOutDir --system-header-dir=$systemOutDir " + + Command: "$syspropCmd --header-dir=$headerOutDir --public-header-dir=$publicOutDir " + "--source-dir=$srcOutDir --include-name=$includeName $in", CommandDeps: []string{"$syspropCmd"}, }, - "headerOutDir", "systemOutDir", "srcOutDir", "includeName") + "headerOutDir", "publicOutDir", "srcOutDir", "includeName") windmc = pctx.AndroidStaticRule("windmc", blueprint.RuleParams{ @@ -70,38 +54,103 @@ var ( "windmcCmd") ) -func genYacc(ctx android.ModuleContext, yaccFile android.Path, outFile android.ModuleGenPath, yaccFlags string) (headerFile android.ModuleGenPath) { - headerFile = android.GenPathWithExt(ctx, "yacc", yaccFile, "h") +type YaccProperties struct { + // list of module-specific flags that will be used for .y and .yy compiles + Flags []string - ctx.Build(pctx, android.BuildParams{ - Rule: yacc, - Description: "yacc " + yaccFile.Rel(), - Output: outFile, - ImplicitOutput: headerFile, - Input: yaccFile, - Args: map[string]string{ - "yaccFlags": yaccFlags, - "hFile": headerFile.String(), - }, - }) + // whether the yacc files will produce a location.hh file + Gen_location_hh *bool - return headerFile + // whether the yacc files will product a position.hh file + Gen_position_hh *bool } -func genAidl(ctx android.ModuleContext, aidlFile android.Path, outFile android.ModuleGenPath, aidlFlags string) android.Paths { - ctx.Build(pctx, android.BuildParams{ - Rule: aidl, - Description: "aidl " + aidlFile.Rel(), - Output: outFile, - Input: aidlFile, - Args: map[string]string{ - "aidlFlags": aidlFlags, - "outDir": android.PathForModuleGen(ctx, "aidl").String(), - }, - }) +func genYacc(ctx android.ModuleContext, rule *android.RuleBuilder, yaccFile android.Path, + outFile android.ModuleGenPath, props *YaccProperties) (headerFiles android.Paths) { + + outDir := android.PathForModuleGen(ctx, "yacc") + headerFile := android.GenPathWithExt(ctx, "yacc", yaccFile, "h") + ret := android.Paths{headerFile} + + cmd := rule.Command() + + // Fix up #line markers to not use the sbox temporary directory + sedCmd := "sed -i.bak 's#__SBOX_OUT_DIR__#" + outDir.String() + "#'" + rule.Command().Text(sedCmd).Input(outFile) + rule.Command().Text(sedCmd).Input(headerFile) + + var flags []string + if props != nil { + flags = props.Flags + + if Bool(props.Gen_location_hh) { + locationHeader := outFile.InSameDir(ctx, "location.hh") + ret = append(ret, locationHeader) + cmd.ImplicitOutput(locationHeader) + rule.Command().Text(sedCmd).Input(locationHeader) + } + if Bool(props.Gen_position_hh) { + positionHeader := outFile.InSameDir(ctx, "position.hh") + ret = append(ret, positionHeader) + cmd.ImplicitOutput(positionHeader) + rule.Command().Text(sedCmd).Input(positionHeader) + } + } - // TODO: This should return the generated headers, not the source file. - return android.Paths{outFile} + cmd.Text("BISON_PKGDATADIR=prebuilts/build-tools/common/bison"). + FlagWithInput("M4=", ctx.Config().PrebuiltBuildTool(ctx, "m4")). + PrebuiltBuildTool(ctx, "bison"). + Flag("-d"). + Flags(flags). + FlagWithOutput("--defines=", headerFile). + Flag("-o").Output(outFile).Input(yaccFile) + + return ret +} + +func genAidl(ctx android.ModuleContext, rule *android.RuleBuilder, aidlFile android.Path, + outFile, depFile android.ModuleGenPath, aidlFlags string) android.Paths { + + aidlPackage := strings.TrimSuffix(aidlFile.Rel(), aidlFile.Base()) + baseName := strings.TrimSuffix(aidlFile.Base(), aidlFile.Ext()) + shortName := baseName + // TODO(b/111362593): aidl_to_cpp_common.cpp uses heuristics to figure out if + // an interface name has a leading I. Those same heuristics have been + // moved here. + if len(baseName) >= 2 && baseName[0] == 'I' && + strings.ToUpper(baseName)[1] == baseName[1] { + shortName = strings.TrimPrefix(baseName, "I") + } + + outDir := android.PathForModuleGen(ctx, "aidl") + headerI := outDir.Join(ctx, aidlPackage, baseName+".h") + headerBn := outDir.Join(ctx, aidlPackage, "Bn"+shortName+".h") + headerBp := outDir.Join(ctx, aidlPackage, "Bp"+shortName+".h") + + baseDir := strings.TrimSuffix(aidlFile.String(), aidlFile.Rel()) + if baseDir != "" { + aidlFlags += " -I" + baseDir + } + + cmd := rule.Command() + cmd.BuiltTool(ctx, "aidl-cpp"). + FlagWithDepFile("-d", depFile). + Flag("--ninja"). + Flag(aidlFlags). + Input(aidlFile). + OutputDir(). + Output(outFile). + ImplicitOutputs(android.WritablePaths{ + headerI, + headerBn, + headerBp, + }) + + return android.Paths{ + headerI, + headerBn, + headerBp, + } } func genLex(ctx android.ModuleContext, lexFile android.Path, outFile android.ModuleGenPath) { @@ -113,26 +162,28 @@ func genLex(ctx android.ModuleContext, lexFile android.Path, outFile android.Mod }) } -func genSysprop(ctx android.ModuleContext, syspropFile android.Path) (android.Path, android.Path) { +func genSysprop(ctx android.ModuleContext, syspropFile android.Path) (android.Path, android.Paths) { headerFile := android.PathForModuleGen(ctx, "sysprop", "include", syspropFile.Rel()+".h") - systemHeaderFile := android.PathForModuleGen(ctx, "sysprop/system", "include", syspropFile.Rel()+".h") + publicHeaderFile := android.PathForModuleGen(ctx, "sysprop/public", "include", syspropFile.Rel()+".h") cppFile := android.PathForModuleGen(ctx, "sysprop", syspropFile.Rel()+".cpp") + headers := android.WritablePaths{headerFile, publicHeaderFile} + ctx.Build(pctx, android.BuildParams{ - Rule: sysprop, - Description: "sysprop " + syspropFile.Rel(), - Output: cppFile, - ImplicitOutput: headerFile, - Input: syspropFile, + Rule: sysprop, + Description: "sysprop " + syspropFile.Rel(), + Output: cppFile, + ImplicitOutputs: headers, + Input: syspropFile, Args: map[string]string{ "headerOutDir": filepath.Dir(headerFile.String()), - "systemOutDir": filepath.Dir(systemHeaderFile.String()), + "publicOutDir": filepath.Dir(publicHeaderFile.String()), "srcOutDir": filepath.Dir(cppFile.String()), "includeName": syspropFile.Rel() + ".h", }, }) - return cppFile, headerFile + return cppFile, headers.Paths() } func genWinMsg(ctx android.ModuleContext, srcFile android.Path, flags builderFlags) (android.Path, android.Path) { @@ -159,19 +210,28 @@ func genSources(ctx android.ModuleContext, srcFiles android.Paths, buildFlags builderFlags) (android.Paths, android.Paths) { var deps android.Paths - var rsFiles android.Paths + var aidlRule *android.RuleBuilder + + var yaccRule_ *android.RuleBuilder + yaccRule := func() *android.RuleBuilder { + if yaccRule_ == nil { + yaccRule_ = android.NewRuleBuilder().Sbox(android.PathForModuleGen(ctx, "yacc")) + } + return yaccRule_ + } + for i, srcFile := range srcFiles { switch srcFile.Ext() { case ".y": cFile := android.GenPathWithExt(ctx, "yacc", srcFile, "c") srcFiles[i] = cFile - deps = append(deps, genYacc(ctx, srcFile, cFile, buildFlags.yaccFlags)) + deps = append(deps, genYacc(ctx, yaccRule(), srcFile, cFile, buildFlags.yacc)...) case ".yy": cppFile := android.GenPathWithExt(ctx, "yacc", srcFile, "cpp") srcFiles[i] = cppFile - deps = append(deps, genYacc(ctx, srcFile, cppFile, buildFlags.yaccFlags)) + deps = append(deps, genYacc(ctx, yaccRule(), srcFile, cppFile, buildFlags.yacc)...) case ".l": cFile := android.GenPathWithExt(ctx, "lex", srcFile, "c") srcFiles[i] = cFile @@ -185,10 +245,14 @@ func genSources(ctx android.ModuleContext, srcFiles android.Paths, srcFiles[i] = ccFile deps = append(deps, headerFile) case ".aidl": + if aidlRule == nil { + aidlRule = android.NewRuleBuilder().Sbox(android.PathForModuleGen(ctx, "aidl")) + } cppFile := android.GenPathWithExt(ctx, "aidl", srcFile, "cpp") + depFile := android.GenPathWithExt(ctx, "aidl", srcFile, "cpp.d") srcFiles[i] = cppFile - deps = append(deps, genAidl(ctx, srcFile, cppFile, buildFlags.aidlFlags)...) - case ".rs", ".fs": + deps = append(deps, genAidl(ctx, aidlRule, srcFile, cppFile, depFile, buildFlags.aidlFlags)...) + case ".rscript", ".fs": cppFile := rsGeneratedCppFile(ctx, srcFile) rsFiles = append(rsFiles, srcFiles[i]) srcFiles[i] = cppFile @@ -197,12 +261,20 @@ func genSources(ctx android.ModuleContext, srcFiles android.Paths, srcFiles[i] = rcFile deps = append(deps, headerFile) case ".sysprop": - cppFile, headerFile := genSysprop(ctx, srcFile) + cppFile, headerFiles := genSysprop(ctx, srcFile) srcFiles[i] = cppFile - deps = append(deps, headerFile) + deps = append(deps, headerFiles...) } } + if aidlRule != nil { + aidlRule.Build(pctx, ctx, "aidl", "gen aidl") + } + + if yaccRule_ != nil { + yaccRule_.Build(pctx, ctx, "yacc", "gen yacc") + } + if len(rsFiles) > 0 { deps = append(deps, rsGenerateCpp(ctx, rsFiles, buildFlags.rsFlags)...) } diff --git a/cc/gen_stub_libs.py b/cc/gen_stub_libs.py index 81bc398a0..0de703ce1 100755 --- a/cc/gen_stub_libs.py +++ b/cc/gen_stub_libs.py @@ -108,7 +108,7 @@ def version_is_private(version): return version.endswith('_PRIVATE') or version.endswith('_PLATFORM') -def should_omit_version(version, arch, api, vndk, apex): +def should_omit_version(version, arch, api, llndk, apex): """Returns True if the version section should be ommitted. We want to omit any sections that do not have any symbols we'll have in the @@ -120,9 +120,9 @@ def should_omit_version(version, arch, api, vndk, apex): if 'platform-only' in version.tags: return True - no_vndk_no_apex = 'vndk' not in version.tags and 'apex' not in version.tags - keep = no_vndk_no_apex or \ - ('vndk' in version.tags and vndk) or \ + no_llndk_no_apex = 'llndk' not in version.tags and 'apex' not in version.tags + keep = no_llndk_no_apex or \ + ('llndk' in version.tags and llndk) or \ ('apex' in version.tags and apex) if not keep: return True @@ -133,11 +133,11 @@ def should_omit_version(version, arch, api, vndk, apex): return False -def should_omit_symbol(symbol, arch, api, vndk, apex): +def should_omit_symbol(symbol, arch, api, llndk, apex): """Returns True if the symbol should be omitted.""" - no_vndk_no_apex = 'vndk' not in symbol.tags and 'apex' not in symbol.tags - keep = no_vndk_no_apex or \ - ('vndk' in symbol.tags and vndk) or \ + no_llndk_no_apex = 'llndk' not in symbol.tags and 'apex' not in symbol.tags + keep = no_llndk_no_apex or \ + ('llndk' in symbol.tags and llndk) or \ ('apex' in symbol.tags and apex) if not keep: return True @@ -250,12 +250,12 @@ class Symbol(object): class SymbolFileParser(object): """Parses NDK symbol files.""" - def __init__(self, input_file, api_map, arch, api, vndk, apex): + def __init__(self, input_file, api_map, arch, api, llndk, apex): self.input_file = input_file self.api_map = api_map self.arch = arch self.api = api - self.vndk = vndk + self.llndk = llndk self.apex = apex self.current_line = None @@ -284,11 +284,11 @@ class SymbolFileParser(object): symbol_names = set() multiply_defined_symbols = set() for version in versions: - if should_omit_version(version, self.arch, self.api, self.vndk, self.apex): + if should_omit_version(version, self.arch, self.api, self.llndk, self.apex): continue for symbol in version.symbols: - if should_omit_symbol(symbol, self.arch, self.api, self.vndk, self.apex): + if should_omit_symbol(symbol, self.arch, self.api, self.llndk, self.apex): continue if symbol.name in symbol_names: @@ -372,12 +372,12 @@ class SymbolFileParser(object): class Generator(object): """Output generator that writes stub source files and version scripts.""" - def __init__(self, src_file, version_script, arch, api, vndk, apex): + def __init__(self, src_file, version_script, arch, api, llndk, apex): self.src_file = src_file self.version_script = version_script self.arch = arch self.api = api - self.vndk = vndk + self.llndk = llndk self.apex = apex def write(self, versions): @@ -387,14 +387,14 @@ class Generator(object): def write_version(self, version): """Writes a single version block's data to the output files.""" - if should_omit_version(version, self.arch, self.api, self.vndk, self.apex): + if should_omit_version(version, self.arch, self.api, self.llndk, self.apex): return section_versioned = symbol_versioned_in_api(version.tags, self.api) version_empty = True pruned_symbols = [] for symbol in version.symbols: - if should_omit_symbol(symbol, self.arch, self.api, self.vndk, self.apex): + if should_omit_symbol(symbol, self.arch, self.api, self.llndk, self.apex): continue if symbol_versioned_in_api(symbol.tags, self.api): @@ -456,7 +456,7 @@ def parse_args(): '--arch', choices=ALL_ARCHITECTURES, required=True, help='Architecture being targeted.') parser.add_argument( - '--vndk', action='store_true', help='Use the VNDK variant.') + '--llndk', action='store_true', help='Use the LLNDK variant.') parser.add_argument( '--apex', action='store_true', help='Use the APEX variant.') @@ -493,14 +493,14 @@ def main(): with open(args.symbol_file) as symbol_file: try: versions = SymbolFileParser(symbol_file, api_map, args.arch, api, - args.vndk, args.apex).parse() + args.llndk, args.apex).parse() except MultiplyDefinedSymbolError as ex: sys.exit('{}: error: {}'.format(args.symbol_file, ex)) with open(args.stub_src, 'w') as src_file: with open(args.version_script, 'w') as version_file: generator = Generator(src_file, version_file, args.arch, api, - args.vndk, args.apex) + args.llndk, args.apex) generator.write(versions) diff --git a/cc/gen_test.go b/cc/gen_test.go index a0f7308c2..4b9a36e6b 100644 --- a/cc/gen_test.go +++ b/cc/gen_test.go @@ -15,6 +15,8 @@ package cc import ( + "path/filepath" + "strings" "testing" ) @@ -29,10 +31,10 @@ func TestGen(t *testing.T) { ], }`) - aidl := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_shared").Rule("aidl") - libfoo := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_shared").Module().(*Module) + aidl := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_shared").Rule("aidl") + libfoo := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_shared").Module().(*Module) - if !inList("-I"+aidl.Args["outDir"], libfoo.flags.GlobalFlags) { + if !inList("-I"+filepath.Dir(aidl.Output.String()), libfoo.flags.Local.CommonFlags) { t.Errorf("missing aidl includes in global flags") } }) @@ -41,7 +43,8 @@ func TestGen(t *testing.T) { ctx := testCc(t, ` filegroup { name: "fg", - srcs: ["b.aidl"], + srcs: ["sub/c.aidl"], + path: "sub", } cc_library_shared { @@ -52,12 +55,18 @@ func TestGen(t *testing.T) { ], }`) - aidl := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_shared").Rule("aidl") - libfoo := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_shared").Module().(*Module) + aidl := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_shared").Rule("aidl") + libfoo := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_shared").Module().(*Module) - if !inList("-I"+aidl.Args["outDir"], libfoo.flags.GlobalFlags) { + if !inList("-I"+filepath.Dir(aidl.Output.String()), libfoo.flags.Local.CommonFlags) { t.Errorf("missing aidl includes in global flags") } + + aidlCommand := aidl.RuleParams.Command + if !strings.Contains(aidlCommand, "-Isub") { + t.Errorf("aidl command for c.aidl should contain \"-Isub\", but was %q", aidlCommand) + } + }) } diff --git a/cc/genrule.go b/cc/genrule.go index decf6ea57..66d178456 100644 --- a/cc/genrule.go +++ b/cc/genrule.go @@ -25,10 +25,9 @@ func init() { type GenruleExtraProperties struct { Vendor_available *bool + Ramdisk_available *bool Recovery_available *bool - - // This genrule is for recovery variant - InRecovery bool `blueprint:"mutated"` + Sdk_version *string } // cc_genrule is a genrule that can depend on other cc_* objects. @@ -37,10 +36,74 @@ type GenruleExtraProperties struct { func genRuleFactory() android.Module { module := genrule.NewGenRule() - module.Extra = &GenruleExtraProperties{} + extra := &GenruleExtraProperties{} + module.Extra = extra + module.ImageInterface = extra module.AddProperties(module.Extra) android.InitAndroidArchModule(module, android.HostAndDeviceSupported, android.MultilibBoth) + android.InitApexModule(module) + return module } + +var _ android.ImageInterface = (*GenruleExtraProperties)(nil) + +func (g *GenruleExtraProperties) ImageMutatorBegin(ctx android.BaseModuleContext) {} + +func (g *GenruleExtraProperties) CoreVariantNeeded(ctx android.BaseModuleContext) bool { + if ctx.DeviceConfig().VndkVersion() == "" { + return true + } + + if ctx.DeviceConfig().ProductVndkVersion() != "" && ctx.ProductSpecific() { + return false + } + + return Bool(g.Vendor_available) || !(ctx.SocSpecific() || ctx.DeviceSpecific()) +} + +func (g *GenruleExtraProperties) RamdiskVariantNeeded(ctx android.BaseModuleContext) bool { + return Bool(g.Ramdisk_available) +} + +func (g *GenruleExtraProperties) RecoveryVariantNeeded(ctx android.BaseModuleContext) bool { + return Bool(g.Recovery_available) +} + +func (g *GenruleExtraProperties) ExtraImageVariations(ctx android.BaseModuleContext) []string { + if ctx.DeviceConfig().VndkVersion() == "" { + return nil + } + + var variants []string + if Bool(g.Vendor_available) || ctx.SocSpecific() || ctx.DeviceSpecific() { + vndkVersion := ctx.DeviceConfig().VndkVersion() + // If vndkVersion is current, we can always use PlatformVndkVersion. + // If not, we assume modules under proprietary paths are compatible for + // BOARD_VNDK_VERSION. The other modules are regarded as AOSP, that is + // PLATFORM_VNDK_VERSION. + if vndkVersion == "current" || !isVendorProprietaryPath(ctx.ModuleDir()) { + variants = append(variants, VendorVariationPrefix+ctx.DeviceConfig().PlatformVndkVersion()) + } else { + variants = append(variants, VendorVariationPrefix+vndkVersion) + } + } + + if ctx.DeviceConfig().ProductVndkVersion() == "" { + return variants + } + + if Bool(g.Vendor_available) || ctx.ProductSpecific() { + variants = append(variants, ProductVariationPrefix+ctx.DeviceConfig().PlatformVndkVersion()) + if vndkVersion := ctx.DeviceConfig().ProductVndkVersion(); vndkVersion != "current" { + variants = append(variants, ProductVariationPrefix+vndkVersion) + } + } + + return variants +} + +func (g *GenruleExtraProperties) SetImageVariation(ctx android.BaseModuleContext, variation string, module android.Module) { +} diff --git a/cc/genrule_test.go b/cc/genrule_test.go index 92024ac14..d38cf27f0 100644 --- a/cc/genrule_test.go +++ b/cc/genrule_test.go @@ -21,31 +21,20 @@ import ( "android/soong/android" ) -func testGenruleContext(config android.Config, bp string, - fs map[string][]byte) *android.TestContext { - +func testGenruleContext(config android.Config) *android.TestContext { ctx := android.NewTestArchContext() - ctx.RegisterModuleType("cc_genrule", android.ModuleFactoryAdaptor(genRuleFactory)) - ctx.Register() - - mockFS := map[string][]byte{ - "Android.bp": []byte(bp), - "tool": nil, - "foo": nil, - "bar": nil, - } - - for k, v := range fs { - mockFS[k] = v - } - - ctx.MockFileSystem(mockFS) + ctx.RegisterModuleType("cc_genrule", genRuleFactory) + ctx.Register(config) return ctx } func TestArchGenruleCmd(t *testing.T) { - config := android.TestArchConfig(buildDir, nil) + fs := map[string][]byte{ + "tool": nil, + "foo": nil, + "bar": nil, + } bp := ` cc_genrule { name: "gen", @@ -63,8 +52,9 @@ func TestArchGenruleCmd(t *testing.T) { }, } ` + config := android.TestArchConfig(buildDir, nil, bp, fs) - ctx := testGenruleContext(config, bp, nil) + ctx := testGenruleContext(config) _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) if errs == nil { diff --git a/cc/installer.go b/cc/installer.go index bd8f9e791..0b4a68cc8 100644 --- a/cc/installer.go +++ b/cc/installer.go @@ -52,7 +52,7 @@ type baseInstaller struct { relative string location installLocation - path android.OutputPath + path android.InstallPath } var _ installer = (*baseInstaller)(nil) @@ -61,16 +61,22 @@ func (installer *baseInstaller) installerProps() []interface{} { return []interface{}{&installer.Properties} } -func (installer *baseInstaller) installDir(ctx ModuleContext) android.OutputPath { +func (installer *baseInstaller) installDir(ctx ModuleContext) android.InstallPath { dir := installer.dir if ctx.toolchain().Is64Bit() && installer.dir64 != "" { dir = installer.dir64 } - if !ctx.Host() && !ctx.Arch().Native { + if ctx.Target().NativeBridge == android.NativeBridgeEnabled { + dir = filepath.Join(dir, ctx.Target().NativeBridgeRelativePath) + } else if !ctx.Host() && ctx.Config().HasMultilibConflict(ctx.Arch().ArchType) { dir = filepath.Join(dir, ctx.Arch().ArchType.String()) } if installer.location == InstallInData && ctx.useVndk() { - dir = filepath.Join(dir, "vendor") + if ctx.inProduct() { + dir = filepath.Join(dir, "product") + } else { + dir = filepath.Join(dir, "vendor") + } } return android.PathForModuleInstall(ctx, dir, installer.subDir, installer.relativeInstallPath(), installer.relative) @@ -80,6 +86,11 @@ func (installer *baseInstaller) install(ctx ModuleContext, file android.Path) { installer.path = ctx.InstallFile(installer.installDir(ctx), file.Base(), file) } +func (installer *baseInstaller) everInstallable() bool { + // Most cc modules are installable. + return true +} + func (installer *baseInstaller) inData() bool { return installer.location == InstallInData } @@ -95,3 +106,7 @@ func (installer *baseInstaller) hostToolPath() android.OptionalPath { func (installer *baseInstaller) relativeInstallPath() string { return String(installer.Properties.Relative_install_path) } + +func (installer *baseInstaller) skipInstall(mod *Module) { + mod.ModuleBase.SkipInstall() +} diff --git a/cc/kernel_headers.go b/cc/kernel_headers.go index 82a779c33..796de6269 100644 --- a/cc/kernel_headers.go +++ b/cc/kernel_headers.go @@ -25,13 +25,16 @@ type kernelHeadersDecorator struct { func (stub *kernelHeadersDecorator) link(ctx ModuleContext, flags Flags, deps PathDeps, objs Objects) android.Path { if ctx.Device() { f := &stub.libraryDecorator.flagExporter - for _, dir := range ctx.DeviceConfig().DeviceKernelHeaderDirs() { - f.flags = append(f.flags, "-isystem "+dir) - } + f.reexportSystemDirs(android.PathsForSource(ctx, ctx.DeviceConfig().DeviceKernelHeaderDirs())...) } return stub.libraryDecorator.linkStatic(ctx, flags, deps, objs) } +// kernel_headers retrieves the list of kernel headers directories from +// TARGET_BOARD_KERNEL_HEADERS and TARGET_PRODUCT_KERNEL_HEADERS variables in +// a makefile for compilation. See +// https://android.googlesource.com/platform/build/+/master/core/config.mk +// for more details on them. func kernelHeadersFactory() android.Module { module, library := NewLibrary(android.HostAndDeviceSupported) library.HeaderOnly() diff --git a/cc/libbuildversion/Android.bp b/cc/libbuildversion/Android.bp index 825b920a0..b63338da3 100644 --- a/cc/libbuildversion/Android.bp +++ b/cc/libbuildversion/Android.bp @@ -10,4 +10,8 @@ cc_library_static { enabled: true, }, }, + apex_available: [ + "//apex_available:platform", + "//apex_available:anyapex", + ], } diff --git a/cc/library.go b/cc/library.go index 39f7a724b..3deb17300 100644 --- a/cc/library.go +++ b/cc/library.go @@ -15,6 +15,8 @@ package cc import ( + "fmt" + "io" "path/filepath" "regexp" "sort" @@ -29,24 +31,7 @@ import ( "android/soong/genrule" ) -type StaticSharedLibraryProperties struct { - Srcs []string `android:"path,arch_variant"` - Cflags []string `android:"path,arch_variant"` - - Enabled *bool `android:"arch_variant"` - Whole_static_libs []string `android:"arch_variant"` - Static_libs []string `android:"arch_variant"` - Shared_libs []string `android:"arch_variant"` - System_shared_libs []string `android:"arch_variant"` - - Export_shared_lib_headers []string `android:"arch_variant"` - Export_static_lib_headers []string `android:"arch_variant"` -} - type LibraryProperties struct { - Static StaticSharedLibraryProperties `android:"arch_variant"` - Shared StaticSharedLibraryProperties `android:"arch_variant"` - // local file name to pass to the linker as -unexported_symbols_list Unexported_symbols_list *string `android:"path,arch_variant"` // local file name to pass to the linker as -force_symbols_not_weak_list @@ -86,6 +71,16 @@ type LibraryProperties struct { // set the name of the output Stem *string `android:"arch_variant"` + // set suffix of the name of the output + Suffix *string `android:"arch_variant"` + + Target struct { + Vendor struct { + // set suffix of the name of the output + Suffix *string `android:"arch_variant"` + } + } + // Names of modules to be overridden. Listed modules can only be other shared libraries // (in Make or Soong). // This does not completely prevent installation of the overridden libraries, but if both @@ -95,6 +90,9 @@ type LibraryProperties struct { // Properties for ABI compatibility checker Header_abi_checker struct { + // Enable ABI checks (even if this is not an LLNDK/VNDK lib) + Enabled *bool + // Path to a symbol file that specifies the symbols to be included in the generated // ABI dump file Symbol_file *string `android:"path"` @@ -105,11 +103,39 @@ type LibraryProperties struct { // Symbol tags that should be ignored from the symbol file Exclude_symbol_tags []string } + + // Order symbols in .bss section by their sizes. Only useful for shared libraries. + Sort_bss_symbols_by_size *bool + + // Inject boringssl hash into the shared library. This is only intended for use by external/boringssl. + Inject_bssl_hash *bool `android:"arch_variant"` } -type LibraryMutatedProperties struct { - VariantName string `blueprint:"mutated"` +type StaticProperties struct { + Static StaticOrSharedProperties `android:"arch_variant"` +} + +type SharedProperties struct { + Shared StaticOrSharedProperties `android:"arch_variant"` +} + +type StaticOrSharedProperties struct { + Srcs []string `android:"path,arch_variant"` + Cflags []string `android:"arch_variant"` + Enabled *bool `android:"arch_variant"` + Whole_static_libs []string `android:"arch_variant"` + Static_libs []string `android:"arch_variant"` + Shared_libs []string `android:"arch_variant"` + System_shared_libs []string `android:"arch_variant"` + + Export_shared_lib_headers []string `android:"arch_variant"` + Export_static_lib_headers []string `android:"arch_variant"` + + Apex_available []string `android:"arch_variant"` +} + +type LibraryMutatedProperties struct { // Build a static variant BuildStatic bool `blueprint:"mutated"` // Build a shared variant @@ -132,6 +158,10 @@ type FlagExporterProperties struct { // listed in local_include_dirs. Export_include_dirs []string `android:"arch_variant"` + // list of directories that will be added to the system include path + // using -isystem for this module and any module that links against this module. + Export_system_include_dirs []string `android:"arch_variant"` + Target struct { Vendor struct { // list of exported include directories, like @@ -144,12 +174,15 @@ type FlagExporterProperties struct { } func init() { - android.RegisterModuleType("cc_library_static", LibraryStaticFactory) - android.RegisterModuleType("cc_library_shared", LibrarySharedFactory) - android.RegisterModuleType("cc_library", LibraryFactory) - android.RegisterModuleType("cc_library_host_static", LibraryHostStaticFactory) - android.RegisterModuleType("cc_library_host_shared", LibraryHostSharedFactory) - android.RegisterModuleType("cc_library_headers", LibraryHeaderFactory) + RegisterLibraryBuildComponents(android.InitRegistrationContext) +} + +func RegisterLibraryBuildComponents(ctx android.RegistrationContext) { + ctx.RegisterModuleType("cc_library_static", LibraryStaticFactory) + ctx.RegisterModuleType("cc_library_shared", LibrarySharedFactory) + ctx.RegisterModuleType("cc_library", LibraryFactory) + ctx.RegisterModuleType("cc_library_host_static", LibraryHostStaticFactory) + ctx.RegisterModuleType("cc_library_host_shared", LibraryHostSharedFactory) } // cc_library creates both static and/or shared libraries for a device and/or @@ -158,6 +191,12 @@ func init() { // host. func LibraryFactory() android.Module { module, _ := NewLibrary(android.HostAndDeviceSupported) + // Can be used as both a static and a shared library. + module.sdkMemberTypes = []android.SdkMemberType{ + sharedLibrarySdkMemberType, + staticLibrarySdkMemberType, + staticAndSharedLibrarySdkMemberType, + } return module.Init() } @@ -165,6 +204,7 @@ func LibraryFactory() android.Module { func LibraryStaticFactory() android.Module { module, library := NewLibrary(android.HostAndDeviceSupported) library.BuildOnlyStatic() + module.sdkMemberTypes = []android.SdkMemberType{staticLibrarySdkMemberType} return module.Init() } @@ -172,6 +212,7 @@ func LibraryStaticFactory() android.Module { func LibrarySharedFactory() android.Module { module, library := NewLibrary(android.HostAndDeviceSupported) library.BuildOnlyShared() + module.sdkMemberTypes = []android.SdkMemberType{sharedLibrarySdkMemberType} return module.Init() } @@ -180,6 +221,7 @@ func LibrarySharedFactory() android.Module { func LibraryHostStaticFactory() android.Module { module, library := NewLibrary(android.HostSupported) library.BuildOnlyStatic() + module.sdkMemberTypes = []android.SdkMemberType{staticLibrarySdkMemberType} return module.Init() } @@ -187,24 +229,18 @@ func LibraryHostStaticFactory() android.Module { func LibraryHostSharedFactory() android.Module { module, library := NewLibrary(android.HostSupported) library.BuildOnlyShared() - return module.Init() -} - -// cc_library_headers contains a set of c/c++ headers which are imported by -// other soong cc modules using the header_libs property. For best practices, -// use export_include_dirs property or LOCAL_EXPORT_C_INCLUDE_DIRS for -// Make. -func LibraryHeaderFactory() android.Module { - module, library := NewLibrary(android.HostAndDeviceSupported) - library.HeaderOnly() + module.sdkMemberTypes = []android.SdkMemberType{sharedLibrarySdkMemberType} return module.Init() } type flagExporter struct { Properties FlagExporterProperties - flags []string - flagsDeps android.Paths + dirs android.Paths + systemDirs android.Paths + flags []string + deps android.Paths + headers android.Paths } func (f *flagExporter) exportedIncludes(ctx ModuleContext) android.Paths { @@ -215,32 +251,69 @@ func (f *flagExporter) exportedIncludes(ctx ModuleContext) android.Paths { } } -func (f *flagExporter) exportIncludes(ctx ModuleContext, inc string) { - includeDirs := f.exportedIncludes(ctx) - for _, dir := range includeDirs.Strings() { - f.flags = append(f.flags, inc+dir) - } +func (f *flagExporter) exportIncludes(ctx ModuleContext) { + f.dirs = append(f.dirs, f.exportedIncludes(ctx)...) + f.systemDirs = append(f.systemDirs, android.PathsForModuleSrc(ctx, f.Properties.Export_system_include_dirs)...) } -func (f *flagExporter) reexportFlags(flags []string) { +func (f *flagExporter) exportIncludesAsSystem(ctx ModuleContext) { + // all dirs are force exported as system + f.systemDirs = append(f.systemDirs, f.exportedIncludes(ctx)...) + f.systemDirs = append(f.systemDirs, android.PathsForModuleSrc(ctx, f.Properties.Export_system_include_dirs)...) +} + +func (f *flagExporter) reexportDirs(dirs ...android.Path) { + f.dirs = append(f.dirs, dirs...) +} + +func (f *flagExporter) reexportSystemDirs(dirs ...android.Path) { + f.systemDirs = append(f.systemDirs, dirs...) +} + +func (f *flagExporter) reexportFlags(flags ...string) { + if android.PrefixInList(flags, "-I") || android.PrefixInList(flags, "-isystem") { + panic(fmt.Errorf("Exporting invalid flag %q: "+ + "use reexportDirs or reexportSystemDirs to export directories", flag)) + } f.flags = append(f.flags, flags...) } -func (f *flagExporter) reexportDeps(deps android.Paths) { - f.flagsDeps = append(f.flagsDeps, deps...) +func (f *flagExporter) reexportDeps(deps ...android.Path) { + f.deps = append(f.deps, deps...) +} + +// addExportedGeneratedHeaders does nothing but collects generated header files. +// This can be differ to exportedDeps which may contain phony files to minimize ninja. +func (f *flagExporter) addExportedGeneratedHeaders(headers ...android.Path) { + f.headers = append(f.headers, headers...) +} + +func (f *flagExporter) exportedDirs() android.Paths { + return f.dirs +} + +func (f *flagExporter) exportedSystemDirs() android.Paths { + return f.systemDirs } func (f *flagExporter) exportedFlags() []string { return f.flags } -func (f *flagExporter) exportedFlagsDeps() android.Paths { - return f.flagsDeps +func (f *flagExporter) exportedDeps() android.Paths { + return f.deps +} + +func (f *flagExporter) exportedGeneratedHeaders() android.Paths { + return f.headers } type exportedFlagsProducer interface { + exportedDirs() android.Paths + exportedSystemDirs() android.Paths exportedFlags() []string - exportedFlagsDeps() android.Paths + exportedDeps() android.Paths + exportedGeneratedHeaders() android.Paths } var _ exportedFlagsProducer = (*flagExporter)(nil) @@ -249,12 +322,12 @@ var _ exportedFlagsProducer = (*flagExporter)(nil) // functionality: static vs. shared linkage, reusing object files for shared libraries type libraryDecorator struct { Properties LibraryProperties + StaticProperties StaticProperties + SharedProperties SharedProperties MutatedProperties LibraryMutatedProperties // For reusing static library objects for shared library - reuseObjects Objects - reuseExportedFlags []string - reuseExportedDeps android.Paths + reuseObjects Objects // table-of-contents file to optimize out relinking when possible tocFile android.OptionalPath @@ -300,22 +373,119 @@ type libraryDecorator struct { // If useCoreVariant is true, the vendor variant of a VNDK library is // not installed. - useCoreVariant bool + useCoreVariant bool + checkSameCoreVariant bool - // Decorated interafaces + // Decorated interfaces *baseCompiler *baseLinker *baseInstaller + + collectedSnapshotHeaders android.Paths +} + +// collectHeadersForSnapshot collects all exported headers from library. +// It globs header files in the source tree for exported include directories, +// and tracks generated header files separately. +// +// This is to be called from GenerateAndroidBuildActions, and then collected +// header files can be retrieved by snapshotHeaders(). +func (l *libraryDecorator) collectHeadersForSnapshot(ctx android.ModuleContext) { + ret := android.Paths{} + + // Headers in the source tree should be globbed. On the contrast, generated headers + // can't be globbed, and they should be manually collected. + // So, we first filter out intermediate directories (which contains generated headers) + // from exported directories, and then glob headers under remaining directories. + for _, path := range append(l.exportedDirs(), l.exportedSystemDirs()...) { + dir := path.String() + // Skip if dir is for generated headers + if strings.HasPrefix(dir, android.PathForOutput(ctx).String()) { + continue + } + // libeigen wrongly exports the root directory "external/eigen". But only two + // subdirectories "Eigen" and "unsupported" contain exported header files. Even worse + // some of them have no extension. So we need special treatment for libeigen in order + // to glob correctly. + if dir == "external/eigen" { + // Only these two directories contains exported headers. + for _, subdir := range []string{"Eigen", "unsupported/Eigen"} { + glob, err := ctx.GlobWithDeps("external/eigen/"+subdir+"/**/*", nil) + if err != nil { + ctx.ModuleErrorf("glob failed: %#v", err) + return + } + for _, header := range glob { + if strings.HasSuffix(header, "/") { + continue + } + ext := filepath.Ext(header) + if ext != "" && ext != ".h" { + continue + } + ret = append(ret, android.PathForSource(ctx, header)) + } + } + continue + } + exts := headerExts + // Glob all files under this special directory, because of C++ headers. + if strings.HasPrefix(dir, "external/libcxx/include") { + exts = []string{""} + } + for _, ext := range exts { + glob, err := ctx.GlobWithDeps(dir+"/**/*"+ext, nil) + if err != nil { + ctx.ModuleErrorf("glob failed: %#v", err) + return + } + for _, header := range glob { + if strings.HasSuffix(header, "/") { + continue + } + ret = append(ret, android.PathForSource(ctx, header)) + } + } + } + + // Collect generated headers + for _, header := range append(l.exportedGeneratedHeaders(), l.exportedDeps()...) { + // TODO(b/148123511): remove exportedDeps after cleaning up genrule + if strings.HasSuffix(header.Base(), "-phony") { + continue + } + ret = append(ret, header) + } + + l.collectedSnapshotHeaders = ret +} + +// This returns all exported header files, both generated ones and headers from source tree. +// collectHeadersForSnapshot() must be called before calling this. +func (l *libraryDecorator) snapshotHeaders() android.Paths { + if l.collectedSnapshotHeaders == nil { + panic("snapshotHeaders() must be called after collectHeadersForSnapshot()") + } + return l.collectedSnapshotHeaders } func (library *libraryDecorator) linkerProps() []interface{} { var props []interface{} props = append(props, library.baseLinker.linkerProps()...) - return append(props, + props = append(props, &library.Properties, &library.MutatedProperties, &library.flagExporter.Properties, &library.stripper.StripProperties) + + if library.MutatedProperties.BuildShared { + props = append(props, &library.SharedProperties) + } + if library.MutatedProperties.BuildStatic { + props = append(props, &library.StaticProperties) + } + + return props } func (library *libraryDecorator) linkerFlags(ctx ModuleContext, flags Flags) Flags { @@ -325,13 +495,13 @@ func (library *libraryDecorator) linkerFlags(ctx ModuleContext, flags Flags) Fla // all code is position independent, and then those warnings get promoted to // errors. if !ctx.Windows() { - flags.CFlags = append(flags.CFlags, "-fPIC") + flags.Global.CFlags = append(flags.Global.CFlags, "-fPIC") } if library.static() { - flags.CFlags = append(flags.CFlags, library.Properties.Static.Cflags...) + flags.Local.CFlags = append(flags.Local.CFlags, library.StaticProperties.Static.Cflags...) } else if library.shared() { - flags.CFlags = append(flags.CFlags, library.Properties.Shared.Cflags...) + flags.Local.CFlags = append(flags.Local.CFlags, library.SharedProperties.Shared.Cflags...) } if library.shared() { @@ -356,12 +526,13 @@ func (library *libraryDecorator) linkerFlags(ctx ModuleContext, flags Flags) Fla ) } } else { - f = append(f, - "-shared", - "-Wl,-soname,"+libName+flags.Toolchain.ShlibSuffix()) + f = append(f, "-shared") + if !ctx.Windows() { + f = append(f, "-Wl,-soname,"+libName+flags.Toolchain.ShlibSuffix()) + } } - flags.LdFlags = append(f, flags.LdFlags...) + flags.Global.LdFlags = append(flags.Global.LdFlags, f...) } return flags @@ -371,8 +542,8 @@ func (library *libraryDecorator) compilerFlags(ctx ModuleContext, flags Flags, d exportIncludeDirs := library.flagExporter.exportedIncludes(ctx) if len(exportIncludeDirs) > 0 { f := includeDirsToFlags(exportIncludeDirs) - flags.GlobalFlags = append(flags.GlobalFlags, f) - flags.YasmFlags = append(flags.YasmFlags, f) + flags.Local.CommonFlags = append(flags.Local.CommonFlags, f) + flags.Local.YasmFlags = append(flags.Local.YasmFlags, f) } flags = library.baseCompiler.compilerFlags(ctx, flags, deps) @@ -392,31 +563,67 @@ func (library *libraryDecorator) compilerFlags(ctx ModuleContext, flags Flags, d } return ret } - flags.GlobalFlags = removeInclude(flags.GlobalFlags) - flags.CFlags = removeInclude(flags.CFlags) + flags.Local.CommonFlags = removeInclude(flags.Local.CommonFlags) + flags.Local.CFlags = removeInclude(flags.Local.CFlags) flags = addStubLibraryCompilerFlags(flags) } return flags } -func extractExportIncludesFromFlags(flags []string) []string { - // This method is used in the generation of rules which produce - // abi-dumps for source files. Exported headers are needed to infer the - // abi exported by a library and filter out the rest of the abi dumped - // from a source. We extract the include flags exported by a library. - // This includes the flags exported which are re-exported from static - // library dependencies, exported header library dependencies and - // generated header dependencies. -isystem headers are not included - // since for bionic libraries, abi-filtering is taken care of by version - // scripts. - var exportedIncludes []string - for _, flag := range flags { - if strings.HasPrefix(flag, "-I") { - exportedIncludes = append(exportedIncludes, flag) +// Returns a string that represents the class of the ABI dump. +// Returns an empty string if ABI check is disabled for this library. +func (library *libraryDecorator) classifySourceAbiDump(ctx ModuleContext) string { + enabled := library.Properties.Header_abi_checker.Enabled + if enabled != nil && !Bool(enabled) { + return "" + } + // Return NDK if the library is both NDK and LLNDK. + if ctx.isNdk() { + return "NDK" + } + if ctx.isLlndkPublic(ctx.Config()) { + return "LLNDK" + } + if ctx.useVndk() && ctx.isVndk() && !ctx.isVndkPrivate(ctx.Config()) { + if ctx.isVndkSp() { + if ctx.isVndkExt() { + return "VNDK-SP-ext" + } else { + return "VNDK-SP" + } + } else { + if ctx.isVndkExt() { + return "VNDK-ext" + } else { + return "VNDK-core" + } } } - return exportedIncludes + if Bool(enabled) || ctx.hasStubsVariants() { + return "PLATFORM" + } + return "" +} + +func (library *libraryDecorator) shouldCreateSourceAbiDump(ctx ModuleContext) bool { + if !ctx.shouldCreateSourceAbiDump() { + return false + } + if !ctx.isForPlatform() { + if !ctx.hasStubsVariants() { + // Skip ABI checks if this library is for APEX but isn't exported. + return false + } + if !Bool(library.Properties.Header_abi_checker.Enabled) { + // Skip ABI checks if this library is for APEX and did not explicitly enable + // ABI checks. + // TODO(b/145608479): ABI checks should be enabled by default. Remove this + // after evaluating the extra build time. + return false + } + } + return library.classifySourceAbiDump(ctx) != "" } func (library *libraryDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) Objects { @@ -430,26 +637,26 @@ func (library *libraryDecorator) compile(ctx ModuleContext, flags Flags, deps Pa if len(library.baseCompiler.Properties.Srcs) > 0 { ctx.PropertyErrorf("srcs", "cc_library_headers must not have any srcs") } - if len(library.Properties.Static.Srcs) > 0 { + if len(library.StaticProperties.Static.Srcs) > 0 { ctx.PropertyErrorf("static.srcs", "cc_library_headers must not have any srcs") } - if len(library.Properties.Shared.Srcs) > 0 { + if len(library.SharedProperties.Shared.Srcs) > 0 { ctx.PropertyErrorf("shared.srcs", "cc_library_headers must not have any srcs") } return Objects{} } - if ctx.shouldCreateVndkSourceAbiDump() || library.sabi.Properties.CreateSAbiDumps { + if library.shouldCreateSourceAbiDump(ctx) || library.sabi.Properties.CreateSAbiDumps { exportIncludeDirs := library.flagExporter.exportedIncludes(ctx) var SourceAbiFlags []string for _, dir := range exportIncludeDirs.Strings() { SourceAbiFlags = append(SourceAbiFlags, "-I"+dir) } - for _, reexportedInclude := range extractExportIncludesFromFlags(library.sabi.Properties.ReexportedIncludeFlags) { - SourceAbiFlags = append(SourceAbiFlags, reexportedInclude) + for _, reexportedInclude := range library.sabi.Properties.ReexportedIncludes { + SourceAbiFlags = append(SourceAbiFlags, "-I"+reexportedInclude) } flags.SAbiFlags = SourceAbiFlags - total_length := len(library.baseCompiler.Properties.Srcs) + len(deps.GeneratedSources) + len(library.Properties.Shared.Srcs) + - len(library.Properties.Static.Srcs) + total_length := len(library.baseCompiler.Properties.Srcs) + len(deps.GeneratedSources) + + len(library.SharedProperties.Shared.Srcs) + len(library.StaticProperties.Static.Srcs) if total_length > 0 { flags.SAbiDump = true } @@ -459,11 +666,11 @@ func (library *libraryDecorator) compile(ctx ModuleContext, flags Flags, deps Pa buildFlags := flagsToBuilderFlags(flags) if library.static() { - srcs := android.PathsForModuleSrc(ctx, library.Properties.Static.Srcs) + srcs := android.PathsForModuleSrc(ctx, library.StaticProperties.Static.Srcs) objs = objs.Append(compileObjs(ctx, buildFlags, android.DeviceStaticLibrary, srcs, library.baseCompiler.pathDeps, library.baseCompiler.cFlagsDeps)) } else if library.shared() { - srcs := android.PathsForModuleSrc(ctx, library.Properties.Shared.Srcs) + srcs := android.PathsForModuleSrc(ctx, library.SharedProperties.Shared.Srcs) objs = objs.Append(compileObjs(ctx, buildFlags, android.DeviceSharedLibrary, srcs, library.baseCompiler.pathDeps, library.baseCompiler.cFlagsDeps)) } @@ -474,8 +681,9 @@ func (library *libraryDecorator) compile(ctx ModuleContext, flags Flags, deps Pa type libraryInterface interface { getWholeStaticMissingDeps() []string static() bool + shared() bool objs() Objects - reuseObjs() (Objects, []string, android.Paths) + reuseObjs() (Objects, exportedFlagsProducer) toc() android.OptionalPath // Returns true if the build options for the module have selected a static or shared build @@ -485,19 +693,42 @@ type libraryInterface interface { // Sets whether a specific variant is static or shared setStatic() setShared() + + // Write LOCAL_ADDITIONAL_DEPENDENCIES for ABI diff + androidMkWriteAdditionalDependenciesForSourceAbiDiff(w io.Writer) + + availableFor(string) bool } -func (library *libraryDecorator) getLibName(ctx ModuleContext) string { +func (library *libraryDecorator) getLibNameHelper(baseModuleName string, useVndk bool) string { name := library.libName if name == "" { name = String(library.Properties.Stem) if name == "" { - name = ctx.baseModuleName() + name = baseModuleName } } + suffix := "" + if useVndk { + suffix = String(library.Properties.Target.Vendor.Suffix) + } + if suffix == "" { + suffix = String(library.Properties.Suffix) + } + + return name + suffix +} + +func (library *libraryDecorator) getLibName(ctx BaseModuleContext) string { + name := library.getLibNameHelper(ctx.baseModuleName(), ctx.useVndk()) + if ctx.isVndkExt() { - name = ctx.getVndkExtendsModuleName() + // vndk-ext lib should have the same name with original lib + ctx.VisitDirectDepsWithTag(vndkExtDepTag, func(module android.Module) { + originalName := module.(*Module).outputFile.Path() + name = strings.TrimSuffix(originalName.Base(), originalName.Ext()) + }) } if ctx.Host() && Bool(library.Properties.Unique_host_soname) { @@ -506,7 +737,7 @@ func (library *libraryDecorator) getLibName(ctx ModuleContext) string { } } - return name + library.MutatedProperties.VariantName + return name } var versioningMacroNamesListMutex sync.Mutex @@ -543,12 +774,14 @@ func (library *libraryDecorator) compilerDeps(ctx DepsContext, deps Deps) Deps { func (library *libraryDecorator) linkerDeps(ctx DepsContext, deps Deps) Deps { if library.static() { - if library.Properties.Static.System_shared_libs != nil { - library.baseLinker.Properties.System_shared_libs = library.Properties.Static.System_shared_libs + // Compare with nil because an empty list needs to be propagated. + if library.StaticProperties.Static.System_shared_libs != nil { + library.baseLinker.Properties.System_shared_libs = library.StaticProperties.Static.System_shared_libs } } else if library.shared() { - if library.Properties.Shared.System_shared_libs != nil { - library.baseLinker.Properties.System_shared_libs = library.Properties.Shared.System_shared_libs + // Compare with nil because an empty list needs to be propagated. + if library.SharedProperties.Shared.System_shared_libs != nil { + library.baseLinker.Properties.System_shared_libs = library.SharedProperties.Shared.System_shared_libs } } @@ -556,12 +789,12 @@ func (library *libraryDecorator) linkerDeps(ctx DepsContext, deps Deps) Deps { if library.static() { deps.WholeStaticLibs = append(deps.WholeStaticLibs, - library.Properties.Static.Whole_static_libs...) - deps.StaticLibs = append(deps.StaticLibs, library.Properties.Static.Static_libs...) - deps.SharedLibs = append(deps.SharedLibs, library.Properties.Static.Shared_libs...) + library.StaticProperties.Static.Whole_static_libs...) + deps.StaticLibs = append(deps.StaticLibs, library.StaticProperties.Static.Static_libs...) + deps.SharedLibs = append(deps.SharedLibs, library.StaticProperties.Static.Shared_libs...) - deps.ReexportSharedLibHeaders = append(deps.ReexportSharedLibHeaders, library.Properties.Static.Export_shared_lib_headers...) - deps.ReexportStaticLibHeaders = append(deps.ReexportStaticLibHeaders, library.Properties.Static.Export_static_lib_headers...) + deps.ReexportSharedLibHeaders = append(deps.ReexportSharedLibHeaders, library.StaticProperties.Static.Export_shared_lib_headers...) + deps.ReexportStaticLibHeaders = append(deps.ReexportStaticLibHeaders, library.StaticProperties.Static.Export_static_lib_headers...) } else if library.shared() { if ctx.toolchain().Bionic() && !Bool(library.baseLinker.Properties.Nocrt) { if !ctx.useSdk() { @@ -580,12 +813,12 @@ func (library *libraryDecorator) linkerDeps(ctx DepsContext, deps Deps) Deps { deps.CrtEnd = "ndk_crtend_so." + version } } - deps.WholeStaticLibs = append(deps.WholeStaticLibs, library.Properties.Shared.Whole_static_libs...) - deps.StaticLibs = append(deps.StaticLibs, library.Properties.Shared.Static_libs...) - deps.SharedLibs = append(deps.SharedLibs, library.Properties.Shared.Shared_libs...) + deps.WholeStaticLibs = append(deps.WholeStaticLibs, library.SharedProperties.Shared.Whole_static_libs...) + deps.StaticLibs = append(deps.StaticLibs, library.SharedProperties.Shared.Static_libs...) + deps.SharedLibs = append(deps.SharedLibs, library.SharedProperties.Shared.Shared_libs...) - deps.ReexportSharedLibHeaders = append(deps.ReexportSharedLibHeaders, library.Properties.Shared.Export_shared_lib_headers...) - deps.ReexportStaticLibHeaders = append(deps.ReexportStaticLibHeaders, library.Properties.Shared.Export_static_lib_headers...) + deps.ReexportSharedLibHeaders = append(deps.ReexportSharedLibHeaders, library.SharedProperties.Shared.Export_shared_lib_headers...) + deps.ReexportStaticLibHeaders = append(deps.ReexportStaticLibHeaders, library.SharedProperties.Shared.Export_static_lib_headers...) } if ctx.useVndk() { deps.WholeStaticLibs = removeListFromList(deps.WholeStaticLibs, library.baseLinker.Properties.Target.Vendor.Exclude_static_libs) @@ -601,17 +834,52 @@ func (library *libraryDecorator) linkerDeps(ctx DepsContext, deps Deps) Deps { deps.ReexportSharedLibHeaders = removeListFromList(deps.ReexportSharedLibHeaders, library.baseLinker.Properties.Target.Recovery.Exclude_shared_libs) deps.ReexportStaticLibHeaders = removeListFromList(deps.ReexportStaticLibHeaders, library.baseLinker.Properties.Target.Recovery.Exclude_static_libs) } + if ctx.inRamdisk() { + deps.WholeStaticLibs = removeListFromList(deps.WholeStaticLibs, library.baseLinker.Properties.Target.Ramdisk.Exclude_static_libs) + deps.SharedLibs = removeListFromList(deps.SharedLibs, library.baseLinker.Properties.Target.Ramdisk.Exclude_shared_libs) + deps.StaticLibs = removeListFromList(deps.StaticLibs, library.baseLinker.Properties.Target.Ramdisk.Exclude_static_libs) + deps.ReexportSharedLibHeaders = removeListFromList(deps.ReexportSharedLibHeaders, library.baseLinker.Properties.Target.Ramdisk.Exclude_shared_libs) + deps.ReexportStaticLibHeaders = removeListFromList(deps.ReexportStaticLibHeaders, library.baseLinker.Properties.Target.Ramdisk.Exclude_static_libs) + } return deps } +func (library *libraryDecorator) linkerSpecifiedDeps(specifiedDeps specifiedDeps) specifiedDeps { + specifiedDeps = library.baseLinker.linkerSpecifiedDeps(specifiedDeps) + var properties StaticOrSharedProperties + if library.static() { + properties = library.StaticProperties.Static + } else if library.shared() { + properties = library.SharedProperties.Shared + } + + specifiedDeps.sharedLibs = append(specifiedDeps.sharedLibs, properties.Shared_libs...) + + // Must distinguish nil and [] in system_shared_libs - ensure that [] in + // either input list doesn't come out as nil. + if specifiedDeps.systemSharedLibs == nil { + specifiedDeps.systemSharedLibs = properties.System_shared_libs + } else { + specifiedDeps.systemSharedLibs = append(specifiedDeps.systemSharedLibs, properties.System_shared_libs...) + } + + specifiedDeps.sharedLibs = android.FirstUniqueStrings(specifiedDeps.sharedLibs) + if len(specifiedDeps.systemSharedLibs) > 0 { + // Skip this if systemSharedLibs is either nil or [], to ensure they are + // retained. + specifiedDeps.systemSharedLibs = android.FirstUniqueStrings(specifiedDeps.systemSharedLibs) + } + return specifiedDeps +} + func (library *libraryDecorator) linkStatic(ctx ModuleContext, flags Flags, deps PathDeps, objs Objects) android.Path { library.objects = deps.WholeStaticLibObjs.Copy() library.objects = library.objects.Append(objs) - fileName := ctx.ModuleName() + library.MutatedProperties.VariantName + staticLibraryExtension + fileName := ctx.ModuleName() + staticLibraryExtension outputFile := android.PathForModuleOut(ctx, fileName) builderFlags := flagsToBuilderFlags(flags) @@ -629,8 +897,7 @@ func (library *libraryDecorator) linkStatic(ctx ModuleContext, TransformObjToStaticLib(ctx, library.objects.objFiles, builderFlags, outputFile, objs.tidyFiles) - library.coverageOutputFile = TransformCoverageFilesToZip(ctx, library.objects, - ctx.ModuleName()+library.MutatedProperties.VariantName) + library.coverageOutputFile = TransformCoverageFilesToZip(ctx, library.objects, ctx.ModuleName()) library.wholeStaticMissingDeps = ctx.GetMissingDependencies() @@ -660,21 +927,21 @@ func (library *libraryDecorator) linkShared(ctx ModuleContext, } } else { if unexportedSymbols.Valid() { - flags.LdFlags = append(flags.LdFlags, "-Wl,-unexported_symbols_list,"+unexportedSymbols.String()) + flags.Local.LdFlags = append(flags.Local.LdFlags, "-Wl,-unexported_symbols_list,"+unexportedSymbols.String()) linkerDeps = append(linkerDeps, unexportedSymbols.Path()) } if forceNotWeakSymbols.Valid() { - flags.LdFlags = append(flags.LdFlags, "-Wl,-force_symbols_not_weak_list,"+forceNotWeakSymbols.String()) + flags.Local.LdFlags = append(flags.Local.LdFlags, "-Wl,-force_symbols_not_weak_list,"+forceNotWeakSymbols.String()) linkerDeps = append(linkerDeps, forceNotWeakSymbols.Path()) } if forceWeakSymbols.Valid() { - flags.LdFlags = append(flags.LdFlags, "-Wl,-force_symbols_weak_list,"+forceWeakSymbols.String()) + flags.Local.LdFlags = append(flags.Local.LdFlags, "-Wl,-force_symbols_weak_list,"+forceWeakSymbols.String()) linkerDeps = append(linkerDeps, forceWeakSymbols.Path()) } } if library.buildStubs() { linkerScriptFlags := "-Wl,--version-script," + library.versionScriptPath.String() - flags.LdFlags = append(flags.LdFlags, linkerScriptFlags) + flags.Local.LdFlags = append(flags.Local.LdFlags, linkerScriptFlags) linkerDeps = append(linkerDeps, library.versionScriptPath) } @@ -682,13 +949,19 @@ func (library *libraryDecorator) linkShared(ctx ModuleContext, outputFile := android.PathForModuleOut(ctx, fileName) ret := outputFile + var implicitOutputs android.WritablePaths + if ctx.Windows() { + importLibraryPath := android.PathForModuleOut(ctx, pathtools.ReplaceExtension(fileName, "lib")) + + flags.Local.LdFlags = append(flags.Local.LdFlags, "-Wl,--out-implib="+importLibraryPath.String()) + implicitOutputs = append(implicitOutputs, importLibraryPath) + } + builderFlags := flagsToBuilderFlags(flags) // Optimize out relinking against shared libraries whose interface hasn't changed by // depending on a table of contents file instead of the library itself. - tocPath := outputFile.RelPathString() - tocPath = pathtools.ReplaceExtension(tocPath, flags.Toolchain.ShlibSuffix()[1:]+".toc") - tocFile := android.PathForOutput(ctx, tocPath) + tocFile := outputFile.ReplaceExtension(ctx, flags.Toolchain.ShlibSuffix()[1:]+".toc") library.tocFile = android.OptionalPathForPath(tocFile) TransformSharedObjectToToc(ctx, outputFile, tocFile, builderFlags) @@ -698,11 +971,12 @@ func (library *libraryDecorator) linkShared(ctx ModuleContext, } strippedOutputFile := outputFile outputFile = android.PathForModuleOut(ctx, "unstripped", fileName) - library.stripper.strip(ctx, outputFile, strippedOutputFile, builderFlags) + library.stripper.stripExecutableOrSharedLib(ctx, outputFile, strippedOutputFile, builderFlags) } - library.unstrippedOutputFile = outputFile + outputFile = maybeInjectBoringSSLHash(ctx, outputFile, library.Properties.Inject_bssl_hash, fileName) + if Bool(library.baseLinker.Properties.Use_version_lib) { if ctx.Host() { versionedOutputFile := outputFile @@ -715,7 +989,7 @@ func (library *libraryDecorator) linkShared(ctx ModuleContext, if library.stripper.needsStrip(ctx) { out := android.PathForModuleOut(ctx, "versioned-stripped", fileName) library.distFile = android.OptionalPathForPath(out) - library.stripper.strip(ctx, versionedOutputFile, out, builderFlags) + library.stripper.stripExecutableOrSharedLib(ctx, versionedOutputFile, out, builderFlags) } library.injectVersionSymbol(ctx, outputFile, versionedOutputFile) @@ -731,9 +1005,21 @@ func (library *libraryDecorator) linkShared(ctx ModuleContext, linkerDeps = append(linkerDeps, deps.LateSharedLibsDeps...) linkerDeps = append(linkerDeps, objs.tidyFiles...) + if Bool(library.Properties.Sort_bss_symbols_by_size) { + unsortedOutputFile := android.PathForModuleOut(ctx, "unsorted", fileName) + TransformObjToDynamicBinary(ctx, objs.objFiles, sharedLibs, + deps.StaticLibs, deps.LateStaticLibs, deps.WholeStaticLibs, + linkerDeps, deps.CrtBegin, deps.CrtEnd, false, builderFlags, unsortedOutputFile, implicitOutputs) + + symbolOrderingFile := android.PathForModuleOut(ctx, "unsorted", fileName+".symbol_order") + symbolOrderingFlag := library.baseLinker.sortBssSymbolsBySize(ctx, unsortedOutputFile, symbolOrderingFile, builderFlags) + builderFlags.localLdFlags += " " + symbolOrderingFlag + linkerDeps = append(linkerDeps, symbolOrderingFile) + } + TransformObjToDynamicBinary(ctx, objs.objFiles, sharedLibs, deps.StaticLibs, deps.LateStaticLibs, deps.WholeStaticLibs, - linkerDeps, deps.CrtBegin, deps.CrtEnd, false, builderFlags, outputFile) + linkerDeps, deps.CrtBegin, deps.CrtEnd, false, builderFlags, outputFile, implicitOutputs) objs.coverageFiles = append(objs.coverageFiles, deps.StaticLibObjs.coverageFiles...) objs.coverageFiles = append(objs.coverageFiles, deps.WholeStaticLibObjs.coverageFiles...) @@ -763,10 +1049,12 @@ func (library *libraryDecorator) coverageOutputFilePath() android.OptionalPath { } func getRefAbiDumpFile(ctx ModuleContext, vndkVersion, fileName string) android.Path { - isLlndk := inList(ctx.baseModuleName(), llndkLibraries) || inList(ctx.baseModuleName(), ndkMigratedLibs) + // The logic must be consistent with classifySourceAbiDump. + isNdk := ctx.isNdk() + isLlndkOrVndk := ctx.isLlndkPublic(ctx.Config()) || (ctx.useVndk() && ctx.isVndk()) - refAbiDumpTextFile := android.PathForVndkRefAbiDump(ctx, vndkVersion, fileName, isLlndk, false) - refAbiDumpGzipFile := android.PathForVndkRefAbiDump(ctx, vndkVersion, fileName, isLlndk, true) + refAbiDumpTextFile := android.PathForVndkRefAbiDump(ctx, vndkVersion, fileName, isNdk, isLlndkOrVndk, false) + refAbiDumpGzipFile := android.PathForVndkRefAbiDump(ctx, vndkVersion, fileName, isNdk, isLlndkOrVndk, true) if refAbiDumpTextFile.Valid() { if refAbiDumpGzipFile.Valid() { @@ -784,7 +1072,7 @@ func getRefAbiDumpFile(ctx ModuleContext, vndkVersion, fileName string) android. } func (library *libraryDecorator) linkSAbiDumpFiles(ctx ModuleContext, objs Objects, fileName string, soFile android.Path) { - if len(objs.sAbiDumpFiles) > 0 && ctx.shouldCreateVndkSourceAbiDump() { + if library.shouldCreateSourceAbiDump(ctx) { vndkVersion := ctx.DeviceConfig().PlatformVndkVersion() if ver := ctx.DeviceConfig().VndkVersion(); ver != "" && ver != "current" { vndkVersion = ver @@ -795,19 +1083,21 @@ func (library *libraryDecorator) linkSAbiDumpFiles(ctx ModuleContext, objs Objec for _, dir := range exportIncludeDirs.Strings() { SourceAbiFlags = append(SourceAbiFlags, "-I"+dir) } - for _, reexportedInclude := range extractExportIncludesFromFlags(library.sabi.Properties.ReexportedIncludeFlags) { - SourceAbiFlags = append(SourceAbiFlags, reexportedInclude) + for _, reexportedInclude := range library.sabi.Properties.ReexportedIncludes { + SourceAbiFlags = append(SourceAbiFlags, "-I"+reexportedInclude) } exportedHeaderFlags := strings.Join(SourceAbiFlags, " ") library.sAbiOutputFile = TransformDumpToLinkedDump(ctx, objs.sAbiDumpFiles, soFile, fileName, exportedHeaderFlags, - android.OptionalPathForModuleSrc(ctx, library.Properties.Header_abi_checker.Symbol_file), + android.OptionalPathForModuleSrc(ctx, library.symbolFileForAbiCheck(ctx)), library.Properties.Header_abi_checker.Exclude_symbol_versions, library.Properties.Header_abi_checker.Exclude_symbol_tags) + addLsdumpPath(library.classifySourceAbiDump(ctx) + ":" + library.sAbiOutputFile.String()) + refAbiDumpFile := getRefAbiDumpFile(ctx, vndkVersion, fileName) if refAbiDumpFile != nil { library.sAbiDiff = SourceAbiDiff(ctx, library.sAbiOutputFile.Path(), - refAbiDumpFile, fileName, exportedHeaderFlags, ctx.isLlndk(), ctx.isVndkExt()) + refAbiDumpFile, fileName, exportedHeaderFlags, ctx.isLlndk(ctx.Config()), ctx.isNdk(), ctx.isVndkExt()) } } } @@ -823,76 +1113,71 @@ func (library *libraryDecorator) link(ctx ModuleContext, out = library.linkShared(ctx, flags, deps, objs) } - library.exportIncludes(ctx, "-I") - library.reexportFlags(deps.ReexportedFlags) - library.reexportDeps(deps.ReexportedFlagsDeps) + library.exportIncludes(ctx) + library.reexportDirs(deps.ReexportedDirs...) + library.reexportSystemDirs(deps.ReexportedSystemDirs...) + library.reexportFlags(deps.ReexportedFlags...) + library.reexportDeps(deps.ReexportedDeps...) + library.addExportedGeneratedHeaders(deps.ReexportedGeneratedHeaders...) if Bool(library.Properties.Aidl.Export_aidl_headers) { if library.baseCompiler.hasSrcExt(".aidl") { - flags := []string{ - "-I" + android.PathForModuleGen(ctx, "aidl").String(), - } - library.reexportFlags(flags) - library.reuseExportedFlags = append(library.reuseExportedFlags, flags...) - library.reexportDeps(library.baseCompiler.pathDeps) // TODO: restrict to aidl deps - library.reuseExportedDeps = append(library.reuseExportedDeps, library.baseCompiler.pathDeps...) + dir := android.PathForModuleGen(ctx, "aidl") + library.reexportDirs(dir) + + // TODO: restrict to aidl deps + library.reexportDeps(library.baseCompiler.pathDeps...) + library.addExportedGeneratedHeaders(library.baseCompiler.pathDeps...) } } if Bool(library.Properties.Proto.Export_proto_headers) { if library.baseCompiler.hasSrcExt(".proto") { - includes := []string{} + var includes android.Paths if flags.proto.CanonicalPathFromRoot { - includes = append(includes, "-I"+flags.proto.SubDir.String()) + includes = append(includes, flags.proto.SubDir) } - includes = append(includes, "-I"+flags.proto.Dir.String()) - library.reexportFlags(includes) - library.reuseExportedFlags = append(library.reuseExportedFlags, includes...) - library.reexportDeps(library.baseCompiler.pathDeps) // TODO: restrict to proto deps - library.reuseExportedDeps = append(library.reuseExportedDeps, library.baseCompiler.pathDeps...) + includes = append(includes, flags.proto.Dir) + library.reexportDirs(includes...) + + // TODO: restrict to proto deps + library.reexportDeps(library.baseCompiler.pathDeps...) + library.addExportedGeneratedHeaders(library.baseCompiler.pathDeps...) } } if library.baseCompiler.hasSrcExt(".sysprop") { - internalFlags := []string{ - "-I" + android.PathForModuleGen(ctx, "sysprop", "include").String(), - } - systemFlags := []string{ - "-I" + android.PathForModuleGen(ctx, "sysprop/system", "include").String(), - } - - flags := internalFlags - + dir := android.PathForModuleGen(ctx, "sysprop", "include") if library.Properties.Sysprop.Platform != nil { isProduct := ctx.ProductSpecific() && !ctx.useVndk() isVendor := ctx.useVndk() isOwnerPlatform := Bool(library.Properties.Sysprop.Platform) - useSystem := isProduct || (isOwnerPlatform == isVendor) - - if useSystem { - flags = systemFlags + if !ctx.inRamdisk() && !ctx.inRecovery() && (isProduct || (isOwnerPlatform == isVendor)) { + dir = android.PathForModuleGen(ctx, "sysprop/public", "include") } } - library.reexportFlags(flags) - library.reexportDeps(library.baseCompiler.pathDeps) - library.reuseExportedFlags = append(library.reuseExportedFlags, flags...) + library.reexportDirs(dir) + library.reexportDeps(library.baseCompiler.pathDeps...) + library.addExportedGeneratedHeaders(library.baseCompiler.pathDeps...) } if library.buildStubs() { - library.reexportFlags([]string{"-D" + versioningMacroName(ctx.ModuleName()) + "=" + library.stubsVersion()}) + library.reexportFlags("-D" + versioningMacroName(ctx.ModuleName()) + "=" + library.stubsVersion()) } return out } func (library *libraryDecorator) buildStatic() bool { - return library.MutatedProperties.BuildStatic && BoolDefault(library.Properties.Static.Enabled, true) + return library.MutatedProperties.BuildStatic && + BoolDefault(library.StaticProperties.Static.Enabled, true) } func (library *libraryDecorator) buildShared() bool { - return library.MutatedProperties.BuildShared && BoolDefault(library.Properties.Shared.Enabled, true) + return library.MutatedProperties.BuildShared && + BoolDefault(library.SharedProperties.Shared.Enabled, true) } func (library *libraryDecorator) getWholeStaticMissingDeps() []string { @@ -903,8 +1188,8 @@ func (library *libraryDecorator) objs() Objects { return library.objects } -func (library *libraryDecorator) reuseObjs() (Objects, []string, android.Paths) { - return library.reuseObjects, library.reuseExportedFlags, library.reuseExportedDeps +func (library *libraryDecorator) reuseObjs() (Objects, exportedFlagsProducer) { + return library.reuseObjects, &library.flagExporter } func (library *libraryDecorator) toc() android.OptionalPath { @@ -925,8 +1210,25 @@ func (library *libraryDecorator) install(ctx ModuleContext, file android.Path) { if ctx.isVndkSp() { library.baseInstaller.subDir = "vndk-sp" } else if ctx.isVndk() { - if ctx.DeviceConfig().VndkUseCoreVariant() && !ctx.mustUseVendorVariant() { - library.useCoreVariant = true + mayUseCoreVariant := true + + if ctx.mustUseVendorVariant() { + mayUseCoreVariant = false + } + + if ctx.isVndkExt() { + mayUseCoreVariant = false + } + + if ctx.Config().CFIEnabledForPath(ctx.ModuleDir()) && ctx.Arch().ArchType == android.Arm64 { + mayUseCoreVariant = false + } + + if mayUseCoreVariant { + library.checkSameCoreVariant = true + if ctx.DeviceConfig().VndkUseCoreVariant() { + library.useCoreVariant = true + } } library.baseInstaller.subDir = "vndk" } @@ -938,24 +1240,29 @@ func (library *libraryDecorator) install(ctx ModuleContext, file android.Path) { library.baseInstaller.subDir += "-" + vndkVersion } } - } else if len(library.Properties.Stubs.Versions) > 0 { + } else if len(library.Properties.Stubs.Versions) > 0 && android.DirectlyInAnyApex(ctx, ctx.ModuleName()) { // Bionic libraries (e.g. libc.so) is installed to the bootstrap subdirectory. // The original path becomes a symlink to the corresponding file in the // runtime APEX. - if installToBootstrap(ctx.baseModuleName(), ctx.Config()) && !library.buildStubs() && ctx.Arch().Native && !ctx.inRecovery() { - if ctx.Device() && isBionic(ctx.baseModuleName()) { + translatedArch := ctx.Target().NativeBridge == android.NativeBridgeEnabled + if InstallToBootstrap(ctx.baseModuleName(), ctx.Config()) && !library.buildStubs() && !translatedArch && !ctx.inRamdisk() && !ctx.inRecovery() { + if ctx.Device() { library.installSymlinkToRuntimeApex(ctx, file) } library.baseInstaller.subDir = "bootstrap" } + } else if android.DirectlyInAnyApex(ctx, ctx.ModuleName()) && ctx.isLlndk(ctx.Config()) && !isBionic(ctx.baseModuleName()) { + // Skip installing LLNDK (non-bionic) libraries moved to APEX. + ctx.Module().SkipInstall() } + library.baseInstaller.install(ctx, file) } if Bool(library.Properties.Static_ndk_lib) && library.static() && - !ctx.useVndk() && !ctx.inRecovery() && ctx.Device() && + !ctx.useVndk() && !ctx.inRamdisk() && !ctx.inRecovery() && ctx.Device() && library.baseLinker.sanitize.isUnsanitizedVariant() && - !library.buildStubs() { + !library.buildStubs() && ctx.sdkVersion() == "" { installPath := getNdkSysrootBase(ctx).Join( ctx, "usr/lib", config.NDKTriple(ctx.toolchain()), file.Base()) @@ -970,6 +1277,12 @@ func (library *libraryDecorator) install(ctx ModuleContext, file android.Path) { } } +func (library *libraryDecorator) everInstallable() bool { + // Only shared and static libraries are installed. Header libraries (which are + // neither static or shared) are not installed. + return library.shared() || library.static() +} + func (library *libraryDecorator) static() bool { return library.MutatedProperties.VariantIsStatic } @@ -1009,10 +1322,50 @@ func (library *libraryDecorator) buildStubs() bool { return library.MutatedProperties.BuildStubs } +func (library *libraryDecorator) symbolFileForAbiCheck(ctx ModuleContext) *string { + if library.Properties.Header_abi_checker.Symbol_file != nil { + return library.Properties.Header_abi_checker.Symbol_file + } + if ctx.hasStubsVariants() && library.Properties.Stubs.Symbol_file != nil { + return library.Properties.Stubs.Symbol_file + } + return nil +} + func (library *libraryDecorator) stubsVersion() string { return library.MutatedProperties.StubsVersion } +func (library *libraryDecorator) isLatestStubVersion() bool { + versions := library.Properties.Stubs.Versions + return versions[len(versions)-1] == library.stubsVersion() +} + +func (library *libraryDecorator) availableFor(what string) bool { + var list []string + if library.static() { + list = library.StaticProperties.Static.Apex_available + } else if library.shared() { + list = library.SharedProperties.Shared.Apex_available + } + if len(list) == 0 { + return false + } + return android.CheckAvailableForApex(what, list) +} + +func (library *libraryDecorator) skipInstall(mod *Module) { + if library.static() && library.buildStatic() && !library.buildStubs() { + // If we're asked to skip installation of a static library (in particular + // when it's not //apex_available:platform) we still want an AndroidMk entry + // for it to ensure we get the relevant NOTICE file targets (cf. + // notice_files.mk) that other libraries might depend on. AndroidMkEntries + // always sets LOCAL_UNINSTALLABLE_MODULE for these entries. + return + } + mod.ModuleBase.SkipInstall() +} + var versioningMacroNamesListKey = android.NewOnceKey("versioningMacroNamesList") func versioningMacroNamesList(config android.Config) *map[string]string { @@ -1028,7 +1381,7 @@ var charsNotForMacro = regexp.MustCompile("[^a-zA-Z0-9_]+") func versioningMacroName(moduleName string) string { macroName := charsNotForMacro.ReplaceAllString(moduleName, "_") - macroName = strings.ToUpper(moduleName) + macroName = strings.ToUpper(macroName) return "__" + macroName + "_API__" } @@ -1061,16 +1414,18 @@ func reuseStaticLibrary(mctx android.BottomUpMutatorContext, static, shared *Mod // Check libraries in addition to cflags, since libraries may be exporting different // include directories. - if len(staticCompiler.Properties.Static.Cflags) == 0 && - len(sharedCompiler.Properties.Shared.Cflags) == 0 && - len(staticCompiler.Properties.Static.Whole_static_libs) == 0 && - len(sharedCompiler.Properties.Shared.Whole_static_libs) == 0 && - len(staticCompiler.Properties.Static.Static_libs) == 0 && - len(sharedCompiler.Properties.Shared.Static_libs) == 0 && - len(staticCompiler.Properties.Static.Shared_libs) == 0 && - len(sharedCompiler.Properties.Shared.Shared_libs) == 0 && - staticCompiler.Properties.Static.System_shared_libs == nil && - sharedCompiler.Properties.Shared.System_shared_libs == nil { + if len(staticCompiler.StaticProperties.Static.Cflags) == 0 && + len(sharedCompiler.SharedProperties.Shared.Cflags) == 0 && + len(staticCompiler.StaticProperties.Static.Whole_static_libs) == 0 && + len(sharedCompiler.SharedProperties.Shared.Whole_static_libs) == 0 && + len(staticCompiler.StaticProperties.Static.Static_libs) == 0 && + len(sharedCompiler.SharedProperties.Shared.Static_libs) == 0 && + len(staticCompiler.StaticProperties.Static.Shared_libs) == 0 && + len(sharedCompiler.SharedProperties.Shared.Shared_libs) == 0 && + // Compare System_shared_libs properties with nil because empty lists are + // semantically significant for them. + staticCompiler.StaticProperties.Static.System_shared_libs == nil && + sharedCompiler.SharedProperties.Shared.System_shared_libs == nil { mctx.AddInterVariantDependency(reuseObjTag, shared, static) sharedCompiler.baseCompiler.Properties.OriginalSrcs = @@ -1085,9 +1440,15 @@ func reuseStaticLibrary(mctx android.BottomUpMutatorContext, static, shared *Mod } func LinkageMutator(mctx android.BottomUpMutatorContext) { + cc_prebuilt := false if m, ok := mctx.Module().(*Module); ok && m.linker != nil { - switch library := m.linker.(type) { - case prebuiltLibraryInterface: + _, cc_prebuilt = m.linker.(prebuiltLibraryInterface) + } + if cc_prebuilt { + library := mctx.Module().(*Module).linker.(prebuiltLibraryInterface) + + // Differentiate between header only and building an actual static/shared library + if library.buildStatic() || library.buildShared() { // Always create both the static and shared variants for prebuilt libraries, and then disable the one // that is not being used. This allows them to share the name of a cc_library module, which requires that // all the variants of the cc_library also exist on the prebuilt. @@ -1104,25 +1465,43 @@ func LinkageMutator(mctx android.BottomUpMutatorContext) { if !library.buildShared() { shared.linker.(prebuiltLibraryInterface).disablePrebuilt() } + } else { + // Header only + } - case libraryInterface: - if library.buildStatic() && library.buildShared() { - modules := mctx.CreateLocalVariations("static", "shared") - static := modules[0].(*Module) - shared := modules[1].(*Module) + } else if library, ok := mctx.Module().(LinkableInterface); ok && library.CcLibraryInterface() { - static.linker.(libraryInterface).setStatic() - shared.linker.(libraryInterface).setShared() + // Non-cc.Modules may need an empty variant for their mutators. + variations := []string{} + if library.NonCcVariants() { + variations = append(variations, "") + } + + if library.BuildStaticVariant() && library.BuildSharedVariant() { + variations := append([]string{"static", "shared"}, variations...) - reuseStaticLibrary(mctx, static, shared) + modules := mctx.CreateLocalVariations(variations...) + static := modules[0].(LinkableInterface) + shared := modules[1].(LinkableInterface) - } else if library.buildStatic() { - modules := mctx.CreateLocalVariations("static") - modules[0].(*Module).linker.(libraryInterface).setStatic() - } else if library.buildShared() { - modules := mctx.CreateLocalVariations("shared") - modules[0].(*Module).linker.(libraryInterface).setShared() + static.SetStatic() + shared.SetShared() + + if _, ok := library.(*Module); ok { + reuseStaticLibrary(mctx, static.(*Module), shared.(*Module)) } + } else if library.BuildStaticVariant() { + variations := append([]string{"static"}, variations...) + + modules := mctx.CreateLocalVariations(variations...) + modules[0].(LinkableInterface).SetStatic() + } else if library.BuildSharedVariant() { + variations := append([]string{"shared"}, variations...) + + modules := mctx.CreateLocalVariations(variations...) + modules[0].(LinkableInterface).SetShared() + } else if len(variations) > 0 { + mctx.CreateLocalVariations(variations...) } } } @@ -1138,7 +1517,7 @@ func stubsVersionsFor(config android.Config) map[string][]string { var stubsVersionsLock sync.Mutex -func latestStubsVersionFor(config android.Config, name string) string { +func LatestStubsVersionFor(config android.Config, name string) string { versions, ok := stubsVersionsFor(config)[name] if ok && len(versions) > 0 { // the versions are alreay sorted in ascending order @@ -1147,56 +1526,124 @@ func latestStubsVersionFor(config android.Config, name string) string { return "" } -// Version mutator splits a module into the mandatory non-stubs variant +func normalizeVersions(ctx android.BaseModuleContext, versions []string) { + numVersions := make([]int, len(versions)) + for i, v := range versions { + numVer, err := android.ApiStrToNum(ctx, v) + if err != nil { + ctx.PropertyErrorf("versions", "%s", err.Error()) + return + } + numVersions[i] = numVer + } + if !sort.IsSorted(sort.IntSlice(numVersions)) { + ctx.PropertyErrorf("versions", "not sorted: %v", versions) + } + for i, v := range numVersions { + versions[i] = strconv.Itoa(v) + } +} + +func createVersionVariations(mctx android.BottomUpMutatorContext, versions []string) { + // "" is for the non-stubs variant + versions = append([]string{""}, versions...) + + modules := mctx.CreateVariations(versions...) + for i, m := range modules { + if versions[i] != "" { + m.(LinkableInterface).SetBuildStubs() + m.(LinkableInterface).SetStubsVersions(versions[i]) + } + } +} + +func VersionVariantAvailable(module interface { + Host() bool + InRamdisk() bool + InRecovery() bool +}) bool { + return !module.Host() && !module.InRamdisk() && !module.InRecovery() +} + +// VersionMutator splits a module into the mandatory non-stubs variant // (which is unnamed) and zero or more stubs variants. func VersionMutator(mctx android.BottomUpMutatorContext) { - if m, ok := mctx.Module().(*Module); ok && !m.inRecovery() && m.linker != nil { - if library, ok := m.linker.(*libraryDecorator); ok && library.buildShared() && - len(library.Properties.Stubs.Versions) > 0 { - versions := []string{} - for _, v := range library.Properties.Stubs.Versions { - if _, err := strconv.Atoi(v); err != nil { - mctx.PropertyErrorf("versions", "%q is not a number", v) - } - versions = append(versions, v) + if library, ok := mctx.Module().(LinkableInterface); ok && VersionVariantAvailable(library) { + if library.CcLibrary() && library.BuildSharedVariant() && len(library.StubsVersions()) > 0 { + versions := library.StubsVersions() + normalizeVersions(mctx, versions) + if mctx.Failed() { + return } - sort.Slice(versions, func(i, j int) bool { - left, _ := strconv.Atoi(versions[i]) - right, _ := strconv.Atoi(versions[j]) - return left < right - }) + stubsVersionsLock.Lock() + defer stubsVersionsLock.Unlock() // save the list of versions for later use - copiedVersions := make([]string, len(versions)) - copy(copiedVersions, versions) + stubsVersionsFor(mctx.Config())[mctx.ModuleName()] = versions + + createVersionVariations(mctx, versions) + return + } + + if c, ok := library.(*Module); ok && c.IsStubs() { stubsVersionsLock.Lock() defer stubsVersionsLock.Unlock() - stubsVersionsFor(mctx.Config())[mctx.ModuleName()] = copiedVersions - - // "" is for the non-stubs variant - versions = append([]string{""}, versions...) - - modules := mctx.CreateVariations(versions...) - for i, m := range modules { - l := m.(*Module).linker.(*libraryDecorator) - if versions[i] != "" { - l.MutatedProperties.BuildStubs = true - l.MutatedProperties.StubsVersion = versions[i] - m.(*Module).Properties.HideFromMake = true - m.(*Module).sanitize = nil - m.(*Module).stl = nil - m.(*Module).Properties.PreventInstall = true - } - } - } else { - mctx.CreateVariations("") + // For LLNDK llndk_library, we borrow vstubs.ersions from its implementation library. + // Since llndk_library has dependency to its implementation library, + // we can safely access stubsVersionsFor() with its baseModuleName. + versions := stubsVersionsFor(mctx.Config())[c.BaseModuleName()] + // save the list of versions for later use + stubsVersionsFor(mctx.Config())[mctx.ModuleName()] = versions + + createVersionVariations(mctx, versions) + return } + + mctx.CreateVariations("") return } if genrule, ok := mctx.Module().(*genrule.Module); ok { - if props, ok := genrule.Extra.(*GenruleExtraProperties); ok && !props.InRecovery { - mctx.CreateVariations("") - return + if _, ok := genrule.Extra.(*GenruleExtraProperties); ok { + if VersionVariantAvailable(genrule) { + mctx.CreateVariations("") + return + } } } } + +// maybeInjectBoringSSLHash adds a rule to run bssl_inject_hash on the output file if the module has the +// inject_bssl_hash or if any static library dependencies have inject_bssl_hash set. It returns the output path +// that the linked output file should be written to. +// TODO(b/137267623): Remove this in favor of a cc_genrule when they support operating on shared libraries. +func maybeInjectBoringSSLHash(ctx android.ModuleContext, outputFile android.ModuleOutPath, + inject *bool, fileName string) android.ModuleOutPath { + // TODO(b/137267623): Remove this in favor of a cc_genrule when they support operating on shared libraries. + injectBoringSSLHash := Bool(inject) + ctx.VisitDirectDeps(func(dep android.Module) { + tag := ctx.OtherModuleDependencyTag(dep) + if tag == StaticDepTag || tag == staticExportDepTag || tag == wholeStaticDepTag || tag == lateStaticDepTag { + if cc, ok := dep.(*Module); ok { + if library, ok := cc.linker.(*libraryDecorator); ok { + if Bool(library.Properties.Inject_bssl_hash) { + injectBoringSSLHash = true + } + } + } + } + }) + if injectBoringSSLHash { + hashedOutputfile := outputFile + outputFile = android.PathForModuleOut(ctx, "unhashed", fileName) + + rule := android.NewRuleBuilder() + rule.Command(). + BuiltTool(ctx, "bssl_inject_hash"). + Flag("-sha256"). + FlagWithInput("-in-object ", outputFile). + FlagWithOutput("-o ", hashedOutputfile) + rule.Build(pctx, ctx, "injectCryptoHash", "inject crypto hash") + } + + return outputFile +} diff --git a/cc/library_headers.go b/cc/library_headers.go new file mode 100644 index 000000000..b7ab3907c --- /dev/null +++ b/cc/library_headers.go @@ -0,0 +1,56 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cc + +import "android/soong/android" + +func init() { + RegisterLibraryHeadersBuildComponents(android.InitRegistrationContext) + + // Register sdk member types. + android.RegisterSdkMemberType(headersLibrarySdkMemberType) +} + +var headersLibrarySdkMemberType = &librarySdkMemberType{ + SdkMemberTypeBase: android.SdkMemberTypeBase{ + PropertyName: "native_header_libs", + SupportsSdk: true, + }, + prebuiltModuleType: "cc_prebuilt_library_headers", + noOutputFiles: true, +} + +func RegisterLibraryHeadersBuildComponents(ctx android.RegistrationContext) { + ctx.RegisterModuleType("cc_library_headers", LibraryHeaderFactory) + ctx.RegisterModuleType("cc_prebuilt_library_headers", prebuiltLibraryHeaderFactory) +} + +// cc_library_headers contains a set of c/c++ headers which are imported by +// other soong cc modules using the header_libs property. For best practices, +// use export_include_dirs property or LOCAL_EXPORT_C_INCLUDE_DIRS for +// Make. +func LibraryHeaderFactory() android.Module { + module, library := NewLibrary(android.HostAndDeviceSupported) + library.HeaderOnly() + module.sdkMemberTypes = []android.SdkMemberType{headersLibrarySdkMemberType} + return module.Init() +} + +// cc_prebuilt_library_headers is a prebuilt version of cc_library_headers +func prebuiltLibraryHeaderFactory() android.Module { + module, library := NewPrebuiltLibrary(android.HostAndDeviceSupported) + library.HeaderOnly() + return module.Init() +} diff --git a/cc/library_headers_test.go b/cc/library_headers_test.go new file mode 100644 index 000000000..564ef61fb --- /dev/null +++ b/cc/library_headers_test.go @@ -0,0 +1,62 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cc + +import ( + "strings" + "testing" +) + +func TestLibraryHeaders(t *testing.T) { + ctx := testCc(t, ` + cc_library_headers { + name: "headers", + export_include_dirs: ["my_include"], + } + cc_library_static { + name: "lib", + srcs: ["foo.c"], + header_libs: ["headers"], + } + `) + + // test if header search paths are correctly added + cc := ctx.ModuleForTests("lib", "android_arm64_armv8-a_static").Rule("cc") + cflags := cc.Args["cFlags"] + if !strings.Contains(cflags, " -Imy_include ") { + t.Errorf("cflags for libsystem must contain -Imy_include, but was %#v.", cflags) + } +} + +func TestPrebuiltLibraryHeaders(t *testing.T) { + ctx := testCc(t, ` + cc_prebuilt_library_headers { + name: "headers", + export_include_dirs: ["my_include"], + } + cc_library_static { + name: "lib", + srcs: ["foo.c"], + header_libs: ["headers"], + } + `) + + // test if header search paths are correctly added + cc := ctx.ModuleForTests("lib", "android_arm64_armv8-a_static").Rule("cc") + cflags := cc.Args["cFlags"] + if !strings.Contains(cflags, " -Imy_include ") { + t.Errorf("cflags for libsystem must contain -Imy_include, but was %#v.", cflags) + } +} diff --git a/cc/library_sdk_member.go b/cc/library_sdk_member.go new file mode 100644 index 000000000..a7a1de251 --- /dev/null +++ b/cc/library_sdk_member.go @@ -0,0 +1,408 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cc + +import ( + "path/filepath" + + "android/soong/android" + + "github.com/google/blueprint" + "github.com/google/blueprint/proptools" +) + +// This file contains support for using cc library modules within an sdk. + +var sharedLibrarySdkMemberType = &librarySdkMemberType{ + SdkMemberTypeBase: android.SdkMemberTypeBase{ + PropertyName: "native_shared_libs", + SupportsSdk: true, + }, + prebuiltModuleType: "cc_prebuilt_library_shared", + linkTypes: []string{"shared"}, +} + +var staticLibrarySdkMemberType = &librarySdkMemberType{ + SdkMemberTypeBase: android.SdkMemberTypeBase{ + PropertyName: "native_static_libs", + SupportsSdk: true, + }, + prebuiltModuleType: "cc_prebuilt_library_static", + linkTypes: []string{"static"}, +} + +var staticAndSharedLibrarySdkMemberType = &librarySdkMemberType{ + SdkMemberTypeBase: android.SdkMemberTypeBase{ + PropertyName: "native_libs", + SupportsSdk: true, + }, + prebuiltModuleType: "cc_prebuilt_library", + linkTypes: []string{"static", "shared"}, +} + +func init() { + // Register sdk member types. + android.RegisterSdkMemberType(sharedLibrarySdkMemberType) + android.RegisterSdkMemberType(staticLibrarySdkMemberType) + android.RegisterSdkMemberType(staticAndSharedLibrarySdkMemberType) +} + +type librarySdkMemberType struct { + android.SdkMemberTypeBase + + prebuiltModuleType string + + noOutputFiles bool // True if there are no srcs files. + + // The set of link types supported. A set of "static", "shared", or nil to + // skip link type variations. + linkTypes []string +} + +func (mt *librarySdkMemberType) AddDependencies(mctx android.BottomUpMutatorContext, dependencyTag blueprint.DependencyTag, names []string) { + targets := mctx.MultiTargets() + for _, lib := range names { + for _, target := range targets { + name, version := StubsLibNameAndVersion(lib) + if version == "" { + version = LatestStubsVersionFor(mctx.Config(), name) + } + if mt.linkTypes == nil { + mctx.AddFarVariationDependencies(append(target.Variations(), []blueprint.Variation{ + {Mutator: "image", Variation: android.CoreVariation}, + {Mutator: "version", Variation: version}, + }...), dependencyTag, name) + } else { + for _, linkType := range mt.linkTypes { + mctx.AddFarVariationDependencies(append(target.Variations(), []blueprint.Variation{ + {Mutator: "image", Variation: android.CoreVariation}, + {Mutator: "link", Variation: linkType}, + {Mutator: "version", Variation: version}, + }...), dependencyTag, name) + } + } + } + } +} + +func (mt *librarySdkMemberType) IsInstance(module android.Module) bool { + // Check the module to see if it can be used with this module type. + if m, ok := module.(*Module); ok { + for _, allowableMemberType := range m.sdkMemberTypes { + if allowableMemberType == mt { + return true + } + } + } + + return false +} + +func (mt *librarySdkMemberType) AddPrebuiltModule(ctx android.SdkMemberContext, member android.SdkMember) android.BpModule { + pbm := ctx.SnapshotBuilder().AddPrebuiltModule(member, mt.prebuiltModuleType) + + ccModule := member.Variants()[0].(*Module) + + sdkVersion := ccModule.SdkVersion() + if sdkVersion != "" { + pbm.AddProperty("sdk_version", sdkVersion) + } + + stl := ccModule.stl.Properties.Stl + if stl != nil { + pbm.AddProperty("stl", proptools.String(stl)) + } + return pbm +} + +func (mt *librarySdkMemberType) CreateVariantPropertiesStruct() android.SdkMemberProperties { + return &nativeLibInfoProperties{memberType: mt} +} + +func isGeneratedHeaderDirectory(p android.Path) bool { + _, gen := p.(android.WritablePath) + return gen +} + +type includeDirsProperty struct { + // Accessor to retrieve the paths + pathsGetter func(libInfo *nativeLibInfoProperties) android.Paths + + // The name of the property in the prebuilt library, "" means there is no property. + propertyName string + + // The directory within the snapshot directory into which items should be copied. + snapshotDir string + + // True if the items on the path should be copied. + copy bool + + // True if the paths represent directories, files if they represent files. + dirs bool +} + +var includeDirProperties = []includeDirsProperty{ + { + // ExportedIncludeDirs lists directories that contains some header files to be + // copied into a directory in the snapshot. The snapshot directories must be added to + // the export_include_dirs property in the prebuilt module in the snapshot. + pathsGetter: func(libInfo *nativeLibInfoProperties) android.Paths { return libInfo.ExportedIncludeDirs }, + propertyName: "export_include_dirs", + snapshotDir: nativeIncludeDir, + copy: true, + dirs: true, + }, + { + // ExportedSystemIncludeDirs lists directories that contains some system header files to + // be copied into a directory in the snapshot. The snapshot directories must be added to + // the export_system_include_dirs property in the prebuilt module in the snapshot. + pathsGetter: func(libInfo *nativeLibInfoProperties) android.Paths { return libInfo.ExportedSystemIncludeDirs }, + propertyName: "export_system_include_dirs", + snapshotDir: nativeIncludeDir, + copy: true, + dirs: true, + }, + { + // exportedGeneratedIncludeDirs lists directories that contains some header files + // that are explicitly listed in the exportedGeneratedHeaders property. So, the contents + // of these directories do not need to be copied, but these directories do need adding to + // the export_include_dirs property in the prebuilt module in the snapshot. + pathsGetter: func(libInfo *nativeLibInfoProperties) android.Paths { return libInfo.exportedGeneratedIncludeDirs }, + propertyName: "export_include_dirs", + snapshotDir: nativeGeneratedIncludeDir, + copy: false, + dirs: true, + }, + { + // exportedGeneratedHeaders lists header files that are in one of the directories + // specified in exportedGeneratedIncludeDirs must be copied into the snapshot. + // As they are in a directory in exportedGeneratedIncludeDirs they do not need adding to a + // property in the prebuilt module in the snapshot. + pathsGetter: func(libInfo *nativeLibInfoProperties) android.Paths { return libInfo.exportedGeneratedHeaders }, + propertyName: "", + snapshotDir: nativeGeneratedIncludeDir, + copy: true, + dirs: false, + }, +} + +// Add properties that may, or may not, be arch specific. +func addPossiblyArchSpecificProperties(sdkModuleContext android.ModuleContext, builder android.SnapshotBuilder, libInfo *nativeLibInfoProperties, outputProperties android.BpPropertySet) { + + // Copy the generated library to the snapshot and add a reference to it in the .bp module. + if libInfo.outputFile != nil { + nativeLibraryPath := nativeLibraryPathFor(libInfo) + builder.CopyToSnapshot(libInfo.outputFile, nativeLibraryPath) + outputProperties.AddProperty("srcs", []string{nativeLibraryPath}) + } + + if len(libInfo.SharedLibs) > 0 { + outputProperties.AddPropertyWithTag("shared_libs", libInfo.SharedLibs, builder.SdkMemberReferencePropertyTag(false)) + } + + // SystemSharedLibs needs to be propagated if it's a list, even if it's empty, + // so check for non-nil instead of nonzero length. + if libInfo.SystemSharedLibs != nil { + outputProperties.AddPropertyWithTag("system_shared_libs", libInfo.SystemSharedLibs, builder.SdkMemberReferencePropertyTag(false)) + } + + // Map from property name to the include dirs to add to the prebuilt module in the snapshot. + includeDirs := make(map[string][]string) + + // Iterate over each include directory property, copying files and collating property + // values where necessary. + for _, propertyInfo := range includeDirProperties { + // Calculate the base directory in the snapshot into which the files will be copied. + // lib.ArchType is "" for common properties. + targetDir := filepath.Join(libInfo.archType, propertyInfo.snapshotDir) + + propertyName := propertyInfo.propertyName + + // Iterate over each path in one of the include directory properties. + for _, path := range propertyInfo.pathsGetter(libInfo) { + + // Copy the files/directories when necessary. + if propertyInfo.copy { + if propertyInfo.dirs { + // When copying a directory glob and copy all the headers within it. + // TODO(jiyong) copy headers having other suffixes + headers, _ := sdkModuleContext.GlobWithDeps(path.String()+"/**/*.h", nil) + for _, file := range headers { + src := android.PathForSource(sdkModuleContext, file) + dest := filepath.Join(targetDir, file) + builder.CopyToSnapshot(src, dest) + } + } else { + // Otherwise, just copy the files. + dest := filepath.Join(targetDir, libInfo.name, path.Rel()) + builder.CopyToSnapshot(path, dest) + } + } + + // Only directories are added to a property. + if propertyInfo.dirs { + var snapshotPath string + if isGeneratedHeaderDirectory(path) { + snapshotPath = filepath.Join(targetDir, libInfo.name) + } else { + snapshotPath = filepath.Join(targetDir, path.String()) + } + + includeDirs[propertyName] = append(includeDirs[propertyName], snapshotPath) + } + } + } + + // Add the collated include dir properties to the output. + for property, dirs := range includeDirs { + outputProperties.AddProperty(property, dirs) + } + + if len(libInfo.StubsVersion) > 0 { + stubsSet := outputProperties.AddPropertySet("stubs") + stubsSet.AddProperty("versions", []string{libInfo.StubsVersion}) + } +} + +const ( + nativeIncludeDir = "include" + nativeGeneratedIncludeDir = "include_gen" + nativeStubDir = "lib" +) + +// path to the native library. Relative to <sdk_root>/<api_dir> +func nativeLibraryPathFor(lib *nativeLibInfoProperties) string { + return filepath.Join(lib.OsPrefix(), lib.archType, + nativeStubDir, lib.outputFile.Base()) +} + +// nativeLibInfoProperties represents properties of a native lib +// +// The exported (capitalized) fields will be examined and may be changed during common value extraction. +// The unexported fields will be left untouched. +type nativeLibInfoProperties struct { + android.SdkMemberPropertiesBase + + memberType *librarySdkMemberType + + // The name of the library, is not exported as this must not be changed during optimization. + name string + + // archType is not exported as if set (to a non default value) it is always arch specific. + // This is "" for common properties. + archType string + + // The list of possibly common exported include dirs. + // + // This field is exported as its contents may not be arch specific. + ExportedIncludeDirs android.Paths `android:"arch_variant"` + + // The list of arch specific exported generated include dirs. + // + // This field is not exported as its contents are always arch specific. + exportedGeneratedIncludeDirs android.Paths + + // The list of arch specific exported generated header files. + // + // This field is not exported as its contents are is always arch specific. + exportedGeneratedHeaders android.Paths + + // The list of possibly common exported system include dirs. + // + // This field is exported as its contents may not be arch specific. + ExportedSystemIncludeDirs android.Paths `android:"arch_variant"` + + // The list of possibly common exported flags. + // + // This field is exported as its contents may not be arch specific. + ExportedFlags []string `android:"arch_variant"` + + // The set of shared libraries + // + // This field is exported as its contents may not be arch specific. + SharedLibs []string `android:"arch_variant"` + + // The set of system shared libraries. Note nil and [] are semantically + // distinct - see BaseLinkerProperties.System_shared_libs. + // + // This field is exported as its contents may not be arch specific. + SystemSharedLibs []string `android:"arch_variant"` + + // The specific stubs version for the lib variant, or empty string if stubs + // are not in use. + // + // Marked 'ignored-on-host' as the StubsVersion() from which this is initialized is + // not set on host and the stubs.versions property which this is written to is does + // not vary by arch so cannot be android specific. + StubsVersion string `sdk:"ignored-on-host"` + + // outputFile is not exported as it is always arch specific. + outputFile android.Path +} + +func (p *nativeLibInfoProperties) PopulateFromVariant(ctx android.SdkMemberContext, variant android.Module) { + ccModule := variant.(*Module) + + // If the library has some link types then it produces an output binary file, otherwise it + // is header only. + if !p.memberType.noOutputFiles { + p.outputFile = getRequiredMemberOutputFile(ctx, ccModule) + } + + // Separate out the generated include dirs (which are arch specific) from the + // include dirs (which may not be). + exportedIncludeDirs, exportedGeneratedIncludeDirs := android.FilterPathListPredicate( + ccModule.ExportedIncludeDirs(), isGeneratedHeaderDirectory) + + p.name = variant.Name() + p.archType = ccModule.Target().Arch.ArchType.String() + + // Make sure that the include directories are unique. + p.ExportedIncludeDirs = android.FirstUniquePaths(exportedIncludeDirs) + p.exportedGeneratedIncludeDirs = android.FirstUniquePaths(exportedGeneratedIncludeDirs) + p.ExportedSystemIncludeDirs = android.FirstUniquePaths(ccModule.ExportedSystemIncludeDirs()) + + p.ExportedFlags = ccModule.ExportedFlags() + if ccModule.linker != nil { + specifiedDeps := specifiedDeps{} + specifiedDeps = ccModule.linker.linkerSpecifiedDeps(specifiedDeps) + + if !ccModule.HasStubsVariants() { + // Propagate dynamic dependencies for implementation libs, but not stubs. + p.SharedLibs = specifiedDeps.sharedLibs + } + p.SystemSharedLibs = specifiedDeps.systemSharedLibs + } + p.exportedGeneratedHeaders = ccModule.ExportedGeneratedHeaders() + + if ccModule.HasStubsVariants() { + p.StubsVersion = ccModule.StubsVersion() + } +} + +func getRequiredMemberOutputFile(ctx android.SdkMemberContext, ccModule *Module) android.Path { + var path android.Path + outputFile := ccModule.OutputFile() + if outputFile.Valid() { + path = outputFile.Path() + } else { + ctx.SdkModuleContext().ModuleErrorf("member variant %s does not have a valid output file", ccModule) + } + return path +} + +func (p *nativeLibInfoProperties) AddToPropertySet(ctx android.SdkMemberContext, propertySet android.BpPropertySet) { + addPossiblyArchSpecificProperties(ctx.SdkModuleContext(), ctx.SnapshotBuilder(), p, propertySet) +} diff --git a/cc/library_test.go b/cc/library_test.go index 859b05aff..cb167252d 100644 --- a/cc/library_test.go +++ b/cc/library_test.go @@ -17,6 +17,8 @@ package cc import ( "reflect" "testing" + + "android/soong/android" ) func TestLibraryReuse(t *testing.T) { @@ -24,23 +26,26 @@ func TestLibraryReuse(t *testing.T) { ctx := testCc(t, ` cc_library { name: "libfoo", - srcs: ["foo.c"], + srcs: ["foo.c", "baz.o"], }`) - libfooShared := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_shared").Rule("ld") - libfooStatic := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_static").Output("libfoo.a") + libfooShared := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_shared").Rule("ld") + libfooStatic := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_static").Output("libfoo.a") - if len(libfooShared.Inputs) != 1 { + if len(libfooShared.Inputs) != 2 { t.Fatalf("unexpected inputs to libfoo shared: %#v", libfooShared.Inputs.Strings()) } - if len(libfooStatic.Inputs) != 1 { + if len(libfooStatic.Inputs) != 2 { t.Fatalf("unexpected inputs to libfoo static: %#v", libfooStatic.Inputs.Strings()) } if libfooShared.Inputs[0] != libfooStatic.Inputs[0] { t.Errorf("static object not reused for shared library") } + if libfooShared.Inputs[1] != libfooStatic.Inputs[1] { + t.Errorf("static object not reused for shared library") + } }) t.Run("extra static source", func(t *testing.T) { @@ -53,8 +58,8 @@ func TestLibraryReuse(t *testing.T) { }, }`) - libfooShared := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_shared").Rule("ld") - libfooStatic := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_static").Output("libfoo.a") + libfooShared := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_shared").Rule("ld") + libfooStatic := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_static").Output("libfoo.a") if len(libfooShared.Inputs) != 1 { t.Fatalf("unexpected inputs to libfoo shared: %#v", libfooShared.Inputs.Strings()) @@ -79,8 +84,8 @@ func TestLibraryReuse(t *testing.T) { }, }`) - libfooShared := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_shared").Rule("ld") - libfooStatic := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_static").Output("libfoo.a") + libfooShared := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_shared").Rule("ld") + libfooStatic := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_static").Output("libfoo.a") if len(libfooShared.Inputs) != 2 { t.Fatalf("unexpected inputs to libfoo shared: %#v", libfooShared.Inputs.Strings()) @@ -105,8 +110,8 @@ func TestLibraryReuse(t *testing.T) { }, }`) - libfooShared := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_shared").Rule("ld") - libfooStatic := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_static").Output("libfoo.a") + libfooShared := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_shared").Rule("ld") + libfooStatic := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_static").Output("libfoo.a") if len(libfooShared.Inputs) != 1 { t.Fatalf("unexpected inputs to libfoo shared: %#v", libfooShared.Inputs.Strings()) @@ -131,8 +136,8 @@ func TestLibraryReuse(t *testing.T) { }, }`) - libfooShared := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_shared").Rule("ld") - libfooStatic := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_static").Output("libfoo.a") + libfooShared := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_shared").Rule("ld") + libfooStatic := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_static").Output("libfoo.a") if len(libfooShared.Inputs) != 1 { t.Fatalf("unexpected inputs to libfoo shared: %#v", libfooShared.Inputs.Strings()) @@ -162,8 +167,8 @@ func TestLibraryReuse(t *testing.T) { }, }`) - libfooShared := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_shared").Rule("ld") - libfooStatic := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_static").Output("libfoo.a") + libfooShared := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_shared").Rule("ld") + libfooStatic := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_static").Output("libfoo.a") if len(libfooShared.Inputs) != 3 { t.Fatalf("unexpected inputs to libfoo shared: %#v", libfooShared.Inputs.Strings()) @@ -177,9 +182,61 @@ func TestLibraryReuse(t *testing.T) { t.Errorf("static objects not reused for shared library") } - libfoo := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_shared").Module().(*Module) - if !inList("-DGOOGLE_PROTOBUF_NO_RTTI", libfoo.flags.CFlags) { + libfoo := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_shared").Module().(*Module) + if !inList("-DGOOGLE_PROTOBUF_NO_RTTI", libfoo.flags.Local.CFlags) { t.Errorf("missing protobuf cflags") } }) } + +func TestStubsVersions(t *testing.T) { + bp := ` + cc_library { + name: "libfoo", + srcs: ["foo.c"], + stubs: { + versions: ["29", "R", "10000"], + }, + } + ` + config := TestConfig(buildDir, android.Android, nil, bp, nil) + config.TestProductVariables.Platform_version_active_codenames = []string{"R"} + ctx := testCcWithConfig(t, config) + + variants := ctx.ModuleVariantsForTests("libfoo") + for _, expectedVer := range []string{"29", "9000", "10000"} { + expectedVariant := "android_arm_armv7-a-neon_shared_" + expectedVer + if !inList(expectedVariant, variants) { + t.Errorf("missing expected variant: %q", expectedVariant) + } + } +} + +func TestStubsVersions_NotSorted(t *testing.T) { + bp := ` + cc_library { + name: "libfoo", + srcs: ["foo.c"], + stubs: { + versions: ["29", "10000", "R"], + }, + } + ` + config := TestConfig(buildDir, android.Android, nil, bp, nil) + config.TestProductVariables.Platform_version_active_codenames = []string{"R"} + testCcErrorWithConfig(t, `"libfoo" .*: versions: not sorted`, config) +} + +func TestStubsVersions_ParseError(t *testing.T) { + bp := ` + cc_library { + name: "libfoo", + srcs: ["foo.c"], + stubs: { + versions: ["29", "10000", "X"], + }, + } + ` + + testCcError(t, `"libfoo" .*: versions: SDK version should be`, bp) +} diff --git a/cc/linkable.go b/cc/linkable.go new file mode 100644 index 000000000..4a70d48f7 --- /dev/null +++ b/cc/linkable.go @@ -0,0 +1,86 @@ +package cc + +import ( + "github.com/google/blueprint" + + "android/soong/android" +) + +type LinkableInterface interface { + Module() android.Module + CcLibrary() bool + CcLibraryInterface() bool + + OutputFile() android.OptionalPath + + IncludeDirs() android.Paths + SetDepsInLinkOrder([]android.Path) + GetDepsInLinkOrder() []android.Path + + HasStaticVariant() bool + GetStaticVariant() LinkableInterface + + NonCcVariants() bool + + StubsVersions() []string + BuildStubs() bool + SetBuildStubs() + SetStubsVersions(string) + StubsVersion() string + HasStubsVariants() bool + SelectedStl() string + ApiLevel() string + + BuildStaticVariant() bool + BuildSharedVariant() bool + SetStatic() + SetShared() + Static() bool + Shared() bool + Toc() android.OptionalPath + + Host() bool + + InRamdisk() bool + OnlyInRamdisk() bool + + InRecovery() bool + OnlyInRecovery() bool + + UseSdk() bool + UseVndk() bool + MustUseVendorVariant() bool + IsVndk() bool + HasVendorVariant() bool + + SdkVersion() string + AlwaysSdk() bool + + ToolchainLibrary() bool + NdkPrebuiltStl() bool + StubDecorator() bool +} + +type DependencyTag struct { + blueprint.BaseDependencyTag + Name string + Library bool + Shared bool + + ReexportFlags bool + + ExplicitlyVersioned bool + + FromStatic bool +} + +var ( + SharedDepTag = DependencyTag{Name: "shared", Library: true, Shared: true} + StaticDepTag = DependencyTag{Name: "static", Library: true} + + // Same as SharedDepTag, but from a static lib + SharedFromStaticDepTag = DependencyTag{Name: "shared from static", Library: true, Shared: true, FromStatic: true} + + CrtBeginDepTag = DependencyTag{Name: "crtbegin"} + CrtEndDepTag = DependencyTag{Name: "crtend"} +) diff --git a/cc/linker.go b/cc/linker.go index e063e441b..00992657c 100644 --- a/cc/linker.go +++ b/cc/linker.go @@ -57,9 +57,6 @@ type BaseLinkerProperties struct { // This flag should only be necessary for compiling low-level libraries like libc. Allow_undefined_symbols *bool `android:"arch_variant"` - // don't link in libgcc.a - No_libgcc *bool - // don't link in libclang_rt.builtins-*.a No_libcrt *bool `android:"arch_variant"` @@ -104,6 +101,10 @@ type BaseLinkerProperties struct { // variant of the C/C++ module. Shared_libs []string + // list of static libs that only should be used to build the vendor + // variant of the C/C++ module. + Static_libs []string + // list of shared libs that should not be used to build the vendor variant // of the C/C++ module. Exclude_shared_libs []string @@ -128,6 +129,10 @@ type BaseLinkerProperties struct { // variant of the C/C++ module. Shared_libs []string + // list of static libs that only should be used to build the recovery + // variant of the C/C++ module. + Static_libs []string + // list of shared libs that should not be used to build // the recovery variant of the C/C++ module. Exclude_shared_libs []string @@ -140,6 +145,26 @@ type BaseLinkerProperties struct { // of the C/C++ module. Exclude_header_libs []string } + Ramdisk struct { + // list of static libs that only should be used to build the recovery + // variant of the C/C++ module. + Static_libs []string + + // list of shared libs that should not be used to build + // the ramdisk variant of the C/C++ module. + Exclude_shared_libs []string + + // list of static libs that should not be used to build + // the ramdisk variant of the C/C++ module. + Exclude_static_libs []string + } + Platform struct { + // list of shared libs that should be use to build the platform variant + // of a module that sets sdk_version. This should rarely be necessary, + // in most cases the same libraries are available for the SDK and platform + // variants. + Shared_libs []string + } } // make android::build:GetBuildNumber() available containing the build ID. @@ -151,8 +176,11 @@ type BaseLinkerProperties struct { // local file name to pass to the linker as --version_script Version_script *string `android:"path,arch_variant"` - // Local file name to pass to the linker as --symbol-ordering-file - Symbol_ordering_file *string `android:"arch_variant"` + // list of static libs that should not be used to build this module + Exclude_static_libs []string `android:"arch_variant"` + + // list of shared libs that should not be used to build this module + Exclude_shared_libs []string `android:"arch_variant"` } func NewBaseLinker(sanitize *sanitize) *baseLinker { @@ -198,6 +226,10 @@ func (linker *baseLinker) linkerDeps(ctx DepsContext, deps Deps) Deps { deps.ReexportSharedLibHeaders = append(deps.ReexportSharedLibHeaders, linker.Properties.Export_shared_lib_headers...) deps.ReexportGeneratedHeaders = append(deps.ReexportGeneratedHeaders, linker.Properties.Export_generated_headers...) + deps.SharedLibs = removeListFromList(deps.SharedLibs, linker.Properties.Exclude_shared_libs) + deps.StaticLibs = removeListFromList(deps.StaticLibs, linker.Properties.Exclude_static_libs) + deps.WholeStaticLibs = removeListFromList(deps.WholeStaticLibs, linker.Properties.Exclude_static_libs) + if Bool(linker.Properties.Use_version_lib) { deps.WholeStaticLibs = append(deps.WholeStaticLibs, "libbuildversion") } @@ -206,6 +238,7 @@ func (linker *baseLinker) linkerDeps(ctx DepsContext, deps Deps) Deps { deps.SharedLibs = append(deps.SharedLibs, linker.Properties.Target.Vendor.Shared_libs...) deps.SharedLibs = removeListFromList(deps.SharedLibs, linker.Properties.Target.Vendor.Exclude_shared_libs) deps.ReexportSharedLibHeaders = removeListFromList(deps.ReexportSharedLibHeaders, linker.Properties.Target.Vendor.Exclude_shared_libs) + deps.StaticLibs = append(deps.StaticLibs, linker.Properties.Target.Vendor.Static_libs...) deps.StaticLibs = removeListFromList(deps.StaticLibs, linker.Properties.Target.Vendor.Exclude_static_libs) deps.HeaderLibs = removeListFromList(deps.HeaderLibs, linker.Properties.Target.Vendor.Exclude_header_libs) deps.ReexportStaticLibHeaders = removeListFromList(deps.ReexportStaticLibHeaders, linker.Properties.Target.Vendor.Exclude_static_libs) @@ -217,6 +250,7 @@ func (linker *baseLinker) linkerDeps(ctx DepsContext, deps Deps) Deps { deps.SharedLibs = append(deps.SharedLibs, linker.Properties.Target.Recovery.Shared_libs...) deps.SharedLibs = removeListFromList(deps.SharedLibs, linker.Properties.Target.Recovery.Exclude_shared_libs) deps.ReexportSharedLibHeaders = removeListFromList(deps.ReexportSharedLibHeaders, linker.Properties.Target.Recovery.Exclude_shared_libs) + deps.StaticLibs = append(deps.StaticLibs, linker.Properties.Target.Recovery.Static_libs...) deps.StaticLibs = removeListFromList(deps.StaticLibs, linker.Properties.Target.Recovery.Exclude_static_libs) deps.HeaderLibs = removeListFromList(deps.HeaderLibs, linker.Properties.Target.Recovery.Exclude_header_libs) deps.ReexportHeaderLibHeaders = removeListFromList(deps.ReexportHeaderLibHeaders, linker.Properties.Target.Recovery.Exclude_header_libs) @@ -224,15 +258,24 @@ func (linker *baseLinker) linkerDeps(ctx DepsContext, deps Deps) Deps { deps.WholeStaticLibs = removeListFromList(deps.WholeStaticLibs, linker.Properties.Target.Recovery.Exclude_static_libs) } + if ctx.inRamdisk() { + deps.SharedLibs = removeListFromList(deps.SharedLibs, linker.Properties.Target.Recovery.Exclude_shared_libs) + deps.ReexportSharedLibHeaders = removeListFromList(deps.ReexportSharedLibHeaders, linker.Properties.Target.Recovery.Exclude_shared_libs) + deps.StaticLibs = append(deps.StaticLibs, linker.Properties.Target.Recovery.Static_libs...) + deps.StaticLibs = removeListFromList(deps.StaticLibs, linker.Properties.Target.Recovery.Exclude_static_libs) + deps.ReexportStaticLibHeaders = removeListFromList(deps.ReexportStaticLibHeaders, linker.Properties.Target.Recovery.Exclude_static_libs) + deps.WholeStaticLibs = removeListFromList(deps.WholeStaticLibs, linker.Properties.Target.Recovery.Exclude_static_libs) + } + + if !ctx.useSdk() { + deps.SharedLibs = append(deps.SharedLibs, linker.Properties.Target.Platform.Shared_libs...) + } + if ctx.toolchain().Bionic() { - // libclang_rt.builtins, libgcc and libatomic have to be last on the command line + // libclang_rt.builtins and libatomic have to be last on the command line if !Bool(linker.Properties.No_libcrt) { deps.LateStaticLibs = append(deps.LateStaticLibs, config.BuiltinsRuntimeLibrary(ctx.toolchain())) deps.LateStaticLibs = append(deps.LateStaticLibs, "libatomic") - deps.LateStaticLibs = append(deps.LateStaticLibs, "libgcc_stripped") - } else if !Bool(linker.Properties.No_libgcc) { - deps.LateStaticLibs = append(deps.LateStaticLibs, "libatomic") - deps.LateStaticLibs = append(deps.LateStaticLibs, "libgcc") } systemSharedLibs := linker.Properties.System_shared_libs @@ -301,10 +344,6 @@ func (linker *baseLinker) useClangLld(ctx ModuleContext) bool { if ctx.Darwin() { return false } - // http://b/110800681 - lld cannot link Android's Windows modules yet. - if ctx.Windows() { - return false - } if linker.Properties.Use_clang_lld != nil { return Bool(linker.Properties.Use_clang_lld) } @@ -339,63 +378,70 @@ func (linker *baseLinker) linkerFlags(ctx ModuleContext, flags Flags) Flags { } if linker.useClangLld(ctx) { - flags.LdFlags = append(flags.LdFlags, fmt.Sprintf("${config.%sGlobalLldflags}", hod)) + flags.Global.LdFlags = append(flags.Global.LdFlags, fmt.Sprintf("${config.%sGlobalLldflags}", hod)) if !BoolDefault(linker.Properties.Pack_relocations, true) { - flags.LdFlags = append(flags.LdFlags, "-Wl,--pack-dyn-relocs=none") + flags.Global.LdFlags = append(flags.Global.LdFlags, "-Wl,--pack-dyn-relocs=none") } else if ctx.Device() { // The SHT_RELR relocations is only supported by API level >= 28. // Do not turn this on if older version NDK is used. if !ctx.useSdk() || CheckSdkVersionAtLeast(ctx, 28) { - flags.LdFlags = append(flags.LdFlags, "-Wl,--pack-dyn-relocs=android+relr") - flags.LdFlags = append(flags.LdFlags, "-Wl,--use-android-relr-tags") + flags.Global.LdFlags = append(flags.Global.LdFlags, + "-Wl,--pack-dyn-relocs=android+relr", + "-Wl,--use-android-relr-tags") + } else if CheckSdkVersionAtLeast(ctx, 23) { + flags.Global.LdFlags = append(flags.Global.LdFlags, "-Wl,--pack-dyn-relocs=android") } } } else { - flags.LdFlags = append(flags.LdFlags, fmt.Sprintf("${config.%sGlobalLdflags}", hod)) + flags.Global.LdFlags = append(flags.Global.LdFlags, fmt.Sprintf("${config.%sGlobalLdflags}", hod)) } if Bool(linker.Properties.Allow_undefined_symbols) { if ctx.Darwin() { // darwin defaults to treating undefined symbols as errors - flags.LdFlags = append(flags.LdFlags, "-Wl,-undefined,dynamic_lookup") + flags.Global.LdFlags = append(flags.Global.LdFlags, "-Wl,-undefined,dynamic_lookup") } - } else if !ctx.Darwin() { - flags.LdFlags = append(flags.LdFlags, "-Wl,--no-undefined") + } else if !ctx.Darwin() && !ctx.Windows() { + flags.Global.LdFlags = append(flags.Global.LdFlags, "-Wl,--no-undefined") } if linker.useClangLld(ctx) { - flags.LdFlags = append(flags.LdFlags, toolchain.ClangLldflags()) + flags.Global.LdFlags = append(flags.Global.LdFlags, toolchain.ClangLldflags()) } else { - flags.LdFlags = append(flags.LdFlags, toolchain.ClangLdflags()) + flags.Global.LdFlags = append(flags.Global.LdFlags, toolchain.ClangLdflags()) } if !ctx.toolchain().Bionic() && !ctx.Fuchsia() { CheckBadHostLdlibs(ctx, "host_ldlibs", linker.Properties.Host_ldlibs) - flags.LdFlags = append(flags.LdFlags, linker.Properties.Host_ldlibs...) + flags.Local.LdFlags = append(flags.Local.LdFlags, linker.Properties.Host_ldlibs...) if !ctx.Windows() { // Add -ldl, -lpthread, -lm and -lrt to host builds to match the default behavior of device // builds - flags.LdFlags = append(flags.LdFlags, + flags.Global.LdFlags = append(flags.Global.LdFlags, "-ldl", "-lpthread", "-lm", ) if !ctx.Darwin() { - flags.LdFlags = append(flags.LdFlags, "-lrt") + flags.Global.LdFlags = append(flags.Global.LdFlags, "-lrt") } } } if ctx.Fuchsia() { - flags.LdFlags = append(flags.LdFlags, "-lfdio", "-lzircon") + flags.Global.LdFlags = append(flags.Global.LdFlags, "-lfdio", "-lzircon") + } + + if ctx.toolchain().LibclangRuntimeLibraryArch() != "" { + flags.Global.LdFlags = append(flags.Global.LdFlags, "-Wl,--exclude-libs="+config.BuiltinsRuntimeLibrary(ctx.toolchain())+".a") } CheckBadLinkerFlags(ctx, "ldflags", linker.Properties.Ldflags) - flags.LdFlags = append(flags.LdFlags, proptools.NinjaAndShellEscapeList(linker.Properties.Ldflags)...) + flags.Local.LdFlags = append(flags.Local.LdFlags, proptools.NinjaAndShellEscapeList(linker.Properties.Ldflags)...) - if ctx.Host() { + if ctx.Host() && !ctx.Windows() { rpath_prefix := `\$$ORIGIN/` if ctx.Darwin() { rpath_prefix = "@loader_path/" @@ -403,7 +449,7 @@ func (linker *baseLinker) linkerFlags(ctx ModuleContext, flags Flags) Flags { if !ctx.static() { for _, rpath := range linker.dynamicProperties.RunPaths { - flags.LdFlags = append(flags.LdFlags, "-Wl,-rpath,"+rpath_prefix+rpath) + flags.Global.LdFlags = append(flags.Global.LdFlags, "-Wl,-rpath,"+rpath_prefix+rpath) } } } @@ -413,10 +459,10 @@ func (linker *baseLinker) linkerFlags(ctx ModuleContext, flags Flags) Flags { // to older devices requires the old style hash. Fortunately, we can build with both and // it'll work anywhere. // This is not currently supported on MIPS architectures. - flags.LdFlags = append(flags.LdFlags, "-Wl,--hash-style=both") + flags.Global.LdFlags = append(flags.Global.LdFlags, "-Wl,--hash-style=both") } - flags.LdFlags = append(flags.LdFlags, toolchain.ToolchainClangLdflags()) + flags.Global.LdFlags = append(flags.Global.LdFlags, toolchain.ToolchainClangLdflags()) if Bool(linker.Properties.Group_static_libs) { flags.GroupStaticLibs = true @@ -438,13 +484,13 @@ func (linker *baseLinker) linkerFlags(ctx ModuleContext, flags Flags) Flags { if ctx.Darwin() { ctx.PropertyErrorf("version_script", "Not supported on Darwin") } else { - flags.LdFlags = append(flags.LdFlags, + flags.Local.LdFlags = append(flags.Local.LdFlags, "-Wl,--version-script,"+versionScript.String()) flags.LdFlagsDeps = append(flags.LdFlagsDeps, versionScript.Path()) if linker.sanitize.isSanitizerEnabled(cfi) { cfiExportsMap := android.PathForSource(ctx, cfiExportsMapPath) - flags.LdFlags = append(flags.LdFlags, + flags.Local.LdFlags = append(flags.Local.LdFlags, "-Wl,--version-script,"+cfiExportsMap.String()) flags.LdFlagsDeps = append(flags.LdFlagsDeps, cfiExportsMap) } @@ -452,16 +498,6 @@ func (linker *baseLinker) linkerFlags(ctx ModuleContext, flags Flags) Flags { } } - if !linker.dynamicProperties.BuildStubs { - symbolOrderingFile := ctx.ExpandOptionalSource( - linker.Properties.Symbol_ordering_file, "Symbol_ordering_file") - if symbolOrderingFile.Valid() { - flags.LdFlags = append(flags.LdFlags, - "-Wl,--symbol-ordering-file,"+symbolOrderingFile.String()) - flags.LdFlagsDeps = append(flags.LdFlagsDeps, symbolOrderingFile.Path()) - } - } - return flags } @@ -470,6 +506,20 @@ func (linker *baseLinker) link(ctx ModuleContext, panic(fmt.Errorf("baseLinker doesn't know how to link")) } +func (linker *baseLinker) linkerSpecifiedDeps(specifiedDeps specifiedDeps) specifiedDeps { + specifiedDeps.sharedLibs = append(specifiedDeps.sharedLibs, linker.Properties.Shared_libs...) + + // Must distinguish nil and [] in system_shared_libs - ensure that [] in + // either input list doesn't come out as nil. + if specifiedDeps.systemSharedLibs == nil { + specifiedDeps.systemSharedLibs = linker.Properties.System_shared_libs + } else { + specifiedDeps.systemSharedLibs = append(specifiedDeps.systemSharedLibs, linker.Properties.System_shared_libs...) + } + + return specifiedDeps +} + // Injecting version symbols // Some host modules want a version number, but we don't want to rebuild it every time. Optionally add a step // after linking that injects a constant placeholder with the current version number. @@ -481,19 +531,47 @@ func init() { var injectVersionSymbol = pctx.AndroidStaticRule("injectVersionSymbol", blueprint.RuleParams{ Command: "$symbolInjectCmd -i $in -o $out -s soong_build_number " + - "-from 'SOONG BUILD NUMBER PLACEHOLDER' -v $buildNumberFromFile", + "-from 'SOONG BUILD NUMBER PLACEHOLDER' -v $$(cat $buildNumberFile)", CommandDeps: []string{"$symbolInjectCmd"}, }, - "buildNumberFromFile") + "buildNumberFile") func (linker *baseLinker) injectVersionSymbol(ctx ModuleContext, in android.Path, out android.WritablePath) { + buildNumberFile := ctx.Config().BuildNumberFile(ctx) ctx.Build(pctx, android.BuildParams{ Rule: injectVersionSymbol, Description: "inject version symbol", Input: in, Output: out, + OrderOnly: android.Paths{buildNumberFile}, + Args: map[string]string{ + "buildNumberFile": buildNumberFile.String(), + }, + }) +} + +// Rule to generate .bss symbol ordering file. + +var ( + _ = pctx.SourcePathVariable("genSortedBssSymbolsPath", "build/soong/scripts/gen_sorted_bss_symbols.sh") + gen_sorted_bss_symbols = pctx.AndroidStaticRule("gen_sorted_bss_symbols", + blueprint.RuleParams{ + Command: "CROSS_COMPILE=$crossCompile $genSortedBssSymbolsPath ${in} ${out}", + CommandDeps: []string{"$genSortedBssSymbolsPath", "${crossCompile}nm"}, + }, + "crossCompile") +) + +func (linker *baseLinker) sortBssSymbolsBySize(ctx ModuleContext, in android.Path, symbolOrderingFile android.ModuleOutPath, flags builderFlags) string { + crossCompile := gccCmd(flags.toolchain, "") + ctx.Build(pctx, android.BuildParams{ + Rule: gen_sorted_bss_symbols, + Description: "generate bss symbol order " + symbolOrderingFile.Base(), + Output: symbolOrderingFile, + Input: in, Args: map[string]string{ - "buildNumberFromFile": ctx.Config().BuildNumberFromFile(), + "crossCompile": crossCompile, }, }) + return "-Wl,--symbol-ordering-file," + symbolOrderingFile.String() } diff --git a/cc/llndk_library.go b/cc/llndk_library.go index 52c58eb78..7ff20f472 100644 --- a/cc/llndk_library.go +++ b/cc/llndk_library.go @@ -19,8 +19,14 @@ import ( "strings" "android/soong/android" + + "github.com/google/blueprint" ) +var llndkImplDep = struct { + blueprint.DependencyTag +}{} + var ( llndkLibrarySuffix = ".llndk" llndkHeadersSuffix = ".llndk" @@ -76,17 +82,15 @@ func (stub *llndkStubDecorator) compilerFlags(ctx ModuleContext, flags Flags, de } func (stub *llndkStubDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) Objects { - vndk_ver := ctx.DeviceConfig().VndkVersion() - if vndk_ver == "current" { - platform_vndk_ver := ctx.DeviceConfig().PlatformVndkVersion() - if !inList(platform_vndk_ver, ctx.Config().PlatformVersionCombinedCodenames()) { - vndk_ver = platform_vndk_ver - } - } else if vndk_ver == "" { - // For non-enforcing devices, use "current" - vndk_ver = "current" + vndkVer := ctx.Module().(*Module).VndkVersion() + if !inList(vndkVer, ctx.Config().PlatformVersionActiveCodenames()) || vndkVer == "" { + // For non-enforcing devices, vndkVer is empty. Use "current" in that case, too. + vndkVer = "current" + } + if stub.stubsVersion() != "" { + vndkVer = stub.stubsVersion() } - objs, versionScript := compileStubLibrary(ctx, flags, String(stub.Properties.Symbol_file), vndk_ver, "--vndk") + objs, versionScript := compileStubLibrary(ctx, flags, String(stub.Properties.Symbol_file), vndkVer, "--llndk") stub.versionScriptPath = versionScript return objs } @@ -133,7 +137,8 @@ func (stub *llndkStubDecorator) link(ctx ModuleContext, flags Flags, deps PathDe if !Bool(stub.Properties.Unversioned) { linkerScriptFlag := "-Wl,--version-script," + stub.versionScriptPath.String() - flags.LdFlags = append(flags.LdFlags, linkerScriptFlag) + flags.Local.LdFlags = append(flags.Local.LdFlags, linkerScriptFlag) + flags.LdFlagsDeps = append(flags.LdFlagsDeps, stub.versionScriptPath) } if len(stub.Properties.Export_preprocessed_headers) > 0 { @@ -144,20 +149,24 @@ func (stub *llndkStubDecorator) link(ctx ModuleContext, flags Flags, deps PathDe timestampFiles = append(timestampFiles, stub.processHeaders(ctx, dir, genHeaderOutDir)) } - includePrefix := "-I" if Bool(stub.Properties.Export_headers_as_system) { - includePrefix = "-isystem " + stub.reexportSystemDirs(genHeaderOutDir) + } else { + stub.reexportDirs(genHeaderOutDir) } - stub.reexportFlags([]string{includePrefix + genHeaderOutDir.String()}) - stub.reexportDeps(timestampFiles) + stub.reexportDeps(timestampFiles...) } if Bool(stub.Properties.Export_headers_as_system) { - stub.exportIncludes(ctx, "-isystem ") + stub.exportIncludesAsSystem(ctx) stub.libraryDecorator.flagExporter.Properties.Export_include_dirs = []string{} } + if stub.stubsVersion() != "" { + stub.reexportFlags("-D" + versioningMacroName(ctx.baseModuleName()) + "=" + stub.stubsVersion()) + } + return stub.libraryDecorator.link(ctx, flags, deps, objs) } @@ -176,7 +185,6 @@ func NewLLndkStubLibrary() *Module { libraryDecorator: library, } stub.Properties.Vendor_available = BoolPtr(true) - module.Properties.UseVndk = true module.compiler = stub module.linker = stub module.installer = nil @@ -190,6 +198,14 @@ func NewLLndkStubLibrary() *Module { return module } +// llndk_library creates a stub llndk shared library based on the provided +// version file. Example: +// +// llndk_library { +// name: "libfoo", +// symbol_file: "libfoo.map.txt", +// export_include_dirs: ["include_vndk"], +// } func LlndkLibraryFactory() android.Module { module := NewLLndkStubLibrary() android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibBoth) @@ -204,6 +220,8 @@ func (headers *llndkHeadersDecorator) Name(name string) string { return name + llndkHeadersSuffix } +// llndk_headers contains a set of c/c++ llndk headers files which are imported +// by other soongs cc modules. func llndkHeadersFactory() android.Module { module, library := NewLibrary(android.DeviceSupported) library.HeaderOnly() @@ -80,6 +80,12 @@ func (lto *lto) useClangLld(ctx BaseModuleContext) bool { } func (lto *lto) flags(ctx BaseModuleContext, flags Flags) Flags { + // TODO(b/131771163): Disable LTO when using explicit fuzzing configurations. + // LTO breaks fuzzer builds. + if inList("-fsanitize=fuzzer-no-link", flags.Local.CFlags) { + return flags + } + if lto.LTO() { var ltoFlag string if Bool(lto.Properties.Lto.Thin) { @@ -88,27 +94,28 @@ func (lto *lto) flags(ctx BaseModuleContext, flags Flags) Flags { ltoFlag = "-flto" } - flags.CFlags = append(flags.CFlags, ltoFlag) - flags.LdFlags = append(flags.LdFlags, ltoFlag) + flags.Local.CFlags = append(flags.Local.CFlags, ltoFlag) + flags.Local.LdFlags = append(flags.Local.LdFlags, ltoFlag) if ctx.Config().IsEnvTrue("USE_THINLTO_CACHE") && Bool(lto.Properties.Lto.Thin) && lto.useClangLld(ctx) { // Set appropriate ThinLTO cache policy cacheDirFormat := "-Wl,--thinlto-cache-dir=" cacheDir := android.PathForOutput(ctx, "thinlto-cache").String() - flags.LdFlags = append(flags.LdFlags, cacheDirFormat+cacheDir) + flags.Local.LdFlags = append(flags.Local.LdFlags, cacheDirFormat+cacheDir) // Limit the size of the ThinLTO cache to the lesser of 10% of available // disk space and 10GB. cachePolicyFormat := "-Wl,--thinlto-cache-policy=" policy := "cache_size=10%:cache_size_bytes=10g" - flags.LdFlags = append(flags.LdFlags, cachePolicyFormat+policy) + flags.Local.LdFlags = append(flags.Local.LdFlags, cachePolicyFormat+policy) } // If the module does not have a profile, be conservative and do not inline // or unroll loops during LTO, in order to prevent significant size bloat. if !ctx.isPgoCompile() { - flags.LdFlags = append(flags.LdFlags, "-Wl,-plugin-opt,-inline-threshold=0") - flags.LdFlags = append(flags.LdFlags, "-Wl,-plugin-opt,-unroll-threshold=0") + flags.Local.LdFlags = append(flags.Local.LdFlags, + "-Wl,-plugin-opt,-inline-threshold=0", + "-Wl,-plugin-opt,-unroll-threshold=0") } } return flags @@ -142,7 +149,7 @@ func ltoDepsMutator(mctx android.TopDownMutatorContext) { mctx.WalkDeps(func(dep android.Module, parent android.Module) bool { tag := mctx.OtherModuleDependencyTag(dep) switch tag { - case staticDepTag, staticExportDepTag, lateStaticDepTag, wholeStaticDepTag, objDepTag, reuseObjTag: + case StaticDepTag, staticExportDepTag, lateStaticDepTag, wholeStaticDepTag, objDepTag, reuseObjTag: if dep, ok := dep.(*Module); ok && dep.lto != nil && !dep.lto.Disabled() { if full && !Bool(dep.lto.Properties.Lto.Full) { diff --git a/cc/makevars.go b/cc/makevars.go index b03e170f3..0f9f4c1e1 100644 --- a/cc/makevars.go +++ b/cc/makevars.go @@ -63,7 +63,16 @@ func makeStringOfWarningAllowedProjects() string { } } +type notOnHostContext struct { +} + +func (c *notOnHostContext) Host() bool { + return false +} + func makeVarsProvider(ctx android.MakeVarsContext) { + vendorPublicLibraries := vendorPublicLibraries(ctx.Config()) + ctx.Strict("LLVM_RELEASE_VERSION", "${config.ClangShortVersion}") ctx.Strict("LLVM_PREBUILTS_VERSION", "${config.ClangVersion}") ctx.Strict("LLVM_PREBUILTS_BASE", "${config.ClangBase}") @@ -75,7 +84,6 @@ func makeVarsProvider(ctx android.MakeVarsContext) { ctx.Strict("LLVM_OBJCOPY", "${config.ClangBin}/llvm-objcopy") ctx.Strict("LLVM_STRIP", "${config.ClangBin}/llvm-strip") ctx.Strict("PATH_TO_CLANG_TIDY", "${config.ClangBin}/clang-tidy") - ctx.Strict("PATH_TO_CLANG_TIDY_SHELL", "${config.ClangTidyShellPath}") ctx.StrictSorted("CLANG_CONFIG_UNKNOWN_CFLAGS", strings.Join(config.ClangUnknownCflags, " ")) ctx.Strict("RS_LLVM_PREBUILTS_VERSION", "${config.RSClangVersion}") @@ -93,31 +101,12 @@ func makeVarsProvider(ctx android.MakeVarsContext) { ctx.Strict("BOARD_VNDK_VERSION", ctx.DeviceConfig().VndkVersion()) - ctx.Strict("VNDK_CORE_LIBRARIES", strings.Join(vndkCoreLibraries, " ")) - ctx.Strict("VNDK_SAMEPROCESS_LIBRARIES", strings.Join(vndkSpLibraries, " ")) - - // Make uses LLNDK_LIBRARIES to determine which libraries to install. - // HWASAN is only part of the LL-NDK in builds in which libc depends on HWASAN. - // Therefore, by removing the library here, we cause it to only be installed if libc - // depends on it. - installedLlndkLibraries := []string{} - for _, lib := range llndkLibraries { - if strings.HasPrefix(lib, "libclang_rt.hwasan-") { - continue - } - installedLlndkLibraries = append(installedLlndkLibraries, lib) - } - ctx.Strict("LLNDK_LIBRARIES", strings.Join(installedLlndkLibraries, " ")) - - ctx.Strict("VNDK_PRIVATE_LIBRARIES", strings.Join(vndkPrivateLibraries, " ")) - ctx.Strict("VNDK_USING_CORE_VARIANT_LIBRARIES", strings.Join(vndkUsingCoreVariantLibraries, " ")) - // Filter vendor_public_library that are exported to make exportedVendorPublicLibraries := []string{} ctx.VisitAllModules(func(module android.Module) { if ccModule, ok := module.(*Module); ok { baseName := ccModule.BaseModuleName() - if inList(baseName, vendorPublicLibraries) && module.ExportedToMake() { + if inList(baseName, *vendorPublicLibraries) && module.ExportedToMake() { if !inList(baseName, exportedVendorPublicLibraries) { exportedVendorPublicLibraries = append(exportedVendorPublicLibraries, baseName) } @@ -137,7 +126,6 @@ func makeVarsProvider(ctx android.MakeVarsContext) { ctx.Strict("ADDRESS_SANITIZER_CONFIG_EXTRA_CFLAGS", strings.Join(asanCflags, " ")) ctx.Strict("ADDRESS_SANITIZER_CONFIG_EXTRA_LDFLAGS", strings.Join(asanLdflags, " ")) - ctx.Strict("ADDRESS_SANITIZER_CONFIG_EXTRA_STATIC_LIBRARIES", strings.Join(asanLibs, " ")) ctx.Strict("HWADDRESS_SANITIZER_CONFIG_EXTRA_CFLAGS", strings.Join(hwasanCflags, " ")) ctx.Strict("HWADDRESS_SANITIZER_GLOBAL_OPTIONS", strings.Join(hwasanGlobalOptions, ",")) @@ -159,6 +147,9 @@ func makeVarsProvider(ctx android.MakeVarsContext) { ctx.Strict("WITH_TIDY_FLAGS", "${config.TidyWithTidyFlags}") ctx.Strict("AIDL_CPP", "${aidlCmd}") + ctx.Strict("ALLOWED_MANUAL_INTERFACE_PATHS", strings.Join(allowedManualInterfacePaths, " ")) + + ctx.Strict("M4", "${m4Cmd}") ctx.Strict("RS_GLOBAL_INCLUDES", "${config.RsGlobalIncludes}") @@ -247,9 +238,9 @@ func makeVarsToolchain(ctx android.MakeVarsContext, secondPrefix string, } clangPrefix := secondPrefix + "CLANG_" + typePrefix - clangExtras := "-target " + toolchain.ClangTriple() - clangExtras += " -B" + config.ToolPath(toolchain) + clangExtras := "-B" + config.ToolPath(toolchain) + ctx.Strict(clangPrefix+"TRIPLE", toolchain.ClangTriple()) ctx.Strict(clangPrefix+"GLOBAL_CFLAGS", strings.Join([]string{ toolchain.ClangCflags(), "${config.CommonClangGlobalCflags}", diff --git a/cc/ndk_headers.go b/cc/ndk_headers.go index 5e45c1ac3..5744bb285 100644 --- a/cc/ndk_headers.go +++ b/cc/ndk_headers.go @@ -16,7 +16,6 @@ package cc import ( "fmt" - "os" "path/filepath" "strings" @@ -48,7 +47,7 @@ func init() { } // Returns the NDK base include path for use with sdk_version current. Usable with -I. -func getCurrentIncludePath(ctx android.ModuleContext) android.OutputPath { +func getCurrentIncludePath(ctx android.ModuleContext) android.InstallPath { return getNdkSysrootBase(ctx).Join(ctx, "usr/include") } @@ -94,7 +93,7 @@ type headerModule struct { } func getHeaderInstallDir(ctx android.ModuleContext, header android.Path, from string, - to string) android.OutputPath { + to string) android.InstallPath { // Output path is the sysroot base + "usr/include" + to directory + directory component // of the file without the leading from directory stripped. // @@ -159,6 +158,16 @@ func (m *headerModule) GenerateAndroidBuildActions(ctx android.ModuleContext) { } } +// ndk_headers installs the sets of ndk headers defined in the srcs property +// to the sysroot base + "usr/include" + to directory + directory component. +// ndk_headers requires the license file to be specified. Example: +// +// Given: +// sysroot base = "ndk/sysroot" +// from = "include/foo" +// to = "bar" +// header = "include/foo/woodly/doodly.h" +// output path = "ndk/sysroot/usr/include/bar/woodly/doodly.h" func ndkHeadersFactory() android.Module { module := &headerModule{} module.AddProperties(&module.properties) @@ -245,16 +254,8 @@ func processHeadersWithVersioner(ctx android.ModuleContext, srcDir, outDir andro depsPath := android.PathForSource(ctx, "bionic/libc/versioner-dependencies") depsGlob := ctx.Glob(filepath.Join(depsPath.String(), "**/*"), nil) for i, path := range depsGlob { - fileInfo, err := os.Lstat(path.String()) - if err != nil { - ctx.ModuleErrorf("os.Lstat(%q) failed: %s", path.String, err) - } - if fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink { - dest, err := os.Readlink(path.String()) - if err != nil { - ctx.ModuleErrorf("os.Readlink(%q) failed: %s", - path.String, err) - } + if ctx.IsSymlink(path) { + dest := ctx.Readlink(path) // Additional .. to account for the symlink itself. depsGlob[i] = android.PathForSource( ctx, filepath.Clean(filepath.Join(path.String(), "..", dest))) @@ -278,6 +279,11 @@ func processHeadersWithVersioner(ctx android.ModuleContext, srcDir, outDir andro return timestampFile } +// versioned_ndk_headers preprocesses the headers with the bionic versioner: +// https://android.googlesource.com/platform/bionic/+/master/tools/versioner/README.md. +// Unlike the ndk_headers soong module, versioned_ndk_headers operates on a +// directory level specified in `from` property. This is only used to process +// the bionic/libc/include directory. func versionedNdkHeadersFactory() android.Module { module := &versionedHeaderModule{} @@ -360,6 +366,8 @@ func (m *preprocessedHeadersModule) GenerateAndroidBuildActions(ctx android.Modu } } +// preprocessed_ndk_headers preprocesses all the ndk headers listed in the srcs +// property by executing the command defined in the preprocessor property. func preprocessedNdkHeadersFactory() android.Module { module := &preprocessedHeadersModule{} diff --git a/cc/ndk_library.go b/cc/ndk_library.go index 7199467b6..119ca403d 100644 --- a/cc/ndk_library.go +++ b/cc/ndk_library.go @@ -38,9 +38,12 @@ var ( ndkLibrarySuffix = ".ndk" ndkPrebuiltSharedLibs = []string{ + "aaudio", + "amidi", "android", "binder_ndk", "c", + "camera2ndk", "dl", "EGL", "GLESv1_CM", @@ -49,7 +52,9 @@ var ( "jnigraphics", "log", "mediandk", + "nativewindow", "m", + "neuralnetworks", "OpenMAXAL", "OpenSLES", "stdc++", @@ -91,6 +96,8 @@ type libraryProperties struct { Unversioned_until *string // Private property for use by the mutator that splits per-API level. + // can be one of <number:sdk_version> or <codename> or "current" + // passed to "gen_stub_libs.py" as it is ApiLevel string `blueprint:"mutated"` // True if this API is not yet ready to be shipped in the NDK. It will be @@ -117,7 +124,7 @@ func intMax(a int, b int) int { } } -func normalizeNdkApiLevel(ctx android.BaseContext, apiLevel string, +func normalizeNdkApiLevel(ctx android.BaseModuleContext, apiLevel string, arch android.Arch) (string, error) { if apiLevel == "current" { @@ -163,7 +170,7 @@ func getFirstGeneratedVersion(firstSupportedVersion string, platformVersion int) return strconv.Atoi(firstSupportedVersion) } -func shouldUseVersionScript(ctx android.BaseContext, stub *stubDecorator) (bool, error) { +func shouldUseVersionScript(ctx android.BaseModuleContext, stub *stubDecorator) (bool, error) { // unversioned_until is normally empty, in which case we should use the version script. if String(stub.properties.Unversioned_until) == "" { return true, nil @@ -223,7 +230,7 @@ func generateStubApiVariants(mctx android.BottomUpMutatorContext, c *stubDecorat } } -func ndkApiMutator(mctx android.BottomUpMutatorContext) { +func NdkApiMutator(mctx android.BottomUpMutatorContext) { if m, ok := mctx.Module().(*Module); ok { if m.Enabled() { if compiler, ok := m.compiler.(*stubDecorator); ok { @@ -252,10 +259,11 @@ func (c *stubDecorator) compilerInit(ctx BaseModuleContext) { } func addStubLibraryCompilerFlags(flags Flags) Flags { - flags.CFlags = append(flags.CFlags, + flags.Global.CFlags = append(flags.Global.CFlags, // We're knowingly doing some otherwise unsightly things with builtin // functions here. We're just generating stub libraries, so ignore it. "-Wno-incompatible-library-redeclaration", + "-Wno-incomplete-setjmp-declaration", "-Wno-builtin-requires-header", "-Wno-invalid-noreturn", "-Wall", @@ -264,6 +272,10 @@ func addStubLibraryCompilerFlags(flags Flags) Flags { // (avoids the need to link an unwinder into a fake library). "-fno-unwind-tables", ) + // All symbols in the stubs library should be visible. + if inList("-fvisibility=hidden", flags.Local.CFlags) { + flags.Local.CFlags = append(flags.Local.CFlags, "-fvisibility=default") + } return flags } @@ -332,7 +344,8 @@ func (stub *stubDecorator) link(ctx ModuleContext, flags Flags, deps PathDeps, if useVersionScript { linkerScriptFlag := "-Wl,--version-script," + stub.versionScriptPath.String() - flags.LdFlags = append(flags.LdFlags, linkerScriptFlag) + flags.Local.LdFlags = append(flags.Local.LdFlags, linkerScriptFlag) + flags.LdFlagsDeps = append(flags.LdFlagsDeps, stub.versionScriptPath) } return stub.libraryDecorator.link(ctx, flags, deps, objs) @@ -372,13 +385,19 @@ func newStubLibrary() *Module { module.linker = stub module.installer = stub + module.Properties.AlwaysSdk = true + module.Properties.Sdk_version = StringPtr("current") + module.AddProperties(&stub.properties, &library.MutatedProperties) return module } -func ndkLibraryFactory() android.Module { +// ndk_library creates a stub library that exposes dummy implementation +// of functions and variables for use at build time only. +func NdkLibraryFactory() android.Module { module := newStubLibrary() android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibBoth) + module.ModuleBase.EnableNativeBridgeSupportByDefault() return module } diff --git a/cc/ndk_prebuilt.go b/cc/ndk_prebuilt.go index 2a7e6573a..c4d770837 100644 --- a/cc/ndk_prebuilt.go +++ b/cc/ndk_prebuilt.go @@ -23,9 +23,9 @@ import ( ) func init() { - android.RegisterModuleType("ndk_prebuilt_object", ndkPrebuiltObjectFactory) - android.RegisterModuleType("ndk_prebuilt_static_stl", ndkPrebuiltStaticStlFactory) - android.RegisterModuleType("ndk_prebuilt_shared_stl", ndkPrebuiltSharedStlFactory) + android.RegisterModuleType("ndk_prebuilt_object", NdkPrebuiltObjectFactory) + android.RegisterModuleType("ndk_prebuilt_static_stl", NdkPrebuiltStaticStlFactory) + android.RegisterModuleType("ndk_prebuilt_shared_stl", NdkPrebuiltSharedStlFactory) } // NDK prebuilt libraries. @@ -64,13 +64,20 @@ func (*ndkPrebuiltObjectLinker) linkerDeps(ctx DepsContext, deps Deps) Deps { return deps } -func ndkPrebuiltObjectFactory() android.Module { +// ndk_prebuilt_object exports a precompiled ndk object file for linking +// operations. Soong's module name format is ndk_<NAME>.o.<sdk_version> where +// the object is located under +// ./prebuilts/ndk/current/platforms/android-<sdk_version>/arch-$(HOST_ARCH)/usr/lib/<NAME>.o. +func NdkPrebuiltObjectFactory() android.Module { module := newBaseModule(android.DeviceSupported, android.MultilibBoth) + module.ModuleBase.EnableNativeBridgeSupportByDefault() module.linker = &ndkPrebuiltObjectLinker{ objectLinker: objectLinker{ baseLinker: NewBaseLinker(nil), }, } + module.Properties.AlwaysSdk = true + module.Properties.Sdk_version = StringPtr("current") module.Properties.HideFromMake = true return module.Init() } @@ -85,6 +92,11 @@ func (c *ndkPrebuiltObjectLinker) link(ctx ModuleContext, flags Flags, return ndkPrebuiltModuleToPath(ctx, flags.Toolchain, objectExtension, ctx.sdkVersion()) } +func (*ndkPrebuiltObjectLinker) availableFor(what string) bool { + // ndk prebuilt objects are available to everywhere + return true +} + type ndkPrebuiltStlLinker struct { *libraryDecorator } @@ -98,7 +110,16 @@ func (*ndkPrebuiltStlLinker) linkerDeps(ctx DepsContext, deps Deps) Deps { return deps } -func ndkPrebuiltSharedStlFactory() android.Module { +func (*ndkPrebuiltStlLinker) availableFor(what string) bool { + // ndk prebuilt objects are available to everywhere + return true +} + +// ndk_prebuilt_shared_stl exports a precompiled ndk shared standard template +// library (stl) library for linking operation. The soong's module name format +// is ndk_<NAME>.so where the library is located under +// ./prebuilts/ndk/current/sources/cxx-stl/llvm-libc++/libs/$(HOST_ARCH)/<NAME>.so. +func NdkPrebuiltSharedStlFactory() android.Module { module, library := NewLibrary(android.DeviceSupported) library.BuildOnlyShared() module.compiler = nil @@ -106,14 +127,17 @@ func ndkPrebuiltSharedStlFactory() android.Module { libraryDecorator: library, } module.installer = nil - minVersionString := "minimum" - noStlString := "none" - module.Properties.Sdk_version = &minVersionString - module.stl.Properties.Stl = &noStlString + module.Properties.Sdk_version = StringPtr("minimum") + module.Properties.AlwaysSdk = true + module.stl.Properties.Stl = StringPtr("none") return module.Init() } -func ndkPrebuiltStaticStlFactory() android.Module { +// ndk_prebuilt_static_stl exports a precompiled ndk static standard template +// library (stl) library for linking operation. The soong's module name format +// is ndk_<NAME>.a where the library is located under +// ./prebuilts/ndk/current/sources/cxx-stl/llvm-libc++/libs/$(HOST_ARCH)/<NAME>.a. +func NdkPrebuiltStaticStlFactory() android.Module { module, library := NewLibrary(android.DeviceSupported) library.BuildOnlyStatic() module.compiler = nil @@ -122,6 +146,10 @@ func ndkPrebuiltStaticStlFactory() android.Module { } module.installer = nil module.Properties.HideFromMake = true + module.Properties.AlwaysSdk = true + module.Properties.Sdk_version = StringPtr("current") + module.stl.Properties.Stl = StringPtr("none") + module.ModuleBase.EnableNativeBridgeSupportByDefault() return module.Init() } @@ -137,7 +165,7 @@ func (ndk *ndkPrebuiltStlLinker) link(ctx ModuleContext, flags Flags, ctx.ModuleErrorf("NDK prebuilt libraries must have an ndk_lib prefixed name") } - ndk.exportIncludes(ctx, "-isystem ") + ndk.exportIncludesAsSystem(ctx) libName := strings.TrimPrefix(ctx.ModuleName(), "ndk_") libExt := flags.Toolchain.ShlibSuffix() diff --git a/cc/ndk_sysroot.go b/cc/ndk_sysroot.go index e39bae559..56fd54b8a 100644 --- a/cc/ndk_sysroot.go +++ b/cc/ndk_sysroot.go @@ -58,7 +58,7 @@ import ( func init() { android.RegisterModuleType("ndk_headers", ndkHeadersFactory) - android.RegisterModuleType("ndk_library", ndkLibraryFactory) + android.RegisterModuleType("ndk_library", NdkLibraryFactory) android.RegisterModuleType("versioned_ndk_headers", versionedNdkHeadersFactory) android.RegisterModuleType("preprocessed_ndk_headers", preprocessedNdkHeadersFactory) android.RegisterSingletonType("ndk", NdkSingleton) @@ -66,12 +66,12 @@ func init() { pctx.Import("android/soong/android") } -func getNdkInstallBase(ctx android.PathContext) android.OutputPath { - return android.PathForOutput(ctx, "ndk") +func getNdkInstallBase(ctx android.PathContext) android.InstallPath { + return android.PathForNdkInstall(ctx) } // Returns the main install directory for the NDK sysroot. Usable with --sysroot. -func getNdkSysrootBase(ctx android.PathContext) android.OutputPath { +func getNdkSysrootBase(ctx android.PathContext) android.InstallPath { return getNdkInstallBase(ctx).Join(ctx, "sysroot") } diff --git a/cc/object.go b/cc/object.go index 2fefd3017..15a529e85 100644 --- a/cc/object.go +++ b/cc/object.go @@ -26,6 +26,16 @@ import ( func init() { android.RegisterModuleType("cc_object", ObjectFactory) + android.RegisterSdkMemberType(ccObjectSdkMemberType) +} + +var ccObjectSdkMemberType = &librarySdkMemberType{ + SdkMemberTypeBase: android.SdkMemberTypeBase{ + PropertyName: "native_objects", + SupportsSdk: true, + }, + prebuiltModuleType: "cc_prebuilt_object", + linkTypes: nil, } type objectLinker struct { @@ -33,20 +43,41 @@ type objectLinker struct { Properties ObjectLinkerProperties } +type ObjectLinkerProperties struct { + // list of modules that should only provide headers for this module. + Header_libs []string `android:"arch_variant,variant_prepend"` + + // names of other cc_object modules to link into this module using partial linking + Objs []string `android:"arch_variant"` + + // if set, add an extra objcopy --prefix-symbols= step + Prefix_symbols *string + + // if set, the path to a linker script to pass to ld -r when combining multiple object files. + Linker_script *string `android:"path,arch_variant"` +} + +func newObject() *Module { + module := newBaseModule(android.HostAndDeviceSupported, android.MultilibBoth) + module.sanitize = &sanitize{} + module.stl = &stl{} + return module +} + // cc_object runs the compiler without running the linker. It is rarely // necessary, but sometimes used to generate .s files from .c files to use as // input to a cc_genrule module. func ObjectFactory() android.Module { - module := newBaseModule(android.HostAndDeviceSupported, android.MultilibBoth) + module := newObject() module.linker = &objectLinker{ - baseLinker: NewBaseLinker(nil), + baseLinker: NewBaseLinker(module.sanitize), } module.compiler = NewBaseCompiler() // Clang's address-significance tables are incompatible with ld -r. module.compiler.appendCflags([]string{"-fno-addrsig"}) - module.stl = &stl{} + module.sdkMemberTypes = []android.SdkMemberType{ccObjectSdkMemberType} return module.Init() } @@ -66,13 +97,18 @@ func (object *objectLinker) linkerDeps(ctx DepsContext, deps Deps) Deps { deps.LateSharedLibs = append(deps.LateSharedLibs, "libc") } + deps.HeaderLibs = append(deps.HeaderLibs, object.Properties.Header_libs...) deps.ObjFiles = append(deps.ObjFiles, object.Properties.Objs...) return deps } -func (*objectLinker) linkerFlags(ctx ModuleContext, flags Flags) Flags { - flags.LdFlags = append(flags.LdFlags, ctx.toolchain().ToolchainClangLdflags()) +func (object *objectLinker) linkerFlags(ctx ModuleContext, flags Flags) Flags { + flags.Global.LdFlags = append(flags.Global.LdFlags, ctx.toolchain().ToolchainClangLdflags()) + if lds := android.OptionalPathForModuleSrc(ctx, object.Properties.Linker_script); lds.Valid() { + flags.Local.LdFlags = append(flags.Local.LdFlags, "-Wl,-T,"+lds.String()) + flags.LdFlagsDeps = append(flags.LdFlagsDeps, lds.Path()) + } return flags } @@ -84,7 +120,7 @@ func (object *objectLinker) link(ctx ModuleContext, var outputFile android.Path builderFlags := flagsToBuilderFlags(flags) - if len(objs.objFiles) == 1 { + if len(objs.objFiles) == 1 && String(object.Properties.Linker_script) == "" { outputFile = objs.objFiles[0] if String(object.Properties.Prefix_symbols) != "" { @@ -104,7 +140,7 @@ func (object *objectLinker) link(ctx ModuleContext, output = input } - TransformObjsToObj(ctx, objs.objFiles, builderFlags, output) + TransformObjsToObj(ctx, objs.objFiles, builderFlags, output, flags.LdFlagsDeps) } ctx.CheckbuildFile(outputFile) @@ -122,3 +158,7 @@ func (object *objectLinker) nativeCoverage() bool { func (object *objectLinker) coverageOutputFilePath() android.OptionalPath { return android.OptionalPath{} } + +func (object *objectLinker) object() bool { + return true +} diff --git a/cc/object_test.go b/cc/object_test.go new file mode 100644 index 000000000..6ff8a00c3 --- /dev/null +++ b/cc/object_test.go @@ -0,0 +1,31 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cc + +import ( + "testing" +) + +func TestLinkerScript(t *testing.T) { + t.Run("script", func(t *testing.T) { + testCc(t, ` + cc_object { + name: "foo", + srcs: ["baz.o"], + linker_script: "foo.lds", + }`) + }) + +} @@ -27,10 +27,9 @@ import ( var ( // Add flags to ignore warnings that profiles are old or missing for - // some functions, and turn on the experimental new pass manager. + // some functions. profileUseOtherFlags = []string{ "-Wno-backend-plugin", - "-fexperimental-new-pass-manager", } globalPgoProfileProjects = []string{ @@ -42,9 +41,9 @@ var ( var pgoProfileProjectsConfigKey = android.NewOnceKey("PgoProfileProjects") const profileInstrumentFlag = "-fprofile-generate=/data/local/tmp" -const profileSamplingFlag = "-gline-tables-only" +const profileSamplingFlag = "-gmlt -fdebug-info-for-profiling" const profileUseInstrumentFormat = "-fprofile-use=%s" -const profileUseSamplingFormat = "-fprofile-sample-use=%s" +const profileUseSamplingFormat = "-fprofile-sample-accurate -fprofile-sample-use=%s" func getPgoProfileProjects(config android.DeviceConfig) []string { return config.OnceStringSlice(pgoProfileProjectsConfigKey, func() []string { @@ -89,20 +88,21 @@ func (pgo *pgo) props() []interface{} { return []interface{}{&pgo.Properties} } -func (props *PgoProperties) addProfileGatherFlags(ctx ModuleContext, flags Flags) Flags { - flags.CFlags = append(flags.CFlags, props.Pgo.Cflags...) +func (props *PgoProperties) addInstrumentationProfileGatherFlags(ctx ModuleContext, flags Flags) Flags { + flags.Local.CFlags = append(flags.Local.CFlags, props.Pgo.Cflags...) - if props.isInstrumentation() { - flags.CFlags = append(flags.CFlags, profileInstrumentFlag) - // The profile runtime is added below in deps(). Add the below - // flag, which is the only other link-time action performed by - // the Clang driver during link. - flags.LdFlags = append(flags.LdFlags, "-u__llvm_profile_runtime") - } - if props.isSampling() { - flags.CFlags = append(flags.CFlags, profileSamplingFlag) - flags.LdFlags = append(flags.LdFlags, profileSamplingFlag) - } + flags.Local.CFlags = append(flags.Local.CFlags, profileInstrumentFlag) + // The profile runtime is added below in deps(). Add the below + // flag, which is the only other link-time action performed by + // the Clang driver during link. + flags.Local.LdFlags = append(flags.Local.LdFlags, "-u__llvm_profile_runtime") + return flags +} +func (props *PgoProperties) addSamplingProfileGatherFlags(ctx ModuleContext, flags Flags) Flags { + flags.Local.CFlags = append(flags.Local.CFlags, props.Pgo.Cflags...) + + flags.Local.CFlags = append(flags.Local.CFlags, profileSamplingFlag) + flags.Local.LdFlags = append(flags.Local.LdFlags, profileSamplingFlag) return flags } @@ -171,13 +171,17 @@ func (props *PgoProperties) addProfileUseFlags(ctx ModuleContext, flags Flags) F profileFilePath := profileFile.Path() profileUseFlags := props.profileUseFlags(ctx, profileFilePath.String()) - flags.CFlags = append(flags.CFlags, profileUseFlags...) - flags.LdFlags = append(flags.LdFlags, profileUseFlags...) + flags.Local.CFlags = append(flags.Local.CFlags, profileUseFlags...) + flags.Local.LdFlags = append(flags.Local.LdFlags, profileUseFlags...) // Update CFlagsDeps and LdFlagsDeps so the module is rebuilt // if profileFile gets updated flags.CFlagsDeps = append(flags.CFlagsDeps, profileFilePath) flags.LdFlagsDeps = append(flags.LdFlagsDeps, profileFilePath) + + if props.isSampling() { + flags.Local.LdFlags = append(flags.Local.LdFlags, "-Wl,-mllvm,-no-warn-sample-unused=true") + } } return flags } @@ -211,11 +215,6 @@ func (props *PgoProperties) isPGO(ctx BaseModuleContext) bool { ctx.ModuleErrorf("PGO specification is missing properties: " + missingProps) } - // Sampling not supported yet - if isSampling { - ctx.PropertyErrorf("pgo.sampling", "\"sampling\" is not supported yet)") - } - if isSampling && isInstrumentation { ctx.PropertyErrorf("pgo", "Exactly one of \"instrumentation\" and \"sampling\" properties must be set") } @@ -258,6 +257,12 @@ func (pgo *pgo) begin(ctx BaseModuleContext) { } } + // PGO profile use is not feasible for a Clang coverage build because + // -fprofile-use and -fprofile-instr-generate are incompatible. + if ctx.DeviceConfig().ClangCoverageEnabled() { + return + } + if !ctx.Config().IsEnvTrue("ANDROID_PGO_NO_PROFILE_USE") && proptools.BoolDefault(pgo.Properties.Pgo.Enable_profile_use, true) { if profileFile := pgo.Properties.getPgoProfileFile(ctx); profileFile.Valid() { @@ -282,8 +287,12 @@ func (pgo *pgo) flags(ctx ModuleContext, flags Flags) Flags { props := pgo.Properties // Add flags to profile this module based on its profile_kind - if props.ShouldProfileModule { - return props.addProfileGatherFlags(ctx, flags) + if props.ShouldProfileModule && props.isInstrumentation() { + return props.addInstrumentationProfileGatherFlags(ctx, flags) + } else if props.ShouldProfileModule && props.isSampling() { + return props.addSamplingProfileGatherFlags(ctx, flags) + } else if ctx.DeviceConfig().SamplingPGO() { + return props.addSamplingProfileGatherFlags(ctx, flags) } if !ctx.Config().IsEnvTrue("ANDROID_PGO_NO_PROFILE_USE") { diff --git a/cc/prebuilt.go b/cc/prebuilt.go index 48e466715..b7c0bf2fd 100644 --- a/cc/prebuilt.go +++ b/cc/prebuilt.go @@ -19,9 +19,15 @@ import ( ) func init() { - android.RegisterModuleType("cc_prebuilt_library_shared", prebuiltSharedLibraryFactory) - android.RegisterModuleType("cc_prebuilt_library_static", prebuiltStaticLibraryFactory) - android.RegisterModuleType("cc_prebuilt_binary", prebuiltBinaryFactory) + RegisterPrebuiltBuildComponents(android.InitRegistrationContext) +} + +func RegisterPrebuiltBuildComponents(ctx android.RegistrationContext) { + ctx.RegisterModuleType("cc_prebuilt_library", PrebuiltLibraryFactory) + ctx.RegisterModuleType("cc_prebuilt_library_shared", PrebuiltSharedLibraryFactory) + ctx.RegisterModuleType("cc_prebuilt_library_static", PrebuiltStaticLibraryFactory) + ctx.RegisterModuleType("cc_prebuilt_object", prebuiltObjectFactory) + ctx.RegisterModuleType("cc_prebuilt_binary", prebuiltBinaryFactory) } type prebuiltLinkerInterface interface { @@ -83,22 +89,32 @@ func (p *prebuiltLibraryLinker) linkerProps() []interface{} { func (p *prebuiltLibraryLinker) link(ctx ModuleContext, flags Flags, deps PathDeps, objs Objects) android.Path { - // TODO(ccross): verify shared library dependencies - if len(p.properties.Srcs) > 0 { - p.libraryDecorator.exportIncludes(ctx, "-I") - p.libraryDecorator.reexportFlags(deps.ReexportedFlags) - p.libraryDecorator.reexportDeps(deps.ReexportedFlagsDeps) + p.libraryDecorator.exportIncludes(ctx) + p.libraryDecorator.reexportDirs(deps.ReexportedDirs...) + p.libraryDecorator.reexportSystemDirs(deps.ReexportedSystemDirs...) + p.libraryDecorator.reexportFlags(deps.ReexportedFlags...) + p.libraryDecorator.reexportDeps(deps.ReexportedDeps...) + p.libraryDecorator.addExportedGeneratedHeaders(deps.ReexportedGeneratedHeaders...) + + // TODO(ccross): verify shared library dependencies + srcs := p.prebuiltSrcs() + if len(srcs) > 0 { builderFlags := flagsToBuilderFlags(flags) - in := p.Prebuilt.SingleSourcePath(ctx) + if len(srcs) > 1 { + ctx.PropertyErrorf("srcs", "multiple prebuilt source files") + return nil + } + + in := android.PathForModuleSrc(ctx, srcs[0]) if p.shared() { p.unstrippedOutputFile = in - libName := ctx.baseModuleName() + flags.Toolchain.ShlibSuffix() + libName := p.libraryDecorator.getLibName(ctx) + flags.Toolchain.ShlibSuffix() if p.needsStrip(ctx) { stripped := android.PathForModuleOut(ctx, "stripped", libName) - p.strip(ctx, in, stripped, builderFlags) + p.stripExecutableOrSharedLib(ctx, in, stripped, builderFlags) in = stripped } @@ -115,6 +131,18 @@ func (p *prebuiltLibraryLinker) link(ctx ModuleContext, return nil } +func (p *prebuiltLibraryLinker) prebuiltSrcs() []string { + srcs := p.properties.Srcs + if p.static() { + srcs = append(srcs, p.libraryDecorator.StaticProperties.Static.Srcs...) + } + if p.shared() { + srcs = append(srcs, p.libraryDecorator.SharedProperties.Shared.Srcs...) + } + + return srcs +} + func (p *prebuiltLibraryLinker) shared() bool { return p.libraryDecorator.shared() } @@ -127,28 +155,56 @@ func (p *prebuiltLibraryLinker) disablePrebuilt() { p.properties.Srcs = nil } -// cc_prebuilt_library_shared installs a precompiled shared library that are -// listed in the srcs property in the device's directory. -func prebuiltSharedLibraryFactory() android.Module { - module, _ := NewPrebuiltSharedLibrary(android.HostAndDeviceSupported) - return module.Init() +func (p *prebuiltLibraryLinker) skipInstall(mod *Module) { + mod.ModuleBase.SkipInstall() } -func NewPrebuiltSharedLibrary(hod android.HostOrDeviceSupported) (*Module, *libraryDecorator) { +func NewPrebuiltLibrary(hod android.HostOrDeviceSupported) (*Module, *libraryDecorator) { module, library := NewLibrary(hod) - library.BuildOnlyShared() module.compiler = nil prebuilt := &prebuiltLibraryLinker{ libraryDecorator: library, } module.linker = prebuilt + module.installer = prebuilt module.AddProperties(&prebuilt.properties) - android.InitPrebuiltModule(module, &prebuilt.properties.Srcs) + srcsSupplier := func() []string { + return prebuilt.prebuiltSrcs() + } + + android.InitPrebuiltModuleWithSrcSupplier(module, srcsSupplier, "srcs") + + // Prebuilt libraries can be used in SDKs. + android.InitSdkAwareModule(module) + return module, library +} - // Prebuilt libraries can be included in APEXes +// cc_prebuilt_library installs a precompiled shared library that are +// listed in the srcs property in the device's directory. +func PrebuiltLibraryFactory() android.Module { + module, _ := NewPrebuiltLibrary(android.HostAndDeviceSupported) + + // Prebuilt shared libraries can be included in APEXes + android.InitApexModule(module) + + return module.Init() +} + +// cc_prebuilt_library_shared installs a precompiled shared library that are +// listed in the srcs property in the device's directory. +func PrebuiltSharedLibraryFactory() android.Module { + module, _ := NewPrebuiltSharedLibrary(android.HostAndDeviceSupported) + return module.Init() +} + +func NewPrebuiltSharedLibrary(hod android.HostOrDeviceSupported) (*Module, *libraryDecorator) { + module, library := NewPrebuiltLibrary(hod) + library.BuildOnlyShared() + + // Prebuilt shared libraries can be included in APEXes android.InitApexModule(module) return module, library @@ -156,25 +212,63 @@ func NewPrebuiltSharedLibrary(hod android.HostOrDeviceSupported) (*Module, *libr // cc_prebuilt_library_static installs a precompiled static library that are // listed in the srcs property in the device's directory. -func prebuiltStaticLibraryFactory() android.Module { +func PrebuiltStaticLibraryFactory() android.Module { module, _ := NewPrebuiltStaticLibrary(android.HostAndDeviceSupported) return module.Init() } func NewPrebuiltStaticLibrary(hod android.HostOrDeviceSupported) (*Module, *libraryDecorator) { - module, library := NewLibrary(hod) + module, library := NewPrebuiltLibrary(hod) library.BuildOnlyStatic() - module.compiler = nil + return module, library +} - prebuilt := &prebuiltLibraryLinker{ - libraryDecorator: library, +type prebuiltObjectProperties struct { + Srcs []string `android:"path,arch_variant"` +} + +type prebuiltObjectLinker struct { + android.Prebuilt + objectLinker + + properties prebuiltObjectProperties +} + +func (p *prebuiltObjectLinker) prebuilt() *android.Prebuilt { + return &p.Prebuilt +} + +var _ prebuiltLinkerInterface = (*prebuiltObjectLinker)(nil) + +func (p *prebuiltObjectLinker) link(ctx ModuleContext, + flags Flags, deps PathDeps, objs Objects) android.Path { + if len(p.properties.Srcs) > 0 { + return p.Prebuilt.SingleSourcePath(ctx) } - module.linker = prebuilt + return nil +} - module.AddProperties(&prebuilt.properties) +func (p *prebuiltObjectLinker) object() bool { + return true +} +func newPrebuiltObject() *Module { + module := newObject() + prebuilt := &prebuiltObjectLinker{ + objectLinker: objectLinker{ + baseLinker: NewBaseLinker(nil), + }, + } + module.linker = prebuilt + module.AddProperties(&prebuilt.properties) android.InitPrebuiltModule(module, &prebuilt.properties.Srcs) - return module, library + android.InitSdkAwareModule(module) + return module +} + +func prebuiltObjectFactory() android.Module { + module := newPrebuiltObject() + return module.Init() } type prebuiltBinaryLinker struct { @@ -197,7 +291,7 @@ func (p *prebuiltBinaryLinker) link(ctx ModuleContext, if p.needsStrip(ctx) { stripped := android.PathForModuleOut(ctx, "stripped", fileName) - p.strip(ctx, in, stripped, builderFlags) + p.stripExecutableOrSharedLib(ctx, in, stripped, builderFlags) in = stripped } @@ -216,6 +310,10 @@ func (p *prebuiltBinaryLinker) link(ctx ModuleContext, return nil } +func (p *prebuiltBinaryLinker) binary() bool { + return true +} + // cc_prebuilt_binary installs a precompiled executable in srcs property in the // device's directory. func prebuiltBinaryFactory() android.Module { diff --git a/cc/prebuilt_test.go b/cc/prebuilt_test.go index 7cc265141..242d835e9 100644 --- a/cc/prebuilt_test.go +++ b/cc/prebuilt_test.go @@ -59,43 +59,49 @@ func TestPrebuilt(t *testing.T) { name: "libe", srcs: ["libe.a"], } - ` - - fs := map[string][]byte{ - "liba.so": nil, - "libb.a": nil, - "libd.so": nil, - "libe.a": nil, - } - - config := android.TestArchConfig(buildDir, nil) - ctx := createTestContext(t, config, bp, fs, android.Android) + cc_library { + name: "libf", + } - ctx.RegisterModuleType("cc_prebuilt_library_shared", android.ModuleFactoryAdaptor(prebuiltSharedLibraryFactory)) - ctx.RegisterModuleType("cc_prebuilt_library_static", android.ModuleFactoryAdaptor(prebuiltStaticLibraryFactory)) - ctx.RegisterModuleType("cc_prebuilt_binary", android.ModuleFactoryAdaptor(prebuiltBinaryFactory)) + cc_prebuilt_library { + name: "libf", + static: { + srcs: ["libf.a"], + }, + shared: { + srcs: ["libf.so"], + }, + } - ctx.PreArchMutators(android.RegisterPrebuiltsPreArchMutators) - ctx.PostDepsMutators(android.RegisterPrebuiltsPostDepsMutators) + cc_object { + name: "crtx", + } - ctx.Register() + cc_prebuilt_object { + name: "crtx", + srcs: ["crtx.o"], + } + ` - _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) - android.FailIfErrored(t, errs) - _, errs = ctx.PrepareBuildActions(config) - android.FailIfErrored(t, errs) + ctx := testPrebuilt(t, bp) // Verify that all the modules exist and that their dependencies were connected correctly - liba := ctx.ModuleForTests("liba", "android_arm64_armv8-a_core_shared").Module() - libb := ctx.ModuleForTests("libb", "android_arm64_armv8-a_core_static").Module() - libd := ctx.ModuleForTests("libd", "android_arm64_armv8-a_core_shared").Module() - libe := ctx.ModuleForTests("libe", "android_arm64_armv8-a_core_static").Module() - - prebuiltLiba := ctx.ModuleForTests("prebuilt_liba", "android_arm64_armv8-a_core_shared").Module() - prebuiltLibb := ctx.ModuleForTests("prebuilt_libb", "android_arm64_armv8-a_core_static").Module() - prebuiltLibd := ctx.ModuleForTests("prebuilt_libd", "android_arm64_armv8-a_core_shared").Module() - prebuiltLibe := ctx.ModuleForTests("prebuilt_libe", "android_arm64_armv8-a_core_static").Module() + liba := ctx.ModuleForTests("liba", "android_arm64_armv8-a_shared").Module() + libb := ctx.ModuleForTests("libb", "android_arm64_armv8-a_static").Module() + libd := ctx.ModuleForTests("libd", "android_arm64_armv8-a_shared").Module() + libe := ctx.ModuleForTests("libe", "android_arm64_armv8-a_static").Module() + libfStatic := ctx.ModuleForTests("libf", "android_arm64_armv8-a_static").Module() + libfShared := ctx.ModuleForTests("libf", "android_arm64_armv8-a_shared").Module() + crtx := ctx.ModuleForTests("crtx", "android_arm64_armv8-a").Module() + + prebuiltLiba := ctx.ModuleForTests("prebuilt_liba", "android_arm64_armv8-a_shared").Module() + prebuiltLibb := ctx.ModuleForTests("prebuilt_libb", "android_arm64_armv8-a_static").Module() + prebuiltLibd := ctx.ModuleForTests("prebuilt_libd", "android_arm64_armv8-a_shared").Module() + prebuiltLibe := ctx.ModuleForTests("prebuilt_libe", "android_arm64_armv8-a_static").Module() + prebuiltLibfStatic := ctx.ModuleForTests("prebuilt_libf", "android_arm64_armv8-a_static").Module() + prebuiltLibfShared := ctx.ModuleForTests("prebuilt_libf", "android_arm64_armv8-a_shared").Module() + prebuiltCrtx := ctx.ModuleForTests("prebuilt_crtx", "android_arm64_armv8-a").Module() hasDep := func(m android.Module, wantDep android.Module) bool { t.Helper() @@ -123,4 +129,95 @@ func TestPrebuilt(t *testing.T) { if !hasDep(libe, prebuiltLibe) { t.Errorf("libe missing dependency on prebuilt_libe") } + + if !hasDep(libfStatic, prebuiltLibfStatic) { + t.Errorf("libf static missing dependency on prebuilt_libf") + } + + if !hasDep(libfShared, prebuiltLibfShared) { + t.Errorf("libf shared missing dependency on prebuilt_libf") + } + + if !hasDep(crtx, prebuiltCrtx) { + t.Errorf("crtx missing dependency on prebuilt_crtx") + } +} + +func testPrebuilt(t *testing.T, bp string) *android.TestContext { + + fs := map[string][]byte{ + "liba.so": nil, + "libb.a": nil, + "libd.so": nil, + "libe.a": nil, + "libf.a": nil, + "libf.so": nil, + "crtx.o": nil, + } + config := TestConfig(buildDir, android.Android, nil, bp, fs) + ctx := CreateTestContext() + + // Enable androidmk support. + // * Register the singleton + // * Configure that we are inside make + // * Add CommonOS to ensure that androidmk processing works. + android.RegisterAndroidMkBuildComponents(ctx) + android.SetInMakeForTests(config) + + ctx.Register(config) + _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) + android.FailIfErrored(t, errs) + _, errs = ctx.PrepareBuildActions(config) + android.FailIfErrored(t, errs) + return ctx +} + +func TestPrebuiltLibraryShared(t *testing.T) { + ctx := testPrebuilt(t, ` + cc_prebuilt_library_shared { + name: "libtest", + srcs: ["libf.so"], + strip: { + none: true, + }, + } + `) + + shared := ctx.ModuleForTests("libtest", "android_arm64_armv8-a_shared").Module().(*Module) + assertString(t, shared.OutputFile().String(), "libf.so") +} + +func TestPrebuiltLibraryStatic(t *testing.T) { + ctx := testPrebuilt(t, ` + cc_prebuilt_library_static { + name: "libtest", + srcs: ["libf.a"], + } + `) + + static := ctx.ModuleForTests("libtest", "android_arm64_armv8-a_static").Module().(*Module) + assertString(t, static.OutputFile().String(), "libf.a") +} + +func TestPrebuiltLibrary(t *testing.T) { + ctx := testPrebuilt(t, ` + cc_prebuilt_library { + name: "libtest", + static: { + srcs: ["libf.a"], + }, + shared: { + srcs: ["libf.so"], + }, + strip: { + none: true, + }, + } + `) + + shared := ctx.ModuleForTests("libtest", "android_arm64_armv8-a_shared").Module().(*Module) + assertString(t, shared.OutputFile().String(), "libf.so") + + static := ctx.ModuleForTests("libtest", "android_arm64_armv8-a_static").Module().(*Module) + assertString(t, static.OutputFile().String(), "libf.a") } diff --git a/cc/proto.go b/cc/proto.go index f818edc95..ae988ec4d 100644 --- a/cc/proto.go +++ b/cc/proto.go @@ -114,13 +114,13 @@ func protoDeps(ctx DepsContext, deps Deps, p *android.ProtoProperties, static bo } func protoFlags(ctx ModuleContext, flags Flags, p *android.ProtoProperties) Flags { - flags.CFlags = append(flags.CFlags, "-DGOOGLE_PROTOBUF_NO_RTTI") + flags.Local.CFlags = append(flags.Local.CFlags, "-DGOOGLE_PROTOBUF_NO_RTTI") flags.proto = android.GetProtoFlags(ctx, p) if flags.proto.CanonicalPathFromRoot { - flags.GlobalFlags = append(flags.GlobalFlags, "-I"+flags.proto.SubDir.String()) + flags.Local.CommonFlags = append(flags.Local.CommonFlags, "-I"+flags.proto.SubDir.String()) } - flags.GlobalFlags = append(flags.GlobalFlags, "-I"+flags.proto.Dir.String()) + flags.Local.CommonFlags = append(flags.Local.CommonFlags, "-I"+flags.proto.Dir.String()) if String(p.Proto.Plugin) == "" { var plugin string diff --git a/cc/proto_test.go b/cc/proto_test.go index a7fcef935..f8bbd2643 100644 --- a/cc/proto_test.go +++ b/cc/proto_test.go @@ -15,7 +15,6 @@ package cc import ( - "runtime" "strings" "testing" @@ -30,7 +29,7 @@ func TestProto(t *testing.T) { srcs: ["a.proto"], }`) - proto := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_shared").Output("proto/a.pb.cc") + proto := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_shared").Output("proto/a.pb.cc") if cmd := proto.RuleParams.Command; !strings.Contains(cmd, "--cpp_out=") { t.Errorf("expected '--cpp_out' in %q", cmd) @@ -38,9 +37,6 @@ func TestProto(t *testing.T) { }) t.Run("plugin", func(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("TODO(b/129763458): cc_binary_host tests fail on mac when trying to exec xcrun") - } ctx := testCc(t, ` cc_binary_host { name: "protoc-gen-foobar", @@ -57,7 +53,7 @@ func TestProto(t *testing.T) { buildOS := android.BuildOs.String() - proto := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_shared").Output("proto/a.pb.cc") + proto := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_shared").Output("proto/a.pb.cc") foobar := ctx.ModuleForTests("protoc-gen-foobar", buildOS+"_x86_64") cmd := proto.RuleParams.Command @@ -29,7 +29,7 @@ func init() { // Use RenderScript prebuilts for unbundled builds but not PDK builds return filepath.Join("prebuilts/sdk/tools", runtime.GOOS, "bin/llvm-rs-cc") } else { - return pctx.HostBinToolPath(ctx, "llvm-rs-cc").String() + return ctx.Config().HostToolPath(ctx, "llvm-rs-cc").String() } }) } @@ -51,7 +51,7 @@ var ( "depFiles", "outDir", "rsFlags", "stampFile") ) -// Takes a path to a .rs or .fs file, and returns a path to a generated ScriptC_*.cpp file +// Takes a path to a .rscript or .fs file, and returns a path to a generated ScriptC_*.cpp file // This has to match the logic in llvm-rs-cc in DetermineOutputFile. func rsGeneratedCppFile(ctx android.ModuleContext, rsFile android.Path) android.WritablePath { fileName := strings.TrimSuffix(rsFile.Base(), rsFile.Ext()) @@ -72,11 +72,12 @@ func rsGenerateCpp(ctx android.ModuleContext, rsFiles android.Paths, rsFlags str stampFile := android.PathForModuleGen(ctx, "rs", "rs.stamp") depFiles := make(android.WritablePaths, 0, len(rsFiles)) genFiles := make(android.WritablePaths, 0, 2*len(rsFiles)) + headers := make(android.Paths, 0, len(rsFiles)) for _, rsFile := range rsFiles { depFiles = append(depFiles, rsGeneratedDepFile(ctx, rsFile)) - genFiles = append(genFiles, - rsGeneratedCppFile(ctx, rsFile), - rsGeneratedHFile(ctx, rsFile)) + headerFile := rsGeneratedHFile(ctx, rsFile) + genFiles = append(genFiles, rsGeneratedCppFile(ctx, rsFile), headerFile) + headers = append(headers, headerFile) } ctx.Build(pctx, android.BuildParams{ @@ -92,7 +93,7 @@ func rsGenerateCpp(ctx android.ModuleContext, rsFiles android.Paths, rsFlags str }, }) - return android.Paths{stampFile} + return headers } func rsFlags(ctx ModuleContext, flags Flags, properties *BaseCompilerProperties) Flags { @@ -122,7 +123,7 @@ func rsFlags(ctx ModuleContext, flags Flags, properties *BaseCompilerProperties) rootRsIncludeDirs := android.PathsForSource(ctx, properties.Renderscript.Include_dirs) flags.rsFlags = append(flags.rsFlags, includeDirsToFlags(rootRsIncludeDirs)) - flags.GlobalFlags = append(flags.GlobalFlags, + flags.Local.CommonFlags = append(flags.Local.CommonFlags, "-I"+android.PathForModuleGen(ctx, "rs").String(), "-Iframeworks/rs", "-Iframeworks/rs/cpp", diff --git a/cc/sabi.go b/cc/sabi.go index 4a8649945..8cef1700c 100644 --- a/cc/sabi.go +++ b/cc/sabi.go @@ -28,8 +28,8 @@ var ( ) type SAbiProperties struct { - CreateSAbiDumps bool `blueprint:"mutated"` - ReexportedIncludeFlags []string + CreateSAbiDumps bool `blueprint:"mutated"` + ReexportedIncludes []string `blueprint:"mutated"` } type sabi struct { @@ -70,20 +70,22 @@ func filterOutWithPrefix(list []string, filter []string) (remainder []string) { func (sabimod *sabi) flags(ctx ModuleContext, flags Flags) Flags { // Assuming that the cflags which clang LibTooling tools cannot // understand have not been converted to ninja variables yet. - flags.ToolingCFlags = filterOutWithPrefix(flags.CFlags, config.ClangLibToolingUnknownCflags) - flags.ToolingCppFlags = filterOutWithPrefix(flags.CppFlags, config.ClangLibToolingUnknownCflags) + flags.Local.ToolingCFlags = filterOutWithPrefix(flags.Local.CFlags, config.ClangLibToolingUnknownCflags) + flags.Global.ToolingCFlags = filterOutWithPrefix(flags.Global.CFlags, config.ClangLibToolingUnknownCflags) + flags.Local.ToolingCppFlags = filterOutWithPrefix(flags.Local.CppFlags, config.ClangLibToolingUnknownCflags) + flags.Global.ToolingCppFlags = filterOutWithPrefix(flags.Global.CppFlags, config.ClangLibToolingUnknownCflags) return flags } func sabiDepsMutator(mctx android.TopDownMutatorContext) { if c, ok := mctx.Module().(*Module); ok && - ((c.isVndk() && c.useVndk()) || inList(c.Name(), llndkLibraries) || + ((c.IsVndk() && c.UseVndk()) || c.isLlndk(mctx.Config()) || (c.sabi != nil && c.sabi.Properties.CreateSAbiDumps)) { mctx.VisitDirectDeps(func(m android.Module) { tag := mctx.OtherModuleDependencyTag(m) switch tag { - case staticDepTag, staticExportDepTag, lateStaticDepTag, wholeStaticDepTag: + case StaticDepTag, staticExportDepTag, lateStaticDepTag, wholeStaticDepTag: cc, _ := m.(*Module) if cc == nil { @@ -94,3 +96,9 @@ func sabiDepsMutator(mctx android.TopDownMutatorContext) { }) } } + +func addLsdumpPath(lsdumpPath string) { + sabiLock.Lock() + lsdumpPaths = append(lsdumpPaths, lsdumpPath) + sabiLock.Unlock() +} diff --git a/cc/sanitize.go b/cc/sanitize.go index c1b055afe..463a02ac2 100644 --- a/cc/sanitize.go +++ b/cc/sanitize.go @@ -31,16 +31,28 @@ var ( // understand also need to be added to ClangLibToolingUnknownCflags in // cc/config/clang.go - asanCflags = []string{"-fno-omit-frame-pointer"} + asanCflags = []string{ + "-fno-omit-frame-pointer", + "-fno-experimental-new-pass-manager", + } asanLdflags = []string{"-Wl,-u,__asan_preinit"} - asanLibs = []string{"libasan"} - // TODO(pcc): Stop passing -hwasan-allow-ifunc here once it has been made - // the default. hwasanCflags = []string{"-fno-omit-frame-pointer", "-Wno-frame-larger-than=", - "-mllvm", "-hwasan-create-frame-descriptions=0", - "-mllvm", "-hwasan-allow-ifunc", - "-fsanitize-hwaddress-abi=platform"} + "-fsanitize-hwaddress-abi=platform", + "-fno-experimental-new-pass-manager", + // The following improves debug location information + // availability at the cost of its accuracy. It increases + // the likelihood of a stack variable's frame offset + // to be recorded in the debug info, which is important + // for the quality of hwasan reports. The downside is a + // higher number of "optimized out" stack variables. + // b/112437883. + "-mllvm", "-instcombine-lower-dbg-declare=0", + // TODO(b/159343917): HWASan and GlobalISel don't play nicely, and + // GlobalISel is the default at -O0 on aarch64. + "-mllvm", "--aarch64-enable-global-isel-at-O=-1", + "-mllvm", "-fast-isel=false", + } cfiCflags = []string{"-flto", "-fsanitize-cfi-cross-dso", "-fsanitize-blacklist=external/compiler-rt/lib/cfi/cfi_blacklist.txt"} @@ -78,6 +90,7 @@ const ( intOverflow cfi scs + fuzzer ) // Name of the sanitizer variation for this sanitizer type @@ -95,6 +108,8 @@ func (t sanitizerType) variationName() string { return "cfi" case scs: return "scs" + case fuzzer: + return "fuzzer" default: panic(fmt.Errorf("unknown sanitizerType %d", t)) } @@ -115,13 +130,15 @@ func (t sanitizerType) name() string { return "cfi" case scs: return "shadow-call-stack" + case fuzzer: + return "fuzzer" default: panic(fmt.Errorf("unknown sanitizerType %d", t)) } } func (t sanitizerType) incompatibleWithCfi() bool { - return t == asan || t == hwasan + return t == asan || t == fuzzer || t == hwasan } type SanitizeProperties struct { @@ -138,7 +155,7 @@ type SanitizeProperties struct { Undefined *bool `android:"arch_variant"` All_undefined *bool `android:"arch_variant"` Misc_undefined []string `android:"arch_variant"` - Coverage *bool `android:"arch_variant"` + Fuzzer *bool `android:"arch_variant"` Safestack *bool `android:"arch_variant"` Cfi *bool `android:"arch_variant"` Integer_overflow *bool `android:"arch_variant"` @@ -166,6 +183,7 @@ type SanitizeProperties struct { SanitizerEnabled bool `blueprint:"mutated"` SanitizeDep bool `blueprint:"mutated"` MinimalRuntimeDep bool `blueprint:"mutated"` + BuiltinsDep bool `blueprint:"mutated"` UbsanRuntimeDep bool `blueprint:"mutated"` InSanitizerDir bool `blueprint:"mutated"` Sanitizers []string `blueprint:"mutated"` @@ -228,22 +246,16 @@ func (sanitize *sanitize) begin(ctx BaseModuleContext) { s.Undefined = boolPtr(true) } - if found, globalSanitizers = removeFromList("address", globalSanitizers); found { - if s.Address == nil { - s.Address = boolPtr(true) - } else if *s.Address == false { - // Coverage w/o address is an error. If globalSanitizers includes both, and the module - // disables address, then disable coverage as well. - _, globalSanitizers = removeFromList("coverage", globalSanitizers) - } + if found, globalSanitizers = removeFromList("address", globalSanitizers); found && s.Address == nil { + s.Address = boolPtr(true) } if found, globalSanitizers = removeFromList("thread", globalSanitizers); found && s.Thread == nil { s.Thread = boolPtr(true) } - if found, globalSanitizers = removeFromList("coverage", globalSanitizers); found && s.Coverage == nil { - s.Coverage = boolPtr(true) + if found, globalSanitizers = removeFromList("fuzzer", globalSanitizers); found && s.Fuzzer == nil { + s.Fuzzer = boolPtr(true) } if found, globalSanitizers = removeFromList("safe-stack", globalSanitizers); found && s.Safestack == nil { @@ -327,8 +339,8 @@ func (sanitize *sanitize) begin(ctx BaseModuleContext) { s.Diag.Cfi = nil } - // Disable sanitizers that depend on the UBSan runtime for host builds. - if ctx.Host() { + // Disable sanitizers that depend on the UBSan runtime for windows/darwin builds. + if !ctx.Os().Linux() { s.Cfi = nil s.Diag.Cfi = nil s.Misc_undefined = nil @@ -343,15 +355,21 @@ func (sanitize *sanitize) begin(ctx BaseModuleContext) { s.Diag.Cfi = nil } + // Also disable CFI if building against snapshot. + vndkVersion := ctx.DeviceConfig().VndkVersion() + if ctx.useVndk() && vndkVersion != "current" && vndkVersion != "" { + s.Cfi = nil + } + // HWASan ramdisk (which is built from recovery) goes over some bootloader limit. - // Keep libc instrumented so that recovery can run hwasan-instrumented code if necessary. - if ctx.inRecovery() && !strings.HasPrefix(ctx.ModuleDir(), "bionic/libc") { + // Keep libc instrumented so that ramdisk / recovery can run hwasan-instrumented code if necessary. + if (ctx.inRamdisk() || ctx.inRecovery()) && !strings.HasPrefix(ctx.ModuleDir(), "bionic/libc") { s.Hwaddress = nil } if ctx.staticBinary() { s.Address = nil - s.Coverage = nil + s.Fuzzer = nil s.Thread = nil } @@ -367,7 +385,7 @@ func (sanitize *sanitize) begin(ctx BaseModuleContext) { } if ctx.Os() != android.Windows && (Bool(s.All_undefined) || Bool(s.Undefined) || Bool(s.Address) || Bool(s.Thread) || - Bool(s.Coverage) || Bool(s.Safestack) || Bool(s.Cfi) || Bool(s.Integer_overflow) || len(s.Misc_undefined) > 0 || + Bool(s.Fuzzer) || Bool(s.Safestack) || Bool(s.Cfi) || Bool(s.Integer_overflow) || len(s.Misc_undefined) > 0 || Bool(s.Scudo) || Bool(s.Hwaddress) || Bool(s.Scs)) { sanitize.Properties.SanitizerEnabled = true } @@ -382,10 +400,10 @@ func (sanitize *sanitize) begin(ctx BaseModuleContext) { s.Thread = nil } - if Bool(s.Coverage) { - if !Bool(s.Address) { - ctx.ModuleErrorf(`Use of "coverage" also requires "address"`) - } + // TODO(b/131771163): CFI transiently depends on LTO, and thus Fuzzer is + // mutually incompatible. + if Bool(s.Fuzzer) { + s.Cfi = nil } } @@ -396,7 +414,6 @@ func (sanitize *sanitize) deps(ctx BaseModuleContext, deps Deps) Deps { if ctx.Device() { if Bool(sanitize.Properties.Sanitize.Address) { - deps.StaticLibs = append(deps.StaticLibs, asanLibs...) // Compiling asan and having libc_scudo in the same // executable will cause the executable to crash. // Remove libc_scudo since it is only used to override @@ -427,11 +444,19 @@ func toDisableImplicitIntegerChange(flags []string) bool { func (sanitize *sanitize) flags(ctx ModuleContext, flags Flags) Flags { minimalRuntimeLib := config.UndefinedBehaviorSanitizerMinimalRuntimeLibrary(ctx.toolchain()) + ".a" minimalRuntimePath := "${config.ClangAsanLibDir}/" + minimalRuntimeLib + builtinsRuntimeLib := config.BuiltinsRuntimeLibrary(ctx.toolchain()) + ".a" + builtinsRuntimePath := "${config.ClangAsanLibDir}/" + builtinsRuntimeLib + + if sanitize.Properties.MinimalRuntimeDep { + flags.Local.LdFlags = append(flags.Local.LdFlags, + minimalRuntimePath, + "-Wl,--exclude-libs,"+minimalRuntimeLib) + } - if ctx.Device() && sanitize.Properties.MinimalRuntimeDep { - flags.LdFlags = append(flags.LdFlags, minimalRuntimePath) - flags.LdFlags = append(flags.LdFlags, "-Wl,--exclude-libs,"+minimalRuntimeLib) + if sanitize.Properties.BuiltinsDep { + flags.libFlags = append([]string{builtinsRuntimePath}, flags.libFlags...) } + if !sanitize.Properties.SanitizerEnabled && !sanitize.Properties.UbsanRuntimeDep { return flags } @@ -442,15 +467,15 @@ func (sanitize *sanitize) flags(ctx ModuleContext, flags Flags) Flags { // TODO: put in flags? flags.RequiredInstructionSet = "arm" } - flags.CFlags = append(flags.CFlags, asanCflags...) - flags.LdFlags = append(flags.LdFlags, asanLdflags...) + flags.Local.CFlags = append(flags.Local.CFlags, asanCflags...) + flags.Local.LdFlags = append(flags.Local.LdFlags, asanLdflags...) if ctx.Host() { // -nodefaultlibs (provided with libc++) prevents the driver from linking // libraries needed with -fsanitize=address. http://b/18650275 (WAI) - flags.LdFlags = append(flags.LdFlags, "-Wl,--no-as-needed") + flags.Local.LdFlags = append(flags.Local.LdFlags, "-Wl,--no-as-needed") } else { - flags.CFlags = append(flags.CFlags, "-mllvm", "-asan-globals=0") + flags.Local.CFlags = append(flags.Local.CFlags, "-mllvm", "-asan-globals=0") if ctx.bootstrap() { flags.DynamicLinker = "/system/bin/bootstrap/linker_asan" } else { @@ -463,11 +488,39 @@ func (sanitize *sanitize) flags(ctx ModuleContext, flags Flags) Flags { } if Bool(sanitize.Properties.Sanitize.Hwaddress) { - flags.CFlags = append(flags.CFlags, hwasanCflags...) + flags.Local.CFlags = append(flags.Local.CFlags, hwasanCflags...) } - if Bool(sanitize.Properties.Sanitize.Coverage) { - flags.CFlags = append(flags.CFlags, "-fsanitize-coverage=trace-pc-guard,indirect-calls,trace-cmp") + if Bool(sanitize.Properties.Sanitize.Fuzzer) { + flags.Local.CFlags = append(flags.Local.CFlags, "-fsanitize=fuzzer-no-link") + + // TODO(b/131771163): LTO and Fuzzer support is mutually incompatible. + _, flags.Local.LdFlags = removeFromList("-flto", flags.Local.LdFlags) + _, flags.Local.CFlags = removeFromList("-flto", flags.Local.CFlags) + flags.Local.LdFlags = append(flags.Local.LdFlags, "-fno-lto") + flags.Local.CFlags = append(flags.Local.CFlags, "-fno-lto") + + // TODO(b/142430592): Upstream linker scripts for sanitizer runtime libraries + // discard the sancov_lowest_stack symbol, because it's emulated TLS (and thus + // doesn't match the linker script due to the "__emutls_v." prefix). + flags.Local.LdFlags = append(flags.Local.LdFlags, "-fno-sanitize-coverage=stack-depth") + flags.Local.CFlags = append(flags.Local.CFlags, "-fno-sanitize-coverage=stack-depth") + + // TODO(b/133876586): Experimental PM breaks sanitizer coverage. + flags.Local.CFlags = append(flags.Local.CFlags, "-fno-experimental-new-pass-manager") + + // Disable fortify for fuzzing builds. Generally, we'll be building with + // UBSan or ASan here and the fortify checks pollute the stack traces. + flags.Local.CFlags = append(flags.Local.CFlags, "-U_FORTIFY_SOURCE") + + // Build fuzzer-sanitized libraries with an $ORIGIN DT_RUNPATH. Android's + // linker uses DT_RUNPATH, not DT_RPATH. When we deploy cc_fuzz targets and + // their libraries to /data/fuzz/<arch>/lib, any transient shared library gets + // the DT_RUNPATH from the shared library above it, and not the executable, + // meaning that the lookup falls back to the system. Adding the $ORIGIN to the + // DT_RUNPATH here means that transient shared libraries can be found + // colocated with their parents. + flags.Local.LdFlags = append(flags.Local.LdFlags, `-Wl,-rpath,\$$ORIGIN`) } if Bool(sanitize.Properties.Sanitize.Cfi) { @@ -477,87 +530,101 @@ func (sanitize *sanitize) flags(ctx ModuleContext, flags Flags) Flags { flags.RequiredInstructionSet = "thumb" } - flags.CFlags = append(flags.CFlags, cfiCflags...) - flags.AsFlags = append(flags.AsFlags, cfiAsflags...) + flags.Local.CFlags = append(flags.Local.CFlags, cfiCflags...) + flags.Local.AsFlags = append(flags.Local.AsFlags, cfiAsflags...) // Only append the default visibility flag if -fvisibility has not already been set // to hidden. - if !inList("-fvisibility=hidden", flags.CFlags) { - flags.CFlags = append(flags.CFlags, "-fvisibility=default") + if !inList("-fvisibility=hidden", flags.Local.CFlags) { + flags.Local.CFlags = append(flags.Local.CFlags, "-fvisibility=default") } - flags.LdFlags = append(flags.LdFlags, cfiLdflags...) + flags.Local.LdFlags = append(flags.Local.LdFlags, cfiLdflags...) if ctx.staticBinary() { - _, flags.CFlags = removeFromList("-fsanitize-cfi-cross-dso", flags.CFlags) - _, flags.LdFlags = removeFromList("-fsanitize-cfi-cross-dso", flags.LdFlags) + _, flags.Local.CFlags = removeFromList("-fsanitize-cfi-cross-dso", flags.Local.CFlags) + _, flags.Local.LdFlags = removeFromList("-fsanitize-cfi-cross-dso", flags.Local.LdFlags) } } if Bool(sanitize.Properties.Sanitize.Integer_overflow) { - flags.CFlags = append(flags.CFlags, intOverflowCflags...) + flags.Local.CFlags = append(flags.Local.CFlags, intOverflowCflags...) } if len(sanitize.Properties.Sanitizers) > 0 { sanitizeArg := "-fsanitize=" + strings.Join(sanitize.Properties.Sanitizers, ",") - flags.CFlags = append(flags.CFlags, sanitizeArg) - flags.AsFlags = append(flags.AsFlags, sanitizeArg) + flags.Local.CFlags = append(flags.Local.CFlags, sanitizeArg) + flags.Local.AsFlags = append(flags.Local.AsFlags, sanitizeArg) if ctx.Host() { - flags.CFlags = append(flags.CFlags, "-fno-sanitize-recover=all") - flags.LdFlags = append(flags.LdFlags, sanitizeArg) // Host sanitizers only link symbols in the final executable, so // there will always be undefined symbols in intermediate libraries. - _, flags.LdFlags = removeFromList("-Wl,--no-undefined", flags.LdFlags) - } else { - flags.CFlags = append(flags.CFlags, "-fsanitize-trap=all", "-ftrap-function=abort") + _, flags.Global.LdFlags = removeFromList("-Wl,--no-undefined", flags.Global.LdFlags) + flags.Local.LdFlags = append(flags.Local.LdFlags, sanitizeArg) + + // non-Bionic toolchain prebuilts are missing UBSan's vptr and function sanitizers + if !ctx.toolchain().Bionic() { + flags.Local.CFlags = append(flags.Local.CFlags, "-fno-sanitize=vptr,function") + } + } - if enableMinimalRuntime(sanitize) { - flags.CFlags = append(flags.CFlags, strings.Join(minimalRuntimeFlags, " ")) - flags.libFlags = append([]string{minimalRuntimePath}, flags.libFlags...) - flags.LdFlags = append(flags.LdFlags, "-Wl,--exclude-libs,"+minimalRuntimeLib) + if enableMinimalRuntime(sanitize) { + flags.Local.CFlags = append(flags.Local.CFlags, strings.Join(minimalRuntimeFlags, " ")) + flags.libFlags = append([]string{minimalRuntimePath}, flags.libFlags...) + flags.Local.LdFlags = append(flags.Local.LdFlags, "-Wl,--exclude-libs,"+minimalRuntimeLib) + if !ctx.toolchain().Bionic() { + flags.libFlags = append([]string{builtinsRuntimePath}, flags.libFlags...) } } + + if Bool(sanitize.Properties.Sanitize.Fuzzer) { + // When fuzzing, we wish to crash with diagnostics on any bug. + flags.Local.CFlags = append(flags.Local.CFlags, "-fno-sanitize-trap=all", "-fno-sanitize-recover=all") + } else if ctx.Host() { + flags.Local.CFlags = append(flags.Local.CFlags, "-fno-sanitize-recover=all") + } else { + flags.Local.CFlags = append(flags.Local.CFlags, "-fsanitize-trap=all", "-ftrap-function=abort") + } // http://b/119329758, Android core does not boot up with this sanitizer yet. - if toDisableImplicitIntegerChange(flags.CFlags) { - flags.CFlags = append(flags.CFlags, "-fno-sanitize=implicit-integer-sign-change") + if toDisableImplicitIntegerChange(flags.Local.CFlags) { + flags.Local.CFlags = append(flags.Local.CFlags, "-fno-sanitize=implicit-integer-sign-change") } } if len(sanitize.Properties.DiagSanitizers) > 0 { - flags.CFlags = append(flags.CFlags, "-fno-sanitize-trap="+strings.Join(sanitize.Properties.DiagSanitizers, ",")) + flags.Local.CFlags = append(flags.Local.CFlags, "-fno-sanitize-trap="+strings.Join(sanitize.Properties.DiagSanitizers, ",")) } // FIXME: enable RTTI if diag + (cfi or vptr) if sanitize.Properties.Sanitize.Recover != nil { - flags.CFlags = append(flags.CFlags, "-fsanitize-recover="+ + flags.Local.CFlags = append(flags.Local.CFlags, "-fsanitize-recover="+ strings.Join(sanitize.Properties.Sanitize.Recover, ",")) } if sanitize.Properties.Sanitize.Diag.No_recover != nil { - flags.CFlags = append(flags.CFlags, "-fno-sanitize-recover="+ + flags.Local.CFlags = append(flags.Local.CFlags, "-fno-sanitize-recover="+ strings.Join(sanitize.Properties.Sanitize.Diag.No_recover, ",")) } blacklist := android.OptionalPathForModuleSrc(ctx, sanitize.Properties.Sanitize.Blacklist) if blacklist.Valid() { - flags.CFlags = append(flags.CFlags, "-fsanitize-blacklist="+blacklist.String()) + flags.Local.CFlags = append(flags.Local.CFlags, "-fsanitize-blacklist="+blacklist.String()) flags.CFlagsDeps = append(flags.CFlagsDeps, blacklist.Path()) } return flags } -func (sanitize *sanitize) AndroidMk(ctx AndroidMkContext, ret *android.AndroidMkData) { +func (sanitize *sanitize) AndroidMkEntries(ctx AndroidMkContext, entries *android.AndroidMkEntries) { // Add a suffix for cfi/hwasan/scs-enabled static/header libraries to allow surfacing // both the sanitized and non-sanitized variants to make without a name conflict. - if ret.Class == "STATIC_LIBRARIES" || ret.Class == "HEADER_LIBRARIES" { + if entries.Class == "STATIC_LIBRARIES" || entries.Class == "HEADER_LIBRARIES" { if Bool(sanitize.Properties.Sanitize.Cfi) { - ret.SubName += ".cfi" + entries.SubName += ".cfi" } if Bool(sanitize.Properties.Sanitize.Hwaddress) { - ret.SubName += ".hwasan" + entries.SubName += ".hwasan" } if Bool(sanitize.Properties.Sanitize.Scs) { - ret.SubName += ".scs" + entries.SubName += ".scs" } } } @@ -580,6 +647,8 @@ func (sanitize *sanitize) getSanitizerBoolPtr(t sanitizerType) *bool { return sanitize.Properties.Sanitize.Cfi case scs: return sanitize.Properties.Sanitize.Scs + case fuzzer: + return sanitize.Properties.Sanitize.Fuzzer default: panic(fmt.Errorf("unknown sanitizerType %d", t)) } @@ -590,22 +659,21 @@ func (sanitize *sanitize) isUnsanitizedVariant() bool { !sanitize.isSanitizerEnabled(hwasan) && !sanitize.isSanitizerEnabled(tsan) && !sanitize.isSanitizerEnabled(cfi) && - !sanitize.isSanitizerEnabled(scs) + !sanitize.isSanitizerEnabled(scs) && + !sanitize.isSanitizerEnabled(fuzzer) } func (sanitize *sanitize) isVariantOnProductionDevice() bool { return !sanitize.isSanitizerEnabled(asan) && !sanitize.isSanitizerEnabled(hwasan) && - !sanitize.isSanitizerEnabled(tsan) + !sanitize.isSanitizerEnabled(tsan) && + !sanitize.isSanitizerEnabled(fuzzer) } func (sanitize *sanitize) SetSanitizer(t sanitizerType, b bool) { switch t { case asan: sanitize.Properties.Sanitize.Address = boolPtr(b) - if !b { - sanitize.Properties.Sanitize.Coverage = nil - } case hwasan: sanitize.Properties.Sanitize.Hwaddress = boolPtr(b) case tsan: @@ -616,6 +684,8 @@ func (sanitize *sanitize) SetSanitizer(t sanitizerType, b bool) { sanitize.Properties.Sanitize.Cfi = boolPtr(b) case scs: sanitize.Properties.Sanitize.Scs = boolPtr(b) + case fuzzer: + sanitize.Properties.Sanitize.Fuzzer = boolPtr(b) default: panic(fmt.Errorf("unknown sanitizerType %d", t)) } @@ -650,8 +720,8 @@ func (sanitize *sanitize) isSanitizerEnabled(t sanitizerType) bool { } func isSanitizableDependencyTag(tag blueprint.DependencyTag) bool { - t, ok := tag.(dependencyTag) - return ok && t.library || t == reuseObjTag + t, ok := tag.(DependencyTag) + return ok && t.Library || t == reuseObjTag || t == objDepTag } // Propagate sanitizer requirements down from binaries @@ -694,20 +764,45 @@ func sanitizerRuntimeDepsMutator(mctx android.TopDownMutatorContext) { if !isSanitizableDependencyTag(mctx.OtherModuleDependencyTag(child)) { return false } - if d, ok := child.(*Module); ok && d.static() && d.sanitize != nil { + d, ok := child.(*Module) + if !ok || !d.static() { + return false + } + if d.sanitize != nil { if enableMinimalRuntime(d.sanitize) { // If a static dependency is built with the minimal runtime, // make sure we include the ubsan minimal runtime. c.sanitize.Properties.MinimalRuntimeDep = true - } else if Bool(d.sanitize.Properties.Sanitize.Diag.Integer_overflow) || - len(d.sanitize.Properties.Sanitize.Diag.Misc_undefined) > 0 { + } else if enableUbsanRuntime(d.sanitize) { // If a static dependency runs with full ubsan diagnostics, // make sure we include the ubsan runtime. c.sanitize.Properties.UbsanRuntimeDep = true } + + if c.sanitize.Properties.MinimalRuntimeDep && + c.sanitize.Properties.UbsanRuntimeDep { + // both flags that this mutator might set are true, so don't bother recursing + return false + } + + if c.Os() == android.Linux { + c.sanitize.Properties.BuiltinsDep = true + } + + return true } - return true + + if p, ok := d.linker.(*vendorSnapshotLibraryDecorator); ok { + if Bool(p.properties.Sanitize_minimal_dep) { + c.sanitize.Properties.MinimalRuntimeDep = true + } + if Bool(p.properties.Sanitize_ubsan_dep) { + c.sanitize.Properties.UbsanRuntimeDep = true + } + } + + return false }) } } @@ -800,19 +895,30 @@ func sanitizerRuntimeMutator(mctx android.BottomUpMutatorContext) { sanitizers = append(sanitizers, "shadow-call-stack") } + if Bool(c.sanitize.Properties.Sanitize.Fuzzer) { + sanitizers = append(sanitizers, "fuzzer-no-link") + } + // Save the list of sanitizers. These will be used again when generating // the build rules (for Cflags, etc.) c.sanitize.Properties.Sanitizers = sanitizers c.sanitize.Properties.DiagSanitizers = diagSanitizers + // TODO(b/150822854) Hosts have a different default behavior and assume the runtime library is used. + if c.Host() { + diagSanitizers = sanitizers + } + // Determine the runtime library required runtimeLibrary := "" + var extraStaticDeps []string toolchain := c.toolchain(mctx) if Bool(c.sanitize.Properties.Sanitize.Address) { runtimeLibrary = config.AddressSanitizerRuntimeLibrary(toolchain) } else if Bool(c.sanitize.Properties.Sanitize.Hwaddress) { if c.staticBinary() { runtimeLibrary = config.HWAddressSanitizerStaticLibrary(toolchain) + extraStaticDeps = []string{"libdl"} } else { runtimeLibrary = config.HWAddressSanitizerRuntimeLibrary(toolchain) } @@ -824,12 +930,16 @@ func sanitizerRuntimeMutator(mctx android.BottomUpMutatorContext) { } else { runtimeLibrary = config.ScudoRuntimeLibrary(toolchain) } - } else if len(diagSanitizers) > 0 || c.sanitize.Properties.UbsanRuntimeDep { + } else if len(diagSanitizers) > 0 || c.sanitize.Properties.UbsanRuntimeDep || + Bool(c.sanitize.Properties.Sanitize.Fuzzer) || + Bool(c.sanitize.Properties.Sanitize.Undefined) || + Bool(c.sanitize.Properties.Sanitize.All_undefined) { runtimeLibrary = config.UndefinedBehaviorSanitizerRuntimeLibrary(toolchain) } - if mctx.Device() && runtimeLibrary != "" { - if inList(runtimeLibrary, llndkLibraries) && !c.static() && c.useVndk() { + if runtimeLibrary != "" && (toolchain.Bionic() || c.sanitize.Properties.UbsanRuntimeDep) { + // UBSan is supported on non-bionic linux host builds as well + if isLlndkLibrary(runtimeLibrary, mctx.Config()) && !c.static() && c.UseVndk() { runtimeLibrary = runtimeLibrary + llndkLibrarySuffix } @@ -841,19 +951,36 @@ func sanitizerRuntimeMutator(mctx android.BottomUpMutatorContext) { // Note that by adding dependency with {static|shared}DepTag, the lib is // added to libFlags and LOCAL_SHARED_LIBRARIES by cc.Module if c.staticBinary() { + deps := append(extraStaticDeps, runtimeLibrary) + // If we're using snapshots and in vendor, redirect to snapshot whenever possible + if c.VndkVersion() == mctx.DeviceConfig().VndkVersion() { + snapshots := vendorSnapshotStaticLibs(mctx.Config()) + for idx, dep := range deps { + if lib, ok := snapshots.get(dep, mctx.Arch().ArchType); ok { + deps[idx] = lib + } + } + } + // static executable gets static runtime libs - mctx.AddFarVariationDependencies([]blueprint.Variation{ + mctx.AddFarVariationDependencies(append(mctx.Target().Variations(), []blueprint.Variation{ {Mutator: "link", Variation: "static"}, - {Mutator: "image", Variation: c.imageVariation()}, - {Mutator: "arch", Variation: mctx.Target().String()}, - }, staticDepTag, runtimeLibrary) + c.ImageVariation(), + }...), StaticDepTag, deps...) } else if !c.static() && !c.header() { + // If we're using snapshots and in vendor, redirect to snapshot whenever possible + if c.VndkVersion() == mctx.DeviceConfig().VndkVersion() { + snapshots := vendorSnapshotSharedLibs(mctx.Config()) + if lib, ok := snapshots.get(runtimeLibrary, mctx.Arch().ArchType); ok { + runtimeLibrary = lib + } + } + // dynamic executable and shared libs get shared runtime libs - mctx.AddFarVariationDependencies([]blueprint.Variation{ + mctx.AddFarVariationDependencies(append(mctx.Target().Variations(), []blueprint.Variation{ {Mutator: "link", Variation: "shared"}, - {Mutator: "image", Variation: c.imageVariation()}, - {Mutator: "arch", Variation: mctx.Target().String()}, - }, earlySharedDepTag, runtimeLibrary) + c.ImageVariation(), + }...), earlySharedDepTag, runtimeLibrary) } // static lib does not have dependency to the runtime library. The // dependency will be added to the executables or shared libs using @@ -877,16 +1004,11 @@ func sanitizerMutator(t sanitizerType) func(android.BottomUpMutatorContext) { modules[0].(*Module).sanitize.SetSanitizer(t, true) } else if c.sanitize.isSanitizerEnabled(t) || c.sanitize.Properties.SanitizeDep { isSanitizerEnabled := c.sanitize.isSanitizerEnabled(t) - if mctx.Device() && t.incompatibleWithCfi() { - // TODO: Make sure that cfi mutator runs "after" any of the sanitizers that - // are incompatible with cfi - c.sanitize.SetSanitizer(cfi, false) - } - if c.static() || c.header() || t == asan { + if c.static() || c.header() || t == asan || t == fuzzer { // Static and header libs are split into non-sanitized and sanitized variants. - // Shared libs are not split. However, for asan, we split even for shared - // libs because a library sanitized for asan can't be linked from a library - // that isn't sanitized for asan. + // Shared libs are not split. However, for asan and fuzzer, we split even for shared + // libs because a library sanitized for asan/fuzzer can't be linked from a library + // that isn't sanitized for asan/fuzzer. // // Note for defaultVariation: since we don't split for shared libs but for static/header // libs, it is possible for the sanitized variant of a static/header lib to depend @@ -903,6 +1025,12 @@ func sanitizerMutator(t sanitizerType) func(android.BottomUpMutatorContext) { modules[0].(*Module).sanitize.Properties.SanitizeDep = false modules[1].(*Module).sanitize.Properties.SanitizeDep = false + if mctx.Device() && t.incompatibleWithCfi() { + // TODO: Make sure that cfi mutator runs "after" any of the sanitizers that + // are incompatible with cfi + modules[1].(*Module).sanitize.SetSanitizer(cfi, false) + } + // For cfi/scs/hwasan, we can export both sanitized and un-sanitized variants // to Make, because the sanitized version has a different suffix in name. // For other types of sanitizers, suppress the variation that is disabled. @@ -915,12 +1043,13 @@ func sanitizerMutator(t sanitizerType) func(android.BottomUpMutatorContext) { modules[1].(*Module).Properties.HideFromMake = true } } + // Export the static lib name to make - if c.static() { + if c.static() && c.ExportedToMake() { if t == cfi { appendStringSync(c.Name(), cfiStaticLibs(mctx.Config()), &cfiStaticLibsMutex) } else if t == hwasan { - if c.useVndk() { + if c.UseVndk() { appendStringSync(c.Name(), hwasanVendorStaticLibs(mctx.Config()), &hwasanStaticLibsMutex) } else { @@ -939,6 +1068,12 @@ func sanitizerMutator(t sanitizerType) func(android.BottomUpMutatorContext) { if mctx.Device() && t == asan && isSanitizerEnabled { modules[0].(*Module).sanitize.Properties.InSanitizerDir = true } + + if mctx.Device() && t.incompatibleWithCfi() { + // TODO: Make sure that cfi mutator runs "after" any of the sanitizers that + // are incompatible with cfi + modules[0].(*Module).sanitize.SetSanitizer(cfi, false) + } } } c.sanitize.Properties.SanitizeDep = false @@ -982,16 +1117,29 @@ func appendStringSync(item string, list *[]string, mutex *sync.Mutex) { func enableMinimalRuntime(sanitize *sanitize) bool { if !Bool(sanitize.Properties.Sanitize.Address) && !Bool(sanitize.Properties.Sanitize.Hwaddress) && + !Bool(sanitize.Properties.Sanitize.Fuzzer) && + (Bool(sanitize.Properties.Sanitize.Integer_overflow) || - len(sanitize.Properties.Sanitize.Misc_undefined) > 0) && + len(sanitize.Properties.Sanitize.Misc_undefined) > 0 || + Bool(sanitize.Properties.Sanitize.Undefined) || + Bool(sanitize.Properties.Sanitize.All_undefined)) && + !(Bool(sanitize.Properties.Sanitize.Diag.Integer_overflow) || Bool(sanitize.Properties.Sanitize.Diag.Cfi) || + Bool(sanitize.Properties.Sanitize.Diag.Undefined) || len(sanitize.Properties.Sanitize.Diag.Misc_undefined) > 0) { + return true } return false } +func enableUbsanRuntime(sanitize *sanitize) bool { + return Bool(sanitize.Properties.Sanitize.Diag.Integer_overflow) || + Bool(sanitize.Properties.Sanitize.Diag.Undefined) || + len(sanitize.Properties.Sanitize.Diag.Misc_undefined) > 0 +} + func cfiMakeVarsProvider(ctx android.MakeVarsContext) { cfiStaticLibs := cfiStaticLibs(ctx.Config()) sort.Strings(*cfiStaticLibs) diff --git a/cc/sdk.go b/cc/sdk.go new file mode 100644 index 000000000..d05a04a12 --- /dev/null +++ b/cc/sdk.go @@ -0,0 +1,65 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cc + +import ( + "android/soong/android" + "android/soong/genrule" +) + +// sdkMutator sets a creates a platform and an SDK variant for modules +// that set sdk_version, and ignores sdk_version for the platform +// variant. The SDK variant will be used for embedding in APKs +// that may be installed on older platforms. Apexes use their own +// variants that enforce backwards compatibility. +func sdkMutator(ctx android.BottomUpMutatorContext) { + if ctx.Os() != android.Android { + return + } + + switch m := ctx.Module().(type) { + case LinkableInterface: + if m.AlwaysSdk() { + if !m.UseSdk() { + ctx.ModuleErrorf("UseSdk() must return true when AlwaysSdk is set, did the factory forget to set Sdk_version?") + } + ctx.CreateVariations("sdk") + } else if m.UseSdk() { + modules := ctx.CreateVariations("", "sdk") + modules[0].(*Module).Properties.Sdk_version = nil + modules[1].(*Module).Properties.IsSdkVariant = true + + if ctx.Config().UnbundledBuild() { + modules[0].(*Module).Properties.HideFromMake = true + } else { + modules[1].(*Module).Properties.SdkAndPlatformVariantVisibleToMake = true + modules[1].(*Module).Properties.PreventInstall = true + } + ctx.AliasVariation("") + } else { + ctx.CreateVariations("") + ctx.AliasVariation("") + } + case *genrule.Module: + if p, ok := m.Extra.(*GenruleExtraProperties); ok { + if String(p.Sdk_version) != "" { + ctx.CreateVariations("", "sdk") + } else { + ctx.CreateVariations("") + } + ctx.AliasVariation("") + } + } +} diff --git a/cc/sdk_test.go b/cc/sdk_test.go new file mode 100644 index 000000000..5a3c181dc --- /dev/null +++ b/cc/sdk_test.go @@ -0,0 +1,102 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cc + +import ( + "testing" + + "android/soong/android" +) + +func TestSdkMutator(t *testing.T) { + bp := ` + cc_library { + name: "libsdk", + shared_libs: ["libsdkdep"], + sdk_version: "current", + stl: "c++_shared", + } + + cc_library { + name: "libsdkdep", + sdk_version: "current", + stl: "c++_shared", + } + + cc_library { + name: "libplatform", + shared_libs: ["libsdk"], + stl: "libc++", + } + + cc_binary { + name: "platformbinary", + shared_libs: ["libplatform"], + stl: "libc++", + } + + cc_binary { + name: "sdkbinary", + shared_libs: ["libsdk"], + sdk_version: "current", + stl: "libc++", + } + ` + + assertDep := func(t *testing.T, from, to android.TestingModule) { + t.Helper() + found := false + + var toFile android.Path + m := to.Module().(*Module) + if toc := m.Toc(); toc.Valid() { + toFile = toc.Path() + } else { + toFile = m.outputFile.Path() + } + + rule := from.Description("link") + for _, dep := range rule.Implicits { + if dep.String() == toFile.String() { + found = true + } + } + if !found { + t.Errorf("expected %q in %q", toFile.String(), rule.Implicits.Strings()) + } + } + + ctx := testCc(t, bp) + + libsdkNDK := ctx.ModuleForTests("libsdk", "android_arm64_armv8-a_sdk_shared") + libsdkPlatform := ctx.ModuleForTests("libsdk", "android_arm64_armv8-a_shared") + libsdkdepNDK := ctx.ModuleForTests("libsdkdep", "android_arm64_armv8-a_sdk_shared") + libsdkdepPlatform := ctx.ModuleForTests("libsdkdep", "android_arm64_armv8-a_shared") + libplatform := ctx.ModuleForTests("libplatform", "android_arm64_armv8-a_shared") + platformbinary := ctx.ModuleForTests("platformbinary", "android_arm64_armv8-a") + sdkbinary := ctx.ModuleForTests("sdkbinary", "android_arm64_armv8-a_sdk") + + libcxxNDK := ctx.ModuleForTests("ndk_libc++_shared", "android_arm64_armv8-a_sdk_shared") + libcxxPlatform := ctx.ModuleForTests("libc++", "android_arm64_armv8-a_shared") + + assertDep(t, libsdkNDK, libsdkdepNDK) + assertDep(t, libsdkPlatform, libsdkdepPlatform) + assertDep(t, libplatform, libsdkPlatform) + assertDep(t, platformbinary, libplatform) + assertDep(t, sdkbinary, libsdkNDK) + + assertDep(t, libsdkNDK, libcxxNDK) + assertDep(t, libsdkPlatform, libcxxPlatform) +} diff --git a/cc/snapshot_utils.go b/cc/snapshot_utils.go new file mode 100644 index 000000000..26e7c8d13 --- /dev/null +++ b/cc/snapshot_utils.go @@ -0,0 +1,95 @@ +// Copyright 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package cc + +import ( + "android/soong/android" +) + +var ( + headerExts = []string{".h", ".hh", ".hpp", ".hxx", ".h++", ".inl", ".inc", ".ipp", ".h.generic"} +) + +type snapshotLibraryInterface interface { + exportedFlagsProducer + libraryInterface + collectHeadersForSnapshot(ctx android.ModuleContext) + snapshotHeaders() android.Paths +} + +var _ snapshotLibraryInterface = (*prebuiltLibraryLinker)(nil) +var _ snapshotLibraryInterface = (*libraryDecorator)(nil) + +type snapshotMap struct { + snapshots map[string]string +} + +func newSnapshotMap() *snapshotMap { + return &snapshotMap{ + snapshots: make(map[string]string), + } +} + +func snapshotMapKey(name string, arch android.ArchType) string { + return name + ":" + arch.String() +} + +// Adds a snapshot name for given module name and architecture. +// e.g. add("libbase", X86, "libbase.vndk.29.x86") +func (s *snapshotMap) add(name string, arch android.ArchType, snapshot string) { + s.snapshots[snapshotMapKey(name, arch)] = snapshot +} + +// Returns snapshot name for given module name and architecture, if found. +// e.g. get("libcutils", X86) => "libcutils.vndk.29.x86", true +func (s *snapshotMap) get(name string, arch android.ArchType) (snapshot string, found bool) { + snapshot, found = s.snapshots[snapshotMapKey(name, arch)] + return snapshot, found +} + +func isSnapshotAware(ctx android.ModuleContext, m *Module) bool { + if _, _, ok := isVndkSnapshotLibrary(ctx.DeviceConfig(), m); ok { + return ctx.Config().VndkSnapshotBuildArtifacts() + } else if isVendorSnapshotModule(m, ctx.ModuleDir()) { + return true + } + return false +} + +func copyFile(ctx android.SingletonContext, path android.Path, out string) android.OutputPath { + outPath := android.PathForOutput(ctx, out) + ctx.Build(pctx, android.BuildParams{ + Rule: android.Cp, + Input: path, + Output: outPath, + Description: "Cp " + out, + Args: map[string]string{ + "cpFlags": "-f -L", + }, + }) + return outPath +} + +func writeStringToFile(ctx android.SingletonContext, content, out string) android.OutputPath { + outPath := android.PathForOutput(ctx, out) + ctx.Build(pctx, android.BuildParams{ + Rule: android.WriteFile, + Output: outPath, + Description: "WriteFile " + out, + Args: map[string]string{ + "content": content, + }, + }) + return outPath +} @@ -20,13 +20,13 @@ import ( "strconv" ) -func getNdkStlFamily(m *Module) string { +func getNdkStlFamily(m LinkableInterface) string { family, _ := getNdkStlFamilyAndLinkType(m) return family } -func getNdkStlFamilyAndLinkType(m *Module) (string, string) { - stl := m.stl.Properties.SelectedStl +func getNdkStlFamilyAndLinkType(m LinkableInterface) (string, string) { + stl := m.SelectedStl() switch stl { case "ndk_libc++_shared": return "libc++", "shared" @@ -115,9 +115,13 @@ func (stl *stl) begin(ctx BaseModuleContext) { switch s { case "libc++", "libc++_static": return s + case "c++_shared": + return "libc++" + case "c++_static": + return "libc++_static" case "none": return "" - case "": + case "", "system": if ctx.static() { return "libc++_static" } else { @@ -151,6 +155,14 @@ func needsLibAndroidSupport(ctx BaseModuleContext) bool { return version < 21 } +func staticUnwinder(ctx android.BaseModuleContext) string { + if ctx.Arch().ArchType == android.Arm { + return "libunwind_llvm" + } else { + return "libgcc_stripped" + } +} + func (stl *stl) deps(ctx BaseModuleContext, deps Deps) Deps { switch stl.Properties.SelectedStl { case "libstdc++": @@ -161,16 +173,27 @@ func (stl *stl) deps(ctx BaseModuleContext, deps Deps) Deps { } else { deps.StaticLibs = append(deps.StaticLibs, stl.Properties.SelectedStl) } + if ctx.Device() && !ctx.useSdk() { + // __cxa_demangle is not a part of libc++.so on the device since + // it's large and most processes don't need it. Statically link + // libc++demangle into every process so that users still have it if + // needed, but the linker won't include this unless it is actually + // called. + // http://b/138245375 + deps.StaticLibs = append(deps.StaticLibs, "libc++demangle") + } if ctx.toolchain().Bionic() { - if ctx.Arch().ArchType == android.Arm { - deps.StaticLibs = append(deps.StaticLibs, "libunwind_llvm") - } if ctx.staticBinary() { - deps.StaticLibs = append(deps.StaticLibs, "libm", "libc", "libdl") + deps.StaticLibs = append(deps.StaticLibs, "libm", "libc", staticUnwinder(ctx)) + } else { + deps.StaticUnwinderIfLegacy = true } } case "": // None or error. + if ctx.toolchain().Bionic() && ctx.Module().Name() == "libc++" { + deps.StaticUnwinderIfLegacy = true + } case "ndk_system": // TODO: Make a system STL prebuilt for the NDK. // The system STL doesn't have a prebuilt (it uses the system's libstdc++), but it does have @@ -187,6 +210,8 @@ func (stl *stl) deps(ctx BaseModuleContext, deps Deps) Deps { } if ctx.Arch().ArchType == android.Arm { deps.StaticLibs = append(deps.StaticLibs, "ndk_libunwind") + } else { + deps.StaticLibs = append(deps.StaticLibs, "libgcc_stripped") } default: panic(fmt.Errorf("Unknown stl: %q", stl.Properties.SelectedStl)) @@ -198,8 +223,6 @@ func (stl *stl) deps(ctx BaseModuleContext, deps Deps) Deps { func (stl *stl) flags(ctx ModuleContext, flags Flags) Flags { switch stl.Properties.SelectedStl { case "libc++", "libc++_static": - flags.CFlags = append(flags.CFlags, "-D_USING_LIBCXX") - if ctx.Darwin() { // libc++'s headers are annotated with availability macros that // indicate which version of Mac OS was the first to ship with a @@ -208,25 +231,25 @@ func (stl *stl) flags(ctx ModuleContext, flags Flags) Flags { // these availability attributes are meaningless for us but cause // build breaks when we try to use code that would not be available // in the system's dylib. - flags.CppFlags = append(flags.CppFlags, + flags.Local.CppFlags = append(flags.Local.CppFlags, "-D_LIBCPP_DISABLE_AVAILABILITY") } if !ctx.toolchain().Bionic() { - flags.CppFlags = append(flags.CppFlags, "-nostdinc++") - flags.LdFlags = append(flags.LdFlags, "-nodefaultlibs") - if ctx.staticBinary() { - flags.LdFlags = append(flags.LdFlags, hostStaticGccLibs[ctx.Os()]...) - } else { - flags.LdFlags = append(flags.LdFlags, hostDynamicGccLibs[ctx.Os()]...) - } + flags.Local.CppFlags = append(flags.Local.CppFlags, "-nostdinc++") + flags.extraLibFlags = append(flags.extraLibFlags, "-nostdlib++") if ctx.Windows() { + if stl.Properties.SelectedStl == "libc++_static" { + // These are transitively needed by libc++_static. + flags.extraLibFlags = append(flags.extraLibFlags, + "-lmsvcrt", "-lucrt") + } // Use SjLj exceptions for 32-bit. libgcc_eh implements SjLj // exception model for 32-bit. if ctx.Arch().ArchType == android.X86 { - flags.CppFlags = append(flags.CppFlags, "-fsjlj-exceptions") + flags.Local.CppFlags = append(flags.Local.CppFlags, "-fsjlj-exceptions") } - flags.CppFlags = append(flags.CppFlags, + flags.Local.CppFlags = append(flags.Local.CppFlags, // Disable visiblity annotations since we're using static // libc++. "-D_LIBCPP_DISABLE_VISIBILITY_ANNOTATIONS", @@ -236,29 +259,24 @@ func (stl *stl) flags(ctx ModuleContext, flags Flags) Flags { } } else { if ctx.Arch().ArchType == android.Arm { - flags.LdFlags = append(flags.LdFlags, "-Wl,--exclude-libs,libunwind_llvm.a") + flags.Local.LdFlags = append(flags.Local.LdFlags, "-Wl,--exclude-libs,libunwind_llvm.a") } } case "libstdc++": // Nothing case "ndk_system": ndkSrcRoot := android.PathForSource(ctx, "prebuilts/ndk/current/sources/cxx-stl/system/include") - flags.CFlags = append(flags.CFlags, "-isystem "+ndkSrcRoot.String()) + flags.Local.CFlags = append(flags.Local.CFlags, "-isystem "+ndkSrcRoot.String()) case "ndk_libc++_shared", "ndk_libc++_static": if ctx.Arch().ArchType == android.Arm { // Make sure the _Unwind_XXX symbols are not re-exported. - flags.LdFlags = append(flags.LdFlags, "-Wl,--exclude-libs,libunwind.a") + flags.Local.LdFlags = append(flags.Local.LdFlags, "-Wl,--exclude-libs,libunwind.a") } case "": // None or error. if !ctx.toolchain().Bionic() { - flags.CppFlags = append(flags.CppFlags, "-nostdinc++") - flags.LdFlags = append(flags.LdFlags, "-nodefaultlibs") - if ctx.staticBinary() { - flags.LdFlags = append(flags.LdFlags, hostStaticGccLibs[ctx.Os()]...) - } else { - flags.LdFlags = append(flags.LdFlags, hostDynamicGccLibs[ctx.Os()]...) - } + flags.Local.CppFlags = append(flags.Local.CppFlags, "-nostdinc++") + flags.extraLibFlags = append(flags.extraLibFlags, "-nostdlib++") } default: panic(fmt.Errorf("Unknown stl: %q", stl.Properties.SelectedStl)) @@ -266,22 +284,3 @@ func (stl *stl) flags(ctx ModuleContext, flags Flags) Flags { return flags } - -var hostDynamicGccLibs, hostStaticGccLibs map[android.OsType][]string - -func init() { - hostDynamicGccLibs = map[android.OsType][]string{ - android.Fuchsia: []string{"-lc", "-lunwind"}, - android.Linux: []string{"-lgcc_s", "-lgcc", "-lc", "-lgcc_s", "-lgcc"}, - android.Darwin: []string{"-lc", "-lSystem"}, - android.Windows: []string{"-Wl,--start-group", "-lmingw32", "-lgcc", "-lgcc_eh", - "-lmoldname", "-lmingwex", "-lmsvcrt", "-lucrt", "-lpthread", - "-ladvapi32", "-lshell32", "-luser32", "-lkernel32", "-lpsapi", - "-Wl,--end-group"}, - } - hostStaticGccLibs = map[android.OsType][]string{ - android.Linux: []string{"-Wl,--start-group", "-lgcc", "-lgcc_eh", "-lc", "-Wl,--end-group"}, - android.Darwin: []string{"NO_STATIC_HOST_BINARIES_ON_DARWIN"}, - android.Windows: []string{"NO_STATIC_HOST_BINARIES_ON_WINDOWS"}, - } -} diff --git a/cc/strip.go b/cc/strip.go index 7122585e3..7e560ec9c 100644 --- a/cc/strip.go +++ b/cc/strip.go @@ -22,11 +22,11 @@ import ( type StripProperties struct { Strip struct { - None *bool `android:"arch_variant"` - All *bool `android:"arch_variant"` - Keep_symbols *bool `android:"arch_variant"` - Keep_symbols_list []string `android:"arch_variant"` - Use_gnu_strip *bool `android:"arch_variant"` + None *bool `android:"arch_variant"` + All *bool `android:"arch_variant"` + Keep_symbols *bool `android:"arch_variant"` + Keep_symbols_list []string `android:"arch_variant"` + Keep_symbols_and_debug_frame *bool `android:"arch_variant"` } `android:"arch_variant"` } @@ -40,23 +40,32 @@ func (stripper *stripper) needsStrip(ctx ModuleContext) bool { } func (stripper *stripper) strip(ctx ModuleContext, in android.Path, out android.ModuleOutPath, - flags builderFlags) { + flags builderFlags, isStaticLib bool) { if ctx.Darwin() { TransformDarwinStrip(ctx, in, out) } else { if Bool(stripper.StripProperties.Strip.Keep_symbols) { flags.stripKeepSymbols = true + } else if Bool(stripper.StripProperties.Strip.Keep_symbols_and_debug_frame) { + flags.stripKeepSymbolsAndDebugFrame = true } else if len(stripper.StripProperties.Strip.Keep_symbols_list) > 0 { flags.stripKeepSymbolsList = strings.Join(stripper.StripProperties.Strip.Keep_symbols_list, ",") } else if !Bool(stripper.StripProperties.Strip.All) { flags.stripKeepMiniDebugInfo = true } - if Bool(stripper.StripProperties.Strip.Use_gnu_strip) { - flags.stripUseGnuStrip = true - } - if ctx.Config().Debuggable() && !flags.stripKeepMiniDebugInfo { + if ctx.Config().Debuggable() && !flags.stripKeepMiniDebugInfo && !isStaticLib { flags.stripAddGnuDebuglink = true } TransformStrip(ctx, in, out, flags) } } + +func (stripper *stripper) stripExecutableOrSharedLib(ctx ModuleContext, in android.Path, + out android.ModuleOutPath, flags builderFlags) { + stripper.strip(ctx, in, out, flags, false) +} + +func (stripper *stripper) stripStaticLib(ctx ModuleContext, in android.Path, out android.ModuleOutPath, + flags builderFlags) { + stripper.strip(ctx, in, out, flags, true) +} diff --git a/cc/sysprop.go b/cc/sysprop.go index 656f79f59..6cac7fb10 100644 --- a/cc/sysprop.go +++ b/cc/sysprop.go @@ -21,6 +21,7 @@ import ( ) type syspropLibraryInterface interface { + BaseModuleName() string CcModuleName() string } @@ -42,6 +43,6 @@ func SyspropMutator(mctx android.BottomUpMutatorContext) { syspropImplLibrariesLock.Lock() defer syspropImplLibrariesLock.Unlock() - syspropImplLibraries[mctx.ModuleName()] = m.CcModuleName() + syspropImplLibraries[m.BaseModuleName()] = m.CcModuleName() } } diff --git a/cc/test.go b/cc/test.go index c735fd91f..95abfbf66 100644 --- a/cc/test.go +++ b/cc/test.go @@ -16,6 +16,7 @@ package cc import ( "path/filepath" + "strconv" "strings" "android/soong/android" @@ -28,6 +29,11 @@ type TestProperties struct { // if set, use the isolated gtest runner. Defaults to false. Isolated *bool + + // List of APEXes that this module tests. The module has access to + // the private part of the listed APEXes even when it is not included in the + // APEXes. + Test_for []string } // Test option struct. @@ -48,7 +54,7 @@ type TestBinaryProperties struct { // list of files or filegroup modules that provide data that should be installed alongside // the test - Data []string `android:"path"` + Data []string `android:"path,arch_variant"` // list of compatibility suites (for example "cts", "vts") that the module should be // installed into. @@ -68,6 +74,26 @@ type TestBinaryProperties struct { // Add RootTargetPreparer to auto generated test config. This guarantees the test to run // with root permission. Require_root *bool + + // Add RunCommandTargetPreparer to stop framework before the test and start it after the test. + Disable_framework *bool + + // Add MinApiLevelModuleController to auto generated test config. If the device property of + // "ro.product.first_api_level" < Test_min_api_level, then skip this module. + Test_min_api_level *int64 + + // Add MinApiLevelModuleController to auto generated test config. If the device property of + // "ro.build.version.sdk" < Test_min_sdk_version, then skip this module. + Test_min_sdk_version *int64 + + // Flag to indicate whether or not to create test config automatically. If AndroidTest.xml + // doesn't exist next to the Android.bp, this attribute doesn't need to be set to true + // explicitly. + Auto_gen_config *bool + + // Add parameterized mainline modules to auto generated test config. The options will be + // handled by TradeFed to download and install the specified modules on the device. + Test_mainline_modules []string } func init() { @@ -121,7 +147,9 @@ func BenchmarkHostFactory() android.Module { type testPerSrc interface { testPerSrc() bool srcs() []string + isAllTestsVariation() bool setSrc(string, string) + unsetSrc() } func (test *testBinary) testPerSrc() bool { @@ -132,45 +160,61 @@ func (test *testBinary) srcs() []string { return test.baseCompiler.Properties.Srcs } +func (test *testBinary) isAllTestsVariation() bool { + stem := test.binaryDecorator.Properties.Stem + return stem != nil && *stem == "" +} + func (test *testBinary) setSrc(name, src string) { test.baseCompiler.Properties.Srcs = []string{src} test.binaryDecorator.Properties.Stem = StringPtr(name) } +func (test *testBinary) unsetSrc() { + test.baseCompiler.Properties.Srcs = nil + test.binaryDecorator.Properties.Stem = StringPtr("") +} + var _ testPerSrc = (*testBinary)(nil) -func testPerSrcMutator(mctx android.BottomUpMutatorContext) { +func TestPerSrcMutator(mctx android.BottomUpMutatorContext) { if m, ok := mctx.Module().(*Module); ok { if test, ok := m.linker.(testPerSrc); ok { - if test.testPerSrc() && len(test.srcs()) > 0 { - if duplicate, found := checkDuplicate(test.srcs()); found { + numTests := len(test.srcs()) + if test.testPerSrc() && numTests > 0 { + if duplicate, found := android.CheckDuplicate(test.srcs()); found { mctx.PropertyErrorf("srcs", "found a duplicate entry %q", duplicate) return } - testNames := make([]string, len(test.srcs())) + testNames := make([]string, numTests) for i, src := range test.srcs() { testNames[i] = strings.TrimSuffix(filepath.Base(src), filepath.Ext(src)) } + // In addition to creating one variation per test source file, + // create an additional "all tests" variation named "", and have it + // depends on all other test_per_src variations. This is useful to + // create subsequent dependencies of a given module on all + // test_per_src variations created above: by depending on + // variation "", that module will transitively depend on all the + // other test_per_src variations without the need to know their + // name or even their number. + testNames = append(testNames, "") tests := mctx.CreateLocalVariations(testNames...) + all_tests := tests[numTests] + all_tests.(*Module).linker.(testPerSrc).unsetSrc() + // Prevent the "all tests" variation from being installable nor + // exporting to Make, as it won't create any output file. + all_tests.(*Module).Properties.PreventInstall = true + all_tests.(*Module).Properties.HideFromMake = true for i, src := range test.srcs() { tests[i].(*Module).linker.(testPerSrc).setSrc(testNames[i], src) + mctx.AddInterVariantDependency(testPerSrcDepTag, all_tests, tests[i]) } } } } } -func checkDuplicate(values []string) (duplicate string, found bool) { - seen := make(map[string]string) - for _, v := range values { - if duplicate, found = seen[v]; found { - return - } - seen[v] = v - } - return -} - type testDecorator struct { Properties TestProperties linker *baseLinker @@ -180,25 +224,29 @@ func (test *testDecorator) gtest() bool { return BoolDefault(test.Properties.Gtest, true) } +func (test *testDecorator) testFor() []string { + return test.Properties.Test_for +} + func (test *testDecorator) linkerFlags(ctx ModuleContext, flags Flags) Flags { if !test.gtest() { return flags } - flags.CFlags = append(flags.CFlags, "-DGTEST_HAS_STD_STRING") + flags.Local.CFlags = append(flags.Local.CFlags, "-DGTEST_HAS_STD_STRING") if ctx.Host() { - flags.CFlags = append(flags.CFlags, "-O0", "-g") + flags.Local.CFlags = append(flags.Local.CFlags, "-O0", "-g") switch ctx.Os() { case android.Windows: - flags.CFlags = append(flags.CFlags, "-DGTEST_OS_WINDOWS") + flags.Local.CFlags = append(flags.Local.CFlags, "-DGTEST_OS_WINDOWS") case android.Linux: - flags.CFlags = append(flags.CFlags, "-DGTEST_OS_LINUX") + flags.Local.CFlags = append(flags.Local.CFlags, "-DGTEST_OS_LINUX") case android.Darwin: - flags.CFlags = append(flags.CFlags, "-DGTEST_OS_MAC") + flags.Local.CFlags = append(flags.Local.CFlags, "-DGTEST_OS_MAC") } } else { - flags.CFlags = append(flags.CFlags, "-DGTEST_OS_LINUX_ANDROID") + flags.Local.CFlags = append(flags.Local.CFlags, "-DGTEST_OS_LINUX_ANDROID") } return flags @@ -210,6 +258,11 @@ func (test *testDecorator) linkerDeps(ctx BaseModuleContext, deps Deps) Deps { deps.StaticLibs = append(deps.StaticLibs, "libgtest_main_ndk_c++", "libgtest_ndk_c++") } else if BoolDefault(test.Properties.Isolated, false) { deps.StaticLibs = append(deps.StaticLibs, "libgtest_isolated_main") + // The isolated library requires liblog, but adding it + // as a static library means unit tests cannot override + // liblog functions. Instead make it a shared library + // dependency. + deps.SharedLibs = append(deps.SharedLibs, "liblog") } else { deps.StaticLibs = append(deps.StaticLibs, "libgtest_main", "libgtest") } @@ -277,19 +330,47 @@ func (test *testBinary) linkerFlags(ctx ModuleContext, flags Flags) Flags { func (test *testBinary) install(ctx ModuleContext, file android.Path) { test.data = android.PathsForModuleSrc(ctx, test.Properties.Data) + var api_level_prop string var configs []tradefed.Config + var min_level string + for _, module := range test.Properties.Test_mainline_modules { + configs = append(configs, tradefed.Option{Name: "config-descriptor:metadata", Key: "mainline-param", Value: module}) + } if Bool(test.Properties.Require_root) { - configs = append(configs, tradefed.Preparer{"com.android.tradefed.targetprep.RootTargetPreparer"}) + configs = append(configs, tradefed.Object{"target_preparer", "com.android.tradefed.targetprep.RootTargetPreparer", nil}) + } else { + var options []tradefed.Option + options = append(options, tradefed.Option{Name: "force-root", Value: "false"}) + configs = append(configs, tradefed.Object{"target_preparer", "com.android.tradefed.targetprep.RootTargetPreparer", options}) + } + if Bool(test.Properties.Disable_framework) { + var options []tradefed.Option + configs = append(configs, tradefed.Object{"target_preparer", "com.android.tradefed.targetprep.StopServicesSetup", options}) } if Bool(test.testDecorator.Properties.Isolated) { - configs = append(configs, tradefed.Option{"not-shardable", "true"}) + configs = append(configs, tradefed.Option{Name: "not-shardable", Value: "true"}) } if test.Properties.Test_options.Run_test_as != nil { - configs = append(configs, tradefed.Option{"run-test-as", String(test.Properties.Test_options.Run_test_as)}) + configs = append(configs, tradefed.Option{Name: "run-test-as", Value: String(test.Properties.Test_options.Run_test_as)}) + } + if test.Properties.Test_min_api_level != nil && test.Properties.Test_min_sdk_version != nil { + ctx.PropertyErrorf("test_min_api_level", "'test_min_api_level' and 'test_min_sdk_version' should not be set at the same time.") + } else if test.Properties.Test_min_api_level != nil { + api_level_prop = "ro.product.first_api_level" + min_level = strconv.FormatInt(int64(*test.Properties.Test_min_api_level), 10) + } else if test.Properties.Test_min_sdk_version != nil { + api_level_prop = "ro.build.version.sdk" + min_level = strconv.FormatInt(int64(*test.Properties.Test_min_sdk_version), 10) + } + if api_level_prop != "" { + var options []tradefed.Option + options = append(options, tradefed.Option{Name: "min-api-level", Value: min_level}) + options = append(options, tradefed.Option{Name: "api-level-prop", Value: api_level_prop}) + configs = append(configs, tradefed.Object{"module_controller", "com.android.tradefed.testtype.suite.module.MinApiLevelModuleController", options}) } test.testConfig = tradefed.AutoGenNativeTestConfig(ctx, test.Properties.Test_config, - test.Properties.Test_config_template, test.Properties.Test_suites, configs) + test.Properties.Test_config_template, test.Properties.Test_suites, configs, test.Properties.Auto_gen_config) test.binaryDecorator.baseInstaller.dir = "nativetest" test.binaryDecorator.baseInstaller.dir64 = "nativetest64" @@ -380,6 +461,11 @@ type BenchmarkProperties struct { // Add RootTargetPreparer to auto generated test config. This guarantees the test to run // with root permission. Require_root *bool + + // Flag to indicate whether or not to create test config automatically. If AndroidTest.xml + // doesn't exist next to the Android.bp, this attribute doesn't need to be set to true + // explicitly. + Auto_gen_config *bool } type benchmarkDecorator struct { @@ -414,10 +500,10 @@ func (benchmark *benchmarkDecorator) install(ctx ModuleContext, file android.Pat benchmark.data = android.PathsForModuleSrc(ctx, benchmark.Properties.Data) var configs []tradefed.Config if Bool(benchmark.Properties.Require_root) { - configs = append(configs, tradefed.Preparer{"com.android.tradefed.targetprep.RootTargetPreparer"}) + configs = append(configs, tradefed.Object{"target_preparer", "com.android.tradefed.targetprep.RootTargetPreparer", nil}) } benchmark.testConfig = tradefed.AutoGenNativeBenchmarkTestConfig(ctx, benchmark.Properties.Test_config, - benchmark.Properties.Test_config_template, benchmark.Properties.Test_suites, configs) + benchmark.Properties.Test_config_template, benchmark.Properties.Test_suites, configs, benchmark.Properties.Auto_gen_config) benchmark.binaryDecorator.baseInstaller.dir = filepath.Join("benchmarktest", ctx.ModuleName()) benchmark.binaryDecorator.baseInstaller.dir64 = filepath.Join("benchmarktest64", ctx.ModuleName()) diff --git a/cc/test_data_test.go b/cc/test_data_test.go index 21ea765eb..ae59e2f0f 100644 --- a/cc/test_data_test.go +++ b/cc/test_data_test.go @@ -115,22 +115,17 @@ func TestDataTests(t *testing.T) { } defer os.RemoveAll(buildDir) - config := android.TestConfig(buildDir, nil) - for _, test := range testDataTests { t.Run(test.name, func(t *testing.T) { - ctx := android.NewTestContext() - ctx.MockFileSystem(map[string][]byte{ - "Blueprints": []byte(`subdirs = ["dir"]`), - "dir/Blueprints": []byte(test.modules), + config := android.TestConfig(buildDir, nil, "", map[string][]byte{ + "dir/Android.bp": []byte(test.modules), "dir/baz": nil, "dir/bar/baz": nil, }) - ctx.RegisterModuleType("filegroup", - android.ModuleFactoryAdaptor(android.FileGroupFactory)) - ctx.RegisterModuleType("test", - android.ModuleFactoryAdaptor(newTest)) - ctx.Register() + ctx := android.NewTestContext() + ctx.RegisterModuleType("filegroup", android.FileGroupFactory) + ctx.RegisterModuleType("test", newTest) + ctx.Register(config) _, errs := ctx.ParseBlueprintsFiles("Blueprints") android.FailIfErrored(t, errs) diff --git a/cc/test_gen_stub_libs.py b/cc/test_gen_stub_libs.py index 2ee988693..0b45e7110 100755 --- a/cc/test_gen_stub_libs.py +++ b/cc/test_gen_stub_libs.py @@ -179,17 +179,17 @@ class OmitVersionTest(unittest.TestCase): gsl.Version('foo', None, ['platform-only'], []), 'arm', 9, False, False)) - def test_omit_vndk(self): + def test_omit_llndk(self): self.assertTrue( gsl.should_omit_version( - gsl.Version('foo', None, ['vndk'], []), 'arm', 9, False, False)) + gsl.Version('foo', None, ['llndk'], []), 'arm', 9, False, False)) self.assertFalse( gsl.should_omit_version( gsl.Version('foo', None, [], []), 'arm', 9, True, False)) self.assertFalse( gsl.should_omit_version( - gsl.Version('foo', None, ['vndk'], []), 'arm', 9, True, False)) + gsl.Version('foo', None, ['llndk'], []), 'arm', 9, True, False)) def test_omit_apex(self): self.assertTrue( @@ -231,16 +231,16 @@ class OmitVersionTest(unittest.TestCase): class OmitSymbolTest(unittest.TestCase): - def test_omit_vndk(self): + def test_omit_llndk(self): self.assertTrue( gsl.should_omit_symbol( - gsl.Symbol('foo', ['vndk']), 'arm', 9, False, False)) + gsl.Symbol('foo', ['llndk']), 'arm', 9, False, False)) self.assertFalse( gsl.should_omit_symbol(gsl.Symbol('foo', []), 'arm', 9, True, False)) self.assertFalse( gsl.should_omit_symbol( - gsl.Symbol('foo', ['vndk']), 'arm', 9, True, False)) + gsl.Symbol('foo', ['llndk']), 'arm', 9, True, False)) def test_omit_apex(self): self.assertTrue( @@ -441,12 +441,12 @@ class SymbolFileParseTest(unittest.TestCase): self.assertEqual(expected, versions) - def test_parse_vndk_apex_symbol(self): + def test_parse_llndk_apex_symbol(self): input_file = io.StringIO(textwrap.dedent("""\ VERSION_1 { foo; - bar; # vndk - baz; # vndk apex + bar; # llndk + baz; # llndk apex qux; # apex }; """)) @@ -459,8 +459,8 @@ class SymbolFileParseTest(unittest.TestCase): expected_symbols = [ gsl.Symbol('foo', []), - gsl.Symbol('bar', ['vndk']), - gsl.Symbol('baz', ['vndk', 'apex']), + gsl.Symbol('bar', ['llndk']), + gsl.Symbol('baz', ['llndk', 'apex']), gsl.Symbol('qux', ['apex']), ] self.assertEqual(expected_symbols, version.symbols) @@ -517,7 +517,7 @@ class GeneratorTest(unittest.TestCase): self.assertEqual('', version_file.getvalue()) version = gsl.Version('VERSION_1', None, [], [ - gsl.Symbol('foo', ['vndk']), + gsl.Symbol('foo', ['llndk']), ]) generator.write_version(version) self.assertEqual('', src_file.getvalue()) @@ -607,7 +607,7 @@ class IntegrationTest(unittest.TestCase): VERSION_4 { # versioned=9 wibble; - wizzes; # vndk + wizzes; # llndk waggle; # apex } VERSION_2; @@ -749,10 +749,10 @@ class IntegrationTest(unittest.TestCase): VERSION_4 { # versioned=9 wibble; - wizzes; # vndk + wizzes; # llndk waggle; # apex - bubble; # apex vndk - duddle; # vndk apex + bubble; # apex llndk + duddle; # llndk apex } VERSION_2; VERSION_5 { # versioned=14 diff --git a/cc/testing.go b/cc/testing.go index 8d76c2f5a..7a86a8d28 100644 --- a/cc/testing.go +++ b/cc/testing.go @@ -18,12 +18,29 @@ import ( "android/soong/android" ) -func GatherRequiredDepsForTest(os android.OsType) string { +func RegisterRequiredBuildComponentsForTest(ctx android.RegistrationContext) { + RegisterPrebuiltBuildComponents(ctx) + android.RegisterPrebuiltMutators(ctx) + + RegisterCCBuildComponents(ctx) + RegisterBinaryBuildComponents(ctx) + RegisterLibraryBuildComponents(ctx) + RegisterLibraryHeadersBuildComponents(ctx) + + ctx.RegisterModuleType("toolchain_library", ToolchainLibraryFactory) + ctx.RegisterModuleType("llndk_library", LlndkLibraryFactory) + ctx.RegisterModuleType("cc_object", ObjectFactory) + ctx.RegisterModuleType("ndk_prebuilt_shared_stl", NdkPrebuiltSharedStlFactory) + ctx.RegisterModuleType("ndk_prebuilt_object", NdkPrebuiltObjectFactory) +} + +func GatherRequiredDepsForTest(oses ...android.OsType) string { ret := ` toolchain_library { name: "libatomic", vendor_available: true, recovery_available: true, + native_bridge_supported: true, src: "", } @@ -38,6 +55,7 @@ func GatherRequiredDepsForTest(os android.OsType) string { name: "libclang_rt.builtins-arm-android", vendor_available: true, recovery_available: true, + native_bridge_supported: true, src: "", } @@ -45,13 +63,29 @@ func GatherRequiredDepsForTest(os android.OsType) string { name: "libclang_rt.builtins-aarch64-android", vendor_available: true, recovery_available: true, + native_bridge_supported: true, src: "", } + cc_prebuilt_library_shared { + name: "libclang_rt.hwasan-aarch64-android", + nocrt: true, + vendor_available: true, + recovery_available: true, + system_shared_libs: [], + stl: "none", + srcs: [""], + check_elf_files: false, + sanitize: { + never: true, + }, + } + toolchain_library { name: "libclang_rt.builtins-i686-android", vendor_available: true, recovery_available: true, + native_bridge_supported: true, src: "", } @@ -59,10 +93,55 @@ func GatherRequiredDepsForTest(os android.OsType) string { name: "libclang_rt.builtins-x86_64-android", vendor_available: true, recovery_available: true, + native_bridge_supported: true, + src: "", + } + + toolchain_library { + name: "libclang_rt.fuzzer-arm-android", + vendor_available: true, + recovery_available: true, + src: "", + } + + toolchain_library { + name: "libclang_rt.fuzzer-aarch64-android", + vendor_available: true, + recovery_available: true, + src: "", + } + + toolchain_library { + name: "libclang_rt.fuzzer-i686-android", + vendor_available: true, + recovery_available: true, + src: "", + } + + toolchain_library { + name: "libclang_rt.fuzzer-x86_64-android", + vendor_available: true, + recovery_available: true, src: "", } toolchain_library { + name: "libclang_rt.fuzzer-x86_64", + vendor_available: true, + recovery_available: true, + src: "", + } + + // Needed for sanitizer + cc_prebuilt_library_shared { + name: "libclang_rt.ubsan_standalone-aarch64-android", + vendor_available: true, + recovery_available: true, + system_shared_libs: [], + srcs: [""], + } + + toolchain_library { name: "libgcc", vendor_available: true, recovery_available: true, @@ -73,67 +152,128 @@ func GatherRequiredDepsForTest(os android.OsType) string { name: "libgcc_stripped", vendor_available: true, recovery_available: true, + sdk_version: "current", src: "", } cc_library { name: "libc", - no_libgcc: true, + no_libcrt: true, nocrt: true, + stl: "none", system_shared_libs: [], recovery_available: true, + stubs: { + versions: ["27", "28", "29"], + }, } llndk_library { name: "libc", symbol_file: "", + sdk_version: "current", } cc_library { name: "libm", - no_libgcc: true, + no_libcrt: true, nocrt: true, + stl: "none", system_shared_libs: [], recovery_available: true, + stubs: { + versions: ["27", "28", "29"], + }, + apex_available: [ + "//apex_available:platform", + "myapex" + ], } llndk_library { name: "libm", symbol_file: "", + sdk_version: "current", } cc_library { name: "libdl", - no_libgcc: true, + no_libcrt: true, nocrt: true, + stl: "none", system_shared_libs: [], recovery_available: true, + stubs: { + versions: ["27", "28", "29"], + }, + apex_available: [ + "//apex_available:platform", + "myapex" + ], } llndk_library { name: "libdl", symbol_file: "", + sdk_version: "current", + } + cc_library { + name: "libft2", + no_libcrt: true, + nocrt: true, + system_shared_libs: [], + recovery_available: true, + } + llndk_library { + name: "libft2", + symbol_file: "", + vendor_available: false, + sdk_version: "current", } cc_library { name: "libc++_static", - no_libgcc: true, + no_libcrt: true, nocrt: true, system_shared_libs: [], stl: "none", vendor_available: true, recovery_available: true, + host_supported: true, + apex_available: [ + "//apex_available:platform", + "//apex_available:anyapex", + ], } cc_library { name: "libc++", - no_libgcc: true, + no_libcrt: true, nocrt: true, system_shared_libs: [], stl: "none", vendor_available: true, recovery_available: true, + host_supported: true, vndk: { enabled: true, support_system_process: true, }, + apex_available: [ + "//apex_available:platform", + "myapex" + ], + } + cc_library { + name: "libc++demangle", + no_libcrt: true, + nocrt: true, + system_shared_libs: [], + stl: "none", + host_supported: false, + vendor_available: true, + recovery_available: true, + apex_available: [ + "//apex_available:platform", + "//apex_available:anyapex", + ], } cc_library { name: "libunwind_llvm", - no_libgcc: true, + no_libcrt: true, nocrt: true, system_shared_libs: [], stl: "none", @@ -141,36 +281,123 @@ func GatherRequiredDepsForTest(os android.OsType) string { recovery_available: true, } + cc_defaults { + name: "crt_defaults", + recovery_available: true, + vendor_available: true, + native_bridge_supported: true, + stl: "none", + apex_available: [ + "//apex_available:platform", + "//apex_available:anyapex", + ], + } + cc_object { name: "crtbegin_so", + defaults: ["crt_defaults"], + recovery_available: true, + vendor_available: true, + native_bridge_supported: true, + stl: "none", + } + + cc_object { + name: "crtbegin_dynamic", + defaults: ["crt_defaults"], recovery_available: true, vendor_available: true, + native_bridge_supported: true, + stl: "none", } cc_object { name: "crtbegin_static", + defaults: ["crt_defaults"], recovery_available: true, vendor_available: true, + native_bridge_supported: true, + stl: "none", } cc_object { name: "crtend_so", + defaults: ["crt_defaults"], recovery_available: true, vendor_available: true, + native_bridge_supported: true, + stl: "none", } cc_object { name: "crtend_android", + defaults: ["crt_defaults"], recovery_available: true, vendor_available: true, + native_bridge_supported: true, + stl: "none", } cc_library { name: "libprotobuf-cpp-lite", } - ` - if os == android.Fuchsia { - ret += ` + + cc_library { + name: "ndk_libunwind", + sdk_version: "current", + stl: "none", + system_shared_libs: [], + } + + cc_library { + name: "libc.ndk.current", + sdk_version: "current", + stl: "none", + system_shared_libs: [], + } + + cc_library { + name: "libm.ndk.current", + sdk_version: "current", + stl: "none", + system_shared_libs: [], + } + + cc_library { + name: "libdl.ndk.current", + sdk_version: "current", + stl: "none", + system_shared_libs: [], + } + + ndk_prebuilt_object { + name: "ndk_crtbegin_so.27", + sdk_version: "27", + } + + ndk_prebuilt_object { + name: "ndk_crtend_so.27", + sdk_version: "27", + } + + ndk_prebuilt_object { + name: "ndk_crtbegin_dynamic.27", + sdk_version: "27", + } + + ndk_prebuilt_object { + name: "ndk_crtend_android.27", + sdk_version: "27", + } + + ndk_prebuilt_shared_stl { + name: "ndk_libc++_shared", + } + ` + + for _, os := range oses { + if os == android.Fuchsia { + ret += ` cc_library { name: "libbioniccompat", stl: "none", @@ -180,6 +407,67 @@ func GatherRequiredDepsForTest(os android.OsType) string { stl: "none", } ` + } + if os == android.Windows { + ret += ` + toolchain_library { + name: "libwinpthread", + host_supported: true, + enabled: false, + target: { + windows: { + enabled: true, + }, + }, + src: "", + } + ` + } } return ret } + +func GatherRequiredFilesForTest(fs map[string][]byte) { +} + +func TestConfig(buildDir string, os android.OsType, env map[string]string, + bp string, fs map[string][]byte) android.Config { + + // add some modules that are required by the compiler and/or linker + bp = bp + GatherRequiredDepsForTest(os) + + mockFS := map[string][]byte{} + + GatherRequiredFilesForTest(mockFS) + + for k, v := range fs { + mockFS[k] = v + } + + var config android.Config + if os == android.Fuchsia { + config = android.TestArchConfigFuchsia(buildDir, env, bp, mockFS) + } else { + config = android.TestArchConfig(buildDir, env, bp, mockFS) + } + + return config +} + +func CreateTestContext() *android.TestContext { + ctx := android.NewTestArchContext() + ctx.RegisterModuleType("cc_fuzz", FuzzFactory) + ctx.RegisterModuleType("cc_test", TestFactory) + ctx.RegisterModuleType("llndk_headers", llndkHeadersFactory) + ctx.RegisterModuleType("ndk_library", NdkLibraryFactory) + ctx.RegisterModuleType("vendor_public_library", vendorPublicLibraryFactory) + ctx.RegisterModuleType("filegroup", android.FileGroupFactory) + ctx.RegisterModuleType("vndk_prebuilt_shared", VndkPrebuiltSharedFactory) + ctx.RegisterModuleType("vndk_libraries_txt", VndkLibrariesTxtFactory) + ctx.PreArchMutators(android.RegisterDefaultsPreArchMutators) + RegisterRequiredBuildComponentsForTest(ctx) + ctx.RegisterSingletonType("vndk-snapshot", VndkSnapshotSingleton) + ctx.RegisterSingletonType("vendor-snapshot", VendorSnapshotSingleton) + + return ctx +} diff --git a/cc/tidy.go b/cc/tidy.go index 545539232..364e56c5d 100644 --- a/cc/tidy.go +++ b/cc/tidy.go @@ -117,6 +117,16 @@ func (tidy *tidyFeature) flags(ctx ModuleContext, flags Flags) Flags { // which is used in many Android files. tidyChecks = tidyChecks + ",-cert-dcl16-c" } + // https://b.corp.google.com/issues/153464409 + // many local projects enable cert-* checks, which + // trigger bugprone-reserved-identifier. + tidyChecks = tidyChecks + ",-bugprone-reserved-identifier*,-cert-dcl51-cpp,-cert-dcl37-c" + // http://b/153757728 + tidyChecks = tidyChecks + ",-readability-qualified-auto" + // http://b/155034563 + tidyChecks = tidyChecks + ",-bugprone-signed-char-misuse" + // http://b/155034972 + tidyChecks = tidyChecks + ",-bugprone-branch-clone" flags.TidyFlags = append(flags.TidyFlags, tidyChecks) if len(tidy.Properties.Tidy_checks_as_errors) > 0 { diff --git a/cc/toolchain_library.go b/cc/toolchain_library.go index 8ab8bc946..042e012d9 100644 --- a/cc/toolchain_library.go +++ b/cc/toolchain_library.go @@ -29,6 +29,9 @@ func init() { type toolchainLibraryProperties struct { // the prebuilt toolchain library, as a path from the top of the source tree Src *string `android:"arch_variant"` + + // Repack the archive with only the selected objects. + Repack_objects_to_keep []string `android:"arch_variant"` } type toolchainLibraryDecorator struct { @@ -50,6 +53,9 @@ func (library *toolchainLibraryDecorator) linkerProps() []interface{} { return append(props, &library.Properties, &library.stripper.StripProperties) } +// toolchain_library is used internally by the build tool to link the specified +// static library in src property to the device libraries that are shipped with +// gcc. func ToolchainLibraryFactory() android.Module { module, library := NewLibrary(android.HostAndDeviceSupported) library.BuildOnlyStatic() @@ -61,6 +67,7 @@ func ToolchainLibraryFactory() android.Module { module.stl = nil module.sanitize = nil module.installer = nil + module.Properties.Sdk_version = StringPtr("current") return module.Init() } @@ -83,7 +90,15 @@ func (library *toolchainLibraryDecorator) link(ctx ModuleContext, fileName := ctx.ModuleName() + staticLibraryExtension outputFile := android.PathForModuleOut(ctx, fileName) buildFlags := flagsToBuilderFlags(flags) - library.stripper.strip(ctx, srcPath, outputFile, buildFlags) + library.stripper.stripStaticLib(ctx, srcPath, outputFile, buildFlags) + return outputFile + } + + if library.Properties.Repack_objects_to_keep != nil { + fileName := ctx.ModuleName() + staticLibraryExtension + outputFile := android.PathForModuleOut(ctx, fileName) + TransformArchiveRepack(ctx, srcPath, outputFile, library.Properties.Repack_objects_to_keep) + return outputFile } diff --git a/cc/util.go b/cc/util.go index 5dcbaeffb..af26268e2 100644 --- a/cc/util.go +++ b/cc/util.go @@ -29,10 +29,6 @@ func includeDirsToFlags(dirs android.Paths) string { return android.JoinWithPrefix(dirs.Strings(), "-I") } -func includeFilesToFlags(files android.Paths) string { - return android.JoinWithPrefix(files.Strings(), "-include ") -} - func ldDirsToFlags(dirs []string) string { return android.JoinWithPrefix(dirs, "-L") } @@ -59,34 +55,48 @@ func moduleToLibName(module string) (string, error) { func flagsToBuilderFlags(in Flags) builderFlags { return builderFlags{ - globalFlags: strings.Join(in.GlobalFlags, " "), - arFlags: strings.Join(in.ArFlags, " "), - asFlags: strings.Join(in.AsFlags, " "), - cFlags: strings.Join(in.CFlags, " "), - toolingCFlags: strings.Join(in.ToolingCFlags, " "), - toolingCppFlags: strings.Join(in.ToolingCppFlags, " "), - conlyFlags: strings.Join(in.ConlyFlags, " "), - cppFlags: strings.Join(in.CppFlags, " "), - yaccFlags: strings.Join(in.YaccFlags, " "), - aidlFlags: strings.Join(in.aidlFlags, " "), - rsFlags: strings.Join(in.rsFlags, " "), - ldFlags: strings.Join(in.LdFlags, " "), - libFlags: strings.Join(in.libFlags, " "), - tidyFlags: strings.Join(in.TidyFlags, " "), - sAbiFlags: strings.Join(in.SAbiFlags, " "), - yasmFlags: strings.Join(in.YasmFlags, " "), - toolchain: in.Toolchain, - coverage: in.Coverage, - tidy: in.Tidy, - sAbiDump: in.SAbiDump, + globalCommonFlags: strings.Join(in.Global.CommonFlags, " "), + globalAsFlags: strings.Join(in.Global.AsFlags, " "), + globalYasmFlags: strings.Join(in.Global.YasmFlags, " "), + globalCFlags: strings.Join(in.Global.CFlags, " "), + globalToolingCFlags: strings.Join(in.Global.ToolingCFlags, " "), + globalToolingCppFlags: strings.Join(in.Global.ToolingCppFlags, " "), + globalConlyFlags: strings.Join(in.Global.ConlyFlags, " "), + globalCppFlags: strings.Join(in.Global.CppFlags, " "), + globalLdFlags: strings.Join(in.Global.LdFlags, " "), + + localCommonFlags: strings.Join(in.Local.CommonFlags, " "), + localAsFlags: strings.Join(in.Local.AsFlags, " "), + localYasmFlags: strings.Join(in.Local.YasmFlags, " "), + localCFlags: strings.Join(in.Local.CFlags, " "), + localToolingCFlags: strings.Join(in.Local.ToolingCFlags, " "), + localToolingCppFlags: strings.Join(in.Local.ToolingCppFlags, " "), + localConlyFlags: strings.Join(in.Local.ConlyFlags, " "), + localCppFlags: strings.Join(in.Local.CppFlags, " "), + localLdFlags: strings.Join(in.Local.LdFlags, " "), + + aidlFlags: strings.Join(in.aidlFlags, " "), + rsFlags: strings.Join(in.rsFlags, " "), + libFlags: strings.Join(in.libFlags, " "), + extraLibFlags: strings.Join(in.extraLibFlags, " "), + tidyFlags: strings.Join(in.TidyFlags, " "), + sAbiFlags: strings.Join(in.SAbiFlags, " "), + toolchain: in.Toolchain, + gcovCoverage: in.GcovCoverage, + tidy: in.Tidy, + sAbiDump: in.SAbiDump, + emitXrefs: in.EmitXrefs, systemIncludeFlags: strings.Join(in.SystemIncludeFlags, " "), - groupStaticLibs: in.GroupStaticLibs, + assemblerWithCpp: in.AssemblerWithCpp, + groupStaticLibs: in.GroupStaticLibs, proto: in.proto, protoC: in.protoC, protoOptionsFile: in.protoOptionsFile, + + yacc: in.Yacc, } } @@ -104,32 +114,6 @@ func addSuffix(list []string, suffix string) []string { return list } -var shlibVersionPattern = regexp.MustCompile("(?:\\.\\d+(?:svn)?)+") - -// splitFileExt splits a file name into root, suffix and ext. root stands for the file name without -// the file extension and the version number (e.g. "libexample"). suffix stands for the -// concatenation of the file extension and the version number (e.g. ".so.1.0"). ext stands for the -// file extension after the version numbers are trimmed (e.g. ".so"). -func splitFileExt(name string) (string, string, string) { - // Extract and trim the shared lib version number if the file name ends with dot digits. - suffix := "" - matches := shlibVersionPattern.FindAllStringIndex(name, -1) - if len(matches) > 0 { - lastMatch := matches[len(matches)-1] - if lastMatch[1] == len(name) { - suffix = name[lastMatch[0]:lastMatch[1]] - name = name[0:lastMatch[0]] - } - } - - // Extract the file name root and the file extension. - ext := filepath.Ext(name) - root := strings.TrimSuffix(name, ext) - suffix = ext + suffix - - return root, suffix, ext -} - // linkDirOnDevice/linkName -> target func makeSymlinkCmd(linkDirOnDevice string, linkName string, target string) string { dir := filepath.Join("$(PRODUCT_OUT)", linkDirOnDevice) diff --git a/cc/util_test.go b/cc/util_test.go deleted file mode 100644 index 7c718eae0..000000000 --- a/cc/util_test.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2018 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cc - -import ( - "testing" -) - -func TestSplitFileExt(t *testing.T) { - t.Run("soname with version", func(t *testing.T) { - root, suffix, ext := splitFileExt("libtest.so.1.0.30") - expected := "libtest" - if root != expected { - t.Errorf("root should be %q but got %q", expected, root) - } - expected = ".so.1.0.30" - if suffix != expected { - t.Errorf("suffix should be %q but got %q", expected, suffix) - } - expected = ".so" - if ext != expected { - t.Errorf("ext should be %q but got %q", expected, ext) - } - }) - - t.Run("soname with svn version", func(t *testing.T) { - root, suffix, ext := splitFileExt("libtest.so.1svn") - expected := "libtest" - if root != expected { - t.Errorf("root should be %q but got %q", expected, root) - } - expected = ".so.1svn" - if suffix != expected { - t.Errorf("suffix should be %q but got %q", expected, suffix) - } - expected = ".so" - if ext != expected { - t.Errorf("ext should be %q but got %q", expected, ext) - } - }) - - t.Run("version numbers in the middle should be ignored", func(t *testing.T) { - root, suffix, ext := splitFileExt("libtest.1.0.30.so") - expected := "libtest.1.0.30" - if root != expected { - t.Errorf("root should be %q but got %q", expected, root) - } - expected = ".so" - if suffix != expected { - t.Errorf("suffix should be %q but got %q", expected, suffix) - } - expected = ".so" - if ext != expected { - t.Errorf("ext should be %q but got %q", expected, ext) - } - }) - - t.Run("no known file extension", func(t *testing.T) { - root, suffix, ext := splitFileExt("test.exe") - expected := "test" - if root != expected { - t.Errorf("root should be %q but got %q", expected, root) - } - expected = ".exe" - if suffix != expected { - t.Errorf("suffix should be %q but got %q", expected, suffix) - } - if ext != expected { - t.Errorf("ext should be %q but got %q", expected, ext) - } - }) -} diff --git a/cc/vendor_public_library.go b/cc/vendor_public_library.go index da41cbc31..e9d1c7378 100644 --- a/cc/vendor_public_library.go +++ b/cc/vendor_public_library.go @@ -24,10 +24,16 @@ import ( var ( vendorPublicLibrarySuffix = ".vendorpublic" - vendorPublicLibraries = []string{} + vendorPublicLibrariesKey = android.NewOnceKey("vendorPublicLibraries") vendorPublicLibrariesLock sync.Mutex ) +func vendorPublicLibraries(config android.Config) *[]string { + return config.Once(vendorPublicLibrariesKey, func() interface{} { + return &[]string{} + }).(*[]string) +} + // Creates a stub shared library for a vendor public library. Vendor public libraries // are vendor libraries (owned by them and installed to /vendor partition) that are // exposed to Android apps via JNI. The libraries are made public by being listed in @@ -82,12 +88,13 @@ func (stub *vendorPublicLibraryStubDecorator) compilerInit(ctx BaseModuleContext vendorPublicLibrariesLock.Lock() defer vendorPublicLibrariesLock.Unlock() - for _, lib := range vendorPublicLibraries { + vendorPublicLibraries := vendorPublicLibraries(ctx.Config()) + for _, lib := range *vendorPublicLibraries { if lib == name { return } } - vendorPublicLibraries = append(vendorPublicLibraries, name) + *vendorPublicLibraries = append(*vendorPublicLibraries, name) } func (stub *vendorPublicLibraryStubDecorator) compilerFlags(ctx ModuleContext, flags Flags, deps PathDeps) Flags { @@ -117,11 +124,21 @@ func (stub *vendorPublicLibraryStubDecorator) link(ctx ModuleContext, flags Flag objs Objects) android.Path { if !Bool(stub.Properties.Unversioned) { linkerScriptFlag := "-Wl,--version-script," + stub.versionScriptPath.String() - flags.LdFlags = append(flags.LdFlags, linkerScriptFlag) + flags.Local.LdFlags = append(flags.Local.LdFlags, linkerScriptFlag) + flags.LdFlagsDeps = append(flags.LdFlagsDeps, stub.versionScriptPath) } return stub.libraryDecorator.link(ctx, flags, deps, objs) } +// vendor_public_library creates a stub shared library for a vendor public +// library. This stub library is a build-time only artifact that provides +// symbols that are exposed from a vendor public library. Example: +// +// vendor_public_library { +// name: "libfoo", +// symbol_file: "libfoo.map.txt", +// export_public_headers: ["libfoo_headers"], +// } func vendorPublicLibraryFactory() android.Module { module, library := NewLibrary(android.DeviceSupported) library.BuildOnlyShared() diff --git a/cc/vendor_snapshot.go b/cc/vendor_snapshot.go new file mode 100644 index 000000000..b11b1a82e --- /dev/null +++ b/cc/vendor_snapshot.go @@ -0,0 +1,983 @@ +// Copyright 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package cc + +import ( + "encoding/json" + "path/filepath" + "sort" + "strings" + "sync" + + "github.com/google/blueprint/proptools" + + "android/soong/android" +) + +const ( + vendorSnapshotHeaderSuffix = ".vendor_header." + vendorSnapshotSharedSuffix = ".vendor_shared." + vendorSnapshotStaticSuffix = ".vendor_static." + vendorSnapshotBinarySuffix = ".vendor_binary." + vendorSnapshotObjectSuffix = ".vendor_object." +) + +var ( + vendorSnapshotsLock sync.Mutex + vendorSuffixModulesKey = android.NewOnceKey("vendorSuffixModules") + vendorSnapshotHeaderLibsKey = android.NewOnceKey("vendorSnapshotHeaderLibs") + vendorSnapshotStaticLibsKey = android.NewOnceKey("vendorSnapshotStaticLibs") + vendorSnapshotSharedLibsKey = android.NewOnceKey("vendorSnapshotSharedLibs") + vendorSnapshotBinariesKey = android.NewOnceKey("vendorSnapshotBinaries") + vendorSnapshotObjectsKey = android.NewOnceKey("vendorSnapshotObjects") +) + +// vendor snapshot maps hold names of vendor snapshot modules per arch +func vendorSuffixModules(config android.Config) map[string]bool { + return config.Once(vendorSuffixModulesKey, func() interface{} { + return make(map[string]bool) + }).(map[string]bool) +} + +func vendorSnapshotHeaderLibs(config android.Config) *snapshotMap { + return config.Once(vendorSnapshotHeaderLibsKey, func() interface{} { + return newSnapshotMap() + }).(*snapshotMap) +} + +func vendorSnapshotSharedLibs(config android.Config) *snapshotMap { + return config.Once(vendorSnapshotSharedLibsKey, func() interface{} { + return newSnapshotMap() + }).(*snapshotMap) +} + +func vendorSnapshotStaticLibs(config android.Config) *snapshotMap { + return config.Once(vendorSnapshotStaticLibsKey, func() interface{} { + return newSnapshotMap() + }).(*snapshotMap) +} + +func vendorSnapshotBinaries(config android.Config) *snapshotMap { + return config.Once(vendorSnapshotBinariesKey, func() interface{} { + return newSnapshotMap() + }).(*snapshotMap) +} + +func vendorSnapshotObjects(config android.Config) *snapshotMap { + return config.Once(vendorSnapshotObjectsKey, func() interface{} { + return newSnapshotMap() + }).(*snapshotMap) +} + +type vendorSnapshotLibraryProperties struct { + // snapshot version. + Version string + + // Target arch name of the snapshot (e.g. 'arm64' for variant 'aosp_arm64') + Target_arch string + + // Prebuilt file for each arch. + Src *string `android:"arch_variant"` + + // list of flags that will be used for any module that links against this module. + Export_flags []string `android:"arch_variant"` + + // Check the prebuilt ELF files (e.g. DT_SONAME, DT_NEEDED, resolution of undefined symbols, + // etc). + Check_elf_files *bool + + // Whether this prebuilt needs to depend on sanitize ubsan runtime or not. + Sanitize_ubsan_dep *bool `android:"arch_variant"` + + // Whether this prebuilt needs to depend on sanitize minimal runtime or not. + Sanitize_minimal_dep *bool `android:"arch_variant"` +} + +type vendorSnapshotLibraryDecorator struct { + *libraryDecorator + properties vendorSnapshotLibraryProperties + androidMkVendorSuffix bool +} + +func (p *vendorSnapshotLibraryDecorator) Name(name string) string { + return name + p.NameSuffix() +} + +func (p *vendorSnapshotLibraryDecorator) NameSuffix() string { + versionSuffix := p.version() + if p.arch() != "" { + versionSuffix += "." + p.arch() + } + + var linkageSuffix string + if p.buildShared() { + linkageSuffix = vendorSnapshotSharedSuffix + } else if p.buildStatic() { + linkageSuffix = vendorSnapshotStaticSuffix + } else { + linkageSuffix = vendorSnapshotHeaderSuffix + } + + return linkageSuffix + versionSuffix +} + +func (p *vendorSnapshotLibraryDecorator) version() string { + return p.properties.Version +} + +func (p *vendorSnapshotLibraryDecorator) arch() string { + return p.properties.Target_arch +} + +func (p *vendorSnapshotLibraryDecorator) linkerFlags(ctx ModuleContext, flags Flags) Flags { + p.libraryDecorator.libName = strings.TrimSuffix(ctx.ModuleName(), p.NameSuffix()) + return p.libraryDecorator.linkerFlags(ctx, flags) +} + +func (p *vendorSnapshotLibraryDecorator) matchesWithDevice(config android.DeviceConfig) bool { + arches := config.Arches() + if len(arches) == 0 || arches[0].ArchType.String() != p.arch() { + return false + } + if !p.header() && p.properties.Src == nil { + return false + } + return true +} + +func (p *vendorSnapshotLibraryDecorator) link(ctx ModuleContext, + flags Flags, deps PathDeps, objs Objects) android.Path { + m := ctx.Module().(*Module) + p.androidMkVendorSuffix = vendorSuffixModules(ctx.Config())[m.BaseModuleName()] + + if p.header() { + return p.libraryDecorator.link(ctx, flags, deps, objs) + } + + if !p.matchesWithDevice(ctx.DeviceConfig()) { + return nil + } + + p.libraryDecorator.exportIncludes(ctx) + p.libraryDecorator.reexportFlags(p.properties.Export_flags...) + + in := android.PathForModuleSrc(ctx, *p.properties.Src) + p.unstrippedOutputFile = in + + if p.shared() { + libName := in.Base() + builderFlags := flagsToBuilderFlags(flags) + + // Optimize out relinking against shared libraries whose interface hasn't changed by + // depending on a table of contents file instead of the library itself. + tocFile := android.PathForModuleOut(ctx, libName+".toc") + p.tocFile = android.OptionalPathForPath(tocFile) + TransformSharedObjectToToc(ctx, in, tocFile, builderFlags) + } + + return in +} + +func (p *vendorSnapshotLibraryDecorator) nativeCoverage() bool { + return false +} + +func (p *vendorSnapshotLibraryDecorator) isSnapshotPrebuilt() bool { + return true +} + +func (p *vendorSnapshotLibraryDecorator) install(ctx ModuleContext, file android.Path) { + if p.matchesWithDevice(ctx.DeviceConfig()) && (p.shared() || p.static()) { + p.baseInstaller.install(ctx, file) + } +} + +type vendorSnapshotInterface interface { + version() string +} + +func vendorSnapshotLoadHook(ctx android.LoadHookContext, p vendorSnapshotInterface) { + if p.version() != ctx.DeviceConfig().VndkVersion() { + ctx.Module().Disable() + return + } +} + +func vendorSnapshotLibrary() (*Module, *vendorSnapshotLibraryDecorator) { + module, library := NewLibrary(android.DeviceSupported) + + module.stl = nil + module.sanitize = nil + library.StripProperties.Strip.None = BoolPtr(true) + + prebuilt := &vendorSnapshotLibraryDecorator{ + libraryDecorator: library, + } + + prebuilt.baseLinker.Properties.No_libcrt = BoolPtr(true) + prebuilt.baseLinker.Properties.Nocrt = BoolPtr(true) + + // Prevent default system libs (libc, libm, and libdl) from being linked + if prebuilt.baseLinker.Properties.System_shared_libs == nil { + prebuilt.baseLinker.Properties.System_shared_libs = []string{} + } + + module.compiler = nil + module.linker = prebuilt + module.installer = prebuilt + + module.AddProperties( + &prebuilt.properties, + ) + + return module, prebuilt +} + +func VendorSnapshotSharedFactory() android.Module { + module, prebuilt := vendorSnapshotLibrary() + prebuilt.libraryDecorator.BuildOnlyShared() + android.AddLoadHook(module, func(ctx android.LoadHookContext) { + vendorSnapshotLoadHook(ctx, prebuilt) + }) + return module.Init() +} + +func VendorSnapshotStaticFactory() android.Module { + module, prebuilt := vendorSnapshotLibrary() + prebuilt.libraryDecorator.BuildOnlyStatic() + android.AddLoadHook(module, func(ctx android.LoadHookContext) { + vendorSnapshotLoadHook(ctx, prebuilt) + }) + return module.Init() +} + +func VendorSnapshotHeaderFactory() android.Module { + module, prebuilt := vendorSnapshotLibrary() + prebuilt.libraryDecorator.HeaderOnly() + android.AddLoadHook(module, func(ctx android.LoadHookContext) { + vendorSnapshotLoadHook(ctx, prebuilt) + }) + return module.Init() +} + +type vendorSnapshotBinaryProperties struct { + // snapshot version. + Version string + + // Target arch name of the snapshot (e.g. 'arm64' for variant 'aosp_arm64_ab') + Target_arch string + + // Prebuilt file for each arch. + Src *string `android:"arch_variant"` +} + +type vendorSnapshotBinaryDecorator struct { + *binaryDecorator + properties vendorSnapshotBinaryProperties + androidMkVendorSuffix bool +} + +func (p *vendorSnapshotBinaryDecorator) Name(name string) string { + return name + p.NameSuffix() +} + +func (p *vendorSnapshotBinaryDecorator) NameSuffix() string { + versionSuffix := p.version() + if p.arch() != "" { + versionSuffix += "." + p.arch() + } + return vendorSnapshotBinarySuffix + versionSuffix +} + +func (p *vendorSnapshotBinaryDecorator) version() string { + return p.properties.Version +} + +func (p *vendorSnapshotBinaryDecorator) arch() string { + return p.properties.Target_arch +} + +func (p *vendorSnapshotBinaryDecorator) matchesWithDevice(config android.DeviceConfig) bool { + if config.DeviceArch() != p.arch() { + return false + } + if p.properties.Src == nil { + return false + } + return true +} + +func (p *vendorSnapshotBinaryDecorator) link(ctx ModuleContext, + flags Flags, deps PathDeps, objs Objects) android.Path { + if !p.matchesWithDevice(ctx.DeviceConfig()) { + return nil + } + + in := android.PathForModuleSrc(ctx, *p.properties.Src) + builderFlags := flagsToBuilderFlags(flags) + p.unstrippedOutputFile = in + binName := in.Base() + if p.needsStrip(ctx) { + stripped := android.PathForModuleOut(ctx, "stripped", binName) + p.stripExecutableOrSharedLib(ctx, in, stripped, builderFlags) + in = stripped + } + + m := ctx.Module().(*Module) + p.androidMkVendorSuffix = vendorSuffixModules(ctx.Config())[m.BaseModuleName()] + + // use cpExecutable to make it executable + outputFile := android.PathForModuleOut(ctx, binName) + ctx.Build(pctx, android.BuildParams{ + Rule: android.CpExecutable, + Description: "prebuilt", + Output: outputFile, + Input: in, + }) + + return outputFile +} + +func (p *vendorSnapshotBinaryDecorator) isSnapshotPrebuilt() bool { + return true +} + +func VendorSnapshotBinaryFactory() android.Module { + module, binary := NewBinary(android.DeviceSupported) + binary.baseLinker.Properties.No_libcrt = BoolPtr(true) + binary.baseLinker.Properties.Nocrt = BoolPtr(true) + + // Prevent default system libs (libc, libm, and libdl) from being linked + if binary.baseLinker.Properties.System_shared_libs == nil { + binary.baseLinker.Properties.System_shared_libs = []string{} + } + + prebuilt := &vendorSnapshotBinaryDecorator{ + binaryDecorator: binary, + } + + module.compiler = nil + module.sanitize = nil + module.stl = nil + module.linker = prebuilt + + android.AddLoadHook(module, func(ctx android.LoadHookContext) { + vendorSnapshotLoadHook(ctx, prebuilt) + }) + + module.AddProperties(&prebuilt.properties) + return module.Init() +} + +type vendorSnapshotObjectProperties struct { + // snapshot version. + Version string + + // Target arch name of the snapshot (e.g. 'arm64' for variant 'aosp_arm64_ab') + Target_arch string + + // Prebuilt file for each arch. + Src *string `android:"arch_variant"` +} + +type vendorSnapshotObjectLinker struct { + objectLinker + properties vendorSnapshotObjectProperties + androidMkVendorSuffix bool +} + +func (p *vendorSnapshotObjectLinker) Name(name string) string { + return name + p.NameSuffix() +} + +func (p *vendorSnapshotObjectLinker) NameSuffix() string { + versionSuffix := p.version() + if p.arch() != "" { + versionSuffix += "." + p.arch() + } + return vendorSnapshotObjectSuffix + versionSuffix +} + +func (p *vendorSnapshotObjectLinker) version() string { + return p.properties.Version +} + +func (p *vendorSnapshotObjectLinker) arch() string { + return p.properties.Target_arch +} + +func (p *vendorSnapshotObjectLinker) matchesWithDevice(config android.DeviceConfig) bool { + if config.DeviceArch() != p.arch() { + return false + } + if p.properties.Src == nil { + return false + } + return true +} + +func (p *vendorSnapshotObjectLinker) link(ctx ModuleContext, + flags Flags, deps PathDeps, objs Objects) android.Path { + if !p.matchesWithDevice(ctx.DeviceConfig()) { + return nil + } + + m := ctx.Module().(*Module) + p.androidMkVendorSuffix = vendorSuffixModules(ctx.Config())[m.BaseModuleName()] + + return android.PathForModuleSrc(ctx, *p.properties.Src) +} + +func (p *vendorSnapshotObjectLinker) nativeCoverage() bool { + return false +} + +func (p *vendorSnapshotObjectLinker) isSnapshotPrebuilt() bool { + return true +} + +func VendorSnapshotObjectFactory() android.Module { + module := newObject() + + prebuilt := &vendorSnapshotObjectLinker{ + objectLinker: objectLinker{ + baseLinker: NewBaseLinker(nil), + }, + } + module.linker = prebuilt + + android.AddLoadHook(module, func(ctx android.LoadHookContext) { + vendorSnapshotLoadHook(ctx, prebuilt) + }) + + module.AddProperties(&prebuilt.properties) + return module.Init() +} + +func init() { + android.RegisterSingletonType("vendor-snapshot", VendorSnapshotSingleton) + android.RegisterModuleType("vendor_snapshot_shared", VendorSnapshotSharedFactory) + android.RegisterModuleType("vendor_snapshot_static", VendorSnapshotStaticFactory) + android.RegisterModuleType("vendor_snapshot_header", VendorSnapshotHeaderFactory) + android.RegisterModuleType("vendor_snapshot_binary", VendorSnapshotBinaryFactory) + android.RegisterModuleType("vendor_snapshot_object", VendorSnapshotObjectFactory) +} + +func VendorSnapshotSingleton() android.Singleton { + return &vendorSnapshotSingleton{} +} + +type vendorSnapshotSingleton struct { + vendorSnapshotZipFile android.OptionalPath +} + +var ( + // Modules under following directories are ignored. They are OEM's and vendor's + // proprietary modules(device/, vendor/, and hardware/). + // TODO(b/65377115): Clean up these with more maintainable way + vendorProprietaryDirs = []string{ + "device", + "vendor", + "hardware", + } + + // Modules under following directories are included as they are in AOSP, + // although hardware/ is normally for vendor's own. + // TODO(b/65377115): Clean up these with more maintainable way + aospDirsUnderProprietary = []string{ + "hardware/interfaces", + "hardware/libhardware", + "hardware/libhardware_legacy", + "hardware/ril", + } +) + +// Determine if a dir under source tree is an SoC-owned proprietary directory, such as +// device/, vendor/, etc. +func isVendorProprietaryPath(dir string) bool { + for _, p := range vendorProprietaryDirs { + if strings.HasPrefix(dir, p) { + // filter out AOSP defined directories, e.g. hardware/interfaces/ + aosp := false + for _, p := range aospDirsUnderProprietary { + if strings.HasPrefix(dir, p) { + aosp = true + break + } + } + if !aosp { + return true + } + } + } + return false +} + +// Determine if a module is going to be included in vendor snapshot or not. +// +// Targets of vendor snapshot are "vendor: true" or "vendor_available: true" modules in +// AOSP. They are not guaranteed to be compatible with older vendor images. (e.g. might +// depend on newer VNDK) So they are captured as vendor snapshot To build older vendor +// image and newer system image altogether. +func isVendorSnapshotModule(m *Module, moduleDir string) bool { + if !m.Enabled() || m.Properties.HideFromMake { + return false + } + // skip proprietary modules, but include all VNDK (static) + if isVendorProprietaryPath(moduleDir) && !m.IsVndk() { + return false + } + if m.Target().Os.Class != android.Device { + return false + } + if m.Target().NativeBridge == android.NativeBridgeEnabled { + return false + } + // the module must be installed in /vendor + if !m.IsForPlatform() || m.isSnapshotPrebuilt() || !m.inVendor() { + return false + } + // skip kernel_headers which always depend on vendor + if _, ok := m.linker.(*kernelHeadersDecorator); ok { + return false + } + + // Libraries + if l, ok := m.linker.(snapshotLibraryInterface); ok { + // TODO(b/65377115): add full support for sanitizer + if m.sanitize != nil { + // cfi, scs and hwasan export both sanitized and unsanitized variants for static and header + // Always use unsanitized variants of them. + for _, t := range []sanitizerType{cfi, scs, hwasan} { + if !l.shared() && m.sanitize.isSanitizerEnabled(t) { + return false + } + } + } + if l.static() { + return m.outputFile.Valid() && proptools.BoolDefault(m.VendorProperties.Vendor_available, true) + } + if l.shared() { + if !m.outputFile.Valid() { + return false + } + if !m.IsVndk() { + return true + } + return m.isVndkExt() + } + return true + } + + // Binaries and Objects + if m.binary() || m.object() { + return m.outputFile.Valid() && proptools.BoolDefault(m.VendorProperties.Vendor_available, true) + } + + return false +} + +func (c *vendorSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContext) { + // BOARD_VNDK_VERSION must be set to 'current' in order to generate a vendor snapshot. + if ctx.DeviceConfig().VndkVersion() != "current" { + return + } + + var snapshotOutputs android.Paths + + /* + Vendor snapshot zipped artifacts directory structure: + {SNAPSHOT_ARCH}/ + arch-{TARGET_ARCH}-{TARGET_ARCH_VARIANT}/ + shared/ + (.so shared libraries) + static/ + (.a static libraries) + header/ + (header only libraries) + binary/ + (executable binaries) + object/ + (.o object files) + arch-{TARGET_2ND_ARCH}-{TARGET_2ND_ARCH_VARIANT}/ + shared/ + (.so shared libraries) + static/ + (.a static libraries) + header/ + (header only libraries) + binary/ + (executable binaries) + object/ + (.o object files) + NOTICE_FILES/ + (notice files, e.g. libbase.txt) + configs/ + (config files, e.g. init.rc files, vintf_fragments.xml files, etc.) + include/ + (header files of same directory structure with source tree) + */ + + snapshotDir := "vendor-snapshot" + snapshotArchDir := filepath.Join(snapshotDir, ctx.DeviceConfig().DeviceArch()) + + includeDir := filepath.Join(snapshotArchDir, "include") + configsDir := filepath.Join(snapshotArchDir, "configs") + noticeDir := filepath.Join(snapshotArchDir, "NOTICE_FILES") + + installedNotices := make(map[string]bool) + installedConfigs := make(map[string]bool) + + var headers android.Paths + + installSnapshot := func(m *Module) android.Paths { + targetArch := "arch-" + m.Target().Arch.ArchType.String() + if m.Target().Arch.ArchVariant != "" { + targetArch += "-" + m.Target().Arch.ArchVariant + } + + var ret android.Paths + + prop := struct { + ModuleName string `json:",omitempty"` + RelativeInstallPath string `json:",omitempty"` + + // library flags + ExportedDirs []string `json:",omitempty"` + ExportedSystemDirs []string `json:",omitempty"` + ExportedFlags []string `json:",omitempty"` + SanitizeMinimalDep bool `json:",omitempty"` + SanitizeUbsanDep bool `json:",omitempty"` + + // binary flags + Symlinks []string `json:",omitempty"` + + // dependencies + SharedLibs []string `json:",omitempty"` + RuntimeLibs []string `json:",omitempty"` + Required []string `json:",omitempty"` + + // extra config files + InitRc []string `json:",omitempty"` + VintfFragments []string `json:",omitempty"` + }{} + + // Common properties among snapshots. + prop.ModuleName = ctx.ModuleName(m) + if m.isVndkExt() { + // vndk exts are installed to /vendor/lib(64)?/vndk(-sp)? + if m.isVndkSp() { + prop.RelativeInstallPath = "vndk-sp" + } else { + prop.RelativeInstallPath = "vndk" + } + } else { + prop.RelativeInstallPath = m.RelativeInstallPath() + } + prop.RuntimeLibs = m.Properties.SnapshotRuntimeLibs + prop.Required = m.RequiredModuleNames() + for _, path := range m.InitRc() { + prop.InitRc = append(prop.InitRc, filepath.Join("configs", path.Base())) + } + for _, path := range m.VintfFragments() { + prop.VintfFragments = append(prop.VintfFragments, filepath.Join("configs", path.Base())) + } + + // install config files. ignores any duplicates. + for _, path := range append(m.InitRc(), m.VintfFragments()...) { + out := filepath.Join(configsDir, path.Base()) + if !installedConfigs[out] { + installedConfigs[out] = true + ret = append(ret, copyFile(ctx, path, out)) + } + } + + var propOut string + + if l, ok := m.linker.(snapshotLibraryInterface); ok { + // library flags + prop.ExportedFlags = l.exportedFlags() + for _, dir := range l.exportedDirs() { + prop.ExportedDirs = append(prop.ExportedDirs, filepath.Join("include", dir.String())) + } + for _, dir := range l.exportedSystemDirs() { + prop.ExportedSystemDirs = append(prop.ExportedSystemDirs, filepath.Join("include", dir.String())) + } + // shared libs dependencies aren't meaningful on static or header libs + if l.shared() { + prop.SharedLibs = m.Properties.SnapshotSharedLibs + } + if l.static() && m.sanitize != nil { + prop.SanitizeMinimalDep = m.sanitize.Properties.MinimalRuntimeDep || enableMinimalRuntime(m.sanitize) + prop.SanitizeUbsanDep = m.sanitize.Properties.UbsanRuntimeDep || enableUbsanRuntime(m.sanitize) + } + + var libType string + if l.static() { + libType = "static" + } else if l.shared() { + libType = "shared" + } else { + libType = "header" + } + + var stem string + + // install .a or .so + if libType != "header" { + libPath := m.outputFile.Path() + stem = libPath.Base() + snapshotLibOut := filepath.Join(snapshotArchDir, targetArch, libType, stem) + ret = append(ret, copyFile(ctx, libPath, snapshotLibOut)) + } else { + stem = ctx.ModuleName(m) + } + + propOut = filepath.Join(snapshotArchDir, targetArch, libType, stem+".json") + } else if m.binary() { + // binary flags + prop.Symlinks = m.Symlinks() + prop.SharedLibs = m.Properties.SnapshotSharedLibs + + // install bin + binPath := m.outputFile.Path() + snapshotBinOut := filepath.Join(snapshotArchDir, targetArch, "binary", binPath.Base()) + ret = append(ret, copyFile(ctx, binPath, snapshotBinOut)) + propOut = snapshotBinOut + ".json" + } else if m.object() { + // object files aren't installed to the device, so their names can conflict. + // Use module name as stem. + objPath := m.outputFile.Path() + snapshotObjOut := filepath.Join(snapshotArchDir, targetArch, "object", + ctx.ModuleName(m)+filepath.Ext(objPath.Base())) + ret = append(ret, copyFile(ctx, objPath, snapshotObjOut)) + propOut = snapshotObjOut + ".json" + } else { + ctx.Errorf("unknown module %q in vendor snapshot", m.String()) + return nil + } + + j, err := json.Marshal(prop) + if err != nil { + ctx.Errorf("json marshal to %q failed: %#v", propOut, err) + return nil + } + ret = append(ret, writeStringToFile(ctx, string(j), propOut)) + + return ret + } + + ctx.VisitAllModules(func(module android.Module) { + m, ok := module.(*Module) + if !ok { + return + } + + moduleDir := ctx.ModuleDir(module) + if !isVendorSnapshotModule(m, moduleDir) { + return + } + + snapshotOutputs = append(snapshotOutputs, installSnapshot(m)...) + if l, ok := m.linker.(snapshotLibraryInterface); ok { + headers = append(headers, l.snapshotHeaders()...) + } + + if m.NoticeFile().Valid() { + noticeName := ctx.ModuleName(m) + ".txt" + noticeOut := filepath.Join(noticeDir, noticeName) + // skip already copied notice file + if !installedNotices[noticeOut] { + installedNotices[noticeOut] = true + snapshotOutputs = append(snapshotOutputs, copyFile( + ctx, m.NoticeFile().Path(), noticeOut)) + } + } + }) + + // install all headers after removing duplicates + for _, header := range android.FirstUniquePaths(headers) { + snapshotOutputs = append(snapshotOutputs, copyFile( + ctx, header, filepath.Join(includeDir, header.String()))) + } + + // All artifacts are ready. Sort them to normalize ninja and then zip. + sort.Slice(snapshotOutputs, func(i, j int) bool { + return snapshotOutputs[i].String() < snapshotOutputs[j].String() + }) + + zipPath := android.PathForOutput(ctx, snapshotDir, "vendor-"+ctx.Config().DeviceName()+".zip") + zipRule := android.NewRuleBuilder() + + // filenames in rspfile from FlagWithRspFileInputList might be single-quoted. Remove it with tr + snapshotOutputList := android.PathForOutput(ctx, snapshotDir, "vendor-"+ctx.Config().DeviceName()+"_list") + zipRule.Command(). + Text("tr"). + FlagWithArg("-d ", "\\'"). + FlagWithRspFileInputList("< ", snapshotOutputs). + FlagWithOutput("> ", snapshotOutputList) + + zipRule.Temporary(snapshotOutputList) + + zipRule.Command(). + BuiltTool(ctx, "soong_zip"). + FlagWithOutput("-o ", zipPath). + FlagWithArg("-C ", android.PathForOutput(ctx, snapshotDir).String()). + FlagWithInput("-l ", snapshotOutputList) + + zipRule.Build(pctx, ctx, zipPath.String(), "vendor snapshot "+zipPath.String()) + zipRule.DeleteTemporaryFiles() + c.vendorSnapshotZipFile = android.OptionalPathForPath(zipPath) +} + +func (c *vendorSnapshotSingleton) MakeVars(ctx android.MakeVarsContext) { + ctx.Strict("SOONG_VENDOR_SNAPSHOT_ZIP", c.vendorSnapshotZipFile.String()) +} + +type snapshotInterface interface { + matchesWithDevice(config android.DeviceConfig) bool +} + +var _ snapshotInterface = (*vndkPrebuiltLibraryDecorator)(nil) +var _ snapshotInterface = (*vendorSnapshotLibraryDecorator)(nil) +var _ snapshotInterface = (*vendorSnapshotBinaryDecorator)(nil) +var _ snapshotInterface = (*vendorSnapshotObjectLinker)(nil) + +// gathers all snapshot modules for vendor, and disable unnecessary snapshots +// TODO(b/145966707): remove mutator and utilize android.Prebuilt to override source modules +func VendorSnapshotMutator(ctx android.BottomUpMutatorContext) { + vndkVersion := ctx.DeviceConfig().VndkVersion() + // don't need snapshot if current + if vndkVersion == "current" || vndkVersion == "" { + return + } + + module, ok := ctx.Module().(*Module) + if !ok || !module.Enabled() || module.VndkVersion() != vndkVersion { + return + } + + if !module.isSnapshotPrebuilt() { + return + } + + // isSnapshotPrebuilt ensures snapshotInterface + if !module.linker.(snapshotInterface).matchesWithDevice(ctx.DeviceConfig()) { + // Disable unnecessary snapshot module, but do not disable + // vndk_prebuilt_shared because they might be packed into vndk APEX + if !module.IsVndk() { + module.Disable() + } + return + } + + var snapshotMap *snapshotMap + + if lib, ok := module.linker.(libraryInterface); ok { + if lib.static() { + snapshotMap = vendorSnapshotStaticLibs(ctx.Config()) + } else if lib.shared() { + snapshotMap = vendorSnapshotSharedLibs(ctx.Config()) + } else { + // header + snapshotMap = vendorSnapshotHeaderLibs(ctx.Config()) + } + } else if _, ok := module.linker.(*vendorSnapshotBinaryDecorator); ok { + snapshotMap = vendorSnapshotBinaries(ctx.Config()) + } else if _, ok := module.linker.(*vendorSnapshotObjectLinker); ok { + snapshotMap = vendorSnapshotObjects(ctx.Config()) + } else { + return + } + + vendorSnapshotsLock.Lock() + defer vendorSnapshotsLock.Unlock() + snapshotMap.add(module.BaseModuleName(), ctx.Arch().ArchType, ctx.ModuleName()) +} + +// Disables source modules which have snapshots +func VendorSnapshotSourceMutator(ctx android.BottomUpMutatorContext) { + if !ctx.Device() { + return + } + + vndkVersion := ctx.DeviceConfig().VndkVersion() + // don't need snapshot if current + if vndkVersion == "current" || vndkVersion == "" { + return + } + + module, ok := ctx.Module().(*Module) + if !ok { + return + } + + // vendor suffix should be added to snapshots if the source module isn't vendor: true. + if !module.SocSpecific() { + // But we can't just check SocSpecific() since we already passed the image mutator. + // Check ramdisk and recovery to see if we are real "vendor: true" module. + ramdisk_available := module.InRamdisk() && !module.OnlyInRamdisk() + recovery_available := module.InRecovery() && !module.OnlyInRecovery() + + if !ramdisk_available && !recovery_available { + vendorSnapshotsLock.Lock() + defer vendorSnapshotsLock.Unlock() + + vendorSuffixModules(ctx.Config())[ctx.ModuleName()] = true + } + } + + if module.isSnapshotPrebuilt() || module.VndkVersion() != ctx.DeviceConfig().VndkVersion() { + // only non-snapshot modules with BOARD_VNDK_VERSION + return + } + + // .. and also filter out llndk library + if module.isLlndk(ctx.Config()) { + return + } + + var snapshotMap *snapshotMap + + if lib, ok := module.linker.(libraryInterface); ok { + if lib.static() { + snapshotMap = vendorSnapshotStaticLibs(ctx.Config()) + } else if lib.shared() { + snapshotMap = vendorSnapshotSharedLibs(ctx.Config()) + } else { + // header + snapshotMap = vendorSnapshotHeaderLibs(ctx.Config()) + } + } else if module.binary() { + snapshotMap = vendorSnapshotBinaries(ctx.Config()) + } else if module.object() { + snapshotMap = vendorSnapshotObjects(ctx.Config()) + } else { + return + } + + if _, ok := snapshotMap.get(ctx.ModuleName(), ctx.Arch().ArchType); !ok { + // Corresponding snapshot doesn't exist + return + } + + // Disables source modules if corresponding snapshot exists. + if lib, ok := module.linker.(libraryInterface); ok && lib.buildStatic() && lib.buildShared() { + // But do not disable because the shared variant depends on the static variant. + module.SkipInstall() + module.Properties.HideFromMake = true + } else { + module.Disable() + } +} diff --git a/cc/vndk.go b/cc/vndk.go index 44a83e76a..a0608696e 100644 --- a/cc/vndk.go +++ b/cc/vndk.go @@ -15,21 +15,52 @@ package cc import ( + "encoding/json" "errors" + "fmt" + "path/filepath" "sort" "strings" "sync" "android/soong/android" "android/soong/cc/config" + "android/soong/etc" ) +const ( + llndkLibrariesTxt = "llndk.libraries.txt" + vndkCoreLibrariesTxt = "vndkcore.libraries.txt" + vndkSpLibrariesTxt = "vndksp.libraries.txt" + vndkPrivateLibrariesTxt = "vndkprivate.libraries.txt" + vndkUsingCoreVariantLibrariesTxt = "vndkcorevariant.libraries.txt" +) + +func VndkLibrariesTxtModules(vndkVersion string) []string { + if vndkVersion == "current" { + return []string{ + llndkLibrariesTxt, + vndkCoreLibrariesTxt, + vndkSpLibrariesTxt, + vndkPrivateLibrariesTxt, + } + } + // Snapshot vndks have their own *.libraries.VER.txt files. + // Note that snapshots don't have "vndkcorevariant.libraries.VER.txt" + return []string{ + insertVndkVersion(llndkLibrariesTxt, vndkVersion), + insertVndkVersion(vndkCoreLibrariesTxt, vndkVersion), + insertVndkVersion(vndkSpLibrariesTxt, vndkVersion), + insertVndkVersion(vndkPrivateLibrariesTxt, vndkVersion), + } +} + type VndkProperties struct { Vndk struct { // declared as a VNDK or VNDK-SP module. The vendor variant // will be installed in /system instead of /vendor partition. // - // `vendor_vailable` must be explicitly set to either true or + // `vendor_available` must be explicitly set to either true or // false together with `vndk: {enabled: true}`. Enabled *bool @@ -96,21 +127,21 @@ func (vndk *vndkdep) typeName() string { return "native:vendor:vndkspext" } -func (vndk *vndkdep) vndkCheckLinkType(ctx android.ModuleContext, to *Module, tag dependencyTag) { +func (vndk *vndkdep) vndkCheckLinkType(ctx android.ModuleContext, to *Module, tag DependencyTag) { if to.linker == nil { return } if !vndk.isVndk() { - // Non-VNDK modules (those installed to /vendor) can't depend on modules marked with - // vendor_available: false. + // Non-VNDK modules (those installed to /vendor, /product, or /system/product) can't depend + // on modules marked with vendor_available: false. violation := false if lib, ok := to.linker.(*llndkStubDecorator); ok && !Bool(lib.Properties.Vendor_available) { violation = true } else { if _, ok := to.linker.(libraryInterface); ok && to.VendorProperties.Vendor_available != nil && !Bool(to.VendorProperties.Vendor_available) { // Vendor_available == nil && !Bool(Vendor_available) should be okay since - // it means a vendor-only library which is a valid dependency for non-VNDK - // modules. + // it means a vendor-only, or product-only library which is a valid dependency + // for non-VNDK modules. violation = true } } @@ -123,7 +154,7 @@ func (vndk *vndkdep) vndkCheckLinkType(ctx android.ModuleContext, to *Module, ta // Other (static and LL-NDK) libraries are allowed to link. return } - if !to.Properties.UseVndk { + if !to.UseVndk() { ctx.ModuleErrorf("(%s) should not link to %q which is not a vendor-available library", vndk.typeName(), to.Name()) return @@ -192,64 +223,623 @@ func vndkIsVndkDepAllowed(from *vndkdep, to *vndkdep) error { } var ( - vndkCoreLibraries []string - vndkSpLibraries []string - llndkLibraries []string - vndkPrivateLibraries []string - vndkUsingCoreVariantLibraries []string - vndkLibrariesLock sync.Mutex + vndkCoreLibrariesKey = android.NewOnceKey("vndkCoreLibrarires") + vndkSpLibrariesKey = android.NewOnceKey("vndkSpLibrarires") + llndkLibrariesKey = android.NewOnceKey("llndkLibrarires") + vndkPrivateLibrariesKey = android.NewOnceKey("vndkPrivateLibrarires") + vndkUsingCoreVariantLibrariesKey = android.NewOnceKey("vndkUsingCoreVariantLibraries") + vndkMustUseVendorVariantListKey = android.NewOnceKey("vndkMustUseVendorVariantListKey") + vndkLibrariesLock sync.Mutex ) +func vndkCoreLibraries(config android.Config) map[string]string { + return config.Once(vndkCoreLibrariesKey, func() interface{} { + return make(map[string]string) + }).(map[string]string) +} + +func vndkSpLibraries(config android.Config) map[string]string { + return config.Once(vndkSpLibrariesKey, func() interface{} { + return make(map[string]string) + }).(map[string]string) +} + +func isLlndkLibrary(baseModuleName string, config android.Config) bool { + _, ok := llndkLibraries(config)[baseModuleName] + return ok +} + +func llndkLibraries(config android.Config) map[string]string { + return config.Once(llndkLibrariesKey, func() interface{} { + return make(map[string]string) + }).(map[string]string) +} + +func isVndkPrivateLibrary(baseModuleName string, config android.Config) bool { + _, ok := vndkPrivateLibraries(config)[baseModuleName] + return ok +} + +func vndkPrivateLibraries(config android.Config) map[string]string { + return config.Once(vndkPrivateLibrariesKey, func() interface{} { + return make(map[string]string) + }).(map[string]string) +} + +func vndkUsingCoreVariantLibraries(config android.Config) map[string]string { + return config.Once(vndkUsingCoreVariantLibrariesKey, func() interface{} { + return make(map[string]string) + }).(map[string]string) +} + +func vndkMustUseVendorVariantList(cfg android.Config) []string { + return cfg.Once(vndkMustUseVendorVariantListKey, func() interface{} { + return config.VndkMustUseVendorVariantList + }).([]string) +} + +// test may call this to override global configuration(config.VndkMustUseVendorVariantList) +// when it is called, it must be before the first call to vndkMustUseVendorVariantList() +func setVndkMustUseVendorVariantListForTest(config android.Config, mustUseVendorVariantList []string) { + config.Once(vndkMustUseVendorVariantListKey, func() interface{} { + return mustUseVendorVariantList + }) +} + +func processLlndkLibrary(mctx android.BottomUpMutatorContext, m *Module) { + lib := m.linker.(*llndkStubDecorator) + name := m.BaseModuleName() + filename := m.BaseModuleName() + ".so" + + vndkLibrariesLock.Lock() + defer vndkLibrariesLock.Unlock() + + llndkLibraries(mctx.Config())[name] = filename + if !Bool(lib.Properties.Vendor_available) { + vndkPrivateLibraries(mctx.Config())[name] = filename + } + if mctx.OtherModuleExists(name) { + mctx.AddFarVariationDependencies(m.Target().Variations(), llndkImplDep, name) + } +} + +func processVndkLibrary(mctx android.BottomUpMutatorContext, m *Module) { + name := m.BaseModuleName() + filename, err := getVndkFileName(m) + if err != nil { + panic(err) + } + + if m.HasStubsVariants() { + mctx.PropertyErrorf("vndk.enabled", "This library provides stubs. Shouldn't be VNDK. Consider making it as LLNDK") + } + + vndkLibrariesLock.Lock() + defer vndkLibrariesLock.Unlock() + + if inList(name, vndkMustUseVendorVariantList(mctx.Config())) { + m.Properties.MustUseVendorVariant = true + } + if mctx.DeviceConfig().VndkUseCoreVariant() && !m.Properties.MustUseVendorVariant { + vndkUsingCoreVariantLibraries(mctx.Config())[name] = filename + } + + if m.vndkdep.isVndkSp() { + vndkSpLibraries(mctx.Config())[name] = filename + } else { + vndkCoreLibraries(mctx.Config())[name] = filename + } + if !Bool(m.VendorProperties.Vendor_available) { + vndkPrivateLibraries(mctx.Config())[name] = filename + } +} + +// Sanity check for modules that mustn't be VNDK +func shouldSkipVndkMutator(m *Module) bool { + if !m.Enabled() { + return true + } + if !m.Device() { + // Skip non-device modules + return true + } + if m.Target().NativeBridge == android.NativeBridgeEnabled { + // Skip native_bridge modules + return true + } + return false +} + +func IsForVndkApex(mctx android.BottomUpMutatorContext, m *Module) bool { + if shouldSkipVndkMutator(m) { + return false + } + + // prebuilt vndk modules should match with device + // TODO(b/142675459): Use enabled: to select target device in vndk_prebuilt_shared + // When b/142675459 is landed, remove following check + if p, ok := m.linker.(*vndkPrebuiltLibraryDecorator); ok && !p.matchesWithDevice(mctx.DeviceConfig()) { + return false + } + + if lib, ok := m.linker.(libraryInterface); ok { + // VNDK APEX for VNDK-Lite devices will have VNDK-SP libraries from core variants + if mctx.DeviceConfig().VndkVersion() == "" { + // b/73296261: filter out libz.so because it is considered as LLNDK for VNDK-lite devices + if mctx.ModuleName() == "libz" { + return false + } + return m.ImageVariation().Variation == android.CoreVariation && lib.shared() && m.isVndkSp() + } + + useCoreVariant := m.VndkVersion() == mctx.DeviceConfig().PlatformVndkVersion() && + mctx.DeviceConfig().VndkUseCoreVariant() && !m.MustUseVendorVariant() + return lib.shared() && m.inVendor() && m.IsVndk() && !m.isVndkExt() && !useCoreVariant + } + return false +} + // gather list of vndk-core, vndk-sp, and ll-ndk libs func VndkMutator(mctx android.BottomUpMutatorContext) { - if m, ok := mctx.Module().(*Module); ok && m.Enabled() { - if lib, ok := m.linker.(*llndkStubDecorator); ok { - vndkLibrariesLock.Lock() - defer vndkLibrariesLock.Unlock() - name := strings.TrimSuffix(m.Name(), llndkLibrarySuffix) - if !inList(name, llndkLibraries) { - llndkLibraries = append(llndkLibraries, name) - sort.Strings(llndkLibraries) - } - if !Bool(lib.Properties.Vendor_available) { - if !inList(name, vndkPrivateLibraries) { - vndkPrivateLibraries = append(vndkPrivateLibraries, name) - sort.Strings(vndkPrivateLibraries) - } + m, ok := mctx.Module().(*Module) + if !ok { + return + } + + if shouldSkipVndkMutator(m) { + return + } + + if _, ok := m.linker.(*llndkStubDecorator); ok { + processLlndkLibrary(mctx, m) + return + } + + lib, is_lib := m.linker.(*libraryDecorator) + prebuilt_lib, is_prebuilt_lib := m.linker.(*prebuiltLibraryLinker) + + if (is_lib && lib.buildShared()) || (is_prebuilt_lib && prebuilt_lib.buildShared()) { + if m.vndkdep != nil && m.vndkdep.isVndk() && !m.vndkdep.isVndkExt() { + processVndkLibrary(mctx, m) + return + } + } +} + +func init() { + android.RegisterModuleType("vndk_libraries_txt", VndkLibrariesTxtFactory) + android.RegisterSingletonType("vndk-snapshot", VndkSnapshotSingleton) +} + +type vndkLibrariesTxt struct { + android.ModuleBase + outputFile android.OutputPath +} + +var _ etc.PrebuiltEtcModule = &vndkLibrariesTxt{} +var _ android.OutputFileProducer = &vndkLibrariesTxt{} + +// vndk_libraries_txt is a special kind of module type in that it name is one of +// - llndk.libraries.txt +// - vndkcore.libraries.txt +// - vndksp.libraries.txt +// - vndkprivate.libraries.txt +// - vndkcorevariant.libraries.txt +// A module behaves like a prebuilt_etc but its content is generated by soong. +// By being a soong module, these files can be referenced by other soong modules. +// For example, apex_vndk can depend on these files as prebuilt. +func VndkLibrariesTxtFactory() android.Module { + m := &vndkLibrariesTxt{} + android.InitAndroidModule(m) + return m +} + +func insertVndkVersion(filename string, vndkVersion string) string { + if index := strings.LastIndex(filename, "."); index != -1 { + return filename[:index] + "." + vndkVersion + filename[index:] + } + return filename +} + +func (txt *vndkLibrariesTxt) GenerateAndroidBuildActions(ctx android.ModuleContext) { + var list []string + switch txt.Name() { + case llndkLibrariesTxt: + for _, filename := range android.SortedStringMapValues(llndkLibraries(ctx.Config())) { + if strings.HasPrefix(filename, "libclang_rt.hwasan-") { + continue } + list = append(list, filename) + } + case vndkCoreLibrariesTxt: + list = android.SortedStringMapValues(vndkCoreLibraries(ctx.Config())) + case vndkSpLibrariesTxt: + list = android.SortedStringMapValues(vndkSpLibraries(ctx.Config())) + case vndkPrivateLibrariesTxt: + list = android.SortedStringMapValues(vndkPrivateLibraries(ctx.Config())) + case vndkUsingCoreVariantLibrariesTxt: + list = android.SortedStringMapValues(vndkUsingCoreVariantLibraries(ctx.Config())) + default: + ctx.ModuleErrorf("name(%s) is unknown.", txt.Name()) + return + } + + var filename string + if txt.Name() != vndkUsingCoreVariantLibrariesTxt { + filename = insertVndkVersion(txt.Name(), ctx.DeviceConfig().PlatformVndkVersion()) + } else { + filename = txt.Name() + } + + txt.outputFile = android.PathForModuleOut(ctx, filename).OutputPath + ctx.Build(pctx, android.BuildParams{ + Rule: android.WriteFile, + Output: txt.outputFile, + Description: "Writing " + txt.outputFile.String(), + Args: map[string]string{ + "content": strings.Join(list, "\\n"), + }, + }) + + installPath := android.PathForModuleInstall(ctx, "etc") + ctx.InstallFile(installPath, filename, txt.outputFile) +} + +func (txt *vndkLibrariesTxt) AndroidMkEntries() []android.AndroidMkEntries { + return []android.AndroidMkEntries{android.AndroidMkEntries{ + Class: "ETC", + OutputFile: android.OptionalPathForPath(txt.outputFile), + ExtraEntries: []android.AndroidMkExtraEntriesFunc{ + func(entries *android.AndroidMkEntries) { + entries.SetString("LOCAL_MODULE_STEM", txt.outputFile.Base()) + }, + }, + }} +} + +func (txt *vndkLibrariesTxt) OutputFile() android.OutputPath { + return txt.outputFile +} + +func (txt *vndkLibrariesTxt) OutputFiles(tag string) (android.Paths, error) { + return android.Paths{txt.outputFile}, nil +} + +func (txt *vndkLibrariesTxt) SubDir() string { + return "" +} + +func VndkSnapshotSingleton() android.Singleton { + return &vndkSnapshotSingleton{} +} + +type vndkSnapshotSingleton struct { + vndkLibrariesFile android.OutputPath + vndkSnapshotZipFile android.OptionalPath +} + +func isVndkSnapshotLibrary(config android.DeviceConfig, m *Module) (i snapshotLibraryInterface, vndkType string, isVndkSnapshotLib bool) { + if m.Target().NativeBridge == android.NativeBridgeEnabled { + return nil, "", false + } + if !m.inVendor() || !m.installable() || m.isSnapshotPrebuilt() { + return nil, "", false + } + l, ok := m.linker.(snapshotLibraryInterface) + if !ok || !l.shared() { + return nil, "", false + } + if m.VndkVersion() == config.PlatformVndkVersion() && m.IsVndk() && !m.isVndkExt() { + if m.isVndkSp() { + return l, "vndk-sp", true } else { - lib, is_lib := m.linker.(*libraryDecorator) - prebuilt_lib, is_prebuilt_lib := m.linker.(*prebuiltLibraryLinker) - if (is_lib && lib.shared()) || (is_prebuilt_lib && prebuilt_lib.shared()) { - name := strings.TrimPrefix(m.Name(), "prebuilt_") - if m.vndkdep.isVndk() && !m.vndkdep.isVndkExt() { - vndkLibrariesLock.Lock() - defer vndkLibrariesLock.Unlock() - if mctx.DeviceConfig().VndkUseCoreVariant() && !inList(name, config.VndkMustUseVendorVariantList) { - if !inList(name, vndkUsingCoreVariantLibraries) { - vndkUsingCoreVariantLibraries = append(vndkUsingCoreVariantLibraries, name) - sort.Strings(vndkUsingCoreVariantLibraries) - } - } - if m.vndkdep.isVndkSp() { - if !inList(name, vndkSpLibraries) { - vndkSpLibraries = append(vndkSpLibraries, name) - sort.Strings(vndkSpLibraries) - } - } else { - if !inList(name, vndkCoreLibraries) { - vndkCoreLibraries = append(vndkCoreLibraries, name) - sort.Strings(vndkCoreLibraries) - } - } - if !Bool(m.VendorProperties.Vendor_available) { - if !inList(name, vndkPrivateLibraries) { - vndkPrivateLibraries = append(vndkPrivateLibraries, name) - sort.Strings(vndkPrivateLibraries) - } - } - } + return l, "vndk-core", true + } + } + + return nil, "", false +} + +func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContext) { + // build these files even if PlatformVndkVersion or BoardVndkVersion is not set + c.buildVndkLibrariesTxtFiles(ctx) + + // BOARD_VNDK_VERSION must be set to 'current' in order to generate a VNDK snapshot. + if ctx.DeviceConfig().VndkVersion() != "current" { + return + } + + if ctx.DeviceConfig().PlatformVndkVersion() == "" { + return + } + + if ctx.DeviceConfig().BoardVndkRuntimeDisable() { + return + } + + var snapshotOutputs android.Paths + + /* + VNDK snapshot zipped artifacts directory structure: + {SNAPSHOT_ARCH}/ + arch-{TARGET_ARCH}-{TARGET_ARCH_VARIANT}/ + shared/ + vndk-core/ + (VNDK-core libraries, e.g. libbinder.so) + vndk-sp/ + (VNDK-SP libraries, e.g. libc++.so) + arch-{TARGET_2ND_ARCH}-{TARGET_2ND_ARCH_VARIANT}/ + shared/ + vndk-core/ + (VNDK-core libraries, e.g. libbinder.so) + vndk-sp/ + (VNDK-SP libraries, e.g. libc++.so) + binder32/ + (This directory is newly introduced in v28 (Android P) to hold + prebuilts built for 32-bit binder interface.) + arch-{TARGET_ARCH}-{TARGE_ARCH_VARIANT}/ + ... + configs/ + (various *.txt configuration files) + include/ + (header files of same directory structure with source tree) + NOTICE_FILES/ + (notice files of libraries, e.g. libcutils.so.txt) + */ + + snapshotDir := "vndk-snapshot" + snapshotArchDir := filepath.Join(snapshotDir, ctx.DeviceConfig().DeviceArch()) + + configsDir := filepath.Join(snapshotArchDir, "configs") + noticeDir := filepath.Join(snapshotArchDir, "NOTICE_FILES") + includeDir := filepath.Join(snapshotArchDir, "include") + + // set of notice files copied. + noticeBuilt := make(map[string]bool) + + // paths of VNDK modules for GPL license checking + modulePaths := make(map[string]string) + + // actual module names of .so files + // e.g. moduleNames["libprotobuf-cpp-full-3.9.1.so"] = "libprotobuf-cpp-full" + moduleNames := make(map[string]string) + + var headers android.Paths + + installVndkSnapshotLib := func(m *Module, l snapshotLibraryInterface, vndkType string) (android.Paths, bool) { + var ret android.Paths + + targetArch := "arch-" + m.Target().Arch.ArchType.String() + if m.Target().Arch.ArchVariant != "" { + targetArch += "-" + m.Target().Arch.ArchVariant + } + + libPath := m.outputFile.Path() + snapshotLibOut := filepath.Join(snapshotArchDir, targetArch, "shared", vndkType, libPath.Base()) + ret = append(ret, copyFile(ctx, libPath, snapshotLibOut)) + + if ctx.Config().VndkSnapshotBuildArtifacts() { + prop := struct { + ExportedDirs []string `json:",omitempty"` + ExportedSystemDirs []string `json:",omitempty"` + ExportedFlags []string `json:",omitempty"` + RelativeInstallPath string `json:",omitempty"` + }{} + prop.ExportedFlags = l.exportedFlags() + prop.ExportedDirs = l.exportedDirs().Strings() + prop.ExportedSystemDirs = l.exportedSystemDirs().Strings() + prop.RelativeInstallPath = m.RelativeInstallPath() + + propOut := snapshotLibOut + ".json" + + j, err := json.Marshal(prop) + if err != nil { + ctx.Errorf("json marshal to %q failed: %#v", propOut, err) + return nil, false } + ret = append(ret, writeStringToFile(ctx, string(j), propOut)) } + return ret, true } + + ctx.VisitAllModules(func(module android.Module) { + m, ok := module.(*Module) + if !ok || !m.Enabled() { + return + } + + l, vndkType, ok := isVndkSnapshotLibrary(ctx.DeviceConfig(), m) + if !ok { + return + } + + // install .so files for appropriate modules. + // Also install .json files if VNDK_SNAPSHOT_BUILD_ARTIFACTS + libs, ok := installVndkSnapshotLib(m, l, vndkType) + if !ok { + return + } + snapshotOutputs = append(snapshotOutputs, libs...) + + // These are for generating module_names.txt and module_paths.txt + stem := m.outputFile.Path().Base() + moduleNames[stem] = ctx.ModuleName(m) + modulePaths[stem] = ctx.ModuleDir(m) + + if m.NoticeFile().Valid() { + noticeName := stem + ".txt" + // skip already copied notice file + if _, ok := noticeBuilt[noticeName]; !ok { + noticeBuilt[noticeName] = true + snapshotOutputs = append(snapshotOutputs, copyFile( + ctx, m.NoticeFile().Path(), filepath.Join(noticeDir, noticeName))) + } + } + + if ctx.Config().VndkSnapshotBuildArtifacts() { + headers = append(headers, l.snapshotHeaders()...) + } + }) + + // install all headers after removing duplicates + for _, header := range android.FirstUniquePaths(headers) { + snapshotOutputs = append(snapshotOutputs, copyFile( + ctx, header, filepath.Join(includeDir, header.String()))) + } + + // install *.libraries.txt except vndkcorevariant.libraries.txt + ctx.VisitAllModules(func(module android.Module) { + m, ok := module.(*vndkLibrariesTxt) + if !ok || !m.Enabled() || m.Name() == vndkUsingCoreVariantLibrariesTxt { + return + } + snapshotOutputs = append(snapshotOutputs, copyFile( + ctx, m.OutputFile(), filepath.Join(configsDir, m.Name()))) + }) + + /* + Dump a map to a list file as: + + {key1} {value1} + {key2} {value2} + ... + */ + installMapListFile := func(m map[string]string, path string) android.OutputPath { + var txtBuilder strings.Builder + for idx, k := range android.SortedStringKeys(m) { + if idx > 0 { + txtBuilder.WriteString("\\n") + } + txtBuilder.WriteString(k) + txtBuilder.WriteString(" ") + txtBuilder.WriteString(m[k]) + } + return writeStringToFile(ctx, txtBuilder.String(), path) + } + + /* + module_paths.txt contains paths on which VNDK modules are defined. + e.g., + libbase.so system/core/base + libc.so bionic/libc + ... + */ + snapshotOutputs = append(snapshotOutputs, installMapListFile(modulePaths, filepath.Join(configsDir, "module_paths.txt"))) + + /* + module_names.txt contains names as which VNDK modules are defined, + because output filename and module name can be different with stem and suffix properties. + + e.g., + libcutils.so libcutils + libprotobuf-cpp-full-3.9.2.so libprotobuf-cpp-full + ... + */ + snapshotOutputs = append(snapshotOutputs, installMapListFile(moduleNames, filepath.Join(configsDir, "module_names.txt"))) + + // All artifacts are ready. Sort them to normalize ninja and then zip. + sort.Slice(snapshotOutputs, func(i, j int) bool { + return snapshotOutputs[i].String() < snapshotOutputs[j].String() + }) + + zipPath := android.PathForOutput(ctx, snapshotDir, "android-vndk-"+ctx.DeviceConfig().DeviceArch()+".zip") + zipRule := android.NewRuleBuilder() + + // filenames in rspfile from FlagWithRspFileInputList might be single-quoted. Remove it with xargs + snapshotOutputList := android.PathForOutput(ctx, snapshotDir, "android-vndk-"+ctx.DeviceConfig().DeviceArch()+"_list") + zipRule.Command(). + Text("tr"). + FlagWithArg("-d ", "\\'"). + FlagWithRspFileInputList("< ", snapshotOutputs). + FlagWithOutput("> ", snapshotOutputList) + + zipRule.Temporary(snapshotOutputList) + + zipRule.Command(). + BuiltTool(ctx, "soong_zip"). + FlagWithOutput("-o ", zipPath). + FlagWithArg("-C ", android.PathForOutput(ctx, snapshotDir).String()). + FlagWithInput("-l ", snapshotOutputList) + + zipRule.Build(pctx, ctx, zipPath.String(), "vndk snapshot "+zipPath.String()) + zipRule.DeleteTemporaryFiles() + c.vndkSnapshotZipFile = android.OptionalPathForPath(zipPath) +} + +func getVndkFileName(m *Module) (string, error) { + if library, ok := m.linker.(*libraryDecorator); ok { + return library.getLibNameHelper(m.BaseModuleName(), true) + ".so", nil + } + if prebuilt, ok := m.linker.(*prebuiltLibraryLinker); ok { + return prebuilt.libraryDecorator.getLibNameHelper(m.BaseModuleName(), true) + ".so", nil + } + return "", fmt.Errorf("VNDK library should have libraryDecorator or prebuiltLibraryLinker as linker: %T", m.linker) +} + +func (c *vndkSnapshotSingleton) buildVndkLibrariesTxtFiles(ctx android.SingletonContext) { + llndk := android.SortedStringMapValues(llndkLibraries(ctx.Config())) + vndkcore := android.SortedStringMapValues(vndkCoreLibraries(ctx.Config())) + vndksp := android.SortedStringMapValues(vndkSpLibraries(ctx.Config())) + vndkprivate := android.SortedStringMapValues(vndkPrivateLibraries(ctx.Config())) + + // Build list of vndk libs as merged & tagged & filter-out(libclang_rt): + // Since each target have different set of libclang_rt.* files, + // keep the common set of files in vndk.libraries.txt + var merged []string + filterOutLibClangRt := func(libList []string) (filtered []string) { + for _, lib := range libList { + if !strings.HasPrefix(lib, "libclang_rt.") { + filtered = append(filtered, lib) + } + } + return + } + merged = append(merged, addPrefix(filterOutLibClangRt(llndk), "LLNDK: ")...) + merged = append(merged, addPrefix(vndksp, "VNDK-SP: ")...) + merged = append(merged, addPrefix(filterOutLibClangRt(vndkcore), "VNDK-core: ")...) + merged = append(merged, addPrefix(vndkprivate, "VNDK-private: ")...) + c.vndkLibrariesFile = android.PathForOutput(ctx, "vndk", "vndk.libraries.txt") + ctx.Build(pctx, android.BuildParams{ + Rule: android.WriteFile, + Output: c.vndkLibrariesFile, + Description: "Writing " + c.vndkLibrariesFile.String(), + Args: map[string]string{ + "content": strings.Join(merged, "\\n"), + }, + }) +} + +func (c *vndkSnapshotSingleton) MakeVars(ctx android.MakeVarsContext) { + // Make uses LLNDK_MOVED_TO_APEX_LIBRARIES to avoid installing libraries on /system if + // they been moved to an apex. + movedToApexLlndkLibraries := []string{} + for lib := range llndkLibraries(ctx.Config()) { + // Skip bionic libs, they are handled in different manner + if android.DirectlyInAnyApex(¬OnHostContext{}, lib) && !isBionic(lib) { + movedToApexLlndkLibraries = append(movedToApexLlndkLibraries, lib) + } + } + ctx.Strict("LLNDK_MOVED_TO_APEX_LIBRARIES", strings.Join(movedToApexLlndkLibraries, " ")) + + // Make uses LLNDK_LIBRARIES to determine which libraries to install. + // HWASAN is only part of the LL-NDK in builds in which libc depends on HWASAN. + // Therefore, by removing the library here, we cause it to only be installed if libc + // depends on it. + installedLlndkLibraries := []string{} + for lib := range llndkLibraries(ctx.Config()) { + if strings.HasPrefix(lib, "libclang_rt.hwasan-") { + continue + } + installedLlndkLibraries = append(installedLlndkLibraries, lib) + } + sort.Strings(installedLlndkLibraries) + ctx.Strict("LLNDK_LIBRARIES", strings.Join(installedLlndkLibraries, " ")) + + ctx.Strict("VNDK_CORE_LIBRARIES", strings.Join(android.SortedStringKeys(vndkCoreLibraries(ctx.Config())), " ")) + ctx.Strict("VNDK_SAMEPROCESS_LIBRARIES", strings.Join(android.SortedStringKeys(vndkSpLibraries(ctx.Config())), " ")) + ctx.Strict("VNDK_PRIVATE_LIBRARIES", strings.Join(android.SortedStringKeys(vndkPrivateLibraries(ctx.Config())), " ")) + ctx.Strict("VNDK_USING_CORE_VARIANT_LIBRARIES", strings.Join(android.SortedStringKeys(vndkUsingCoreVariantLibraries(ctx.Config())), " ")) + + ctx.Strict("VNDK_LIBRARIES_FILE", c.vndkLibrariesFile.String()) + ctx.Strict("SOONG_VNDK_SNAPSHOT_ZIP", c.vndkSnapshotZipFile.String()) } diff --git a/cc/vndk_prebuilt.go b/cc/vndk_prebuilt.go index 74f7f270c..5a44c4663 100644 --- a/cc/vndk_prebuilt.go +++ b/cc/vndk_prebuilt.go @@ -31,7 +31,8 @@ var ( // // vndk_prebuilt_shared { // name: "libfoo", -// version: "27.1.0", +// version: "27", +// target_arch: "arm64", // vendor_available: true, // vndk: { // enabled: true, @@ -61,6 +62,9 @@ type vndkPrebuiltProperties struct { // Prebuilt files for each arch. Srcs []string `android:"arch_variant"` + // list of flags that will be used for any module that links against this module. + Export_flags []string `android:"arch_variant"` + // Check the prebuilt ELF files (e.g. DT_SONAME, DT_NEEDED, resolution of undefined symbols, // etc). Check_elf_files *bool @@ -68,7 +72,8 @@ type vndkPrebuiltProperties struct { type vndkPrebuiltLibraryDecorator struct { *libraryDecorator - properties vndkPrebuiltProperties + properties vndkPrebuiltProperties + androidMkSuffix string } func (p *vndkPrebuiltLibraryDecorator) Name(name string) string { @@ -122,13 +127,69 @@ func (p *vndkPrebuiltLibraryDecorator) singleSourcePath(ctx ModuleContext) andro func (p *vndkPrebuiltLibraryDecorator) link(ctx ModuleContext, flags Flags, deps PathDeps, objs Objects) android.Path { + + if !p.matchesWithDevice(ctx.DeviceConfig()) { + ctx.Module().SkipInstall() + return nil + } + if len(p.properties.Srcs) > 0 && p.shared() { + p.libraryDecorator.exportIncludes(ctx) + p.libraryDecorator.reexportFlags(p.properties.Export_flags...) // current VNDK prebuilts are only shared libs. - return p.singleSourcePath(ctx) + + in := p.singleSourcePath(ctx) + builderFlags := flagsToBuilderFlags(flags) + p.unstrippedOutputFile = in + libName := in.Base() + if p.needsStrip(ctx) { + stripped := android.PathForModuleOut(ctx, "stripped", libName) + p.stripExecutableOrSharedLib(ctx, in, stripped, builderFlags) + in = stripped + } + + // Optimize out relinking against shared libraries whose interface hasn't changed by + // depending on a table of contents file instead of the library itself. + tocFile := android.PathForModuleOut(ctx, libName+".toc") + p.tocFile = android.OptionalPathForPath(tocFile) + TransformSharedObjectToToc(ctx, in, tocFile, builderFlags) + + p.androidMkSuffix = p.NameSuffix() + + vndkVersion := ctx.DeviceConfig().VndkVersion() + if vndkVersion == p.version() { + p.androidMkSuffix = "" + } + + return in } + + ctx.Module().SkipInstall() return nil } +func (p *vndkPrebuiltLibraryDecorator) matchesWithDevice(config android.DeviceConfig) bool { + arches := config.Arches() + if len(arches) == 0 || arches[0].ArchType.String() != p.arch() { + return false + } + if config.BinderBitness() != p.binderBit() { + return false + } + if len(p.properties.Srcs) == 0 { + return false + } + return true +} + +func (p *vndkPrebuiltLibraryDecorator) nativeCoverage() bool { + return false +} + +func (p *vndkPrebuiltLibraryDecorator) isSnapshotPrebuilt() bool { + return true +} + func (p *vndkPrebuiltLibraryDecorator) install(ctx ModuleContext, file android.Path) { arches := ctx.DeviceConfig().Arches() if len(arches) == 0 || arches[0].ArchType.String() != p.arch() { @@ -153,13 +214,19 @@ func vndkPrebuiltSharedLibrary() *Module { module.stl = nil module.sanitize = nil library.StripProperties.Strip.None = BoolPtr(true) - module.Properties.UseVndk = true prebuilt := &vndkPrebuiltLibraryDecorator{ libraryDecorator: library, } prebuilt.properties.Check_elf_files = BoolPtr(false) + prebuilt.baseLinker.Properties.No_libcrt = BoolPtr(true) + prebuilt.baseLinker.Properties.Nocrt = BoolPtr(true) + + // Prevent default system libs (libc, libm, and libdl) from being linked + if prebuilt.baseLinker.Properties.System_shared_libs == nil { + prebuilt.baseLinker.Properties.System_shared_libs = []string{} + } module.compiler = nil module.linker = prebuilt @@ -172,11 +239,32 @@ func vndkPrebuiltSharedLibrary() *Module { return module } -func vndkPrebuiltSharedFactory() android.Module { +// vndk_prebuilt_shared installs Vendor Native Development kit (VNDK) snapshot +// shared libraries for system build. Example: +// +// vndk_prebuilt_shared { +// name: "libfoo", +// version: "27", +// target_arch: "arm64", +// vendor_available: true, +// vndk: { +// enabled: true, +// }, +// export_include_dirs: ["include/external/libfoo/vndk_include"], +// arch: { +// arm64: { +// srcs: ["arm/lib64/libfoo.so"], +// }, +// arm: { +// srcs: ["arm/lib/libfoo.so"], +// }, +// }, +// } +func VndkPrebuiltSharedFactory() android.Module { module := vndkPrebuiltSharedLibrary() return module.Init() } func init() { - android.RegisterModuleType("vndk_prebuilt_shared", vndkPrebuiltSharedFactory) + android.RegisterModuleType("vndk_prebuilt_shared", VndkPrebuiltSharedFactory) } diff --git a/cc/xom.go b/cc/xom.go deleted file mode 100644 index 9337990f8..000000000 --- a/cc/xom.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright 2018 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cc - -import ( - "android/soong/android" -) - -type XomProperties struct { - Xom *bool -} - -type xom struct { - Properties XomProperties -} - -func (xom *xom) props() []interface{} { - return []interface{}{&xom.Properties} -} - -func (xom *xom) begin(ctx BaseModuleContext) {} - -func (xom *xom) deps(ctx BaseModuleContext, deps Deps) Deps { - return deps -} - -func (xom *xom) flags(ctx ModuleContext, flags Flags) Flags { - disableXom := false - - if !ctx.Config().EnableXOM() || ctx.Config().XOMDisabledForPath(ctx.ModuleDir()) { - disableXom = true - } - - if xom.Properties.Xom != nil && !*xom.Properties.Xom { - return flags - } - - // If any static dependencies have XOM disabled, we should disable XOM in this module, - // the assumption being if it's been explicitly disabled then there's probably incompatible - // code in the library which may get pulled in. - if !disableXom { - ctx.VisitDirectDeps(func(m android.Module) { - cc, ok := m.(*Module) - if !ok || cc.xom == nil || !cc.static() { - return - } - if cc.xom.Properties.Xom != nil && !*cc.xom.Properties.Xom { - disableXom = true - return - } - }) - } - - // Enable execute-only if none of the dependencies disable it, - // also if it's explicitly set true (allows overriding dependencies disabling it). - if !disableXom || (xom.Properties.Xom != nil && *xom.Properties.Xom) { - // XOM is only supported on AArch64 when using lld. - if ctx.Arch().ArchType == android.Arm64 && ctx.useClangLld(ctx) { - flags.LdFlags = append(flags.LdFlags, "-Wl,-execute-only") - } - } - - return flags -} diff --git a/cmd/diff_target_files/known_nondeterminism.whitelist b/cmd/diff_target_files/known_nondeterminism.whitelist index 6d714035a..a8ade4984 100644 --- a/cmd/diff_target_files/known_nondeterminism.whitelist +++ b/cmd/diff_target_files/known_nondeterminism.whitelist @@ -3,8 +3,6 @@ [ { "Paths": [ - // b/120039850 - "system/framework/oat/*/services.art" ] } ] diff --git a/cmd/diff_target_files/target_files.go b/cmd/diff_target_files/target_files.go index 8705ca700..0fa04e8c3 100644 --- a/cmd/diff_target_files/target_files.go +++ b/cmd/diff_target_files/target_files.go @@ -28,7 +28,7 @@ var targetZipPartitions = []string{ "ODM/", "OEM/", "PRODUCT/", - "PRODUCT_SERVICES/", + "SYSTEM_EXT/", "ROOT/", "SYSTEM/", "SYSTEM_OTHER/", diff --git a/cmd/extract_apks/main.go b/cmd/extract_apks/main.go index 85dd6d122..db54ffbaf 100644 --- a/cmd/extract_apks/main.go +++ b/cmd/extract_apks/main.go @@ -411,7 +411,7 @@ func (apkSet *ApkSet) writeApks(selected SelectionResult, config TargetConfig, } if partition != "" { apkcerts = append(apkcerts, fmt.Sprintf( - `name="%s" certificate="PRESIGNED" private_key=""`, outName)) + `name="%s" certificate="PRESIGNED" private_key="" partition="%s"`, outName, partition)) } } sort.Strings(apkcerts) diff --git a/cmd/extract_apks/main_test.go b/cmd/extract_apks/main_test.go index 356faab82..c3e6a2def 100644 --- a/cmd/extract_apks/main_test.go +++ b/cmd/extract_apks/main_test.go @@ -453,8 +453,8 @@ func TestWriteApks(t *testing.T) { "Foo-xhdpi.apk": "splits/mybase-xhdpi.apk", }, expectedApkcerts: []string{ - `name="Foo-xhdpi.apk" certificate="PRESIGNED" private_key=""`, - `name="Foo.apk" certificate="PRESIGNED" private_key=""`, + `name="Foo-xhdpi.apk" certificate="PRESIGNED" private_key="" partition="system"`, + `name="Foo.apk" certificate="PRESIGNED" private_key="" partition="system"`, }, }, { @@ -466,7 +466,7 @@ func TestWriteApks(t *testing.T) { "Bar.apk": "universal.apk", }, expectedApkcerts: []string{ - `name="Bar.apk" certificate="PRESIGNED" private_key=""`, + `name="Bar.apk" certificate="PRESIGNED" private_key="" partition="product"`, }, }, } diff --git a/cmd/javac_wrapper/javac_wrapper.go b/cmd/javac_wrapper/javac_wrapper.go index 7a448ba33..4679906d5 100644 --- a/cmd/javac_wrapper/javac_wrapper.go +++ b/cmd/javac_wrapper/javac_wrapper.go @@ -31,6 +31,7 @@ import ( "os" "os/exec" "regexp" + "strconv" "syscall" ) @@ -80,10 +81,11 @@ func Main(out io.Writer, name string, args []string) (int, error) { pw.Close() + proc := processor{} // Process subprocess stdout asynchronously errCh := make(chan error) go func() { - errCh <- process(pr, out) + errCh <- proc.process(pr, out) }() // Wait for subprocess to finish @@ -117,14 +119,18 @@ func Main(out io.Writer, name string, args []string) (int, error) { return 0, nil } -func process(r io.Reader, w io.Writer) error { +type processor struct { + silencedWarnings int +} + +func (proc *processor) process(r io.Reader, w io.Writer) error { scanner := bufio.NewScanner(r) // Some javac wrappers output the entire list of java files being // compiled on a single line, which can be very large, set the maximum // buffer size to 2MB. scanner.Buffer(nil, 2*1024*1024) for scanner.Scan() { - processLine(w, scanner.Text()) + proc.processLine(w, scanner.Text()) } err := scanner.Err() if err != nil { @@ -133,12 +139,32 @@ func process(r io.Reader, w io.Writer) error { return nil } -func processLine(w io.Writer, line string) { +func (proc *processor) processLine(w io.Writer, line string) { + for _, f := range warningFilters { + if f.MatchString(line) { + proc.silencedWarnings++ + return + } + } for _, f := range filters { if f.MatchString(line) { return } } + if match := warningCount.FindStringSubmatch(line); match != nil { + c, err := strconv.Atoi(match[1]) + if err == nil { + c -= proc.silencedWarnings + if c == 0 { + return + } else { + line = fmt.Sprintf("%d warning", c) + if c > 1 { + line += "s" + } + } + } + } for _, p := range colorPatterns { var matched bool if line, matched = applyColor(line, p.color, p.re); matched { @@ -170,12 +196,17 @@ var colorPatterns = []struct { {markerRe, green}, } +var warningCount = regexp.MustCompile(`^([0-9]+) warning(s)?$`) + +var warningFilters = []*regexp.Regexp{ + regexp.MustCompile(`bootstrap class path not set in conjunction with -source`), +} + var filters = []*regexp.Regexp{ regexp.MustCompile(`Note: (Some input files|.*\.java) uses? or overrides? a deprecated API.`), regexp.MustCompile(`Note: Recompile with -Xlint:deprecation for details.`), regexp.MustCompile(`Note: (Some input files|.*\.java) uses? unchecked or unsafe operations.`), regexp.MustCompile(`Note: Recompile with -Xlint:unchecked for details.`), - regexp.MustCompile(`bootstrap class path not set in conjunction with -source`), regexp.MustCompile(`javadoc: warning - The old Doclet and Taglet APIs in the packages`), regexp.MustCompile(`com.sun.javadoc, com.sun.tools.doclets and their implementations`), diff --git a/cmd/javac_wrapper/javac_wrapper_test.go b/cmd/javac_wrapper/javac_wrapper_test.go index ad657e7d4..ad230012e 100644 --- a/cmd/javac_wrapper/javac_wrapper_test.go +++ b/cmd/javac_wrapper/javac_wrapper_test.go @@ -75,13 +75,29 @@ javadoc: option --boot-class-path not allowed with target 1.9 `, out: "\n", }, + { + in: ` +warning: [options] bootstrap class path not set in conjunction with -source 1.9\n +1 warning +`, + out: "\n", + }, + { + in: ` +warning: foo +warning: [options] bootstrap class path not set in conjunction with -source 1.9\n +2 warnings +`, + out: "\n\x1b[1m\x1b[35mwarning:\x1b[0m\x1b[1m foo\x1b[0m\n1 warning\n", + }, } func TestJavacColorize(t *testing.T) { for i, test := range testCases { t.Run(strconv.Itoa(i), func(t *testing.T) { buf := new(bytes.Buffer) - err := process(bytes.NewReader([]byte(test.in)), buf) + proc := processor{} + err := proc.process(bytes.NewReader([]byte(test.in)), buf) if err != nil { t.Errorf("error: %q", err) } diff --git a/cmd/merge_zips/Android.bp b/cmd/merge_zips/Android.bp index ab658fd0d..f70c86eb6 100644 --- a/cmd/merge_zips/Android.bp +++ b/cmd/merge_zips/Android.bp @@ -18,6 +18,7 @@ blueprint_go_binary { "android-archive-zip", "blueprint-pathtools", "soong-jar", + "soong-zip", ], srcs: [ "merge_zips.go", diff --git a/cmd/merge_zips/merge_zips.go b/cmd/merge_zips/merge_zips.go index 68fe2592d..a95aca9f0 100644 --- a/cmd/merge_zips/merge_zips.go +++ b/cmd/merge_zips/merge_zips.go @@ -30,457 +30,725 @@ import ( "android/soong/jar" "android/soong/third_party/zip" + soongZip "android/soong/zip" ) -type fileList []string +// Input zip: we can open it, close it, and obtain an array of entries +type InputZip interface { + Name() string + Open() error + Close() error + Entries() []*zip.File + IsOpen() bool +} -func (f *fileList) String() string { - return `""` +// An entry that can be written to the output zip +type ZipEntryContents interface { + String() string + IsDir() bool + CRC32() uint32 + Size() uint64 + WriteToZip(dest string, zw *zip.Writer) error } -func (f *fileList) Set(name string) error { - *f = append(*f, filepath.Clean(name)) +// a ZipEntryFromZip is a ZipEntryContents that pulls its content from another zip +// identified by the input zip and the index of the entry in its entries array +type ZipEntryFromZip struct { + inputZip InputZip + index int + name string + isDir bool + crc32 uint32 + size uint64 +} - return nil +func NewZipEntryFromZip(inputZip InputZip, entryIndex int) *ZipEntryFromZip { + fi := inputZip.Entries()[entryIndex] + newEntry := ZipEntryFromZip{inputZip: inputZip, + index: entryIndex, + name: fi.Name, + isDir: fi.FileInfo().IsDir(), + crc32: fi.CRC32, + size: fi.UncompressedSize64, + } + return &newEntry } -type zipsToNotStripSet map[string]bool +func (ze ZipEntryFromZip) String() string { + return fmt.Sprintf("%s!%s", ze.inputZip.Name(), ze.name) +} -func (s zipsToNotStripSet) String() string { - return `""` +func (ze ZipEntryFromZip) IsDir() bool { + return ze.isDir } -func (s zipsToNotStripSet) Set(zip_path string) error { - s[zip_path] = true +func (ze ZipEntryFromZip) CRC32() uint32 { + return ze.crc32 +} - return nil +func (ze ZipEntryFromZip) Size() uint64 { + return ze.size } -var ( - sortEntries = flag.Bool("s", false, "sort entries (defaults to the order from the input zip files)") - emulateJar = flag.Bool("j", false, "sort zip entries using jar ordering (META-INF first)") - emulatePar = flag.Bool("p", false, "merge zip entries based on par format") - stripDirs fileList - stripFiles fileList - zipsToNotStrip = make(zipsToNotStripSet) - stripDirEntries = flag.Bool("D", false, "strip directory entries from the output zip file") - manifest = flag.String("m", "", "manifest file to insert in jar") - pyMain = flag.String("pm", "", "__main__.py file to insert in par") - prefix = flag.String("prefix", "", "A file to prefix to the zip file") - ignoreDuplicates = flag.Bool("ignore-duplicates", false, "take each entry from the first zip it exists in and don't warn") -) +func (ze ZipEntryFromZip) WriteToZip(dest string, zw *zip.Writer) error { + if err := ze.inputZip.Open(); err != nil { + return err + } + return zw.CopyFrom(ze.inputZip.Entries()[ze.index], dest) +} -func init() { - flag.Var(&stripDirs, "stripDir", "directories to be excluded from the output zip, accepts wildcards") - flag.Var(&stripFiles, "stripFile", "files to be excluded from the output zip, accepts wildcards") - flag.Var(&zipsToNotStrip, "zipToNotStrip", "the input zip file which is not applicable for stripping") +// a ZipEntryFromBuffer is a ZipEntryContents that pulls its content from a []byte +type ZipEntryFromBuffer struct { + fh *zip.FileHeader + content []byte } -func main() { - flag.Usage = func() { - fmt.Fprintln(os.Stderr, "usage: merge_zips [-jpsD] [-m manifest] [--prefix script] [-pm __main__.py] output [inputs...]") - flag.PrintDefaults() - } +func (be ZipEntryFromBuffer) String() string { + return "internal buffer" +} - // parse args - flag.Parse() - args := flag.Args() - if len(args) < 1 { - flag.Usage() - os.Exit(1) - } - outputPath := args[0] - inputs := args[1:] +func (be ZipEntryFromBuffer) IsDir() bool { + return be.fh.FileInfo().IsDir() +} - log.SetFlags(log.Lshortfile) +func (be ZipEntryFromBuffer) CRC32() uint32 { + return crc32.ChecksumIEEE(be.content) +} - // make writer - output, err := os.Create(outputPath) +func (be ZipEntryFromBuffer) Size() uint64 { + return uint64(len(be.content)) +} + +func (be ZipEntryFromBuffer) WriteToZip(dest string, zw *zip.Writer) error { + w, err := zw.CreateHeader(be.fh) if err != nil { - log.Fatal(err) + return err } - defer output.Close() - var offset int64 - if *prefix != "" { - prefixFile, err := os.Open(*prefix) - if err != nil { - log.Fatal(err) - } - offset, err = io.Copy(output, prefixFile) + if !be.IsDir() { + _, err = w.Write(be.content) if err != nil { - log.Fatal(err) + return err } } - writer := zip.NewWriter(output) - defer func() { - err := writer.Close() - if err != nil { - log.Fatal(err) - } - }() - writer.SetOffset(offset) + return nil +} - // make readers - readers := []namedZipReader{} - for _, input := range inputs { - reader, err := zip.OpenReader(input) - if err != nil { - log.Fatal(err) - } - defer reader.Close() - namedReader := namedZipReader{path: input, reader: &reader.Reader} - readers = append(readers, namedReader) - } +// Processing state. +type OutputZip struct { + outputWriter *zip.Writer + stripDirEntries bool + emulateJar bool + sortEntries bool + ignoreDuplicates bool + excludeDirs []string + excludeFiles []string + sourceByDest map[string]ZipEntryContents +} - if *manifest != "" && !*emulateJar { - log.Fatal(errors.New("must specify -j when specifying a manifest via -m")) +func NewOutputZip(outputWriter *zip.Writer, sortEntries, emulateJar, stripDirEntries, ignoreDuplicates bool) *OutputZip { + return &OutputZip{ + outputWriter: outputWriter, + stripDirEntries: stripDirEntries, + emulateJar: emulateJar, + sortEntries: sortEntries, + sourceByDest: make(map[string]ZipEntryContents, 0), + ignoreDuplicates: ignoreDuplicates, } +} - if *pyMain != "" && !*emulatePar { - log.Fatal(errors.New("must specify -p when specifying a Python __main__.py via -pm")) +func (oz *OutputZip) setExcludeDirs(excludeDirs []string) { + oz.excludeDirs = make([]string, len(excludeDirs)) + for i, dir := range excludeDirs { + oz.excludeDirs[i] = filepath.Clean(dir) } +} - // do merge - err = mergeZips(readers, writer, *manifest, *pyMain, *sortEntries, *emulateJar, *emulatePar, - *stripDirEntries, *ignoreDuplicates, []string(stripFiles), []string(stripDirs), map[string]bool(zipsToNotStrip)) - if err != nil { - log.Fatal(err) +func (oz *OutputZip) setExcludeFiles(excludeFiles []string) { + oz.excludeFiles = excludeFiles +} + +// Adds an entry with given name whose source is given ZipEntryContents. Returns old ZipEntryContents +// if entry with given name already exists. +func (oz *OutputZip) addZipEntry(name string, source ZipEntryContents) (ZipEntryContents, error) { + if existingSource, exists := oz.sourceByDest[name]; exists { + return existingSource, nil + } + oz.sourceByDest[name] = source + // Delay writing an entry if entries need to be rearranged. + if oz.emulateJar || oz.sortEntries { + return nil, nil } + return nil, source.WriteToZip(name, oz.outputWriter) } -// a namedZipReader reads a .zip file and can say which file it's reading -type namedZipReader struct { - path string - reader *zip.Reader +// Adds an entry for the manifest (META-INF/MANIFEST.MF from the given file +func (oz *OutputZip) addManifest(manifestPath string) error { + if !oz.stripDirEntries { + if _, err := oz.addZipEntry(jar.MetaDir, ZipEntryFromBuffer{jar.MetaDirFileHeader(), nil}); err != nil { + return err + } + } + contents, err := ioutil.ReadFile(manifestPath) + if err == nil { + fh, buf, err := jar.ManifestFileContents(contents) + if err == nil { + _, err = oz.addZipEntry(jar.ManifestFile, ZipEntryFromBuffer{fh, buf}) + } + } + return err } -// a zipEntryPath refers to a file contained in a zip -type zipEntryPath struct { - zipName string - entryName string +// Adds an entry with given name and contents read from given file +func (oz *OutputZip) addZipEntryFromFile(name string, path string) error { + buf, err := ioutil.ReadFile(path) + if err == nil { + fh := &zip.FileHeader{ + Name: name, + Method: zip.Store, + UncompressedSize64: uint64(len(buf)), + } + fh.SetMode(0700) + fh.SetModTime(jar.DefaultTime) + _, err = oz.addZipEntry(name, ZipEntryFromBuffer{fh, buf}) + } + return err } -func (p zipEntryPath) String() string { - return p.zipName + "/" + p.entryName +func (oz *OutputZip) addEmptyEntry(entry string) error { + var emptyBuf []byte + fh := &zip.FileHeader{ + Name: entry, + Method: zip.Store, + UncompressedSize64: uint64(len(emptyBuf)), + } + fh.SetMode(0700) + fh.SetModTime(jar.DefaultTime) + _, err := oz.addZipEntry(entry, ZipEntryFromBuffer{fh, emptyBuf}) + return err } -// a zipEntry is a zipSource that pulls its content from another zip -type zipEntry struct { - path zipEntryPath - content *zip.File +// Returns true if given entry is to be excluded +func (oz *OutputZip) isEntryExcluded(name string) bool { + for _, dir := range oz.excludeDirs { + dir = filepath.Clean(dir) + patterns := []string{ + dir + "/", // the directory itself + dir + "/**/*", // files recursively in the directory + dir + "/**/*/", // directories recursively in the directory + } + + for _, pattern := range patterns { + match, err := pathtools.Match(pattern, name) + if err != nil { + panic(fmt.Errorf("%s: %s", err.Error(), pattern)) + } + if match { + if oz.emulateJar { + // When merging jar files, don't strip META-INF/MANIFEST.MF even if stripping META-INF is + // requested. + // TODO(ccross): which files does this affect? + if name != jar.MetaDir && name != jar.ManifestFile { + return true + } + } + return true + } + } + } + + for _, pattern := range oz.excludeFiles { + match, err := pathtools.Match(pattern, name) + if err != nil { + panic(fmt.Errorf("%s: %s", err.Error(), pattern)) + } + if match { + return true + } + } + return false } -func (ze zipEntry) String() string { - return ze.path.String() +// Creates a zip entry whose contents is an entry from the given input zip. +func (oz *OutputZip) copyEntry(inputZip InputZip, index int) error { + entry := NewZipEntryFromZip(inputZip, index) + if oz.stripDirEntries && entry.IsDir() { + return nil + } + existingEntry, err := oz.addZipEntry(entry.name, entry) + if err != nil { + return err + } + if existingEntry == nil { + return nil + } + + // File types should match + if existingEntry.IsDir() != entry.IsDir() { + return fmt.Errorf("Directory/file mismatch at %v from %v and %v\n", + entry.name, existingEntry, entry) + } + + if oz.ignoreDuplicates || + // Skip manifest and module info files that are not from the first input file + (oz.emulateJar && entry.name == jar.ManifestFile || entry.name == jar.ModuleInfoClass) || + // Identical entries + (existingEntry.CRC32() == entry.CRC32() && existingEntry.Size() == entry.Size()) || + // Directory entries + entry.IsDir() { + return nil + } + + return fmt.Errorf("Duplicate path %v found in %v and %v\n", entry.name, existingEntry, inputZip.Name()) } -func (ze zipEntry) IsDir() bool { - return ze.content.FileInfo().IsDir() +func (oz *OutputZip) entriesArray() []string { + entries := make([]string, len(oz.sourceByDest)) + i := 0 + for entry := range oz.sourceByDest { + entries[i] = entry + i++ + } + return entries } -func (ze zipEntry) CRC32() uint32 { - return ze.content.FileHeader.CRC32 +func (oz *OutputZip) jarSorted() []string { + entries := oz.entriesArray() + sort.SliceStable(entries, func(i, j int) bool { return jar.EntryNamesLess(entries[i], entries[j]) }) + return entries } -func (ze zipEntry) Size() uint64 { - return ze.content.FileHeader.UncompressedSize64 +func (oz *OutputZip) alphanumericSorted() []string { + entries := oz.entriesArray() + sort.Strings(entries) + return entries } -func (ze zipEntry) WriteToZip(dest string, zw *zip.Writer) error { - return zw.CopyFrom(ze.content, dest) +func (oz *OutputZip) writeEntries(entries []string) error { + for _, entry := range entries { + source, _ := oz.sourceByDest[entry] + if err := source.WriteToZip(entry, oz.outputWriter); err != nil { + return err + } + } + return nil } -// a bufferEntry is a zipSource that pulls its content from a []byte -type bufferEntry struct { - fh *zip.FileHeader - content []byte +func (oz *OutputZip) getUninitializedPythonPackages(inputZips []InputZip) ([]string, error) { + // the runfiles packages needs to be populated with "__init__.py". + // the runfiles dirs have been treated as packages. + allPackages := make(map[string]bool) + initedPackages := make(map[string]bool) + getPackage := func(path string) string { + ret := filepath.Dir(path) + // filepath.Dir("abc") -> "." and filepath.Dir("/abc") -> "/". + if ret == "." || ret == "/" { + return "" + } + return ret + } + + // put existing __init__.py files to a set first. This set is used for preventing + // generated __init__.py files from overwriting existing ones. + for _, inputZip := range inputZips { + if err := inputZip.Open(); err != nil { + return nil, err + } + for _, file := range inputZip.Entries() { + pyPkg := getPackage(file.Name) + if filepath.Base(file.Name) == "__init__.py" { + if _, found := initedPackages[pyPkg]; found { + panic(fmt.Errorf("found __init__.py path duplicates during pars merging: %q", file.Name)) + } + initedPackages[pyPkg] = true + } + for pyPkg != "" { + if _, found := allPackages[pyPkg]; found { + break + } + allPackages[pyPkg] = true + pyPkg = getPackage(pyPkg) + } + } + } + noInitPackages := make([]string, 0) + for pyPkg := range allPackages { + if _, found := initedPackages[pyPkg]; !found { + noInitPackages = append(noInitPackages, pyPkg) + } + } + return noInitPackages, nil } -func (be bufferEntry) String() string { - return "internal buffer" +// An InputZip owned by the InputZipsManager. Opened ManagedInputZip's are chained in the open order. +type ManagedInputZip struct { + owner *InputZipsManager + realInputZip InputZip + older *ManagedInputZip + newer *ManagedInputZip } -func (be bufferEntry) IsDir() bool { - return be.fh.FileInfo().IsDir() +// Maintains the array of ManagedInputZips, keeping track of open input ones. When an InputZip is opened, +// may close some other InputZip to limit the number of open ones. +type InputZipsManager struct { + inputZips []*ManagedInputZip + nOpenZips int + maxOpenZips int + openInputZips *ManagedInputZip } -func (be bufferEntry) CRC32() uint32 { - return crc32.ChecksumIEEE(be.content) +func (miz *ManagedInputZip) unlink() { + olderMiz := miz.older + newerMiz := miz.newer + if newerMiz.older != miz || olderMiz.newer != miz { + panic(fmt.Errorf("removing %p:%#v: broken list between %p:%#v and %p:%#v", + miz, miz, newerMiz, newerMiz, olderMiz, olderMiz)) + } + olderMiz.newer = newerMiz + newerMiz.older = olderMiz + miz.newer = nil + miz.older = nil } -func (be bufferEntry) Size() uint64 { - return uint64(len(be.content)) +func (miz *ManagedInputZip) link(olderMiz *ManagedInputZip) { + if olderMiz.newer != nil || olderMiz.older != nil { + panic(fmt.Errorf("inputZip is already open")) + } + oldOlderMiz := miz.older + if oldOlderMiz.newer != miz { + panic(fmt.Errorf("broken list between %p:%#v and %p:%#v", miz, miz, oldOlderMiz, oldOlderMiz)) + } + miz.older = olderMiz + olderMiz.older = oldOlderMiz + oldOlderMiz.newer = olderMiz + olderMiz.newer = miz } -func (be bufferEntry) WriteToZip(dest string, zw *zip.Writer) error { - w, err := zw.CreateHeader(be.fh) - if err != nil { - return err +func NewInputZipsManager(nInputZips, maxOpenZips int) *InputZipsManager { + if maxOpenZips < 3 { + panic(fmt.Errorf("open zips limit should be above 3")) + } + // In the dummy element .older points to the most recently opened InputZip, and .newer points to the oldest. + head := new(ManagedInputZip) + head.older = head + head.newer = head + return &InputZipsManager{ + inputZips: make([]*ManagedInputZip, 0, nInputZips), + maxOpenZips: maxOpenZips, + openInputZips: head, } +} - if !be.IsDir() { - _, err = w.Write(be.content) - if err != nil { +// InputZip factory +func (izm *InputZipsManager) Manage(inz InputZip) InputZip { + iz := &ManagedInputZip{owner: izm, realInputZip: inz} + izm.inputZips = append(izm.inputZips, iz) + return iz +} + +// Opens or reopens ManagedInputZip. +func (izm *InputZipsManager) reopen(miz *ManagedInputZip) error { + if miz.realInputZip.IsOpen() { + if miz != izm.openInputZips { + miz.unlink() + izm.openInputZips.link(miz) + } + return nil + } + if izm.nOpenZips >= izm.maxOpenZips { + if err := izm.close(izm.openInputZips.older); err != nil { return err } } + if err := miz.realInputZip.Open(); err != nil { + return err + } + izm.openInputZips.link(miz) + izm.nOpenZips++ + return nil +} +func (izm *InputZipsManager) close(miz *ManagedInputZip) error { + if miz.IsOpen() { + err := miz.realInputZip.Close() + izm.nOpenZips-- + miz.unlink() + return err + } return nil } -type zipSource interface { - String() string - IsDir() bool - CRC32() uint32 - Size() uint64 - WriteToZip(dest string, zw *zip.Writer) error +// Checks that openInputZips deque is valid +func (izm *InputZipsManager) checkOpenZipsDeque() { + nReallyOpen := 0 + el := izm.openInputZips + for { + elNext := el.older + if elNext.newer != el { + panic(fmt.Errorf("Element:\n %p: %v\nNext:\n %p %v", el, el, elNext, elNext)) + } + if elNext == izm.openInputZips { + break + } + el = elNext + if !el.IsOpen() { + panic(fmt.Errorf("Found unopened element")) + } + nReallyOpen++ + if nReallyOpen > izm.nOpenZips { + panic(fmt.Errorf("found %d open zips, should be %d", nReallyOpen, izm.nOpenZips)) + } + } + if nReallyOpen > izm.nOpenZips { + panic(fmt.Errorf("found %d open zips, should be %d", nReallyOpen, izm.nOpenZips)) + } } -// a fileMapping specifies to copy a zip entry from one place to another -type fileMapping struct { - dest string - source zipSource +func (miz *ManagedInputZip) Name() string { + return miz.realInputZip.Name() } -func mergeZips(readers []namedZipReader, writer *zip.Writer, manifest, pyMain string, - sortEntries, emulateJar, emulatePar, stripDirEntries, ignoreDuplicates bool, - stripFiles, stripDirs []string, zipsToNotStrip map[string]bool) error { +func (miz *ManagedInputZip) Open() error { + return miz.owner.reopen(miz) +} - sourceByDest := make(map[string]zipSource, 0) - orderedMappings := []fileMapping{} +func (miz *ManagedInputZip) Close() error { + return miz.owner.close(miz) +} - // if dest already exists returns a non-null zipSource for the existing source - addMapping := func(dest string, source zipSource) zipSource { - mapKey := filepath.Clean(dest) - if existingSource, exists := sourceByDest[mapKey]; exists { - return existingSource - } +func (miz *ManagedInputZip) IsOpen() bool { + return miz.realInputZip.IsOpen() +} - sourceByDest[mapKey] = source - orderedMappings = append(orderedMappings, fileMapping{source: source, dest: dest}) - return nil +func (miz *ManagedInputZip) Entries() []*zip.File { + if !miz.IsOpen() { + panic(fmt.Errorf("%s: is not open", miz.Name())) } + return miz.realInputZip.Entries() +} - if manifest != "" { - if !stripDirEntries { - dirHeader := jar.MetaDirFileHeader() - dirSource := bufferEntry{dirHeader, nil} - addMapping(jar.MetaDir, dirSource) - } +// Actual processing. +func mergeZips(inputZips []InputZip, writer *zip.Writer, manifest, pyMain string, + sortEntries, emulateJar, emulatePar, stripDirEntries, ignoreDuplicates bool, + excludeFiles, excludeDirs []string, zipsToNotStrip map[string]bool) error { - contents, err := ioutil.ReadFile(manifest) - if err != nil { + out := NewOutputZip(writer, sortEntries, emulateJar, stripDirEntries, ignoreDuplicates) + out.setExcludeFiles(excludeFiles) + out.setExcludeDirs(excludeDirs) + if manifest != "" { + if err := out.addManifest(manifest); err != nil { return err } - - fh, buf, err := jar.ManifestFileContents(contents) - if err != nil { + } + if pyMain != "" { + if err := out.addZipEntryFromFile("__main__.py", pyMain); err != nil { return err } - - fileSource := bufferEntry{fh, buf} - addMapping(jar.ManifestFile, fileSource) } - if pyMain != "" { - buf, err := ioutil.ReadFile(pyMain) + if emulatePar { + noInitPackages, err := out.getUninitializedPythonPackages(inputZips) if err != nil { return err } - fh := &zip.FileHeader{ - Name: "__main__.py", - Method: zip.Store, - UncompressedSize64: uint64(len(buf)), + for _, uninitializedPyPackage := range noInitPackages { + if err = out.addEmptyEntry(filepath.Join(uninitializedPyPackage, "__init__.py")); err != nil { + return err + } } - fh.SetMode(0700) - fh.SetModTime(jar.DefaultTime) - fileSource := bufferEntry{fh, buf} - addMapping("__main__.py", fileSource) } - if emulatePar { - // the runfiles packages needs to be populated with "__init__.py". - newPyPkgs := []string{} - // the runfiles dirs have been treated as packages. - existingPyPkgSet := make(map[string]bool) - // put existing __init__.py files to a set first. This set is used for preventing - // generated __init__.py files from overwriting existing ones. - for _, namedReader := range readers { - for _, file := range namedReader.reader.File { - if filepath.Base(file.Name) != "__init__.py" { - continue - } - pyPkg := pathBeforeLastSlash(file.Name) - if _, found := existingPyPkgSet[pyPkg]; found { - panic(fmt.Errorf("found __init__.py path duplicates during pars merging: %q.", file.Name)) - } else { - existingPyPkgSet[pyPkg] = true - } - } + // Finally, add entries from all the input zips. + for _, inputZip := range inputZips { + _, copyFully := zipsToNotStrip[inputZip.Name()] + if err := inputZip.Open(); err != nil { + return err } - for _, namedReader := range readers { - for _, file := range namedReader.reader.File { - var parentPath string /* the path after trimming last "/" */ - if filepath.Base(file.Name) == "__init__.py" { - // for existing __init__.py files, we should trim last "/" for twice. - // eg. a/b/c/__init__.py ---> a/b - parentPath = pathBeforeLastSlash(pathBeforeLastSlash(file.Name)) - } else { - parentPath = pathBeforeLastSlash(file.Name) + + for i, entry := range inputZip.Entries() { + if copyFully || !out.isEntryExcluded(entry.Name) { + if err := out.copyEntry(inputZip, i); err != nil { + return err } - populateNewPyPkgs(parentPath, existingPyPkgSet, &newPyPkgs) } } - for _, pkg := range newPyPkgs { - var emptyBuf []byte - fh := &zip.FileHeader{ - Name: filepath.Join(pkg, "__init__.py"), - Method: zip.Store, - UncompressedSize64: uint64(len(emptyBuf)), + // Unless we need to rearrange the entries, the input zip can now be closed. + if !(emulateJar || sortEntries) { + if err := inputZip.Close(); err != nil { + return err } - fh.SetMode(0700) - fh.SetModTime(jar.DefaultTime) - fileSource := bufferEntry{fh, emptyBuf} - addMapping(filepath.Join(pkg, "__init__.py"), fileSource) } } - for _, namedReader := range readers { - _, skipStripThisZip := zipsToNotStrip[namedReader.path] - for _, file := range namedReader.reader.File { - if !skipStripThisZip { - if skip, err := shouldStripEntry(emulateJar, stripFiles, stripDirs, file.Name); err != nil { - return err - } else if skip { - continue - } - } - if stripDirEntries && file.FileInfo().IsDir() { - continue - } + if emulateJar { + return out.writeEntries(out.jarSorted()) + } else if sortEntries { + return out.writeEntries(out.alphanumericSorted()) + } + return nil +} - // check for other files or directories destined for the same path - dest := file.Name +// Process command line +type fileList []string - // make a new entry to add - source := zipEntry{path: zipEntryPath{zipName: namedReader.path, entryName: file.Name}, content: file} +func (f *fileList) String() string { + return `""` +} - if existingSource := addMapping(dest, source); existingSource != nil { - // handle duplicates - if existingSource.IsDir() != source.IsDir() { - return fmt.Errorf("Directory/file mismatch at %v from %v and %v\n", - dest, existingSource, source) - } +func (f *fileList) Set(name string) error { + *f = append(*f, filepath.Clean(name)) - if ignoreDuplicates { - continue - } + return nil +} - if emulateJar && - file.Name == jar.ManifestFile || file.Name == jar.ModuleInfoClass { - // Skip manifest and module info files that are not from the first input file - continue - } +type zipsToNotStripSet map[string]bool - if source.IsDir() { - continue - } +func (s zipsToNotStripSet) String() string { + return `""` +} - if existingSource.CRC32() == source.CRC32() && existingSource.Size() == source.Size() { - continue - } +func (s zipsToNotStripSet) Set(path string) error { + s[path] = true + return nil +} - return fmt.Errorf("Duplicate path %v found in %v and %v\n", - dest, existingSource, source) - } - } - } +var ( + sortEntries = flag.Bool("s", false, "sort entries (defaults to the order from the input zip files)") + emulateJar = flag.Bool("j", false, "sort zip entries using jar ordering (META-INF first)") + emulatePar = flag.Bool("p", false, "merge zip entries based on par format") + excludeDirs fileList + excludeFiles fileList + zipsToNotStrip = make(zipsToNotStripSet) + stripDirEntries = flag.Bool("D", false, "strip directory entries from the output zip file") + manifest = flag.String("m", "", "manifest file to insert in jar") + pyMain = flag.String("pm", "", "__main__.py file to insert in par") + prefix = flag.String("prefix", "", "A file to prefix to the zip file") + ignoreDuplicates = flag.Bool("ignore-duplicates", false, "take each entry from the first zip it exists in and don't warn") +) - if emulateJar { - jarSort(orderedMappings) - } else if sortEntries { - alphanumericSort(orderedMappings) - } +func init() { + flag.Var(&excludeDirs, "stripDir", "directories to be excluded from the output zip, accepts wildcards") + flag.Var(&excludeFiles, "stripFile", "files to be excluded from the output zip, accepts wildcards") + flag.Var(&zipsToNotStrip, "zipToNotStrip", "the input zip file which is not applicable for stripping") +} - for _, entry := range orderedMappings { - if err := entry.source.WriteToZip(entry.dest, writer); err != nil { - return err - } - } +type FileInputZip struct { + name string + reader *zip.ReadCloser +} +func (fiz *FileInputZip) Name() string { + return fiz.name +} + +func (fiz *FileInputZip) Close() error { + if fiz.IsOpen() { + reader := fiz.reader + fiz.reader = nil + return reader.Close() + } return nil } -// Sets the given directory and all its ancestor directories as Python packages. -func populateNewPyPkgs(pkgPath string, existingPyPkgSet map[string]bool, newPyPkgs *[]string) { - for pkgPath != "" { - if _, found := existingPyPkgSet[pkgPath]; !found { - existingPyPkgSet[pkgPath] = true - *newPyPkgs = append(*newPyPkgs, pkgPath) - // Gets its ancestor directory by trimming last slash. - pkgPath = pathBeforeLastSlash(pkgPath) - } else { - break - } +func (fiz *FileInputZip) Entries() []*zip.File { + if !fiz.IsOpen() { + panic(fmt.Errorf("%s: is not open", fiz.Name())) } + return fiz.reader.File } -func pathBeforeLastSlash(path string) string { - ret := filepath.Dir(path) - // filepath.Dir("abc") -> "." and filepath.Dir("/abc") -> "/". - if ret == "." || ret == "/" { - return "" +func (fiz *FileInputZip) IsOpen() bool { + return fiz.reader != nil +} + +func (fiz *FileInputZip) Open() error { + if fiz.IsOpen() { + return nil } - return ret + var err error + if fiz.reader, err = zip.OpenReader(fiz.Name()); err != nil { + return fmt.Errorf("%s: %s", fiz.Name(), err.Error()) + } + return nil } -func shouldStripEntry(emulateJar bool, stripFiles, stripDirs []string, name string) (bool, error) { - for _, dir := range stripDirs { - dir = filepath.Clean(dir) - patterns := []string{ - dir + "/", // the directory itself - dir + "/**/*", // files recursively in the directory - dir + "/**/*/", // directories recursively in the directory - } +func main() { + flag.Usage = func() { + fmt.Fprintln(os.Stderr, "usage: merge_zips [-jpsD] [-m manifest] [--prefix script] [-pm __main__.py] OutputZip [inputs...]") + flag.PrintDefaults() + } - for _, pattern := range patterns { - match, err := pathtools.Match(pattern, name) + // parse args + flag.Parse() + args := flag.Args() + if len(args) < 1 { + flag.Usage() + os.Exit(1) + } + outputPath := args[0] + inputs := make([]string, 0) + for _, input := range args[1:] { + if input[0] == '@' { + bytes, err := ioutil.ReadFile(input[1:]) if err != nil { - return false, fmt.Errorf("%s: %s", err.Error(), pattern) - } else if match { - if emulateJar { - // When merging jar files, don't strip META-INF/MANIFEST.MF even if stripping META-INF is - // requested. - // TODO(ccross): which files does this affect? - if name != jar.MetaDir && name != jar.ManifestFile { - return true, nil - } - } - return true, nil + log.Fatal(err) } + inputs = append(inputs, soongZip.ReadRespFile(bytes)...) + continue } + inputs = append(inputs, input) + continue } - for _, pattern := range stripFiles { - if match, err := pathtools.Match(pattern, name); err != nil { - return false, fmt.Errorf("%s: %s", err.Error(), pattern) - } else if match { - return true, nil + log.SetFlags(log.Lshortfile) + + // make writer + outputZip, err := os.Create(outputPath) + if err != nil { + log.Fatal(err) + } + defer outputZip.Close() + + var offset int64 + if *prefix != "" { + prefixFile, err := os.Open(*prefix) + if err != nil { + log.Fatal(err) + } + offset, err = io.Copy(outputZip, prefixFile) + if err != nil { + log.Fatal(err) } } - return false, nil -} -func jarSort(files []fileMapping) { - sort.SliceStable(files, func(i, j int) bool { - return jar.EntryNamesLess(files[i].dest, files[j].dest) - }) -} + writer := zip.NewWriter(outputZip) + defer func() { + err := writer.Close() + if err != nil { + log.Fatal(err) + } + }() + writer.SetOffset(offset) + + if *manifest != "" && !*emulateJar { + log.Fatal(errors.New("must specify -j when specifying a manifest via -m")) + } + + if *pyMain != "" && !*emulatePar { + log.Fatal(errors.New("must specify -p when specifying a Python __main__.py via -pm")) + } -func alphanumericSort(files []fileMapping) { - sort.SliceStable(files, func(i, j int) bool { - return files[i].dest < files[j].dest - }) + // do merge + inputZipsManager := NewInputZipsManager(len(inputs), 1000) + inputZips := make([]InputZip, len(inputs)) + for i, input := range inputs { + inputZips[i] = inputZipsManager.Manage(&FileInputZip{name: input}) + } + err = mergeZips(inputZips, writer, *manifest, *pyMain, *sortEntries, *emulateJar, *emulatePar, + *stripDirEntries, *ignoreDuplicates, []string(excludeFiles), []string(excludeDirs), + map[string]bool(zipsToNotStrip)) + if err != nil { + log.Fatal(err) + } } diff --git a/cmd/merge_zips/merge_zips_test.go b/cmd/merge_zips/merge_zips_test.go index dbde27058..cb5843607 100644 --- a/cmd/merge_zips/merge_zips_test.go +++ b/cmd/merge_zips/merge_zips_test.go @@ -51,6 +51,39 @@ var ( moduleInfoFile = testZipEntry{jar.ModuleInfoClass, 0755, []byte("module-info")} ) +type testInputZip struct { + name string + entries []testZipEntry + reader *zip.Reader +} + +func (tiz *testInputZip) Name() string { + return tiz.name +} + +func (tiz *testInputZip) Open() error { + if tiz.reader == nil { + tiz.reader = testZipEntriesToZipReader(tiz.entries) + } + return nil +} + +func (tiz *testInputZip) Close() error { + tiz.reader = nil + return nil +} + +func (tiz *testInputZip) Entries() []*zip.File { + if tiz.reader == nil { + panic(fmt.Errorf("%s: should be open to get entries", tiz.Name())) + } + return tiz.reader.File +} + +func (tiz *testInputZip) IsOpen() bool { + return tiz.reader != nil +} + func TestMergeZips(t *testing.T) { testCases := []struct { name string @@ -207,13 +240,9 @@ func TestMergeZips(t *testing.T) { for _, test := range testCases { t.Run(test.name, func(t *testing.T) { - var readers []namedZipReader + inputZips := make([]InputZip, len(test.in)) for i, in := range test.in { - r := testZipEntriesToZipReader(in) - readers = append(readers, namedZipReader{ - path: "in" + strconv.Itoa(i), - reader: r, - }) + inputZips[i] = &testInputZip{name: "in" + strconv.Itoa(i), entries: in} } want := testZipEntriesToBuf(test.out) @@ -221,7 +250,7 @@ func TestMergeZips(t *testing.T) { out := &bytes.Buffer{} writer := zip.NewWriter(out) - err := mergeZips(readers, writer, "", "", + err := mergeZips(inputZips, writer, "", "", test.sort, test.jar, false, test.stripDirEntries, test.ignoreDuplicates, test.stripFiles, test.stripDirs, test.zipsToNotStrip) @@ -304,3 +333,60 @@ func dumpZip(buf []byte) string { return ret } + +type DummyInpuZip struct { + isOpen bool +} + +func (diz *DummyInpuZip) Name() string { + return "dummy" +} + +func (diz *DummyInpuZip) Open() error { + diz.isOpen = true + return nil +} + +func (diz *DummyInpuZip) Close() error { + diz.isOpen = false + return nil +} + +func (DummyInpuZip) Entries() []*zip.File { + panic("implement me") +} + +func (diz *DummyInpuZip) IsOpen() bool { + return diz.isOpen +} + +func TestInputZipsManager(t *testing.T) { + const nInputZips = 20 + const nMaxOpenZips = 10 + izm := NewInputZipsManager(20, 10) + managedZips := make([]InputZip, nInputZips) + for i := 0; i < nInputZips; i++ { + managedZips[i] = izm.Manage(&DummyInpuZip{}) + } + + t.Run("InputZipsManager", func(t *testing.T) { + for i, iz := range managedZips { + if err := iz.Open(); err != nil { + t.Fatalf("Step %d: open failed: %s", i, err) + return + } + if izm.nOpenZips > nMaxOpenZips { + t.Errorf("Step %d: should be <=%d open zips", i, nMaxOpenZips) + } + } + if !managedZips[nInputZips-1].IsOpen() { + t.Error("The last input should stay open") + } + for _, iz := range managedZips { + iz.Close() + } + if izm.nOpenZips > 0 { + t.Error("Some input zips are still open") + } + }) +} diff --git a/cmd/multiproduct_kati/Android.bp b/cmd/multiproduct_kati/Android.bp index 13b3679ed..d34f8c3c1 100644 --- a/cmd/multiproduct_kati/Android.bp +++ b/cmd/multiproduct_kati/Android.bp @@ -24,4 +24,7 @@ blueprint_go_binary { srcs: [ "main.go", ], + testSrcs: [ + "main_test.go", + ], } diff --git a/cmd/multiproduct_kati/main.go b/cmd/multiproduct_kati/main.go index 330c5dd27..7329aee88 100644 --- a/cmd/multiproduct_kati/main.go +++ b/cmd/multiproduct_kati/main.go @@ -37,18 +37,7 @@ import ( "android/soong/zip" ) -// We default to number of cpus / 4, which seems to be the sweet spot for my -// system. I suspect this is mostly due to memory or disk bandwidth though, and -// may depend on the size ofthe source tree, so this probably isn't a great -// default. -func detectNumJobs() int { - if runtime.NumCPU() < 4 { - return 1 - } - return runtime.NumCPU() / 4 -} - -var numJobs = flag.Int("j", detectNumJobs(), "number of parallel kati jobs") +var numJobs = flag.Int("j", 0, "number of parallel jobs [0=autodetect]") var keepArtifacts = flag.Bool("keep", false, "keep archives of artifacts") var incremental = flag.Bool("incremental", false, "run in incremental mode (saving intermediates)") @@ -64,6 +53,9 @@ var buildVariant = flag.String("variant", "eng", "build variant to use") var skipProducts = flag.String("skip-products", "", "comma-separated list of products to skip (known failures, etc)") var includeProducts = flag.String("products", "", "comma-separated list of products to build") +var shardCount = flag.Int("shard-count", 1, "split the products into multiple shards (to spread the build onto multiple machines, etc)") +var shard = flag.Int("shard", 1, "1-indexed shard to execute") + const errorLeadingLines = 20 const errorTrailingLines = 20 @@ -156,10 +148,12 @@ type mpContext struct { } func main() { - writer := terminal.NewWriter(terminal.StdioImpl{}) - defer writer.Finish() + stdio := terminal.StdioImpl{} - log := logger.New(writer) + output := terminal.NewStatusOutput(stdio.Stdout(), "", false, + build.OsEnvironment().IsEnvTrue("ANDROID_QUIET_BUILD")) + + log := logger.New(output) defer log.Cleanup() flag.Parse() @@ -172,8 +166,7 @@ func main() { stat := &status.Status{} defer stat.Finish() - stat.AddOutput(terminal.NewStatusOutput(writer, "", - build.OsEnvironment().IsEnvTrue("ANDROID_QUIET_BUILD"))) + stat.AddOutput(output) var failures failureCount stat.AddOutput(&failures) @@ -188,7 +181,7 @@ func main() { Context: ctx, Logger: log, Tracer: trace, - Writer: writer, + Writer: output, Status: stat, }} @@ -229,6 +222,21 @@ func main() { trace.SetOutput(filepath.Join(config.OutDir(), "build.trace")) } + var jobs = *numJobs + if jobs < 1 { + jobs = runtime.NumCPU() / 4 + + ramGb := int(config.TotalRAM() / 1024 / 1024 / 1024) + if ramJobs := ramGb / 20; ramGb > 0 && jobs > ramJobs { + jobs = ramJobs + } + + if jobs < 1 { + jobs = 1 + } + } + log.Verbosef("Using %d parallel jobs", jobs) + setMaxFiles(log) finder := build.NewSourceFinder(buildCtx, config) @@ -277,6 +285,17 @@ func main() { } } + if *shard < 1 { + log.Fatalf("--shard value must be >= 1, not %d\n", *shard) + } else if *shardCount < 1 { + log.Fatalf("--shard-count value must be >= 1, not %d\n", *shardCount) + } else if *shard > *shardCount { + log.Fatalf("--shard (%d) must not be greater than --shard-count (%d)\n", *shard, + *shardCount) + } else if *shardCount > 1 { + finalProductsList = splitList(finalProductsList, *shardCount)[*shard-1] + } + log.Verbose("Got product list: ", finalProductsList) s := buildCtx.Status.StartTool() @@ -303,7 +322,7 @@ func main() { }() var wg sync.WaitGroup - for i := 0; i < *numJobs; i++ { + for i := 0; i < jobs; i++ { wg.Add(1) go func() { defer wg.Done() @@ -341,7 +360,7 @@ func main() { } else if failures > 1 { log.Fatalf("%d failures", failures) } else { - writer.Print("Success") + fmt.Fprintln(output, "Success") } } @@ -386,11 +405,11 @@ func buildProduct(mpctx *mpContext, product string) { Context: mpctx.Context, Logger: log, Tracer: mpctx.Tracer, - Writer: terminal.NewWriter(terminal.NewCustomStdio(nil, f, f)), + Writer: f, Thread: mpctx.Tracer.NewThread(product), Status: &status.Status{}, }} - ctx.Status.AddOutput(terminal.NewStatusOutput(ctx.Writer, "", + ctx.Status.AddOutput(terminal.NewStatusOutput(ctx.Writer, "", false, build.OsEnvironment().IsEnvTrue("ANDROID_QUIET_BUILD"))) config := build.NewConfig(ctx, flag.Args()...) @@ -466,3 +485,23 @@ func (f *failureCount) Message(level status.MsgLevel, message string) { } func (f *failureCount) Flush() {} + +func (f *failureCount) Write(p []byte) (int, error) { + // discard writes + return len(p), nil +} + +func splitList(list []string, shardCount int) (ret [][]string) { + each := len(list) / shardCount + extra := len(list) % shardCount + for i := 0; i < shardCount; i++ { + count := each + if extra > 0 { + count += 1 + extra -= 1 + } + ret = append(ret, list[:count]) + list = list[count:] + } + return +} diff --git a/cmd/multiproduct_kati/main_test.go b/cmd/multiproduct_kati/main_test.go new file mode 100644 index 000000000..263a124fb --- /dev/null +++ b/cmd/multiproduct_kati/main_test.go @@ -0,0 +1,93 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "reflect" + "testing" +) + +func TestSplitList(t *testing.T) { + testcases := []struct { + inputCount int + shardCount int + want [][]string + }{ + { + inputCount: 1, + shardCount: 1, + want: [][]string{{"1"}}, + }, + { + inputCount: 1, + shardCount: 2, + want: [][]string{{"1"}, {}}, + }, + { + inputCount: 4, + shardCount: 2, + want: [][]string{{"1", "2"}, {"3", "4"}}, + }, + { + inputCount: 19, + shardCount: 10, + want: [][]string{ + {"1", "2"}, + {"3", "4"}, + {"5", "6"}, + {"7", "8"}, + {"9", "10"}, + {"11", "12"}, + {"13", "14"}, + {"15", "16"}, + {"17", "18"}, + {"19"}, + }, + }, + { + inputCount: 15, + shardCount: 10, + want: [][]string{ + {"1", "2"}, + {"3", "4"}, + {"5", "6"}, + {"7", "8"}, + {"9", "10"}, + {"11"}, + {"12"}, + {"13"}, + {"14"}, + {"15"}, + }, + }, + } + + for _, tc := range testcases { + t.Run(fmt.Sprintf("%d/%d", tc.inputCount, tc.shardCount), func(t *testing.T) { + input := []string{} + for i := 1; i <= tc.inputCount; i++ { + input = append(input, fmt.Sprintf("%d", i)) + } + + got := splitList(input, tc.shardCount) + + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("unexpected result for splitList([]string{...%d...}, %d):\nwant: %v\n got: %v\n", + tc.inputCount, tc.shardCount, tc.want, got) + } + }) + } +} diff --git a/cmd/pom2bp/pom2bp.go b/cmd/pom2bp/pom2bp.go index a399b2834..d341b8c10 100644 --- a/cmd/pom2bp/pom2bp.go +++ b/cmd/pom2bp/pom2bp.go @@ -89,7 +89,9 @@ func (d ExtraDeps) Set(v string) error { return nil } -var extraDeps = make(ExtraDeps) +var extraStaticLibs = make(ExtraDeps) + +var extraLibs = make(ExtraDeps) type Exclude map[string]bool @@ -122,8 +124,29 @@ func (n HostModuleNames) Set(v string) error { var hostModuleNames = HostModuleNames{} +type HostAndDeviceModuleNames map[string]bool + +func (n HostAndDeviceModuleNames) IsHostAndDeviceModule(groupId string, artifactId string) bool { + _, found := n[groupId+":"+artifactId] + + return found +} + +func (n HostAndDeviceModuleNames) String() string { + return "" +} + +func (n HostAndDeviceModuleNames) Set(v string) error { + n[v] = true + return nil +} + +var hostAndDeviceModuleNames = HostAndDeviceModuleNames{} + var sdkVersion string var useVersion string +var staticDeps bool +var jetifier bool func InList(s string, list []string) bool { for _, l := range list { @@ -186,10 +209,18 @@ func (p Pom) IsDeviceModule() bool { return !p.IsHostModule() } +func (p Pom) IsHostAndDeviceModule() bool { + return hostAndDeviceModuleNames.IsHostAndDeviceModule(p.GroupId, p.ArtifactId) +} + +func (p Pom) IsHostOnly() bool { + return p.IsHostModule() && !p.IsHostAndDeviceModule() +} + func (p Pom) ModuleType() string { if p.IsAar() { return "android_library" - } else if p.IsHostModule() { + } else if p.IsHostOnly() { return "java_library_host" } else { return "java_library_static" @@ -199,7 +230,7 @@ func (p Pom) ModuleType() string { func (p Pom) ImportModuleType() string { if p.IsAar() { return "android_library_import" - } else if p.IsHostModule() { + } else if p.IsHostOnly() { return "java_import_host" } else { return "java_import" @@ -229,8 +260,12 @@ func (p Pom) BpAarDeps() []string { return p.BpDeps("aar", []string{"compile", "runtime"}) } -func (p Pom) BpExtraDeps() []string { - return extraDeps[p.BpName()] +func (p Pom) BpExtraStaticLibs() []string { + return extraStaticLibs[p.BpName()] +} + +func (p Pom) BpExtraLibs() []string { + return extraLibs[p.BpName()] } // BpDeps obtains dependencies filtered by type and scope. The results of this @@ -251,6 +286,10 @@ func (p Pom) SdkVersion() string { return sdkVersion } +func (p Pom) Jetifier() bool { + return jetifier +} + func (p *Pom) FixDeps(modules map[string]*Pom) { for _, d := range p.Dependencies { if d.Type == "" { @@ -322,19 +361,82 @@ func (p *Pom) ExtractMinSdkVersion() error { var bpTemplate = template.Must(template.New("bp").Parse(` {{.ImportModuleType}} { + name: "{{.BpName}}", + {{.ImportProperty}}: ["{{.ArtifactFile}}"], + sdk_version: "{{.SdkVersion}}", + {{- if .Jetifier}} + jetifier: true, + {{- end}} + {{- if .IsHostAndDeviceModule}} + host_supported: true, + {{- end}} + {{- if not .IsHostOnly}} + apex_available: [ + "//apex_available:platform", + "//apex_available:anyapex", + ], + {{- end}} + {{- if .IsAar}} + min_sdk_version: "{{.MinSdkVersion}}", + static_libs: [ + {{- range .BpJarDeps}} + "{{.}}", + {{- end}} + {{- range .BpAarDeps}} + "{{.}}", + {{- end}} + {{- range .BpExtraStaticLibs}} + "{{.}}", + {{- end}} + ], + {{- if .BpExtraLibs}} + libs: [ + {{- range .BpExtraLibs}} + "{{.}}", + {{- end}} + ], + {{- end}} + {{- end}} +} +`)) + +var bpDepsTemplate = template.Must(template.New("bp").Parse(` +{{.ImportModuleType}} { name: "{{.BpName}}-nodeps", {{.ImportProperty}}: ["{{.ArtifactFile}}"], sdk_version: "{{.SdkVersion}}", + {{- if .Jetifier}} + jetifier: true, + {{- end}} + {{- if .IsHostAndDeviceModule}} + host_supported: true, + {{- end}} + {{- if not .IsHostOnly}} + apex_available: [ + "//apex_available:platform", + "//apex_available:anyapex", + ], + {{- end}} {{- if .IsAar}} min_sdk_version: "{{.MinSdkVersion}}", static_libs: [ + {{- range .BpJarDeps}} + "{{.}}", + {{- end}} {{- range .BpAarDeps}} "{{.}}", {{- end}} - {{- range .BpExtraDeps}} + {{- range .BpExtraStaticLibs}} "{{.}}", {{- end}} ], + {{- if .BpExtraLibs}} + libs: [ + {{- range .BpExtraLibs}} + "{{.}}", + {{- end}} + ], + {{- end}} {{- end}} } @@ -342,23 +444,41 @@ var bpTemplate = template.Must(template.New("bp").Parse(` name: "{{.BpName}}", {{- if .IsDeviceModule}} sdk_version: "{{.SdkVersion}}", + {{- if .IsHostAndDeviceModule}} + host_supported: true, + {{- end}} + {{- if not .IsHostOnly}} + apex_available: [ + "//apex_available:platform", + "//apex_available:anyapex", + ], + {{- end}} {{- if .IsAar}} min_sdk_version: "{{.MinSdkVersion}}", manifest: "manifests/{{.BpName}}/AndroidManifest.xml", + {{- else if not .IsHostOnly}} + min_sdk_version: "24", {{- end}} {{- end}} static_libs: [ "{{.BpName}}-nodeps", - {{- range .BpJarDeps}} + {{- range .BpJarDeps}} "{{.}}", {{- end}} {{- range .BpAarDeps}} "{{.}}", {{- end}} - {{- range .BpExtraDeps}} + {{- range .BpExtraStaticLibs}} + "{{.}}", + {{- end}} + ], + {{- if .BpExtraLibs}} + libs: [ + {{- range .BpExtraLibs}} "{{.}}", {{- end}} ], + {{- end}} java_version: "1.7", } `)) @@ -458,7 +578,7 @@ func main() { The tool will extract the necessary information from *.pom files to create an Android.bp whose aar libraries can be linked against when using AAPT2. -Usage: %s [--rewrite <regex>=<replace>] [-exclude <module>] [--extra-deps <module>=<module>[,<module>]] [<dir>] [-regen <file>] +Usage: %s [--rewrite <regex>=<replace>] [-exclude <module>] [--extra-static-libs <module>=<module>[,<module>]] [--extra-libs <module>=<module>[,<module>]] [<dir>] [-regen <file>] -rewrite <regex>=<replace> rewrite can be used to specify mappings between Maven projects and Android.bp modules. The -rewrite @@ -468,15 +588,21 @@ Usage: %s [--rewrite <regex>=<replace>] [-exclude <module>] [--extra-deps <modul the Android.bp module name using <replace>. If no matches are found, <artifactId> is used. -exclude <module> Don't put the specified module in the Android.bp file. - -extra-deps <module>=<module>[,<module>] - Some Android.bp modules have transitive dependencies that must be specified when they are - depended upon (like android-support-v7-mediarouter requires android-support-v7-appcompat). + -extra-static-libs <module>=<module>[,<module>] + Some Android.bp modules have transitive static dependencies that must be specified when they + are depended upon (like android-support-v7-mediarouter requires android-support-v7-appcompat). + This may be specified multiple times to declare these dependencies. + -extra-libs <module>=<module>[,<module>] + Some Android.bp modules have transitive runtime dependencies that must be specified when they + are depended upon (like androidx.test.rules requires android.test.base). This may be specified multiple times to declare these dependencies. -sdk-version <version> - Sets LOCAL_SDK_VERSION := <version> for all modules. + Sets sdk_version: "<version>" for all modules. -use-version <version> If the maven directory contains multiple versions of artifacts and their pom files, -use-version can be used to only write Android.bp files for a specific version of those artifacts. + -jetifier + Sets jetifier: true for all modules. <dir> The directory to search for *.pom files under. The contents are written to stdout, to be put in the current directory (often as Android.bp) @@ -490,12 +616,15 @@ Usage: %s [--rewrite <regex>=<replace>] [-exclude <module>] [--extra-deps <modul var regen string flag.Var(&excludes, "exclude", "Exclude module") - flag.Var(&extraDeps, "extra-deps", "Extra dependencies needed when depending on a module") + flag.Var(&extraStaticLibs, "extra-static-libs", "Extra static dependencies needed when depending on a module") + flag.Var(&extraLibs, "extra-libs", "Extra runtime dependencies needed when depending on a module") flag.Var(&rewriteNames, "rewrite", "Regex(es) to rewrite artifact names") flag.Var(&hostModuleNames, "host", "Specifies that the corresponding module (specified in the form 'module.group:module.artifact') is a host module") - flag.StringVar(&sdkVersion, "sdk-version", "", "What to write to LOCAL_SDK_VERSION") + flag.Var(&hostAndDeviceModuleNames, "host-and-device", "Specifies that the corresponding module (specified in the form 'module.group:module.artifact') is both a host and device module.") + flag.StringVar(&sdkVersion, "sdk-version", "", "What to write to sdk_version") flag.StringVar(&useVersion, "use-version", "", "Only read artifacts of a specific version") - flag.Bool("static-deps", false, "Ignored") + flag.BoolVar(&staticDeps, "static-deps", false, "Statically include direct dependencies") + flag.BoolVar(&jetifier, "jetifier", false, "Sets jetifier: true on all modules") flag.StringVar(®en, "regen", "", "Rewrite specified file") flag.Parse() @@ -609,7 +738,11 @@ Usage: %s [--rewrite <regex>=<replace>] [-exclude <module>] [--extra-deps <modul for _, pom := range poms { var err error - err = bpTemplate.Execute(buf, pom) + if staticDeps { + err = bpDepsTemplate.Execute(buf, pom) + } else { + err = bpTemplate.Execute(buf, pom) + } if err != nil { fmt.Fprintln(os.Stderr, "Error writing", pom.PomFile, pom.BpName(), err) os.Exit(1) diff --git a/cmd/pom2mk/pom2mk.go b/cmd/pom2mk/pom2mk.go index 94e561943..b3471556a 100644 --- a/cmd/pom2mk/pom2mk.go +++ b/cmd/pom2mk/pom2mk.go @@ -104,6 +104,7 @@ var excludes = make(Exclude) var sdkVersion string var useVersion string var staticDeps bool +var jetifier bool func InList(s string, list []string) bool { for _, l := range list { @@ -195,6 +196,10 @@ func (p Pom) SdkVersion() string { return sdkVersion } +func (p Pom) Jetifier() bool { + return jetifier +} + func (p *Pom) FixDeps(modules map[string]*Pom) { for _, d := range p.Dependencies { if d.Type == "" { @@ -229,6 +234,7 @@ LOCAL_STATIC_JAVA_LIBRARIES :={{range .MkJarDeps}} \ {{.}}{{end}} LOCAL_STATIC_ANDROID_LIBRARIES :={{range .MkAarDeps}} \ {{.}}{{end}} +LOCAL_JETIFIER_ENABLED := {{if .Jetifier}}true{{end}} include $(BUILD_PREBUILT) `)) @@ -367,6 +373,8 @@ Usage: %s [--rewrite <regex>=<replace>] [-exclude <module>] [--extra-deps <modul -use-version can be used to only write makefiles for a specific version of those artifacts. -static-deps Whether to statically include direct dependencies. + -jetifier + Enable jetifier in order to use androidx <dir> The directory to search for *.pom files under. The makefile is written to stdout, to be put in the current directory (often as Android.mk) @@ -383,6 +391,7 @@ Usage: %s [--rewrite <regex>=<replace>] [-exclude <module>] [--extra-deps <modul flag.StringVar(&sdkVersion, "sdk-version", "", "What to write to LOCAL_SDK_VERSION") flag.StringVar(&useVersion, "use-version", "", "Only read artifacts of a specific version") flag.BoolVar(&staticDeps, "static-deps", false, "Statically include direct dependencies") + flag.BoolVar(&jetifier, "jetifier", false, "Enable jetifier in order to use androidx") flag.StringVar(®en, "regen", "", "Rewrite specified file") flag.Parse() diff --git a/cmd/sbox/sbox.go b/cmd/sbox/sbox.go index 7057b33f0..65a34fdf4 100644 --- a/cmd/sbox/sbox.go +++ b/cmd/sbox/sbox.go @@ -36,6 +36,7 @@ var ( outputRoot string keepOutDir bool depfileOut string + inputHash string ) func init() { @@ -51,6 +52,8 @@ func init() { flag.StringVar(&depfileOut, "depfile-out", "", "file path of the depfile to generate. This value will replace '__SBOX_DEPFILE__' in the command and will be treated as an output but won't be added to __SBOX_OUT_FILES__") + flag.StringVar(&inputHash, "input-hash", "", + "This option is ignored. Typical usage is to supply a hash of the list of input names so that the module will be rebuilt if the list (and thus the hash) changes.") } func usageViolation(violation string) { @@ -59,7 +62,7 @@ func usageViolation(violation string) { } fmt.Fprintf(os.Stderr, - "Usage: sbox -c <commandToRun> --sandbox-path <sandboxPath> --output-root <outputRoot> [--depfile-out depFile] <outputFile> [<outputFile>...]\n"+ + "Usage: sbox -c <commandToRun> --sandbox-path <sandboxPath> --output-root <outputRoot> [--depfile-out depFile] [--input-hash hash] <outputFile> [<outputFile>...]\n"+ "\n"+ "Deletes <outputRoot>,"+ "runs <commandToRun>,"+ diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index 41c7d46ce..30381e088 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -18,7 +18,12 @@ import ( "flag" "fmt" "os" + "os/exec" "path/filepath" + "strconv" + "strings" + "syscall" + "time" "github.com/google/blueprint/bootstrap" @@ -50,6 +55,42 @@ func newNameResolver(config android.Config) *android.NameResolver { } func main() { + if android.SoongDelveListen != "" { + if android.SoongDelvePath == "" { + fmt.Fprintln(os.Stderr, "SOONG_DELVE is set but failed to find dlv") + os.Exit(1) + } + pid := strconv.Itoa(os.Getpid()) + cmd := []string{android.SoongDelvePath, + "attach", pid, + "--headless", + "-l", android.SoongDelveListen, + "--api-version=2", + "--accept-multiclient", + "--log", + } + + fmt.Println("Starting", strings.Join(cmd, " ")) + dlv := exec.Command(cmd[0], cmd[1:]...) + dlv.Stdout = os.Stdout + dlv.Stderr = os.Stderr + dlv.Stdin = nil + + // Put dlv into its own process group so we can kill it and the child process it starts. + dlv.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + + err := dlv.Start() + if err != nil { + // Print the error starting dlv and continue. + fmt.Println(err) + } else { + // Kill the process group for dlv when soong_build exits. + defer syscall.Kill(-dlv.Process.Pid, syscall.SIGKILL) + // Wait to give dlv a chance to connect and pause the process. + time.Sleep(time.Second) + } + } + flag.Parse() // The top-level Blueprints file is passed as the first argument. @@ -72,7 +113,17 @@ func main() { ctx.SetAllowMissingDependencies(configuration.AllowMissingDependencies()) - bootstrap.Main(ctx.Context, configuration, configuration.ConfigFileName, configuration.ProductVariablesFileName) + extraNinjaDeps := []string{configuration.ConfigFileName, configuration.ProductVariablesFileName} + + // Read the SOONG_DELVE again through configuration so that there is a dependency on the environment variable + // and soong_build will rerun when it is set for the first time. + if listen := configuration.Getenv("SOONG_DELVE"); listen != "" { + // Add a non-existent file to the dependencies so that soong_build will rerun when the debugger is + // enabled even if it completed successfully. + extraNinjaDeps = append(extraNinjaDeps, filepath.Join(configuration.BuildDir(), "always_rerun_for_delve")) + } + + bootstrap.Main(ctx.Context, configuration, extraNinjaDeps...) if docFile != "" { if err := writeDocs(ctx, docFile); err != nil { diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go index 345a821bb..a880eea72 100644 --- a/cmd/soong_ui/main.go +++ b/cmd/soong_ui/main.go @@ -32,50 +32,105 @@ import ( "android/soong/ui/tracer" ) +// A command represents an operation to be executed in the soong build +// system. +type command struct { + // the flag name (must have double dashes) + flag string + + // description for the flag (to display when running help) + description string + + // Forces the status output into dumb terminal mode. + forceDumbOutput bool + + // Sets a prefix string to use for filenames of log files. + logsPrefix string + + // Creates the build configuration based on the args and build context. + config func(ctx build.Context, args ...string) build.Config + + // Returns what type of IO redirection this Command requires. + stdio func() terminal.StdioInterface + + // run the command + run func(ctx build.Context, config build.Config, args []string, logsDir string) +} + +const makeModeFlagName = "--make-mode" + +// list of supported commands (flags) supported by soong ui +var commands []command = []command{ + { + flag: makeModeFlagName, + description: "build the modules by the target name (i.e. soong_docs)", + config: func(ctx build.Context, args ...string) build.Config { + return build.NewConfig(ctx, args...) + }, + stdio: stdio, + run: make, + }, { + flag: "--dumpvar-mode", + description: "print the value of the legacy make variable VAR to stdout", + forceDumbOutput: true, + logsPrefix: "dumpvars-", + config: dumpVarConfig, + stdio: customStdio, + run: dumpVar, + }, { + flag: "--dumpvars-mode", + description: "dump the values of one or more legacy make variables, in shell syntax", + forceDumbOutput: true, + logsPrefix: "dumpvars-", + config: dumpVarConfig, + stdio: customStdio, + run: dumpVars, + }, { + flag: "--build-mode", + description: "build modules based on the specified build action", + config: buildActionConfig, + stdio: stdio, + run: make, + }, +} + +// indexList returns the index of first found s. -1 is return if s is not +// found. func indexList(s string, list []string) int { for i, l := range list { if l == s { return i } } - return -1 } +// inList returns true if one or more of s is in the list. func inList(s string, list []string) bool { return indexList(s, list) != -1 } +// Main execution of soong_ui. The command format is as follows: +// +// soong_ui <command> [<arg 1> <arg 2> ... <arg n>] +// +// Command is the type of soong_ui execution. Only one type of +// execution is specified. The args are specific to the command. func main() { buildStarted := time.Now() - var stdio terminal.StdioInterface - stdio = terminal.StdioImpl{} - simpleOutput := false - logsPrefix := "" - - // dumpvar uses stdout, everything else should be in stderr - if os.Args[1] == "--dumpvar-mode" || os.Args[1] == "--dumpvars-mode" { - // Any metrics files add the prefix to distinguish the type of metrics being - // collected to further aggregate the metrics. For dump-var mode, it is usually - // related to the execution of lunch command. - logsPrefix = "dumpvars-" - simpleOutput = true - stdio = terminal.NewCustomStdio(os.Stdin, os.Stderr, os.Stderr) + + c, args := getCommand(os.Args) + if c == nil { + fmt.Fprintf(os.Stderr, "The `soong` native UI is not yet available.\n") + os.Exit(1) } - writer := terminal.NewWriter(stdio) - defer writer.Finish() + output := terminal.NewStatusOutput(c.stdio().Stdout(), os.Getenv("NINJA_STATUS"), c.forceDumbOutput, + build.OsEnvironment().IsEnvTrue("ANDROID_QUIET_BUILD")) - log := logger.New(writer) + log := logger.New(output) defer log.Cleanup() - if len(os.Args) < 2 || !(inList("--make-mode", os.Args) || - os.Args[1] == "--dumpvars-mode" || - os.Args[1] == "--dumpvar-mode") { - - log.Fatalln("The `soong` native UI is not yet available.") - } - ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -88,8 +143,7 @@ func main() { stat := &status.Status{} defer stat.Finish() - stat.AddOutput(terminal.NewStatusOutput(writer, os.Getenv("NINJA_STATUS"), - build.OsEnvironment().IsEnvTrue("ANDROID_QUIET_BUILD"))) + stat.AddOutput(output) stat.AddOutput(trace.StatusTracer()) build.SetupSignals(log, cancel, func() { @@ -103,15 +157,11 @@ func main() { Logger: log, Metrics: met, Tracer: trace, - Writer: writer, + Writer: output, Status: stat, }} - var config build.Config - if os.Args[1] == "--dumpvars-mode" || os.Args[1] == "--dumpvar-mode" { - config = build.NewConfig(buildCtx) - } else { - config = build.NewConfig(buildCtx, os.Args[1:]...) - } + + config := c.config(buildCtx, args...) build.SetupOutDir(buildCtx, config) @@ -120,17 +170,22 @@ func main() { logsDir = filepath.Join(config.DistDir(), "logs") } - buildErrorFile := filepath.Join(logsDir, logsPrefix+"build_error") - rbeMetricsFile := filepath.Join(logsDir, logsPrefix+"rbe_metrics.pb") - soongMetricsFile := filepath.Join(logsDir, logsPrefix+"soong_metrics") - defer build.UploadMetrics(buildCtx, config, simpleOutput, buildStarted, buildErrorFile, rbeMetricsFile, soongMetricsFile) + buildErrorFile := filepath.Join(logsDir, c.logsPrefix+"build_error") + rbeMetricsFile := filepath.Join(logsDir, c.logsPrefix+"rbe_metrics.pb") + soongMetricsFile := filepath.Join(logsDir, c.logsPrefix+"soong_metrics") + defer build.UploadMetrics(buildCtx, config, c.forceDumbOutput, buildStarted, buildErrorFile, rbeMetricsFile, soongMetricsFile) os.MkdirAll(logsDir, 0777) + log.SetOutput(filepath.Join(logsDir, c.logsPrefix+"soong.log")) + trace.SetOutput(filepath.Join(logsDir, c.logsPrefix+"build.trace")) + stat.AddOutput(status.NewVerboseLog(log, filepath.Join(logsDir, c.logsPrefix+"verbose.log"))) + stat.AddOutput(status.NewErrorLog(log, filepath.Join(logsDir, c.logsPrefix+"error.log"))) + stat.AddOutput(status.NewProtoErrorLog(log, buildErrorFile)) + stat.AddOutput(status.NewCriticalPath(log)) - log.SetOutput(filepath.Join(logsDir, "soong.log")) - trace.SetOutput(filepath.Join(logsDir, "build.trace")) - stat.AddOutput(status.NewVerboseLog(log, filepath.Join(logsDir, "verbose.log"))) - stat.AddOutput(status.NewErrorLog(log, filepath.Join(logsDir, "error.log"))) + buildCtx.Verbosef("Detected %.3v GB total RAM", float32(config.TotalRAM())/(1024*1024*1024)) + buildCtx.Verbosef("Parallelism (local/remote/highmem): %v/%v/%v", + config.Parallel(), config.RemoteParallel(), config.HighmemParallel()) defer met.Dump(soongMetricsFile) defer build.DumpRBEMetrics(buildCtx, config, rbeMetricsFile) @@ -158,28 +213,7 @@ func main() { defer f.Shutdown() build.FindSources(buildCtx, config, f) - if os.Args[1] == "--dumpvar-mode" { - dumpVar(buildCtx, config, os.Args[2:]) - } else if os.Args[1] == "--dumpvars-mode" { - dumpVars(buildCtx, config, os.Args[2:]) - } else { - if config.IsVerbose() { - writer.Print("! The argument `showcommands` is no longer supported.") - writer.Print("! Instead, the verbose log is always written to a compressed file in the output dir:") - writer.Print("!") - writer.Print(fmt.Sprintf("! gzip -cd %s/verbose.log.gz | less -R", logsDir)) - writer.Print("!") - writer.Print("! Older versions are saved in verbose.log.#.gz files") - writer.Print("") - time.Sleep(5 * time.Second) - } - - toBuild := build.BuildAll - if config.Checkbuild() { - toBuild |= build.RunBuildTests - } - build.Build(buildCtx, config, toBuild) - } + c.run(buildCtx, config, args, logsDir) } func fixBadDanglingLink(ctx build.Context, name string) { @@ -196,16 +230,16 @@ func fixBadDanglingLink(ctx build.Context, name string) { } } -func dumpVar(ctx build.Context, config build.Config, args []string) { +func dumpVar(ctx build.Context, config build.Config, args []string, _ string) { flags := flag.NewFlagSet("dumpvar", flag.ExitOnError) flags.Usage = func() { - fmt.Fprintf(os.Stderr, "usage: %s --dumpvar-mode [--abs] <VAR>\n\n", os.Args[0]) - fmt.Fprintln(os.Stderr, "In dumpvar mode, print the value of the legacy make variable VAR to stdout") - fmt.Fprintln(os.Stderr, "") + fmt.Fprintf(ctx.Writer, "usage: %s --dumpvar-mode [--abs] <VAR>\n\n", os.Args[0]) + fmt.Fprintln(ctx.Writer, "In dumpvar mode, print the value of the legacy make variable VAR to stdout") + fmt.Fprintln(ctx.Writer, "") - fmt.Fprintln(os.Stderr, "'report_config' is a special case that prints the human-readable config banner") - fmt.Fprintln(os.Stderr, "from the beginning of the build.") - fmt.Fprintln(os.Stderr, "") + fmt.Fprintln(ctx.Writer, "'report_config' is a special case that prints the human-readable config banner") + fmt.Fprintln(ctx.Writer, "from the beginning of the build.") + fmt.Fprintln(ctx.Writer, "") flags.PrintDefaults() } abs := flags.Bool("abs", false, "Print the absolute path of the value") @@ -246,18 +280,18 @@ func dumpVar(ctx build.Context, config build.Config, args []string) { } } -func dumpVars(ctx build.Context, config build.Config, args []string) { +func dumpVars(ctx build.Context, config build.Config, args []string, _ string) { flags := flag.NewFlagSet("dumpvars", flag.ExitOnError) flags.Usage = func() { - fmt.Fprintf(os.Stderr, "usage: %s --dumpvars-mode [--vars=\"VAR VAR ...\"]\n\n", os.Args[0]) - fmt.Fprintln(os.Stderr, "In dumpvars mode, dump the values of one or more legacy make variables, in") - fmt.Fprintln(os.Stderr, "shell syntax. The resulting output may be sourced directly into a shell to") - fmt.Fprintln(os.Stderr, "set corresponding shell variables.") - fmt.Fprintln(os.Stderr, "") - - fmt.Fprintln(os.Stderr, "'report_config' is a special case that dumps a variable containing the") - fmt.Fprintln(os.Stderr, "human-readable config banner from the beginning of the build.") - fmt.Fprintln(os.Stderr, "") + fmt.Fprintf(ctx.Writer, "usage: %s --dumpvars-mode [--vars=\"VAR VAR ...\"]\n\n", os.Args[0]) + fmt.Fprintln(ctx.Writer, "In dumpvars mode, dump the values of one or more legacy make variables, in") + fmt.Fprintln(ctx.Writer, "shell syntax. The resulting output may be sourced directly into a shell to") + fmt.Fprintln(ctx.Writer, "set corresponding shell variables.") + fmt.Fprintln(ctx.Writer, "") + + fmt.Fprintln(ctx.Writer, "'report_config' is a special case that dumps a variable containing the") + fmt.Fprintln(ctx.Writer, "human-readable config banner from the beginning of the build.") + fmt.Fprintln(ctx.Writer, "") flags.PrintDefaults() } @@ -313,3 +347,158 @@ func dumpVars(ctx build.Context, config build.Config, args []string) { fmt.Printf("%s%s='%s'\n", *absVarPrefix, name, strings.Join(res, " ")) } } + +func stdio() terminal.StdioInterface { + return terminal.StdioImpl{} +} + +func customStdio() terminal.StdioInterface { + return terminal.NewCustomStdio(os.Stdin, os.Stderr, os.Stderr) +} + +// dumpVarConfig does not require any arguments to be parsed by the NewConfig. +func dumpVarConfig(ctx build.Context, args ...string) build.Config { + return build.NewConfig(ctx) +} + +func buildActionConfig(ctx build.Context, args ...string) build.Config { + flags := flag.NewFlagSet("build-mode", flag.ContinueOnError) + flags.Usage = func() { + fmt.Fprintf(ctx.Writer, "usage: %s --build-mode --dir=<path> <build action> [<build arg 1> <build arg 2> ...]\n\n", os.Args[0]) + fmt.Fprintln(ctx.Writer, "In build mode, build the set of modules based on the specified build") + fmt.Fprintln(ctx.Writer, "action. The --dir flag is required to determine what is needed to") + fmt.Fprintln(ctx.Writer, "build in the source tree based on the build action. See below for") + fmt.Fprintln(ctx.Writer, "the list of acceptable build action flags.") + fmt.Fprintln(ctx.Writer, "") + flags.PrintDefaults() + } + + buildActionFlags := []struct { + name string + description string + action build.BuildAction + set bool + }{{ + name: "all-modules", + description: "Build action: build from the top of the source tree.", + action: build.BUILD_MODULES, + }, { + // This is redirecting to mma build command behaviour. Once it has soaked for a + // while, the build command is deleted from here once it has been removed from the + // envsetup.sh. + name: "modules-in-a-dir-no-deps", + description: "Build action: builds all of the modules in the current directory without their dependencies.", + action: build.BUILD_MODULES_IN_A_DIRECTORY, + }, { + // This is redirecting to mmma build command behaviour. Once it has soaked for a + // while, the build command is deleted from here once it has been removed from the + // envsetup.sh. + name: "modules-in-dirs-no-deps", + description: "Build action: builds all of the modules in the supplied directories without their dependencies.", + action: build.BUILD_MODULES_IN_DIRECTORIES, + }, { + name: "modules-in-a-dir", + description: "Build action: builds all of the modules in the current directory and their dependencies.", + action: build.BUILD_MODULES_IN_A_DIRECTORY, + }, { + name: "modules-in-dirs", + description: "Build action: builds all of the modules in the supplied directories and their dependencies.", + action: build.BUILD_MODULES_IN_DIRECTORIES, + }} + for i, flag := range buildActionFlags { + flags.BoolVar(&buildActionFlags[i].set, flag.name, false, flag.description) + } + dir := flags.String("dir", "", "Directory of the executed build command.") + + // Only interested in the first two args which defines the build action and the directory. + // The remaining arguments are passed down to the config. + const numBuildActionFlags = 2 + if len(args) < numBuildActionFlags { + flags.Usage() + ctx.Fatalln("Improper build action arguments.") + } + flags.Parse(args[0:numBuildActionFlags]) + + // The next block of code is to validate that exactly one build action is set and the dir flag + // is specified. + buildActionCount := 0 + var buildAction build.BuildAction + for _, flag := range buildActionFlags { + if flag.set { + buildActionCount++ + buildAction = flag.action + } + } + if buildActionCount != 1 { + ctx.Fatalln("Build action not defined.") + } + if *dir == "" { + ctx.Fatalln("-dir not specified.") + } + + // Remove the build action flags from the args as they are not recognized by the config. + args = args[numBuildActionFlags:] + return build.NewBuildActionConfig(buildAction, *dir, ctx, args...) +} + +func make(ctx build.Context, config build.Config, _ []string, logsDir string) { + if config.IsVerbose() { + writer := ctx.Writer + fmt.Fprintln(writer, "! The argument `showcommands` is no longer supported.") + fmt.Fprintln(writer, "! Instead, the verbose log is always written to a compressed file in the output dir:") + fmt.Fprintln(writer, "!") + fmt.Fprintf(writer, "! gzip -cd %s/verbose.log.gz | less -R\n", logsDir) + fmt.Fprintln(writer, "!") + fmt.Fprintln(writer, "! Older versions are saved in verbose.log.#.gz files") + fmt.Fprintln(writer, "") + select { + case <-time.After(5 * time.Second): + case <-ctx.Done(): + return + } + } + + if _, ok := config.Environment().Get("ONE_SHOT_MAKEFILE"); ok { + writer := ctx.Writer + fmt.Fprintln(writer, "! The variable `ONE_SHOT_MAKEFILE` is obsolete.") + fmt.Fprintln(writer, "!") + fmt.Fprintln(writer, "! If you're using `mm`, you'll need to run `source build/envsetup.sh` to update.") + fmt.Fprintln(writer, "!") + fmt.Fprintln(writer, "! Otherwise, either specify a module name with m, or use mma / MODULES-IN-...") + fmt.Fprintln(writer, "") + ctx.Fatal("done") + } + + toBuild := build.BuildAll + if config.Checkbuild() { + toBuild |= build.RunBuildTests + } + build.Build(ctx, config, toBuild) +} + +// getCommand finds the appropriate command based on args[1] flag. args[0] +// is the soong_ui filename. +func getCommand(args []string) (*command, []string) { + if len(args) < 2 { + return nil, args + } + + for _, c := range commands { + if c.flag == args[1] { + return &c, args[2:] + } + + // special case for --make-mode: if soong_ui was called from + // build/make/core/main.mk, the makeparallel with --ninja + // option specified puts the -j<num> before --make-mode. + // TODO: Remove this hack once it has been fixed. + if c.flag == makeModeFlagName { + if inList(makeModeFlagName, args) { + return &c, args[1:] + } + } + } + + // command not found + return nil, args +} diff --git a/cmd/zipsync/zipsync.go b/cmd/zipsync/zipsync.go index ea755f5b2..294e5ef0a 100644 --- a/cmd/zipsync/zipsync.go +++ b/cmd/zipsync/zipsync.go @@ -30,6 +30,7 @@ var ( outputDir = flag.String("d", "", "output dir") outputFile = flag.String("l", "", "output list file") filter = flag.String("f", "", "optional filter pattern") + zipPrefix = flag.String("zip-prefix", "", "optional prefix within the zip file to extract, stripping the prefix") ) func must(err error) { @@ -77,6 +78,10 @@ func main() { var files []string seen := make(map[string]string) + if *zipPrefix != "" { + *zipPrefix = filepath.Clean(*zipPrefix) + "/" + } + for _, input := range inputs { reader, err := zip.OpenReader(input) if err != nil { @@ -85,25 +90,32 @@ func main() { defer reader.Close() for _, f := range reader.File { + name := f.Name + if *zipPrefix != "" { + if !strings.HasPrefix(name, *zipPrefix) { + continue + } + name = strings.TrimPrefix(name, *zipPrefix) + } if *filter != "" { - if match, err := filepath.Match(*filter, filepath.Base(f.Name)); err != nil { + if match, err := filepath.Match(*filter, filepath.Base(name)); err != nil { log.Fatal(err) } else if !match { continue } } - if filepath.IsAbs(f.Name) { - log.Fatalf("%q in %q is an absolute path", f.Name, input) + if filepath.IsAbs(name) { + log.Fatalf("%q in %q is an absolute path", name, input) } - if prev, exists := seen[f.Name]; exists { - log.Fatalf("%q found in both %q and %q", f.Name, prev, input) + if prev, exists := seen[name]; exists { + log.Fatalf("%q found in both %q and %q", name, prev, input) } - seen[f.Name] = input + seen[name] = input - filename := filepath.Join(*outputDir, f.Name) + filename := filepath.Join(*outputDir, name) if f.FileInfo().IsDir() { - must(os.MkdirAll(filename, f.FileInfo().Mode())) + must(os.MkdirAll(filename, 0777)) } else { must(os.MkdirAll(filepath.Dir(filename), 0777)) in, err := f.Open() diff --git a/cuj/Android.bp b/cuj/Android.bp new file mode 100644 index 000000000..21d667f3c --- /dev/null +++ b/cuj/Android.bp @@ -0,0 +1,12 @@ +blueprint_go_binary { + name: "cuj_tests", + deps: [ + "soong-ui-build", + "soong-ui-logger", + "soong-ui-terminal", + "soong-ui-tracer", + ], + srcs: [ + "cuj.go", + ], +} diff --git a/cuj/cuj.go b/cuj/cuj.go new file mode 100644 index 000000000..c7ff8ffa8 --- /dev/null +++ b/cuj/cuj.go @@ -0,0 +1,190 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This executable runs a series of build commands to test and benchmark some critical user journeys. +package main + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "android/soong/ui/build" + "android/soong/ui/logger" + "android/soong/ui/metrics" + "android/soong/ui/status" + "android/soong/ui/terminal" + "android/soong/ui/tracer" +) + +type Test struct { + name string + args []string + + results TestResults +} + +type TestResults struct { + metrics *metrics.Metrics + err error +} + +// Run runs a single build command. It emulates the "m" command line by calling into Soong UI directly. +func (t *Test) Run(logsDir string) { + output := terminal.NewStatusOutput(os.Stdout, "", false, false) + + log := logger.New(output) + defer log.Cleanup() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + trace := tracer.New(log) + defer trace.Close() + + met := metrics.New() + + stat := &status.Status{} + defer stat.Finish() + stat.AddOutput(output) + stat.AddOutput(trace.StatusTracer()) + + build.SetupSignals(log, cancel, func() { + trace.Close() + log.Cleanup() + stat.Finish() + }) + + buildCtx := build.Context{ContextImpl: &build.ContextImpl{ + Context: ctx, + Logger: log, + Metrics: met, + Tracer: trace, + Writer: output, + Status: stat, + }} + + defer logger.Recover(func(err error) { + t.results.err = err + }) + + config := build.NewConfig(buildCtx, t.args...) + build.SetupOutDir(buildCtx, config) + + os.MkdirAll(logsDir, 0777) + log.SetOutput(filepath.Join(logsDir, "soong.log")) + trace.SetOutput(filepath.Join(logsDir, "build.trace")) + stat.AddOutput(status.NewVerboseLog(log, filepath.Join(logsDir, "verbose.log"))) + stat.AddOutput(status.NewErrorLog(log, filepath.Join(logsDir, "error.log"))) + stat.AddOutput(status.NewProtoErrorLog(log, filepath.Join(logsDir, "build_error"))) + stat.AddOutput(status.NewCriticalPath(log)) + + defer met.Dump(filepath.Join(logsDir, "soong_metrics")) + + if start, ok := os.LookupEnv("TRACE_BEGIN_SOONG"); ok { + if !strings.HasSuffix(start, "N") { + if start_time, err := strconv.ParseUint(start, 10, 64); err == nil { + log.Verbosef("Took %dms to start up.", + time.Since(time.Unix(0, int64(start_time))).Nanoseconds()/time.Millisecond.Nanoseconds()) + buildCtx.CompleteTrace(metrics.RunSetupTool, "startup", start_time, uint64(time.Now().UnixNano())) + } + } + + if executable, err := os.Executable(); err == nil { + trace.ImportMicrofactoryLog(filepath.Join(filepath.Dir(executable), "."+filepath.Base(executable)+".trace")) + } + } + + f := build.NewSourceFinder(buildCtx, config) + defer f.Shutdown() + build.FindSources(buildCtx, config, f) + + build.Build(buildCtx, config, build.BuildAll) + + t.results.metrics = met +} + +func main() { + outDir := os.Getenv("OUT_DIR") + if outDir == "" { + outDir = "out" + } + + cujDir := filepath.Join(outDir, "cuj_tests") + + // Use a subdirectory for the out directory for the tests to keep them isolated. + os.Setenv("OUT_DIR", filepath.Join(cujDir, "out")) + + // Each of these tests is run in sequence without resetting the output tree. The state of the output tree will + // affect each successive test. To maintain the validity of the benchmarks across changes, care must be taken + // to avoid changing the state of the tree when a test is run. This is most easily accomplished by adding tests + // at the end. + tests := []Test{ + { + // Reset the out directory to get reproducible results. + name: "clean", + args: []string{"clean"}, + }, + { + // Parse the build files. + name: "nothing", + args: []string{"nothing"}, + }, + { + // Parse the build files again to monitor issues like globs rerunning. + name: "nothing_rebuild", + args: []string{"nothing"}, + }, + { + // Parse the build files again, this should always be very short. + name: "nothing_rebuild_twice", + args: []string{"nothing"}, + }, + { + // Build the framework as a common developer task and one that keeps getting longer. + name: "framework", + args: []string{"framework"}, + }, + { + // Build the framework again to make sure it doesn't rebuild anything. + name: "framework_rebuild", + args: []string{"framework"}, + }, + { + // Build the framework again to make sure it doesn't rebuild anything even if it did the second time. + name: "framework_rebuild_twice", + args: []string{"framework"}, + }, + } + + cujMetrics := metrics.NewCriticalUserJourneysMetrics() + defer cujMetrics.Dump(filepath.Join(cujDir, "logs", "cuj_metrics.pb")) + + for i, t := range tests { + logsSubDir := fmt.Sprintf("%02d_%s", i, t.name) + logsDir := filepath.Join(cujDir, "logs", logsSubDir) + t.Run(logsDir) + if t.results.err != nil { + fmt.Printf("error running test %q: %s\n", t.name, t.results.err) + break + } + if t.results.metrics != nil { + cujMetrics.Add(t.name, t.results.metrics) + } + } +} diff --git a/cuj/run_cuj_tests.sh b/cuj/run_cuj_tests.sh new file mode 100755 index 000000000..b4f9f8895 --- /dev/null +++ b/cuj/run_cuj_tests.sh @@ -0,0 +1,29 @@ +#!/bin/bash -e + +readonly UNAME="$(uname)" +case "$UNAME" in +Linux) + readonly OS='linux' + ;; +Darwin) + readonly OS='darwin' + ;; +*) + echo "Unsupported OS '$UNAME'" + exit 1 + ;; +esac + +readonly ANDROID_TOP="$(cd $(dirname $0)/../../..; pwd)" +cd "$ANDROID_TOP" + +export OUT_DIR="${OUT_DIR:-out}" +readonly SOONG_OUT="${OUT_DIR}/soong" + +build/soong/soong_ui.bash --make-mode "${SOONG_OUT}/host/${OS}-x86/bin/cuj_tests" + +"${SOONG_OUT}/host/${OS}-x86/bin/cuj_tests" || true + +if [ -n "${DIST_DIR}" ]; then + cp -r "${OUT_DIR}/cuj_tests/logs" "${DIST_DIR}" +fi diff --git a/dexpreopt/Android.bp b/dexpreopt/Android.bp index c5f24e273..b8f7ea682 100644 --- a/dexpreopt/Android.bp +++ b/dexpreopt/Android.bp @@ -4,6 +4,7 @@ bootstrap_go_package { srcs: [ "config.go", "dexpreopt.go", + "testing.go", ], testSrcs: [ "dexpreopt_test.go", diff --git a/dexpreopt/config.go b/dexpreopt/config.go index 3b77042ba..98850e51a 100644 --- a/dexpreopt/config.go +++ b/dexpreopt/config.go @@ -16,23 +16,23 @@ package dexpreopt import ( "encoding/json" - "io/ioutil" + "fmt" "strings" + "github.com/google/blueprint" + "android/soong/android" ) -// GlobalConfig stores the configuration for dex preopting set by the product +// GlobalConfig stores the configuration for dex preopting. The fields are set +// from product variables via dex_preopt_config.mk. type GlobalConfig struct { - DefaultNoStripping bool // don't strip dex files by default - DisablePreopt bool // disable preopt for all modules DisablePreoptModules []string // modules with preopt disabled by product-specific config OnlyPreoptBootImageAndSystemServer bool // only preopt jars in the boot image or system server - GenerateApexImage bool // generate an extra boot image only containing jars from the runtime apex - UseApexImage bool // use the apex image by default + UseArtImage bool // use the art image (use other boot class path dex files without image) HasSystemOther bool // store odex files that match PatternsOnSystemOther on the system_other partition PatternsOnSystemOther []string // patterns (using '%' to denote a prefix match) to put odex on the system_other partition @@ -40,23 +40,22 @@ type GlobalConfig struct { DisableGenerateProfile bool // don't generate profiles ProfileDir string // directory to find profiles in - BootJars []string // modules for jars that form the boot class path + BootJars []string // modules for jars that form the boot class path + UpdatableBootJars []string // jars within apex that form the boot class path - RuntimeApexJars []string // modules for jars that are in the runtime apex - ProductUpdatableBootModules []string - ProductUpdatableBootLocations []string + ArtApexJars []string // modules for jars that are in the ART APEX - SystemServerJars []string // jars that form the system server - SystemServerApps []string // apps that are loaded into system server - SpeedApps []string // apps that should be speed optimized + SystemServerJars []string // jars that form the system server + SystemServerApps []string // apps that are loaded into system server + UpdatableSystemServerJars []string // jars within apex that are loaded into system server + SpeedApps []string // apps that should be speed optimized PreoptFlags []string // global dex2oat flags that should be used if no module-specific dex2oat flags are specified DefaultCompilerFilter string // default compiler filter to pass to dex2oat, overridden by --compiler-filter= in module-specific dex2oat flags SystemServerCompilerFilter string // default compiler filter to pass to dex2oat for system server jars - GenerateDMFiles bool // generate Dex Metadata files - NeverAllowStripping bool // whether stripping should not be done - used as build time check to make sure dex files are always available + GenerateDMFiles bool // generate Dex Metadata files NoDebugInfo bool // don't generate debug info by default DontResolveStartupStrings bool // don't resolve string literals loaded during application startup. @@ -65,8 +64,6 @@ type GlobalConfig struct { AlwaysOtherDebugInfo bool // always generate mini debug info for non-system server modules (overrides NoDebugInfo=true) NeverOtherDebugInfo bool // never generate mini debug info for non-system server modules (overrides NoDebugInfo=true) - MissingUsesLibraries []string // libraries that may be listed in OptionalUsesLibraries but will not be installed by the product - IsEng bool // build is a eng variant SanitizeLite bool // build is the second phase of a SANITIZE_LITE build @@ -81,28 +78,24 @@ type GlobalConfig struct { InstructionSetFeatures map[android.ArchType]string // instruction set for each architecture // Only used for boot image - DirtyImageObjects android.OptionalPath // path to a dirty-image-objects file - PreloadedClasses android.OptionalPath // path to a preloaded-classes file - BootImageProfiles android.Paths // path to a boot-image-profile.txt file - UseProfileForBootImage bool // whether a profile should be used to compile the boot image - BootFlags string // extra flags to pass to dex2oat for the boot image - Dex2oatImageXmx string // max heap size for dex2oat for the boot image - Dex2oatImageXms string // initial heap size for dex2oat for the boot image - - Tools Tools // paths to tools possibly used by the generated commands + DirtyImageObjects android.OptionalPath // path to a dirty-image-objects file + BootImageProfiles android.Paths // path to a boot-image-profile.txt file + BootFlags string // extra flags to pass to dex2oat for the boot image + Dex2oatImageXmx string // max heap size for dex2oat for the boot image + Dex2oatImageXms string // initial heap size for dex2oat for the boot image } -// Tools contains paths to tools possibly used by the generated commands. If you add a new tool here you MUST add it -// to the order-only dependency list in DEXPREOPT_GEN_DEPS. -type Tools struct { - Profman android.Path - Dex2oat android.Path - Aapt android.Path - SoongZip android.Path - Zip2zip android.Path - - VerifyUsesLibraries android.Path - ConstructContext android.Path +// GlobalSoongConfig contains the global config that is generated from Soong, +// stored in dexpreopt_soong.config. +type GlobalSoongConfig struct { + // Paths to tools possibly used by the generated commands. + Profman android.Path + Dex2oat android.Path + Aapt android.Path + SoongZip android.Path + Zip2zip android.Path + ManifestCheck android.Path + ConstructContext android.Path } type ModuleConfig struct { @@ -110,20 +103,24 @@ type ModuleConfig struct { DexLocation string // dex location on device BuildPath android.OutputPath DexPath android.Path + ManifestPath android.Path UncompressedDex bool HasApkLibraries bool PreoptFlags []string ProfileClassListing android.OptionalPath ProfileIsTextListing bool + ProfileBootListing android.OptionalPath - EnforceUsesLibraries bool - OptionalUsesLibraries []string - UsesLibraries []string - LibraryPaths map[string]android.Path + EnforceUsesLibraries bool + PresentOptionalUsesLibraries []string + UsesLibraries []string + LibraryPaths map[string]android.Path - Archs []android.ArchType - DexPreoptImages []android.Path + Archs []android.ArchType + DexPreoptImages []android.Path + DexPreoptImagesDeps []android.OutputPaths + DexPreoptImageLocations []string PreoptBootClassPathDexFiles android.Paths // file paths of boot class path files PreoptBootClassPathDexLocations []string // virtual locations of boot class path files @@ -134,10 +131,17 @@ type ModuleConfig struct { ForceCreateAppImage bool PresignedPrebuilt bool +} + +type globalSoongConfigSingleton struct{} - NoStripping bool - StripInputPath android.Path - StripOutputPath android.WritablePath +var pctx = android.NewPackageContext("android/soong/dexpreopt") + +func init() { + pctx.Import("android/soong/android") + android.RegisterSingletonType("dexpreopt-soong-config", func() android.Singleton { + return &globalSoongConfigSingleton{} + }) } func constructPath(ctx android.PathContext, path string) android.Path { @@ -174,74 +178,108 @@ func constructWritablePath(ctx android.PathContext, path string) android.Writabl return constructPath(ctx, path).(android.WritablePath) } -// LoadGlobalConfig reads the global dexpreopt.config file into a GlobalConfig struct. It is used directly in Soong -// and in dexpreopt_gen called from Make to read the $OUT/dexpreopt.config written by Make. -func LoadGlobalConfig(ctx android.PathContext, path string) (GlobalConfig, error) { +// ParseGlobalConfig parses the given data assumed to be read from the global +// dexpreopt.config file into a GlobalConfig struct. +func ParseGlobalConfig(ctx android.PathContext, data []byte) (*GlobalConfig, error) { type GlobalJSONConfig struct { - GlobalConfig + *GlobalConfig // Copies of entries in GlobalConfig that are not constructable without extra parameters. They will be // used to construct the real value manually below. DirtyImageObjects string - PreloadedClasses string BootImageProfiles []string - - Tools struct { - Profman string - Dex2oat string - Aapt string - SoongZip string - Zip2zip string - - VerifyUsesLibraries string - ConstructContext string - } } config := GlobalJSONConfig{} - err := loadConfig(ctx, path, &config) + err := json.Unmarshal(data, &config) if err != nil { return config.GlobalConfig, err } // Construct paths that require a PathContext. config.GlobalConfig.DirtyImageObjects = android.OptionalPathForPath(constructPath(ctx, config.DirtyImageObjects)) - config.GlobalConfig.PreloadedClasses = android.OptionalPathForPath(constructPath(ctx, config.PreloadedClasses)) config.GlobalConfig.BootImageProfiles = constructPaths(ctx, config.BootImageProfiles) - config.GlobalConfig.Tools.Profman = constructPath(ctx, config.Tools.Profman) - config.GlobalConfig.Tools.Dex2oat = constructPath(ctx, config.Tools.Dex2oat) - config.GlobalConfig.Tools.Aapt = constructPath(ctx, config.Tools.Aapt) - config.GlobalConfig.Tools.SoongZip = constructPath(ctx, config.Tools.SoongZip) - config.GlobalConfig.Tools.Zip2zip = constructPath(ctx, config.Tools.Zip2zip) - config.GlobalConfig.Tools.VerifyUsesLibraries = constructPath(ctx, config.Tools.VerifyUsesLibraries) - config.GlobalConfig.Tools.ConstructContext = constructPath(ctx, config.Tools.ConstructContext) - return config.GlobalConfig, nil } -// LoadModuleConfig reads a per-module dexpreopt.config file into a ModuleConfig struct. It is not used in Soong, which -// receives a ModuleConfig struct directly from java/dexpreopt.go. It is used in dexpreopt_gen called from oMake to -// read the module dexpreopt.config written by Make. -func LoadModuleConfig(ctx android.PathContext, path string) (ModuleConfig, error) { +type globalConfigAndRaw struct { + global *GlobalConfig + data []byte +} + +// GetGlobalConfig returns the global dexpreopt.config that's created in the +// make config phase. It is loaded once the first time it is called for any +// ctx.Config(), and returns the same data for all future calls with the same +// ctx.Config(). A value can be inserted for tests using +// setDexpreoptTestGlobalConfig. +func GetGlobalConfig(ctx android.PathContext) *GlobalConfig { + return getGlobalConfigRaw(ctx).global +} + +// GetGlobalConfigRawData is the same as GetGlobalConfig, except that it returns +// the literal content of dexpreopt.config. +func GetGlobalConfigRawData(ctx android.PathContext) []byte { + return getGlobalConfigRaw(ctx).data +} + +var globalConfigOnceKey = android.NewOnceKey("DexpreoptGlobalConfig") +var testGlobalConfigOnceKey = android.NewOnceKey("TestDexpreoptGlobalConfig") + +func getGlobalConfigRaw(ctx android.PathContext) globalConfigAndRaw { + return ctx.Config().Once(globalConfigOnceKey, func() interface{} { + if data, err := ctx.Config().DexpreoptGlobalConfig(ctx); err != nil { + panic(err) + } else if data != nil { + globalConfig, err := ParseGlobalConfig(ctx, data) + if err != nil { + panic(err) + } + return globalConfigAndRaw{globalConfig, data} + } + + // No global config filename set, see if there is a test config set + return ctx.Config().Once(testGlobalConfigOnceKey, func() interface{} { + // Nope, return a config with preopting disabled + return globalConfigAndRaw{&GlobalConfig{ + DisablePreopt: true, + DisableGenerateProfile: true, + }, nil} + }) + }).(globalConfigAndRaw) +} + +// SetTestGlobalConfig sets a GlobalConfig that future calls to GetGlobalConfig +// will return. It must be called before the first call to GetGlobalConfig for +// the config. +func SetTestGlobalConfig(config android.Config, globalConfig *GlobalConfig) { + config.Once(testGlobalConfigOnceKey, func() interface{} { return globalConfigAndRaw{globalConfig, nil} }) +} + +// ParseModuleConfig parses a per-module dexpreopt.config file into a +// ModuleConfig struct. It is not used in Soong, which receives a ModuleConfig +// struct directly from java/dexpreopt.go. It is used in dexpreopt_gen called +// from Make to read the module dexpreopt.config written in the Make config +// stage. +func ParseModuleConfig(ctx android.PathContext, data []byte) (*ModuleConfig, error) { type ModuleJSONConfig struct { - ModuleConfig + *ModuleConfig // Copies of entries in ModuleConfig that are not constructable without extra parameters. They will be // used to construct the real value manually below. BuildPath string DexPath string + ManifestPath string ProfileClassListing string LibraryPaths map[string]string DexPreoptImages []string + DexPreoptImageLocations []string PreoptBootClassPathDexFiles []string - StripInputPath string - StripOutputPath string } config := ModuleJSONConfig{} - err := loadConfig(ctx, path, &config) + err := json.Unmarshal(data, &config) if err != nil { return config.ModuleConfig, err } @@ -249,39 +287,220 @@ func LoadModuleConfig(ctx android.PathContext, path string) (ModuleConfig, error // Construct paths that require a PathContext. config.ModuleConfig.BuildPath = constructPath(ctx, config.BuildPath).(android.OutputPath) config.ModuleConfig.DexPath = constructPath(ctx, config.DexPath) + config.ModuleConfig.ManifestPath = constructPath(ctx, config.ManifestPath) config.ModuleConfig.ProfileClassListing = android.OptionalPathForPath(constructPath(ctx, config.ProfileClassListing)) config.ModuleConfig.LibraryPaths = constructPathMap(ctx, config.LibraryPaths) config.ModuleConfig.DexPreoptImages = constructPaths(ctx, config.DexPreoptImages) + config.ModuleConfig.DexPreoptImageLocations = config.DexPreoptImageLocations config.ModuleConfig.PreoptBootClassPathDexFiles = constructPaths(ctx, config.PreoptBootClassPathDexFiles) - config.ModuleConfig.StripInputPath = constructPath(ctx, config.StripInputPath) - config.ModuleConfig.StripOutputPath = constructWritablePath(ctx, config.StripOutputPath) + + // This needs to exist, but dependencies are already handled in Make, so we don't need to pass them through JSON. + config.ModuleConfig.DexPreoptImagesDeps = make([]android.OutputPaths, len(config.ModuleConfig.DexPreoptImages)) return config.ModuleConfig, nil } -func loadConfig(ctx android.PathContext, path string, config interface{}) error { - r, err := ctx.Fs().Open(path) - if err != nil { - return err +// dex2oatModuleName returns the name of the module to use for the dex2oat host +// tool. It should be a binary module with public visibility that is compiled +// and installed for host. +func dex2oatModuleName(config android.Config) string { + // Default to the debug variant of dex2oat to help find bugs. + // Set USE_DEX2OAT_DEBUG to false for only building non-debug versions. + if config.Getenv("USE_DEX2OAT_DEBUG") == "false" { + return "dex2oat" + } else { + return "dex2oatd" + } +} + +var dex2oatDepTag = struct { + blueprint.BaseDependencyTag +}{} + +// RegisterToolDeps adds the necessary dependencies to binary modules for tools +// that are required later when Get(Cached)GlobalSoongConfig is called. It +// should be called from a mutator that's registered with +// android.RegistrationContext.FinalDepsMutators. +func RegisterToolDeps(ctx android.BottomUpMutatorContext) { + dex2oatBin := dex2oatModuleName(ctx.Config()) + v := ctx.Config().BuildOSTarget.Variations() + ctx.AddFarVariationDependencies(v, dex2oatDepTag, dex2oatBin) +} + +func dex2oatPathFromDep(ctx android.ModuleContext) android.Path { + dex2oatBin := dex2oatModuleName(ctx.Config()) + + dex2oatModule := ctx.GetDirectDepWithTag(dex2oatBin, dex2oatDepTag) + if dex2oatModule == nil { + // If this happens there's probably a missing call to AddToolDeps in DepsMutator. + panic(fmt.Sprintf("Failed to lookup %s dependency", dex2oatBin)) } - defer r.Close() - data, err := ioutil.ReadAll(r) + dex2oatPath := dex2oatModule.(android.HostToolProvider).HostToolPath() + if !dex2oatPath.Valid() { + panic(fmt.Sprintf("Failed to find host tool path in %s", dex2oatModule)) + } + + return dex2oatPath.Path() +} + +// createGlobalSoongConfig creates a GlobalSoongConfig from the current context. +// Should not be used in dexpreopt_gen. +func createGlobalSoongConfig(ctx android.ModuleContext) *GlobalSoongConfig { + if ctx.Config().TestProductVariables != nil { + // If we're called in a test there'll be a confusing error from the path + // functions below that gets reported without a stack trace, so let's panic + // properly with a more helpful message. + panic("This should not be called from tests. Please call GlobalSoongConfigForTests somewhere in the test setup.") + } + + return &GlobalSoongConfig{ + Profman: ctx.Config().HostToolPath(ctx, "profman"), + Dex2oat: dex2oatPathFromDep(ctx), + Aapt: ctx.Config().HostToolPath(ctx, "aapt"), + SoongZip: ctx.Config().HostToolPath(ctx, "soong_zip"), + Zip2zip: ctx.Config().HostToolPath(ctx, "zip2zip"), + ManifestCheck: ctx.Config().HostToolPath(ctx, "manifest_check"), + ConstructContext: android.PathForSource(ctx, "build/make/core/construct_context.sh"), + } +} + +// The main reason for this Once cache for GlobalSoongConfig is to make the +// dex2oat path available to singletons. In ordinary modules we get it through a +// dex2oatDepTag dependency, but in singletons there's no simple way to do the +// same thing and ensure the right variant is selected, hence this cache to make +// the resolved path available to singletons. This means we depend on there +// being at least one ordinary module with a dex2oatDepTag dependency. +// +// TODO(b/147613152): Implement a way to deal with dependencies from singletons, +// and then possibly remove this cache altogether (but the use in +// GlobalSoongConfigForTests also needs to be rethought). +var globalSoongConfigOnceKey = android.NewOnceKey("DexpreoptGlobalSoongConfig") + +// GetGlobalSoongConfig creates a GlobalSoongConfig the first time it's called, +// and later returns the same cached instance. +func GetGlobalSoongConfig(ctx android.ModuleContext) *GlobalSoongConfig { + globalSoong := ctx.Config().Once(globalSoongConfigOnceKey, func() interface{} { + return createGlobalSoongConfig(ctx) + }).(*GlobalSoongConfig) + + // Always resolve the tool path from the dependency, to ensure that every + // module has the dependency added properly. + myDex2oat := dex2oatPathFromDep(ctx) + if myDex2oat != globalSoong.Dex2oat { + panic(fmt.Sprintf("Inconsistent dex2oat path in cached config: expected %s, got %s", globalSoong.Dex2oat, myDex2oat)) + } + + return globalSoong +} + +// GetCachedGlobalSoongConfig returns a cached GlobalSoongConfig created by an +// earlier GetGlobalSoongConfig call. This function works with any context +// compatible with a basic PathContext, since it doesn't try to create a +// GlobalSoongConfig with the proper paths (which requires a full +// ModuleContext). If there has been no prior call to GetGlobalSoongConfig, nil +// is returned. +func GetCachedGlobalSoongConfig(ctx android.PathContext) *GlobalSoongConfig { + return ctx.Config().Once(globalSoongConfigOnceKey, func() interface{} { + return (*GlobalSoongConfig)(nil) + }).(*GlobalSoongConfig) +} + +type globalJsonSoongConfig struct { + Profman string + Dex2oat string + Aapt string + SoongZip string + Zip2zip string + ManifestCheck string + ConstructContext string +} + +// ParseGlobalSoongConfig parses the given data assumed to be read from the +// global dexpreopt_soong.config file into a GlobalSoongConfig struct. It is +// only used in dexpreopt_gen. +func ParseGlobalSoongConfig(ctx android.PathContext, data []byte) (*GlobalSoongConfig, error) { + var jc globalJsonSoongConfig + + err := json.Unmarshal(data, &jc) if err != nil { - return err + return &GlobalSoongConfig{}, err } - err = json.Unmarshal(data, config) + config := &GlobalSoongConfig{ + Profman: constructPath(ctx, jc.Profman), + Dex2oat: constructPath(ctx, jc.Dex2oat), + Aapt: constructPath(ctx, jc.Aapt), + SoongZip: constructPath(ctx, jc.SoongZip), + Zip2zip: constructPath(ctx, jc.Zip2zip), + ManifestCheck: constructPath(ctx, jc.ManifestCheck), + ConstructContext: constructPath(ctx, jc.ConstructContext), + } + + return config, nil +} + +func (s *globalSoongConfigSingleton) GenerateBuildActions(ctx android.SingletonContext) { + if GetGlobalConfig(ctx).DisablePreopt { + return + } + + config := GetCachedGlobalSoongConfig(ctx) + if config == nil { + // No module has enabled dexpreopting, so we assume there will be no calls + // to dexpreopt_gen. + return + } + + jc := globalJsonSoongConfig{ + Profman: config.Profman.String(), + Dex2oat: config.Dex2oat.String(), + Aapt: config.Aapt.String(), + SoongZip: config.SoongZip.String(), + Zip2zip: config.Zip2zip.String(), + ManifestCheck: config.ManifestCheck.String(), + ConstructContext: config.ConstructContext.String(), + } + + data, err := json.Marshal(jc) if err != nil { - return err + ctx.Errorf("failed to JSON marshal GlobalSoongConfig: %v", err) + return } - return nil + ctx.Build(pctx, android.BuildParams{ + Rule: android.WriteFile, + Output: android.PathForOutput(ctx, "dexpreopt_soong.config"), + Args: map[string]string{ + "content": string(data), + }, + }) } -func GlobalConfigForTests(ctx android.PathContext) GlobalConfig { - return GlobalConfig{ - DefaultNoStripping: false, +func (s *globalSoongConfigSingleton) MakeVars(ctx android.MakeVarsContext) { + if GetGlobalConfig(ctx).DisablePreopt { + return + } + + config := GetCachedGlobalSoongConfig(ctx) + if config == nil { + return + } + + ctx.Strict("DEX2OAT", config.Dex2oat.String()) + ctx.Strict("DEXPREOPT_GEN_DEPS", strings.Join([]string{ + config.Profman.String(), + config.Dex2oat.String(), + config.Aapt.String(), + config.SoongZip.String(), + config.Zip2zip.String(), + config.ManifestCheck.String(), + config.ConstructContext.String(), + }, " ")) +} + +func GlobalConfigForTests(ctx android.PathContext) *GlobalConfig { + return &GlobalConfig{ DisablePreopt: false, DisablePreoptModules: nil, OnlyPreoptBootImageAndSystemServer: false, @@ -290,24 +509,22 @@ func GlobalConfigForTests(ctx android.PathContext) GlobalConfig { DisableGenerateProfile: false, ProfileDir: "", BootJars: nil, - RuntimeApexJars: nil, - ProductUpdatableBootModules: nil, - ProductUpdatableBootLocations: nil, + UpdatableBootJars: nil, + ArtApexJars: nil, SystemServerJars: nil, SystemServerApps: nil, + UpdatableSystemServerJars: nil, SpeedApps: nil, PreoptFlags: nil, DefaultCompilerFilter: "", SystemServerCompilerFilter: "", GenerateDMFiles: false, - NeverAllowStripping: false, NoDebugInfo: false, DontResolveStartupStrings: false, AlwaysSystemServerDebugInfo: false, NeverSystemServerDebugInfo: false, AlwaysOtherDebugInfo: false, NeverOtherDebugInfo: false, - MissingUsesLibraries: nil, IsEng: false, SanitizeLite: false, DefaultAppImages: false, @@ -317,20 +534,25 @@ func GlobalConfigForTests(ctx android.PathContext) GlobalConfig { CpuVariant: nil, InstructionSetFeatures: nil, DirtyImageObjects: android.OptionalPath{}, - PreloadedClasses: android.OptionalPath{}, BootImageProfiles: nil, - UseProfileForBootImage: false, BootFlags: "", Dex2oatImageXmx: "", Dex2oatImageXms: "", - Tools: Tools{ - Profman: android.PathForTesting("profman"), - Dex2oat: android.PathForTesting("dex2oat"), - Aapt: android.PathForTesting("aapt"), - SoongZip: android.PathForTesting("soong_zip"), - Zip2zip: android.PathForTesting("zip2zip"), - VerifyUsesLibraries: android.PathForTesting("verify_uses_libraries.sh"), - ConstructContext: android.PathForTesting("construct_context.sh"), - }, } } + +func GlobalSoongConfigForTests(config android.Config) *GlobalSoongConfig { + // Install the test GlobalSoongConfig in the Once cache so that later calls to + // Get(Cached)GlobalSoongConfig returns it without trying to create a real one. + return config.Once(globalSoongConfigOnceKey, func() interface{} { + return &GlobalSoongConfig{ + Profman: android.PathForTesting("profman"), + Dex2oat: android.PathForTesting("dex2oat"), + Aapt: android.PathForTesting("aapt"), + SoongZip: android.PathForTesting("soong_zip"), + Zip2zip: android.PathForTesting("zip2zip"), + ManifestCheck: android.PathForTesting("manifest_check"), + ConstructContext: android.PathForTesting("construct_context.sh"), + } + }).(*GlobalSoongConfig) +} diff --git a/dexpreopt/dexpreopt.go b/dexpreopt/dexpreopt.go index 5b658d989..f984966b4 100644 --- a/dexpreopt/dexpreopt.go +++ b/dexpreopt/dexpreopt.go @@ -13,7 +13,7 @@ // limitations under the License. // The dexpreopt package converts a global dexpreopt config and a module dexpreopt config into rules to perform -// dexpreopting and to strip the dex files from the APK or JAR. +// dexpreopting. // // It is used in two places; in the dexpeopt_gen binary for modules defined in Make, and directly linked into Soong. // @@ -22,8 +22,7 @@ // changed. One script takes an APK or JAR as an input and produces a zip file containing any outputs of preopting, // in the location they should be on the device. The Make build rules will unzip the zip file into $(PRODUCT_OUT) when // installing the APK, which will install the preopt outputs into $(PRODUCT_OUT)/system or $(PRODUCT_OUT)/system_other -// as necessary. The zip file may be empty if preopting was disabled for any reason. The second script takes an APK or -// JAR as an input and strips the dex files in it as necessary. +// as necessary. The zip file may be empty if preopting was disabled for any reason. // // The intermediate shell scripts allow changes to this package or to the global config to regenerate the shell scripts // but only require re-executing preopting if the script has changed. @@ -48,49 +47,12 @@ import ( const SystemPartition = "/system/" const SystemOtherPartition = "/system_other/" -// GenerateStripRule generates a set of commands that will take an APK or JAR as an input and strip the dex files if -// they are no longer necessary after preopting. -func GenerateStripRule(global GlobalConfig, module ModuleConfig) (rule *android.RuleBuilder, err error) { - defer func() { - if r := recover(); r != nil { - if _, ok := r.(runtime.Error); ok { - panic(r) - } else if e, ok := r.(error); ok { - err = e - rule = nil - } else { - panic(r) - } - } - }() - - tools := global.Tools - - rule = android.NewRuleBuilder() - - strip := shouldStripDex(module, global) - - if strip { - if global.NeverAllowStripping { - panic(fmt.Errorf("Stripping requested on %q, though the product does not allow it", module.DexLocation)) - } - // Only strips if the dex files are not already uncompressed - rule.Command(). - Textf(`if (zipinfo %s '*.dex' 2>/dev/null | grep -v ' stor ' >/dev/null) ; then`, module.StripInputPath). - Tool(tools.Zip2zip).FlagWithInput("-i ", module.StripInputPath).FlagWithOutput("-o ", module.StripOutputPath). - FlagWithArg("-x ", `"classes*.dex"`). - Textf(`; else cp -f %s %s; fi`, module.StripInputPath, module.StripOutputPath) - } else { - rule.Command().Text("cp -f").Input(module.StripInputPath).Output(module.StripOutputPath) - } - - return rule, nil -} +var DexpreoptRunningInSoong = false // GenerateDexpreoptRule generates a set of commands that will preopt a module based on a GlobalConfig and a // ModuleConfig. The produced files and their install locations will be available through rule.Installs(). -func GenerateDexpreoptRule(ctx android.PathContext, - global GlobalConfig, module ModuleConfig) (rule *android.RuleBuilder, err error) { +func GenerateDexpreoptRule(ctx android.PathContext, globalSoong *GlobalSoongConfig, + global *GlobalConfig, module *ModuleConfig) (rule *android.RuleBuilder, err error) { defer func() { if r := recover(); r != nil { @@ -108,24 +70,26 @@ func GenerateDexpreoptRule(ctx android.PathContext, rule = android.NewRuleBuilder() generateProfile := module.ProfileClassListing.Valid() && !global.DisableGenerateProfile + generateBootProfile := module.ProfileBootListing.Valid() && !global.DisableGenerateProfile var profile android.WritablePath if generateProfile { - profile = profileCommand(ctx, global, module, rule) + profile = profileCommand(ctx, globalSoong, global, module, rule) + } + if generateBootProfile { + bootProfileCommand(ctx, globalSoong, global, module, rule) } - if !dexpreoptDisabled(global, module) { + if !dexpreoptDisabled(ctx, global, module) { // Don't preopt individual boot jars, they will be preopted together. - // This check is outside dexpreoptDisabled because they still need to be stripped. if !contains(global.BootJars, module.Name) { appImage := (generateProfile || module.ForceCreateAppImage || global.DefaultAppImages) && !module.NoCreateAppImage generateDM := shouldGenerateDM(module, global) - for i, arch := range module.Archs { - image := module.DexPreoptImages[i] - dexpreoptCommand(ctx, global, module, rule, arch, profile, image, appImage, generateDM) + for archIdx, _ := range module.Archs { + dexpreoptCommand(ctx, globalSoong, global, module, rule, archIdx, profile, appImage, generateDM) } } } @@ -133,11 +97,18 @@ func GenerateDexpreoptRule(ctx android.PathContext, return rule, nil } -func dexpreoptDisabled(global GlobalConfig, module ModuleConfig) bool { +func dexpreoptDisabled(ctx android.PathContext, global *GlobalConfig, module *ModuleConfig) bool { if contains(global.DisablePreoptModules, module.Name) { return true } + // Don't preopt system server jars that are updatable. + for _, p := range global.UpdatableSystemServerJars { + if _, jar := android.SplitApexJarPair(p); jar == module.Name { + return true + } + } + // If OnlyPreoptBootImageAndSystemServer=true and module is not in boot class path skip // Also preopt system server jars since selinux prevents system server from loading anything from // /data. If we don't do this they will need to be extracted which is not favorable for RAM usage @@ -150,8 +121,8 @@ func dexpreoptDisabled(global GlobalConfig, module ModuleConfig) bool { return false } -func profileCommand(ctx android.PathContext, global GlobalConfig, module ModuleConfig, - rule *android.RuleBuilder) android.WritablePath { +func profileCommand(ctx android.PathContext, globalSoong *GlobalSoongConfig, global *GlobalConfig, + module *ModuleConfig, rule *android.RuleBuilder) android.WritablePath { profilePath := module.BuildPath.InSameDir(ctx, "profile.prof") profileInstalledPath := module.DexLocation + ".prof" @@ -162,7 +133,7 @@ func profileCommand(ctx android.PathContext, global GlobalConfig, module ModuleC cmd := rule.Command(). Text(`ANDROID_LOG_TAGS="*:e"`). - Tool(global.Tools.Profman) + Tool(globalSoong.Profman) if module.ProfileIsTextListing { // The profile is a test listing of classes (used for framework jars). @@ -189,8 +160,43 @@ func profileCommand(ctx android.PathContext, global GlobalConfig, module ModuleC return profilePath } -func dexpreoptCommand(ctx android.PathContext, global GlobalConfig, module ModuleConfig, rule *android.RuleBuilder, - arch android.ArchType, profile, bootImage android.Path, appImage, generateDM bool) { +func bootProfileCommand(ctx android.PathContext, globalSoong *GlobalSoongConfig, global *GlobalConfig, + module *ModuleConfig, rule *android.RuleBuilder) android.WritablePath { + + profilePath := module.BuildPath.InSameDir(ctx, "profile.bprof") + profileInstalledPath := module.DexLocation + ".bprof" + + if !module.ProfileIsTextListing { + rule.Command().FlagWithOutput("touch ", profilePath) + } + + cmd := rule.Command(). + Text(`ANDROID_LOG_TAGS="*:e"`). + Tool(globalSoong.Profman) + + // The profile is a test listing of methods. + // We need to generate the actual binary profile. + cmd.FlagWithInput("--create-profile-from=", module.ProfileBootListing.Path()) + + cmd. + Flag("--generate-boot-profile"). + FlagWithInput("--apk=", module.DexPath). + Flag("--dex-location="+module.DexLocation). + FlagWithOutput("--reference-profile-file=", profilePath) + + if !module.ProfileIsTextListing { + cmd.Text(fmt.Sprintf(`|| echo "Profile out of date for %s"`, module.DexPath)) + } + rule.Install(profilePath, profileInstalledPath) + + return profilePath +} + +func dexpreoptCommand(ctx android.PathContext, globalSoong *GlobalSoongConfig, global *GlobalConfig, + module *ModuleConfig, rule *android.RuleBuilder, archIdx int, profile android.WritablePath, + appImage bool, generateDM bool) { + + arch := module.Archs[archIdx] // HACK: make soname in Soong-generated .odex files match Make. base := filepath.Base(module.DexLocation) @@ -211,7 +217,7 @@ func dexpreoptCommand(ctx android.PathContext, global GlobalConfig, module Modul odexPath := module.BuildPath.InSameDir(ctx, "oat", arch.String(), pathtools.ReplaceExtension(base, "odex")) odexInstallPath := toOdexPath(module.DexLocation) if odexOnSystemOther(module, global) { - odexInstallPath = strings.Replace(odexInstallPath, SystemPartition, SystemOtherPartition, 1) + odexInstallPath = filepath.Join(SystemOtherPartition, odexInstallPath) } vdexPath := odexPath.ReplaceExtension(ctx, "vdex") @@ -219,21 +225,7 @@ func dexpreoptCommand(ctx android.PathContext, global GlobalConfig, module Modul invocationPath := odexPath.ReplaceExtension(ctx, "invocation") - // bootImage is .../dex_bootjars/system/framework/arm64/boot.art, but dex2oat wants - // .../dex_bootjars/system/framework/boot.art on the command line - var bootImageLocation string - if bootImage != nil { - bootImageLocation = PathToLocation(bootImage, arch) - } - - // Lists of used and optional libraries from the build config to be verified against the manifest in the APK - var verifyUsesLibs []string - var verifyOptionalUsesLibs []string - - // Lists of used and optional libraries from the build config, with optional libraries that are known to not - // be present in the current product removed. - var filteredUsesLibs []string - var filteredOptionalUsesLibs []string + systemServerJars := NonUpdatableSystemServerJars(ctx, global) // The class loader context using paths in the build var classLoaderContextHost android.Paths @@ -249,17 +241,14 @@ func dexpreoptCommand(ctx android.PathContext, global GlobalConfig, module Modul var conditionalClassLoaderContextHost29 android.Paths var conditionalClassLoaderContextTarget29 []string - var classLoaderContextHostString string + // A flag indicating if the '&' class loader context is used. + unknownClassLoaderContext := false if module.EnforceUsesLibraries { - verifyUsesLibs = copyOf(module.UsesLibraries) - verifyOptionalUsesLibs = copyOf(module.OptionalUsesLibraries) - - filteredOptionalUsesLibs = filterOut(global.MissingUsesLibraries, module.OptionalUsesLibraries) - filteredUsesLibs = append(copyOf(module.UsesLibraries), filteredOptionalUsesLibs...) + usesLibs := append(copyOf(module.UsesLibraries), module.PresentOptionalUsesLibraries...) // Create class loader context for dex2oat from uses libraries and filtered optional libraries - for _, l := range filteredUsesLibs { + for _, l := range usesLibs { classLoaderContextHost = append(classLoaderContextHost, pathForLibrary(module, l)) @@ -270,11 +259,13 @@ func dexpreoptCommand(ctx android.PathContext, global GlobalConfig, module Modul const httpLegacy = "org.apache.http.legacy" const httpLegacyImpl = "org.apache.http.legacy.impl" - // Fix up org.apache.http.legacy.impl since it should be org.apache.http.legacy in the manifest. - replace(verifyUsesLibs, httpLegacyImpl, httpLegacy) - replace(verifyOptionalUsesLibs, httpLegacyImpl, httpLegacy) + // org.apache.http.legacy contains classes that were in the default classpath until API 28. If the + // targetSdkVersion in the manifest or APK is < 28, and the module does not explicitly depend on + // org.apache.http.legacy, then implicitly add the classes to the classpath for dexpreopt. One the + // device the classes will be in a file called org.apache.http.legacy.impl.jar. + module.LibraryPaths[httpLegacyImpl] = module.LibraryPaths[httpLegacy] - if !contains(verifyUsesLibs, httpLegacy) && !contains(verifyOptionalUsesLibs, httpLegacy) { + if !contains(module.UsesLibraries, httpLegacy) && !contains(module.PresentOptionalUsesLibraries, httpLegacy) { conditionalClassLoaderContextHost28 = append(conditionalClassLoaderContextHost28, pathForLibrary(module, httpLegacyImpl)) conditionalClassLoaderContextTarget28 = append(conditionalClassLoaderContextTarget28, @@ -284,6 +275,9 @@ func dexpreoptCommand(ctx android.PathContext, global GlobalConfig, module Modul const hidlBase = "android.hidl.base-V1.0-java" const hidlManager = "android.hidl.manager-V1.0-java" + // android.hidl.base-V1.0-java and android.hidl.manager-V1.0 contain classes that were in the default + // classpath until API 29. If the targetSdkVersion in the manifest or APK is < 29 then implicitly add + // the classes to the classpath for dexpreopt. conditionalClassLoaderContextHost29 = append(conditionalClassLoaderContextHost29, pathForLibrary(module, hidlManager)) conditionalClassLoaderContextTarget29 = append(conditionalClassLoaderContextTarget29, @@ -292,26 +286,56 @@ func dexpreoptCommand(ctx android.PathContext, global GlobalConfig, module Modul pathForLibrary(module, hidlBase)) conditionalClassLoaderContextTarget29 = append(conditionalClassLoaderContextTarget29, filepath.Join("/system/framework", hidlBase+".jar")) + } else if jarIndex := android.IndexList(module.Name, systemServerJars); jarIndex >= 0 { + // System server jars should be dexpreopted together: class loader context of each jar + // should include all preceding jars on the system server classpath. + for _, otherJar := range systemServerJars[:jarIndex] { + classLoaderContextHost = append(classLoaderContextHost, SystemServerDexJarHostPath(ctx, otherJar)) + classLoaderContextTarget = append(classLoaderContextTarget, "/system/framework/"+otherJar+".jar") + } - classLoaderContextHostString = strings.Join(classLoaderContextHost.Strings(), ":") + // Copy the system server jar to a predefined location where dex2oat will find it. + dexPathHost := SystemServerDexJarHostPath(ctx, module.Name) + rule.Command().Text("mkdir -p").Flag(filepath.Dir(dexPathHost.String())) + rule.Command().Text("cp -f").Input(module.DexPath).Output(dexPathHost) } else { // Pass special class loader context to skip the classpath and collision check. // This will get removed once LOCAL_USES_LIBRARIES is enforced. // Right now LOCAL_USES_LIBRARIES is opt in, for the case where it's not specified we still default // to the &. - classLoaderContextHostString = `\&` + unknownClassLoaderContext = true } rule.Command().FlagWithArg("mkdir -p ", filepath.Dir(odexPath.String())) rule.Command().FlagWithOutput("rm -f ", odexPath) // Set values in the environment of the rule. These may be modified by construct_context.sh. - rule.Command().FlagWithArg("class_loader_context_arg=--class-loader-context=", classLoaderContextHostString) - rule.Command().Text(`stored_class_loader_context_arg=""`) + if unknownClassLoaderContext { + rule.Command(). + Text(`class_loader_context_arg=--class-loader-context=\&`). + Text(`stored_class_loader_context_arg=""`) + } else { + rule.Command(). + Text("class_loader_context_arg=--class-loader-context=PCL[" + strings.Join(classLoaderContextHost.Strings(), ":") + "]"). + Implicits(classLoaderContextHost). + Text("stored_class_loader_context_arg=--stored-class-loader-context=PCL[" + strings.Join(classLoaderContextTarget, ":") + "]") + } if module.EnforceUsesLibraries { - rule.Command().Textf(`uses_library_names="%s"`, strings.Join(verifyUsesLibs, " ")) - rule.Command().Textf(`optional_uses_library_names="%s"`, strings.Join(verifyOptionalUsesLibs, " ")) - rule.Command().Textf(`aapt_binary="%s"`, global.Tools.Aapt) + if module.ManifestPath != nil { + rule.Command().Text(`target_sdk_version="$(`). + Tool(globalSoong.ManifestCheck). + Flag("--extract-target-sdk-version"). + Input(module.ManifestPath). + Text(`)"`) + } else { + // No manifest to extract targetSdkVersion from, hope that DexJar is an APK + rule.Command().Text(`target_sdk_version="$(`). + Tool(globalSoong.Aapt). + Flag("dump badging"). + Input(module.DexPath). + Text(`| grep "targetSdkVersion" | sed -n "s/targetSdkVersion:'\(.*\)'/\1/p"`). + Text(`)"`) + } rule.Command().Textf(`dex_preopt_host_libraries="%s"`, strings.Join(classLoaderContextHost.Strings(), " ")). Implicits(classLoaderContextHost) @@ -327,8 +351,7 @@ func dexpreoptCommand(ctx android.PathContext, global GlobalConfig, module Modul Implicits(conditionalClassLoaderContextHost29) rule.Command().Textf(`conditional_target_libs_29="%s"`, strings.Join(conditionalClassLoaderContextTarget29, " ")) - rule.Command().Text("source").Tool(global.Tools.VerifyUsesLibraries).Input(module.DexPath) - rule.Command().Text("source").Tool(global.Tools.ConstructContext) + rule.Command().Text("source").Tool(globalSoong.ConstructContext).Input(module.DexPath) } // Devices that do not have a product partition use a symlink from /product to /system/product. @@ -341,7 +364,7 @@ func dexpreoptCommand(ctx android.PathContext, global GlobalConfig, module Modul cmd := rule.Command(). Text(`ANDROID_LOG_TAGS="*:e"`). - Tool(global.Tools.Dex2oat). + Tool(globalSoong.Dex2oat). Flag("--avoid-storing-invocation"). FlagWithOutput("--write-invocation-to=", invocationPath).ImplicitOutput(invocationPath). Flag("--runtime-arg").FlagWithArg("-Xms", global.Dex2oatXms). @@ -350,7 +373,7 @@ func dexpreoptCommand(ctx android.PathContext, global GlobalConfig, module Modul Flag("--runtime-arg").FlagWithList("-Xbootclasspath-locations:", module.PreoptBootClassPathDexLocations, ":"). Flag("${class_loader_context_arg}"). Flag("${stored_class_loader_context_arg}"). - FlagWithArg("--boot-image=", bootImageLocation).Implicit(bootImage). + FlagWithArg("--boot-image=", strings.Join(module.DexPreoptImageLocations, ":")).Implicits(module.DexPreoptImagesDeps[archIdx].Paths()). FlagWithInput("--dex-file=", module.DexPath). FlagWithArg("--dex-location=", dexLocationArg). FlagWithOutput("--oat-file=", odexPath).ImplicitOutput(vdexPath). @@ -380,7 +403,7 @@ func dexpreoptCommand(ctx android.PathContext, global GlobalConfig, module Modul cmd.FlagWithArg("--copy-dex-files=", "false") } - if !anyHavePrefix(preoptFlags, "--compiler-filter=") { + if !android.PrefixInList(preoptFlags, "--compiler-filter=") { var compilerFilter string if contains(global.SystemServerJars, module.Name) { // Jars of system server, use the product option if it is set, speed otherwise. @@ -410,7 +433,7 @@ func dexpreoptCommand(ctx android.PathContext, global GlobalConfig, module Modul dmInstalledPath := pathtools.ReplaceExtension(module.DexLocation, "dm") tmpPath := module.BuildPath.InSameDir(ctx, "primary.vdex") rule.Command().Text("cp -f").Input(vdexPath).Output(tmpPath) - rule.Command().Tool(global.Tools.SoongZip). + rule.Command().Tool(globalSoong.SoongZip). FlagWithArg("-L", "9"). FlagWithOutput("-o", dmPath). Flag("-j"). @@ -475,59 +498,14 @@ func dexpreoptCommand(ctx android.PathContext, global GlobalConfig, module Modul rule.Install(vdexPath, vdexInstallPath) } -// Return if the dex file in the APK should be stripped. If an APK is found to contain uncompressed dex files at -// dex2oat time it will not be stripped even if strip=true. -func shouldStripDex(module ModuleConfig, global GlobalConfig) bool { - strip := !global.DefaultNoStripping - - if dexpreoptDisabled(global, module) { - strip = false - } - - if module.NoStripping { - strip = false - } - - // Don't strip modules that are not on the system partition in case the oat/vdex version in system ROM - // doesn't match the one in other partitions. It needs to be able to fall back to the APK for that case. - if !strings.HasPrefix(module.DexLocation, SystemPartition) { - strip = false - } - - // system_other isn't there for an OTA, so don't strip if module is on system, and odex is on system_other. - if odexOnSystemOther(module, global) { - strip = false - } - - if module.HasApkLibraries { - strip = false - } - - // Don't strip with dex files we explicitly uncompress (dexopt will not store the dex code). - if module.UncompressedDex { - strip = false - } - - if shouldGenerateDM(module, global) { - strip = false - } - - if module.PresignedPrebuilt { - // Only strip out files if we can re-sign the package. - strip = false - } - - return strip -} - -func shouldGenerateDM(module ModuleConfig, global GlobalConfig) bool { +func shouldGenerateDM(module *ModuleConfig, global *GlobalConfig) bool { // Generating DM files only makes sense for verify, avoid doing for non verify compiler filter APKs. // No reason to use a dm file if the dex is already uncompressed. return global.GenerateDMFiles && !module.UncompressedDex && contains(module.PreoptFlags, "--compiler-filter=verify") } -func OdexOnSystemOtherByName(name string, dexLocation string, global GlobalConfig) bool { +func OdexOnSystemOtherByName(name string, dexLocation string, global *GlobalConfig) bool { if !global.HasSystemOther { return false } @@ -549,7 +527,7 @@ func OdexOnSystemOtherByName(name string, dexLocation string, global GlobalConfi return false } -func odexOnSystemOther(module ModuleConfig, global GlobalConfig) bool { +func odexOnSystemOther(module *ModuleConfig, global *GlobalConfig) bool { return OdexOnSystemOtherByName(module.Name, module.DexLocation, global) } @@ -562,7 +540,7 @@ func PathToLocation(path android.Path, arch android.ArchType) string { return filepath.Join(filepath.Dir(filepath.Dir(path.String())), filepath.Base(path.String())) } -func pathForLibrary(module ModuleConfig, lib string) android.Path { +func pathForLibrary(module *ModuleConfig, lib string) android.Path { path, ok := module.LibraryPaths[lib] if !ok { panic(fmt.Errorf("unknown library path for %q", lib)) @@ -582,41 +560,54 @@ func makefileMatch(pattern, s string) bool { } } -func contains(l []string, s string) bool { - for _, e := range l { - if e == s { - return true - } - } - return false +// Expected format for apexJarValue = <apex name>:<jar name> +func GetJarLocationFromApexJarPair(apexJarValue string) string { + apex, jar := android.SplitApexJarPair(apexJarValue) + return filepath.Join("/apex", apex, "javalib", jar+".jar") } -// remove all elements in a from b, returning a new slice -func filterOut(a []string, b []string) []string { - var ret []string - for _, x := range b { - if !contains(a, x) { - ret = append(ret, x) - } +func GetJarsFromApexJarPairs(apexJarPairs []string) []string { + modules := make([]string, len(apexJarPairs)) + for i, p := range apexJarPairs { + _, jar := android.SplitApexJarPair(p) + modules[i] = jar } - return ret + return modules } -func replace(l []string, from, to string) { - for i := range l { - if l[i] == from { - l[i] = to - } - } +var nonUpdatableSystemServerJarsKey = android.NewOnceKey("nonUpdatableSystemServerJars") + +// TODO: eliminate the superficial global config parameter by moving global config definition +// from java subpackage to dexpreopt. +func NonUpdatableSystemServerJars(ctx android.PathContext, global *GlobalConfig) []string { + return ctx.Config().Once(nonUpdatableSystemServerJarsKey, func() interface{} { + return android.RemoveListFromList(global.SystemServerJars, + GetJarsFromApexJarPairs(global.UpdatableSystemServerJars)) + }).([]string) } -var copyOf = android.CopyOf +// A predefined location for the system server dex jars. This is needed in order to generate +// class loader context for dex2oat, as the path to the jar in the Soong module may be unknown +// at that time (Soong processes the jars in dependency order, which may be different from the +// the system server classpath order). +func SystemServerDexJarHostPath(ctx android.PathContext, jar string) android.OutputPath { + if DexpreoptRunningInSoong { + // Soong module, just use the default output directory $OUT/soong. + return android.PathForOutput(ctx, "system_server_dexjars", jar+".jar") + } else { + // Make module, default output directory is $OUT (passed via the "null config" created + // by dexpreopt_gen). Append Soong subdirectory to match Soong module paths. + return android.PathForOutput(ctx, "soong", "system_server_dexjars", jar+".jar") + } +} -func anyHavePrefix(l []string, prefix string) bool { - for _, x := range l { - if strings.HasPrefix(x, prefix) { +func contains(l []string, s string) bool { + for _, e := range l { + if e == s { return true } } return false } + +var copyOf = android.CopyOf diff --git a/dexpreopt/dexpreopt_gen/dexpreopt_gen.go b/dexpreopt/dexpreopt_gen/dexpreopt_gen.go index c72f6842d..e89f04591 100644 --- a/dexpreopt/dexpreopt_gen/dexpreopt_gen.go +++ b/dexpreopt/dexpreopt_gen/dexpreopt_gen.go @@ -18,6 +18,7 @@ import ( "bytes" "flag" "fmt" + "io/ioutil" "os" "path/filepath" "runtime" @@ -30,18 +31,17 @@ import ( ) var ( - dexpreoptScriptPath = flag.String("dexpreopt_script", "", "path to output dexpreopt script") - stripScriptPath = flag.String("strip_script", "", "path to output strip script") - globalConfigPath = flag.String("global", "", "path to global configuration file") - moduleConfigPath = flag.String("module", "", "path to module configuration file") - outDir = flag.String("out_dir", "", "path to output directory") + dexpreoptScriptPath = flag.String("dexpreopt_script", "", "path to output dexpreopt script") + globalSoongConfigPath = flag.String("global_soong", "", "path to global configuration file for settings originating from Soong") + globalConfigPath = flag.String("global", "", "path to global configuration file") + moduleConfigPath = flag.String("module", "", "path to module configuration file") + outDir = flag.String("out_dir", "", "path to output directory") ) type pathContext struct { config android.Config } -func (x *pathContext) Fs() pathtools.FileSystem { return pathtools.OsFs } func (x *pathContext) Config() android.Config { return x.config } func (x *pathContext) AddNinjaFileDeps(...string) {} @@ -64,35 +64,55 @@ func main() { usage("path to output dexpreopt script is required") } - if *stripScriptPath == "" { - usage("path to output strip script is required") + if *globalSoongConfigPath == "" { + usage("--global_soong configuration file is required") } if *globalConfigPath == "" { - usage("path to global configuration file is required") + usage("--global configuration file is required") } if *moduleConfigPath == "" { - usage("path to module configuration file is required") + usage("--module configuration file is required") } - ctx := &pathContext{android.TestConfig(*outDir, nil)} + ctx := &pathContext{android.NullConfig(*outDir)} - globalConfig, err := dexpreopt.LoadGlobalConfig(ctx, *globalConfigPath) + globalSoongConfigData, err := ioutil.ReadFile(*globalSoongConfigPath) if err != nil { - fmt.Fprintf(os.Stderr, "error loading global config %q: %s\n", *globalConfigPath, err) + fmt.Fprintf(os.Stderr, "error reading global Soong config %q: %s\n", *globalSoongConfigPath, err) os.Exit(2) } - moduleConfig, err := dexpreopt.LoadModuleConfig(ctx, *moduleConfigPath) + globalSoongConfig, err := dexpreopt.ParseGlobalSoongConfig(ctx, globalSoongConfigData) if err != nil { - fmt.Fprintf(os.Stderr, "error loading module config %q: %s\n", *moduleConfigPath, err) + fmt.Fprintf(os.Stderr, "error parsing global Soong config %q: %s\n", *globalSoongConfigPath, err) os.Exit(2) } - // This shouldn't be using *PathForTesting, but it's outside of soong_build so its OK for now. - moduleConfig.StripInputPath = android.PathForTesting("$1") - moduleConfig.StripOutputPath = android.WritablePathForTesting("$2") + globalConfigData, err := ioutil.ReadFile(*globalConfigPath) + if err != nil { + fmt.Fprintf(os.Stderr, "error reading global config %q: %s\n", *globalConfigPath, err) + os.Exit(2) + } + + globalConfig, err := dexpreopt.ParseGlobalConfig(ctx, globalConfigData) + if err != nil { + fmt.Fprintf(os.Stderr, "error parsing global config %q: %s\n", *globalConfigPath, err) + os.Exit(2) + } + + moduleConfigData, err := ioutil.ReadFile(*moduleConfigPath) + if err != nil { + fmt.Fprintf(os.Stderr, "error reading module config %q: %s\n", *moduleConfigPath, err) + os.Exit(2) + } + + moduleConfig, err := dexpreopt.ParseModuleConfig(ctx, moduleConfigData) + if err != nil { + fmt.Fprintf(os.Stderr, "error parsing module config %q: %s\n", *moduleConfigPath, err) + os.Exit(2) + } moduleConfig.DexPath = android.PathForTesting("$1") @@ -110,12 +130,12 @@ func main() { } }() - writeScripts(ctx, globalConfig, moduleConfig, *dexpreoptScriptPath, *stripScriptPath) + writeScripts(ctx, globalSoongConfig, globalConfig, moduleConfig, *dexpreoptScriptPath) } -func writeScripts(ctx android.PathContext, global dexpreopt.GlobalConfig, module dexpreopt.ModuleConfig, - dexpreoptScriptPath, stripScriptPath string) { - dexpreoptRule, err := dexpreopt.GenerateDexpreoptRule(ctx, global, module) +func writeScripts(ctx android.PathContext, globalSoong *dexpreopt.GlobalSoongConfig, + global *dexpreopt.GlobalConfig, module *dexpreopt.ModuleConfig, dexpreoptScriptPath string) { + dexpreoptRule, err := dexpreopt.GenerateDexpreoptRule(ctx, globalSoong, global, module) if err != nil { panic(err) } @@ -130,16 +150,11 @@ func writeScripts(ctx android.PathContext, global dexpreopt.GlobalConfig, module dexpreoptRule.Command().Text("mkdir -p").Flag(filepath.Dir(installPath.String())) dexpreoptRule.Command().Text("cp -f").Input(install.From).Output(installPath) } - dexpreoptRule.Command().Tool(global.Tools.SoongZip). + dexpreoptRule.Command().Tool(globalSoong.SoongZip). FlagWithArg("-o ", "$2"). FlagWithArg("-C ", installDir.String()). FlagWithArg("-D ", installDir.String()) - stripRule, err := dexpreopt.GenerateStripRule(global, module) - if err != nil { - panic(err) - } - write := func(rule *android.RuleBuilder, file string) { script := &bytes.Buffer{} script.WriteString(scriptHeader) @@ -180,15 +195,8 @@ func writeScripts(ctx android.PathContext, global dexpreopt.GlobalConfig, module if module.DexPath.String() != "$1" { panic(fmt.Errorf("module.DexPath must be '$1', was %q", module.DexPath)) } - if module.StripInputPath.String() != "$1" { - panic(fmt.Errorf("module.StripInputPath must be '$1', was %q", module.StripInputPath)) - } - if module.StripOutputPath.String() != "$2" { - panic(fmt.Errorf("module.StripOutputPath must be '$2', was %q", module.StripOutputPath)) - } write(dexpreoptRule, dexpreoptScriptPath) - write(stripRule, stripScriptPath) } const scriptHeader = `#!/bin/bash diff --git a/dexpreopt/dexpreopt_test.go b/dexpreopt/dexpreopt_test.go index 6dfa9d266..d23999328 100644 --- a/dexpreopt/dexpreopt_test.go +++ b/dexpreopt/dexpreopt_test.go @@ -16,45 +16,58 @@ package dexpreopt import ( "android/soong/android" - "reflect" - "strings" + "fmt" "testing" ) -func testModuleConfig(ctx android.PathContext) ModuleConfig { - return ModuleConfig{ - Name: "test", - DexLocation: "/system/app/test/test.apk", - BuildPath: android.PathForOutput(ctx, "test/test.apk"), - DexPath: android.PathForOutput(ctx, "test/dex/test.jar"), +func testSystemModuleConfig(ctx android.PathContext, name string) *ModuleConfig { + return testModuleConfig(ctx, name, "system") +} + +func testSystemProductModuleConfig(ctx android.PathContext, name string) *ModuleConfig { + return testModuleConfig(ctx, name, "system/product") +} + +func testProductModuleConfig(ctx android.PathContext, name string) *ModuleConfig { + return testModuleConfig(ctx, name, "product") +} + +func testModuleConfig(ctx android.PathContext, name, partition string) *ModuleConfig { + return &ModuleConfig{ + Name: name, + DexLocation: fmt.Sprintf("/%s/app/test/%s.apk", partition, name), + BuildPath: android.PathForOutput(ctx, fmt.Sprintf("%s/%s.apk", name, name)), + DexPath: android.PathForOutput(ctx, fmt.Sprintf("%s/dex/%s.jar", name, name)), UncompressedDex: false, HasApkLibraries: false, PreoptFlags: nil, ProfileClassListing: android.OptionalPath{}, ProfileIsTextListing: false, EnforceUsesLibraries: false, - OptionalUsesLibraries: nil, + PresentOptionalUsesLibraries: nil, UsesLibraries: nil, LibraryPaths: nil, Archs: []android.ArchType{android.Arm}, DexPreoptImages: android.Paths{android.PathForTesting("system/framework/arm/boot.art")}, + DexPreoptImagesDeps: []android.OutputPaths{android.OutputPaths{}}, + DexPreoptImageLocations: []string{}, PreoptBootClassPathDexFiles: nil, PreoptBootClassPathDexLocations: nil, PreoptExtractedApk: false, NoCreateAppImage: false, ForceCreateAppImage: false, PresignedPrebuilt: false, - NoStripping: false, - StripInputPath: android.PathForOutput(ctx, "unstripped/test.apk"), - StripOutputPath: android.PathForOutput(ctx, "stripped/test.apk"), } } func TestDexPreopt(t *testing.T) { - ctx := android.PathContextForTesting(android.TestConfig("out", nil), nil) - global, module := GlobalConfigForTests(ctx), testModuleConfig(ctx) + config := android.TestConfig("out", nil, "", nil) + ctx := android.PathContextForTesting(config) + globalSoong := GlobalSoongConfigForTests(config) + global := GlobalConfigForTests(ctx) + module := testSystemModuleConfig(ctx, "test") - rule, err := GenerateDexpreoptRule(ctx, global, module) + rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module) if err != nil { t.Fatal(err) } @@ -69,49 +82,76 @@ func TestDexPreopt(t *testing.T) { } } -func TestDexPreoptStrip(t *testing.T) { - // Test that we panic if we strip in a configuration where stripping is not allowed. - ctx := android.PathContextForTesting(android.TestConfig("out", nil), nil) - global, module := GlobalConfigForTests(ctx), testModuleConfig(ctx) - - global.NeverAllowStripping = true - module.NoStripping = false - - _, err := GenerateStripRule(global, module) - if err == nil { - t.Errorf("Expected an error when calling GenerateStripRule on a stripped module") - } -} - func TestDexPreoptSystemOther(t *testing.T) { - ctx := android.PathContextForTesting(android.TestConfig("out", nil), nil) - global, module := GlobalConfigForTests(ctx), testModuleConfig(ctx) + config := android.TestConfig("out", nil, "", nil) + ctx := android.PathContextForTesting(config) + globalSoong := GlobalSoongConfigForTests(config) + global := GlobalConfigForTests(ctx) + systemModule := testSystemModuleConfig(ctx, "Stest") + systemProductModule := testSystemProductModuleConfig(ctx, "SPtest") + productModule := testProductModuleConfig(ctx, "Ptest") global.HasSystemOther = true - global.PatternsOnSystemOther = []string{"app/%"} - rule, err := GenerateDexpreoptRule(ctx, global, module) - if err != nil { - t.Fatal(err) + type moduleTest struct { + module *ModuleConfig + expectedPartition string } - - wantInstalls := android.RuleBuilderInstalls{ - {android.PathForOutput(ctx, "test/oat/arm/package.odex"), "/system_other/app/test/oat/arm/test.odex"}, - {android.PathForOutput(ctx, "test/oat/arm/package.vdex"), "/system_other/app/test/oat/arm/test.vdex"}, + tests := []struct { + patterns []string + moduleTests []moduleTest + }{ + { + patterns: []string{"app/%"}, + moduleTests: []moduleTest{ + {module: systemModule, expectedPartition: "system_other/system"}, + {module: systemProductModule, expectedPartition: "system/product"}, + {module: productModule, expectedPartition: "product"}, + }, + }, + // product/app/% only applies to product apps inside the system partition + { + patterns: []string{"app/%", "product/app/%"}, + moduleTests: []moduleTest{ + {module: systemModule, expectedPartition: "system_other/system"}, + {module: systemProductModule, expectedPartition: "system_other/system/product"}, + {module: productModule, expectedPartition: "product"}, + }, + }, } - if rule.Installs().String() != wantInstalls.String() { - t.Errorf("\nwant installs:\n %v\ngot:\n %v", wantInstalls, rule.Installs()) + for _, test := range tests { + global.PatternsOnSystemOther = test.patterns + for _, mt := range test.moduleTests { + rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, mt.module) + if err != nil { + t.Fatal(err) + } + + name := mt.module.Name + wantInstalls := android.RuleBuilderInstalls{ + {android.PathForOutput(ctx, name+"/oat/arm/package.odex"), fmt.Sprintf("/%s/app/test/oat/arm/%s.odex", mt.expectedPartition, name)}, + {android.PathForOutput(ctx, name+"/oat/arm/package.vdex"), fmt.Sprintf("/%s/app/test/oat/arm/%s.vdex", mt.expectedPartition, name)}, + } + + if rule.Installs().String() != wantInstalls.String() { + t.Errorf("\nwant installs:\n %v\ngot:\n %v", wantInstalls, rule.Installs()) + } + } } + } func TestDexPreoptProfile(t *testing.T) { - ctx := android.PathContextForTesting(android.TestConfig("out", nil), nil) - global, module := GlobalConfigForTests(ctx), testModuleConfig(ctx) + config := android.TestConfig("out", nil, "", nil) + ctx := android.PathContextForTesting(config) + globalSoong := GlobalSoongConfigForTests(config) + global := GlobalConfigForTests(ctx) + module := testSystemModuleConfig(ctx, "test") module.ProfileClassListing = android.OptionalPathForPath(android.PathForTesting("profile")) - rule, err := GenerateDexpreoptRule(ctx, global, module) + rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module) if err != nil { t.Fatal(err) } @@ -127,56 +167,3 @@ func TestDexPreoptProfile(t *testing.T) { t.Errorf("\nwant installs:\n %v\ngot:\n %v", wantInstalls, rule.Installs()) } } - -func TestStripDex(t *testing.T) { - tests := []struct { - name string - setup func(global *GlobalConfig, module *ModuleConfig) - strip bool - }{ - { - name: "default strip", - setup: func(global *GlobalConfig, module *ModuleConfig) {}, - strip: true, - }, - { - name: "global no stripping", - setup: func(global *GlobalConfig, module *ModuleConfig) { global.DefaultNoStripping = true }, - strip: false, - }, - { - name: "module no stripping", - setup: func(global *GlobalConfig, module *ModuleConfig) { module.NoStripping = true }, - strip: false, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - - ctx := android.PathContextForTesting(android.TestConfig("out", nil), nil) - global, module := GlobalConfigForTests(ctx), testModuleConfig(ctx) - - test.setup(&global, &module) - - rule, err := GenerateStripRule(global, module) - if err != nil { - t.Fatal(err) - } - - if test.strip { - want := `zip2zip -i out/unstripped/test.apk -o out/stripped/test.apk -x "classes*.dex"` - if len(rule.Commands()) < 1 || !strings.Contains(rule.Commands()[0], want) { - t.Errorf("\nwant commands[0] to have:\n %v\ngot:\n %v", want, rule.Commands()[0]) - } - } else { - wantCommands := []string{ - "cp -f out/unstripped/test.apk out/stripped/test.apk", - } - if !reflect.DeepEqual(rule.Commands(), wantCommands) { - t.Errorf("\nwant commands:\n %v\ngot:\n %v", wantCommands, rule.Commands()) - } - } - }) - } -} diff --git a/dexpreopt/testing.go b/dexpreopt/testing.go new file mode 100644 index 000000000..b572eb351 --- /dev/null +++ b/dexpreopt/testing.go @@ -0,0 +1,47 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dexpreopt + +import ( + "android/soong/android" +) + +type dummyToolBinary struct { + android.ModuleBase +} + +func (m *dummyToolBinary) GenerateAndroidBuildActions(ctx android.ModuleContext) {} + +func (m *dummyToolBinary) HostToolPath() android.OptionalPath { + return android.OptionalPathForPath(android.PathForTesting("dex2oat")) +} + +func dummyToolBinaryFactory() android.Module { + module := &dummyToolBinary{} + android.InitAndroidArchModule(module, android.HostSupported, android.MultilibFirst) + return module +} + +func RegisterToolModulesForTest(ctx *android.TestContext) { + ctx.RegisterModuleType("dummy_tool_binary", dummyToolBinaryFactory) +} + +func BpToolModulesForTest() string { + return ` + dummy_tool_binary { + name: "dex2oatd", + } + ` +} diff --git a/docs/best_practices.md b/docs/best_practices.md index 546e19e23..bc760b876 100644 --- a/docs/best_practices.md +++ b/docs/best_practices.md @@ -146,3 +146,145 @@ they're used dynamically via `dlopen`. If they're only used via `LOCAL_SHARED_LIBRARIES` / `shared_libs`, then those dependencies will trigger them to be installed when necessary. Adding unnecessary libraries into `PRODUCT_PACKAGES` will force them to always be installed, wasting space. + +## Removing conditionals + +Over-use of conditionals in the build files results in an untestable number +of build combinations, leading to more build breakages. It also makes the +code less testable, as it must be built with each combination of flags to +be tested. + +### Conditionally compiled module + +Conditionally compiling a module can generally be replaced with conditional +installation: + +``` +ifeq (some condition) +# body of the Android.mk file +LOCAL_MODULE:= bt_logger +include $(BUILD_EXECUTABLE) +endif +``` + +Becomes: + +``` +cc_binary { + name: "bt_logger", + // body of the module +} +``` + +And in a product Makefile somewhere (something included with +`$(call inherit-product, ...)`: + +``` +ifeq (some condition) # Or no condition +PRODUCT_PACKAGES += bt_logger +endif +``` + +If the condition was on a type of board or product, it can often be dropped +completely by putting the `PRODUCT_PACKAGES` entry in a product makefile that +is included only by the correct products or boards. + +### Conditionally compiled module with multiple implementations + +If there are multiple implementations of the same module with one selected +for compilation via a conditional, the implementations can sometimes be renamed +to unique values. + +For example, the name of the gralloc HAL module can be overridden by the +`ro.hardware.gralloc` system property: + +``` +# In hardware/acme/soc_a/gralloc/Android.mk: +ifeq ($(TARGET_BOARD_PLATFORM),soc_a) +LOCAL_MODULE := gralloc.acme +... +include $(BUILD_SHARED_LIBRARY) +endif + +# In hardware/acme/soc_b/gralloc/Android.mk: +ifeq ($(TARGET_BOARD_PLATFORM),soc_b) +LOCAL_MODULE := gralloc.acme +... +include $(BUILD_SHARED_LIBRARY) +endif +``` + +Becomes: +``` +# In hardware/acme/soc_a/gralloc/Android.bp: +cc_library { + name: "gralloc.soc_a", + ... +} + +# In hardware/acme/soc_b/gralloc/Android.bp: +cc_library { + name: "gralloc.soc_b", + ... +} +``` + +Then to select the correct gralloc implementation, a product makefile inherited +by products that use soc_a should contain: + +``` +PRODUCT_PACKAGES += gralloc.soc_a +PRODUCT_PROPERTY_OVERRIDES += ro.hardware.gralloc=soc_a +``` + +In cases where the names cannot be made unique a `soong_namespace` should be +used to partition a set of modules so that they are built only when the +namespace is listed in `PRODUCT_SOONG_NAMESPACES`. See the +[Referencing Modules](../README.md#referencing-modules) section of the Soong +README.md for more on namespaces. + +### Module with name based on variable + +HAL modules sometimes use variables like `$(TARGET_BOARD_PLATFORM)` in their +module name. These can be renamed to a fixed name. + +For example, the name of the gralloc HAL module can be overridden by the +`ro.hardware.gralloc` system property: + +``` +LOCAL_MODULE := gralloc.$(TARGET_BOARD_PLATFORM) +... +include $(BUILD_SHARED_LIBRARY) +``` + +Becomes: +``` +cc_library { + name: "gralloc.acme", + ... +} +``` + +Then to select the correct gralloc implementation, a product makefile should +contain: + +``` +PRODUCT_PACKAGES += gralloc.acme +PRODUCT_PROPERTY_OVERRIDES += ro.hardware.gralloc=acme +``` + +### Conditionally used source files, libraries or flags + +The preferred solution is to convert the conditional to runtime, either by +autodetecting the correct value or loading the value from a system property +or a configuration file. + +As a last resort, if the conditional cannot be removed, a Soong plugin can +be written in Go that can implement additional features for specific module +types. Soong plugins are inherently tightly coupled to the build system +and will require ongoing maintenance as the build system is changed; so +plugins should be used only when absolutely required. + +See [art/build/art.go](https://android.googlesource.com/platform/art/+/master/build/art.go) +or [external/llvm/soong/llvm.go](https://android.googlesource.com/platform/external/llvm/+/master/soong/llvm.go) +for examples of more complex conditionals on product variables or environment variables. diff --git a/docs/perf.md b/docs/perf.md index c3a26474f..538adffb5 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -8,7 +8,7 @@ soong_ui has tracing built in, so that every build execution's trace can be viewed. Just open `$OUT_DIR/build.trace.gz` in Chrome's <chrome://tracing>, or with [catapult's trace viewer][catapult trace_viewer]. The last few traces are stored in `build.trace.#.gz` (larger numbers are older). The associated logs -are stored in `soong.#.log`. +are stored in `soong.#.log` and `verbose.#.log.gz`.  @@ -31,29 +31,29 @@ or read a file that's going to change on every build. In most cases, we've found that the fast-path is slow because all of the `$(shell)` commands need to be re-executed to determine if their output changed. -The `$OUT_DIR/soong.log` contains statistics from the regen check: - -``` -.../kati.go:127: *kati*: regen check time: 1.699207 -.../kati.go:127: *kati*: glob time (regen): 0.377193 / 33609 -.../kati.go:127: *kati*: shell time (regen): 1.313529 / 184 -.../kati.go:127: *kati*: 0.217 find device vendor -type f -name \*.pk8 -o -name verifiedboot\* -o -name \*.x509.pem -o -name oem\*.prop | sort -.../kati.go:127: *kati*: 0.105 cd packages/apps/Dialer ; find -L . -type d -name "res" -.../kati.go:127: *kati*: 0.035 find device vendor -maxdepth 4 -name '*_aux_variant_config.mk' -o -name '*_aux_os_config.mk' | sort -.../kati.go:127: *kati*: 0.029 cd frameworks/base ; find -L core/java graphics/java location/java media/java media/mca/effect/java media/mca/filterfw/java media/mca/filterpacks/java drm/java opengl/java sax/java telecomm/java telephony/java wifi/java lowpan/java keystore/java rs/java ../opt/telephony/src/java/android/telephony ../opt/telephony/src/java/android/telephony/gsm ../opt/net/voip/src/java/android/net/rtp ../opt/net/voip/src/java/android/net/sip -name "*.html" -and -not -name ".*" -.../kati.go:127: *kati*: 0.025 test -d device && find -L device -maxdepth 4 -path '*/marlin/BoardConfig.mk' -.../kati.go:127: *kati*: 0.023 find packages/apps/Settings/tests/robotests -type f -name '*Test.java' | sed -e 's!.*\(com/google.*Test\)\.java!\1!' -e 's!.*\(com/android.*Test\)\.java!\1!' | sed 's!/!\.!g' | cat -.../kati.go:127: *kati*: 0.022 test -d vendor && find -L vendor -maxdepth 4 -path '*/marlin/BoardConfig.mk' -.../kati.go:127: *kati*: 0.017 cd cts/tests/tests/shortcutmanager/packages/launchermanifest ; find -L ../src -name "*.java" -and -not -name ".*" -.../kati.go:127: *kati*: 0.016 cd cts/tests/tests/shortcutmanager/packages/launchermanifest ; find -L ../../common/src -name "*.java" -and -not -name ".*" -.../kati.go:127: *kati*: 0.015 cd libcore && (find luni/src/test/java -name "*.java" 2> /dev/null) | grep -v -f java_tests_blacklist -.../kati.go:127: *kati*: stat time (regen): 0.250384 / 4405 -``` - -In this case, the total time spent checking was 1.69 seconds, even though the +The `$OUT_DIR/verbose.log.gz` contains statistics from the regen check: + +``` +verbose: *kati*: regen check time: 0.754030 +verbose: *kati*: glob time (regen): 0.545859 / 43840 +verbose: *kati*: shell time (regen): 0.278095 / 66 (59 unique) +verbose: *kati*: 0.012 / 1 mkdir -p out/target/product/generic && echo Android/aosp_arm/generic:R/AOSP.MASTER/$(date -d @$(cat out/build_date.txt) +%m%d%H%M):eng/test-keys >out/target/product/generic/build_fingerprint.txt && grep " " out/target/product/generic/build_fingerprint.txt +verbose: *kati*: 0.010 / 1 echo 'com.android.launcher3.config.FlagOverrideSampleTest com.android.launcher3.logging.FileLogTest com.android.launcher3.model.AddWorkspaceItemsTaskTest com.android.launcher3.model.CacheDataUpdatedTaskTest com.android.launcher3.model.DbDowngradeHelperTest com.android.launcher3.model.GridBackupTableTest com.android.launcher3.model.GridSizeMigrationTaskTest com.android.launcher3.model.PackageInstallStateChangedTaskTest com.android.launcher3.popup.PopupPopulatorTest com.android.launcher3.util.GridOccupancyTest com.android.launcher3.util.IntSetTest' | tr ' ' '\n' | cat +verbose: *kati*: 0.010 / 1 cd cts/tests/framework/base/windowmanager ; find -L * -name "Components.java" -and -not -name ".*" +verbose: *kati*: 0.010 / 1 git -C test/framework/build log -s -n 1 --format="%cd" --date=format:"%Y%m%d_%H%M%S" 2>/dev/null +verbose: *kati*: 0.009 / 2 cd development/samples/ShortcutDemo/publisher ; find -L ../common/src -name "*.java" -and -not -name ".*" +verbose: *kati*: 0.009 / 2 cd development/samples/ShortcutDemo/launcher ; find -L ../common/src -name "*.java" -and -not -name ".*" +verbose: *kati*: 0.009 / 1 if ! cmp -s out/target/product/generic/obj/CONFIG/kati_packaging/dist.mk.tmp out/target/product/generic/obj/CONFIG/kati_packaging/dist.mk; then mv out/target/product/generic/obj/CONFIG/kati_packaging/dist.mk.tmp out/target/product/generic/obj/CONFIG/kati_packaging/dist.mk; else rm out/target/product/generic/obj/CONFIG/kati_packaging/dist.mk.tmp; fi +verbose: *kati*: 0.008 / 1 mkdir -p out/target/product/generic && echo R/AOSP.MASTER/$(cat out/build_number.txt):eng/test-keys >out/target/product/generic/build_thumbprint.txt && grep " " out/target/product/generic/build_thumbprint.txt +verbose: *kati*: 0.007 / 1 echo 'com.android.customization.model.clock.BaseClockManagerTest com.android.customization.model.clock.ClockManagerTest com.android.customization.model.grid.GridOptionsManagerTest com.android.customization.model.theme.ThemeManagerTest' | tr ' ' '\n' | cat +verbose: *kati*: 0.007 / 1 uname -sm +verbose: *kati*: stat time (regen): 0.361907 / 1241 +``` + +In this case, the total time spent checking was 0.75 seconds, even though the other "(regen)" numbers add up to more than that (some parts are parallelized -where possible). The biggest contributor is the `$(shell)` times -- 184 -executions took a total of 1.31 seconds. The top 10 longest shell functions are +where possible). Often times, the biggest contributor is the `$(shell)` times +-- in this case, 66 calls took 0.27s. The top 10 longest shell functions are printed. All the longest commands in this case are all variants of a call to `find`, but @@ -96,7 +96,8 @@ cached by Kati: $(sort $(shell find device vendor -type -f -a -name \*.pk8 -o -name verifiedboot\* -o -name \*.x509.pem -o -name oem\*.prop)) ``` -Kati is learning about the implicit `-a` in [this change](https://github.com/google/kati/pull/132) +Kati has now learned about the implicit `-a`, so this particular change is no +longer necessary, but the basic concept holds. #### Kati regens too often @@ -113,6 +114,46 @@ read with the `ckati_stamp_dump` tool in prebuilts/build-tools. More debugging is available when ckati is run with `--regen_debug`, but that can be a lot of data to understand. +#### Debugging the slow path + +Kati will now dump out information about which Makefiles took the most time to +execute. This is also in the `verbose.log.gz` file: + +``` +verbose: *kati*: included makefiles: 73.640833 / 232810 (1066 unique) +verbose: *kati*: 18.389 / 1 out/soong/Android-aosp_arm.mk +verbose: *kati*: 13.137 / 20144 build/make/core/soong_cc_prebuilt.mk +verbose: *kati*: 11.743 / 27666 build/make/core/base_rules.mk +verbose: *kati*: 2.289 / 1 art/Android.mk +verbose: *kati*: 2.054 / 1 art/build/Android.cpplint.mk +verbose: *kati*: 1.955 / 28269 build/make/core/clear_vars.mk +verbose: *kati*: 1.795 / 283 build/make/core/package.mk +verbose: *kati*: 1.790 / 283 build/make/core/package_internal.mk +verbose: *kati*: 1.757 / 17382 build/make/core/link_type.mk +verbose: *kati*: 1.078 / 297 build/make/core/aapt2.mk +``` + +This shows that soong_cc_prebuilt.mk was included 20144 times, for a total time +spent of 13.137 secounds. While Android-aosp_arm.mk was only included once, and +took 18.389 seconds. In this case, Android-aosp_arm.mk is the only file that +includes soong_cc_prebuilt.mk, so we can safely assume that 13 of the 18 seconds +in Android-aosp_arm.mk was actually spent within soong_cc_prebuilt.mk (or +something that it included, like base_rules.mk). + +By default this only includes the top 10 entries, but you can ask for the stats +for any makefile to be printed with `$(KATI_profile_makefile)`: + +``` +$(KATI_profile_makefile build/make/core/product.mk) +``` + +With these primitives, it's possible to get the timing information for small +chunks, or even single lines, of a makefile. Just move the lines you want to +measure into a new makefile, and replace their use with an `include` of the +new makefile. It's possible to analyze where the time is being spent by doing +a binary search using this method, but you do need to be careful not to split +conditionals across two files (the ifeq/else/endif must all be in the same file). + ### Ninja #### Understanding why something rebuilt @@ -164,15 +205,14 @@ use, disk bandwidth, etc) may also be necessary. ### Common -#### mm +### <= Android 10 (Q): mm Soong always loads the entire module graph, so as modules convert from Make to Soong, `mm` is becoming closer to `mma`. This produces more correct builds, but does slow down builds, as we need to verify/produce/load a larger build graph. -We're exploring a few options to speed up build startup, one being [an -experimental set of ninja patches][ninja parse optimization], -though that's not the current path we're working towards. +As of Android Q, loading large build graphs is fast, and in Android R, `mm` is +now an alias of `mma`. ### Android 8.1 (Oreo MR1) diff --git a/env/Android.bp b/env/Android.bp new file mode 100644 index 000000000..90c604729 --- /dev/null +++ b/env/Android.bp @@ -0,0 +1,7 @@ +bootstrap_go_package { + name: "soong-env", + pkgPath: "android/soong/env", + srcs: [ + "env.go", + ], +} diff --git a/env/env.go b/env/env.go index bf58a9914..a98e1f6a8 100644 --- a/env/env.go +++ b/env/env.go @@ -27,7 +27,7 @@ import ( type envFileEntry struct{ Key, Value string } type envFileData []envFileEntry -func WriteEnvFile(filename string, envDeps map[string]string) error { +func EnvFileContents(envDeps map[string]string) ([]byte, error) { contents := make(envFileData, 0, len(envDeps)) for key, value := range envDeps { contents = append(contents, envFileEntry{key, value}) @@ -37,17 +37,12 @@ func WriteEnvFile(filename string, envDeps map[string]string) error { data, err := json.MarshalIndent(contents, "", " ") if err != nil { - return err + return nil, err } data = append(data, '\n') - err = ioutil.WriteFile(filename, data, 0664) - if err != nil { - return err - } - - return nil + return data, nil } func StaleEnvFile(filename string) (bool, error) { diff --git a/etc/Android.bp b/etc/Android.bp new file mode 100644 index 000000000..cfd303ec8 --- /dev/null +++ b/etc/Android.bp @@ -0,0 +1,16 @@ +bootstrap_go_package { + name: "soong-etc", + pkgPath: "android/soong/etc", + deps: [ + "blueprint", + "soong", + "soong-android", + ], + srcs: [ + "prebuilt_etc.go", + ], + testSrcs: [ + "prebuilt_etc_test.go", + ], + pluginFor: ["soong_build"], +} diff --git a/etc/prebuilt_etc.go b/etc/prebuilt_etc.go new file mode 100644 index 000000000..842d9eec1 --- /dev/null +++ b/etc/prebuilt_etc.go @@ -0,0 +1,289 @@ +// Copyright 2016 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package etc + +import ( + "strconv" + + "github.com/google/blueprint/proptools" + + "android/soong/android" +) + +var pctx = android.NewPackageContext("android/soong/etc") + +// TODO(jungw): Now that it handles more than the ones in etc/, consider renaming this file. + +func init() { + pctx.Import("android/soong/android") + + android.RegisterModuleType("prebuilt_etc", PrebuiltEtcFactory) + android.RegisterModuleType("prebuilt_etc_host", PrebuiltEtcHostFactory) + android.RegisterModuleType("prebuilt_usr_share", PrebuiltUserShareFactory) + android.RegisterModuleType("prebuilt_usr_share_host", PrebuiltUserShareHostFactory) + android.RegisterModuleType("prebuilt_font", PrebuiltFontFactory) + android.RegisterModuleType("prebuilt_firmware", PrebuiltFirmwareFactory) +} + +type prebuiltEtcProperties struct { + // Source file of this prebuilt. + Src *string `android:"path,arch_variant"` + + // optional subdirectory under which this file is installed into + Sub_dir *string `android:"arch_variant"` + + // optional name for the installed file. If unspecified, name of the module is used as the file name + Filename *string `android:"arch_variant"` + + // when set to true, and filename property is not set, the name for the installed file + // is the same as the file name of the source file. + Filename_from_src *bool `android:"arch_variant"` + + // Make this module available when building for ramdisk. + Ramdisk_available *bool + + // Make this module available when building for recovery. + Recovery_available *bool + + // Whether this module is directly installable to one of the partitions. Default: true. + Installable *bool +} + +type PrebuiltEtcModule interface { + android.Module + SubDir() string + OutputFile() android.OutputPath +} + +type PrebuiltEtc struct { + android.ModuleBase + + properties prebuiltEtcProperties + + sourceFilePath android.Path + outputFilePath android.OutputPath + // The base install location, e.g. "etc" for prebuilt_etc, "usr/share" for prebuilt_usr_share. + installDirBase string + // The base install location when soc_specific property is set to true, e.g. "firmware" for prebuilt_firmware. + socInstallDirBase string + installDirPath android.InstallPath + additionalDependencies *android.Paths +} + +func (p *PrebuiltEtc) inRamdisk() bool { + return p.ModuleBase.InRamdisk() || p.ModuleBase.InstallInRamdisk() +} + +func (p *PrebuiltEtc) onlyInRamdisk() bool { + return p.ModuleBase.InstallInRamdisk() +} + +func (p *PrebuiltEtc) InstallInRamdisk() bool { + return p.inRamdisk() +} + +func (p *PrebuiltEtc) inRecovery() bool { + return p.ModuleBase.InRecovery() || p.ModuleBase.InstallInRecovery() +} + +func (p *PrebuiltEtc) onlyInRecovery() bool { + return p.ModuleBase.InstallInRecovery() +} + +func (p *PrebuiltEtc) InstallInRecovery() bool { + return p.inRecovery() +} + +var _ android.ImageInterface = (*PrebuiltEtc)(nil) + +func (p *PrebuiltEtc) ImageMutatorBegin(ctx android.BaseModuleContext) {} + +func (p *PrebuiltEtc) CoreVariantNeeded(ctx android.BaseModuleContext) bool { + return !p.ModuleBase.InstallInRecovery() && !p.ModuleBase.InstallInRamdisk() +} + +func (p *PrebuiltEtc) RamdiskVariantNeeded(ctx android.BaseModuleContext) bool { + return proptools.Bool(p.properties.Ramdisk_available) || p.ModuleBase.InstallInRamdisk() +} + +func (p *PrebuiltEtc) RecoveryVariantNeeded(ctx android.BaseModuleContext) bool { + return proptools.Bool(p.properties.Recovery_available) || p.ModuleBase.InstallInRecovery() +} + +func (p *PrebuiltEtc) ExtraImageVariations(ctx android.BaseModuleContext) []string { + return nil +} + +func (p *PrebuiltEtc) SetImageVariation(ctx android.BaseModuleContext, variation string, module android.Module) { +} + +func (p *PrebuiltEtc) DepsMutator(ctx android.BottomUpMutatorContext) { + if p.properties.Src == nil { + ctx.PropertyErrorf("src", "missing prebuilt source file") + } +} + +func (p *PrebuiltEtc) SourceFilePath(ctx android.ModuleContext) android.Path { + return android.PathForModuleSrc(ctx, android.String(p.properties.Src)) +} + +func (p *PrebuiltEtc) InstallDirPath() android.InstallPath { + return p.installDirPath +} + +// This allows other derivative modules (e.g. prebuilt_etc_xml) to perform +// additional steps (like validating the src) before the file is installed. +func (p *PrebuiltEtc) SetAdditionalDependencies(paths android.Paths) { + p.additionalDependencies = &paths +} + +func (p *PrebuiltEtc) OutputFile() android.OutputPath { + return p.outputFilePath +} + +func (p *PrebuiltEtc) SubDir() string { + return android.String(p.properties.Sub_dir) +} + +func (p *PrebuiltEtc) Installable() bool { + return p.properties.Installable == nil || android.Bool(p.properties.Installable) +} + +func (p *PrebuiltEtc) GenerateAndroidBuildActions(ctx android.ModuleContext) { + p.sourceFilePath = android.PathForModuleSrc(ctx, android.String(p.properties.Src)) + filename := android.String(p.properties.Filename) + filename_from_src := android.Bool(p.properties.Filename_from_src) + if filename == "" { + if filename_from_src { + filename = p.sourceFilePath.Base() + } else { + filename = ctx.ModuleName() + } + } else if filename_from_src { + ctx.PropertyErrorf("filename_from_src", "filename is set. filename_from_src can't be true") + return + } + p.outputFilePath = android.PathForModuleOut(ctx, filename).OutputPath + + // If soc install dir was specified and SOC specific is set, set the installDirPath to the specified + // socInstallDirBase. + installBaseDir := p.installDirBase + if ctx.SocSpecific() && p.socInstallDirBase != "" { + installBaseDir = p.socInstallDirBase + } + p.installDirPath = android.PathForModuleInstall(ctx, installBaseDir, proptools.String(p.properties.Sub_dir)) + + // This ensures that outputFilePath has the correct name for others to + // use, as the source file may have a different name. + ctx.Build(pctx, android.BuildParams{ + Rule: android.Cp, + Output: p.outputFilePath, + Input: p.sourceFilePath, + }) +} + +func (p *PrebuiltEtc) AndroidMkEntries() []android.AndroidMkEntries { + nameSuffix := "" + if p.inRamdisk() && !p.onlyInRamdisk() { + nameSuffix = ".ramdisk" + } + if p.inRecovery() && !p.onlyInRecovery() { + nameSuffix = ".recovery" + } + return []android.AndroidMkEntries{android.AndroidMkEntries{ + Class: "ETC", + SubName: nameSuffix, + OutputFile: android.OptionalPathForPath(p.outputFilePath), + ExtraEntries: []android.AndroidMkExtraEntriesFunc{ + func(entries *android.AndroidMkEntries) { + entries.SetString("LOCAL_MODULE_TAGS", "optional") + entries.SetString("LOCAL_MODULE_PATH", p.installDirPath.ToMakePath().String()) + entries.SetString("LOCAL_INSTALLED_MODULE_STEM", p.outputFilePath.Base()) + entries.SetString("LOCAL_UNINSTALLABLE_MODULE", strconv.FormatBool(!p.Installable())) + if p.additionalDependencies != nil { + for _, path := range *p.additionalDependencies { + entries.SetString("LOCAL_ADDITIONAL_DEPENDENCIES", path.String()) + } + } + }, + }, + }} +} + +func InitPrebuiltEtcModule(p *PrebuiltEtc, dirBase string) { + p.installDirBase = dirBase + p.AddProperties(&p.properties) +} + +// prebuilt_etc is for a prebuilt artifact that is installed in +// <partition>/etc/<sub_dir> directory. +func PrebuiltEtcFactory() android.Module { + module := &PrebuiltEtc{} + InitPrebuiltEtcModule(module, "etc") + // This module is device-only + android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibFirst) + return module +} + +// prebuilt_etc_host is for a host prebuilt artifact that is installed in +// $(HOST_OUT)/etc/<sub_dir> directory. +func PrebuiltEtcHostFactory() android.Module { + module := &PrebuiltEtc{} + InitPrebuiltEtcModule(module, "etc") + // This module is host-only + android.InitAndroidArchModule(module, android.HostSupported, android.MultilibCommon) + return module +} + +// prebuilt_usr_share is for a prebuilt artifact that is installed in +// <partition>/usr/share/<sub_dir> directory. +func PrebuiltUserShareFactory() android.Module { + module := &PrebuiltEtc{} + InitPrebuiltEtcModule(module, "usr/share") + // This module is device-only + android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibFirst) + return module +} + +// prebuild_usr_share_host is for a host prebuilt artifact that is installed in +// $(HOST_OUT)/usr/share/<sub_dir> directory. +func PrebuiltUserShareHostFactory() android.Module { + module := &PrebuiltEtc{} + InitPrebuiltEtcModule(module, "usr/share") + // This module is host-only + android.InitAndroidArchModule(module, android.HostSupported, android.MultilibCommon) + return module +} + +// prebuilt_font installs a font in <partition>/fonts directory. +func PrebuiltFontFactory() android.Module { + module := &PrebuiltEtc{} + InitPrebuiltEtcModule(module, "fonts") + // This module is device-only + android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibFirst) + return module +} + +// prebuilt_firmware installs a firmware file to <partition>/etc/firmware directory for system image. +// If soc_specific property is set to true, the firmware file is installed to the vendor <partition>/firmware +// directory for vendor image. +func PrebuiltFirmwareFactory() android.Module { + module := &PrebuiltEtc{} + module.socInstallDirBase = "firmware" + InitPrebuiltEtcModule(module, "etc/firmware") + // This module is device-only + android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibFirst) + return module +} diff --git a/etc/prebuilt_etc_test.go b/etc/prebuilt_etc_test.go new file mode 100644 index 000000000..e13cb3cb7 --- /dev/null +++ b/etc/prebuilt_etc_test.go @@ -0,0 +1,283 @@ +// Copyright 2018 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package etc + +import ( + "io/ioutil" + "os" + "path/filepath" + "reflect" + "testing" + + "android/soong/android" +) + +var buildDir string + +func setUp() { + var err error + buildDir, err = ioutil.TempDir("", "soong_etc_test") + if err != nil { + panic(err) + } +} + +func tearDown() { + os.RemoveAll(buildDir) +} + +func TestMain(m *testing.M) { + run := func() int { + setUp() + defer tearDown() + + return m.Run() + } + + os.Exit(run()) +} + +func testPrebuiltEtc(t *testing.T, bp string) (*android.TestContext, android.Config) { + fs := map[string][]byte{ + "foo.conf": nil, + "bar.conf": nil, + "baz.conf": nil, + } + + config := android.TestArchConfig(buildDir, nil, bp, fs) + + ctx := android.NewTestArchContext() + ctx.RegisterModuleType("prebuilt_etc", PrebuiltEtcFactory) + ctx.RegisterModuleType("prebuilt_etc_host", PrebuiltEtcHostFactory) + ctx.RegisterModuleType("prebuilt_usr_share", PrebuiltUserShareFactory) + ctx.RegisterModuleType("prebuilt_usr_share_host", PrebuiltUserShareHostFactory) + ctx.RegisterModuleType("prebuilt_font", PrebuiltFontFactory) + ctx.RegisterModuleType("prebuilt_firmware", PrebuiltFirmwareFactory) + ctx.Register(config) + _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) + android.FailIfErrored(t, errs) + _, errs = ctx.PrepareBuildActions(config) + android.FailIfErrored(t, errs) + + return ctx, config +} + +func TestPrebuiltEtcVariants(t *testing.T) { + ctx, _ := testPrebuiltEtc(t, ` + prebuilt_etc { + name: "foo.conf", + src: "foo.conf", + } + prebuilt_etc { + name: "bar.conf", + src: "bar.conf", + recovery_available: true, + } + prebuilt_etc { + name: "baz.conf", + src: "baz.conf", + recovery: true, + } + `) + + foo_variants := ctx.ModuleVariantsForTests("foo.conf") + if len(foo_variants) != 1 { + t.Errorf("expected 1, got %#v", foo_variants) + } + + bar_variants := ctx.ModuleVariantsForTests("bar.conf") + if len(bar_variants) != 2 { + t.Errorf("expected 2, got %#v", bar_variants) + } + + baz_variants := ctx.ModuleVariantsForTests("baz.conf") + if len(baz_variants) != 1 { + t.Errorf("expected 1, got %#v", bar_variants) + } +} + +func TestPrebuiltEtcOutputPath(t *testing.T) { + ctx, _ := testPrebuiltEtc(t, ` + prebuilt_etc { + name: "foo.conf", + src: "foo.conf", + filename: "foo.installed.conf", + } + `) + + p := ctx.ModuleForTests("foo.conf", "android_arm64_armv8-a").Module().(*PrebuiltEtc) + if p.outputFilePath.Base() != "foo.installed.conf" { + t.Errorf("expected foo.installed.conf, got %q", p.outputFilePath.Base()) + } +} + +func TestPrebuiltEtcGlob(t *testing.T) { + ctx, _ := testPrebuiltEtc(t, ` + prebuilt_etc { + name: "my_foo", + src: "foo.*", + } + prebuilt_etc { + name: "my_bar", + src: "bar.*", + filename_from_src: true, + } + `) + + p := ctx.ModuleForTests("my_foo", "android_arm64_armv8-a").Module().(*PrebuiltEtc) + if p.outputFilePath.Base() != "my_foo" { + t.Errorf("expected my_foo, got %q", p.outputFilePath.Base()) + } + + p = ctx.ModuleForTests("my_bar", "android_arm64_armv8-a").Module().(*PrebuiltEtc) + if p.outputFilePath.Base() != "bar.conf" { + t.Errorf("expected bar.conf, got %q", p.outputFilePath.Base()) + } +} + +func TestPrebuiltEtcAndroidMk(t *testing.T) { + ctx, config := testPrebuiltEtc(t, ` + prebuilt_etc { + name: "foo", + src: "foo.conf", + owner: "abc", + filename_from_src: true, + required: ["modA", "moduleB"], + host_required: ["hostModA", "hostModB"], + target_required: ["targetModA"], + } + `) + + expected := map[string][]string{ + "LOCAL_MODULE": {"foo"}, + "LOCAL_MODULE_CLASS": {"ETC"}, + "LOCAL_MODULE_OWNER": {"abc"}, + "LOCAL_INSTALLED_MODULE_STEM": {"foo.conf"}, + "LOCAL_REQUIRED_MODULES": {"modA", "moduleB"}, + "LOCAL_HOST_REQUIRED_MODULES": {"hostModA", "hostModB"}, + "LOCAL_TARGET_REQUIRED_MODULES": {"targetModA"}, + } + + mod := ctx.ModuleForTests("foo", "android_arm64_armv8-a").Module().(*PrebuiltEtc) + entries := android.AndroidMkEntriesForTest(t, config, "", mod)[0] + for k, expectedValue := range expected { + if value, ok := entries.EntryMap[k]; ok { + if !reflect.DeepEqual(value, expectedValue) { + t.Errorf("Incorrect %s '%s', expected '%s'", k, value, expectedValue) + } + } else { + t.Errorf("No %s defined, saw %q", k, entries.EntryMap) + } + } +} + +func TestPrebuiltEtcHost(t *testing.T) { + ctx, _ := testPrebuiltEtc(t, ` + prebuilt_etc_host { + name: "foo.conf", + src: "foo.conf", + } + `) + + buildOS := android.BuildOs.String() + p := ctx.ModuleForTests("foo.conf", buildOS+"_common").Module().(*PrebuiltEtc) + if !p.Host() { + t.Errorf("host bit is not set for a prebuilt_etc_host module.") + } +} + +func TestPrebuiltUserShareInstallDirPath(t *testing.T) { + ctx, _ := testPrebuiltEtc(t, ` + prebuilt_usr_share { + name: "foo.conf", + src: "foo.conf", + sub_dir: "bar", + } + `) + + p := ctx.ModuleForTests("foo.conf", "android_arm64_armv8-a").Module().(*PrebuiltEtc) + expected := buildDir + "/target/product/test_device/system/usr/share/bar" + if p.installDirPath.String() != expected { + t.Errorf("expected %q, got %q", expected, p.installDirPath.String()) + } +} + +func TestPrebuiltUserShareHostInstallDirPath(t *testing.T) { + ctx, config := testPrebuiltEtc(t, ` + prebuilt_usr_share_host { + name: "foo.conf", + src: "foo.conf", + sub_dir: "bar", + } + `) + + buildOS := android.BuildOs.String() + p := ctx.ModuleForTests("foo.conf", buildOS+"_common").Module().(*PrebuiltEtc) + expected := filepath.Join(buildDir, "host", config.PrebuiltOS(), "usr", "share", "bar") + if p.installDirPath.String() != expected { + t.Errorf("expected %q, got %q", expected, p.installDirPath.String()) + } +} + +func TestPrebuiltFontInstallDirPath(t *testing.T) { + ctx, _ := testPrebuiltEtc(t, ` + prebuilt_font { + name: "foo.conf", + src: "foo.conf", + } + `) + + p := ctx.ModuleForTests("foo.conf", "android_arm64_armv8-a").Module().(*PrebuiltEtc) + expected := buildDir + "/target/product/test_device/system/fonts" + if p.installDirPath.String() != expected { + t.Errorf("expected %q, got %q", expected, p.installDirPath.String()) + } +} + +func TestPrebuiltFirmwareDirPath(t *testing.T) { + targetPath := buildDir + "/target/product/test_device" + tests := []struct { + description string + config string + expectedPath string + }{{ + description: "prebuilt: system firmware", + config: ` + prebuilt_firmware { + name: "foo.conf", + src: "foo.conf", + }`, + expectedPath: filepath.Join(targetPath, "system/etc/firmware"), + }, { + description: "prebuilt: vendor firmware", + config: ` + prebuilt_firmware { + name: "foo.conf", + src: "foo.conf", + soc_specific: true, + sub_dir: "sub_dir", + }`, + expectedPath: filepath.Join(targetPath, "vendor/firmware/sub_dir"), + }} + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + ctx, _ := testPrebuiltEtc(t, tt.config) + p := ctx.ModuleForTests("foo.conf", "android_arm64_armv8-a").Module().(*PrebuiltEtc) + if p.installDirPath.String() != tt.expectedPath { + t.Errorf("expected %q, got %q", tt.expectedPath, p.installDirPath) + } + }) + } +} diff --git a/finder/finder_test.go b/finder/finder_test.go index 29711fc2a..f6d0aa9e8 100644 --- a/finder/finder_test.go +++ b/finder/finder_test.go @@ -891,8 +891,8 @@ func TestFileAdded(t *testing.T) { IncludeFiles: []string{"findme.txt"}, }, ) - foundPaths := finder.FindNamedAt("/tmp", "findme.txt") filesystem.Clock.Tick() + foundPaths := finder.FindNamedAt("/tmp", "findme.txt") finder.Shutdown() // check the response of the first finder assertSameResponse(t, foundPaths, []string{"/tmp/a/findme.txt"}) @@ -1522,8 +1522,8 @@ func TestUpdatingDbIffChanged(t *testing.T) { IncludeFiles: []string{"hi.txt"}, }, ) - foundPaths := finder.FindAll() filesystem.Clock.Tick() + foundPaths := finder.FindAll() finder.Shutdown() // check results assertSameResponse(t, foundPaths, []string{"/tmp/a/hi.txt"}) @@ -1583,8 +1583,8 @@ func TestDirectoryNotPermitted(t *testing.T) { IncludeFiles: []string{"hi.txt"}, }, ) - foundPaths := finder.FindAll() filesystem.Clock.Tick() + foundPaths := finder.FindAll() finder.Shutdown() allPaths := []string{"/tmp/hi.txt", "/tmp/a/hi.txt", "/tmp/a/a/hi.txt", "/tmp/b/hi.txt"} // check results @@ -1629,8 +1629,8 @@ func TestFileNotPermitted(t *testing.T) { IncludeFiles: []string{"hi.txt"}, }, ) - foundPaths := finder.FindAll() filesystem.Clock.Tick() + foundPaths := finder.FindAll() finder.Shutdown() // check results assertSameResponse(t, foundPaths, []string{"/tmp/hi.txt"}) @@ -1650,8 +1650,8 @@ func TestCacheEntryPathUnexpectedError(t *testing.T) { IncludeFiles: []string{"hi.txt"}, }, ) - foundPaths := finder.FindAll() filesystem.Clock.Tick() + foundPaths := finder.FindAll() finder.Shutdown() // check results assertSameResponse(t, foundPaths, []string{"/tmp/a/hi.txt"}) diff --git a/genrule/Android.bp b/genrule/Android.bp new file mode 100644 index 000000000..ff543a61f --- /dev/null +++ b/genrule/Android.bp @@ -0,0 +1,18 @@ +bootstrap_go_package { + name: "soong-genrule", + pkgPath: "android/soong/genrule", + deps: [ + "blueprint", + "blueprint-pathtools", + "soong", + "soong-android", + "soong-shared", + ], + srcs: [ + "genrule.go", + ], + testSrcs: [ + "genrule_test.go", + ], + pluginFor: ["soong_build"], +} diff --git a/genrule/genrule.go b/genrule/genrule.go index 55c7e628e..fe877fe8d 100644 --- a/genrule/genrule.go +++ b/genrule/genrule.go @@ -26,14 +26,23 @@ import ( "android/soong/android" "android/soong/shared" + "crypto/sha256" "path/filepath" ) func init() { - android.RegisterModuleType("genrule_defaults", defaultsFactory) + registerGenruleBuildComponents(android.InitRegistrationContext) +} + +func registerGenruleBuildComponents(ctx android.RegistrationContext) { + ctx.RegisterModuleType("genrule_defaults", defaultsFactory) + + ctx.RegisterModuleType("gensrcs", GenSrcsFactory) + ctx.RegisterModuleType("genrule", GenRuleFactory) - android.RegisterModuleType("gensrcs", GenSrcsFactory) - android.RegisterModuleType("genrule", GenRuleFactory) + ctx.FinalDepsMutators(func(ctx android.RegisterMutatorsContext) { + ctx.BottomUp("genrule_tool_deps", toolDepsMutator).Parallel() + }) } var ( @@ -48,6 +57,7 @@ var ( ) func init() { + pctx.Import("android/soong/android") pctx.HostBinToolVariable("sboxCmd", "sbox") pctx.HostBinToolVariable("soongZip", "soong_zip") @@ -112,10 +122,12 @@ type generatorProperties struct { type Module struct { android.ModuleBase android.DefaultableModuleBase + android.ApexModuleBase // For other packages to make their own genrules with extra // properties Extra interface{} + android.ImageInterface properties generatorProperties @@ -163,16 +175,14 @@ func (g *Module) GeneratedDeps() android.Paths { return g.outputDeps } -func (g *Module) DepsMutator(ctx android.BottomUpMutatorContext) { +func toolDepsMutator(ctx android.BottomUpMutatorContext) { if g, ok := ctx.Module().(*Module); ok { for _, tool := range g.properties.Tools { tag := hostToolDependencyTag{label: tool} if m := android.SrcIsModule(tool); m != "" { tool = m } - ctx.AddFarVariationDependencies([]blueprint.Variation{ - {Mutator: "arch", Variation: ctx.Config().BuildOsVariant}, - }, tag, tool) + ctx.AddFarVariationDependencies(ctx.Config().BuildOSTarget.Variations(), tag, tool) } } } @@ -300,14 +310,15 @@ func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) { addLocationLabel(out.Rel(), []string{filepath.Join("__SBOX_OUT_DIR__", out.Rel())}) } + referencedIn := false referencedDepfile := false - rawCommand, err := android.Expand(task.cmd, func(name string) (string, error) { + rawCommand, err := android.ExpandNinjaEscaped(task.cmd, func(name string) (string, bool, error) { // report the error directly without returning an error to android.Expand to catch multiple errors in a // single run - reportError := func(fmt string, args ...interface{}) (string, error) { + reportError := func(fmt string, args ...interface{}) (string, bool, error) { ctx.PropertyErrorf("cmd", fmt, args...) - return "SOONG_ERROR", nil + return "SOONG_ERROR", false, nil } switch name { @@ -322,19 +333,20 @@ func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) { return reportError("default label %q has multiple files, use $(locations %s) to reference it", firstLabel, firstLabel) } - return locationLabels[firstLabel][0], nil + return locationLabels[firstLabel][0], false, nil case "in": - return "${in}", nil + referencedIn = true + return "${in}", true, nil case "out": - return "__SBOX_OUT_FILES__", nil + return "__SBOX_OUT_FILES__", false, nil case "depfile": referencedDepfile = true if !Bool(g.properties.Depfile) { return reportError("$(depfile) used without depfile property") } - return "__SBOX_DEPFILE__", nil + return "__SBOX_DEPFILE__", false, nil case "genDir": - return "__SBOX_OUT_DIR__", nil + return "__SBOX_OUT_DIR__", false, nil default: if strings.HasPrefix(name, "location ") { label := strings.TrimSpace(strings.TrimPrefix(name, "location ")) @@ -342,10 +354,10 @@ func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) { if len(paths) == 0 { return reportError("label %q has no files", label) } else if len(paths) > 1 { - return reportError("label %q has multiple files, use $(locations %s) to reference it", + return reportError("label %q has multiple files, use $(locations %s) to reference it", label, label) } - return paths[0], nil + return paths[0], false, nil } else { return reportError("unknown location label %q", label) } @@ -355,7 +367,7 @@ func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) { if len(paths) == 0 { return reportError("label %q has no files", label) } - return strings.Join(paths, " "), nil + return strings.Join(paths, " "), false, nil } else { return reportError("unknown locations label %q", label) } @@ -389,8 +401,16 @@ func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) { // Escape the command for the shell rawCommand = "'" + strings.Replace(rawCommand, "'", `'\''`, -1) + "'" g.rawCommands = append(g.rawCommands, rawCommand) - sandboxCommand := fmt.Sprintf("rm -rf %s && $sboxCmd --sandbox-path %s --output-root %s -c %s %s $allouts", - task.genDir, sandboxPath, task.genDir, rawCommand, depfilePlaceholder) + + sandboxCommand := fmt.Sprintf("rm -rf %s && $sboxCmd --sandbox-path %s --output-root %s", + task.genDir, sandboxPath, task.genDir) + + if !referencedIn { + sandboxCommand = sandboxCommand + hashSrcFiles(srcFiles) + } + + sandboxCommand = sandboxCommand + fmt.Sprintf(" -c %s %s $allouts", + rawCommand, depfilePlaceholder) ruleParams := blueprint.RuleParams{ Command: sandboxCommand, @@ -433,9 +453,33 @@ func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) { } g.outputFiles = outputFiles.Paths() - if len(g.outputFiles) > 0 { - g.outputDeps = append(g.outputDeps, g.outputFiles[0]) + + // For <= 6 outputs, just embed those directly in the users. Right now, that covers >90% of + // the genrules on AOSP. That will make things simpler to look at the graph in the common + // case. For larger sets of outputs, inject a phony target in between to limit ninja file + // growth. + if len(g.outputFiles) <= 6 { + g.outputDeps = g.outputFiles + } else { + phonyFile := android.PathForModuleGen(ctx, "genrule-phony") + + ctx.Build(pctx, android.BuildParams{ + Rule: blueprint.Phony, + Output: phonyFile, + Inputs: g.outputFiles, + }) + + g.outputDeps = android.Paths{phonyFile} + } + +} + +func hashSrcFiles(srcFiles android.Paths) string { + h := sha256.New() + for _, src := range srcFiles { + h.Write([]byte(src.String())) } + return fmt.Sprintf(" --input-hash %x", h.Sum(nil)) } func (g *Module) generateSourceFile(ctx android.ModuleContext, task generateTask, rule blueprint.Rule) { @@ -516,9 +560,21 @@ func generatorFactory(taskGenerator taskFunc, props ...interface{}) *Module { module.AddProperties(props...) module.AddProperties(&module.properties) + module.ImageInterface = noopImageInterface{} + return module } +type noopImageInterface struct{} + +func (x noopImageInterface) ImageMutatorBegin(android.BaseModuleContext) {} +func (x noopImageInterface) CoreVariantNeeded(android.BaseModuleContext) bool { return false } +func (x noopImageInterface) RamdiskVariantNeeded(android.BaseModuleContext) bool { return false } +func (x noopImageInterface) RecoveryVariantNeeded(android.BaseModuleContext) bool { return false } +func (x noopImageInterface) ExtraImageVariations(ctx android.BaseModuleContext) []string { return nil } +func (x noopImageInterface) SetImageVariation(ctx android.BaseModuleContext, variation string, module android.Module) { +} + // replace "out" with "__SBOX_OUT_DIR__/<the value of ${out}>" func pathToSandboxOut(path android.Path, genDir android.Path) string { relOut, err := filepath.Rel(genDir.String(), path.String()) @@ -670,9 +726,6 @@ type Defaults struct { android.DefaultsModuleBase } -func (*Defaults) GenerateAndroidBuildActions(ctx android.ModuleContext) { -} - func defaultsFactory() android.Module { return DefaultsFactory() } diff --git a/genrule/genrule_test.go b/genrule/genrule_test.go index e8dc3b55b..4b36600a6 100644 --- a/genrule/genrule_test.go +++ b/genrule/genrule_test.go @@ -51,18 +51,21 @@ func TestMain(m *testing.M) { os.Exit(run()) } -func testContext(config android.Config, bp string, - fs map[string][]byte) *android.TestContext { +func testContext(config android.Config) *android.TestContext { ctx := android.NewTestArchContext() - ctx.RegisterModuleType("filegroup", android.ModuleFactoryAdaptor(android.FileGroupFactory)) - ctx.RegisterModuleType("genrule", android.ModuleFactoryAdaptor(GenRuleFactory)) - ctx.RegisterModuleType("gensrcs", android.ModuleFactoryAdaptor(GenSrcsFactory)) - ctx.RegisterModuleType("genrule_defaults", android.ModuleFactoryAdaptor(defaultsFactory)) - ctx.RegisterModuleType("tool", android.ModuleFactoryAdaptor(toolFactory)) + ctx.RegisterModuleType("filegroup", android.FileGroupFactory) + ctx.RegisterModuleType("tool", toolFactory) + + registerGenruleBuildComponents(ctx) + ctx.PreArchMutators(android.RegisterDefaultsPreArchMutators) - ctx.Register() + ctx.Register(config) + return ctx +} + +func testConfig(bp string, fs map[string][]byte) android.Config { bp += ` tool { name: "tool", @@ -104,7 +107,6 @@ func testContext(config android.Config, bp string, ` mockFS := map[string][]byte{ - "Android.bp": []byte(bp), "tool": nil, "tool_file1": nil, "tool_file2": nil, @@ -119,9 +121,7 @@ func testContext(config android.Config, bp string, mockFS[k] = v } - ctx.MockFileSystem(mockFS) - - return ctx + return android.TestArchConfig(buildDir, nil, bp, mockFS) } func TestGenruleCmd(t *testing.T) { @@ -461,15 +461,15 @@ func TestGenruleCmd(t *testing.T) { for _, test := range testcases { t.Run(test.name, func(t *testing.T) { - config := android.TestArchConfig(buildDir, nil) bp := "genrule {\n" bp += "name: \"gen\",\n" bp += test.prop bp += "}\n" + config := testConfig(bp, nil) config.TestProductVariables.Allow_missing_dependencies = proptools.BoolPtr(test.allowMissingDependencies) - ctx := testContext(config, bp, nil) + ctx := testContext(config) ctx.SetAllowMissingDependencies(test.allowMissingDependencies) _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) @@ -502,6 +502,93 @@ func TestGenruleCmd(t *testing.T) { } } +func TestGenruleHashInputs(t *testing.T) { + + // The basic idea here is to verify that the sbox command (which is + // in the Command field of the generate rule) contains a hash of the + // inputs, but only if $(in) is not referenced in the genrule cmd + // property. + + // By including a hash of the inputs, we cause the rule to re-run if + // the list of inputs changes (because the sbox command changes). + + // However, if the genrule cmd property already contains $(in), then + // the dependency is already expressed, so we don't need to include the + // hash in that case. + + bp := ` + genrule { + name: "hash0", + srcs: ["in1.txt", "in2.txt"], + out: ["out"], + cmd: "echo foo > $(out)", + } + genrule { + name: "hash1", + srcs: ["*.txt"], + out: ["out"], + cmd: "echo bar > $(out)", + } + genrule { + name: "hash2", + srcs: ["*.txt"], + out: ["out"], + cmd: "echo $(in) > $(out)", + } + ` + testcases := []struct { + name string + expectedHash string + }{ + { + name: "hash0", + // sha256 value obtained from: echo -n 'in1.txtin2.txt' | sha256sum + expectedHash: "031097e11e0a8c822c960eb9742474f46336360a515744000d086d94335a9cb9", + }, + { + name: "hash1", + // sha256 value obtained from: echo -n 'in1.txtin2.txtin3.txt' | sha256sum + expectedHash: "de5d22a4a7ab50d250cc59fcdf7a7e0775790d270bfca3a7a9e1f18a70dd996c", + }, + { + name: "hash2", + // $(in) is present, option should not appear + expectedHash: "", + }, + } + + config := testConfig(bp, nil) + ctx := testContext(config) + _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) + if errs == nil { + _, errs = ctx.PrepareBuildActions(config) + } + if errs != nil { + t.Fatal(errs) + } + + for _, test := range testcases { + t.Run(test.name, func(t *testing.T) { + gen := ctx.ModuleForTests(test.name, "") + command := gen.Rule("generator").RuleParams.Command + + if len(test.expectedHash) > 0 { + // We add spaces before and after to make sure that + // this option doesn't abutt another sbox option. + expectedInputHashOption := " --input-hash " + test.expectedHash + " " + + if !strings.Contains(command, expectedInputHashOption) { + t.Errorf("Expected command \"%s\" to contain \"%s\"", command, expectedInputHashOption) + } + } else { + if strings.Contains(command, "--input-hash") { + t.Errorf("Unexpected \"--input-hash\" found in command: \"%s\"", command) + } + } + }) + } +} + func TestGenSrcs(t *testing.T) { testcases := []struct { name string @@ -524,7 +611,7 @@ func TestGenSrcs(t *testing.T) { cmds: []string{ "'bash -c '\\''out/tool in1.txt > __SBOX_OUT_DIR__/in1.h'\\'' && bash -c '\\''out/tool in2.txt > __SBOX_OUT_DIR__/in2.h'\\'''", }, - deps: []string{buildDir + "/.intermediates/gen/gen/gensrcs/in1.h"}, + deps: []string{buildDir + "/.intermediates/gen/gen/gensrcs/in1.h", buildDir + "/.intermediates/gen/gen/gensrcs/in2.h"}, files: []string{buildDir + "/.intermediates/gen/gen/gensrcs/in1.h", buildDir + "/.intermediates/gen/gen/gensrcs/in2.h"}, }, { @@ -539,21 +626,21 @@ func TestGenSrcs(t *testing.T) { "'bash -c '\\''out/tool in1.txt > __SBOX_OUT_DIR__/in1.h'\\'' && bash -c '\\''out/tool in2.txt > __SBOX_OUT_DIR__/in2.h'\\'''", "'bash -c '\\''out/tool in3.txt > __SBOX_OUT_DIR__/in3.h'\\'''", }, - deps: []string{buildDir + "/.intermediates/gen/gen/gensrcs/in1.h"}, + deps: []string{buildDir + "/.intermediates/gen/gen/gensrcs/in1.h", buildDir + "/.intermediates/gen/gen/gensrcs/in2.h", buildDir + "/.intermediates/gen/gen/gensrcs/in3.h"}, files: []string{buildDir + "/.intermediates/gen/gen/gensrcs/in1.h", buildDir + "/.intermediates/gen/gen/gensrcs/in2.h", buildDir + "/.intermediates/gen/gen/gensrcs/in3.h"}, }, } for _, test := range testcases { t.Run(test.name, func(t *testing.T) { - config := android.TestArchConfig(buildDir, nil) bp := "gensrcs {\n" bp += `name: "gen",` + "\n" bp += `output_extension: "h",` + "\n" bp += test.prop bp += "}\n" - ctx := testContext(config, bp, nil) + config := testConfig(bp, nil) + ctx := testContext(config) _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) if errs == nil { @@ -595,7 +682,6 @@ func TestGenSrcs(t *testing.T) { } func TestGenruleDefaults(t *testing.T) { - config := android.TestArchConfig(buildDir, nil) bp := ` genrule_defaults { name: "gen_defaults1", @@ -613,7 +699,8 @@ func TestGenruleDefaults(t *testing.T) { defaults: ["gen_defaults1", "gen_defaults2"], } ` - ctx := testContext(config, bp, nil) + config := testConfig(bp, nil) + ctx := testContext(config) _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) if errs == nil { _, errs = ctx.PrepareBuildActions(config) @@ -7,3 +7,5 @@ require github.com/google/blueprint v0.0.0 replace github.com/golang/protobuf v0.0.0 => ../../external/golang-protobuf replace github.com/google/blueprint v0.0.0 => ../blueprint + +go 1.13 diff --git a/jar/Android.bp b/jar/Android.bp index 6c2e60e47..2563474af 100644 --- a/jar/Android.bp +++ b/jar/Android.bp @@ -18,8 +18,10 @@ bootstrap_go_package { srcs: [ "jar.go", ], + testSrcs: [ + "jar_test.go", + ], deps: [ "android-archive-zip", ], } - diff --git a/jar/jar.go b/jar/jar.go index fa0e69350..a8f06a4f3 100644 --- a/jar/jar.go +++ b/jar/jar.go @@ -17,9 +17,12 @@ package jar import ( "bytes" "fmt" + "io" "os" "strings" + "text/scanner" "time" + "unicode" "android/soong/third_party/zip" ) @@ -112,3 +115,111 @@ func manifestContents(contents []byte) ([]byte, error) { return finalBytes, nil } + +var javaIgnorableIdentifier = &unicode.RangeTable{ + R16: []unicode.Range16{ + {0x00, 0x08, 1}, + {0x0e, 0x1b, 1}, + {0x7f, 0x9f, 1}, + }, + LatinOffset: 3, +} + +func javaIdentRune(ch rune, i int) bool { + if unicode.IsLetter(ch) { + return true + } + if unicode.IsDigit(ch) && i > 0 { + return true + } + + if unicode.In(ch, + unicode.Nl, // letter number + unicode.Sc, // currency symbol + unicode.Pc, // connecting punctuation + ) { + return true + } + + if unicode.In(ch, + unicode.Cf, // format + unicode.Mc, // combining mark + unicode.Mn, // non-spacing mark + javaIgnorableIdentifier, + ) && i > 0 { + return true + } + + return false +} + +// JavaPackage parses the package out of a java source file by looking for the package statement, or the first valid +// non-package statement, in which case it returns an empty string for the package. +func JavaPackage(r io.Reader, src string) (string, error) { + var s scanner.Scanner + var sErr error + + s.Init(r) + s.Filename = src + s.Error = func(s *scanner.Scanner, msg string) { + sErr = fmt.Errorf("error parsing %q: %s", src, msg) + } + s.IsIdentRune = javaIdentRune + + tok := s.Scan() + if sErr != nil { + return "", sErr + } + if tok == scanner.Ident { + switch s.TokenText() { + case "package": + // Nothing + case "import": + // File has no package statement, first keyword is an import + return "", nil + case "class", "enum", "interface": + // File has no package statement, first keyword is a type declaration + return "", nil + case "public", "protected", "private", "abstract", "static", "final", "strictfp": + // File has no package statement, first keyword is a modifier + return "", nil + case "module", "open": + // File has no package statement, first keyword is a module declaration + return "", nil + default: + return "", fmt.Errorf(`expected first token of java file to be "package", got %q`, s.TokenText()) + } + } else if tok == '@' { + // File has no package statement, first token is an annotation + return "", nil + } else if tok == scanner.EOF { + // File no package statement, it has no non-whitespace non-comment tokens + return "", nil + } else { + return "", fmt.Errorf(`expected first token of java file to be "package", got %q`, s.TokenText()) + } + + var pkg string + for { + tok = s.Scan() + if sErr != nil { + return "", sErr + } + if tok != scanner.Ident { + return "", fmt.Errorf(`expected "package <package>;", got "package %s%s"`, pkg, s.TokenText()) + } + pkg += s.TokenText() + + tok = s.Scan() + if sErr != nil { + return "", sErr + } + if tok == ';' { + return pkg, nil + } else if tok == '.' { + pkg += "." + } else { + return "", fmt.Errorf(`expected "package <package>;", got "package %s%s"`, pkg, s.TokenText()) + } + } +} diff --git a/jar/jar_test.go b/jar/jar_test.go new file mode 100644 index 000000000..c92011e12 --- /dev/null +++ b/jar/jar_test.go @@ -0,0 +1,182 @@ +// Copyright 2017 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package jar + +import ( + "bytes" + "io" + "testing" +) + +func TestGetJavaPackage(t *testing.T) { + type args struct { + r io.Reader + src string + } + tests := []struct { + name string + in string + want string + wantErr bool + }{ + { + name: "simple", + in: "package foo.bar;", + want: "foo.bar", + }, + { + name: "comment", + in: "/* test */\npackage foo.bar;", + want: "foo.bar", + }, + { + name: "no package", + in: "import foo.bar;", + want: "", + }, + { + name: "missing semicolon error", + in: "package foo.bar", + wantErr: true, + }, + { + name: "parser error", + in: "/*", + wantErr: true, + }, + { + name: "parser ident error", + in: "package 0foo.bar;", + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + buf := bytes.NewBufferString(tt.in) + got, err := JavaPackage(buf, "<test>") + if (err != nil) != tt.wantErr { + t.Errorf("JavaPackage() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("JavaPackage() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_javaIdentRune(t *testing.T) { + // runes that should be valid anywhere in an identifier + validAnywhere := []rune{ + // letters, $, _ + 'a', + 'A', + '$', + '_', + + // assorted unicode + '𐐀', + '𐐨', + 'Dž', + 'ῼ', + 'ʰ', + 'ﾟ', + 'ƻ', + '㡢', + '￦', + '＿', + 'Ⅰ', + '𐍊', + } + + // runes that should be invalid as the first rune in an identifier, but valid anywhere else + validAfterFirst := []rune{ + // digits + '0', + + // assorted unicode + '᥍', + '𝟎', + 'ྂ', + '𝆀', + + // control characters + '\x00', + '\b', + '\u000e', + '\u001b', + '\u007f', + '\u009f', + '\u00ad', + 0xE007F, + + // zero width space + '\u200b', + } + + // runes that should never be valid in an identifier + invalid := []rune{ + ';', + 0x110000, + } + + validFirst := validAnywhere + invalidFirst := append(validAfterFirst, invalid...) + validPart := append(validAnywhere, validAfterFirst...) + invalidPart := invalid + + check := func(t *testing.T, ch rune, i int, want bool) { + t.Helper() + if got := javaIdentRune(ch, i); got != want { + t.Errorf("javaIdentRune() = %v, want %v", got, want) + } + } + + t.Run("first", func(t *testing.T) { + t.Run("valid", func(t *testing.T) { + for _, ch := range validFirst { + t.Run(string(ch), func(t *testing.T) { + check(t, ch, 0, true) + }) + } + }) + + t.Run("invalid", func(t *testing.T) { + for _, ch := range invalidFirst { + t.Run(string(ch), func(t *testing.T) { + check(t, ch, 0, false) + }) + } + }) + }) + + t.Run("part", func(t *testing.T) { + t.Run("valid", func(t *testing.T) { + for _, ch := range validPart { + t.Run(string(ch), func(t *testing.T) { + check(t, ch, 1, true) + }) + } + }) + + t.Run("invalid", func(t *testing.T) { + for _, ch := range invalidPart { + t.Run(string(ch), func(t *testing.T) { + check(t, ch, 1, false) + }) + } + }) + }) +} diff --git a/java/Android.bp b/java/Android.bp new file mode 100644 index 000000000..1fda7f71d --- /dev/null +++ b/java/Android.bp @@ -0,0 +1,67 @@ +bootstrap_go_package { + name: "soong-java", + pkgPath: "android/soong/java", + deps: [ + "blueprint", + "blueprint-pathtools", + "soong", + "soong-android", + "soong-cc", + "soong-dexpreopt", + "soong-genrule", + "soong-java-config", + "soong-remoteexec", + "soong-tradefed", + ], + srcs: [ + "aapt2.go", + "aar.go", + "android_manifest.go", + "android_resources.go", + "androidmk.go", + "app_builder.go", + "app.go", + "builder.go", + "device_host_converter.go", + "dex.go", + "dexpreopt.go", + "dexpreopt_bootjars.go", + "dexpreopt_config.go", + "droiddoc.go", + "gen.go", + "genrule.go", + "hiddenapi.go", + "hiddenapi_singleton.go", + "jacoco.go", + "java.go", + "jdeps.go", + "java_resources.go", + "kotlin.go", + "lint.go", + "platform_compat_config.go", + "plugin.go", + "prebuilt_apis.go", + "proto.go", + "robolectric.go", + "sdk.go", + "sdk_library.go", + "support_libraries.go", + "sysprop.go", + "system_modules.go", + "testing.go", + "tradefed.go", + ], + testSrcs: [ + "androidmk_test.go", + "app_test.go", + "device_host_converter_test.go", + "dexpreopt_test.go", + "dexpreopt_bootjars_test.go", + "java_test.go", + "jdeps_test.go", + "kotlin_test.go", + "plugin_test.go", + "sdk_test.go", + ], + pluginFor: ["soong_build"], +} diff --git a/java/OWNERS b/java/OWNERS index d68a5b0f3..16ef4d812 100644 --- a/java/OWNERS +++ b/java/OWNERS @@ -1 +1 @@ -per-file dexpreopt.go = ngeoffray@google.com,calin@google.com,mathieuc@google.com +per-file dexpreopt*.go = ngeoffray@google.com,calin@google.com,mathieuc@google.com diff --git a/java/aapt2.go b/java/aapt2.go index f21408f97..04e4de52c 100644 --- a/java/aapt2.go +++ b/java/aapt2.go @@ -55,12 +55,14 @@ func pathsToAapt2Paths(ctx android.ModuleContext, resPaths android.Paths) androi var aapt2CompileRule = pctx.AndroidStaticRule("aapt2Compile", blueprint.RuleParams{ - Command: `${config.Aapt2Cmd} compile -o $outDir $cFlags --legacy $in`, + Command: `${config.Aapt2Cmd} compile -o $outDir $cFlags $in`, CommandDeps: []string{"${config.Aapt2Cmd}"}, }, "outDir", "cFlags") -func aapt2Compile(ctx android.ModuleContext, dir android.Path, paths android.Paths) android.WritablePaths { +func aapt2Compile(ctx android.ModuleContext, dir android.Path, paths android.Paths, + flags []string) android.WritablePaths { + shards := android.ShardPaths(paths, AAPT2_SHARD_SIZE) ret := make(android.WritablePaths, 0, len(paths)) @@ -81,9 +83,7 @@ func aapt2Compile(ctx android.ModuleContext, dir android.Path, paths android.Pat Outputs: outPaths, Args: map[string]string{ "outDir": android.PathForModuleOut(ctx, "aapt2", dir.String()).String(), - // Always set --pseudo-localize, it will be stripped out later for release - // builds that don't want it. - "cFlags": "--pseudo-localize", + "cFlags": strings.Join(flags, " "), }, }) } @@ -94,42 +94,31 @@ func aapt2Compile(ctx android.ModuleContext, dir android.Path, paths android.Pat return ret } -func aapt2CompileDirs(ctx android.ModuleContext, flata android.WritablePath, dirs android.Paths, deps android.Paths) { - ctx.Build(pctx, android.BuildParams{ - Rule: aapt2CompileRule, - Description: "aapt2 compile dirs", - Implicits: deps, - Output: flata, - Args: map[string]string{ - "outDir": flata.String(), - // Always set --pseudo-localize, it will be stripped out later for release - // builds that don't want it. - "cFlags": "--pseudo-localize " + android.JoinWithPrefix(dirs.Strings(), "--dir "), - }, - }) -} - var aapt2CompileZipRule = pctx.AndroidStaticRule("aapt2CompileZip", blueprint.RuleParams{ - Command: `${config.ZipSyncCmd} -d $resZipDir $in && ` + - `${config.Aapt2Cmd} compile -o $out $cFlags --legacy --dir $resZipDir`, + Command: `${config.ZipSyncCmd} -d $resZipDir $zipSyncFlags $in && ` + + `${config.Aapt2Cmd} compile -o $out $cFlags --dir $resZipDir`, CommandDeps: []string{ "${config.Aapt2Cmd}", "${config.ZipSyncCmd}", }, - }, "cFlags", "resZipDir") + }, "cFlags", "resZipDir", "zipSyncFlags") + +func aapt2CompileZip(ctx android.ModuleContext, flata android.WritablePath, zip android.Path, zipPrefix string, + flags []string) { -func aapt2CompileZip(ctx android.ModuleContext, flata android.WritablePath, zip android.Path) { + if zipPrefix != "" { + zipPrefix = "--zip-prefix " + zipPrefix + } ctx.Build(pctx, android.BuildParams{ Rule: aapt2CompileZipRule, Description: "aapt2 compile zip", Input: zip, Output: flata, Args: map[string]string{ - // Always set --pseudo-localize, it will be stripped out later for release - // builds that don't want it. - "cFlags": "--pseudo-localize", - "resZipDir": android.PathForModuleOut(ctx, "aapt2", "reszip", flata.Base()).String(), + "cFlags": strings.Join(flags, " "), + "resZipDir": android.PathForModuleOut(ctx, "aapt2", "reszip", flata.Base()).String(), + "zipSyncFlags": zipPrefix, }, }) } @@ -158,10 +147,16 @@ var fileListToFileRule = pctx.AndroidStaticRule("fileListToFile", RspfileContent: "$in", }) +var mergeAssetsRule = pctx.AndroidStaticRule("mergeAssets", + blueprint.RuleParams{ + Command: `${config.MergeZipsCmd} ${out} ${in}`, + CommandDeps: []string{"${config.MergeZipsCmd}"}, + }) + func aapt2Link(ctx android.ModuleContext, packageRes, genJar, proguardOptions, rTxt, extraPackages android.WritablePath, flags []string, deps android.Paths, - compiledRes, compiledOverlay android.Paths, splitPackages android.WritablePaths) { + compiledRes, compiledOverlay, assetPackages android.Paths, splitPackages android.WritablePaths) { genDir := android.PathForModuleGen(ctx, "aapt2", "R") @@ -197,12 +192,25 @@ func aapt2Link(ctx android.ModuleContext, } implicitOutputs := append(splitPackages, proguardOptions, genJar, rTxt, extraPackages) + linkOutput := packageRes + + // AAPT2 ignores assets in overlays. Merge them after linking. + if len(assetPackages) > 0 { + linkOutput = android.PathForModuleOut(ctx, "aapt2", "package-res.apk") + inputZips := append(android.Paths{linkOutput}, assetPackages...) + ctx.Build(pctx, android.BuildParams{ + Rule: mergeAssetsRule, + Inputs: inputZips, + Output: packageRes, + Description: "merge assets from dependencies", + }) + } ctx.Build(pctx, android.BuildParams{ Rule: aapt2LinkRule, Description: "aapt2 link", Implicits: deps, - Output: packageRes, + Output: linkOutput, ImplicitOutputs: implicitOutputs, Args: map[string]string{ "flags": strings.Join(flags, " "), diff --git a/java/aar.go b/java/aar.go index 6273a9b50..8dd752f12 100644 --- a/java/aar.go +++ b/java/aar.go @@ -15,11 +15,12 @@ package java import ( - "android/soong/android" "fmt" "path/filepath" "strings" + "android/soong/android" + "github.com/google/blueprint" "github.com/google/blueprint/proptools" ) @@ -30,12 +31,17 @@ type AndroidLibraryDependency interface { ExportedProguardFlagFiles() android.Paths ExportedRRODirs() []rroDir ExportedStaticPackages() android.Paths - ExportedManifest() android.Path + ExportedManifests() android.Paths + ExportedAssets() android.OptionalPath } func init() { - android.RegisterModuleType("android_library_import", AARImportFactory) - android.RegisterModuleType("android_library", AndroidLibraryFactory) + RegisterAARBuildComponents(android.InitRegistrationContext) +} + +func RegisterAARBuildComponents(ctx android.RegistrationContext) { + ctx.RegisterModuleType("android_library_import", AARImportFactory) + ctx.RegisterModuleType("android_library", AndroidLibraryFactory) } // @@ -69,21 +75,34 @@ type aaptProperties struct { // path to AndroidManifest.xml. If unset, defaults to "AndroidManifest.xml". Manifest *string `android:"path"` + + // paths to additional manifest files to merge with main manifest. + Additional_manifests []string `android:"path"` + + // do not include AndroidManifest from dependent libraries + Dont_merge_manifests *bool } type aapt struct { - aaptSrcJar android.Path - exportPackage android.Path - manifestPath android.Path - proguardOptionsFile android.Path - rroDirs []rroDir - rTxt android.Path - extraAaptPackagesFile android.Path - noticeFile android.OptionalPath - isLibrary bool - uncompressedJNI bool - useEmbeddedDex bool - usesNonSdkApis bool + aaptSrcJar android.Path + exportPackage android.Path + manifestPath android.Path + transitiveManifestPaths android.Paths + proguardOptionsFile android.Path + rroDirs []rroDir + rTxt android.Path + extraAaptPackagesFile android.Path + mergedManifestFile android.Path + noticeFile android.OptionalPath + assetPackage android.OptionalPath + isLibrary bool + useEmbeddedNativeLibs bool + useEmbeddedDex bool + usesNonSdkApis bool + sdkLibraries []string + hasNoCode bool + LoggingParent string + resourceFiles android.Paths splitNames []string splits []split @@ -105,24 +124,20 @@ func (a *aapt) ExportedRRODirs() []rroDir { return a.rroDirs } -func (a *aapt) ExportedManifest() android.Path { - return a.manifestPath +func (a *aapt) ExportedManifests() android.Paths { + return a.transitiveManifestPaths } -func (a *aapt) aapt2Flags(ctx android.ModuleContext, sdkContext sdkContext, manifestPath android.Path) (flags []string, - deps android.Paths, resDirs, overlayDirs []globbedResourceDir, rroDirs []rroDir, resZips android.Paths) { +func (a *aapt) ExportedAssets() android.OptionalPath { + return a.assetPackage +} - hasVersionCode := false - hasVersionName := false - for _, f := range a.aaptProperties.Aaptflags { - if strings.HasPrefix(f, "--version-code") { - hasVersionCode = true - } else if strings.HasPrefix(f, "--version-name") { - hasVersionName = true - } - } +func (a *aapt) aapt2Flags(ctx android.ModuleContext, sdkContext sdkContext, + manifestPath android.Path) (compileFlags, linkFlags []string, linkDeps android.Paths, + resDirs, overlayDirs []globbedResourceDir, rroDirs []rroDir, resZips android.Paths) { - var linkFlags []string + hasVersionCode := android.PrefixInList(a.aaptProperties.Aaptflags, "--version-code") + hasVersionName := android.PrefixInList(a.aaptProperties.Aaptflags, "--version-name") // Flags specified in Android.bp linkFlags = append(linkFlags, a.aaptProperties.Aaptflags...) @@ -134,8 +149,6 @@ func (a *aapt) aapt2Flags(ctx android.ModuleContext, sdkContext sdkContext, mani resourceDirs := android.PathsWithOptionalDefaultForModuleSrc(ctx, a.aaptProperties.Resource_dirs, "res") resourceZips := android.PathsForModuleSrc(ctx, a.aaptProperties.Resource_zips) - var linkDeps android.Paths - // Glob directories into lists of paths for _, dir := range resourceDirs { resDirs = append(resDirs, globbedResourceDir{ @@ -165,7 +178,10 @@ func (a *aapt) aapt2Flags(ctx android.ModuleContext, sdkContext sdkContext, mani linkDeps = append(linkDeps, assetFiles...) // SDK version flags - minSdkVersion := sdkVersionOrDefault(ctx, sdkContext.minSdkVersion()) + minSdkVersion, err := sdkContext.minSdkVersion().effectiveVersionString(ctx) + if err != nil { + ctx.ModuleErrorf("invalid minSdkVersion: %s", err) + } linkFlags = append(linkFlags, "--min-sdk-version "+minSdkVersion) linkFlags = append(linkFlags, "--target-sdk-version "+minSdkVersion) @@ -189,27 +205,58 @@ func (a *aapt) aapt2Flags(ctx android.ModuleContext, sdkContext sdkContext, mani linkFlags = append(linkFlags, "--version-name ", versionName) } - return linkFlags, linkDeps, resDirs, overlayDirs, rroDirs, resourceZips + linkFlags, compileFlags = android.FilterList(linkFlags, []string{"--legacy"}) + + // Always set --pseudo-localize, it will be stripped out later for release + // builds that don't want it. + compileFlags = append(compileFlags, "--pseudo-localize") + + return compileFlags, linkFlags, linkDeps, resDirs, overlayDirs, rroDirs, resourceZips } -func (a *aapt) deps(ctx android.BottomUpMutatorContext, sdkContext sdkContext) { - sdkDep := decodeSdkDep(ctx, sdkContext) +func (a *aapt) deps(ctx android.BottomUpMutatorContext, sdkDep sdkDep) { if sdkDep.frameworkResModule != "" { ctx.AddVariationDependencies(nil, frameworkResTag, sdkDep.frameworkResModule) } } +var extractAssetsRule = pctx.AndroidStaticRule("extractAssets", + blueprint.RuleParams{ + Command: `${config.Zip2ZipCmd} -i ${in} -o ${out} "assets/**/*"`, + CommandDeps: []string{"${config.Zip2ZipCmd}"}, + }) + func (a *aapt) buildActions(ctx android.ModuleContext, sdkContext sdkContext, extraLinkFlags ...string) { - transitiveStaticLibs, staticLibManifests, staticRRODirs, libDeps, libFlags := aaptLibs(ctx, sdkContext) + + transitiveStaticLibs, transitiveStaticLibManifests, staticRRODirs, assetPackages, libDeps, libFlags, sdkLibraries := + aaptLibs(ctx, sdkContext) // App manifest file manifestFile := proptools.StringDefault(a.aaptProperties.Manifest, "AndroidManifest.xml") manifestSrcPath := android.PathForModuleSrc(ctx, manifestFile) - manifestPath := manifestMerger(ctx, manifestSrcPath, sdkContext, staticLibManifests, a.isLibrary, - a.uncompressedJNI, a.useEmbeddedDex, a.usesNonSdkApis) + manifestPath := manifestFixer(ctx, manifestSrcPath, sdkContext, sdkLibraries, + a.isLibrary, a.useEmbeddedNativeLibs, a.usesNonSdkApis, a.useEmbeddedDex, a.hasNoCode, + a.LoggingParent) + + // Add additional manifest files to transitive manifests. + additionalManifests := android.PathsForModuleSrc(ctx, a.aaptProperties.Additional_manifests) + a.transitiveManifestPaths = append(android.Paths{manifestPath}, additionalManifests...) + a.transitiveManifestPaths = append(a.transitiveManifestPaths, transitiveStaticLibManifests...) + + if len(a.transitiveManifestPaths) > 1 && !Bool(a.aaptProperties.Dont_merge_manifests) { + a.mergedManifestFile = manifestMerger(ctx, a.transitiveManifestPaths[0], a.transitiveManifestPaths[1:], a.isLibrary) + if !a.isLibrary { + // Only use the merged manifest for applications. For libraries, the transitive closure of manifests + // will be propagated to the final application and merged there. The merged manifest for libraries is + // only passed to Make, which can't handle transitive dependencies. + manifestPath = a.mergedManifestFile + } + } else { + a.mergedManifestFile = manifestPath + } - linkFlags, linkDeps, resDirs, overlayDirs, rroDirs, resZips := a.aapt2Flags(ctx, sdkContext, manifestPath) + compileFlags, linkFlags, linkDeps, resDirs, overlayDirs, rroDirs, resZips := a.aapt2Flags(ctx, sdkContext, manifestPath) rroDirs = append(rroDirs, staticRRODirs...) linkFlags = append(linkFlags, libFlags...) @@ -220,7 +267,8 @@ func (a *aapt) buildActions(ctx android.ModuleContext, sdkContext sdkContext, ex } packageRes := android.PathForModuleOut(ctx, "package-res.apk") - srcJar := android.PathForModuleGen(ctx, "R.jar") + // the subdir "android" is required to be filtered by package names + srcJar := android.PathForModuleGen(ctx, "android", "R.srcjar") proguardOptionsFile := android.PathForModuleGen(ctx, "proguard.options") rTxt := android.PathForModuleOut(ctx, "R.txt") // This file isn't used by Soong, but is generated for exporting @@ -228,12 +276,13 @@ func (a *aapt) buildActions(ctx android.ModuleContext, sdkContext sdkContext, ex var compiledResDirs []android.Paths for _, dir := range resDirs { - compiledResDirs = append(compiledResDirs, aapt2Compile(ctx, dir.dir, dir.files).Paths()) + a.resourceFiles = append(a.resourceFiles, dir.files...) + compiledResDirs = append(compiledResDirs, aapt2Compile(ctx, dir.dir, dir.files, compileFlags).Paths()) } for i, zip := range resZips { flata := android.PathForModuleOut(ctx, fmt.Sprintf("reszip.%d.flata", i)) - aapt2CompileZip(ctx, flata, zip) + aapt2CompileZip(ctx, flata, zip, "", compileFlags) compiledResDirs = append(compiledResDirs, android.Paths{flata}) } @@ -262,7 +311,7 @@ func (a *aapt) buildActions(ctx android.ModuleContext, sdkContext sdkContext, ex } for _, dir := range overlayDirs { - compiledOverlay = append(compiledOverlay, aapt2Compile(ctx, dir.dir, dir.files).Paths()...) + compiledOverlay = append(compiledOverlay, aapt2Compile(ctx, dir.dir, dir.files, compileFlags).Paths()...) } var splitPackages android.WritablePaths @@ -281,7 +330,20 @@ func (a *aapt) buildActions(ctx android.ModuleContext, sdkContext sdkContext, ex } aapt2Link(ctx, packageRes, srcJar, proguardOptionsFile, rTxt, extraPackages, - linkFlags, linkDeps, compiledRes, compiledOverlay, splitPackages) + linkFlags, linkDeps, compiledRes, compiledOverlay, assetPackages, splitPackages) + + // Extract assets from the resource package output so that they can be used later in aapt2link + // for modules that depend on this one. + if android.PrefixInList(linkFlags, "-A ") || len(assetPackages) > 0 { + assets := android.PathForModuleOut(ctx, "assets.zip") + ctx.Build(pctx, android.BuildParams{ + Rule: extractAssetsRule, + Input: packageRes, + Output: assets, + Description: "extract assets from built resource file", + }) + a.assetPackage = android.OptionalPathForPath(assets) + } a.aaptSrcJar = srcJar a.exportPackage = packageRes @@ -294,8 +356,8 @@ func (a *aapt) buildActions(ctx android.ModuleContext, sdkContext sdkContext, ex } // aaptLibs collects libraries from dependencies and sdk_version and converts them into paths -func aaptLibs(ctx android.ModuleContext, sdkContext sdkContext) (transitiveStaticLibs, staticLibManifests android.Paths, - staticRRODirs []rroDir, deps android.Paths, flags []string) { +func aaptLibs(ctx android.ModuleContext, sdkContext sdkContext) (transitiveStaticLibs, transitiveStaticLibManifests android.Paths, + staticRRODirs []rroDir, assets, deps android.Paths, flags []string, sdkLibraries []string) { var sharedLibs android.Paths @@ -314,7 +376,19 @@ func aaptLibs(ctx android.ModuleContext, sdkContext sdkContext) (transitiveStati switch ctx.OtherModuleDependencyTag(module) { case instrumentationForTag: // Nothing, instrumentationForTag is treated as libTag for javac but not for aapt2. - case libTag, frameworkResTag: + case libTag: + if exportPackage != nil { + sharedLibs = append(sharedLibs, exportPackage) + } + + // If the module is (or possibly could be) a component of a java_sdk_library + // (including the java_sdk_library) itself then append any implicit sdk library + // names to the list of sdk libraries to be added to the manifest. + if component, ok := module.(SdkLibraryComponentDependency); ok { + sdkLibraries = append(sdkLibraries, component.OptionalImplicitSdkLibrary()...) + } + + case frameworkResTag: if exportPackage != nil { sharedLibs = append(sharedLibs, exportPackage) } @@ -322,7 +396,11 @@ func aaptLibs(ctx android.ModuleContext, sdkContext sdkContext) (transitiveStati if exportPackage != nil { transitiveStaticLibs = append(transitiveStaticLibs, aarDep.ExportedStaticPackages()...) transitiveStaticLibs = append(transitiveStaticLibs, exportPackage) - staticLibManifests = append(staticLibManifests, aarDep.ExportedManifest()) + transitiveStaticLibManifests = append(transitiveStaticLibManifests, aarDep.ExportedManifests()...) + sdkLibraries = append(sdkLibraries, aarDep.ExportedSdkLibs()...) + if aarDep.ExportedAssets().Valid() { + assets = append(assets, aarDep.ExportedAssets().Path()) + } outer: for _, d := range aarDep.ExportedRRODirs() { @@ -349,8 +427,10 @@ func aaptLibs(ctx android.ModuleContext, sdkContext sdkContext) (transitiveStati } transitiveStaticLibs = android.FirstUniquePaths(transitiveStaticLibs) + transitiveStaticLibManifests = android.FirstUniquePaths(transitiveStaticLibManifests) + sdkLibraries = android.FirstUniqueStrings(sdkLibraries) - return transitiveStaticLibs, staticLibManifests, staticRRODirs, deps, flags + return transitiveStaticLibs, transitiveStaticLibManifests, staticRRODirs, assets, deps, flags, sdkLibraries } type AndroidLibrary struct { @@ -377,13 +457,15 @@ var _ AndroidLibraryDependency = (*AndroidLibrary)(nil) func (a *AndroidLibrary) DepsMutator(ctx android.BottomUpMutatorContext) { a.Module.deps(ctx) - if !Bool(a.properties.No_framework_libs) && !Bool(a.properties.No_standard_libs) { - a.aapt.deps(ctx, sdkContext(a)) + sdkDep := decodeSdkDep(ctx, sdkContext(a)) + if sdkDep.hasFrameworkLibs() { + a.aapt.deps(ctx, sdkDep) } } func (a *AndroidLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) { a.aapt.isLibrary = true + a.aapt.sdkLibraries = a.exportedSdkLibs a.aapt.buildActions(ctx, sdkContext(a)) ctx.CheckbuildFile(a.proguardOptionsFile) @@ -393,6 +475,10 @@ func (a *AndroidLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) // apps manifests are handled by aapt, don't let Module see them a.properties.Manifest = nil + a.linter.mergedManifest = a.aapt.mergedManifestFile + a.linter.manifest = a.aapt.manifestPath + a.linter.resources = a.aapt.resourceFiles + a.Module.extraProguardFlagFiles = append(a.Module.extraProguardFlagFiles, a.proguardOptionsFile) @@ -426,16 +512,15 @@ func (a *AndroidLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) func AndroidLibraryFactory() android.Module { module := &AndroidLibrary{} + module.Module.addHostAndDeviceProperties() module.AddProperties( - &module.Module.properties, - &module.Module.deviceProperties, - &module.Module.dexpreoptProperties, - &module.Module.protoProperties, &module.aaptProperties, &module.androidLibraryProperties) module.androidLibraryProperties.BuildAAR = true + module.Module.linter.library = true + android.InitApexModule(module) InitJavaModule(module, android.DeviceSupported) return module } @@ -460,8 +545,12 @@ type AARImportProperties struct { type AARImport struct { android.ModuleBase android.DefaultableModuleBase + android.ApexModuleBase prebuilt android.Prebuilt + // Functionality common to Module and Import. + embeddableInModuleAndImport + properties AARImportProperties classpathFile android.WritablePath @@ -473,21 +562,29 @@ type AARImport struct { exportedStaticPackages android.Paths } -func (a *AARImport) sdkVersion() string { - return String(a.properties.Sdk_version) +func (a *AARImport) sdkVersion() sdkSpec { + return sdkSpecFrom(String(a.properties.Sdk_version)) } -func (a *AARImport) minSdkVersion() string { +func (a *AARImport) systemModules() string { + return "" +} + +func (a *AARImport) minSdkVersion() sdkSpec { if a.properties.Min_sdk_version != nil { - return *a.properties.Min_sdk_version + return sdkSpecFrom(*a.properties.Min_sdk_version) } return a.sdkVersion() } -func (a *AARImport) targetSdkVersion() string { +func (a *AARImport) targetSdkVersion() sdkSpec { return a.sdkVersion() } +func (a *AARImport) javaVersion() string { + return "" +} + var _ AndroidLibraryDependency = (*AARImport)(nil) func (a *AARImport) ExportPackage() android.Path { @@ -506,8 +603,13 @@ func (a *AARImport) ExportedStaticPackages() android.Paths { return a.exportedStaticPackages } -func (a *AARImport) ExportedManifest() android.Path { - return a.manifest +func (a *AARImport) ExportedManifests() android.Paths { + return android.Paths{a.manifest} +} + +// TODO(jungjw): Decide whether we want to implement this. +func (a *AARImport) ExportedAssets() android.OptionalPath { + return android.OptionalPath{} } func (a *AARImport) Prebuilt() *android.Prebuilt { @@ -518,6 +620,10 @@ func (a *AARImport) Name() string { return a.prebuilt.Name(a.ModuleBase.Name()) } +func (a *AARImport) JacocoReportClassesFile() android.Path { + return nil +} + func (a *AARImport) DepsMutator(ctx android.BottomUpMutatorContext) { if !ctx.Config().UnbundledBuildUsePrebuiltSdks() { sdkDep := decodeSdkDep(ctx, sdkContext(a)) @@ -531,13 +637,13 @@ func (a *AARImport) DepsMutator(ctx android.BottomUpMutatorContext) { } // Unzip an AAR into its constituent files and directories. Any files in Outputs that don't exist in the AAR will be -// touched to create an empty file, and any directories in $expectedDirs will be created. +// touched to create an empty file. The res directory is not extracted, as it will be extracted in its own rule. var unzipAAR = pctx.AndroidStaticRule("unzipAAR", blueprint.RuleParams{ - Command: `rm -rf $outDir && mkdir -p $outDir $expectedDirs && ` + - `unzip -qo -d $outDir $in && touch $out`, + Command: `rm -rf $outDir && mkdir -p $outDir && ` + + `unzip -qoDD -d $outDir $in && rm -rf $outDir/res && touch $out`, }, - "expectedDirs", "outDir") + "outDir") func (a *AARImport) GenerateAndroidBuildActions(ctx android.ModuleContext) { if len(a.properties.Aars) != 1 { @@ -555,7 +661,6 @@ func (a *AARImport) GenerateAndroidBuildActions(ctx android.ModuleContext) { } extractedAARDir := android.PathForModuleOut(ctx, "aar") - extractedResDir := extractedAARDir.Join(ctx, "res") a.classpathFile = extractedAARDir.Join(ctx, "classes.jar") a.proguardFlags = extractedAARDir.Join(ctx, "proguard.txt") a.manifest = extractedAARDir.Join(ctx, "AndroidManifest.xml") @@ -566,19 +671,20 @@ func (a *AARImport) GenerateAndroidBuildActions(ctx android.ModuleContext) { Outputs: android.WritablePaths{a.classpathFile, a.proguardFlags, a.manifest}, Description: "unzip AAR", Args: map[string]string{ - "expectedDirs": extractedResDir.String(), - "outDir": extractedAARDir.String(), + "outDir": extractedAARDir.String(), }, }) + // Always set --pseudo-localize, it will be stripped out later for release + // builds that don't want it. + compileFlags := []string{"--pseudo-localize"} compiledResDir := android.PathForModuleOut(ctx, "flat-res") - aaptCompileDeps := android.Paths{a.classpathFile} - aaptCompileDirs := android.Paths{extractedResDir} flata := compiledResDir.Join(ctx, "gen_res.flata") - aapt2CompileDirs(ctx, flata, aaptCompileDirs, aaptCompileDeps) + aapt2CompileZip(ctx, flata, aar, "res", compileFlags) a.exportPackage = android.PathForModuleOut(ctx, "package-res.apk") - srcJar := android.PathForModuleGen(ctx, "R.jar") + // the subdir "android" is required to be filtered by package names + srcJar := android.PathForModuleGen(ctx, "android", "R.srcjar") proguardOptionsFile := android.PathForModuleGen(ctx, "proguard.options") rTxt := android.PathForModuleOut(ctx, "R.txt") a.extraAaptPackagesFile = android.PathForModuleOut(ctx, "extra_packages") @@ -594,10 +700,12 @@ func (a *AARImport) GenerateAndroidBuildActions(ctx android.ModuleContext) { linkFlags = append(linkFlags, "--manifest "+a.manifest.String()) linkDeps = append(linkDeps, a.manifest) - transitiveStaticLibs, staticLibManifests, staticRRODirs, libDeps, libFlags := aaptLibs(ctx, sdkContext(a)) + transitiveStaticLibs, staticLibManifests, staticRRODirs, transitiveAssets, libDeps, libFlags, sdkLibraries := + aaptLibs(ctx, sdkContext(a)) _ = staticLibManifests _ = staticRRODirs + _ = sdkLibraries linkDeps = append(linkDeps, libDeps...) linkFlags = append(linkFlags, libFlags...) @@ -605,7 +713,7 @@ func (a *AARImport) GenerateAndroidBuildActions(ctx android.ModuleContext) { overlayRes := append(android.Paths{flata}, transitiveStaticLibs...) aapt2Link(ctx, a.exportPackage, srcJar, proguardOptionsFile, rTxt, a.extraAaptPackagesFile, - linkFlags, linkDeps, nil, overlayRes, nil) + linkFlags, linkDeps, nil, overlayRes, transitiveAssets, nil) } var _ Dependency = (*AARImport)(nil) @@ -638,6 +746,18 @@ func (a *AARImport) ExportedSdkLibs() []string { return nil } +func (d *AARImport) ExportedPlugins() (android.Paths, []string) { + return nil, nil +} + +func (a *AARImport) SrcJarArgs() ([]string, android.Paths) { + return nil, nil +} + +func (a *AARImport) DepIsInSameApex(ctx android.BaseModuleContext, dep android.Module) bool { + return a.depIsInSameApex(ctx, dep) +} + var _ android.PrebuiltInterface = (*Import)(nil) // android_library_import imports an `.aar` file into the build graph as if it was built with android_library. @@ -650,6 +770,7 @@ func AARImportFactory() android.Module { module.AddProperties(&module.properties) android.InitPrebuiltModule(module, &module.properties.Aars) + android.InitApexModule(module) InitJavaModule(module, android.DeviceSupported) return module } diff --git a/java/android_manifest.go b/java/android_manifest.go index 8dc3b4752..8280cb1b1 100644 --- a/java/android_manifest.go +++ b/java/android_manifest.go @@ -36,25 +36,36 @@ var manifestFixerRule = pctx.AndroidStaticRule("manifestFixer", var manifestMergerRule = pctx.AndroidStaticRule("manifestMerger", blueprint.RuleParams{ - Command: `${config.ManifestMergerCmd} --main $in $libs --out $out`, + Command: `${config.ManifestMergerCmd} $args --main $in $libs --out $out`, CommandDeps: []string{"${config.ManifestMergerCmd}"}, }, - "libs") + "args", "libs") -func manifestMerger(ctx android.ModuleContext, manifest android.Path, sdkContext sdkContext, - staticLibManifests android.Paths, isLibrary, uncompressedJNI, useEmbeddedDex, usesNonSdkApis bool) android.Path { +// These two libs are added as optional dependencies (<uses-library> with +// android:required set to false). This is because they haven't existed in pre-P +// devices, but classes in them were in bootclasspath jars, etc. So making them +// hard dependencies (android:required=true) would prevent apps from being +// installed to such legacy devices. +var optionalUsesLibs = []string{ + "android.test.base", + "android.test.mock", +} + +// Uses manifest_fixer.py to inject minSdkVersion, etc. into an AndroidManifest.xml +func manifestFixer(ctx android.ModuleContext, manifest android.Path, sdkContext sdkContext, sdkLibraries []string, + isLibrary, useEmbeddedNativeLibs, usesNonSdkApis, useEmbeddedDex, hasNoCode bool, loggingParent string) android.Path { var args []string if isLibrary { args = append(args, "--library") } else { - minSdkVersion, err := sdkVersionToNumber(ctx, sdkContext.minSdkVersion()) + minSdkVersion, err := sdkContext.minSdkVersion().effectiveVersion(ctx) if err != nil { ctx.ModuleErrorf("invalid minSdkVersion: %s", err) } if minSdkVersion >= 23 { - args = append(args, fmt.Sprintf("--extract-native-libs=%v", !uncompressedJNI)) - } else if uncompressedJNI { + args = append(args, fmt.Sprintf("--extract-native-libs=%v", !useEmbeddedNativeLibs)) + } else if useEmbeddedNativeLibs { ctx.ModuleErrorf("module attempted to store uncompressed native libraries, but minSdkVersion=%d doesn't support it", minSdkVersion) } @@ -65,49 +76,84 @@ func manifestMerger(ctx android.ModuleContext, manifest android.Path, sdkContext } if useEmbeddedDex { - args = append(args, "--use-embedded-dex=true") + args = append(args, "--use-embedded-dex") + } + + for _, usesLib := range sdkLibraries { + if inList(usesLib, optionalUsesLibs) { + args = append(args, "--optional-uses-library", usesLib) + } else { + args = append(args, "--uses-library", usesLib) + } } + if hasNoCode { + args = append(args, "--has-no-code") + } + + if loggingParent != "" { + args = append(args, "--logging-parent", loggingParent) + } var deps android.Paths - targetSdkVersion := sdkVersionOrDefault(ctx, sdkContext.targetSdkVersion()) - if targetSdkVersion == ctx.Config().PlatformSdkCodename() && - ctx.Config().UnbundledBuild() && - !ctx.Config().UnbundledBuildUsePrebuiltSdks() && - ctx.Config().IsEnvTrue("UNBUNDLED_BUILD_TARGET_SDK_WITH_API_FINGERPRINT") { - apiFingerprint := ApiFingerprintPath(ctx) - targetSdkVersion += fmt.Sprintf(".$$(cat %s)", apiFingerprint.String()) - deps = append(deps, apiFingerprint) + targetSdkVersion, err := sdkContext.targetSdkVersion().effectiveVersionString(ctx) + if err != nil { + ctx.ModuleErrorf("invalid targetSdkVersion: %s", err) + } + if UseApiFingerprint(ctx) { + targetSdkVersion = ctx.Config().PlatformSdkCodename() + fmt.Sprintf(".$$(cat %s)", ApiFingerprintPath(ctx).String()) + deps = append(deps, ApiFingerprintPath(ctx)) + } + + minSdkVersion, err := sdkContext.minSdkVersion().effectiveVersionString(ctx) + if err != nil { + ctx.ModuleErrorf("invalid minSdkVersion: %s", err) + } + if UseApiFingerprint(ctx) { + minSdkVersion = ctx.Config().PlatformSdkCodename() + fmt.Sprintf(".$$(cat %s)", ApiFingerprintPath(ctx).String()) + deps = append(deps, ApiFingerprintPath(ctx)) } - // Inject minSdkVersion into the manifest fixedManifest := android.PathForModuleOut(ctx, "manifest_fixer", "AndroidManifest.xml") + if err != nil { + ctx.ModuleErrorf("invalid minSdkVersion: %s", err) + } ctx.Build(pctx, android.BuildParams{ - Rule: manifestFixerRule, - Input: manifest, - Implicits: deps, - Output: fixedManifest, + Rule: manifestFixerRule, + Description: "fix manifest", + Input: manifest, + Implicits: deps, + Output: fixedManifest, Args: map[string]string{ - "minSdkVersion": sdkVersionOrDefault(ctx, sdkContext.minSdkVersion()), + "minSdkVersion": minSdkVersion, "targetSdkVersion": targetSdkVersion, "args": strings.Join(args, " "), }, }) - manifest = fixedManifest - - // Merge static aar dependency manifests if necessary - if len(staticLibManifests) > 0 { - mergedManifest := android.PathForModuleOut(ctx, "manifest_merger", "AndroidManifest.xml") - ctx.Build(pctx, android.BuildParams{ - Rule: manifestMergerRule, - Input: manifest, - Implicits: staticLibManifests, - Output: mergedManifest, - Args: map[string]string{ - "libs": android.JoinWithPrefix(staticLibManifests.Strings(), "--libs "), - }, - }) - manifest = mergedManifest + + return fixedManifest +} + +func manifestMerger(ctx android.ModuleContext, manifest android.Path, staticLibManifests android.Paths, + isLibrary bool) android.Path { + + var args string + if !isLibrary { + // Follow Gradle's behavior, only pass --remove-tools-declarations when merging app manifests. + args = "--remove-tools-declarations" } - return manifest + mergedManifest := android.PathForModuleOut(ctx, "manifest_merger", "AndroidManifest.xml") + ctx.Build(pctx, android.BuildParams{ + Rule: manifestMergerRule, + Description: "merge manifest", + Input: manifest, + Implicits: staticLibManifests, + Output: mergedManifest, + Args: map[string]string{ + "libs": android.JoinWithPrefix(staticLibManifests.Strings(), "--libs "), + "args": args, + }, + }) + + return mergedManifest } diff --git a/java/androidmk.go b/java/androidmk.go index 45fd1c1c0..ae257d7ad 100644 --- a/java/androidmk.go +++ b/java/androidmk.go @@ -17,265 +17,330 @@ package java import ( "fmt" "io" - "strings" "android/soong/android" ) -func (library *Library) AndroidMkHostDex(w io.Writer, name string, data android.AndroidMkData) { - if Bool(library.deviceProperties.Hostdex) && !library.Host() { - fmt.Fprintln(w, "include $(CLEAR_VARS)") - fmt.Fprintln(w, "LOCAL_MODULE := "+name+"-hostdex") - fmt.Fprintln(w, "LOCAL_IS_HOST_MODULE := true") - fmt.Fprintln(w, "LOCAL_MODULE_CLASS := JAVA_LIBRARIES") +func (library *Library) AndroidMkEntriesHostDex() android.AndroidMkEntries { + hostDexNeeded := Bool(library.deviceProperties.Hostdex) && !library.Host() + if !library.IsForPlatform() { + // Don't emit hostdex modules from the APEX variants + hostDexNeeded = false + } + + if hostDexNeeded { + var output android.Path if library.dexJarFile != nil { - fmt.Fprintln(w, "LOCAL_PREBUILT_MODULE_FILE :=", library.dexJarFile.String()) + output = library.dexJarFile } else { - fmt.Fprintln(w, "LOCAL_PREBUILT_MODULE_FILE :=", library.implementationAndResourcesJar.String()) + output = library.implementationAndResourcesJar } - if library.dexJarFile != nil { - fmt.Fprintln(w, "LOCAL_SOONG_DEX_JAR :=", library.dexJarFile.String()) - } - fmt.Fprintln(w, "LOCAL_SOONG_HEADER_JAR :=", library.headerJarFile.String()) - fmt.Fprintln(w, "LOCAL_SOONG_CLASSES_JAR :=", library.implementationAndResourcesJar.String()) - fmt.Fprintln(w, "LOCAL_REQUIRED_MODULES := "+strings.Join(data.Required, " ")) - if r := library.deviceProperties.Target.Hostdex.Required; len(r) > 0 { - fmt.Fprintln(w, "LOCAL_REQUIRED_MODULES +=", strings.Join(r, " ")) + return android.AndroidMkEntries{ + Class: "JAVA_LIBRARIES", + SubName: "-hostdex", + OutputFile: android.OptionalPathForPath(output), + Required: library.deviceProperties.Target.Hostdex.Required, + Include: "$(BUILD_SYSTEM)/soong_java_prebuilt.mk", + ExtraEntries: []android.AndroidMkExtraEntriesFunc{ + func(entries *android.AndroidMkEntries) { + entries.SetBool("LOCAL_IS_HOST_MODULE", true) + entries.SetPath("LOCAL_PREBUILT_MODULE_FILE", output) + if library.dexJarFile != nil { + entries.SetPath("LOCAL_SOONG_DEX_JAR", library.dexJarFile) + } + entries.SetPath("LOCAL_SOONG_HEADER_JAR", library.headerJarFile) + entries.SetPath("LOCAL_SOONG_CLASSES_JAR", library.implementationAndResourcesJar) + entries.SetString("LOCAL_MODULE_STEM", library.Stem()+"-hostdex") + }, + }, } - fmt.Fprintln(w, "include $(BUILD_SYSTEM)/soong_java_prebuilt.mk") } + return android.AndroidMkEntries{Disabled: true} } -func (library *Library) AndroidMk() android.AndroidMkData { - return android.AndroidMkData{ - Class: "JAVA_LIBRARIES", - OutputFile: android.OptionalPathForPath(library.outputFile), - Include: "$(BUILD_SYSTEM)/soong_java_prebuilt.mk", - Extra: []android.AndroidMkExtraFunc{ - func(w io.Writer, outputFile android.Path) { - if len(library.logtagsSrcs) > 0 { - var logtags []string - for _, l := range library.logtagsSrcs { - logtags = append(logtags, l.Rel()) +func (library *Library) AndroidMkEntries() []android.AndroidMkEntries { + var entriesList []android.AndroidMkEntries + + mainEntries := android.AndroidMkEntries{Disabled: true} + + // For a java library built for an APEX, we don't need Make module + hideFromMake := !library.IsForPlatform() + // If not available for platform, don't emit to make. + if !library.ApexModuleBase.AvailableFor(android.AvailableToPlatform) { + hideFromMake = true + } + if hideFromMake { + // May still need to add some additional dependencies. This will be called + // once for the platform variant (even if it is not being used) and once each + // for the APEX specific variants. In order to avoid adding the dependency + // multiple times only add it for the platform variant. + checkedModulePaths := library.additionalCheckedModules + if library.IsForPlatform() && len(checkedModulePaths) != 0 { + mainEntries = android.AndroidMkEntries{ + Class: "FAKE", + // Need at least one output file in order for this to take effect. + OutputFile: android.OptionalPathForPath(checkedModulePaths[0]), + Include: "$(BUILD_PHONY_PACKAGE)", + ExtraEntries: []android.AndroidMkExtraEntriesFunc{ + func(entries *android.AndroidMkEntries) { + entries.AddStrings("LOCAL_ADDITIONAL_CHECKED_MODULE", checkedModulePaths.Strings()...) + }, + }, + } + } + } else { + mainEntries = android.AndroidMkEntries{ + Class: "JAVA_LIBRARIES", + DistFile: android.OptionalPathForPath(library.distFile), + OutputFile: android.OptionalPathForPath(library.outputFile), + Include: "$(BUILD_SYSTEM)/soong_java_prebuilt.mk", + ExtraEntries: []android.AndroidMkExtraEntriesFunc{ + func(entries *android.AndroidMkEntries) { + if len(library.logtagsSrcs) > 0 { + var logtags []string + for _, l := range library.logtagsSrcs { + logtags = append(logtags, l.Rel()) + } + entries.AddStrings("LOCAL_LOGTAGS_FILES", logtags...) } - fmt.Fprintln(w, "LOCAL_LOGTAGS_FILES :=", strings.Join(logtags, " ")) - } - if library.installFile == nil { - fmt.Fprintln(w, "LOCAL_UNINSTALLABLE_MODULE := true") - } - if library.dexJarFile != nil { - fmt.Fprintln(w, "LOCAL_SOONG_DEX_JAR :=", library.dexJarFile.String()) - } - if len(library.dexpreopter.builtInstalled) > 0 { - fmt.Fprintln(w, "LOCAL_SOONG_BUILT_INSTALLED :=", library.dexpreopter.builtInstalled) - } - fmt.Fprintln(w, "LOCAL_SDK_VERSION :=", library.sdkVersion()) - fmt.Fprintln(w, "LOCAL_SOONG_CLASSES_JAR :=", library.implementationAndResourcesJar.String()) - fmt.Fprintln(w, "LOCAL_SOONG_HEADER_JAR :=", library.headerJarFile.String()) + if library.installFile == nil { + entries.SetBoolIfTrue("LOCAL_UNINSTALLABLE_MODULE", true) + } + if library.dexJarFile != nil { + entries.SetPath("LOCAL_SOONG_DEX_JAR", library.dexJarFile) + } + if len(library.dexpreopter.builtInstalled) > 0 { + entries.SetString("LOCAL_SOONG_BUILT_INSTALLED", library.dexpreopter.builtInstalled) + } + entries.SetString("LOCAL_SDK_VERSION", library.sdkVersion().raw) + entries.SetPath("LOCAL_SOONG_CLASSES_JAR", library.implementationAndResourcesJar) + entries.SetPath("LOCAL_SOONG_HEADER_JAR", library.headerJarFile) - if library.jacocoReportClassesFile != nil { - fmt.Fprintln(w, "LOCAL_SOONG_JACOCO_REPORT_CLASSES_JAR :=", library.jacocoReportClassesFile.String()) - } + if library.jacocoReportClassesFile != nil { + entries.SetPath("LOCAL_SOONG_JACOCO_REPORT_CLASSES_JAR", library.jacocoReportClassesFile) + } - if len(library.exportedSdkLibs) != 0 { - fmt.Fprintln(w, "LOCAL_EXPORT_SDK_LIBRARIES :=", strings.Join(library.exportedSdkLibs, " ")) - } + entries.AddStrings("LOCAL_EXPORT_SDK_LIBRARIES", library.exportedSdkLibs...) - if len(library.additionalCheckedModules) != 0 { - fmt.Fprintln(w, "LOCAL_ADDITIONAL_CHECKED_MODULE +=", strings.Join(library.additionalCheckedModules.Strings(), " ")) - } + if len(library.additionalCheckedModules) != 0 { + entries.AddStrings("LOCAL_ADDITIONAL_CHECKED_MODULE", library.additionalCheckedModules.Strings()...) + } - // Temporary hack: export sources used to compile framework.jar to Make - // to be used for droiddoc - // TODO(ccross): remove this once droiddoc is in soong - if (library.Name() == "framework") || (library.Name() == "framework-annotation-proc") { - fmt.Fprintln(w, "SOONG_FRAMEWORK_SRCS :=", strings.Join(library.compiledJavaSrcs.Strings(), " ")) - fmt.Fprintln(w, "SOONG_FRAMEWORK_SRCJARS :=", strings.Join(library.compiledSrcJars.Strings(), " ")) - } + if library.proguardDictionary != nil { + entries.SetPath("LOCAL_SOONG_PROGUARD_DICT", library.proguardDictionary) + } + entries.SetString("LOCAL_MODULE_STEM", library.Stem()) + + entries.SetOptionalPaths("LOCAL_SOONG_LINT_REPORTS", library.linter.reports) + }, }, - }, - Custom: func(w io.Writer, name, prefix, moduleDir string, data android.AndroidMkData) { - android.WriteAndroidMkData(w, data) - library.AndroidMkHostDex(w, name, data) - }, + } } + + hostDexEntries := library.AndroidMkEntriesHostDex() + + entriesList = append(entriesList, mainEntries, hostDexEntries) + return entriesList } // Called for modules that are a component of a test suite. -func testSuiteComponent(w io.Writer, test_suites []string) { - fmt.Fprintln(w, "LOCAL_MODULE_TAGS := tests") +func testSuiteComponent(entries *android.AndroidMkEntries, test_suites []string) { + entries.SetString("LOCAL_MODULE_TAGS", "tests") if len(test_suites) > 0 { - fmt.Fprintln(w, "LOCAL_COMPATIBILITY_SUITE :=", - strings.Join(test_suites, " ")) + entries.AddStrings("LOCAL_COMPATIBILITY_SUITE", test_suites...) } else { - fmt.Fprintln(w, "LOCAL_COMPATIBILITY_SUITE := null-suite") + entries.SetString("LOCAL_COMPATIBILITY_SUITE", "null-suite") } } -func (j *Test) AndroidMk() android.AndroidMkData { - data := j.Library.AndroidMk() - data.Extra = append(data.Extra, func(w io.Writer, outputFile android.Path) { - testSuiteComponent(w, j.testProperties.Test_suites) +func (j *Test) AndroidMkEntries() []android.AndroidMkEntries { + entriesList := j.Library.AndroidMkEntries() + entries := &entriesList[0] + entries.ExtraEntries = append(entries.ExtraEntries, func(entries *android.AndroidMkEntries) { + testSuiteComponent(entries, j.testProperties.Test_suites) if j.testConfig != nil { - fmt.Fprintln(w, "LOCAL_FULL_TEST_CONFIG :=", j.testConfig.String()) + entries.SetPath("LOCAL_FULL_TEST_CONFIG", j.testConfig) + } + androidMkWriteTestData(j.data, entries) + if !BoolDefault(j.testProperties.Auto_gen_config, true) { + entries.SetString("LOCAL_DISABLE_AUTO_GENERATE_TEST_CONFIG", "true") } }) - androidMkWriteTestData(j.data, &data) - - return data + return entriesList } -func (j *TestHelperLibrary) AndroidMk() android.AndroidMkData { - data := j.Library.AndroidMk() - data.Extra = append(data.Extra, func(w io.Writer, outputFile android.Path) { - testSuiteComponent(w, j.testHelperLibraryProperties.Test_suites) +func (j *TestHelperLibrary) AndroidMkEntries() []android.AndroidMkEntries { + entriesList := j.Library.AndroidMkEntries() + entries := &entriesList[0] + entries.ExtraEntries = append(entries.ExtraEntries, func(entries *android.AndroidMkEntries) { + testSuiteComponent(entries, j.testHelperLibraryProperties.Test_suites) }) - return data + return entriesList } -func (prebuilt *Import) AndroidMk() android.AndroidMkData { - return android.AndroidMkData{ +func (prebuilt *Import) AndroidMkEntries() []android.AndroidMkEntries { + if !prebuilt.IsForPlatform() || !prebuilt.ContainingSdk().Unversioned() { + return []android.AndroidMkEntries{android.AndroidMkEntries{ + Disabled: true, + }} + } + return []android.AndroidMkEntries{android.AndroidMkEntries{ Class: "JAVA_LIBRARIES", OutputFile: android.OptionalPathForPath(prebuilt.combinedClasspathFile), Include: "$(BUILD_SYSTEM)/soong_java_prebuilt.mk", - Extra: []android.AndroidMkExtraFunc{ - func(w io.Writer, outputFile android.Path) { - fmt.Fprintln(w, "LOCAL_UNINSTALLABLE_MODULE := ", !Bool(prebuilt.properties.Installable)) - fmt.Fprintln(w, "LOCAL_SOONG_HEADER_JAR :=", prebuilt.combinedClasspathFile.String()) - fmt.Fprintln(w, "LOCAL_SOONG_CLASSES_JAR :=", prebuilt.combinedClasspathFile.String()) - fmt.Fprintln(w, "LOCAL_SDK_VERSION :=", prebuilt.sdkVersion()) + ExtraEntries: []android.AndroidMkExtraEntriesFunc{ + func(entries *android.AndroidMkEntries) { + entries.SetBool("LOCAL_UNINSTALLABLE_MODULE", !Bool(prebuilt.properties.Installable)) + entries.SetPath("LOCAL_SOONG_HEADER_JAR", prebuilt.combinedClasspathFile) + entries.SetPath("LOCAL_SOONG_CLASSES_JAR", prebuilt.combinedClasspathFile) + entries.SetString("LOCAL_SDK_VERSION", prebuilt.sdkVersion().raw) + entries.SetString("LOCAL_MODULE_STEM", prebuilt.Stem()) }, }, - } + }} } -func (prebuilt *DexImport) AndroidMk() android.AndroidMkData { - return android.AndroidMkData{ +func (prebuilt *DexImport) AndroidMkEntries() []android.AndroidMkEntries { + if !prebuilt.IsForPlatform() { + return []android.AndroidMkEntries{android.AndroidMkEntries{ + Disabled: true, + }} + } + return []android.AndroidMkEntries{android.AndroidMkEntries{ Class: "JAVA_LIBRARIES", OutputFile: android.OptionalPathForPath(prebuilt.maybeStrippedDexJarFile), Include: "$(BUILD_SYSTEM)/soong_java_prebuilt.mk", - Extra: []android.AndroidMkExtraFunc{ - func(w io.Writer, outputFile android.Path) { + ExtraEntries: []android.AndroidMkExtraEntriesFunc{ + func(entries *android.AndroidMkEntries) { if prebuilt.dexJarFile != nil { - fmt.Fprintln(w, "LOCAL_SOONG_DEX_JAR :=", prebuilt.dexJarFile.String()) + entries.SetPath("LOCAL_SOONG_DEX_JAR", prebuilt.dexJarFile) // TODO(b/125517186): export the dex jar as a classes jar to match some mis-uses in Make until // boot_jars_package_check.mk can check dex jars. - fmt.Fprintln(w, "LOCAL_SOONG_HEADER_JAR :=", prebuilt.dexJarFile.String()) - fmt.Fprintln(w, "LOCAL_SOONG_CLASSES_JAR :=", prebuilt.dexJarFile.String()) + entries.SetPath("LOCAL_SOONG_HEADER_JAR", prebuilt.dexJarFile) + entries.SetPath("LOCAL_SOONG_CLASSES_JAR", prebuilt.dexJarFile) } if len(prebuilt.dexpreopter.builtInstalled) > 0 { - fmt.Fprintln(w, "LOCAL_SOONG_BUILT_INSTALLED :=", prebuilt.dexpreopter.builtInstalled) + entries.SetString("LOCAL_SOONG_BUILT_INSTALLED", prebuilt.dexpreopter.builtInstalled) } + entries.SetString("LOCAL_MODULE_STEM", prebuilt.Stem()) }, }, - } + }} } -func (prebuilt *AARImport) AndroidMk() android.AndroidMkData { - return android.AndroidMkData{ +func (prebuilt *AARImport) AndroidMkEntries() []android.AndroidMkEntries { + if !prebuilt.IsForPlatform() { + return []android.AndroidMkEntries{{ + Disabled: true, + }} + } + return []android.AndroidMkEntries{android.AndroidMkEntries{ Class: "JAVA_LIBRARIES", OutputFile: android.OptionalPathForPath(prebuilt.classpathFile), Include: "$(BUILD_SYSTEM)/soong_java_prebuilt.mk", - Extra: []android.AndroidMkExtraFunc{ - func(w io.Writer, outputFile android.Path) { - fmt.Fprintln(w, "LOCAL_UNINSTALLABLE_MODULE := true") - fmt.Fprintln(w, "LOCAL_SOONG_HEADER_JAR :=", prebuilt.classpathFile.String()) - fmt.Fprintln(w, "LOCAL_SOONG_CLASSES_JAR :=", prebuilt.classpathFile.String()) - fmt.Fprintln(w, "LOCAL_SOONG_RESOURCE_EXPORT_PACKAGE :=", prebuilt.exportPackage.String()) - fmt.Fprintln(w, "LOCAL_SOONG_EXPORT_PROGUARD_FLAGS :=", prebuilt.proguardFlags.String()) - fmt.Fprintln(w, "LOCAL_SOONG_STATIC_LIBRARY_EXTRA_PACKAGES :=", prebuilt.extraAaptPackagesFile.String()) - fmt.Fprintln(w, "LOCAL_FULL_MANIFEST_FILE :=", prebuilt.manifest.String()) - fmt.Fprintln(w, "LOCAL_SDK_VERSION :=", prebuilt.sdkVersion()) + ExtraEntries: []android.AndroidMkExtraEntriesFunc{ + func(entries *android.AndroidMkEntries) { + entries.SetBool("LOCAL_UNINSTALLABLE_MODULE", true) + entries.SetPath("LOCAL_SOONG_HEADER_JAR", prebuilt.classpathFile) + entries.SetPath("LOCAL_SOONG_CLASSES_JAR", prebuilt.classpathFile) + entries.SetPath("LOCAL_SOONG_RESOURCE_EXPORT_PACKAGE", prebuilt.exportPackage) + entries.SetPath("LOCAL_SOONG_EXPORT_PROGUARD_FLAGS", prebuilt.proguardFlags) + entries.SetPath("LOCAL_SOONG_STATIC_LIBRARY_EXTRA_PACKAGES", prebuilt.extraAaptPackagesFile) + entries.SetPath("LOCAL_FULL_MANIFEST_FILE", prebuilt.manifest) + entries.SetString("LOCAL_SDK_VERSION", prebuilt.sdkVersion().raw) }, }, - } + }} } -func (binary *Binary) AndroidMk() android.AndroidMkData { +func (binary *Binary) AndroidMkEntries() []android.AndroidMkEntries { if !binary.isWrapperVariant { - return android.AndroidMkData{ + return []android.AndroidMkEntries{android.AndroidMkEntries{ Class: "JAVA_LIBRARIES", OutputFile: android.OptionalPathForPath(binary.outputFile), Include: "$(BUILD_SYSTEM)/soong_java_prebuilt.mk", - Extra: []android.AndroidMkExtraFunc{ - func(w io.Writer, outputFile android.Path) { - fmt.Fprintln(w, "LOCAL_SOONG_HEADER_JAR :=", binary.headerJarFile.String()) - fmt.Fprintln(w, "LOCAL_SOONG_CLASSES_JAR :=", binary.implementationAndResourcesJar.String()) + ExtraEntries: []android.AndroidMkExtraEntriesFunc{ + func(entries *android.AndroidMkEntries) { + entries.SetPath("LOCAL_SOONG_HEADER_JAR", binary.headerJarFile) + entries.SetPath("LOCAL_SOONG_CLASSES_JAR", binary.implementationAndResourcesJar) if binary.dexJarFile != nil { - fmt.Fprintln(w, "LOCAL_SOONG_DEX_JAR :=", binary.dexJarFile.String()) + entries.SetPath("LOCAL_SOONG_DEX_JAR", binary.dexJarFile) } if len(binary.dexpreopter.builtInstalled) > 0 { - fmt.Fprintln(w, "LOCAL_SOONG_BUILT_INSTALLED :=", binary.dexpreopter.builtInstalled) + entries.SetString("LOCAL_SOONG_BUILT_INSTALLED", binary.dexpreopter.builtInstalled) } }, }, - Custom: func(w io.Writer, name, prefix, moduleDir string, data android.AndroidMkData) { - android.WriteAndroidMkData(w, data) - - fmt.Fprintln(w, "jar_installed_module := $(LOCAL_INSTALLED_MODULE)") + ExtraFooters: []android.AndroidMkExtraFootersFunc{ + func(w io.Writer, name, prefix, moduleDir string, entries *android.AndroidMkEntries) { + fmt.Fprintln(w, "jar_installed_module := $(LOCAL_INSTALLED_MODULE)") + }, }, - } + }} } else { - return android.AndroidMkData{ + return []android.AndroidMkEntries{android.AndroidMkEntries{ Class: "EXECUTABLES", OutputFile: android.OptionalPathForPath(binary.wrapperFile), - Extra: []android.AndroidMkExtraFunc{ - func(w io.Writer, outputFile android.Path) { - fmt.Fprintln(w, "LOCAL_STRIP_MODULE := false") + ExtraEntries: []android.AndroidMkExtraEntriesFunc{ + func(entries *android.AndroidMkEntries) { + entries.SetBool("LOCAL_STRIP_MODULE", false) }, }, - Custom: func(w io.Writer, name, prefix, moduleDir string, data android.AndroidMkData) { - android.WriteAndroidMkData(w, data) - - // Ensure that the wrapper script timestamp is always updated when the jar is updated - fmt.Fprintln(w, "$(LOCAL_INSTALLED_MODULE): $(jar_installed_module)") - fmt.Fprintln(w, "jar_installed_module :=") + ExtraFooters: []android.AndroidMkExtraFootersFunc{ + func(w io.Writer, name, prefix, moduleDir string, entries *android.AndroidMkEntries) { + // Ensure that the wrapper script timestamp is always updated when the jar is updated + fmt.Fprintln(w, "$(LOCAL_INSTALLED_MODULE): $(jar_installed_module)") + fmt.Fprintln(w, "jar_installed_module :=") + }, }, - } + }} } } -func (app *AndroidApp) AndroidMk() android.AndroidMkData { - return android.AndroidMkData{ +func (app *AndroidApp) AndroidMkEntries() []android.AndroidMkEntries { + if !app.IsForPlatform() || app.appProperties.HideFromMake { + return []android.AndroidMkEntries{android.AndroidMkEntries{ + Disabled: true, + }} + } + return []android.AndroidMkEntries{android.AndroidMkEntries{ Class: "APPS", OutputFile: android.OptionalPathForPath(app.outputFile), Include: "$(BUILD_SYSTEM)/soong_app_prebuilt.mk", - Extra: []android.AndroidMkExtraFunc{ - func(w io.Writer, outputFile android.Path) { - // TODO(jungjw): This, outputting two LOCAL_MODULE lines, works, but is not ideal. Find a better solution. - if app.Name() != app.installApkName { - fmt.Fprintln(w, "# Overridden by PRODUCT_PACKAGE_NAME_OVERRIDES") - fmt.Fprintln(w, "LOCAL_MODULE :=", app.installApkName) - } - fmt.Fprintln(w, "LOCAL_SOONG_RESOURCE_EXPORT_PACKAGE :=", app.exportPackage.String()) + ExtraEntries: []android.AndroidMkExtraEntriesFunc{ + func(entries *android.AndroidMkEntries) { + // App module names can be overridden. + entries.SetString("LOCAL_MODULE", app.installApkName) + entries.SetBoolIfTrue("LOCAL_UNINSTALLABLE_MODULE", app.appProperties.PreventInstall) + entries.SetPath("LOCAL_SOONG_RESOURCE_EXPORT_PACKAGE", app.exportPackage) if app.dexJarFile != nil { - fmt.Fprintln(w, "LOCAL_SOONG_DEX_JAR :=", app.dexJarFile.String()) + entries.SetPath("LOCAL_SOONG_DEX_JAR", app.dexJarFile) } if app.implementationAndResourcesJar != nil { - fmt.Fprintln(w, "LOCAL_SOONG_CLASSES_JAR :=", app.implementationAndResourcesJar.String()) + entries.SetPath("LOCAL_SOONG_CLASSES_JAR", app.implementationAndResourcesJar) } if app.headerJarFile != nil { - fmt.Fprintln(w, "LOCAL_SOONG_HEADER_JAR :=", app.headerJarFile.String()) + entries.SetPath("LOCAL_SOONG_HEADER_JAR", app.headerJarFile) } if app.bundleFile != nil { - fmt.Fprintln(w, "LOCAL_SOONG_BUNDLE :=", app.bundleFile.String()) + entries.SetPath("LOCAL_SOONG_BUNDLE", app.bundleFile) } if app.jacocoReportClassesFile != nil { - fmt.Fprintln(w, "LOCAL_SOONG_JACOCO_REPORT_CLASSES_JAR :=", app.jacocoReportClassesFile.String()) + entries.SetPath("LOCAL_SOONG_JACOCO_REPORT_CLASSES_JAR", app.jacocoReportClassesFile) } if app.proguardDictionary != nil { - fmt.Fprintln(w, "LOCAL_SOONG_PROGUARD_DICT :=", app.proguardDictionary.String()) + entries.SetPath("LOCAL_SOONG_PROGUARD_DICT", app.proguardDictionary) } if app.Name() == "framework-res" { - fmt.Fprintln(w, "LOCAL_MODULE_PATH := $(TARGET_OUT_JAVA_LIBRARIES)") + entries.SetString("LOCAL_MODULE_PATH", "$(TARGET_OUT_JAVA_LIBRARIES)") // Make base_rules.mk not put framework-res in a subdirectory called // framework_res. - fmt.Fprintln(w, "LOCAL_NO_STANDARD_LIBRARIES := true") + entries.SetBoolIfTrue("LOCAL_NO_STANDARD_LIBRARIES", true) } filterRRO := func(filter overlayType) android.Paths { @@ -291,41 +356,62 @@ func (app *AndroidApp) AndroidMk() android.AndroidMkData { } deviceRRODirs := filterRRO(device) if len(deviceRRODirs) > 0 { - fmt.Fprintln(w, "LOCAL_SOONG_DEVICE_RRO_DIRS :=", strings.Join(deviceRRODirs.Strings(), " ")) + entries.AddStrings("LOCAL_SOONG_DEVICE_RRO_DIRS", deviceRRODirs.Strings()...) } productRRODirs := filterRRO(product) if len(productRRODirs) > 0 { - fmt.Fprintln(w, "LOCAL_SOONG_PRODUCT_RRO_DIRS :=", strings.Join(productRRODirs.Strings(), " ")) + entries.AddStrings("LOCAL_SOONG_PRODUCT_RRO_DIRS", productRRODirs.Strings()...) } - if Bool(app.appProperties.Export_package_resources) { - fmt.Fprintln(w, "LOCAL_EXPORT_PACKAGE_RESOURCES := true") - } + entries.SetBoolIfTrue("LOCAL_EXPORT_PACKAGE_RESOURCES", Bool(app.appProperties.Export_package_resources)) - fmt.Fprintln(w, "LOCAL_FULL_MANIFEST_FILE :=", app.manifestPath.String()) + entries.SetPath("LOCAL_FULL_MANIFEST_FILE", app.manifestPath) - if Bool(app.appProperties.Privileged) { - fmt.Fprintln(w, "LOCAL_PRIVILEGED_MODULE := true") - } + entries.SetBoolIfTrue("LOCAL_PRIVILEGED_MODULE", app.Privileged()) + + entries.SetString("LOCAL_CERTIFICATE", app.certificate.AndroidMkString()) + entries.AddStrings("LOCAL_OVERRIDES_PACKAGES", app.getOverriddenPackages()...) - fmt.Fprintln(w, "LOCAL_CERTIFICATE :=", app.certificate.Pem.String()) - if overriddenPkgs := app.getOverriddenPackages(); len(overriddenPkgs) > 0 { - fmt.Fprintln(w, "LOCAL_OVERRIDES_PACKAGES :=", strings.Join(overriddenPkgs, " ")) + if app.embeddedJniLibs { + jniSymbols := app.JNISymbolsInstalls(app.installPathForJNISymbols.String()) + entries.SetString("LOCAL_SOONG_JNI_LIBS_SYMBOLS", jniSymbols.String()) + } else { + for _, jniLib := range app.jniLibs { + entries.AddStrings("LOCAL_SOONG_JNI_LIBS_"+jniLib.target.Arch.ArchType.String(), jniLib.name) + } } - for _, jniLib := range app.installJniLibs { - fmt.Fprintln(w, "LOCAL_SOONG_JNI_LIBS_"+jniLib.target.Arch.ArchType.String(), "+=", jniLib.name) + if len(app.jniCoverageOutputs) > 0 { + entries.AddStrings("LOCAL_PREBUILT_COVERAGE_ARCHIVE", app.jniCoverageOutputs.Strings()...) } if len(app.dexpreopter.builtInstalled) > 0 { - fmt.Fprintln(w, "LOCAL_SOONG_BUILT_INSTALLED :=", app.dexpreopter.builtInstalled) + entries.SetString("LOCAL_SOONG_BUILT_INSTALLED", app.dexpreopter.builtInstalled) } - for _, split := range app.aapt.splits { - install := "$(LOCAL_MODULE_PATH)/" + strings.TrimSuffix(app.installApkName, ".apk") + split.suffix + ".apk" - fmt.Fprintln(w, "LOCAL_SOONG_BUILT_INSTALLED +=", split.path.String()+":"+install) + for _, extra := range app.extraOutputFiles { + install := app.onDeviceDir + "/" + extra.Base() + entries.AddStrings("LOCAL_SOONG_BUILT_INSTALLED", extra.String()+":"+install) } + + entries.SetOptionalPaths("LOCAL_SOONG_LINT_REPORTS", app.linter.reports) }, }, - } + ExtraFooters: []android.AndroidMkExtraFootersFunc{ + func(w io.Writer, name, prefix, moduleDir string, entries *android.AndroidMkEntries) { + if app.noticeOutputs.Merged.Valid() { + fmt.Fprintf(w, "$(call dist-for-goals,%s,%s:%s)\n", + app.installApkName, app.noticeOutputs.Merged.String(), app.installApkName+"_NOTICE") + } + if app.noticeOutputs.TxtOutput.Valid() { + fmt.Fprintf(w, "$(call dist-for-goals,%s,%s:%s)\n", + app.installApkName, app.noticeOutputs.TxtOutput.String(), app.installApkName+"_NOTICE.txt") + } + if app.noticeOutputs.HtmlOutput.Valid() { + fmt.Fprintf(w, "$(call dist-for-goals,%s,%s:%s)\n", + app.installApkName, app.noticeOutputs.HtmlOutput.String(), app.installApkName+"_NOTICE.html") + } + }, + }, + }} } func (a *AndroidApp) getOverriddenPackages() []string { @@ -339,88 +425,96 @@ func (a *AndroidApp) getOverriddenPackages() []string { return overridden } -func (a *AndroidTest) AndroidMk() android.AndroidMkData { - data := a.AndroidApp.AndroidMk() - data.Extra = append(data.Extra, func(w io.Writer, outputFile android.Path) { - testSuiteComponent(w, a.testProperties.Test_suites) +func (a *AndroidTest) AndroidMkEntries() []android.AndroidMkEntries { + entriesList := a.AndroidApp.AndroidMkEntries() + entries := &entriesList[0] + entries.ExtraEntries = append(entries.ExtraEntries, func(entries *android.AndroidMkEntries) { + testSuiteComponent(entries, a.testProperties.Test_suites) if a.testConfig != nil { - fmt.Fprintln(w, "LOCAL_FULL_TEST_CONFIG :=", a.testConfig.String()) + entries.SetPath("LOCAL_FULL_TEST_CONFIG", a.testConfig) } + androidMkWriteTestData(a.data, entries) }) - androidMkWriteTestData(a.data, &data) - return data + return entriesList } -func (a *AndroidTestHelperApp) AndroidMk() android.AndroidMkData { - data := a.AndroidApp.AndroidMk() - data.Extra = append(data.Extra, func(w io.Writer, outputFile android.Path) { - testSuiteComponent(w, a.appTestHelperAppProperties.Test_suites) +func (a *AndroidTestHelperApp) AndroidMkEntries() []android.AndroidMkEntries { + entriesList := a.AndroidApp.AndroidMkEntries() + entries := &entriesList[0] + entries.ExtraEntries = append(entries.ExtraEntries, func(entries *android.AndroidMkEntries) { + testSuiteComponent(entries, a.appTestHelperAppProperties.Test_suites) }) - return data + return entriesList } -func (a *AndroidLibrary) AndroidMk() android.AndroidMkData { - data := a.Library.AndroidMk() +func (a *AndroidLibrary) AndroidMkEntries() []android.AndroidMkEntries { + if !a.IsForPlatform() { + return []android.AndroidMkEntries{{ + Disabled: true, + }} + } + entriesList := a.Library.AndroidMkEntries() + entries := &entriesList[0] - data.Extra = append(data.Extra, func(w io.Writer, outputFile android.Path) { + entries.ExtraEntries = append(entries.ExtraEntries, func(entries *android.AndroidMkEntries) { if a.aarFile != nil { - fmt.Fprintln(w, "LOCAL_SOONG_AAR :=", a.aarFile.String()) - } - if a.proguardDictionary != nil { - fmt.Fprintln(w, "LOCAL_SOONG_PROGUARD_DICT :=", a.proguardDictionary.String()) + entries.SetPath("LOCAL_SOONG_AAR", a.aarFile) } if a.Name() == "framework-res" { - fmt.Fprintln(w, "LOCAL_MODULE_PATH := $(TARGET_OUT_JAVA_LIBRARIES)") + entries.SetString("LOCAL_MODULE_PATH", "$(TARGET_OUT_JAVA_LIBRARIES)") // Make base_rules.mk not put framework-res in a subdirectory called // framework_res. - fmt.Fprintln(w, "LOCAL_NO_STANDARD_LIBRARIES := true") + entries.SetBoolIfTrue("LOCAL_NO_STANDARD_LIBRARIES", true) } - fmt.Fprintln(w, "LOCAL_SOONG_RESOURCE_EXPORT_PACKAGE :=", a.exportPackage.String()) - fmt.Fprintln(w, "LOCAL_SOONG_STATIC_LIBRARY_EXTRA_PACKAGES :=", a.extraAaptPackagesFile.String()) - fmt.Fprintln(w, "LOCAL_FULL_MANIFEST_FILE :=", a.manifestPath.String()) - fmt.Fprintln(w, "LOCAL_SOONG_EXPORT_PROGUARD_FLAGS :=", - strings.Join(a.exportedProguardFlagFiles.Strings(), " ")) - fmt.Fprintln(w, "LOCAL_UNINSTALLABLE_MODULE := true") + entries.SetPath("LOCAL_SOONG_RESOURCE_EXPORT_PACKAGE", a.exportPackage) + entries.SetPath("LOCAL_SOONG_STATIC_LIBRARY_EXTRA_PACKAGES", a.extraAaptPackagesFile) + entries.SetPath("LOCAL_FULL_MANIFEST_FILE", a.mergedManifestFile) + entries.AddStrings("LOCAL_SOONG_EXPORT_PROGUARD_FLAGS", a.exportedProguardFlagFiles.Strings()...) + entries.SetBoolIfTrue("LOCAL_UNINSTALLABLE_MODULE", true) }) - return data + return entriesList } -func (jd *Javadoc) AndroidMk() android.AndroidMkData { - return android.AndroidMkData{ +func (jd *Javadoc) AndroidMkEntries() []android.AndroidMkEntries { + return []android.AndroidMkEntries{android.AndroidMkEntries{ Class: "JAVA_LIBRARIES", OutputFile: android.OptionalPathForPath(jd.stubsSrcJar), Include: "$(BUILD_SYSTEM)/soong_droiddoc_prebuilt.mk", - Extra: []android.AndroidMkExtraFunc{ - func(w io.Writer, outputFile android.Path) { + ExtraEntries: []android.AndroidMkExtraEntriesFunc{ + func(entries *android.AndroidMkEntries) { if BoolDefault(jd.properties.Installable, true) { - fmt.Fprintln(w, "LOCAL_DROIDDOC_DOC_ZIP := ", jd.docZip.String()) + entries.SetPath("LOCAL_DROIDDOC_DOC_ZIP", jd.docZip) } if jd.stubsSrcJar != nil { - fmt.Fprintln(w, "LOCAL_DROIDDOC_STUBS_SRCJAR := ", jd.stubsSrcJar.String()) + entries.SetPath("LOCAL_DROIDDOC_STUBS_SRCJAR", jd.stubsSrcJar) } }, }, - } + }} } -func (ddoc *Droiddoc) AndroidMk() android.AndroidMkData { - return android.AndroidMkData{ +func (ddoc *Droiddoc) AndroidMkEntries() []android.AndroidMkEntries { + return []android.AndroidMkEntries{android.AndroidMkEntries{ Class: "JAVA_LIBRARIES", OutputFile: android.OptionalPathForPath(ddoc.stubsSrcJar), Include: "$(BUILD_SYSTEM)/soong_droiddoc_prebuilt.mk", - Extra: []android.AndroidMkExtraFunc{ - func(w io.Writer, outputFile android.Path) { + ExtraEntries: []android.AndroidMkExtraEntriesFunc{ + func(entries *android.AndroidMkEntries) { if BoolDefault(ddoc.Javadoc.properties.Installable, true) && ddoc.Javadoc.docZip != nil { - fmt.Fprintln(w, "LOCAL_DROIDDOC_DOC_ZIP := ", ddoc.Javadoc.docZip.String()) + entries.SetPath("LOCAL_DROIDDOC_DOC_ZIP", ddoc.Javadoc.docZip) } if ddoc.Javadoc.stubsSrcJar != nil { - fmt.Fprintln(w, "LOCAL_DROIDDOC_STUBS_SRCJAR := ", ddoc.Javadoc.stubsSrcJar.String()) + entries.SetPath("LOCAL_DROIDDOC_STUBS_SRCJAR", ddoc.Javadoc.stubsSrcJar) } + }, + }, + ExtraFooters: []android.AndroidMkExtraFootersFunc{ + func(w io.Writer, name, prefix, moduleDir string, entries *android.AndroidMkEntries) { if ddoc.checkCurrentApiTimestamp != nil { fmt.Fprintln(w, ".PHONY:", ddoc.Name()+"-check-current-api") fmt.Fprintln(w, ddoc.Name()+"-check-current-api:", @@ -456,60 +550,56 @@ func (ddoc *Droiddoc) AndroidMk() android.AndroidMkData { fmt.Fprintln(w, "droidcore: checkapi") } } - apiFilePrefix := "INTERNAL_PLATFORM_" - if String(ddoc.properties.Api_tag_name) != "" { - apiFilePrefix += String(ddoc.properties.Api_tag_name) + "_" - } - if ddoc.apiFile != nil { - fmt.Fprintln(w, apiFilePrefix+"API_FILE := ", ddoc.apiFile.String()) - } - if ddoc.dexApiFile != nil { - fmt.Fprintln(w, apiFilePrefix+"DEX_API_FILE := ", ddoc.dexApiFile.String()) - } - if ddoc.privateApiFile != nil { - fmt.Fprintln(w, apiFilePrefix+"PRIVATE_API_FILE := ", ddoc.privateApiFile.String()) - } - if ddoc.privateDexApiFile != nil { - fmt.Fprintln(w, apiFilePrefix+"PRIVATE_DEX_API_FILE := ", ddoc.privateDexApiFile.String()) - } - if ddoc.removedApiFile != nil { - fmt.Fprintln(w, apiFilePrefix+"REMOVED_API_FILE := ", ddoc.removedApiFile.String()) - } - if ddoc.removedDexApiFile != nil { - fmt.Fprintln(w, apiFilePrefix+"REMOVED_DEX_API_FILE := ", ddoc.removedDexApiFile.String()) - } - if ddoc.exactApiFile != nil { - fmt.Fprintln(w, apiFilePrefix+"EXACT_API_FILE := ", ddoc.exactApiFile.String()) - } - if ddoc.proguardFile != nil { - fmt.Fprintln(w, apiFilePrefix+"PROGUARD_FILE := ", ddoc.proguardFile.String()) - } }, }, - } + }} } -func (dstubs *Droidstubs) AndroidMk() android.AndroidMkData { - return android.AndroidMkData{ +func (dstubs *Droidstubs) AndroidMkEntries() []android.AndroidMkEntries { + // If the stubsSrcJar is not generated (because generate_stubs is false) then + // use the api file as the output file to ensure the relevant phony targets + // are created in make if only the api txt file is being generated. This is + // needed because an invalid output file would prevent the make entries from + // being written. + // TODO(b/146727827): Revert when we do not need to generate stubs and API separately. + distFile := android.OptionalPathForPath(dstubs.apiFile) + outputFile := android.OptionalPathForPath(dstubs.stubsSrcJar) + if !outputFile.Valid() { + outputFile = distFile + } + return []android.AndroidMkEntries{android.AndroidMkEntries{ Class: "JAVA_LIBRARIES", - OutputFile: android.OptionalPathForPath(dstubs.stubsSrcJar), + DistFile: distFile, + OutputFile: outputFile, Include: "$(BUILD_SYSTEM)/soong_droiddoc_prebuilt.mk", - Extra: []android.AndroidMkExtraFunc{ - func(w io.Writer, outputFile android.Path) { + ExtraEntries: []android.AndroidMkExtraEntriesFunc{ + func(entries *android.AndroidMkEntries) { if dstubs.Javadoc.stubsSrcJar != nil { - fmt.Fprintln(w, "LOCAL_DROIDDOC_STUBS_SRCJAR := ", dstubs.Javadoc.stubsSrcJar.String()) + entries.SetPath("LOCAL_DROIDDOC_STUBS_SRCJAR", dstubs.Javadoc.stubsSrcJar) } if dstubs.apiVersionsXml != nil { - fmt.Fprintln(w, "LOCAL_DROIDDOC_API_VERSIONS_XML := ", dstubs.apiVersionsXml.String()) + entries.SetPath("LOCAL_DROIDDOC_API_VERSIONS_XML", dstubs.apiVersionsXml) } if dstubs.annotationsZip != nil { - fmt.Fprintln(w, "LOCAL_DROIDDOC_ANNOTATIONS_ZIP := ", dstubs.annotationsZip.String()) + entries.SetPath("LOCAL_DROIDDOC_ANNOTATIONS_ZIP", dstubs.annotationsZip) } if dstubs.jdiffDocZip != nil { - fmt.Fprintln(w, "LOCAL_DROIDDOC_JDIFF_DOC_ZIP := ", dstubs.jdiffDocZip.String()) + entries.SetPath("LOCAL_DROIDDOC_JDIFF_DOC_ZIP", dstubs.jdiffDocZip) } if dstubs.metadataZip != nil { - fmt.Fprintln(w, "LOCAL_DROIDDOC_METADATA_ZIP := ", dstubs.metadataZip.String()) + entries.SetPath("LOCAL_DROIDDOC_METADATA_ZIP", dstubs.metadataZip) + } + }, + }, + ExtraFooters: []android.AndroidMkExtraFootersFunc{ + func(w io.Writer, name, prefix, moduleDir string, entries *android.AndroidMkEntries) { + if dstubs.apiFile != nil { + fmt.Fprintf(w, ".PHONY: %s %s.txt\n", dstubs.Name(), dstubs.Name()) + fmt.Fprintf(w, "%s %s.txt: %s\n", dstubs.Name(), dstubs.Name(), dstubs.apiFile) + } + if dstubs.removedApiFile != nil { + fmt.Fprintf(w, ".PHONY: %s %s.txt\n", dstubs.Name(), dstubs.Name()) + fmt.Fprintf(w, "%s %s.txt: %s\n", dstubs.Name(), dstubs.Name(), dstubs.removedApiFile) } if dstubs.checkCurrentApiTimestamp != nil { fmt.Fprintln(w, ".PHONY:", dstubs.Name()+"-check-current-api") @@ -537,13 +627,28 @@ func (dstubs *Droidstubs) AndroidMk() android.AndroidMkData { fmt.Fprintln(w, dstubs.Name()+"-check-last-released-api:", dstubs.checkLastReleasedApiTimestamp.String()) - if dstubs.Name() == "api-stubs-docs" || dstubs.Name() == "system-api-stubs-docs" { - fmt.Fprintln(w, ".PHONY: checkapi") - fmt.Fprintln(w, "checkapi:", - dstubs.checkLastReleasedApiTimestamp.String()) + fmt.Fprintln(w, ".PHONY: checkapi") + fmt.Fprintln(w, "checkapi:", + dstubs.checkLastReleasedApiTimestamp.String()) - fmt.Fprintln(w, ".PHONY: droidcore") - fmt.Fprintln(w, "droidcore: checkapi") + fmt.Fprintln(w, ".PHONY: droidcore") + fmt.Fprintln(w, "droidcore: checkapi") + } + if dstubs.apiLintTimestamp != nil { + fmt.Fprintln(w, ".PHONY:", dstubs.Name()+"-api-lint") + fmt.Fprintln(w, dstubs.Name()+"-api-lint:", + dstubs.apiLintTimestamp.String()) + + fmt.Fprintln(w, ".PHONY: checkapi") + fmt.Fprintln(w, "checkapi:", + dstubs.Name()+"-api-lint") + + fmt.Fprintln(w, ".PHONY: droidcore") + fmt.Fprintln(w, "droidcore: checkapi") + + if dstubs.apiLintReport != nil { + fmt.Fprintf(w, "$(call dist-for-goals,%s,%s:%s)\n", dstubs.Name()+"-api-lint", + dstubs.apiLintReport.String(), "apilint/"+dstubs.Name()+"-lint-report.txt") } } if dstubs.checkNullabilityWarningsTimestamp != nil { @@ -554,61 +659,76 @@ func (dstubs *Droidstubs) AndroidMk() android.AndroidMkData { fmt.Fprintln(w, ".PHONY:", "droidcore") fmt.Fprintln(w, "droidcore: ", dstubs.Name()+"-check-nullability-warnings") } - apiFilePrefix := "INTERNAL_PLATFORM_" - if String(dstubs.properties.Api_tag_name) != "" { - apiFilePrefix += String(dstubs.properties.Api_tag_name) + "_" - } - if dstubs.apiFile != nil { - fmt.Fprintln(w, apiFilePrefix+"API_FILE := ", dstubs.apiFile.String()) - } - if dstubs.dexApiFile != nil { - fmt.Fprintln(w, apiFilePrefix+"DEX_API_FILE := ", dstubs.dexApiFile.String()) - } - if dstubs.privateApiFile != nil { - fmt.Fprintln(w, apiFilePrefix+"PRIVATE_API_FILE := ", dstubs.privateApiFile.String()) - } - if dstubs.privateDexApiFile != nil { - fmt.Fprintln(w, apiFilePrefix+"PRIVATE_DEX_API_FILE := ", dstubs.privateDexApiFile.String()) - } - if dstubs.removedApiFile != nil { - fmt.Fprintln(w, apiFilePrefix+"REMOVED_API_FILE := ", dstubs.removedApiFile.String()) - } - if dstubs.removedDexApiFile != nil { - fmt.Fprintln(w, apiFilePrefix+"REMOVED_DEX_API_FILE := ", dstubs.removedDexApiFile.String()) - } - if dstubs.exactApiFile != nil { - fmt.Fprintln(w, apiFilePrefix+"EXACT_API_FILE := ", dstubs.exactApiFile.String()) - } }, }, - } + }} } -func androidMkWriteTestData(data android.Paths, ret *android.AndroidMkData) { +func (a *AndroidAppImport) AndroidMkEntries() []android.AndroidMkEntries { + return []android.AndroidMkEntries{android.AndroidMkEntries{ + Class: "APPS", + OutputFile: android.OptionalPathForPath(a.outputFile), + Include: "$(BUILD_SYSTEM)/soong_app_prebuilt.mk", + ExtraEntries: []android.AndroidMkExtraEntriesFunc{ + func(entries *android.AndroidMkEntries) { + entries.SetBoolIfTrue("LOCAL_PRIVILEGED_MODULE", a.Privileged()) + entries.SetString("LOCAL_CERTIFICATE", a.certificate.AndroidMkString()) + entries.AddStrings("LOCAL_OVERRIDES_PACKAGES", a.properties.Overrides...) + if len(a.dexpreopter.builtInstalled) > 0 { + entries.SetString("LOCAL_SOONG_BUILT_INSTALLED", a.dexpreopter.builtInstalled) + } + entries.AddStrings("LOCAL_INSTALLED_MODULE_STEM", a.installPath.Rel()) + }, + }, + }} +} + +func (a *AndroidTestImport) AndroidMkEntries() []android.AndroidMkEntries { + entriesList := a.AndroidAppImport.AndroidMkEntries() + entries := &entriesList[0] + entries.ExtraEntries = append(entries.ExtraEntries, func(entries *android.AndroidMkEntries) { + testSuiteComponent(entries, a.testProperties.Test_suites) + androidMkWriteTestData(a.data, entries) + }) + return entriesList +} + +func androidMkWriteTestData(data android.Paths, entries *android.AndroidMkEntries) { var testFiles []string for _, d := range data { testFiles = append(testFiles, d.String()+":"+d.Rel()) } - if len(testFiles) > 0 { - ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) { - fmt.Fprintln(w, "LOCAL_COMPATIBILITY_SUPPORT_FILES := "+strings.Join(testFiles, " ")) - }) - } + entries.AddStrings("LOCAL_COMPATIBILITY_SUPPORT_FILES", testFiles...) } -func (apkSet *AndroidAppSet) AndroidMk() android.AndroidMkData { - return android.AndroidMkData{ - Class: "APPS", - OutputFile: android.OptionalPathForPath(apkSet.packedOutput), - Include: "$(BUILD_SYSTEM)/soong_android_app_set.mk", - Extra: []android.AndroidMkExtraFunc{ - func(w io.Writer, outputFile android.Path) { - if apkSet.Privileged() { - fmt.Fprintln(w, "LOCAL_PRIVILEGED_MODULE := true") - } - fmt.Fprintln(w, "LOCAL_APK_SET_MASTER_FILE := ", apkSet.masterFile) - fmt.Fprintln(w, "LOCAL_APKCERTS_FILE := ", apkSet.apkcertsFile) - fmt.Fprintln(w, "LOCAL_OVERRIDES_PACKAGES :=", strings.Join(apkSet.properties.Overrides, " ")) +func (r *RuntimeResourceOverlay) AndroidMkEntries() []android.AndroidMkEntries { + return []android.AndroidMkEntries{android.AndroidMkEntries{ + Class: "ETC", + OutputFile: android.OptionalPathForPath(r.outputFile), + Include: "$(BUILD_SYSTEM)/soong_app_prebuilt.mk", + ExtraEntries: []android.AndroidMkExtraEntriesFunc{ + func(entries *android.AndroidMkEntries) { + entries.SetString("LOCAL_CERTIFICATE", r.certificate.AndroidMkString()) + entries.SetPath("LOCAL_MODULE_PATH", r.installDir.ToMakePath()) + entries.AddStrings("LOCAL_OVERRIDES_PACKAGES", r.properties.Overrides...) + }, + }, + }} +} + +func (apkSet *AndroidAppSet) AndroidMkEntries() []android.AndroidMkEntries { + return []android.AndroidMkEntries{ + android.AndroidMkEntries{ + Class: "APPS", + OutputFile: android.OptionalPathForPath(apkSet.packedOutput), + Include: "$(BUILD_SYSTEM)/soong_android_app_set.mk", + ExtraEntries: []android.AndroidMkExtraEntriesFunc{ + func(entries *android.AndroidMkEntries) { + entries.SetBoolIfTrue("LOCAL_PRIVILEGED_MODULE", apkSet.Privileged()) + entries.SetString("LOCAL_APK_SET_MASTER_FILE", apkSet.masterFile) + entries.SetPath("LOCAL_APKCERTS_FILE", apkSet.apkcertsFile) + entries.AddStrings("LOCAL_OVERRIDES_PACKAGES", apkSet.properties.Overrides...) + }, }, }, } diff --git a/java/androidmk_test.go b/java/androidmk_test.go new file mode 100644 index 000000000..7daa6244f --- /dev/null +++ b/java/androidmk_test.go @@ -0,0 +1,171 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package java + +import ( + "reflect" + "strings" + "testing" + + "android/soong/android" +) + +func TestRequired(t *testing.T) { + ctx, config := testJava(t, ` + java_library { + name: "foo", + srcs: ["a.java"], + required: ["libfoo"], + } + `) + + mod := ctx.ModuleForTests("foo", "android_common").Module() + entries := android.AndroidMkEntriesForTest(t, config, "", mod)[0] + + expected := []string{"libfoo"} + actual := entries.EntryMap["LOCAL_REQUIRED_MODULES"] + if !reflect.DeepEqual(expected, actual) { + t.Errorf("Unexpected required modules - expected: %q, actual: %q", expected, actual) + } +} + +func TestHostdex(t *testing.T) { + ctx, config := testJava(t, ` + java_library { + name: "foo", + srcs: ["a.java"], + hostdex: true, + } + `) + + mod := ctx.ModuleForTests("foo", "android_common").Module() + entriesList := android.AndroidMkEntriesForTest(t, config, "", mod) + if len(entriesList) != 2 { + t.Errorf("two entries are expected, but got %d", len(entriesList)) + } + + mainEntries := &entriesList[0] + expected := []string{"foo"} + actual := mainEntries.EntryMap["LOCAL_MODULE"] + if !reflect.DeepEqual(expected, actual) { + t.Errorf("Unexpected module name - expected: %q, actual: %q", expected, actual) + } + + subEntries := &entriesList[1] + expected = []string{"foo-hostdex"} + actual = subEntries.EntryMap["LOCAL_MODULE"] + if !reflect.DeepEqual(expected, actual) { + t.Errorf("Unexpected module name - expected: %q, actual: %q", expected, actual) + } +} + +func TestHostdexRequired(t *testing.T) { + ctx, config := testJava(t, ` + java_library { + name: "foo", + srcs: ["a.java"], + hostdex: true, + required: ["libfoo"], + } + `) + + mod := ctx.ModuleForTests("foo", "android_common").Module() + entriesList := android.AndroidMkEntriesForTest(t, config, "", mod) + if len(entriesList) != 2 { + t.Errorf("two entries are expected, but got %d", len(entriesList)) + } + + mainEntries := &entriesList[0] + expected := []string{"libfoo"} + actual := mainEntries.EntryMap["LOCAL_REQUIRED_MODULES"] + if !reflect.DeepEqual(expected, actual) { + t.Errorf("Unexpected required modules - expected: %q, actual: %q", expected, actual) + } + + subEntries := &entriesList[1] + expected = []string{"libfoo"} + actual = subEntries.EntryMap["LOCAL_REQUIRED_MODULES"] + if !reflect.DeepEqual(expected, actual) { + t.Errorf("Unexpected required modules - expected: %q, actual: %q", expected, actual) + } +} + +func TestHostdexSpecificRequired(t *testing.T) { + ctx, config := testJava(t, ` + java_library { + name: "foo", + srcs: ["a.java"], + hostdex: true, + target: { + hostdex: { + required: ["libfoo"], + }, + }, + } + `) + + mod := ctx.ModuleForTests("foo", "android_common").Module() + entriesList := android.AndroidMkEntriesForTest(t, config, "", mod) + if len(entriesList) != 2 { + t.Errorf("two entries are expected, but got %d", len(entriesList)) + } + + mainEntries := &entriesList[0] + if r, ok := mainEntries.EntryMap["LOCAL_REQUIRED_MODULES"]; ok { + t.Errorf("Unexpected required modules: %q", r) + } + + subEntries := &entriesList[1] + expected := []string{"libfoo"} + actual := subEntries.EntryMap["LOCAL_REQUIRED_MODULES"] + if !reflect.DeepEqual(expected, actual) { + t.Errorf("Unexpected required modules - expected: %q, actual: %q", expected, actual) + } +} + +func TestDistWithTag(t *testing.T) { + ctx, config := testJava(t, ` + java_library { + name: "foo_without_tag", + srcs: ["a.java"], + compile_dex: true, + dist: { + targets: ["hi"], + }, + } + java_library { + name: "foo_with_tag", + srcs: ["a.java"], + compile_dex: true, + dist: { + targets: ["hi"], + tag: ".jar", + }, + } + `) + + without_tag_entries := android.AndroidMkEntriesForTest(t, config, "", ctx.ModuleForTests("foo_without_tag", "android_common").Module()) + with_tag_entries := android.AndroidMkEntriesForTest(t, config, "", ctx.ModuleForTests("foo_with_tag", "android_common").Module()) + + if len(without_tag_entries) != 2 || len(with_tag_entries) != 2 { + t.Errorf("two mk entries per module expected, got %d and %d", len(without_tag_entries), len(with_tag_entries)) + } + if !with_tag_entries[0].DistFile.Valid() || !strings.Contains(with_tag_entries[0].DistFile.String(), "/javac/foo_with_tag.jar") { + t.Errorf("expected classes.jar DistFile, got %v", with_tag_entries[0].DistFile) + } + if without_tag_entries[0].DistFile.Valid() { + t.Errorf("did not expect explicit DistFile, got %v", without_tag_entries[0].DistFile) + } +} diff --git a/java/app.go b/java/app.go index 586b66d85..2ec27f3df 100644..100755 --- a/java/app.go +++ b/java/app.go @@ -18,6 +18,7 @@ package java import ( "path/filepath" + "reflect" "sort" "strconv" "strings" @@ -30,18 +31,31 @@ import ( "android/soong/tradefed" ) +var supportedDpis = []string{"ldpi", "mdpi", "hdpi", "xhdpi", "xxhdpi", "xxxhdpi"} + func init() { - android.RegisterModuleType("android_app", AndroidAppFactory) - android.RegisterModuleType("android_test", AndroidTestFactory) - android.RegisterModuleType("android_test_helper_app", AndroidTestHelperAppFactory) - android.RegisterModuleType("android_app_certificate", AndroidAppCertificateFactory) - android.RegisterModuleType("override_android_app", OverrideAndroidAppModuleFactory) - android.RegisterModuleType("android_app_set", AndroidApkSetFactory) + RegisterAppBuildComponents(android.InitRegistrationContext) + + initAndroidAppImportVariantGroupTypes() +} + +func RegisterAppBuildComponents(ctx android.RegistrationContext) { + ctx.RegisterModuleType("android_app", AndroidAppFactory) + ctx.RegisterModuleType("android_test", AndroidTestFactory) + ctx.RegisterModuleType("android_test_helper_app", AndroidTestHelperAppFactory) + ctx.RegisterModuleType("android_app_certificate", AndroidAppCertificateFactory) + ctx.RegisterModuleType("override_android_app", OverrideAndroidAppModuleFactory) + ctx.RegisterModuleType("override_android_test", OverrideAndroidTestModuleFactory) + ctx.RegisterModuleType("override_runtime_resource_overlay", OverrideRuntimeResourceOverlayModuleFactory) + ctx.RegisterModuleType("android_app_import", AndroidAppImportFactory) + ctx.RegisterModuleType("android_test_import", AndroidTestImportFactory) + ctx.RegisterModuleType("runtime_resource_overlay", RuntimeResourceOverlayFactory) + ctx.RegisterModuleType("android_app_set", AndroidApkSetFactory) } type AndroidAppSetProperties struct { // APK Set path - Set string + Set *string // Specifies that this app should be installed to the priv-app directory, // where the system will grant it additional privileges not available to @@ -83,6 +97,18 @@ func (as *AndroidAppSet) Privileged() bool { return Bool(as.properties.Privileged) } +func (as *AndroidAppSet) OutputFile() android.Path { + return as.packedOutput +} + +func (as *AndroidAppSet) MasterFile() string { + return as.masterFile +} + +func (as *AndroidAppSet) APKCertsFile() android.Path { + return as.apkcertsFile +} + var TargetCpuAbi = map[string]string{ "arm": "ARMEABI_V7A", "arm64": "ARM64_V8A", @@ -91,28 +117,28 @@ var TargetCpuAbi = map[string]string{ } func SupportedAbis(ctx android.ModuleContext) []string { - abiName := func(archVar string, deviceArch string) string { + abiName := func(targetIdx int, deviceArch string) string { if abi, found := TargetCpuAbi[deviceArch]; found { return abi } - ctx.ModuleErrorf("Invalid %s: %s", archVar, deviceArch) + ctx.ModuleErrorf("Target %d has invalid Arch: %s", targetIdx, deviceArch) return "BAD_ABI" } - result := []string{abiName("TARGET_ARCH", ctx.DeviceConfig().DeviceArch())} - if s := ctx.DeviceConfig().DeviceSecondaryArch(); s != "" { - result = append(result, abiName("TARGET_2ND_ARCH", s)) + var result []string + for i, target := range ctx.Config().Targets[android.Android] { + result = append(result, abiName(i, target.Arch.ArchType.String())) } return result } func (as *AndroidAppSet) GenerateAndroidBuildActions(ctx android.ModuleContext) { - as.packedOutput = android.PathForModuleOut(ctx, "extracted.zip") + as.packedOutput = android.PathForModuleOut(ctx, ctx.ModuleName()+".zip") as.apkcertsFile = android.PathForModuleOut(ctx, "apkcerts.txt") // We are assuming here that the master file in the APK // set has `.apk` suffix. If it doesn't the build will fail. // APK sets containing APEX files are handled elsewhere. - as.masterFile = ctx.ModuleName() + ".apk" + as.masterFile = as.BaseModuleName() + ".apk" screenDensities := "all" if dpis := ctx.Config().ProductAAPTPrebuiltDPI(); len(dpis) > 0 { screenDensities = strings.ToUpper(strings.Join(dpis, ",")) @@ -131,36 +157,27 @@ func (as *AndroidAppSet) GenerateAndroidBuildActions(ctx android.ModuleContext) "allow-prereleased": strconv.FormatBool(proptools.Bool(as.properties.Prerelease)), "screen-densities": screenDensities, "sdk-version": ctx.Config().PlatformSdkVersion(), - "stem": ctx.ModuleName(), + "stem": as.BaseModuleName(), "apkcerts": as.apkcertsFile.String(), "partition": as.PartitionTag(ctx.DeviceConfig()), }, }) - // TODO(asmundak): add this (it's wrong now, will cause copying extracted.zip) - /* - var installDir android.InstallPath - if Bool(as.properties.Privileged) { - installDir = android.PathForModuleInstall(ctx, "priv-app", as.BaseModuleName()) - } else if ctx.InstallInTestcases() { - installDir = android.PathForModuleInstall(ctx, as.BaseModuleName(), ctx.DeviceConfig().DeviceArch()) - } else { - installDir = android.PathForModuleInstall(ctx, "app", as.BaseModuleName()) - } - ctx.InstallFile(installDir, as.masterFile", as.packedOutput) - */ } // android_app_set extracts a set of APKs based on the target device // configuration and installs this set as "split APKs". -// The set will always contain `base-master.apk` and every APK built -// to the target device. All density-specific APK will be included, too, -// unless PRODUCT_APPT_PREBUILT_DPI is defined (should contain comma-sepearated -// list of density names (LDPI, MDPI, HDPI, etc.) +// The extracted set always contains 'master' APK whose name is +// _module_name_.apk and every split APK matching target device. +// The extraction of the density-specific splits depends on +// PRODUCT_AAPT_PREBUILT_DPI variable. If present (its value should +// be a list density names: LDPI, MDPI, HDPI, etc.), only listed +// splits will be extracted. Otherwise all density-specific splits +// will be extracted. func AndroidApkSetFactory() android.Module { module := &AndroidAppSet{} module.AddProperties(&module.properties) InitJavaModule(module, android.DeviceSupported) - android.InitSingleSourcePrebuiltModule(module, &module.properties.Set) + android.InitSingleSourcePrebuiltModule(module, &module.properties, "Set") return module } @@ -193,10 +210,22 @@ type appProperties struct { // list of native libraries that will be provided in or alongside the resulting jar Jni_libs []string `android:"arch_variant"` + // if true, use JNI libraries that link against platform APIs even if this module sets + // sdk_version. + Jni_uses_platform_apis *bool + + // if true, use JNI libraries that link against SDK APIs even if this module does not set + // sdk_version. + Jni_uses_sdk_apis *bool + + // STL library to use for JNI libraries. + Stl *string `android:"arch_variant"` + // Store native libraries uncompressed in the APK and set the android:extractNativeLibs="false" manifest // flag so that they are used from inside the APK at runtime. Defaults to true for android_test modules unless - // sdk_version or min_sdk_version is set to a version that doesn't support it (<23), defaults to false for other - // module types where the native libraries are generally preinstalled outside the APK. + // sdk_version or min_sdk_version is set to a version that doesn't support it (<23), defaults to true for + // android_app modules that are embedded to APEXes, defaults to false for other module types where the native + // libraries are generally preinstalled outside the APK. Use_embedded_native_libs *bool // Store dex files uncompressed in the APK and set the android:useEmbeddedDex="true" manifest attribute so that @@ -211,6 +240,17 @@ type appProperties struct { // If set, find and merge all NOTICE files that this module and its dependencies have and store // it in the APK as an asset. Embed_notices *bool + + // cc.Coverage related properties + PreventInstall bool `blueprint:"mutated"` + HideFromMake bool `blueprint:"mutated"` + IsCoverageVariant bool `blueprint:"mutated"` + + // Whether this app is considered mainline updatable or not. When set to true, this will enforce + // additional rules to make sure an app can safely be updated. Default is false. + // Prefer using other specific properties if build behaviour must be changed; avoid using this + // flag for anything but neverallow rules (unless the behaviour change is invisible to owners). + Updatable *bool } // android_app properties that can be overridden by override_android_app @@ -219,8 +259,23 @@ type overridableAppProperties struct { // or an android_app_certificate module name in the form ":module". Certificate *string + // Name of the signing certificate lineage file. + Lineage *string + + // the package name of this app. The package name in the manifest file is used if one was not given. + Package_name *string + + // the logging parent of this app. + Logging_parent *string +} + +// runtime_resource_overlay properties that can be overridden by override_runtime_resource_overlay +type OverridableRuntimeResourceOverlayProperties struct { // the package name of this app. The package name in the manifest file is used if one was not given. Package_name *string + + // the target package name of this overlay app. The target package name in the manifest file is used if one was not given. + Target_package_name *string } type AndroidApp struct { @@ -228,20 +283,39 @@ type AndroidApp struct { aapt android.OverridableModuleBase + usesLibrary usesLibrary + certificate Certificate appProperties appProperties overridableAppProperties overridableAppProperties - installJniLibs []jniLib + jniLibs []jniLib + installPathForJNISymbols android.Path + embeddedJniLibs bool + jniCoverageOutputs android.Paths bundleFile android.Path // the install APK name is normally the same as the module name, but can be overridden with PRODUCT_PACKAGE_NAME_OVERRIDES. installApkName string + installDir android.InstallPath + + onDeviceDir string + additionalAaptFlags []string + + noticeOutputs android.NoticeOutputs + + overriddenManifestPackageName string + + android.ApexBundleDepsInfo +} + +func (a *AndroidApp) IsInstallable() bool { + return Bool(a.properties.Installable) } func (a *AndroidApp) ExportedProguardFlagFiles() android.Paths { @@ -252,30 +326,78 @@ func (a *AndroidApp) ExportedStaticPackages() android.Paths { return nil } +func (a *AndroidApp) OutputFile() android.Path { + return a.outputFile +} + +func (a *AndroidApp) Certificate() Certificate { + return a.certificate +} + +func (a *AndroidApp) JniCoverageOutputs() android.Paths { + return a.jniCoverageOutputs +} + var _ AndroidLibraryDependency = (*AndroidApp)(nil) type Certificate struct { - Pem, Key android.Path + Pem, Key android.Path + presigned bool +} + +var PresignedCertificate = Certificate{presigned: true} + +func (c Certificate) AndroidMkString() string { + if c.presigned { + return "PRESIGNED" + } else { + return c.Pem.String() + } } func (a *AndroidApp) DepsMutator(ctx android.BottomUpMutatorContext) { a.Module.deps(ctx) - if !Bool(a.properties.No_framework_libs) && !Bool(a.properties.No_standard_libs) { - a.aapt.deps(ctx, sdkContext(a)) + if String(a.appProperties.Stl) == "c++_shared" && !a.sdkVersion().specified() { + ctx.PropertyErrorf("stl", "sdk_version must be set in order to use c++_shared") + } + + sdkDep := decodeSdkDep(ctx, sdkContext(a)) + if sdkDep.hasFrameworkLibs() { + a.aapt.deps(ctx, sdkDep) + } + + usesSDK := a.sdkVersion().specified() && a.sdkVersion().kind != sdkCorePlatform + + if usesSDK && Bool(a.appProperties.Jni_uses_sdk_apis) { + ctx.PropertyErrorf("jni_uses_sdk_apis", + "can only be set for modules that do not set sdk_version") + } else if !usesSDK && Bool(a.appProperties.Jni_uses_platform_apis) { + ctx.PropertyErrorf("jni_uses_platform_apis", + "can only be set for modules that set sdk_version") } + tag := &jniDependencyTag{} for _, jniTarget := range ctx.MultiTargets() { - variation := []blueprint.Variation{ - {Mutator: "arch", Variation: jniTarget.String()}, - {Mutator: "link", Variation: "shared"}, - } - tag := &jniDependencyTag{ - target: jniTarget, + variation := append(jniTarget.Variations(), + blueprint.Variation{Mutator: "link", Variation: "shared"}) + + // If the app builds against an Android SDK use the SDK variant of JNI dependencies + // unless jni_uses_platform_apis is set. + // Don't require the SDK variant for apps that are shipped on vendor, etc., as they already + // have stable APIs through the VNDK. + if (usesSDK && !a.RequiresStableAPIs(ctx) && + !Bool(a.appProperties.Jni_uses_platform_apis)) || + Bool(a.appProperties.Jni_uses_sdk_apis) { + variation = append(variation, blueprint.Variation{Mutator: "sdk", Variation: "sdk"}) } ctx.AddFarVariationDependencies(variation, tag, a.appProperties.Jni_libs...) } + a.usesLibrary.deps(ctx, sdkDep.hasFrameworkLibs()) +} + +func (a *AndroidApp) OverridablePropertiesDepsMutator(ctx android.BottomUpMutatorContext) { cert := android.SrcIsModule(a.getCertString(ctx)) if cert != "" { ctx.AddDependency(ctx.Module(), certificateTag, cert) @@ -292,21 +414,70 @@ func (a *AndroidApp) DepsMutator(ctx android.BottomUpMutatorContext) { } } +func (a *AndroidTestHelperApp) GenerateAndroidBuildActions(ctx android.ModuleContext) { + a.generateAndroidBuildActions(ctx) +} + func (a *AndroidApp) GenerateAndroidBuildActions(ctx android.ModuleContext) { - a.aapt.uncompressedJNI = a.shouldUncompressJNI(ctx) - a.aapt.useEmbeddedDex = Bool(a.appProperties.Use_embedded_dex) + a.checkAppSdkVersions(ctx) a.generateAndroidBuildActions(ctx) } -// shouldUncompressJNI returns true if the native libraries should be stored in the APK uncompressed and the +func (a *AndroidApp) checkAppSdkVersions(ctx android.ModuleContext) { + if a.Updatable() { + if !a.sdkVersion().stable() { + ctx.PropertyErrorf("sdk_version", "Updatable apps must use stable SDKs, found %v", a.sdkVersion()) + } + if String(a.deviceProperties.Min_sdk_version) == "" { + ctx.PropertyErrorf("updatable", "updatable apps must set min_sdk_version.") + } + if minSdkVersion, err := a.minSdkVersion().effectiveVersion(ctx); err == nil { + a.checkJniLibsSdkVersion(ctx, minSdkVersion) + } else { + ctx.PropertyErrorf("min_sdk_version", "%s", err.Error()) + } + } + + a.checkPlatformAPI(ctx) + a.checkSdkVersions(ctx) +} + +// If an updatable APK sets min_sdk_version, min_sdk_vesion of JNI libs should match with it. +// This check is enforced for "updatable" APKs (including APK-in-APEX). +// b/155209650: until min_sdk_version is properly supported, use sdk_version instead. +// because, sdk_version is overridden by min_sdk_version (if set as smaller) +// and linkType is checked with dependencies so we can be sure that the whole dependency tree +// will meet the requirements. +func (a *AndroidApp) checkJniLibsSdkVersion(ctx android.ModuleContext, minSdkVersion sdkVersion) { + // It's enough to check direct JNI deps' sdk_version because all transitive deps from JNI deps are checked in cc.checkLinkType() + ctx.VisitDirectDeps(func(m android.Module) { + if !IsJniDepTag(ctx.OtherModuleDependencyTag(m)) { + return + } + dep, _ := m.(*cc.Module) + // The domain of cc.sdk_version is "current" and <number> + // We can rely on sdkSpec to convert it to <number> so that "current" is handled + // properly regardless of sdk finalization. + jniSdkVersion, err := sdkSpecFrom(dep.SdkVersion()).effectiveVersion(ctx) + if err != nil || minSdkVersion < jniSdkVersion { + ctx.OtherModuleErrorf(dep, "sdk_version(%v) is higher than min_sdk_version(%v) of the containing android_app(%v)", + dep.SdkVersion(), minSdkVersion, ctx.ModuleName()) + return + } + + }) +} + +// Returns true if the native libraries should be stored in the APK uncompressed and the // extractNativeLibs application flag should be set to false in the manifest. -func (a *AndroidApp) shouldUncompressJNI(ctx android.ModuleContext) bool { - minSdkVersion, err := sdkVersionToNumber(ctx, a.minSdkVersion()) +func (a *AndroidApp) useEmbeddedNativeLibs(ctx android.ModuleContext) bool { + minSdkVersion, err := a.minSdkVersion().effectiveVersion(ctx) if err != nil { ctx.PropertyErrorf("min_sdk_version", "invalid value %q: %s", a.minSdkVersion(), err) } - return minSdkVersion >= 23 && Bool(a.appProperties.Use_embedded_native_libs) + return (minSdkVersion >= 23 && Bool(a.appProperties.Use_embedded_native_libs)) || + !a.IsForPlatform() } // Returns whether this module should have the dex file stored uncompressed in the APK. @@ -315,11 +486,9 @@ func (a *AndroidApp) shouldUncompressDex(ctx android.ModuleContext) bool { return true } - // Uncompress dex in APKs of privileged apps, and modules used by privileged apps - // (even for unbundled builds, they may be preinstalled as prebuilts). - if ctx.Config().UncompressPrivAppDex() && - (Bool(a.appProperties.Privileged) || - inList(ctx.ModuleName(), ctx.Config().ModulesLoadedByPrivilegedModules())) { + // Uncompress dex in APKs of privileged apps (even for unbundled builds, they may + // be preinstalled as prebuilts). + if ctx.Config().UncompressPrivAppDex() && a.Privileged() { return true } @@ -327,27 +496,28 @@ func (a *AndroidApp) shouldUncompressDex(ctx android.ModuleContext) bool { return false } - // Uncompress if the dex files is preopted on /system. - if !a.dexpreopter.dexpreoptDisabled(ctx) && (ctx.Host() || !odexOnSystemOther(ctx, a.dexpreopter.installPath)) { - return true - } + return shouldUncompressDex(ctx, &a.dexpreopter) +} + +func (a *AndroidApp) shouldEmbedJnis(ctx android.BaseModuleContext) bool { + return ctx.Config().UnbundledBuild() || Bool(a.appProperties.Use_embedded_native_libs) || + !a.IsForPlatform() || a.appProperties.AlwaysPackageNativeLibs +} - return false +func (a *AndroidApp) OverriddenManifestPackageName() string { + return a.overriddenManifestPackageName } func (a *AndroidApp) aaptBuildActions(ctx android.ModuleContext) { a.aapt.usesNonSdkApis = Bool(a.Module.deviceProperties.Platform_apis) + // Ask manifest_fixer to add or update the application element indicating this app has no code. + a.aapt.hasNoCode = !a.hasCode(ctx) + aaptLinkFlags := []string{} // Add TARGET_AAPT_CHARACTERISTICS values to AAPT link flags if they exist and --product flags were not provided. - hasProduct := false - for _, f := range a.aaptProperties.Aaptflags { - if strings.HasPrefix(f, "--product") { - hasProduct = true - break - } - } + hasProduct := android.PrefixInList(a.aaptProperties.Aaptflags, "--product") if !hasProduct && len(ctx.Config().ProductAAPTCharacteristics()) > 0 { aaptLinkFlags = append(aaptLinkFlags, "--product", ctx.Config().ProductAAPTCharacteristics()) } @@ -371,12 +541,14 @@ func (a *AndroidApp) aaptBuildActions(ctx android.ModuleContext) { manifestPackageName = *a.overridableAppProperties.Package_name } aaptLinkFlags = append(aaptLinkFlags, "--rename-manifest-package "+manifestPackageName) + a.overriddenManifestPackageName = manifestPackageName } aaptLinkFlags = append(aaptLinkFlags, a.additionalAaptFlags...) a.aapt.splitNames = a.appProperties.Package_splits - + a.aapt.sdkLibraries = a.exportedSdkLibs + a.aapt.LoggingParent = String(a.overridableAppProperties.Logging_parent) a.aapt.buildActions(ctx, sdkContext(a), aaptLinkFlags...) // apps manifests are handled by aapt, don't let Module see them @@ -397,21 +569,32 @@ func (a *AndroidApp) proguardBuildActions(ctx android.ModuleContext) { a.Module.extraProguardFlagFiles = append(a.Module.extraProguardFlagFiles, a.proguardOptionsFile) } -func (a *AndroidApp) dexBuildActions(ctx android.ModuleContext) android.Path { - +func (a *AndroidApp) installPath(ctx android.ModuleContext) android.InstallPath { var installDir string if ctx.ModuleName() == "framework-res" { // framework-res.apk is installed as system/framework/framework-res.apk installDir = "framework" - } else if Bool(a.appProperties.Privileged) { + } else if a.Privileged() { installDir = filepath.Join("priv-app", a.installApkName) } else { installDir = filepath.Join("app", a.installApkName) } - a.dexpreopter.installPath = android.PathForModuleInstall(ctx, installDir, a.installApkName+".apk") - a.dexpreopter.isInstallable = Bool(a.properties.Installable) - a.dexpreopter.uncompressedDex = a.shouldUncompressDex(ctx) - a.deviceProperties.UncompressDex = a.dexpreopter.uncompressedDex + + return android.PathForModuleInstall(ctx, installDir, a.installApkName+".apk") +} + +func (a *AndroidApp) dexBuildActions(ctx android.ModuleContext) android.Path { + a.dexpreopter.installPath = a.installPath(ctx) + if a.deviceProperties.Uncompress_dex == nil { + // If the value was not force-set by the user, use reasonable default based on the module. + a.deviceProperties.Uncompress_dex = proptools.BoolPtr(a.shouldUncompressDex(ctx)) + } + a.dexpreopter.uncompressedDex = *a.deviceProperties.Uncompress_dex + a.dexpreopter.enforceUsesLibs = a.usesLibrary.enforceUsesLibraries() + a.dexpreopter.usesLibs = a.usesLibrary.usesLibraryProperties.Uses_libs + a.dexpreopter.optionalUsesLibs = a.usesLibrary.presentOptionalUsesLibs(ctx) + a.dexpreopter.libraryPaths = a.usesLibrary.usesLibraryPaths(ctx) + a.dexpreopter.manifestFile = a.mergedManifestFile if ctx.ModuleName() != "framework-res" { a.Module.compile(ctx, a.aaptSrcJar) @@ -423,77 +606,64 @@ func (a *AndroidApp) dexBuildActions(ctx android.ModuleContext) android.Path { func (a *AndroidApp) jniBuildActions(jniLibs []jniLib, ctx android.ModuleContext) android.WritablePath { var jniJarFile android.WritablePath if len(jniLibs) > 0 { - embedJni := ctx.Config().UnbundledBuild() || Bool(a.appProperties.Use_embedded_native_libs) || - a.appProperties.AlwaysPackageNativeLibs - if embedJni { + a.jniLibs = jniLibs + if a.shouldEmbedJnis(ctx) { jniJarFile = android.PathForModuleOut(ctx, "jnilibs.zip") - TransformJniLibsToJar(ctx, jniJarFile, jniLibs, a.shouldUncompressJNI(ctx)) - } else { - a.installJniLibs = jniLibs + a.installPathForJNISymbols = a.installPath(ctx).ToMakePath() + TransformJniLibsToJar(ctx, jniJarFile, jniLibs, a.useEmbeddedNativeLibs(ctx)) + for _, jni := range jniLibs { + if jni.coverageFile.Valid() { + // Only collect coverage for the first target arch if this is a multilib target. + // TODO(jungjw): Ideally, we want to collect both reports, but that would cause coverage + // data file path collisions since the current coverage file path format doesn't contain + // arch-related strings. This is fine for now though; the code coverage team doesn't use + // multi-arch targets such as test_suite_* for coverage collections yet. + // + // Work with the team to come up with a new format that handles multilib modules properly + // and change this. + if len(ctx.Config().Targets[android.Android]) == 1 || + ctx.Config().Targets[android.Android][0].Arch.ArchType == jni.target.Arch.ArchType { + a.jniCoverageOutputs = append(a.jniCoverageOutputs, jni.coverageFile.Path()) + } + } + } + a.embeddedJniLibs = true } } return jniJarFile } -func (a *AndroidApp) certificateBuildActions(certificateDeps []Certificate, ctx android.ModuleContext) []Certificate { - cert := a.getCertString(ctx) - certModule := android.SrcIsModule(cert) - if certModule != "" { - a.certificate = certificateDeps[0] - certificateDeps = certificateDeps[1:] - } else if cert != "" { - defaultDir := ctx.Config().DefaultAppCertificateDir(ctx) - a.certificate = Certificate{ - defaultDir.Join(ctx, cert+".x509.pem"), - defaultDir.Join(ctx, cert+".pk8"), +func (a *AndroidApp) JNISymbolsInstalls(installPath string) android.RuleBuilderInstalls { + var jniSymbols android.RuleBuilderInstalls + for _, jniLib := range a.jniLibs { + if jniLib.unstrippedFile != nil { + jniSymbols = append(jniSymbols, android.RuleBuilderInstall{ + From: jniLib.unstrippedFile, + To: filepath.Join(installPath, targetToJniDir(jniLib.target), jniLib.unstrippedFile.Base()), + }) } - } else { - pem, key := ctx.Config().DefaultAppCertificate(ctx) - a.certificate = Certificate{pem, key} } - - if !a.Module.Platform() { - certPath := a.certificate.Pem.String() - systemCertPath := ctx.Config().DefaultAppCertificateDir(ctx).String() - if strings.HasPrefix(certPath, systemCertPath) { - enforceSystemCert := ctx.Config().EnforceSystemCertificate() - whitelist := ctx.Config().EnforceSystemCertificateWhitelist() - - if enforceSystemCert && !inList(a.Module.Name(), whitelist) { - ctx.PropertyErrorf("certificate", "The module in product partition cannot be signed with certificate in system.") - } - } - } - - return append([]Certificate{a.certificate}, certificateDeps...) + return jniSymbols } -func (a *AndroidApp) noticeBuildActions(ctx android.ModuleContext, installDir android.OutputPath) android.OptionalPath { - if !Bool(a.appProperties.Embed_notices) && !ctx.Config().IsEnvTrue("ALWAYS_EMBED_NOTICES") { - return android.OptionalPath{} - } - +func (a *AndroidApp) noticeBuildActions(ctx android.ModuleContext) { // Collect NOTICE files from all dependencies. seenModules := make(map[android.Module]bool) noticePathSet := make(map[android.Path]bool) - ctx.WalkDepsBlueprint(func(child blueprint.Module, parent blueprint.Module) bool { - if _, ok := child.(android.Module); !ok { - return false - } - module := child.(android.Module) + ctx.WalkDeps(func(child android.Module, parent android.Module) bool { // Have we already seen this? - if _, ok := seenModules[module]; ok { + if _, ok := seenModules[child]; ok { return false } - seenModules[module] = true + seenModules[child] = true // Skip host modules. - if module.Target().Os.Class == android.Host || module.Target().Os.Class == android.HostCross { + if child.Target().Os.Class == android.Host || child.Target().Os.Class == android.HostCross { return false } - path := module.NoticeFile() + path := child.(android.Module).NoticeFile() if path.Valid() { noticePathSet[path.Path()] = true } @@ -506,7 +676,7 @@ func (a *AndroidApp) noticeBuildActions(ctx android.ModuleContext, installDir an } if len(noticePathSet) == 0 { - return android.OptionalPath{} + return } var noticePaths []android.Path for path := range noticePathSet { @@ -515,54 +685,132 @@ func (a *AndroidApp) noticeBuildActions(ctx android.ModuleContext, installDir an sort.Slice(noticePaths, func(i, j int) bool { return noticePaths[i].String() < noticePaths[j].String() }) - noticeFile := android.BuildNoticeOutput(ctx, installDir, a.installApkName+".apk", noticePaths) - return android.OptionalPathForPath(noticeFile) + a.noticeOutputs = android.BuildNoticeOutput(ctx, a.installDir, a.installApkName+".apk", noticePaths) +} + +// Reads and prepends a main cert from the default cert dir if it hasn't been set already, i.e. it +// isn't a cert module reference. Also checks and enforces system cert restriction if applicable. +func processMainCert(m android.ModuleBase, certPropValue string, certificates []Certificate, ctx android.ModuleContext) []Certificate { + if android.SrcIsModule(certPropValue) == "" { + var mainCert Certificate + if certPropValue != "" { + defaultDir := ctx.Config().DefaultAppCertificateDir(ctx) + mainCert = Certificate{ + Pem: defaultDir.Join(ctx, certPropValue+".x509.pem"), + Key: defaultDir.Join(ctx, certPropValue+".pk8"), + } + } else { + pem, key := ctx.Config().DefaultAppCertificate(ctx) + mainCert = Certificate{ + Pem: pem, + Key: key, + } + } + certificates = append([]Certificate{mainCert}, certificates...) + } + + if !m.Platform() { + certPath := certificates[0].Pem.String() + systemCertPath := ctx.Config().DefaultAppCertificateDir(ctx).String() + if strings.HasPrefix(certPath, systemCertPath) { + enforceSystemCert := ctx.Config().EnforceSystemCertificate() + allowed := ctx.Config().EnforceSystemCertificateAllowList() + + if enforceSystemCert && !inList(m.Name(), allowed) { + ctx.PropertyErrorf("certificate", "The module in product partition cannot be signed with certificate in system.") + } + } + } + + return certificates +} + +func (a *AndroidApp) InstallApkName() string { + return a.installApkName } func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) { + var apkDeps android.Paths + + a.aapt.useEmbeddedNativeLibs = a.useEmbeddedNativeLibs(ctx) + a.aapt.useEmbeddedDex = Bool(a.appProperties.Use_embedded_dex) + // Check if the install APK name needs to be overridden. a.installApkName = ctx.DeviceConfig().OverridePackageNameFor(a.Name()) - var installDir android.OutputPath if ctx.ModuleName() == "framework-res" { // framework-res.apk is installed as system/framework/framework-res.apk - installDir = android.PathForModuleInstall(ctx, "framework") - } else if Bool(a.appProperties.Privileged) { - installDir = android.PathForModuleInstall(ctx, "priv-app", a.installApkName) + a.installDir = android.PathForModuleInstall(ctx, "framework") + } else if a.Privileged() { + a.installDir = android.PathForModuleInstall(ctx, "priv-app", a.installApkName) + } else if ctx.InstallInTestcases() { + a.installDir = android.PathForModuleInstall(ctx, a.installApkName, ctx.DeviceConfig().DeviceArch()) } else { - installDir = android.PathForModuleInstall(ctx, "app", a.installApkName) + a.installDir = android.PathForModuleInstall(ctx, "app", a.installApkName) } + a.onDeviceDir = android.InstallPathToOnDevicePath(ctx, a.installDir) - a.aapt.noticeFile = a.noticeBuildActions(ctx, installDir) + a.noticeBuildActions(ctx) + if Bool(a.appProperties.Embed_notices) || ctx.Config().IsEnvTrue("ALWAYS_EMBED_NOTICES") { + a.aapt.noticeFile = a.noticeOutputs.HtmlGzOutput + } // Process all building blocks, from AAPT to certificates. a.aaptBuildActions(ctx) + if a.usesLibrary.enforceUsesLibraries() { + manifestCheckFile := a.usesLibrary.verifyUsesLibrariesManifest(ctx, a.mergedManifestFile) + apkDeps = append(apkDeps, manifestCheckFile) + } + a.proguardBuildActions(ctx) + a.linter.mergedManifest = a.aapt.mergedManifestFile + a.linter.manifest = a.aapt.manifestPath + a.linter.resources = a.aapt.resourceFiles + a.linter.buildModuleReportZip = ctx.Config().UnbundledBuild() + dexJarFile := a.dexBuildActions(ctx) - jniLibs, certificateDeps := a.collectAppDeps(ctx) + jniLibs, certificateDeps := collectAppDeps(ctx, a, a.shouldEmbedJnis(ctx), !Bool(a.appProperties.Jni_uses_platform_apis)) jniJarFile := a.jniBuildActions(jniLibs, ctx) if ctx.Failed() { return } - certificates := a.certificateBuildActions(certificateDeps, ctx) + certificates := processMainCert(a.ModuleBase, a.getCertString(ctx), certificateDeps, ctx) + a.certificate = certificates[0] // Build a final signed app package. - // TODO(jungjw): Consider changing this to installApkName. - packageFile := android.PathForModuleOut(ctx, ctx.ModuleName()+".apk") - CreateAppPackage(ctx, packageFile, a.exportPackage, jniJarFile, dexJarFile, certificates) + packageFile := android.PathForModuleOut(ctx, a.installApkName+".apk") + v4SigningRequested := Bool(a.Module.deviceProperties.V4_signature) + var v4SignatureFile android.WritablePath = nil + if v4SigningRequested { + v4SignatureFile = android.PathForModuleOut(ctx, a.installApkName+".apk.idsig") + } + var lineageFile android.Path + if lineage := String(a.overridableAppProperties.Lineage); lineage != "" { + lineageFile = android.PathForModuleSrc(ctx, lineage) + } + CreateAndSignAppPackage(ctx, packageFile, a.exportPackage, jniJarFile, dexJarFile, certificates, apkDeps, v4SignatureFile, lineageFile) a.outputFile = packageFile + if v4SigningRequested { + a.extraOutputFiles = append(a.extraOutputFiles, v4SignatureFile) + } for _, split := range a.aapt.splits { // Sign the split APKs - packageFile := android.PathForModuleOut(ctx, ctx.ModuleName()+"_"+split.suffix+".apk") - CreateAppPackage(ctx, packageFile, split.path, nil, nil, certificates) + packageFile := android.PathForModuleOut(ctx, a.installApkName+"_"+split.suffix+".apk") + if v4SigningRequested { + v4SignatureFile = android.PathForModuleOut(ctx, a.installApkName+"_"+split.suffix+".apk.idsig") + } + CreateAndSignAppPackage(ctx, packageFile, split.path, nil, nil, certificates, apkDeps, v4SignatureFile, lineageFile) a.extraOutputFiles = append(a.extraOutputFiles, packageFile) + if v4SigningRequested { + a.extraOutputFiles = append(a.extraOutputFiles, v4SignatureFile) + } } // Build an app bundle. @@ -571,49 +819,137 @@ func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) { a.bundleFile = bundleFile // Install the app package. - ctx.InstallFile(installDir, a.installApkName+".apk", a.outputFile) - for _, split := range a.aapt.splits { - ctx.InstallFile(installDir, a.installApkName+"_"+split.suffix+".apk", split.path) + if (Bool(a.Module.properties.Installable) || ctx.Host()) && a.IsForPlatform() { + ctx.InstallFile(a.installDir, a.outputFile.Base(), a.outputFile) + for _, extra := range a.extraOutputFiles { + ctx.InstallFile(a.installDir, extra.Base(), extra) + } } + + a.buildAppDependencyInfo(ctx) } -func (a *AndroidApp) collectAppDeps(ctx android.ModuleContext) ([]jniLib, []Certificate) { +type appDepsInterface interface { + sdkVersion() sdkSpec + minSdkVersion() sdkSpec + RequiresStableAPIs(ctx android.BaseModuleContext) bool +} + +func collectAppDeps(ctx android.ModuleContext, app appDepsInterface, + shouldCollectRecursiveNativeDeps bool, + checkNativeSdkVersion bool) ([]jniLib, []Certificate) { + var jniLibs []jniLib var certificates []Certificate + seenModulePaths := make(map[string]bool) + + if checkNativeSdkVersion { + checkNativeSdkVersion = app.sdkVersion().specified() && + app.sdkVersion().kind != sdkCorePlatform && !app.RequiresStableAPIs(ctx) + } - ctx.VisitDirectDeps(func(module android.Module) { + ctx.WalkDeps(func(module android.Module, parent android.Module) bool { otherName := ctx.OtherModuleName(module) tag := ctx.OtherModuleDependencyTag(module) - if jniTag, ok := tag.(*jniDependencyTag); ok { + if IsJniDepTag(tag) || tag == cc.SharedDepTag { if dep, ok := module.(*cc.Module); ok { + if dep.IsNdk() || dep.IsStubs() { + return false + } + lib := dep.OutputFile() + path := lib.Path() + if seenModulePaths[path.String()] { + return false + } + seenModulePaths[path.String()] = true + + if checkNativeSdkVersion && dep.SdkVersion() == "" { + ctx.PropertyErrorf("jni_libs", "JNI dependency %q uses platform APIs, but this module does not", + otherName) + } + if lib.Valid() { jniLibs = append(jniLibs, jniLib{ - name: ctx.OtherModuleName(module), - path: lib.Path(), - target: jniTag.target, + name: ctx.OtherModuleName(module), + path: path, + target: module.Target(), + coverageFile: dep.CoverageOutputFile(), + unstrippedFile: dep.UnstrippedOutputFile(), }) } else { ctx.ModuleErrorf("dependency %q missing output file", otherName) } } else { ctx.ModuleErrorf("jni_libs dependency %q must be a cc library", otherName) - } - } else if tag == certificateTag { + + return shouldCollectRecursiveNativeDeps + } + + if tag == certificateTag { if dep, ok := module.(*AndroidAppCertificate); ok { certificates = append(certificates, dep.Certificate) } else { ctx.ModuleErrorf("certificate dependency %q must be an android_app_certificate module", otherName) } } + + return false }) return jniLibs, certificates } -func (a *AndroidApp) getCertString(ctx android.BaseContext) string { +func (a *AndroidApp) walkPayloadDeps(ctx android.ModuleContext, + do func(ctx android.ModuleContext, from blueprint.Module, to android.ApexModule, externalDep bool)) { + + ctx.WalkDeps(func(child, parent android.Module) bool { + isExternal := !a.DepIsInSameApex(ctx, child) + if am, ok := child.(android.ApexModule); ok { + do(ctx, parent, am, isExternal) + } + return !isExternal + }) +} + +func (a *AndroidApp) buildAppDependencyInfo(ctx android.ModuleContext) { + if ctx.Host() { + return + } + + depsInfo := android.DepNameToDepInfoMap{} + a.walkPayloadDeps(ctx, func(ctx android.ModuleContext, from blueprint.Module, to android.ApexModule, externalDep bool) { + depName := to.Name() + if info, exist := depsInfo[depName]; exist { + info.From = append(info.From, from.Name()) + info.IsExternal = info.IsExternal && externalDep + depsInfo[depName] = info + } else { + toMinSdkVersion := "(no version)" + if m, ok := to.(interface{ MinSdkVersion() string }); ok { + if v := m.MinSdkVersion(); v != "" { + toMinSdkVersion = v + } + } + depsInfo[depName] = android.ApexModuleDepInfo{ + To: depName, + From: []string{from.Name()}, + IsExternal: externalDep, + MinSdkVersion: toMinSdkVersion, + } + } + }) + + a.ApexBundleDepsInfo.BuildDepsInfoLists(ctx, a.MinSdkVersion(), depsInfo) +} + +func (a *AndroidApp) Updatable() bool { + return Bool(a.appProperties.Updatable) || a.ApexModuleBase.Updatable() +} + +func (a *AndroidApp) getCertString(ctx android.BaseModuleContext) string { certificate, overridden := ctx.DeviceConfig().OverrideCertificateFor(ctx.ModuleName()) if overridden { return ":" + certificate @@ -621,6 +957,46 @@ func (a *AndroidApp) getCertString(ctx android.BaseContext) string { return String(a.overridableAppProperties.Certificate) } +func (a *AndroidApp) DepIsInSameApex(ctx android.BaseModuleContext, dep android.Module) bool { + if IsJniDepTag(ctx.OtherModuleDependencyTag(dep)) { + return true + } + return a.Library.DepIsInSameApex(ctx, dep) +} + +// For OutputFileProducer interface +func (a *AndroidApp) OutputFiles(tag string) (android.Paths, error) { + switch tag { + case ".aapt.srcjar": + return []android.Path{a.aaptSrcJar}, nil + case ".export-package.apk": + return []android.Path{a.exportPackage}, nil + } + return a.Library.OutputFiles(tag) +} + +func (a *AndroidApp) Privileged() bool { + return Bool(a.appProperties.Privileged) +} + +func (a *AndroidApp) IsNativeCoverageNeeded(ctx android.BaseModuleContext) bool { + return ctx.Device() && ctx.DeviceConfig().NativeCoverageEnabled() +} + +func (a *AndroidApp) PreventInstall() { + a.appProperties.PreventInstall = true +} + +func (a *AndroidApp) HideFromMake() { + a.appProperties.HideFromMake = true +} + +func (a *AndroidApp) MarkAsCoverageVariant(coverage bool) { + a.appProperties.IsCoverageVariant = coverage +} + +var _ cc.Coverage = (*AndroidApp)(nil) + // android_app compiles sources and Android resources into an Android application package `.apk` file. func AndroidAppFactory() android.Module { module := &AndroidApp{} @@ -631,14 +1007,12 @@ func AndroidAppFactory() android.Module { module.Module.properties.Instrument = true module.Module.properties.Installable = proptools.BoolPtr(true) + module.addHostAndDeviceProperties() module.AddProperties( - &module.Module.properties, - &module.Module.deviceProperties, - &module.Module.dexpreoptProperties, - &module.Module.protoProperties, &module.aaptProperties, &module.appProperties, - &module.overridableAppProperties) + &module.overridableAppProperties, + &module.usesLibrary.usesLibraryProperties) module.Prefer32(func(ctx android.BaseModuleContext, base *android.ModuleBase, class android.OsClass) bool { return class == android.Device && ctx.Config().DevicePrefer32BitApps() @@ -647,12 +1021,16 @@ func AndroidAppFactory() android.Module { android.InitAndroidMultiTargetsArchModule(module, android.DeviceSupported, android.MultilibCommon) android.InitDefaultableModule(module) android.InitOverridableModule(module, &module.appProperties.Overrides) + android.InitApexModule(module) return module } type appTestProperties struct { Instrumentation_for *string + + // if specified, the instrumentation target package name in the manifest is overwritten by it. + Instrumentation_target_package *string } type AndroidTest struct { @@ -666,9 +1044,17 @@ type AndroidTest struct { data android.Paths } +func (a *AndroidTest) InstallInTestcases() bool { + return true +} + func (a *AndroidTest) GenerateAndroidBuildActions(ctx android.ModuleContext) { - // Check if the instrumentation target package is overridden before generating build actions. - if a.appTestProperties.Instrumentation_for != nil { + var configs []tradefed.Config + if a.appTestProperties.Instrumentation_target_package != nil { + a.additionalAaptFlags = append(a.additionalAaptFlags, + "--rename-instrumentation-target-package "+*a.appTestProperties.Instrumentation_target_package) + } else if a.appTestProperties.Instrumentation_for != nil { + // Check if the instrumentation target package is overridden. manifestPackageName, overridden := ctx.DeviceConfig().OverrideManifestPackageNameFor(*a.appTestProperties.Instrumentation_for) if overridden { a.additionalAaptFlags = append(a.additionalAaptFlags, "--rename-instrumentation-target-package "+manifestPackageName) @@ -676,12 +1062,50 @@ func (a *AndroidTest) GenerateAndroidBuildActions(ctx android.ModuleContext) { } a.generateAndroidBuildActions(ctx) - a.testConfig = tradefed.AutoGenInstrumentationTestConfig(ctx, a.testProperties.Test_config, a.testProperties.Test_config_template, a.manifestPath, a.testProperties.Test_suites) + for _, module := range a.testProperties.Test_mainline_modules { + configs = append(configs, tradefed.Option{Name: "config-descriptor:metadata", Key: "mainline-param", Value: module}) + } + + testConfig := tradefed.AutoGenInstrumentationTestConfig(ctx, a.testProperties.Test_config, + a.testProperties.Test_config_template, a.manifestPath, a.testProperties.Test_suites, a.testProperties.Auto_gen_config, configs) + a.testConfig = a.FixTestConfig(ctx, testConfig) a.data = android.PathsForModuleSrc(ctx, a.testProperties.Data) } +func (a *AndroidTest) FixTestConfig(ctx android.ModuleContext, testConfig android.Path) android.Path { + if testConfig == nil { + return nil + } + + fixedConfig := android.PathForModuleOut(ctx, "test_config_fixer", "AndroidTest.xml") + rule := android.NewRuleBuilder() + command := rule.Command().BuiltTool(ctx, "test_config_fixer").Input(testConfig).Output(fixedConfig) + fixNeeded := false + + if ctx.ModuleName() != a.installApkName { + fixNeeded = true + command.FlagWithArg("--test-file-name ", a.installApkName+".apk") + } + + if a.overridableAppProperties.Package_name != nil { + fixNeeded = true + command.FlagWithInput("--manifest ", a.manifestPath). + FlagWithArg("--package-name ", *a.overridableAppProperties.Package_name) + } + + if fixNeeded { + rule.Build(pctx, ctx, "fix_test_config", "fix test config") + return fixedConfig + } + return testConfig +} + func (a *AndroidTest) DepsMutator(ctx android.BottomUpMutatorContext) { a.AndroidApp.DepsMutator(ctx) +} + +func (a *AndroidTest) OverridablePropertiesDepsMutator(ctx android.BottomUpMutatorContext) { + a.AndroidApp.OverridablePropertiesDepsMutator(ctx) if a.appTestProperties.Instrumentation_for != nil { // The android_app dependency listed in instrumentation_for needs to be added to the classpath for javac, // but not added to the aapt2 link includes like a normal android_app or android_library dependency, so @@ -702,20 +1126,20 @@ func AndroidTestFactory() android.Module { module.appProperties.Use_embedded_native_libs = proptools.BoolPtr(true) module.appProperties.AlwaysPackageNativeLibs = true module.Module.dexpreopter.isTest = true + module.Module.linter.test = true + module.addHostAndDeviceProperties() module.AddProperties( - &module.Module.properties, - &module.Module.deviceProperties, - &module.Module.dexpreoptProperties, - &module.Module.protoProperties, &module.aaptProperties, &module.appProperties, &module.appTestProperties, &module.overridableAppProperties, + &module.usesLibrary.usesLibraryProperties, &module.testProperties) android.InitAndroidMultiTargetsArchModule(module, android.DeviceSupported, android.MultilibCommon) android.InitDefaultableModule(module) + android.InitOverridableModule(module, &module.appProperties.Overrides) return module } @@ -723,6 +1147,11 @@ type appTestHelperAppProperties struct { // list of compatibility suites (for example "cts", "vts") that the module should be // installed into. Test_suites []string `android:"arch_variant"` + + // Flag to indicate whether or not to create test config automatically. If AndroidTest.xml + // doesn't exist next to the Android.bp, this attribute doesn't need to be set to true + // explicitly. + Auto_gen_config *bool } type AndroidTestHelperApp struct { @@ -731,6 +1160,10 @@ type AndroidTestHelperApp struct { appTestHelperAppProperties appTestHelperAppProperties } +func (a *AndroidTestHelperApp) InstallInTestcases() bool { + return true +} + // android_test_helper_app compiles sources and Android resources into an Android application package `.apk` file that // will be used by tests, but does not produce an `AndroidTest.xml` file so the module will not be run directly as a // test. @@ -743,19 +1176,19 @@ func AndroidTestHelperAppFactory() android.Module { module.appProperties.Use_embedded_native_libs = proptools.BoolPtr(true) module.appProperties.AlwaysPackageNativeLibs = true module.Module.dexpreopter.isTest = true + module.Module.linter.test = true + module.addHostAndDeviceProperties() module.AddProperties( - &module.Module.properties, - &module.Module.deviceProperties, - &module.Module.dexpreoptProperties, - &module.Module.protoProperties, &module.aaptProperties, &module.appProperties, &module.appTestHelperAppProperties, - &module.overridableAppProperties) + &module.overridableAppProperties, + &module.usesLibrary.usesLibraryProperties) android.InitAndroidMultiTargetsArchModule(module, android.DeviceSupported, android.MultilibCommon) android.InitDefaultableModule(module) + android.InitApexModule(module) return module } @@ -782,8 +1215,8 @@ func AndroidAppCertificateFactory() android.Module { func (c *AndroidAppCertificate) GenerateAndroidBuildActions(ctx android.ModuleContext) { cert := String(c.properties.Certificate) c.Certificate = Certificate{ - android.PathForModuleSrc(ctx, cert+".x509.pem"), - android.PathForModuleSrc(ctx, cert+".pk8"), + Pem: android.PathForModuleSrc(ctx, cert+".x509.pem"), + Key: android.PathForModuleSrc(ctx, cert+".pk8"), } } @@ -792,7 +1225,7 @@ type OverrideAndroidApp struct { android.OverrideModuleBase } -func (i *OverrideAndroidApp) GenerateAndroidBuildActions(ctx android.ModuleContext) { +func (i *OverrideAndroidApp) GenerateAndroidBuildActions(_ android.ModuleContext) { // All the overrides happen in the base module. // TODO(jungjw): Check the base module type. } @@ -803,7 +1236,723 @@ func OverrideAndroidAppModuleFactory() android.Module { m := &OverrideAndroidApp{} m.AddProperties(&overridableAppProperties{}) - android.InitAndroidModule(m) + android.InitAndroidMultiTargetsArchModule(m, android.DeviceSupported, android.MultilibCommon) android.InitOverrideModule(m) return m } + +type OverrideAndroidTest struct { + android.ModuleBase + android.OverrideModuleBase +} + +func (i *OverrideAndroidTest) GenerateAndroidBuildActions(_ android.ModuleContext) { + // All the overrides happen in the base module. + // TODO(jungjw): Check the base module type. +} + +// override_android_test is used to create an android_app module based on another android_test by overriding +// some of its properties. +func OverrideAndroidTestModuleFactory() android.Module { + m := &OverrideAndroidTest{} + m.AddProperties(&overridableAppProperties{}) + m.AddProperties(&appTestProperties{}) + + android.InitAndroidMultiTargetsArchModule(m, android.DeviceSupported, android.MultilibCommon) + android.InitOverrideModule(m) + return m +} + +type OverrideRuntimeResourceOverlay struct { + android.ModuleBase + android.OverrideModuleBase +} + +func (i *OverrideRuntimeResourceOverlay) GenerateAndroidBuildActions(_ android.ModuleContext) { + // All the overrides happen in the base module. + // TODO(jungjw): Check the base module type. +} + +// override_runtime_resource_overlay is used to create a module based on another +// runtime_resource_overlay module by overriding some of its properties. +func OverrideRuntimeResourceOverlayModuleFactory() android.Module { + m := &OverrideRuntimeResourceOverlay{} + m.AddProperties(&OverridableRuntimeResourceOverlayProperties{}) + + android.InitAndroidMultiTargetsArchModule(m, android.DeviceSupported, android.MultilibCommon) + android.InitOverrideModule(m) + return m +} + +type AndroidAppImport struct { + android.ModuleBase + android.DefaultableModuleBase + prebuilt android.Prebuilt + + properties AndroidAppImportProperties + dpiVariants interface{} + archVariants interface{} + + outputFile android.Path + certificate Certificate + + dexpreopter + + usesLibrary usesLibrary + + preprocessed bool + + installPath android.InstallPath +} + +type AndroidAppImportProperties struct { + // A prebuilt apk to import + Apk *string + + // The name of a certificate in the default certificate directory or an android_app_certificate + // module name in the form ":module". Should be empty if presigned or default_dev_cert is set. + Certificate *string + + // Set this flag to true if the prebuilt apk is already signed. The certificate property must not + // be set for presigned modules. + Presigned *bool + + // Name of the signing certificate lineage file. + Lineage *string + + // Sign with the default system dev certificate. Must be used judiciously. Most imported apps + // need to either specify a specific certificate or be presigned. + Default_dev_cert *bool + + // Specifies that this app should be installed to the priv-app directory, + // where the system will grant it additional privileges not available to + // normal apps. + Privileged *bool + + // Names of modules to be overridden. Listed modules can only be other binaries + // (in Make or Soong). + // This does not completely prevent installation of the overridden binaries, but if both + // binaries would be installed by default (in PRODUCT_PACKAGES) the other binary will be removed + // from PRODUCT_PACKAGES. + Overrides []string + + // Optional name for the installed app. If unspecified, it is derived from the module name. + Filename *string +} + +func (a *AndroidAppImport) IsInstallable() bool { + return true +} + +// Updates properties with variant-specific values. +func (a *AndroidAppImport) processVariants(ctx android.LoadHookContext) { + config := ctx.Config() + + dpiProps := reflect.ValueOf(a.dpiVariants).Elem().FieldByName("Dpi_variants") + // Try DPI variant matches in the reverse-priority order so that the highest priority match + // overwrites everything else. + // TODO(jungjw): Can we optimize this by making it priority order? + for i := len(config.ProductAAPTPrebuiltDPI()) - 1; i >= 0; i-- { + MergePropertiesFromVariant(ctx, &a.properties, dpiProps, config.ProductAAPTPrebuiltDPI()[i]) + } + if config.ProductAAPTPreferredConfig() != "" { + MergePropertiesFromVariant(ctx, &a.properties, dpiProps, config.ProductAAPTPreferredConfig()) + } + + archProps := reflect.ValueOf(a.archVariants).Elem().FieldByName("Arch") + archType := ctx.Config().Targets[android.Android][0].Arch.ArchType + MergePropertiesFromVariant(ctx, &a.properties, archProps, archType.Name) +} + +func MergePropertiesFromVariant(ctx android.EarlyModuleContext, + dst interface{}, variantGroup reflect.Value, variant string) { + src := variantGroup.FieldByName(proptools.FieldNameForProperty(variant)) + if !src.IsValid() { + return + } + + err := proptools.ExtendMatchingProperties([]interface{}{dst}, src.Interface(), nil, proptools.OrderAppend) + if err != nil { + if propertyErr, ok := err.(*proptools.ExtendPropertyError); ok { + ctx.PropertyErrorf(propertyErr.Property, "%s", propertyErr.Err.Error()) + } else { + panic(err) + } + } +} + +func (a *AndroidAppImport) DepsMutator(ctx android.BottomUpMutatorContext) { + cert := android.SrcIsModule(String(a.properties.Certificate)) + if cert != "" { + ctx.AddDependency(ctx.Module(), certificateTag, cert) + } + + a.usesLibrary.deps(ctx, true) +} + +func (a *AndroidAppImport) uncompressEmbeddedJniLibs( + ctx android.ModuleContext, inputPath android.Path, outputPath android.OutputPath) { + // Test apps don't need their JNI libraries stored uncompressed. As a matter of fact, messing + // with them may invalidate pre-existing signature data. + if ctx.InstallInTestcases() && (Bool(a.properties.Presigned) || a.preprocessed) { + ctx.Build(pctx, android.BuildParams{ + Rule: android.Cp, + Output: outputPath, + Input: inputPath, + }) + return + } + rule := android.NewRuleBuilder() + rule.Command(). + Textf(`if (zipinfo %s 'lib/*.so' 2>/dev/null | grep -v ' stor ' >/dev/null) ; then`, inputPath). + BuiltTool(ctx, "zip2zip"). + FlagWithInput("-i ", inputPath). + FlagWithOutput("-o ", outputPath). + FlagWithArg("-0 ", "'lib/**/*.so'"). + Textf(`; else cp -f %s %s; fi`, inputPath, outputPath) + rule.Build(pctx, ctx, "uncompress-embedded-jni-libs", "Uncompress embedded JIN libs") +} + +// Returns whether this module should have the dex file stored uncompressed in the APK. +func (a *AndroidAppImport) shouldUncompressDex(ctx android.ModuleContext) bool { + if ctx.Config().UnbundledBuild() || a.preprocessed { + return false + } + + // Uncompress dex in APKs of privileged apps + if ctx.Config().UncompressPrivAppDex() && a.Privileged() { + return true + } + + return shouldUncompressDex(ctx, &a.dexpreopter) +} + +func (a *AndroidAppImport) uncompressDex( + ctx android.ModuleContext, inputPath android.Path, outputPath android.OutputPath) { + rule := android.NewRuleBuilder() + rule.Command(). + Textf(`if (zipinfo %s '*.dex' 2>/dev/null | grep -v ' stor ' >/dev/null) ; then`, inputPath). + BuiltTool(ctx, "zip2zip"). + FlagWithInput("-i ", inputPath). + FlagWithOutput("-o ", outputPath). + FlagWithArg("-0 ", "'classes*.dex'"). + Textf(`; else cp -f %s %s; fi`, inputPath, outputPath) + rule.Build(pctx, ctx, "uncompress-dex", "Uncompress dex files") +} + +func (a *AndroidAppImport) GenerateAndroidBuildActions(ctx android.ModuleContext) { + a.generateAndroidBuildActions(ctx) +} + +func (a *AndroidAppImport) InstallApkName() string { + return a.BaseModuleName() +} + +func (a *AndroidAppImport) generateAndroidBuildActions(ctx android.ModuleContext) { + numCertPropsSet := 0 + if String(a.properties.Certificate) != "" { + numCertPropsSet++ + } + if Bool(a.properties.Presigned) { + numCertPropsSet++ + } + if Bool(a.properties.Default_dev_cert) { + numCertPropsSet++ + } + if numCertPropsSet != 1 { + ctx.ModuleErrorf("One and only one of certficate, presigned, and default_dev_cert properties must be set") + } + + _, certificates := collectAppDeps(ctx, a, false, false) + + // TODO: LOCAL_EXTRACT_APK/LOCAL_EXTRACT_DPI_APK + // TODO: LOCAL_PACKAGE_SPLITS + + srcApk := a.prebuilt.SingleSourcePath(ctx) + + if a.usesLibrary.enforceUsesLibraries() { + srcApk = a.usesLibrary.verifyUsesLibrariesAPK(ctx, srcApk) + } + + // TODO: Install or embed JNI libraries + + // Uncompress JNI libraries in the apk + jnisUncompressed := android.PathForModuleOut(ctx, "jnis-uncompressed", ctx.ModuleName()+".apk") + a.uncompressEmbeddedJniLibs(ctx, srcApk, jnisUncompressed.OutputPath) + + var installDir android.InstallPath + if Bool(a.properties.Privileged) { + installDir = android.PathForModuleInstall(ctx, "priv-app", a.BaseModuleName()) + } else if ctx.InstallInTestcases() { + installDir = android.PathForModuleInstall(ctx, a.BaseModuleName(), ctx.DeviceConfig().DeviceArch()) + } else { + installDir = android.PathForModuleInstall(ctx, "app", a.BaseModuleName()) + } + + a.dexpreopter.installPath = installDir.Join(ctx, a.BaseModuleName()+".apk") + a.dexpreopter.isPresignedPrebuilt = Bool(a.properties.Presigned) + a.dexpreopter.uncompressedDex = a.shouldUncompressDex(ctx) + + a.dexpreopter.enforceUsesLibs = a.usesLibrary.enforceUsesLibraries() + a.dexpreopter.usesLibs = a.usesLibrary.usesLibraryProperties.Uses_libs + a.dexpreopter.optionalUsesLibs = a.usesLibrary.presentOptionalUsesLibs(ctx) + a.dexpreopter.libraryPaths = a.usesLibrary.usesLibraryPaths(ctx) + + dexOutput := a.dexpreopter.dexpreopt(ctx, jnisUncompressed) + if a.dexpreopter.uncompressedDex { + dexUncompressed := android.PathForModuleOut(ctx, "dex-uncompressed", ctx.ModuleName()+".apk") + a.uncompressDex(ctx, dexOutput, dexUncompressed.OutputPath) + dexOutput = dexUncompressed + } + + apkFilename := proptools.StringDefault(a.properties.Filename, a.BaseModuleName()+".apk") + + // TODO: Handle EXTERNAL + + // Sign or align the package if package has not been preprocessed + if a.preprocessed { + a.outputFile = srcApk + a.certificate = PresignedCertificate + } else if !Bool(a.properties.Presigned) { + // If the certificate property is empty at this point, default_dev_cert must be set to true. + // Which makes processMainCert's behavior for the empty cert string WAI. + certificates = processMainCert(a.ModuleBase, String(a.properties.Certificate), certificates, ctx) + if len(certificates) != 1 { + ctx.ModuleErrorf("Unexpected number of certificates were extracted: %q", certificates) + } + a.certificate = certificates[0] + signed := android.PathForModuleOut(ctx, "signed", apkFilename) + var lineageFile android.Path + if lineage := String(a.properties.Lineage); lineage != "" { + lineageFile = android.PathForModuleSrc(ctx, lineage) + } + SignAppPackage(ctx, signed, dexOutput, certificates, nil, lineageFile) + a.outputFile = signed + } else { + alignedApk := android.PathForModuleOut(ctx, "zip-aligned", apkFilename) + TransformZipAlign(ctx, alignedApk, dexOutput) + a.outputFile = alignedApk + a.certificate = PresignedCertificate + } + + // TODO: Optionally compress the output apk. + + a.installPath = ctx.InstallFile(installDir, apkFilename, a.outputFile) + + // TODO: androidmk converter jni libs +} + +func (a *AndroidAppImport) Prebuilt() *android.Prebuilt { + return &a.prebuilt +} + +func (a *AndroidAppImport) Name() string { + return a.prebuilt.Name(a.ModuleBase.Name()) +} + +func (a *AndroidAppImport) OutputFile() android.Path { + return a.outputFile +} + +func (a *AndroidAppImport) JacocoReportClassesFile() android.Path { + return nil +} + +func (a *AndroidAppImport) Certificate() Certificate { + return a.certificate +} + +var dpiVariantGroupType reflect.Type +var archVariantGroupType reflect.Type + +func initAndroidAppImportVariantGroupTypes() { + dpiVariantGroupType = createVariantGroupType(supportedDpis, "Dpi_variants") + + archNames := make([]string, len(android.ArchTypeList())) + for i, archType := range android.ArchTypeList() { + archNames[i] = archType.Name + } + archVariantGroupType = createVariantGroupType(archNames, "Arch") +} + +// Populates all variant struct properties at creation time. +func (a *AndroidAppImport) populateAllVariantStructs() { + a.dpiVariants = reflect.New(dpiVariantGroupType).Interface() + a.AddProperties(a.dpiVariants) + + a.archVariants = reflect.New(archVariantGroupType).Interface() + a.AddProperties(a.archVariants) +} + +func (a *AndroidAppImport) Privileged() bool { + return Bool(a.properties.Privileged) +} + +func (a *AndroidAppImport) sdkVersion() sdkSpec { + return sdkSpecFrom("") +} + +func (a *AndroidAppImport) minSdkVersion() sdkSpec { + return sdkSpecFrom("") +} + +func createVariantGroupType(variants []string, variantGroupName string) reflect.Type { + props := reflect.TypeOf((*AndroidAppImportProperties)(nil)) + + variantFields := make([]reflect.StructField, len(variants)) + for i, variant := range variants { + variantFields[i] = reflect.StructField{ + Name: proptools.FieldNameForProperty(variant), + Type: props, + } + } + + variantGroupStruct := reflect.StructOf(variantFields) + return reflect.StructOf([]reflect.StructField{ + { + Name: variantGroupName, + Type: variantGroupStruct, + }, + }) +} + +// android_app_import imports a prebuilt apk with additional processing specified in the module. +// DPI-specific apk source files can be specified using dpi_variants. Example: +// +// android_app_import { +// name: "example_import", +// apk: "prebuilts/example.apk", +// dpi_variants: { +// mdpi: { +// apk: "prebuilts/example_mdpi.apk", +// }, +// xhdpi: { +// apk: "prebuilts/example_xhdpi.apk", +// }, +// }, +// certificate: "PRESIGNED", +// } +func AndroidAppImportFactory() android.Module { + module := &AndroidAppImport{} + module.AddProperties(&module.properties) + module.AddProperties(&module.dexpreoptProperties) + module.AddProperties(&module.usesLibrary.usesLibraryProperties) + module.populateAllVariantStructs() + android.AddLoadHook(module, func(ctx android.LoadHookContext) { + module.processVariants(ctx) + }) + + android.InitAndroidMultiTargetsArchModule(module, android.DeviceSupported, android.MultilibCommon) + android.InitDefaultableModule(module) + android.InitSingleSourcePrebuiltModule(module, &module.properties, "Apk") + + return module +} + +type androidTestImportProperties struct { + // Whether the prebuilt apk can be installed without additional processing. Default is false. + Preprocessed *bool +} + +type AndroidTestImport struct { + AndroidAppImport + + testProperties testProperties + + testImportProperties androidTestImportProperties + + data android.Paths +} + +func (a *AndroidTestImport) GenerateAndroidBuildActions(ctx android.ModuleContext) { + a.preprocessed = Bool(a.testImportProperties.Preprocessed) + + a.generateAndroidBuildActions(ctx) + + a.data = android.PathsForModuleSrc(ctx, a.testProperties.Data) +} + +func (a *AndroidTestImport) InstallInTestcases() bool { + return true +} + +// android_test_import imports a prebuilt test apk with additional processing specified in the +// module. DPI or arch variant configurations can be made as with android_app_import. +func AndroidTestImportFactory() android.Module { + module := &AndroidTestImport{} + module.AddProperties(&module.properties) + module.AddProperties(&module.dexpreoptProperties) + module.AddProperties(&module.usesLibrary.usesLibraryProperties) + module.AddProperties(&module.testProperties) + module.AddProperties(&module.testImportProperties) + module.populateAllVariantStructs() + android.AddLoadHook(module, func(ctx android.LoadHookContext) { + module.processVariants(ctx) + }) + + module.dexpreopter.isTest = true + + android.InitAndroidMultiTargetsArchModule(module, android.DeviceSupported, android.MultilibCommon) + android.InitDefaultableModule(module) + android.InitSingleSourcePrebuiltModule(module, &module.properties, "Apk") + + return module +} + +type RuntimeResourceOverlay struct { + android.ModuleBase + android.DefaultableModuleBase + android.OverridableModuleBase + aapt + + properties RuntimeResourceOverlayProperties + overridableProperties OverridableRuntimeResourceOverlayProperties + + certificate Certificate + + outputFile android.Path + installDir android.InstallPath +} + +type RuntimeResourceOverlayProperties struct { + // the name of a certificate in the default certificate directory or an android_app_certificate + // module name in the form ":module". + Certificate *string + + // Name of the signing certificate lineage file. + Lineage *string + + // optional theme name. If specified, the overlay package will be applied + // only when the ro.boot.vendor.overlay.theme system property is set to the same value. + Theme *string + + // if not blank, set to the version of the sdk to compile against. + // Defaults to compiling against the current platform. + Sdk_version *string + + // if not blank, set the minimum version of the sdk that the compiled artifacts will run against. + // Defaults to sdk_version if not set. + Min_sdk_version *string + + // list of android_library modules whose resources are extracted and linked against statically + Static_libs []string + + // list of android_app modules whose resources are extracted and linked against + Resource_libs []string + + // Names of modules to be overridden. Listed modules can only be other overlays + // (in Make or Soong). + // This does not completely prevent installation of the overridden overlays, but if both + // overlays would be installed by default (in PRODUCT_PACKAGES) the other overlay will be removed + // from PRODUCT_PACKAGES. + Overrides []string +} + +func (r *RuntimeResourceOverlay) DepsMutator(ctx android.BottomUpMutatorContext) { + sdkDep := decodeSdkDep(ctx, sdkContext(r)) + if sdkDep.hasFrameworkLibs() { + r.aapt.deps(ctx, sdkDep) + } + + cert := android.SrcIsModule(String(r.properties.Certificate)) + if cert != "" { + ctx.AddDependency(ctx.Module(), certificateTag, cert) + } + + ctx.AddVariationDependencies(nil, staticLibTag, r.properties.Static_libs...) + ctx.AddVariationDependencies(nil, libTag, r.properties.Resource_libs...) +} + +func (r *RuntimeResourceOverlay) GenerateAndroidBuildActions(ctx android.ModuleContext) { + // Compile and link resources + r.aapt.hasNoCode = true + // Do not remove resources without default values nor dedupe resource configurations with the same value + aaptLinkFlags := []string{"--no-resource-deduping", "--no-resource-removal"} + // Allow the override of "package name" and "overlay target package name" + manifestPackageName, overridden := ctx.DeviceConfig().OverrideManifestPackageNameFor(ctx.ModuleName()) + if overridden || r.overridableProperties.Package_name != nil { + // The product override variable has a priority over the package_name property. + if !overridden { + manifestPackageName = *r.overridableProperties.Package_name + } + aaptLinkFlags = append(aaptLinkFlags, "--rename-manifest-package "+manifestPackageName) + } + if r.overridableProperties.Target_package_name != nil { + aaptLinkFlags = append(aaptLinkFlags, + "--rename-overlay-target-package "+*r.overridableProperties.Target_package_name) + } + r.aapt.buildActions(ctx, r, aaptLinkFlags...) + + // Sign the built package + _, certificates := collectAppDeps(ctx, r, false, false) + certificates = processMainCert(r.ModuleBase, String(r.properties.Certificate), certificates, ctx) + signed := android.PathForModuleOut(ctx, "signed", r.Name()+".apk") + var lineageFile android.Path + if lineage := String(r.properties.Lineage); lineage != "" { + lineageFile = android.PathForModuleSrc(ctx, lineage) + } + SignAppPackage(ctx, signed, r.aapt.exportPackage, certificates, nil, lineageFile) + r.certificate = certificates[0] + + r.outputFile = signed + r.installDir = android.PathForModuleInstall(ctx, "overlay", String(r.properties.Theme)) + ctx.InstallFile(r.installDir, r.outputFile.Base(), r.outputFile) +} + +func (r *RuntimeResourceOverlay) sdkVersion() sdkSpec { + return sdkSpecFrom(String(r.properties.Sdk_version)) +} + +func (r *RuntimeResourceOverlay) systemModules() string { + return "" +} + +func (r *RuntimeResourceOverlay) minSdkVersion() sdkSpec { + if r.properties.Min_sdk_version != nil { + return sdkSpecFrom(*r.properties.Min_sdk_version) + } + return r.sdkVersion() +} + +func (r *RuntimeResourceOverlay) targetSdkVersion() sdkSpec { + return r.sdkVersion() +} + +// runtime_resource_overlay generates a resource-only apk file that can overlay application and +// system resources at run time. +func RuntimeResourceOverlayFactory() android.Module { + module := &RuntimeResourceOverlay{} + module.AddProperties( + &module.properties, + &module.aaptProperties, + &module.overridableProperties) + + android.InitAndroidMultiTargetsArchModule(module, android.DeviceSupported, android.MultilibCommon) + android.InitDefaultableModule(module) + android.InitOverridableModule(module, &module.properties.Overrides) + return module +} + +type UsesLibraryProperties struct { + // A list of shared library modules that will be listed in uses-library tags in the AndroidManifest.xml file. + Uses_libs []string + + // A list of shared library modules that will be listed in uses-library tags in the AndroidManifest.xml file with + // required=false. + Optional_uses_libs []string + + // If true, the list of uses_libs and optional_uses_libs modules must match the AndroidManifest.xml file. Defaults + // to true if either uses_libs or optional_uses_libs is set. Will unconditionally default to true in the future. + Enforce_uses_libs *bool +} + +// usesLibrary provides properties and helper functions for AndroidApp and AndroidAppImport to verify that the +// <uses-library> tags that end up in the manifest of an APK match the ones known to the build system through the +// uses_libs and optional_uses_libs properties. The build system's values are used by dexpreopt to preopt apps +// with knowledge of their shared libraries. +type usesLibrary struct { + usesLibraryProperties UsesLibraryProperties +} + +func (u *usesLibrary) deps(ctx android.BottomUpMutatorContext, hasFrameworkLibs bool) { + if !ctx.Config().UnbundledBuild() { + ctx.AddVariationDependencies(nil, usesLibTag, u.usesLibraryProperties.Uses_libs...) + ctx.AddVariationDependencies(nil, usesLibTag, u.presentOptionalUsesLibs(ctx)...) + // Only add these extra dependencies if the module depends on framework libs. This avoids + // creating a cyclic dependency: + // e.g. framework-res -> org.apache.http.legacy -> ... -> framework-res. + if hasFrameworkLibs { + // dexpreopt/dexpreopt.go needs the paths to the dex jars of these libraries in case construct_context.sh needs + // to pass them to dex2oat. Add them as a dependency so we can determine the path to the dex jar of each + // library to dexpreopt. + ctx.AddVariationDependencies(nil, usesLibTag, + "org.apache.http.legacy", + "android.hidl.base-V1.0-java", + "android.hidl.manager-V1.0-java") + } + } +} + +// presentOptionalUsesLibs returns optional_uses_libs after filtering out MissingUsesLibraries, which don't exist in the +// build. +func (u *usesLibrary) presentOptionalUsesLibs(ctx android.BaseModuleContext) []string { + optionalUsesLibs, _ := android.FilterList(u.usesLibraryProperties.Optional_uses_libs, ctx.Config().MissingUsesLibraries()) + return optionalUsesLibs +} + +// usesLibraryPaths returns a map of module names of shared library dependencies to the paths to their dex jars. +func (u *usesLibrary) usesLibraryPaths(ctx android.ModuleContext) map[string]android.Path { + usesLibPaths := make(map[string]android.Path) + + if !ctx.Config().UnbundledBuild() { + ctx.VisitDirectDepsWithTag(usesLibTag, func(m android.Module) { + if lib, ok := m.(Dependency); ok { + if dexJar := lib.DexJar(); dexJar != nil { + usesLibPaths[ctx.OtherModuleName(m)] = dexJar + } else { + ctx.ModuleErrorf("module %q in uses_libs or optional_uses_libs must produce a dex jar, does it have installable: true?", + ctx.OtherModuleName(m)) + } + } else if ctx.Config().AllowMissingDependencies() { + ctx.AddMissingDependencies([]string{ctx.OtherModuleName(m)}) + } else { + ctx.ModuleErrorf("module %q in uses_libs or optional_uses_libs must be a java library", + ctx.OtherModuleName(m)) + } + }) + } + + return usesLibPaths +} + +// enforceUsesLibraries returns true of <uses-library> tags should be checked against uses_libs and optional_uses_libs +// properties. Defaults to true if either of uses_libs or optional_uses_libs is specified. Will default to true +// unconditionally in the future. +func (u *usesLibrary) enforceUsesLibraries() bool { + defaultEnforceUsesLibs := len(u.usesLibraryProperties.Uses_libs) > 0 || + len(u.usesLibraryProperties.Optional_uses_libs) > 0 + return BoolDefault(u.usesLibraryProperties.Enforce_uses_libs, defaultEnforceUsesLibs) +} + +// verifyUsesLibrariesManifest checks the <uses-library> tags in an AndroidManifest.xml against the ones specified +// in the uses_libs and optional_uses_libs properties. It returns the path to a copy of the manifest. +func (u *usesLibrary) verifyUsesLibrariesManifest(ctx android.ModuleContext, manifest android.Path) android.Path { + outputFile := android.PathForModuleOut(ctx, "manifest_check", "AndroidManifest.xml") + + rule := android.NewRuleBuilder() + cmd := rule.Command().BuiltTool(ctx, "manifest_check"). + Flag("--enforce-uses-libraries"). + Input(manifest). + FlagWithOutput("-o ", outputFile) + + for _, lib := range u.usesLibraryProperties.Uses_libs { + cmd.FlagWithArg("--uses-library ", lib) + } + + for _, lib := range u.usesLibraryProperties.Optional_uses_libs { + cmd.FlagWithArg("--optional-uses-library ", lib) + } + + rule.Build(pctx, ctx, "verify_uses_libraries", "verify <uses-library>") + + return outputFile +} + +// verifyUsesLibrariesAPK checks the <uses-library> tags in the manifest of an APK against the ones specified +// in the uses_libs and optional_uses_libs properties. It returns the path to a copy of the APK. +func (u *usesLibrary) verifyUsesLibrariesAPK(ctx android.ModuleContext, apk android.Path) android.Path { + outputFile := android.PathForModuleOut(ctx, "verify_uses_libraries", apk.Base()) + + rule := android.NewRuleBuilder() + aapt := ctx.Config().HostToolPath(ctx, "aapt") + rule.Command(). + Textf("aapt_binary=%s", aapt.String()).Implicit(aapt). + Textf(`uses_library_names="%s"`, strings.Join(u.usesLibraryProperties.Uses_libs, " ")). + Textf(`optional_uses_library_names="%s"`, strings.Join(u.usesLibraryProperties.Optional_uses_libs, " ")). + Tool(android.PathForSource(ctx, "build/make/core/verify_uses_libraries.sh")).Input(apk) + rule.Command().Text("cp -f").Input(apk).Output(outputFile) + + rule.Build(pctx, ctx, "verify_uses_libraries", "verify <uses-library>") + + return outputFile +} diff --git a/java/app_builder.go b/java/app_builder.go index 5bacb6776..97ec269ee 100644 --- a/java/app_builder.go +++ b/java/app_builder.go @@ -26,44 +26,33 @@ import ( "github.com/google/blueprint/proptools" "android/soong/android" + "android/soong/remoteexec" ) var ( - Signapk = pctx.AndroidStaticRule("signapk", + Signapk, SignapkRE = remoteexec.StaticRules(pctx, "signapk", blueprint.RuleParams{ - Command: `${config.JavaCmd} -Djava.library.path=$$(dirname $signapkJniLibrary) ` + - `-jar $signapkCmd $flags $certificates $in $out`, - CommandDeps: []string{"$signapkCmd", "$signapkJniLibrary"}, + Command: `$reTemplate${config.JavaCmd} ${config.JavaVmFlags} -Djava.library.path=$$(dirname ${config.SignapkJniLibrary}) ` + + `-jar ${config.SignapkCmd} $flags $certificates $in $out`, + CommandDeps: []string{"${config.SignapkCmd}", "${config.SignapkJniLibrary}"}, }, - "flags", "certificates") - - androidManifestMerger = pctx.AndroidStaticRule("androidManifestMerger", - blueprint.RuleParams{ - Command: "java -classpath $androidManifestMergerCmd com.android.manifmerger.Main merge " + - "--main $in --libs $libsManifests --out $out", - CommandDeps: []string{"$androidManifestMergerCmd"}, - Description: "merge manifest files", - }, - "libsManifests") + &remoteexec.REParams{Labels: map[string]string{"type": "tool", "name": "signapk"}, + ExecStrategy: "${config.RESignApkExecStrategy}", + Inputs: []string{"${config.SignapkCmd}", "$in", "$$(dirname ${config.SignapkJniLibrary})", "$implicits"}, + OutputFiles: []string{"$outCommaList"}, + ToolchainInputs: []string{"${config.JavaCmd}"}, + Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, + }, []string{"flags", "certificates"}, []string{"implicits", "outCommaList"}) ) -func init() { - pctx.SourcePathVariable("androidManifestMergerCmd", "prebuilts/devtools/tools/lib/manifest-merger.jar") - pctx.HostBinToolVariable("aaptCmd", "aapt") - pctx.HostJavaToolVariable("signapkCmd", "signapk.jar") - // TODO(ccross): this should come from the signapk dependencies, but we don't have any way - // to express host JNI dependencies yet. - pctx.HostJNIToolVariable("signapkJniLibrary", "libconscrypt_openjdk_jni") -} - var combineApk = pctx.AndroidStaticRule("combineApk", blueprint.RuleParams{ Command: `${config.MergeZipsCmd} $out $in`, CommandDeps: []string{"${config.MergeZipsCmd}"}, }) -func CreateAppPackage(ctx android.ModuleContext, outputFile android.WritablePath, - packageFile, jniJarFile, dexJarFile android.Path, certificates []Certificate) { +func CreateAndSignAppPackage(ctx android.ModuleContext, outputFile android.WritablePath, + packageFile, jniJarFile, dexJarFile android.Path, certificates []Certificate, deps android.Paths, v4SignatureFile android.WritablePath, lineageFile android.Path) { unsignedApkName := strings.TrimSuffix(outputFile.Base(), ".apk") + "-unsigned.apk" unsignedApk := android.PathForModuleOut(ctx, unsignedApkName) @@ -78,11 +67,17 @@ func CreateAppPackage(ctx android.ModuleContext, outputFile android.WritablePath } ctx.Build(pctx, android.BuildParams{ - Rule: combineApk, - Inputs: inputs, - Output: unsignedApk, + Rule: combineApk, + Inputs: inputs, + Output: unsignedApk, + Implicits: deps, }) + SignAppPackage(ctx, outputFile, unsignedApk, certificates, v4SignatureFile, lineageFile) +} + +func SignAppPackage(ctx android.ModuleContext, signedApk android.WritablePath, unsignedApk android.Path, certificates []Certificate, v4SignatureFile android.WritablePath, lineageFile android.Path) { + var certificateArgs []string var deps android.Paths for _, c := range certificates { @@ -90,15 +85,35 @@ func CreateAppPackage(ctx android.ModuleContext, outputFile android.WritablePath deps = append(deps, c.Pem, c.Key) } + outputFiles := android.WritablePaths{signedApk} + var flags []string + if v4SignatureFile != nil { + outputFiles = append(outputFiles, v4SignatureFile) + flags = append(flags, "--enable-v4") + } + + if lineageFile != nil { + flags = append(flags, "--lineage", lineageFile.String()) + deps = append(deps, lineageFile) + } + + rule := Signapk + args := map[string]string{ + "certificates": strings.Join(certificateArgs, " "), + "flags": strings.Join(flags, " "), + } + if ctx.Config().IsEnvTrue("RBE_SIGNAPK") { + rule = SignapkRE + args["implicits"] = strings.Join(deps.Strings(), ",") + args["outCommaList"] = strings.Join(outputFiles.Strings(), ",") + } ctx.Build(pctx, android.BuildParams{ - Rule: Signapk, + Rule: rule, Description: "signapk", - Output: outputFile, + Outputs: outputFiles, Input: unsignedApk, Implicits: deps, - Args: map[string]string{ - "certificates": strings.Join(certificateArgs, " "), - }, + Args: args, }) } @@ -212,24 +227,30 @@ func TransformJniLibsToJar(ctx android.ModuleContext, outputFile android.Writabl } if uncompressJNI { - jarArgs = append(jarArgs, "-L 0") + jarArgs = append(jarArgs, "-L", "0") } for _, j := range jniLibs { deps = append(deps, j.path) jarArgs = append(jarArgs, - "-P "+targetToJniDir(j.target), - "-f "+j.path.String()) + "-P", targetToJniDir(j.target), + "-f", j.path.String()) } + rule := zip + args := map[string]string{ + "jarArgs": strings.Join(proptools.NinjaAndShellEscapeList(jarArgs), " "), + } + if ctx.Config().IsEnvTrue("RBE_ZIP") { + rule = zipRE + args["implicits"] = strings.Join(deps.Strings(), ",") + } ctx.Build(pctx, android.BuildParams{ - Rule: zip, + Rule: rule, Description: "zip jni libs", Output: outputFile, Implicits: deps, - Args: map[string]string{ - "jarArgs": strings.Join(proptools.NinjaAndShellEscapeList(jarArgs), " "), - }, + Args: args, }) } diff --git a/java/app_test.go b/java/app_test.go index 1d497703a..8ef315206 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -15,17 +15,18 @@ package java import ( - "android/soong/android" - "android/soong/cc" - "fmt" "path/filepath" "reflect" + "regexp" "sort" "strings" "testing" "github.com/google/blueprint/proptools" + + "android/soong/android" + "android/soong/cc" ) var ( @@ -42,7 +43,7 @@ var ( } ) -func testAppContext(config android.Config, bp string, fs map[string][]byte) *android.TestContext { +func testAppConfig(env map[string]string, bp string, fs map[string][]byte) android.Config { appFS := map[string][]byte{} for k, v := range fs { appFS[k] = v @@ -52,13 +53,13 @@ func testAppContext(config android.Config, bp string, fs map[string][]byte) *and appFS[file] = nil } - return testContext(config, bp, appFS) + return testConfig(env, bp, appFS) } func testApp(t *testing.T, bp string) *android.TestContext { - config := testConfig(nil) + config := testAppConfig(nil, bp, nil) - ctx := testAppContext(config, bp, nil) + ctx := testContext() run(t, ctx, config) @@ -71,6 +72,7 @@ func TestApp(t *testing.T) { ctx := testApp(t, moduleType+` { name: "foo", srcs: ["a.java"], + sdk_version: "current" } `) @@ -116,6 +118,7 @@ func TestAppSplits(t *testing.T) { name: "foo", srcs: ["a.java"], package_splits: ["v4", "v7,hdpi"], + sdk_version: "current" }`) foo := ctx.ModuleForTests("foo", "android_common") @@ -129,23 +132,24 @@ func TestAppSplits(t *testing.T) { foo.Output(expectedOutput) } - if g, w := foo.Module().(*AndroidApp).Srcs().Strings(), expectedOutputs; !reflect.DeepEqual(g, w) { - t.Errorf("want Srcs() = %q, got %q", w, g) + outputFiles, err := foo.Module().(*AndroidApp).OutputFiles("") + if err != nil { + t.Fatal(err) + } + if g, w := outputFiles.Strings(), expectedOutputs; !reflect.DeepEqual(g, w) { + t.Errorf(`want OutputFiles("") = %q, got %q`, w, g) } } func TestAndroidAppSet(t *testing.T) { - config := testConfig(nil) - config.TestProductVariables.DeviceArch = proptools.StringPtr("arm64") - ctx := testAppContext(config, ` - android_app_set { - name: "foo", - set: "prebuilts/apks/app.apks", - prerelease: true, - }`, nil) - run(t, ctx, config) + ctx, config := testJava(t, ` + android_app_set { + name: "foo", + set: "prebuilts/apks/app.apks", + prerelease: true, + }`) module := ctx.ModuleForTests("foo", "android_common") - const packedSplitApks = "extracted.zip" + const packedSplitApks = "foo.zip" params := module.Output(packedSplitApks) if params.Rule == nil { t.Errorf("expected output %s is missing", packedSplitApks) @@ -156,6 +160,13 @@ func TestAndroidAppSet(t *testing.T) { if s := params.Args["partition"]; s != "system" { t.Errorf("wrong partition value: '%s', expected 'system'", s) } + mkEntries := android.AndroidMkEntriesForTest(t, config, "", module.Module())[0] + actualMaster := mkEntries.EntryMap["LOCAL_APK_SET_MASTER_FILE"] + expectedMaster := []string{"foo.apk"} + if !reflect.DeepEqual(actualMaster, expectedMaster) { + t.Errorf("Unexpected LOCAL_APK_SET_MASTER_FILE value: '%s', expected: '%s',", + actualMaster, expectedMaster) + } } func TestAndroidAppSet_Variants(t *testing.T) { @@ -165,16 +176,17 @@ func TestAndroidAppSet_Variants(t *testing.T) { set: "prebuilts/apks/app.apks", }` testCases := []struct { - name string - deviceArch *string - deviceSecondaryArch *string - aaptPrebuiltDPI []string - sdkVersion int - expected map[string]string + name string + targets []android.Target + aaptPrebuiltDPI []string + sdkVersion int + expected map[string]string }{ { - name: "One", - deviceArch: proptools.StringPtr("x86"), + name: "One", + targets: []android.Target{ + {Os: android.Android, Arch: android.Arch{ArchType: android.X86}}, + }, aaptPrebuiltDPI: []string{"ldpi", "xxhdpi"}, sdkVersion: 29, expected: map[string]string{ @@ -186,11 +198,13 @@ func TestAndroidAppSet_Variants(t *testing.T) { }, }, { - name: "Two", - deviceArch: proptools.StringPtr("x86_64"), - deviceSecondaryArch: proptools.StringPtr("x86"), - aaptPrebuiltDPI: nil, - sdkVersion: 30, + name: "Two", + targets: []android.Target{ + {Os: android.Android, Arch: android.Arch{ArchType: android.X86_64}}, + {Os: android.Android, Arch: android.Arch{ArchType: android.X86}}, + }, + aaptPrebuiltDPI: nil, + sdkVersion: 30, expected: map[string]string{ "abis": "X86_64,X86", "allow-prereleased": "false", @@ -202,15 +216,14 @@ func TestAndroidAppSet_Variants(t *testing.T) { } for _, test := range testCases { - config := testConfig(nil) + config := testAppConfig(nil, bp, nil) config.TestProductVariables.AAPTPrebuiltDPI = test.aaptPrebuiltDPI config.TestProductVariables.Platform_sdk_version = &test.sdkVersion - config.TestProductVariables.DeviceArch = test.deviceArch - config.TestProductVariables.DeviceSecondaryArch = test.deviceSecondaryArch - ctx := testAppContext(config, bp, nil) + config.Targets[android.Android] = test.targets + ctx := testContext() run(t, ctx, config) module := ctx.ModuleForTests("foo", "android_common") - const packedSplitApks = "extracted.zip" + const packedSplitApks = "foo.zip" params := module.Output(packedSplitApks) for k, v := range test.expected { if actual := params.Args[k]; actual != v { @@ -221,6 +234,371 @@ func TestAndroidAppSet_Variants(t *testing.T) { } } +func TestPlatformAPIs(t *testing.T) { + testJava(t, ` + android_app { + name: "foo", + srcs: ["a.java"], + platform_apis: true, + } + `) + + testJava(t, ` + android_app { + name: "foo", + srcs: ["a.java"], + sdk_version: "current", + } + `) + + testJavaError(t, "platform_apis must be true when sdk_version is empty.", ` + android_app { + name: "bar", + srcs: ["b.java"], + } + `) + + testJavaError(t, "platform_apis must be false when sdk_version is not empty.", ` + android_app { + name: "bar", + srcs: ["b.java"], + sdk_version: "system_current", + platform_apis: true, + } + `) +} + +func TestAndroidAppLinkType(t *testing.T) { + testJava(t, ` + android_app { + name: "foo", + srcs: ["a.java"], + libs: ["bar"], + static_libs: ["baz"], + platform_apis: true, + } + + java_library { + name: "bar", + sdk_version: "current", + srcs: ["b.java"], + } + + android_library { + name: "baz", + sdk_version: "system_current", + srcs: ["c.java"], + } + `) + + testJavaError(t, "Adjust sdk_version: property of the source or target module so that target module is built with the same or smaller API set than the source.", ` + android_app { + name: "foo", + srcs: ["a.java"], + libs: ["bar"], + sdk_version: "current", + static_libs: ["baz"], + } + + java_library { + name: "bar", + sdk_version: "current", + srcs: ["b.java"], + } + + android_library { + name: "baz", + sdk_version: "system_current", + srcs: ["c.java"], + } + `) + + testJava(t, ` + android_app { + name: "foo", + srcs: ["a.java"], + libs: ["bar"], + sdk_version: "system_current", + static_libs: ["baz"], + } + + java_library { + name: "bar", + sdk_version: "current", + srcs: ["b.java"], + } + + android_library { + name: "baz", + sdk_version: "system_current", + srcs: ["c.java"], + } + `) + + testJavaError(t, "Adjust sdk_version: property of the source or target module so that target module is built with the same or smaller API set than the source.", ` + android_app { + name: "foo", + srcs: ["a.java"], + libs: ["bar"], + sdk_version: "system_current", + static_libs: ["baz"], + } + + java_library { + name: "bar", + sdk_version: "current", + srcs: ["b.java"], + } + + android_library { + name: "baz", + srcs: ["c.java"], + } + `) +} + +func TestUpdatableApps(t *testing.T) { + testCases := []struct { + name string + bp string + expectedError string + }{ + { + name: "Stable public SDK", + bp: `android_app { + name: "foo", + srcs: ["a.java"], + sdk_version: "29", + min_sdk_version: "29", + updatable: true, + }`, + }, + { + name: "Stable system SDK", + bp: `android_app { + name: "foo", + srcs: ["a.java"], + sdk_version: "system_29", + min_sdk_version: "29", + updatable: true, + }`, + }, + { + name: "Current public SDK", + bp: `android_app { + name: "foo", + srcs: ["a.java"], + sdk_version: "current", + min_sdk_version: "29", + updatable: true, + }`, + }, + { + name: "Current system SDK", + bp: `android_app { + name: "foo", + srcs: ["a.java"], + sdk_version: "system_current", + min_sdk_version: "29", + updatable: true, + }`, + }, + { + name: "Current module SDK", + bp: `android_app { + name: "foo", + srcs: ["a.java"], + sdk_version: "module_current", + min_sdk_version: "29", + updatable: true, + }`, + }, + { + name: "Current core SDK", + bp: `android_app { + name: "foo", + srcs: ["a.java"], + sdk_version: "core_current", + min_sdk_version: "29", + updatable: true, + }`, + }, + { + name: "No Platform APIs", + bp: `android_app { + name: "foo", + srcs: ["a.java"], + platform_apis: true, + min_sdk_version: "29", + updatable: true, + }`, + expectedError: "Updatable apps must use stable SDKs", + }, + { + name: "No Core Platform APIs", + bp: `android_app { + name: "foo", + srcs: ["a.java"], + sdk_version: "core_platform", + min_sdk_version: "29", + updatable: true, + }`, + expectedError: "Updatable apps must use stable SDKs", + }, + { + name: "No unspecified APIs", + bp: `android_app { + name: "foo", + srcs: ["a.java"], + updatable: true, + min_sdk_version: "29", + }`, + expectedError: "Updatable apps must use stable SDK", + }, + { + name: "Must specify min_sdk_version", + bp: `android_app { + name: "app_without_min_sdk_version", + srcs: ["a.java"], + sdk_version: "29", + updatable: true, + }`, + expectedError: "updatable apps must set min_sdk_version.", + }, + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + if test.expectedError == "" { + testJava(t, test.bp) + } else { + testJavaError(t, test.expectedError, test.bp) + } + }) + } +} + +func TestUpdatableApps_JniLibsShouldShouldSupportMinSdkVersion(t *testing.T) { + testJava(t, cc.GatherRequiredDepsForTest(android.Android)+` + android_app { + name: "foo", + srcs: ["a.java"], + updatable: true, + sdk_version: "current", + min_sdk_version: "current", + jni_libs: ["libjni"], + } + + cc_library { + name: "libjni", + stl: "none", + system_shared_libs: [], + sdk_version: "current", + } + `) +} + +func TestUpdatableApps_JniLibShouldBeBuiltAgainstMinSdkVersion(t *testing.T) { + bp := cc.GatherRequiredDepsForTest(android.Android) + ` + android_app { + name: "foo", + srcs: ["a.java"], + updatable: true, + sdk_version: "current", + min_sdk_version: "29", + jni_libs: ["libjni"], + } + + cc_library { + name: "libjni", + stl: "none", + system_shared_libs: [], + sdk_version: "29", + } + + ndk_prebuilt_object { + name: "ndk_crtbegin_so.29", + sdk_version: "29", + } + + ndk_prebuilt_object { + name: "ndk_crtend_so.29", + sdk_version: "29", + } + ` + fs := map[string][]byte{ + "prebuilts/ndk/current/platforms/android-29/arch-arm64/usr/lib/crtbegin_so.o": nil, + "prebuilts/ndk/current/platforms/android-29/arch-arm64/usr/lib/crtend_so.o": nil, + "prebuilts/ndk/current/platforms/android-29/arch-arm/usr/lib/crtbegin_so.o": nil, + "prebuilts/ndk/current/platforms/android-29/arch-arm/usr/lib/crtend_so.o": nil, + } + + ctx, _ := testJavaWithConfig(t, testConfig(nil, bp, fs)) + + inputs := ctx.ModuleForTests("libjni", "android_arm64_armv8-a_sdk_shared").Description("link").Implicits + var crtbeginFound, crtendFound bool + for _, input := range inputs { + switch input.String() { + case "prebuilts/ndk/current/platforms/android-29/arch-arm64/usr/lib/crtbegin_so.o": + crtbeginFound = true + case "prebuilts/ndk/current/platforms/android-29/arch-arm64/usr/lib/crtend_so.o": + crtendFound = true + } + } + if !crtbeginFound || !crtendFound { + t.Error("should link with ndk_crtbegin_so.29 and ndk_crtend_so.29") + } +} + +func TestUpdatableApps_ErrorIfJniLibDoesntSupportMinSdkVersion(t *testing.T) { + bp := cc.GatherRequiredDepsForTest(android.Android) + ` + android_app { + name: "foo", + srcs: ["a.java"], + updatable: true, + sdk_version: "current", + min_sdk_version: "29", // this APK should support 29 + jni_libs: ["libjni"], + } + + cc_library { + name: "libjni", + stl: "none", + sdk_version: "current", + } + ` + testJavaError(t, `"libjni" .*: sdk_version\(current\) is higher than min_sdk_version\(29\)`, bp) +} + +func TestUpdatableApps_ErrorIfDepSdkVersionIsHigher(t *testing.T) { + bp := cc.GatherRequiredDepsForTest(android.Android) + ` + android_app { + name: "foo", + srcs: ["a.java"], + updatable: true, + sdk_version: "current", + min_sdk_version: "29", // this APK should support 29 + jni_libs: ["libjni"], + } + + cc_library { + name: "libjni", + stl: "none", + shared_libs: ["libbar"], + system_shared_libs: [], + sdk_version: "27", + } + + cc_library { + name: "libbar", + stl: "none", + system_shared_libs: [], + sdk_version: "current", + } + ` + testJavaError(t, `"libjni" .*: links "libbar" built against newer API version "current"`, bp) +} + func TestResourceDirs(t *testing.T) { testCases := []struct { name string @@ -251,14 +629,15 @@ func TestResourceDirs(t *testing.T) { bp := ` android_app { name: "foo", + sdk_version: "current", %s } ` for _, testCase := range testCases { t.Run(testCase.name, func(t *testing.T) { - config := testConfig(nil) - ctx := testContext(config, fmt.Sprintf(bp, testCase.prop), fs) + config := testConfig(nil, fmt.Sprintf(bp, testCase.prop), fs) + ctx := testContext() run(t, ctx, config) module := ctx.ModuleForTests("foo", "android_common") @@ -279,6 +658,107 @@ func TestResourceDirs(t *testing.T) { } } +func TestLibraryAssets(t *testing.T) { + bp := ` + android_app { + name: "foo", + sdk_version: "current", + static_libs: ["lib1", "lib2", "lib3"], + } + + android_library { + name: "lib1", + sdk_version: "current", + asset_dirs: ["assets_a"], + } + + android_library { + name: "lib2", + sdk_version: "current", + } + + android_library { + name: "lib3", + sdk_version: "current", + static_libs: ["lib4"], + } + + android_library { + name: "lib4", + sdk_version: "current", + asset_dirs: ["assets_b"], + } + ` + + testCases := []struct { + name string + assetFlag string + assetPackages []string + }{ + { + name: "foo", + // lib1 has its own asset. lib3 doesn't have any, but provides lib4's transitively. + assetPackages: []string{ + buildDir + "/.intermediates/foo/android_common/aapt2/package-res.apk", + buildDir + "/.intermediates/lib1/android_common/assets.zip", + buildDir + "/.intermediates/lib3/android_common/assets.zip", + }, + }, + { + name: "lib1", + assetFlag: "-A assets_a", + }, + { + name: "lib2", + }, + { + name: "lib3", + assetPackages: []string{ + buildDir + "/.intermediates/lib3/android_common/aapt2/package-res.apk", + buildDir + "/.intermediates/lib4/android_common/assets.zip", + }, + }, + { + name: "lib4", + assetFlag: "-A assets_b", + }, + } + ctx := testApp(t, bp) + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + m := ctx.ModuleForTests(test.name, "android_common") + + // Check asset flag in aapt2 link flags + var aapt2link android.TestingBuildParams + if len(test.assetPackages) > 0 { + aapt2link = m.Output("aapt2/package-res.apk") + } else { + aapt2link = m.Output("package-res.apk") + } + aapt2Flags := aapt2link.Args["flags"] + if test.assetFlag != "" { + if !strings.Contains(aapt2Flags, test.assetFlag) { + t.Errorf("Can't find asset flag %q in aapt2 link flags %q", test.assetFlag, aapt2Flags) + } + } else { + if strings.Contains(aapt2Flags, " -A ") { + t.Errorf("aapt2 link flags %q contain unexpected asset flag", aapt2Flags) + } + } + + // Check asset merge rule. + if len(test.assetPackages) > 0 { + mergeAssets := m.Output("package-res.apk") + if !reflect.DeepEqual(test.assetPackages, mergeAssets.Inputs.Strings()) { + t.Errorf("Unexpected mergeAssets inputs: %v, expected: %v", + mergeAssets.Inputs.Strings(), test.assetPackages) + } + } + }) + } +} + func TestAndroidResources(t *testing.T) { testCases := []struct { name string @@ -431,36 +911,41 @@ func TestAndroidResources(t *testing.T) { bp := ` android_app { name: "foo", + sdk_version: "current", resource_dirs: ["foo/res"], static_libs: ["lib", "lib3"], } android_app { name: "bar", + sdk_version: "current", resource_dirs: ["bar/res"], } android_library { name: "lib", + sdk_version: "current", resource_dirs: ["lib/res"], static_libs: ["lib2"], } android_library { name: "lib2", + sdk_version: "current", resource_dirs: ["lib2/res"], } // This library has the same resources as lib (should not lead to dupe RROs) android_library { name: "lib3", + sdk_version: "current", resource_dirs: ["lib/res"] } ` for _, testCase := range testCases { t.Run(testCase.name, func(t *testing.T) { - config := testConfig(nil) + config := testAppConfig(nil, bp, fs) config.TestProductVariables.DeviceResourceOverlays = deviceResourceOverlays config.TestProductVariables.ProductResourceOverlays = productResourceOverlays if testCase.enforceRROTargets != nil { @@ -470,7 +955,7 @@ func TestAndroidResources(t *testing.T) { config.TestProductVariables.EnforceRROExcludedOverlays = testCase.enforceRROExcludedOverlays } - ctx := testAppContext(config, bp, fs) + ctx := testContext() run(t, ctx, config) resourceListToFiles := func(module android.TestingModule, list []string) (files []string) { @@ -543,6 +1028,7 @@ func TestAppSdkVersion(t *testing.T) { platformSdkCodename string platformSdkFinal bool expectedMinSdkVersion string + platformApis bool }{ { name: "current final SDK", @@ -563,6 +1049,7 @@ func TestAppSdkVersion(t *testing.T) { { name: "default final SDK", sdkVersion: "", + platformApis: true, platformSdkInt: 27, platformSdkCodename: "REL", platformSdkFinal: true, @@ -571,6 +1058,7 @@ func TestAppSdkVersion(t *testing.T) { { name: "default non-final SDK", sdkVersion: "", + platformApis: true, platformSdkInt: 27, platformSdkCodename: "OMR1", platformSdkFinal: false, @@ -586,18 +1074,23 @@ func TestAppSdkVersion(t *testing.T) { for _, moduleType := range []string{"android_app", "android_library"} { for _, test := range testCases { t.Run(moduleType+" "+test.name, func(t *testing.T) { + platformApiProp := "" + if test.platformApis { + platformApiProp = "platform_apis: true," + } bp := fmt.Sprintf(`%s { name: "foo", srcs: ["a.java"], sdk_version: "%s", - }`, moduleType, test.sdkVersion) + %s + }`, moduleType, test.sdkVersion, platformApiProp) - config := testConfig(nil) + config := testAppConfig(nil, bp, nil) config.TestProductVariables.Platform_sdk_version = &test.platformSdkInt config.TestProductVariables.Platform_sdk_codename = &test.platformSdkCodename config.TestProductVariables.Platform_sdk_final = &test.platformSdkFinal - ctx := testAppContext(config, bp, nil) + ctx := testContext() run(t, ctx, config) @@ -629,43 +1122,44 @@ func TestAppSdkVersion(t *testing.T) { } func TestJNIABI(t *testing.T) { - ctx := testJava(t, cc.GatherRequiredDepsForTest(android.Android)+` + ctx, _ := testJava(t, cc.GatherRequiredDepsForTest(android.Android)+` cc_library { name: "libjni", system_shared_libs: [], + sdk_version: "current", stl: "none", } android_test { name: "test", - no_framework_libs: true, + sdk_version: "core_platform", jni_libs: ["libjni"], } android_test { name: "test_first", - no_framework_libs: true, + sdk_version: "core_platform", compile_multilib: "first", jni_libs: ["libjni"], } android_test { name: "test_both", - no_framework_libs: true, + sdk_version: "core_platform", compile_multilib: "both", jni_libs: ["libjni"], } android_test { name: "test_32", - no_framework_libs: true, + sdk_version: "core_platform", compile_multilib: "32", jni_libs: ["libjni"], } android_test { name: "test_64", - no_framework_libs: true, + sdk_version: "core_platform", compile_multilib: "64", jni_libs: ["libjni"], } @@ -701,53 +1195,95 @@ func TestJNIABI(t *testing.T) { } } +func TestAppSdkVersionByPartition(t *testing.T) { + testJavaError(t, "sdk_version must have a value when the module is located at vendor or product", ` + android_app { + name: "foo", + srcs: ["a.java"], + vendor: true, + platform_apis: true, + } + `) + + testJava(t, ` + android_app { + name: "bar", + srcs: ["b.java"], + platform_apis: true, + } + `) + + for _, enforce := range []bool{true, false} { + bp := ` + android_app { + name: "foo", + srcs: ["a.java"], + product_specific: true, + platform_apis: true, + } + ` + + config := testAppConfig(nil, bp, nil) + config.TestProductVariables.EnforceProductPartitionInterface = proptools.BoolPtr(enforce) + if enforce { + testJavaErrorWithConfig(t, "sdk_version must have a value when the module is located at vendor or product", config) + } else { + testJavaWithConfig(t, config) + } + } +} + func TestJNIPackaging(t *testing.T) { - ctx := testJava(t, cc.GatherRequiredDepsForTest(android.Android)+` + ctx, _ := testJava(t, cc.GatherRequiredDepsForTest(android.Android)+` cc_library { name: "libjni", system_shared_libs: [], stl: "none", + sdk_version: "current", } android_app { name: "app", jni_libs: ["libjni"], + sdk_version: "current", } android_app { name: "app_noembed", jni_libs: ["libjni"], use_embedded_native_libs: false, + sdk_version: "current", } android_app { name: "app_embed", jni_libs: ["libjni"], use_embedded_native_libs: true, + sdk_version: "current", } android_test { name: "test", - no_framework_libs: true, + sdk_version: "current", jni_libs: ["libjni"], } android_test { name: "test_noembed", - no_framework_libs: true, + sdk_version: "current", jni_libs: ["libjni"], use_embedded_native_libs: false, } android_test_helper_app { name: "test_helper", - no_framework_libs: true, + sdk_version: "current", jni_libs: ["libjni"], } android_test_helper_app { name: "test_helper_noembed", - no_framework_libs: true, + sdk_version: "current", jni_libs: ["libjni"], use_embedded_native_libs: false, } @@ -779,9 +1315,129 @@ func TestJNIPackaging(t *testing.T) { if g, w := !strings.Contains(jniLibZip.Args["jarArgs"], "-L 0"), test.compressed; g != w { t.Errorf("expected jni compressed %v, got %v", w, g) } + + if !strings.Contains(jniLibZip.Implicits[0].String(), "_sdk_") { + t.Errorf("expected input %q to use sdk variant", jniLibZip.Implicits[0].String()) + } } }) } +} + +func TestJNISDK(t *testing.T) { + ctx, _ := testJava(t, cc.GatherRequiredDepsForTest(android.Android)+` + cc_library { + name: "libjni", + system_shared_libs: [], + stl: "none", + sdk_version: "current", + } + + android_test { + name: "app_platform", + jni_libs: ["libjni"], + platform_apis: true, + } + + android_test { + name: "app_sdk", + jni_libs: ["libjni"], + sdk_version: "current", + } + + android_test { + name: "app_force_platform", + jni_libs: ["libjni"], + sdk_version: "current", + jni_uses_platform_apis: true, + } + + android_test { + name: "app_force_sdk", + jni_libs: ["libjni"], + platform_apis: true, + jni_uses_sdk_apis: true, + } + + cc_library { + name: "libvendorjni", + system_shared_libs: [], + stl: "none", + vendor: true, + } + + android_test { + name: "app_vendor", + jni_libs: ["libvendorjni"], + sdk_version: "current", + vendor: true, + } + `) + + testCases := []struct { + name string + sdkJNI bool + vendorJNI bool + }{ + {name: "app_platform"}, + {name: "app_sdk", sdkJNI: true}, + {name: "app_force_platform"}, + {name: "app_force_sdk", sdkJNI: true}, + {name: "app_vendor", vendorJNI: true}, + } + + platformJNI := ctx.ModuleForTests("libjni", "android_arm64_armv8-a_shared"). + Output("libjni.so").Output.String() + sdkJNI := ctx.ModuleForTests("libjni", "android_arm64_armv8-a_sdk_shared"). + Output("libjni.so").Output.String() + vendorJNI := ctx.ModuleForTests("libvendorjni", "android_arm64_armv8-a_shared"). + Output("libvendorjni.so").Output.String() + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + app := ctx.ModuleForTests(test.name, "android_common") + + jniLibZip := app.MaybeOutput("jnilibs.zip") + if len(jniLibZip.Implicits) != 1 { + t.Fatalf("expected exactly one jni library, got %q", jniLibZip.Implicits.Strings()) + } + gotJNI := jniLibZip.Implicits[0].String() + + if test.sdkJNI { + if gotJNI != sdkJNI { + t.Errorf("expected SDK JNI library %q, got %q", sdkJNI, gotJNI) + } + } else if test.vendorJNI { + if gotJNI != vendorJNI { + t.Errorf("expected platform JNI library %q, got %q", vendorJNI, gotJNI) + } + } else { + if gotJNI != platformJNI { + t.Errorf("expected platform JNI library %q, got %q", platformJNI, gotJNI) + } + } + }) + } + + t.Run("jni_uses_platform_apis_error", func(t *testing.T) { + testJavaError(t, `jni_uses_platform_apis: can only be set for modules that set sdk_version`, ` + android_test { + name: "app_platform", + platform_apis: true, + jni_uses_platform_apis: true, + } + `) + }) + + t.Run("jni_uses_sdk_apis_error", func(t *testing.T) { + testJavaError(t, `jni_uses_sdk_apis: can only be set for modules that do not set sdk_version`, ` + android_test { + name: "app_sdk", + sdk_version: "current", + jni_uses_sdk_apis: true, + } + `) + }) } @@ -790,7 +1446,8 @@ func TestCertificates(t *testing.T) { name string bp string certificateOverride string - expected string + expectedLineage string + expectedCertificate string }{ { name: "default", @@ -798,10 +1455,12 @@ func TestCertificates(t *testing.T) { android_app { name: "foo", srcs: ["a.java"], + sdk_version: "current", } `, certificateOverride: "", - expected: "build/target/product/security/testkey.x509.pem build/target/product/security/testkey.pk8", + expectedLineage: "", + expectedCertificate: "build/make/target/product/security/testkey.x509.pem build/make/target/product/security/testkey.pk8", }, { name: "module certificate property", @@ -809,16 +1468,18 @@ func TestCertificates(t *testing.T) { android_app { name: "foo", srcs: ["a.java"], - certificate: ":new_certificate" + certificate: ":new_certificate", + sdk_version: "current", } android_app_certificate { name: "new_certificate", - certificate: "cert/new_cert", + certificate: "cert/new_cert", } `, certificateOverride: "", - expected: "cert/new_cert.x509.pem cert/new_cert.pk8", + expectedLineage: "", + expectedCertificate: "cert/new_cert.x509.pem cert/new_cert.pk8", }, { name: "path certificate property", @@ -826,11 +1487,13 @@ func TestCertificates(t *testing.T) { android_app { name: "foo", srcs: ["a.java"], - certificate: "expiredkey" + certificate: "expiredkey", + sdk_version: "current", } `, certificateOverride: "", - expected: "build/target/product/security/expiredkey.x509.pem build/target/product/security/expiredkey.pk8", + expectedLineage: "", + expectedCertificate: "build/make/target/product/security/expiredkey.x509.pem build/make/target/product/security/expiredkey.pk8", }, { name: "certificate overrides", @@ -838,32 +1501,119 @@ func TestCertificates(t *testing.T) { android_app { name: "foo", srcs: ["a.java"], - certificate: "expiredkey" + certificate: "expiredkey", + sdk_version: "current", } android_app_certificate { name: "new_certificate", - certificate: "cert/new_cert", + certificate: "cert/new_cert", } `, certificateOverride: "foo:new_certificate", - expected: "cert/new_cert.x509.pem cert/new_cert.pk8", + expectedLineage: "", + expectedCertificate: "cert/new_cert.x509.pem cert/new_cert.pk8", + }, + { + name: "certificate lineage", + bp: ` + android_app { + name: "foo", + srcs: ["a.java"], + certificate: ":new_certificate", + lineage: "lineage.bin", + sdk_version: "current", + } + + android_app_certificate { + name: "new_certificate", + certificate: "cert/new_cert", + } + `, + certificateOverride: "", + expectedLineage: "--lineage lineage.bin", + expectedCertificate: "cert/new_cert.x509.pem cert/new_cert.pk8", }, } for _, test := range testCases { t.Run(test.name, func(t *testing.T) { - config := testConfig(nil) + config := testAppConfig(nil, test.bp, nil) if test.certificateOverride != "" { config.TestProductVariables.CertificateOverrides = []string{test.certificateOverride} } - ctx := testAppContext(config, test.bp, nil) + ctx := testContext() + + run(t, ctx, config) + foo := ctx.ModuleForTests("foo", "android_common") + + signapk := foo.Output("foo.apk") + signCertificateFlags := signapk.Args["certificates"] + if test.expectedCertificate != signCertificateFlags { + t.Errorf("Incorrect signing flags, expected: %q, got: %q", test.expectedCertificate, signCertificateFlags) + } + + signFlags := signapk.Args["flags"] + if test.expectedLineage != signFlags { + t.Errorf("Incorrect signing flags, expected: %q, got: %q", test.expectedLineage, signFlags) + } + }) + } +} + +func TestRequestV4SigningFlag(t *testing.T) { + testCases := []struct { + name string + bp string + expected string + }{ + { + name: "default", + bp: ` + android_app { + name: "foo", + srcs: ["a.java"], + sdk_version: "current", + } + `, + expected: "", + }, + { + name: "default", + bp: ` + android_app { + name: "foo", + srcs: ["a.java"], + sdk_version: "current", + v4_signature: false, + } + `, + expected: "", + }, + { + name: "module certificate property", + bp: ` + android_app { + name: "foo", + srcs: ["a.java"], + sdk_version: "current", + v4_signature: true, + } + `, + expected: "--enable-v4", + }, + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + config := testAppConfig(nil, test.bp, nil) + ctx := testContext() run(t, ctx, config) foo := ctx.ModuleForTests("foo", "android_common") signapk := foo.Output("foo.apk") - signFlags := signapk.Args["certificates"] + signFlags := signapk.Args["flags"] if test.expected != signFlags { t.Errorf("Incorrect signing flags, expected: %q, got: %q", test.expected, signFlags) } @@ -884,6 +1634,7 @@ func TestPackageNameOverride(t *testing.T) { android_app { name: "foo", srcs: ["a.java"], + sdk_version: "current", } `, packageNameOverride: "", @@ -898,12 +1649,13 @@ func TestPackageNameOverride(t *testing.T) { android_app { name: "foo", srcs: ["a.java"], + sdk_version: "current", } `, packageNameOverride: "foo:bar", expected: []string{ // The package apk should be still be the original name for test dependencies. - buildDir + "/.intermediates/foo/android_common/foo.apk", + buildDir + "/.intermediates/foo/android_common/bar.apk", buildDir + "/target/product/test_device/system/app/bar/bar.apk", }, }, @@ -911,11 +1663,11 @@ func TestPackageNameOverride(t *testing.T) { for _, test := range testCases { t.Run(test.name, func(t *testing.T) { - config := testConfig(nil) + config := testAppConfig(nil, test.bp, nil) if test.packageNameOverride != "" { config.TestProductVariables.PackageNameOverrides = []string{test.packageNameOverride} } - ctx := testAppContext(config, test.bp, nil) + ctx := testContext() run(t, ctx, config) foo := ctx.ModuleForTests("foo", "android_common") @@ -939,16 +1691,18 @@ func TestInstrumentationTargetOverridden(t *testing.T) { android_app { name: "foo", srcs: ["a.java"], + sdk_version: "current", } android_test { name: "bar", instrumentation_for: "foo", + sdk_version: "current", } ` - config := testConfig(nil) + config := testAppConfig(nil, bp, nil) config.TestProductVariables.ManifestPackageNameOverrides = []string{"foo:org.dandroid.bp"} - ctx := testAppContext(config, bp, nil) + ctx := testContext() run(t, ctx, config) @@ -962,18 +1716,21 @@ func TestInstrumentationTargetOverridden(t *testing.T) { } func TestOverrideAndroidApp(t *testing.T) { - ctx := testJava(t, ` + ctx, _ := testJava(t, ` android_app { name: "foo", srcs: ["a.java"], certificate: "expiredkey", - overrides: ["baz"], + overrides: ["qux"], + sdk_version: "current", } override_android_app { name: "bar", base: "foo", certificate: ":new_certificate", + lineage: "lineage.bin", + logging_parent: "bah", } android_app_certificate { @@ -989,33 +1746,45 @@ func TestOverrideAndroidApp(t *testing.T) { `) expectedVariants := []struct { - variantName string - apkName string - apkPath string - signFlag string - overrides []string - aaptFlag string + moduleName string + variantName string + apkName string + apkPath string + certFlag string + lineageFlag string + overrides []string + aaptFlag string + logging_parent string }{ { - variantName: "android_common", - apkPath: "/target/product/test_device/system/app/foo/foo.apk", - signFlag: "build/target/product/security/expiredkey.x509.pem build/target/product/security/expiredkey.pk8", - overrides: []string{"baz"}, - aaptFlag: "", + moduleName: "foo", + variantName: "android_common", + apkPath: "/target/product/test_device/system/app/foo/foo.apk", + certFlag: "build/make/target/product/security/expiredkey.x509.pem build/make/target/product/security/expiredkey.pk8", + lineageFlag: "", + overrides: []string{"qux"}, + aaptFlag: "", + logging_parent: "", }, { - variantName: "bar_android_common", - apkPath: "/target/product/test_device/system/app/bar/bar.apk", - signFlag: "cert/new_cert.x509.pem cert/new_cert.pk8", - overrides: []string{"baz", "foo"}, - aaptFlag: "", + moduleName: "bar", + variantName: "android_common_bar", + apkPath: "/target/product/test_device/system/app/bar/bar.apk", + certFlag: "cert/new_cert.x509.pem cert/new_cert.pk8", + lineageFlag: "--lineage lineage.bin", + overrides: []string{"qux", "foo"}, + aaptFlag: "", + logging_parent: "bah", }, { - variantName: "baz_android_common", - apkPath: "/target/product/test_device/system/app/baz/baz.apk", - signFlag: "build/target/product/security/expiredkey.x509.pem build/target/product/security/expiredkey.pk8", - overrides: []string{"baz", "foo"}, - aaptFlag: "--rename-manifest-package org.dandroid.bp", + moduleName: "baz", + variantName: "android_common_baz", + apkPath: "/target/product/test_device/system/app/baz/baz.apk", + certFlag: "build/make/target/product/security/expiredkey.x509.pem build/make/target/product/security/expiredkey.pk8", + lineageFlag: "", + overrides: []string{"qux", "foo"}, + aaptFlag: "--rename-manifest-package org.dandroid.bp", + logging_parent: "", }, } for _, expected := range expectedVariants { @@ -1036,10 +1805,16 @@ func TestOverrideAndroidApp(t *testing.T) { } // Check the certificate paths - signapk := variant.Output("foo.apk") - signFlag := signapk.Args["certificates"] - if expected.signFlag != signFlag { - t.Errorf("Incorrect signing flags, expected: %q, got: %q", expected.signFlag, signFlag) + signapk := variant.Output(expected.moduleName + ".apk") + certFlag := signapk.Args["certificates"] + if expected.certFlag != certFlag { + t.Errorf("Incorrect signing flags, expected: %q, got: %q", expected.certFlag, certFlag) + } + + // Check the lineage flags + lineageFlag := signapk.Args["flags"] + if expected.lineageFlag != lineageFlag { + t.Errorf("Incorrect signing flags, expected: %q, got: %q", expected.lineageFlag, lineageFlag) } // Check if the overrides field values are correctly aggregated. @@ -1049,6 +1824,13 @@ func TestOverrideAndroidApp(t *testing.T) { expected.overrides, mod.appProperties.Overrides) } + // Test Overridable property: Logging_parent + logging_parent := mod.aapt.LoggingParent + if expected.logging_parent != logging_parent { + t.Errorf("Incorrect overrides property value for logging parent, expected: %q, got: %q", + expected.logging_parent, logging_parent) + } + // Check the package renaming flag, if exists. res := variant.Output("package-res.apk") aapt2Flags := res.Args["flags"] @@ -1058,8 +1840,889 @@ func TestOverrideAndroidApp(t *testing.T) { } } +func TestOverrideAndroidAppDependency(t *testing.T) { + ctx, _ := testJava(t, ` + android_app { + name: "foo", + srcs: ["a.java"], + sdk_version: "current", + } + + override_android_app { + name: "bar", + base: "foo", + package_name: "org.dandroid.bp", + } + + android_test { + name: "baz", + srcs: ["b.java"], + instrumentation_for: "foo", + } + + android_test { + name: "qux", + srcs: ["b.java"], + instrumentation_for: "bar", + } + `) + + // Verify baz, which depends on the overridden module foo, has the correct classpath javac arg. + javac := ctx.ModuleForTests("baz", "android_common").Rule("javac") + fooTurbine := filepath.Join(buildDir, ".intermediates", "foo", "android_common", "turbine-combined", "foo.jar") + if !strings.Contains(javac.Args["classpath"], fooTurbine) { + t.Errorf("baz classpath %v does not contain %q", javac.Args["classpath"], fooTurbine) + } + + // Verify qux, which depends on the overriding module bar, has the correct classpath javac arg. + javac = ctx.ModuleForTests("qux", "android_common").Rule("javac") + barTurbine := filepath.Join(buildDir, ".intermediates", "foo", "android_common_bar", "turbine-combined", "foo.jar") + if !strings.Contains(javac.Args["classpath"], barTurbine) { + t.Errorf("qux classpath %v does not contain %q", javac.Args["classpath"], barTurbine) + } +} + +func TestOverrideAndroidTest(t *testing.T) { + ctx, _ := testJava(t, ` + android_app { + name: "foo", + srcs: ["a.java"], + package_name: "com.android.foo", + sdk_version: "current", + } + + override_android_app { + name: "bar", + base: "foo", + package_name: "com.android.bar", + } + + android_test { + name: "foo_test", + srcs: ["b.java"], + instrumentation_for: "foo", + } + + override_android_test { + name: "bar_test", + base: "foo_test", + package_name: "com.android.bar.test", + instrumentation_for: "bar", + instrumentation_target_package: "com.android.bar", + } + `) + + expectedVariants := []struct { + moduleName string + variantName string + apkPath string + overrides []string + targetVariant string + packageFlag string + targetPackageFlag string + }{ + { + variantName: "android_common", + apkPath: "/target/product/test_device/testcases/foo_test/arm64/foo_test.apk", + overrides: nil, + targetVariant: "android_common", + packageFlag: "", + targetPackageFlag: "", + }, + { + variantName: "android_common_bar_test", + apkPath: "/target/product/test_device/testcases/bar_test/arm64/bar_test.apk", + overrides: []string{"foo_test"}, + targetVariant: "android_common_bar", + packageFlag: "com.android.bar.test", + targetPackageFlag: "com.android.bar", + }, + } + for _, expected := range expectedVariants { + variant := ctx.ModuleForTests("foo_test", expected.variantName) + + // Check the final apk name + outputs := variant.AllOutputs() + expectedApkPath := buildDir + expected.apkPath + found := false + for _, o := range outputs { + if o == expectedApkPath { + found = true + break + } + } + if !found { + t.Errorf("Can't find %q in output files.\nAll outputs:%v", expectedApkPath, outputs) + } + + // Check if the overrides field values are correctly aggregated. + mod := variant.Module().(*AndroidTest) + if !reflect.DeepEqual(expected.overrides, mod.appProperties.Overrides) { + t.Errorf("Incorrect overrides property value, expected: %q, got: %q", + expected.overrides, mod.appProperties.Overrides) + } + + // Check if javac classpath has the correct jar file path. This checks instrumentation_for overrides. + javac := variant.Rule("javac") + turbine := filepath.Join(buildDir, ".intermediates", "foo", expected.targetVariant, "turbine-combined", "foo.jar") + if !strings.Contains(javac.Args["classpath"], turbine) { + t.Errorf("classpath %q does not contain %q", javac.Args["classpath"], turbine) + } + + // Check aapt2 flags. + res := variant.Output("package-res.apk") + aapt2Flags := res.Args["flags"] + checkAapt2LinkFlag(t, aapt2Flags, "rename-manifest-package", expected.packageFlag) + checkAapt2LinkFlag(t, aapt2Flags, "rename-instrumentation-target-package", expected.targetPackageFlag) + } +} + +func TestAndroidTest_FixTestConfig(t *testing.T) { + ctx, _ := testJava(t, ` + android_app { + name: "foo", + srcs: ["a.java"], + package_name: "com.android.foo", + sdk_version: "current", + } + + android_test { + name: "foo_test", + srcs: ["b.java"], + instrumentation_for: "foo", + } + + android_test { + name: "bar_test", + srcs: ["b.java"], + package_name: "com.android.bar.test", + instrumentation_for: "foo", + } + + override_android_test { + name: "baz_test", + base: "foo_test", + package_name: "com.android.baz.test", + } + `) + + testCases := []struct { + moduleName string + variantName string + expectedFlags []string + }{ + { + moduleName: "foo_test", + variantName: "android_common", + }, + { + moduleName: "bar_test", + variantName: "android_common", + expectedFlags: []string{ + "--manifest " + buildDir + "/.intermediates/bar_test/android_common/manifest_fixer/AndroidManifest.xml", + "--package-name com.android.bar.test", + }, + }, + { + moduleName: "foo_test", + variantName: "android_common_baz_test", + expectedFlags: []string{ + "--manifest " + buildDir + + "/.intermediates/foo_test/android_common_baz_test/manifest_fixer/AndroidManifest.xml", + "--package-name com.android.baz.test", + "--test-file-name baz_test.apk", + }, + }, + } + + for _, test := range testCases { + variant := ctx.ModuleForTests(test.moduleName, test.variantName) + params := variant.MaybeOutput("test_config_fixer/AndroidTest.xml") + + if len(test.expectedFlags) > 0 { + if params.Rule == nil { + t.Errorf("test_config_fixer was expected to run, but didn't") + } else { + for _, flag := range test.expectedFlags { + if !strings.Contains(params.RuleParams.Command, flag) { + t.Errorf("Flag %q was not found in command: %q", flag, params.RuleParams.Command) + } + } + } + } else { + if params.Rule != nil { + t.Errorf("test_config_fixer was not expected to run, but did: %q", params.RuleParams.Command) + } + } + + } +} + +func TestAndroidAppImport(t *testing.T) { + ctx, _ := testJava(t, ` + android_app_import { + name: "foo", + apk: "prebuilts/apk/app.apk", + certificate: "platform", + dex_preopt: { + enabled: true, + }, + } + `) + + variant := ctx.ModuleForTests("foo", "android_common") + + // Check dexpreopt outputs. + if variant.MaybeOutput("dexpreopt/oat/arm64/package.vdex").Rule == nil || + variant.MaybeOutput("dexpreopt/oat/arm64/package.odex").Rule == nil { + t.Errorf("can't find dexpreopt outputs") + } + + // Check cert signing flag. + signedApk := variant.Output("signed/foo.apk") + signingFlag := signedApk.Args["certificates"] + expected := "build/make/target/product/security/platform.x509.pem build/make/target/product/security/platform.pk8" + if expected != signingFlag { + t.Errorf("Incorrect signing flags, expected: %q, got: %q", expected, signingFlag) + } +} + +func TestAndroidAppImport_NoDexPreopt(t *testing.T) { + ctx, _ := testJava(t, ` + android_app_import { + name: "foo", + apk: "prebuilts/apk/app.apk", + certificate: "platform", + dex_preopt: { + enabled: false, + }, + } + `) + + variant := ctx.ModuleForTests("foo", "android_common") + + // Check dexpreopt outputs. They shouldn't exist. + if variant.MaybeOutput("dexpreopt/oat/arm64/package.vdex").Rule != nil || + variant.MaybeOutput("dexpreopt/oat/arm64/package.odex").Rule != nil { + t.Errorf("dexpreopt shouldn't have run.") + } +} + +func TestAndroidAppImport_Presigned(t *testing.T) { + ctx, _ := testJava(t, ` + android_app_import { + name: "foo", + apk: "prebuilts/apk/app.apk", + presigned: true, + dex_preopt: { + enabled: true, + }, + } + `) + + variant := ctx.ModuleForTests("foo", "android_common") + + // Check dexpreopt outputs. + if variant.MaybeOutput("dexpreopt/oat/arm64/package.vdex").Rule == nil || + variant.MaybeOutput("dexpreopt/oat/arm64/package.odex").Rule == nil { + t.Errorf("can't find dexpreopt outputs") + } + // Make sure signing was skipped and aligning was done. + if variant.MaybeOutput("signed/foo.apk").Rule != nil { + t.Errorf("signing rule shouldn't be included.") + } + if variant.MaybeOutput("zip-aligned/foo.apk").Rule == nil { + t.Errorf("can't find aligning rule") + } +} + +func TestAndroidAppImport_SigningLineage(t *testing.T) { + ctx, _ := testJava(t, ` + android_app_import { + name: "foo", + apk: "prebuilts/apk/app.apk", + certificate: "platform", + lineage: "lineage.bin", + } + `) + + variant := ctx.ModuleForTests("foo", "android_common") + + // Check cert signing lineage flag. + signedApk := variant.Output("signed/foo.apk") + signingFlag := signedApk.Args["flags"] + expected := "--lineage lineage.bin" + if expected != signingFlag { + t.Errorf("Incorrect signing flags, expected: %q, got: %q", expected, signingFlag) + } +} + +func TestAndroidAppImport_DefaultDevCert(t *testing.T) { + ctx, _ := testJava(t, ` + android_app_import { + name: "foo", + apk: "prebuilts/apk/app.apk", + default_dev_cert: true, + dex_preopt: { + enabled: true, + }, + } + `) + + variant := ctx.ModuleForTests("foo", "android_common") + + // Check dexpreopt outputs. + if variant.MaybeOutput("dexpreopt/oat/arm64/package.vdex").Rule == nil || + variant.MaybeOutput("dexpreopt/oat/arm64/package.odex").Rule == nil { + t.Errorf("can't find dexpreopt outputs") + } + + // Check cert signing flag. + signedApk := variant.Output("signed/foo.apk") + signingFlag := signedApk.Args["certificates"] + expected := "build/make/target/product/security/testkey.x509.pem build/make/target/product/security/testkey.pk8" + if expected != signingFlag { + t.Errorf("Incorrect signing flags, expected: %q, got: %q", expected, signingFlag) + } +} + +func TestAndroidAppImport_DpiVariants(t *testing.T) { + bp := ` + android_app_import { + name: "foo", + apk: "prebuilts/apk/app.apk", + dpi_variants: { + xhdpi: { + apk: "prebuilts/apk/app_xhdpi.apk", + }, + xxhdpi: { + apk: "prebuilts/apk/app_xxhdpi.apk", + }, + }, + presigned: true, + dex_preopt: { + enabled: true, + }, + } + ` + testCases := []struct { + name string + aaptPreferredConfig *string + aaptPrebuiltDPI []string + expected string + }{ + { + name: "no preferred", + aaptPreferredConfig: nil, + aaptPrebuiltDPI: []string{}, + expected: "prebuilts/apk/app.apk", + }, + { + name: "AAPTPreferredConfig matches", + aaptPreferredConfig: proptools.StringPtr("xhdpi"), + aaptPrebuiltDPI: []string{"xxhdpi", "ldpi"}, + expected: "prebuilts/apk/app_xhdpi.apk", + }, + { + name: "AAPTPrebuiltDPI matches", + aaptPreferredConfig: proptools.StringPtr("mdpi"), + aaptPrebuiltDPI: []string{"xxhdpi", "xhdpi"}, + expected: "prebuilts/apk/app_xxhdpi.apk", + }, + { + name: "non-first AAPTPrebuiltDPI matches", + aaptPreferredConfig: proptools.StringPtr("mdpi"), + aaptPrebuiltDPI: []string{"ldpi", "xhdpi"}, + expected: "prebuilts/apk/app_xhdpi.apk", + }, + { + name: "no matches", + aaptPreferredConfig: proptools.StringPtr("mdpi"), + aaptPrebuiltDPI: []string{"ldpi", "xxxhdpi"}, + expected: "prebuilts/apk/app.apk", + }, + } + + jniRuleRe := regexp.MustCompile("^if \\(zipinfo (\\S+)") + for _, test := range testCases { + config := testAppConfig(nil, bp, nil) + config.TestProductVariables.AAPTPreferredConfig = test.aaptPreferredConfig + config.TestProductVariables.AAPTPrebuiltDPI = test.aaptPrebuiltDPI + ctx := testContext() + + run(t, ctx, config) + + variant := ctx.ModuleForTests("foo", "android_common") + jniRuleCommand := variant.Output("jnis-uncompressed/foo.apk").RuleParams.Command + matches := jniRuleRe.FindStringSubmatch(jniRuleCommand) + if len(matches) != 2 { + t.Errorf("failed to extract the src apk path from %q", jniRuleCommand) + } + if test.expected != matches[1] { + t.Errorf("wrong src apk, expected: %q got: %q", test.expected, matches[1]) + } + } +} + +func TestAndroidAppImport_Filename(t *testing.T) { + ctx, config := testJava(t, ` + android_app_import { + name: "foo", + apk: "prebuilts/apk/app.apk", + presigned: true, + } + + android_app_import { + name: "bar", + apk: "prebuilts/apk/app.apk", + presigned: true, + filename: "bar_sample.apk" + } + `) + + testCases := []struct { + name string + expected string + }{ + { + name: "foo", + expected: "foo.apk", + }, + { + name: "bar", + expected: "bar_sample.apk", + }, + } + + for _, test := range testCases { + variant := ctx.ModuleForTests(test.name, "android_common") + if variant.MaybeOutput(test.expected).Rule == nil { + t.Errorf("can't find output named %q - all outputs: %v", test.expected, variant.AllOutputs()) + } + + a := variant.Module().(*AndroidAppImport) + expectedValues := []string{test.expected} + actualValues := android.AndroidMkEntriesForTest( + t, config, "", a)[0].EntryMap["LOCAL_INSTALLED_MODULE_STEM"] + if !reflect.DeepEqual(actualValues, expectedValues) { + t.Errorf("Incorrect LOCAL_INSTALLED_MODULE_STEM value '%s', expected '%s'", + actualValues, expectedValues) + } + } +} + +func TestAndroidAppImport_ArchVariants(t *testing.T) { + // The test config's target arch is ARM64. + testCases := []struct { + name string + bp string + expected string + }{ + { + name: "matching arch", + bp: ` + android_app_import { + name: "foo", + apk: "prebuilts/apk/app.apk", + arch: { + arm64: { + apk: "prebuilts/apk/app_arm64.apk", + }, + }, + presigned: true, + dex_preopt: { + enabled: true, + }, + } + `, + expected: "prebuilts/apk/app_arm64.apk", + }, + { + name: "no matching arch", + bp: ` + android_app_import { + name: "foo", + apk: "prebuilts/apk/app.apk", + arch: { + arm: { + apk: "prebuilts/apk/app_arm.apk", + }, + }, + presigned: true, + dex_preopt: { + enabled: true, + }, + } + `, + expected: "prebuilts/apk/app.apk", + }, + } + + jniRuleRe := regexp.MustCompile("^if \\(zipinfo (\\S+)") + for _, test := range testCases { + ctx, _ := testJava(t, test.bp) + + variant := ctx.ModuleForTests("foo", "android_common") + jniRuleCommand := variant.Output("jnis-uncompressed/foo.apk").RuleParams.Command + matches := jniRuleRe.FindStringSubmatch(jniRuleCommand) + if len(matches) != 2 { + t.Errorf("failed to extract the src apk path from %q", jniRuleCommand) + } + if test.expected != matches[1] { + t.Errorf("wrong src apk, expected: %q got: %q", test.expected, matches[1]) + } + } +} + +func TestAndroidTestImport(t *testing.T) { + ctx, config := testJava(t, ` + android_test_import { + name: "foo", + apk: "prebuilts/apk/app.apk", + presigned: true, + data: [ + "testdata/data", + ], + } + `) + + test := ctx.ModuleForTests("foo", "android_common").Module().(*AndroidTestImport) + + // Check android mks. + entries := android.AndroidMkEntriesForTest(t, config, "", test)[0] + expected := []string{"tests"} + actual := entries.EntryMap["LOCAL_MODULE_TAGS"] + if !reflect.DeepEqual(expected, actual) { + t.Errorf("Unexpected module tags - expected: %q, actual: %q", expected, actual) + } + expected = []string{"testdata/data:testdata/data"} + actual = entries.EntryMap["LOCAL_COMPATIBILITY_SUPPORT_FILES"] + if !reflect.DeepEqual(expected, actual) { + t.Errorf("Unexpected test data - expected: %q, actual: %q", expected, actual) + } +} + +func TestAndroidTestImport_NoJinUncompressForPresigned(t *testing.T) { + ctx, _ := testJava(t, ` + android_test_import { + name: "foo", + apk: "prebuilts/apk/app.apk", + certificate: "cert/new_cert", + data: [ + "testdata/data", + ], + } + + android_test_import { + name: "foo_presigned", + apk: "prebuilts/apk/app.apk", + presigned: true, + data: [ + "testdata/data", + ], + } + `) + + variant := ctx.ModuleForTests("foo", "android_common") + jniRule := variant.Output("jnis-uncompressed/foo.apk").RuleParams.Command + if !strings.HasPrefix(jniRule, "if (zipinfo") { + t.Errorf("Unexpected JNI uncompress rule command: " + jniRule) + } + + variant = ctx.ModuleForTests("foo_presigned", "android_common") + jniRule = variant.Output("jnis-uncompressed/foo_presigned.apk").BuildParams.Rule.String() + if jniRule != android.Cp.String() { + t.Errorf("Unexpected JNI uncompress rule: " + jniRule) + } + if variant.MaybeOutput("zip-aligned/foo_presigned.apk").Rule == nil { + t.Errorf("Presigned test apk should be aligned") + } +} + +func TestAndroidTestImport_Preprocessed(t *testing.T) { + ctx, _ := testJava(t, ` + android_test_import { + name: "foo", + apk: "prebuilts/apk/app.apk", + presigned: true, + preprocessed: true, + } + + android_test_import { + name: "foo_cert", + apk: "prebuilts/apk/app.apk", + certificate: "cert/new_cert", + preprocessed: true, + } + `) + + testModules := []string{"foo", "foo_cert"} + for _, m := range testModules { + apkName := m + ".apk" + variant := ctx.ModuleForTests(m, "android_common") + jniRule := variant.Output("jnis-uncompressed/" + apkName).BuildParams.Rule.String() + if jniRule != android.Cp.String() { + t.Errorf("Unexpected JNI uncompress rule: " + jniRule) + } + + // Make sure signing and aligning were skipped. + if variant.MaybeOutput("signed/"+apkName).Rule != nil { + t.Errorf("signing rule shouldn't be included for preprocessed.") + } + if variant.MaybeOutput("zip-aligned/"+apkName).Rule != nil { + t.Errorf("aligning rule shouldn't be for preprocessed") + } + } +} + +func TestStl(t *testing.T) { + ctx, _ := testJava(t, cc.GatherRequiredDepsForTest(android.Android)+` + cc_library { + name: "libjni", + sdk_version: "current", + stl: "c++_shared", + } + + android_test { + name: "stl", + jni_libs: ["libjni"], + compile_multilib: "both", + sdk_version: "current", + stl: "c++_shared", + } + + android_test { + name: "system", + jni_libs: ["libjni"], + compile_multilib: "both", + sdk_version: "current", + } + `) + + testCases := []struct { + name string + jnis []string + }{ + {"stl", + []string{ + "libjni.so", + "libc++_shared.so", + }, + }, + {"system", + []string{ + "libjni.so", + }, + }, + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + app := ctx.ModuleForTests(test.name, "android_common") + jniLibZip := app.Output("jnilibs.zip") + var jnis []string + args := strings.Fields(jniLibZip.Args["jarArgs"]) + for i := 0; i < len(args); i++ { + if args[i] == "-f" { + jnis = append(jnis, args[i+1]) + i += 1 + } + } + jnisJoined := strings.Join(jnis, " ") + for _, jni := range test.jnis { + if !strings.Contains(jnisJoined, jni) { + t.Errorf("missing jni %q in %q", jni, jnis) + } + } + }) + } +} + +func TestUsesLibraries(t *testing.T) { + bp := ` + java_sdk_library { + name: "foo", + srcs: ["a.java"], + api_packages: ["foo"], + sdk_version: "current", + } + + java_sdk_library { + name: "qux", + srcs: ["a.java"], + api_packages: ["qux"], + sdk_version: "current", + } + + java_sdk_library { + name: "quuz", + srcs: ["a.java"], + api_packages: ["quuz"], + sdk_version: "current", + } + + java_sdk_library { + name: "bar", + srcs: ["a.java"], + api_packages: ["bar"], + sdk_version: "current", + } + + android_app { + name: "app", + srcs: ["a.java"], + libs: ["qux", "quuz.stubs"], + uses_libs: ["foo"], + sdk_version: "current", + optional_uses_libs: [ + "bar", + "baz", + ], + } + + android_app_import { + name: "prebuilt", + apk: "prebuilts/apk/app.apk", + certificate: "platform", + uses_libs: ["foo"], + optional_uses_libs: [ + "bar", + "baz", + ], + } + ` + + config := testAppConfig(nil, bp, nil) + config.TestProductVariables.MissingUsesLibraries = []string{"baz"} + + ctx := testContext() + + run(t, ctx, config) + + app := ctx.ModuleForTests("app", "android_common") + prebuilt := ctx.ModuleForTests("prebuilt", "android_common") + + // Test that implicit dependencies on java_sdk_library instances are passed to the manifest. + manifestFixerArgs := app.Output("manifest_fixer/AndroidManifest.xml").Args["args"] + if w := "--uses-library qux"; !strings.Contains(manifestFixerArgs, w) { + t.Errorf("unexpected manifest_fixer args: wanted %q in %q", w, manifestFixerArgs) + } + if w := "--uses-library quuz"; !strings.Contains(manifestFixerArgs, w) { + t.Errorf("unexpected manifest_fixer args: wanted %q in %q", w, manifestFixerArgs) + } + + // Test that all libraries are verified + cmd := app.Rule("verify_uses_libraries").RuleParams.Command + if w := "--uses-library foo"; !strings.Contains(cmd, w) { + t.Errorf("wanted %q in %q", w, cmd) + } + + if w := "--optional-uses-library bar --optional-uses-library baz"; !strings.Contains(cmd, w) { + t.Errorf("wanted %q in %q", w, cmd) + } + + cmd = prebuilt.Rule("verify_uses_libraries").RuleParams.Command + + if w := `uses_library_names="foo"`; !strings.Contains(cmd, w) { + t.Errorf("wanted %q in %q", w, cmd) + } + + if w := `optional_uses_library_names="bar baz"`; !strings.Contains(cmd, w) { + t.Errorf("wanted %q in %q", w, cmd) + } + + // Test that only present libraries are preopted + cmd = app.Rule("dexpreopt").RuleParams.Command + + if w := `dex_preopt_target_libraries="/system/framework/foo.jar /system/framework/bar.jar"`; !strings.Contains(cmd, w) { + t.Errorf("wanted %q in %q", w, cmd) + } + + cmd = prebuilt.Rule("dexpreopt").RuleParams.Command + + if w := `dex_preopt_target_libraries="/system/framework/foo.jar /system/framework/bar.jar"`; !strings.Contains(cmd, w) { + t.Errorf("wanted %q in %q", w, cmd) + } +} + +func TestCodelessApp(t *testing.T) { + testCases := []struct { + name string + bp string + noCode bool + }{ + { + name: "normal", + bp: ` + android_app { + name: "foo", + srcs: ["a.java"], + sdk_version: "current", + } + `, + noCode: false, + }, + { + name: "app without sources", + bp: ` + android_app { + name: "foo", + sdk_version: "current", + } + `, + noCode: true, + }, + { + name: "app with libraries", + bp: ` + android_app { + name: "foo", + static_libs: ["lib"], + sdk_version: "current", + } + + java_library { + name: "lib", + srcs: ["a.java"], + sdk_version: "current", + } + `, + noCode: false, + }, + { + name: "app with sourceless libraries", + bp: ` + android_app { + name: "foo", + static_libs: ["lib"], + sdk_version: "current", + } + + java_library { + name: "lib", + sdk_version: "current", + } + `, + // TODO(jungjw): this should probably be true + noCode: false, + }, + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + ctx := testApp(t, test.bp) + + foo := ctx.ModuleForTests("foo", "android_common") + manifestFixerArgs := foo.Output("manifest_fixer/AndroidManifest.xml").Args["args"] + if strings.Contains(manifestFixerArgs, "--has-no-code") != test.noCode { + t.Errorf("unexpected manifest_fixer args: %q", manifestFixerArgs) + } + }) + } +} + func TestEmbedNotice(t *testing.T) { - ctx := testJava(t, cc.GatherRequiredDepsForTest(android.Android)+` + ctx, _ := testJavaWithFS(t, cc.GatherRequiredDepsForTest(android.Android)+` android_app { name: "foo", srcs: ["a.java"], @@ -1067,6 +2730,7 @@ func TestEmbedNotice(t *testing.T) { jni_libs: ["libjni"], notice: "APP_NOTICE", embed_notices: true, + sdk_version: "current", } // No embed_notice flag @@ -1075,6 +2739,7 @@ func TestEmbedNotice(t *testing.T) { srcs: ["a.java"], jni_libs: ["libjni"], notice: "APP_NOTICE", + sdk_version: "current", } // No NOTICE files @@ -1082,6 +2747,7 @@ func TestEmbedNotice(t *testing.T) { name: "baz", srcs: ["a.java"], embed_notices: true, + sdk_version: "current", } cc_library { @@ -1089,6 +2755,7 @@ func TestEmbedNotice(t *testing.T) { system_shared_libs: [], stl: "none", notice: "LIB_NOTICE", + sdk_version: "current", } java_library { @@ -1096,6 +2763,7 @@ func TestEmbedNotice(t *testing.T) { srcs: [ ":gen", ], + sdk_version: "current", } genrule { @@ -1110,7 +2778,12 @@ func TestEmbedNotice(t *testing.T) { srcs: ["b.java"], notice: "TOOL_NOTICE", } - `) + `, map[string][]byte{ + "APP_NOTICE": nil, + "GENRULE_NOTICE": nil, + "LIB_NOTICE": nil, + "TOOL_NOTICE": nil, + }) // foo has NOTICE files to process, and embed_notices is true. foo := ctx.ModuleForTests("foo", "android_common") @@ -1140,16 +2813,20 @@ func TestEmbedNotice(t *testing.T) { // bar has NOTICE files to process, but embed_notices is not set. bar := ctx.ModuleForTests("bar", "android_common") - mergeNotices = bar.MaybeRule("mergeNoticesRule") - if mergeNotices.Rule != nil { - t.Errorf("mergeNotices shouldn't have run for bar") + res = bar.Output("package-res.apk") + aapt2Flags = res.Args["flags"] + e = "-A " + buildDir + "/.intermediates/bar/android_common/NOTICE" + if strings.Contains(aapt2Flags, e) { + t.Errorf("bar shouldn't have the asset dir flag for NOTICE: %q", e) } // baz's embed_notice is true, but it doesn't have any NOTICE files. baz := ctx.ModuleForTests("baz", "android_common") - mergeNotices = baz.MaybeRule("mergeNoticesRule") - if mergeNotices.Rule != nil { - t.Errorf("mergeNotices shouldn't have run for baz") + res = baz.Output("package-res.apk") + aapt2Flags = res.Args["flags"] + e = "-A " + buildDir + "/.intermediates/baz/android_common/NOTICE" + if strings.Contains(aapt2Flags, e) { + t.Errorf("baz shouldn't have the asset dir flag for NOTICE: %q", e) } } @@ -1167,6 +2844,7 @@ func TestUncompressDex(t *testing.T) { android_app { name: "foo", srcs: ["a.java"], + sdk_version: "current", } `, uncompressedPlatform: true, @@ -1179,6 +2857,7 @@ func TestUncompressDex(t *testing.T) { name: "foo", use_embedded_dex: true, srcs: ["a.java"], + sdk_version: "current", } `, uncompressedPlatform: true, @@ -1191,22 +2870,49 @@ func TestUncompressDex(t *testing.T) { name: "foo", privileged: true, srcs: ["a.java"], + sdk_version: "current", + } + `, + uncompressedPlatform: true, + uncompressedUnbundled: true, + }, + { + name: "normal_uncompress_dex_true", + bp: ` + android_app { + name: "foo", + srcs: ["a.java"], + sdk_version: "current", + uncompress_dex: true, } `, uncompressedPlatform: true, uncompressedUnbundled: true, }, + { + name: "normal_uncompress_dex_false", + bp: ` + android_app { + name: "foo", + srcs: ["a.java"], + sdk_version: "current", + uncompress_dex: false, + } + `, + uncompressedPlatform: false, + uncompressedUnbundled: false, + }, } test := func(t *testing.T, bp string, want bool, unbundled bool) { t.Helper() - config := testConfig(nil) + config := testAppConfig(nil, bp, nil) if unbundled { config.TestProductVariables.Unbundled_build = proptools.BoolPtr(true) } - ctx := testAppContext(config, bp, nil) + ctx := testContext() run(t, ctx, config) @@ -1235,3 +2941,255 @@ func TestUncompressDex(t *testing.T) { }) } } + +func checkAapt2LinkFlag(t *testing.T, aapt2Flags, flagName, expectedValue string) { + if expectedValue != "" { + expectedFlag := "--" + flagName + " " + expectedValue + if !strings.Contains(aapt2Flags, expectedFlag) { + t.Errorf("%q is missing in aapt2 link flags, %q", expectedFlag, aapt2Flags) + } + } else { + unexpectedFlag := "--" + flagName + if strings.Contains(aapt2Flags, unexpectedFlag) { + t.Errorf("unexpected flag, %q is found in aapt2 link flags, %q", unexpectedFlag, aapt2Flags) + } + } +} + +func TestRuntimeResourceOverlay(t *testing.T) { + fs := map[string][]byte{ + "baz/res/res/values/strings.xml": nil, + "bar/res/res/values/strings.xml": nil, + } + bp := ` + runtime_resource_overlay { + name: "foo", + certificate: "platform", + lineage: "lineage.bin", + product_specific: true, + static_libs: ["bar"], + resource_libs: ["baz"], + aaptflags: ["--keep-raw-values"], + } + + runtime_resource_overlay { + name: "foo_themed", + certificate: "platform", + product_specific: true, + theme: "faza", + overrides: ["foo"], + } + + android_library { + name: "bar", + resource_dirs: ["bar/res"], + } + + android_app { + name: "baz", + sdk_version: "current", + resource_dirs: ["baz/res"], + } + ` + config := testAppConfig(nil, bp, fs) + ctx := testContext() + run(t, ctx, config) + + m := ctx.ModuleForTests("foo", "android_common") + + // Check AAPT2 link flags. + aapt2Flags := m.Output("package-res.apk").Args["flags"] + expectedFlags := []string{"--keep-raw-values", "--no-resource-deduping", "--no-resource-removal"} + absentFlags := android.RemoveListFromList(expectedFlags, strings.Split(aapt2Flags, " ")) + if len(absentFlags) > 0 { + t.Errorf("expected values, %q are missing in aapt2 link flags, %q", absentFlags, aapt2Flags) + } + + // Check overlay.list output for static_libs dependency. + overlayList := m.Output("aapt2/overlay.list").Inputs.Strings() + staticLibPackage := buildDir + "/.intermediates/bar/android_common/package-res.apk" + if !inList(staticLibPackage, overlayList) { + t.Errorf("Stactic lib res package %q missing in overlay list: %q", staticLibPackage, overlayList) + } + + // Check AAPT2 link flags for resource_libs dependency. + resourceLibFlag := "-I " + buildDir + "/.intermediates/baz/android_common/package-res.apk" + if !strings.Contains(aapt2Flags, resourceLibFlag) { + t.Errorf("Resource lib flag %q missing in aapt2 link flags: %q", resourceLibFlag, aapt2Flags) + } + + // Check cert signing flag. + signedApk := m.Output("signed/foo.apk") + lineageFlag := signedApk.Args["flags"] + expectedLineageFlag := "--lineage lineage.bin" + if expectedLineageFlag != lineageFlag { + t.Errorf("Incorrect signing lineage flags, expected: %q, got: %q", expectedLineageFlag, lineageFlag) + } + signingFlag := signedApk.Args["certificates"] + expected := "build/make/target/product/security/platform.x509.pem build/make/target/product/security/platform.pk8" + if expected != signingFlag { + t.Errorf("Incorrect signing flags, expected: %q, got: %q", expected, signingFlag) + } + androidMkEntries := android.AndroidMkEntriesForTest(t, config, "", m.Module())[0] + path := androidMkEntries.EntryMap["LOCAL_CERTIFICATE"] + expectedPath := []string{"build/make/target/product/security/platform.x509.pem"} + if !reflect.DeepEqual(path, expectedPath) { + t.Errorf("Unexpected LOCAL_CERTIFICATE value: %v, expected: %v", path, expectedPath) + } + + // Check device location. + path = androidMkEntries.EntryMap["LOCAL_MODULE_PATH"] + expectedPath = []string{"/tmp/target/product/test_device/product/overlay"} + if !reflect.DeepEqual(path, expectedPath) { + t.Errorf("Unexpected LOCAL_MODULE_PATH value: %v, expected: %v", path, expectedPath) + } + + // A themed module has a different device location + m = ctx.ModuleForTests("foo_themed", "android_common") + androidMkEntries = android.AndroidMkEntriesForTest(t, config, "", m.Module())[0] + path = androidMkEntries.EntryMap["LOCAL_MODULE_PATH"] + expectedPath = []string{"/tmp/target/product/test_device/product/overlay/faza"} + if !reflect.DeepEqual(path, expectedPath) { + t.Errorf("Unexpected LOCAL_MODULE_PATH value: %v, expected: %v", path, expectedPath) + } + + overrides := androidMkEntries.EntryMap["LOCAL_OVERRIDES_PACKAGES"] + expectedOverrides := []string{"foo"} + if !reflect.DeepEqual(overrides, expectedOverrides) { + t.Errorf("Unexpected LOCAL_OVERRIDES_PACKAGES value: %v, expected: %v", overrides, expectedOverrides) + } +} + +func TestRuntimeResourceOverlay_JavaDefaults(t *testing.T) { + ctx, config := testJava(t, ` + java_defaults { + name: "rro_defaults", + theme: "default_theme", + product_specific: true, + aaptflags: ["--keep-raw-values"], + } + + runtime_resource_overlay { + name: "foo_with_defaults", + defaults: ["rro_defaults"], + } + + runtime_resource_overlay { + name: "foo_barebones", + } + `) + + // + // RRO module with defaults + // + m := ctx.ModuleForTests("foo_with_defaults", "android_common") + + // Check AAPT2 link flags. + aapt2Flags := strings.Split(m.Output("package-res.apk").Args["flags"], " ") + expectedFlags := []string{"--keep-raw-values", "--no-resource-deduping", "--no-resource-removal"} + absentFlags := android.RemoveListFromList(expectedFlags, aapt2Flags) + if len(absentFlags) > 0 { + t.Errorf("expected values, %q are missing in aapt2 link flags, %q", absentFlags, aapt2Flags) + } + + // Check device location. + path := android.AndroidMkEntriesForTest(t, config, "", m.Module())[0].EntryMap["LOCAL_MODULE_PATH"] + expectedPath := []string{"/tmp/target/product/test_device/product/overlay/default_theme"} + if !reflect.DeepEqual(path, expectedPath) { + t.Errorf("Unexpected LOCAL_MODULE_PATH value: %q, expected: %q", path, expectedPath) + } + + // + // RRO module without defaults + // + m = ctx.ModuleForTests("foo_barebones", "android_common") + + // Check AAPT2 link flags. + aapt2Flags = strings.Split(m.Output("package-res.apk").Args["flags"], " ") + unexpectedFlags := "--keep-raw-values" + if inList(unexpectedFlags, aapt2Flags) { + t.Errorf("unexpected value, %q is present in aapt2 link flags, %q", unexpectedFlags, aapt2Flags) + } + + // Check device location. + path = android.AndroidMkEntriesForTest(t, config, "", m.Module())[0].EntryMap["LOCAL_MODULE_PATH"] + expectedPath = []string{"/tmp/target/product/test_device/system/overlay"} + if !reflect.DeepEqual(path, expectedPath) { + t.Errorf("Unexpected LOCAL_MODULE_PATH value: %v, expected: %v", path, expectedPath) + } +} + +func TestOverrideRuntimeResourceOverlay(t *testing.T) { + ctx, _ := testJava(t, ` + runtime_resource_overlay { + name: "foo_overlay", + certificate: "platform", + product_specific: true, + sdk_version: "current", + } + + override_runtime_resource_overlay { + name: "bar_overlay", + base: "foo_overlay", + package_name: "com.android.bar.overlay", + target_package_name: "com.android.bar", + } + `) + + expectedVariants := []struct { + moduleName string + variantName string + apkPath string + overrides []string + targetVariant string + packageFlag string + targetPackageFlag string + }{ + { + variantName: "android_common", + apkPath: "/target/product/test_device/product/overlay/foo_overlay.apk", + overrides: nil, + targetVariant: "android_common", + packageFlag: "", + targetPackageFlag: "", + }, + { + variantName: "android_common_bar_overlay", + apkPath: "/target/product/test_device/product/overlay/bar_overlay.apk", + overrides: []string{"foo_overlay"}, + targetVariant: "android_common_bar", + packageFlag: "com.android.bar.overlay", + targetPackageFlag: "com.android.bar", + }, + } + for _, expected := range expectedVariants { + variant := ctx.ModuleForTests("foo_overlay", expected.variantName) + + // Check the final apk name + outputs := variant.AllOutputs() + expectedApkPath := buildDir + expected.apkPath + found := false + for _, o := range outputs { + if o == expectedApkPath { + found = true + break + } + } + if !found { + t.Errorf("Can't find %q in output files.\nAll outputs:%v", expectedApkPath, outputs) + } + + // Check if the overrides field values are correctly aggregated. + mod := variant.Module().(*RuntimeResourceOverlay) + if !reflect.DeepEqual(expected.overrides, mod.properties.Overrides) { + t.Errorf("Incorrect overrides property value, expected: %q, got: %q", + expected.overrides, mod.properties.Overrides) + } + + // Check aapt2 flags. + res := variant.Output("package-res.apk") + aapt2Flags := res.Args["flags"] + checkAapt2LinkFlag(t, aapt2Flags, "rename-manifest-package", expected.packageFlag) + checkAapt2LinkFlag(t, aapt2Flags, "rename-overlay-target-package", expected.targetPackageFlag) + } +} diff --git a/java/builder.go b/java/builder.go index 0e7574e63..7318fcbad 100644 --- a/java/builder.go +++ b/java/builder.go @@ -40,16 +40,17 @@ var ( // (if the rule produces .class files) or a .srcjar file (if the rule produces .java files). // .srcjar files are unzipped into a temporary directory when compiled with javac. // TODO(b/143658984): goma can't handle the --system argument to javac. - javac, javacRE = remoteexec.StaticRules(pctx, "javac", + javac, javacRE = remoteexec.MultiCommandStaticRules(pctx, "javac", blueprint.RuleParams{ Command: `rm -rf "$outDir" "$annoDir" "$srcJarDir" && mkdir -p "$outDir" "$annoDir" "$srcJarDir" && ` + `${config.ZipSyncCmd} -d $srcJarDir -l $srcJarDir/list -f "*.java" $srcJars && ` + `(if [ -s $srcJarDir/list ] || [ -s $out.rsp ] ; then ` + - `${config.SoongJavacWrapper} $reTemplate${config.JavacCmd} ${config.JavacHeapFlags} ${config.CommonJdkFlags} ` + + `${config.SoongJavacWrapper} $javaTemplate${config.JavacCmd} ` + + `${config.JavacHeapFlags} ${config.JavacVmFlags} ${config.CommonJdkFlags} ` + `$processorpath $processor $javacFlags $bootClasspath $classpath ` + `-source $javaVersion -target $javaVersion ` + `-d $outDir -s $annoDir @$out.rsp @$srcJarDir/list ; fi ) && ` + - `${config.SoongZipCmd} -jar -o $out -C $outDir -D $outDir && ` + + `$zipTemplate${config.SoongZipCmd} -jar -o $out -C $outDir -D $outDir && ` + `rm -rf "$srcJarDir"`, CommandDeps: []string{ "${config.JavacCmd}", @@ -59,13 +60,59 @@ var ( CommandOrderOnly: []string{"${config.SoongJavacWrapper}"}, Rspfile: "$out.rsp", RspfileContent: "$in", - }, &remoteexec.REParams{ - Labels: map[string]string{"type": "compile", "lang": "java", "compiler": "javac"}, - ExecStrategy: "${config.REJavacExecStrategy}", - Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, + }, map[string]*remoteexec.REParams{ + "$javaTemplate": &remoteexec.REParams{ + Labels: map[string]string{"type": "compile", "lang": "java", "compiler": "javac"}, + ExecStrategy: "${config.REJavacExecStrategy}", + Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, + }, + "$zipTemplate": &remoteexec.REParams{ + Labels: map[string]string{"type": "tool", "name": "soong_zip"}, + Inputs: []string{"${config.SoongZipCmd}", "$outDir"}, + OutputFiles: []string{"$out"}, + ExecStrategy: "${config.REJavacExecStrategy}", + Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, + }, }, []string{"javacFlags", "bootClasspath", "classpath", "processorpath", "processor", "srcJars", "srcJarDir", "outDir", "annoDir", "javaVersion"}, nil) + _ = pctx.VariableFunc("kytheCorpus", + func(ctx android.PackageVarContext) string { return ctx.Config().XrefCorpusName() }) + _ = pctx.VariableFunc("kytheCuEncoding", + func(ctx android.PackageVarContext) string { return ctx.Config().XrefCuEncoding() }) + _ = pctx.SourcePathVariable("kytheVnames", "build/soong/vnames.json") + // Run it with -add-opens=java.base/java.nio=ALL-UNNAMED to avoid JDK9's warning about + // "Illegal reflective access by com.google.protobuf.Utf8$UnsafeProcessor ... + // to field java.nio.Buffer.address" + kytheExtract = pctx.AndroidStaticRule("kythe", + blueprint.RuleParams{ + Command: `${config.ZipSyncCmd} -d $srcJarDir ` + + `-l $srcJarDir/list -f "*.java" $srcJars && ` + + `( [ ! -s $srcJarDir/list -a ! -s $out.rsp ] || ` + + `KYTHE_ROOT_DIRECTORY=. KYTHE_OUTPUT_FILE=$out ` + + `KYTHE_CORPUS=${kytheCorpus} ` + + `KYTHE_VNAMES=${kytheVnames} ` + + `KYTHE_KZIP_ENCODING=${kytheCuEncoding} ` + + `${config.SoongJavacWrapper} ${config.JavaCmd} ` + + `--add-opens=java.base/java.nio=ALL-UNNAMED ` + + `-jar ${config.JavaKytheExtractorJar} ` + + `${config.JavacHeapFlags} ${config.CommonJdkFlags} ` + + `$processorpath $processor $javacFlags $bootClasspath $classpath ` + + `-source $javaVersion -target $javaVersion ` + + `-d $outDir -s $annoDir @$out.rsp @$srcJarDir/list)`, + CommandDeps: []string{ + "${config.JavaCmd}", + "${config.JavaKytheExtractorJar}", + "${kytheVnames}", + "${config.ZipSyncCmd}", + }, + CommandOrderOnly: []string{"${config.SoongJavacWrapper}"}, + Rspfile: "$out.rsp", + RspfileContent: "$in", + }, + "javacFlags", "bootClasspath", "classpath", "processorpath", "processor", "srcJars", "srcJarDir", + "outDir", "annoDir", "javaVersion") + extractMatchingApks = pctx.StaticRule( "extractMatchingApks", blueprint.RuleParams{ @@ -79,10 +126,10 @@ var ( }, "abis", "allow-prereleased", "screen-densities", "sdk-version", "stem", "apkcerts", "partition") - turbine = pctx.AndroidStaticRule("turbine", + turbine, turbineRE = remoteexec.StaticRules(pctx, "turbine", blueprint.RuleParams{ Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` + - `${config.JavaCmd} -jar ${config.TurbineJar} --output $out.tmp ` + + `$reTemplate${config.JavaCmd} ${config.JavaVmFlags} -jar ${config.TurbineJar} --output $out.tmp ` + `--temp_dir "$outDir" --sources @$out.rsp --source_jars $srcJars ` + `--javacopts ${config.CommonJdkFlags} ` + `$javacFlags -source $javaVersion -target $javaVersion -- $bootClasspath $classpath && ` + @@ -97,25 +144,45 @@ var ( RspfileContent: "$in", Restat: true, }, - "javacFlags", "bootClasspath", "classpath", "srcJars", "outDir", "javaVersion") - - jar = pctx.AndroidStaticRule("jar", + &remoteexec.REParams{Labels: map[string]string{"type": "tool", "name": "turbine"}, + ExecStrategy: "${config.RETurbineExecStrategy}", + Inputs: []string{"${config.TurbineJar}", "${out}.rsp", "$implicits"}, + RSPFile: "${out}.rsp", + OutputFiles: []string{"$out.tmp"}, + OutputDirectories: []string{"$outDir"}, + ToolchainInputs: []string{"${config.JavaCmd}"}, + Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, + }, []string{"javacFlags", "bootClasspath", "classpath", "srcJars", "outDir", "javaVersion"}, []string{"implicits"}) + + jar, jarRE = remoteexec.StaticRules(pctx, "jar", blueprint.RuleParams{ - Command: `${config.SoongZipCmd} -jar -o $out @$out.rsp`, + Command: `$reTemplate${config.SoongZipCmd} -jar -o $out @$out.rsp`, CommandDeps: []string{"${config.SoongZipCmd}"}, Rspfile: "$out.rsp", RspfileContent: "$jarArgs", }, - "jarArgs") + &remoteexec.REParams{ + ExecStrategy: "${config.REJarExecStrategy}", + Inputs: []string{"${config.SoongZipCmd}", "${out}.rsp"}, + RSPFile: "${out}.rsp", + OutputFiles: []string{"$out"}, + Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, + }, []string{"jarArgs"}, nil) - zip = pctx.AndroidStaticRule("zip", + zip, zipRE = remoteexec.StaticRules(pctx, "zip", blueprint.RuleParams{ Command: `${config.SoongZipCmd} -o $out @$out.rsp`, CommandDeps: []string{"${config.SoongZipCmd}"}, Rspfile: "$out.rsp", RspfileContent: "$jarArgs", }, - "jarArgs") + &remoteexec.REParams{ + ExecStrategy: "${config.REZipExecStrategy}", + Inputs: []string{"${config.SoongZipCmd}", "${out}.rsp", "$implicits"}, + RSPFile: "${out}.rsp", + OutputFiles: []string{"$out"}, + Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, + }, []string{"jarArgs"}, []string{"implicits"}) combineJar = pctx.AndroidStaticRule("combineJar", blueprint.RuleParams{ @@ -126,7 +193,12 @@ var ( jarjar = pctx.AndroidStaticRule("jarjar", blueprint.RuleParams{ - Command: "${config.JavaCmd} -jar ${config.JarjarCmd} process $rulesFile $in $out", + Command: "${config.JavaCmd} ${config.JavaVmFlags}" + + // b/146418363 Enable Android specific jarjar transformer to drop compat annotations + // for newly repackaged classes. Dropping @UnsupportedAppUsage on repackaged classes + // avoids adding new hiddenapis after jarjar'ing. + " -DremoveAndroidCompatAnnotations=true" + + " -jar ${config.JarjarCmd} process $rulesFile $in $out", CommandDeps: []string{"${config.JavaCmd}", "${config.JarjarCmd}", "$rulesFile"}, }, "rulesFile") @@ -142,7 +214,7 @@ var ( jetifier = pctx.AndroidStaticRule("jetifier", blueprint.RuleParams{ - Command: "${config.JavaCmd} -jar ${config.JetifierJar} -l error -o $out -i $in", + Command: "${config.JavaCmd} ${config.JavaVmFlags} -jar ${config.JetifierJar} -l error -o $out -i $in", CommandDeps: []string{"${config.JavaCmd}", "${config.JetifierJar}"}, }, ) @@ -166,15 +238,16 @@ func init() { } type javaBuilderFlags struct { - javacFlags string - bootClasspath classpath - classpath classpath - processorPath classpath - processor string - systemModules classpath - aidlFlags string - aidlDeps android.Paths - javaVersion string + javacFlags string + bootClasspath classpath + classpath classpath + java9Classpath classpath + processorPath classpath + processors []string + systemModules *systemModules + aidlFlags string + aidlDeps android.Paths + javaVersion javaVersion errorProneExtraJavacFlags string errorProneProcessorPath classpath @@ -214,37 +287,115 @@ func RunErrorProne(ctx android.ModuleContext, outputFile android.WritablePath, "errorprone", "errorprone") } +// Emits the rule to generate Xref input file (.kzip file) for the given set of source files and source jars +// to compile with given set of builder flags, etc. +func emitXrefRule(ctx android.ModuleContext, xrefFile android.WritablePath, idx int, + srcFiles, srcJars android.Paths, + flags javaBuilderFlags, deps android.Paths) { + + deps = append(deps, srcJars...) + classpath := flags.classpath + + var bootClasspath string + if flags.javaVersion.usesJavaModules() { + var systemModuleDeps android.Paths + bootClasspath, systemModuleDeps = flags.systemModules.FormJavaSystemModulesPath(ctx.Device()) + deps = append(deps, systemModuleDeps...) + classpath = append(flags.java9Classpath, classpath...) + } else { + deps = append(deps, flags.bootClasspath...) + if len(flags.bootClasspath) == 0 && ctx.Device() { + // explicitly specify -bootclasspath "" if the bootclasspath is empty to + // ensure java does not fall back to the default bootclasspath. + bootClasspath = `-bootclasspath ""` + } else { + bootClasspath = flags.bootClasspath.FormJavaClassPath("-bootclasspath") + } + } + + deps = append(deps, classpath...) + deps = append(deps, flags.processorPath...) + + processor := "-proc:none" + if len(flags.processors) > 0 { + processor = "-processor " + strings.Join(flags.processors, ",") + } + + intermediatesDir := "xref" + if idx >= 0 { + intermediatesDir += strconv.Itoa(idx) + } + + ctx.Build(pctx, + android.BuildParams{ + Rule: kytheExtract, + Description: "Xref Java extractor", + Output: xrefFile, + Inputs: srcFiles, + Implicits: deps, + Args: map[string]string{ + "annoDir": android.PathForModuleOut(ctx, intermediatesDir, "anno").String(), + "bootClasspath": bootClasspath, + "classpath": classpath.FormJavaClassPath("-classpath"), + "javacFlags": flags.javacFlags, + "javaVersion": flags.javaVersion.String(), + "outDir": android.PathForModuleOut(ctx, "javac", "classes.xref").String(), + "processorpath": flags.processorPath.FormJavaClassPath("-processorpath"), + "processor": processor, + "srcJarDir": android.PathForModuleOut(ctx, intermediatesDir, "srcjars.xref").String(), + "srcJars": strings.Join(srcJars.Strings(), " "), + }, + }) +} + func TransformJavaToHeaderClasses(ctx android.ModuleContext, outputFile android.WritablePath, srcFiles, srcJars android.Paths, flags javaBuilderFlags) { var deps android.Paths deps = append(deps, srcJars...) - deps = append(deps, flags.bootClasspath...) - deps = append(deps, flags.classpath...) + + classpath := flags.classpath var bootClasspath string - if len(flags.bootClasspath) == 0 && ctx.Device() { - // explicitly specify -bootclasspath "" if the bootclasspath is empty to - // ensure java does not fall back to the default bootclasspath. - bootClasspath = `--bootclasspath ""` + if flags.javaVersion.usesJavaModules() { + var systemModuleDeps android.Paths + bootClasspath, systemModuleDeps = flags.systemModules.FormTurbineSystemModulesPath(ctx.Device()) + deps = append(deps, systemModuleDeps...) + classpath = append(flags.java9Classpath, classpath...) } else { - bootClasspath = strings.Join(flags.bootClasspath.FormTurbineClasspath("--bootclasspath "), " ") + deps = append(deps, flags.bootClasspath...) + if len(flags.bootClasspath) == 0 && ctx.Device() { + // explicitly specify -bootclasspath "" if the bootclasspath is empty to + // ensure turbine does not fall back to the default bootclasspath. + bootClasspath = `--bootclasspath ""` + } else { + bootClasspath = flags.bootClasspath.FormTurbineClassPath("--bootclasspath ") + } } + deps = append(deps, classpath...) + deps = append(deps, flags.processorPath...) + + rule := turbine + args := map[string]string{ + "javacFlags": flags.javacFlags, + "bootClasspath": bootClasspath, + "srcJars": strings.Join(srcJars.Strings(), " "), + "classpath": classpath.FormTurbineClassPath("--classpath "), + "outDir": android.PathForModuleOut(ctx, "turbine", "classes").String(), + "javaVersion": flags.javaVersion.String(), + } + if ctx.Config().IsEnvTrue("RBE_TURBINE") { + rule = turbineRE + args["implicits"] = strings.Join(deps.Strings(), ",") + } ctx.Build(pctx, android.BuildParams{ - Rule: turbine, + Rule: rule, Description: "turbine", Output: outputFile, Inputs: srcFiles, Implicits: deps, - Args: map[string]string{ - "javacFlags": flags.javacFlags, - "bootClasspath": bootClasspath, - "srcJars": strings.Join(srcJars.Strings(), " "), - "classpath": strings.Join(flags.classpath.FormTurbineClasspath("--classpath "), " "), - "outDir": android.PathForModuleOut(ctx, "turbine", "classes").String(), - "javaVersion": flags.javaVersion, - }, + Args: args, }) } @@ -264,10 +415,14 @@ func transformJavaToClasses(ctx android.ModuleContext, outputFile android.Writab deps = append(deps, srcJars...) + classpath := flags.classpath + var bootClasspath string - if flags.javaVersion == "1.9" { - deps = append(deps, flags.systemModules...) - bootClasspath = flags.systemModules.FormJavaSystemModulesPath("--system=", ctx.Device()) + if flags.javaVersion.usesJavaModules() { + var systemModuleDeps android.Paths + bootClasspath, systemModuleDeps = flags.systemModules.FormJavaSystemModulesPath(ctx.Device()) + deps = append(deps, systemModuleDeps...) + classpath = append(flags.java9Classpath, classpath...) } else { deps = append(deps, flags.bootClasspath...) if len(flags.bootClasspath) == 0 && ctx.Device() { @@ -279,12 +434,12 @@ func transformJavaToClasses(ctx android.ModuleContext, outputFile android.Writab } } - deps = append(deps, flags.classpath...) + deps = append(deps, classpath...) deps = append(deps, flags.processorPath...) processor := "-proc:none" - if flags.processor != "" { - processor = "-processor " + flags.processor + if len(flags.processors) > 0 { + processor = "-processor " + strings.Join(flags.processors, ",") } srcJarDir := "srcjars" @@ -309,14 +464,14 @@ func transformJavaToClasses(ctx android.ModuleContext, outputFile android.Writab Args: map[string]string{ "javacFlags": flags.javacFlags, "bootClasspath": bootClasspath, - "classpath": flags.classpath.FormJavaClassPath("-classpath"), + "classpath": classpath.FormJavaClassPath("-classpath"), "processorpath": flags.processorPath.FormJavaClassPath("-processorpath"), "processor": processor, "srcJars": strings.Join(srcJars.Strings(), " "), "srcJarDir": android.PathForModuleOut(ctx, intermediatesDir, srcJarDir).String(), "outDir": android.PathForModuleOut(ctx, intermediatesDir, outDir).String(), "annoDir": android.PathForModuleOut(ctx, intermediatesDir, annoDir).String(), - "javaVersion": flags.javaVersion, + "javaVersion": flags.javaVersion.String(), }, }) } @@ -324,8 +479,12 @@ func transformJavaToClasses(ctx android.ModuleContext, outputFile android.Writab func TransformResourcesToJar(ctx android.ModuleContext, outputFile android.WritablePath, jarArgs []string, deps android.Paths) { + rule := jar + if ctx.Config().IsEnvTrue("RBE_JAR") { + rule = jarRE + } ctx.Build(pctx, android.BuildParams{ - Rule: jar, + Rule: rule, Description: "jar", Output: outputFile, Implicits: deps, @@ -432,35 +591,28 @@ func TransformZipAlign(ctx android.ModuleContext, outputFile android.WritablePat }) } -type classpath []android.Path +type classpath android.Paths -func (x *classpath) FormJavaClassPath(optName string) string { +func (x *classpath) formJoinedClassPath(optName string, sep string) string { if optName != "" && !strings.HasSuffix(optName, "=") && !strings.HasSuffix(optName, " ") { optName += " " } if len(*x) > 0 { - return optName + strings.Join(x.Strings(), ":") + return optName + strings.Join(x.Strings(), sep) } else { return "" } } +func (x *classpath) FormJavaClassPath(optName string) string { + return x.formJoinedClassPath(optName, ":") +} -// Returns a --system argument in the form javac expects with -source 1.9. If forceEmpty is true, -// returns --system=none if the list is empty to ensure javac does not fall back to the default -// system modules. -func (x *classpath) FormJavaSystemModulesPath(optName string, forceEmpty bool) string { - if len(*x) > 1 { - panic("more than one system module") - } else if len(*x) == 1 { - return optName + strings.TrimSuffix((*x)[0].String(), "lib/modules") - } else if forceEmpty { - return optName + "none" - } else { - return "" - } +func (x *classpath) FormTurbineClassPath(optName string) string { + return x.formJoinedClassPath(optName, " ") } -func (x *classpath) FormTurbineClasspath(optName string) []string { +// FormRepeatedClassPath returns a list of arguments with the given optName prefixed to each element of the classpath. +func (x *classpath) FormRepeatedClassPath(optName string) []string { if x == nil || *x == nil { return nil } @@ -487,3 +639,34 @@ func (x *classpath) Strings() []string { } return ret } + +type systemModules struct { + dir android.Path + deps android.Paths +} + +// Returns a --system argument in the form javac expects with -source 1.9 and the list of files to +// depend on. If forceEmpty is true, returns --system=none if the list is empty to ensure javac +// does not fall back to the default system modules. +func (x *systemModules) FormJavaSystemModulesPath(forceEmpty bool) (string, android.Paths) { + if x != nil { + return "--system=" + x.dir.String(), x.deps + } else if forceEmpty { + return "--system=none", nil + } else { + return "", nil + } +} + +// Returns a --system argument in the form turbine expects with -source 1.9 and the list of files to +// depend on. If forceEmpty is true, returns --bootclasspath "" if the list is empty to ensure turbine +// does not fall back to the default bootclasspath. +func (x *systemModules) FormTurbineSystemModulesPath(forceEmpty bool) (string, android.Paths) { + if x != nil { + return "--system " + x.dir.String(), x.deps + } else if forceEmpty { + return `--bootclasspath ""`, nil + } else { + return "", nil + } +} diff --git a/java/config/Android.bp b/java/config/Android.bp new file mode 100644 index 000000000..198352187 --- /dev/null +++ b/java/config/Android.bp @@ -0,0 +1,15 @@ +bootstrap_go_package { + name: "soong-java-config", + pkgPath: "android/soong/java/config", + deps: [ + "blueprint-proptools", + "soong-android", + "soong-remoteexec", + ], + srcs: [ + "config.go", + "error_prone.go", + "kotlin.go", + "makevars.go", + ], +} diff --git a/java/config/config.go b/java/config/config.go index c3523bee4..1d0dd617e 100644 --- a/java/config/config.go +++ b/java/config/config.go @@ -30,31 +30,43 @@ var ( DefaultBootclasspathLibraries = []string{"core.platform.api.stubs", "core-lambda-stubs"} DefaultSystemModules = "core-platform-api-stubs-system-modules" - DefaultLibraries = []string{"ext", "framework", "updatable_media_stubs"} + DefaultLibraries = []string{"ext", "framework"} DefaultLambdaStubsLibrary = "core-lambda-stubs" SdkLambdaStubsPath = "prebuilts/sdk/tools/core-lambda-stubs.jar" - DefaultJacocoExcludeFilter = []string{"org.junit.*", "org.jacoco.*", "org.mockito.*"} + DefaultMakeJacocoExcludeFilter = []string{"org.junit.*", "org.jacoco.*", "org.mockito.*"} + DefaultJacocoExcludeFilter = []string{"org.junit.**", "org.jacoco.**", "org.mockito.**"} InstrumentFrameworkModules = []string{ "framework", + "framework-minus-apex", "telephony-common", "services", "android.car", "android.car7", "conscrypt", + "core-icu4j", "core-oj", "core-libart", + // TODO: Could this be all updatable bootclasspath jars? "updatable-media", + "framework-mediaprovider", + "framework-sdkextensions", + "android.net.ipsec.ike", } ) +const ( + JavaVmFlags = `-XX:OnError="cat hs_err_pid%p.log" -XX:CICompilerCount=6 -XX:+UseDynamicNumberOfGCThreads` + JavacVmFlags = `-J-XX:OnError="cat hs_err_pid%p.log" -J-XX:CICompilerCount=6 -J-XX:+UseDynamicNumberOfGCThreads` +) + func init() { pctx.Import("github.com/google/blueprint/bootstrap") pctx.StaticVariable("JavacHeapSize", "2048M") pctx.StaticVariable("JavacHeapFlags", "-J-Xmx${JavacHeapSize}") - pctx.StaticVariable("DexFlags", "-JXX:+TieredCompilation -JXX:TieredStopAtLevel=1") + pctx.StaticVariable("DexFlags", "-JXX:OnError='cat hs_err_pid%p.log' -JXX:CICompilerCount=6 -JXX:+UseDynamicNumberOfGCThreads") pctx.StaticVariable("CommonJdkFlags", strings.Join([]string{ `-Xmaxerrs 9999999`, @@ -72,12 +84,21 @@ func init() { `-XDstringConcat=inline`, }, " ")) + pctx.StaticVariable("JavaVmFlags", JavaVmFlags) + pctx.StaticVariable("JavacVmFlags", JavacVmFlags) + pctx.VariableConfigMethod("hostPrebuiltTag", android.Config.PrebuiltOS) pctx.VariableFunc("JavaHome", func(ctx android.PackageVarContext) string { // This is set up and guaranteed by soong_ui return ctx.Config().Getenv("ANDROID_JAVA_HOME") }) + pctx.VariableFunc("JlinkVersion", func(ctx android.PackageVarContext) string { + if override := ctx.Config().Getenv("OVERRIDE_JLINK_VERSION_NUMBER"); override != "" { + return override + } + return "11" + }) pctx.SourcePathVariable("JavaToolchain", "${JavaHome}/bin") pctx.SourcePathVariableWithEnvOverride("JavacCmd", @@ -88,6 +109,7 @@ func init() { pctx.SourcePathVariable("JlinkCmd", "${JavaToolchain}/jlink") pctx.SourcePathVariable("JmodCmd", "${JavaToolchain}/jmod") pctx.SourcePathVariable("JrtFsJar", "${JavaHome}/lib/jrt-fs.jar") + pctx.SourcePathVariable("JavaKytheExtractorJar", "prebuilts/build-tools/common/framework/javac_extractor.jar") pctx.SourcePathVariable("Ziptime", "prebuilts/build-tools/${hostPrebuiltTag}/bin/ziptime") pctx.SourcePathVariable("GenKotlinBuildFileCmd", "build/soong/scripts/gen-kotlin-build-file.sh") @@ -109,7 +131,7 @@ func init() { if ctx.Config().UnbundledBuild() { return "prebuilts/build-tools/common/framework/" + turbine } else { - return pctx.HostJavaToolPath(ctx, turbine).String() + return ctx.Config().HostJavaToolPath(ctx, turbine).String() } }) @@ -129,27 +151,109 @@ func init() { pctx.VariableFunc("REJavacExecStrategy", remoteexec.EnvOverrideFunc("RBE_JAVAC_EXEC_STRATEGY", remoteexec.RemoteLocalFallbackExecStrategy)) pctx.VariableFunc("RED8ExecStrategy", remoteexec.EnvOverrideFunc("RBE_D8_EXEC_STRATEGY", remoteexec.RemoteLocalFallbackExecStrategy)) pctx.VariableFunc("RER8ExecStrategy", remoteexec.EnvOverrideFunc("RBE_R8_EXEC_STRATEGY", remoteexec.RemoteLocalFallbackExecStrategy)) + pctx.VariableFunc("RETurbineExecStrategy", remoteexec.EnvOverrideFunc("RBE_TURBINE_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) + pctx.VariableFunc("RESignApkExecStrategy", remoteexec.EnvOverrideFunc("RBE_SIGNAPK_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) + pctx.VariableFunc("REJarExecStrategy", remoteexec.EnvOverrideFunc("RBE_JAR_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) + pctx.VariableFunc("REZipExecStrategy", remoteexec.EnvOverrideFunc("RBE_ZIP_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) pctx.HostJavaToolVariable("JacocoCLIJar", "jacoco-cli.jar") - hostBinToolVariableWithPrebuilt := func(name, prebuiltDir, tool string) { - pctx.VariableFunc(name, func(ctx android.PackageVarContext) string { - if ctx.Config().UnbundledBuild() || ctx.Config().IsPdkBuild() { - return filepath.Join(prebuiltDir, runtime.GOOS, "bin", tool) - } else { - return pctx.HostBinToolPath(ctx, tool).String() + pctx.HostBinToolVariable("ManifestCheckCmd", "manifest_check") + pctx.HostBinToolVariable("ManifestFixerCmd", "manifest_fixer") + + pctx.HostBinToolVariable("ManifestMergerCmd", "manifest-merger") + + pctx.HostBinToolVariable("Class2Greylist", "class2greylist") + pctx.HostBinToolVariable("HiddenAPI", "hiddenapi") + + hostBinToolVariableWithSdkToolsPrebuilt("Aapt2Cmd", "aapt2") + hostBinToolVariableWithBuildToolsPrebuilt("AidlCmd", "aidl") + hostBinToolVariableWithBuildToolsPrebuilt("ZipAlign", "zipalign") + + hostJavaToolVariableWithSdkToolsPrebuilt("SignapkCmd", "signapk") + // TODO(ccross): this should come from the signapk dependencies, but we don't have any way + // to express host JNI dependencies yet. + hostJNIToolVariableWithSdkToolsPrebuilt("SignapkJniLibrary", "libconscrypt_openjdk_jni") +} + +func hostBinToolVariableWithSdkToolsPrebuilt(name, tool string) { + pctx.VariableFunc(name, func(ctx android.PackageVarContext) string { + if ctx.Config().UnbundledBuild() || ctx.Config().IsPdkBuild() { + return filepath.Join("prebuilts/sdk/tools", runtime.GOOS, "bin", tool) + } else { + return ctx.Config().HostToolPath(ctx, tool).String() + } + }) +} + +func hostJavaToolVariableWithSdkToolsPrebuilt(name, tool string) { + pctx.VariableFunc(name, func(ctx android.PackageVarContext) string { + if ctx.Config().UnbundledBuild() || ctx.Config().IsPdkBuild() { + return filepath.Join("prebuilts/sdk/tools/lib", tool+".jar") + } else { + return ctx.Config().HostJavaToolPath(ctx, tool+".jar").String() + } + }) +} + +func hostJNIToolVariableWithSdkToolsPrebuilt(name, tool string) { + pctx.VariableFunc(name, func(ctx android.PackageVarContext) string { + if ctx.Config().UnbundledBuild() || ctx.Config().IsPdkBuild() { + ext := ".so" + if runtime.GOOS == "darwin" { + ext = ".dylib" } - }) - } + return filepath.Join("prebuilts/sdk/tools", runtime.GOOS, "lib64", tool+ext) + } else { + return ctx.Config().HostJNIToolPath(ctx, tool).String() + } + }) +} - hostBinToolVariableWithPrebuilt("Aapt2Cmd", "prebuilts/sdk/tools", "aapt2") +func hostBinToolVariableWithBuildToolsPrebuilt(name, tool string) { + pctx.VariableFunc(name, func(ctx android.PackageVarContext) string { + if ctx.Config().UnbundledBuild() || ctx.Config().IsPdkBuild() { + return filepath.Join("prebuilts/build-tools", ctx.Config().PrebuiltOS(), "bin", tool) + } else { + return ctx.Config().HostToolPath(ctx, tool).String() + } + }) +} - pctx.SourcePathVariable("ManifestFixerCmd", "build/soong/scripts/manifest_fixer.py") +// JavaCmd returns a SourcePath object with the path to the java command. +func JavaCmd(ctx android.PathContext) android.SourcePath { + return javaTool(ctx, "java") +} - pctx.HostBinToolVariable("ManifestMergerCmd", "manifest-merger") +// JavadocCmd returns a SourcePath object with the path to the java command. +func JavadocCmd(ctx android.PathContext) android.SourcePath { + return javaTool(ctx, "javadoc") +} - pctx.HostBinToolVariable("ZipAlign", "zipalign") +func javaTool(ctx android.PathContext, tool string) android.SourcePath { + type javaToolKey string - pctx.HostBinToolVariable("Class2Greylist", "class2greylist") - pctx.HostBinToolVariable("HiddenAPI", "hiddenapi") + key := android.NewCustomOnceKey(javaToolKey(tool)) + + return ctx.Config().OnceSourcePath(key, func() android.SourcePath { + return javaToolchain(ctx).Join(ctx, tool) + }) + +} + +var javaToolchainKey = android.NewOnceKey("javaToolchain") + +func javaToolchain(ctx android.PathContext) android.SourcePath { + return ctx.Config().OnceSourcePath(javaToolchainKey, func() android.SourcePath { + return javaHome(ctx).Join(ctx, "bin") + }) +} + +var javaHomeKey = android.NewOnceKey("javaHome") + +func javaHome(ctx android.PathContext) android.SourcePath { + return ctx.Config().OnceSourcePath(javaHomeKey, func() android.SourcePath { + // This is set up and guaranteed by soong_ui + return android.PathForSource(ctx, ctx.Config().Getenv("ANDROID_JAVA_HOME")) + }) } diff --git a/java/config/kotlin.go b/java/config/kotlin.go index 2af7b3c3a..fd8e3dbe9 100644 --- a/java/config/kotlin.go +++ b/java/config/kotlin.go @@ -27,7 +27,12 @@ var ( func init() { pctx.SourcePathVariable("KotlincCmd", "external/kotlinc/bin/kotlinc") pctx.SourcePathVariable("KotlinCompilerJar", "external/kotlinc/lib/kotlin-compiler.jar") + pctx.SourcePathVariable("KotlinPreloaderJar", "external/kotlinc/lib/kotlin-preloader.jar") + pctx.SourcePathVariable("KotlinReflectJar", "external/kotlinc/lib/kotlin-reflect.jar") + pctx.SourcePathVariable("KotlinScriptRuntimeJar", "external/kotlinc/lib/kotlin-script-runtime.jar") + pctx.SourcePathVariable("KotlinTrove4jJar", "external/kotlinc/lib/trove4j.jar") pctx.SourcePathVariable("KotlinKaptJar", "external/kotlinc/lib/kotlin-annotation-processing.jar") + pctx.SourcePathVariable("KotlinAnnotationJar", "external/kotlinc/lib/annotations-13.0.jar") pctx.SourcePathVariable("KotlinStdlibJar", KotlinStdlibJar) // These flags silence "Illegal reflective access" warnings when running kotlinc in OpenJDK9 diff --git a/java/config/makevars.go b/java/config/makevars.go index 6881caff6..b355fad87 100644 --- a/java/config/makevars.go +++ b/java/config/makevars.go @@ -29,18 +29,13 @@ func makeVarsProvider(ctx android.MakeVarsContext) { ctx.Strict("TARGET_DEFAULT_BOOTCLASSPATH_LIBRARIES", strings.Join(DefaultBootclasspathLibraries, " ")) ctx.Strict("DEFAULT_SYSTEM_MODULES", DefaultSystemModules) - if ctx.Config().TargetOpenJDK9() { - ctx.Strict("DEFAULT_JAVA_LANGUAGE_VERSION", "1.9") - } else { - ctx.Strict("DEFAULT_JAVA_LANGUAGE_VERSION", "1.8") - } - ctx.Strict("ANDROID_JAVA_HOME", "${JavaHome}") ctx.Strict("ANDROID_JAVA8_HOME", "prebuilts/jdk/jdk8/${hostPrebuiltTag}") ctx.Strict("ANDROID_JAVA9_HOME", "prebuilts/jdk/jdk9/${hostPrebuiltTag}") + ctx.Strict("ANDROID_JAVA11_HOME", "prebuilts/jdk/jdk11/${hostPrebuiltTag}") ctx.Strict("ANDROID_JAVA_TOOLCHAIN", "${JavaToolchain}") - ctx.Strict("JAVA", "${JavaCmd}") - ctx.Strict("JAVAC", "${JavacCmd}") + ctx.Strict("JAVA", "${JavaCmd} ${JavaVmFlags}") + ctx.Strict("JAVAC", "${JavacCmd} ${JavacVmFlags}") ctx.Strict("JAR", "${JarCmd}") ctx.Strict("JAR_ARGS", "${JarArgsCmd}") ctx.Strict("JAVADOC", "${JavadocCmd}") @@ -58,8 +53,8 @@ func makeVarsProvider(ctx android.MakeVarsContext) { ctx.Strict("ERROR_PRONE_CHECKS", "${ErrorProneChecks}") } - ctx.Strict("TARGET_JAVAC", "${JavacCmd} ${CommonJdkFlags}") - ctx.Strict("HOST_JAVAC", "${JavacCmd} ${CommonJdkFlags}") + ctx.Strict("TARGET_JAVAC", "${JavacCmd} ${JavacVmFlags} ${CommonJdkFlags}") + ctx.Strict("HOST_JAVAC", "${JavacCmd} ${JavacVmFlags} ${CommonJdkFlags}") ctx.Strict("JLINK", "${JlinkCmd}") ctx.Strict("JMOD", "${JmodCmd}") @@ -69,10 +64,11 @@ func makeVarsProvider(ctx android.MakeVarsContext) { ctx.Strict("ZIPSYNC", "${ZipSyncCmd}") ctx.Strict("JACOCO_CLI_JAR", "${JacocoCLIJar}") - ctx.Strict("DEFAULT_JACOCO_EXCLUDE_FILTER", strings.Join(DefaultJacocoExcludeFilter, ",")) + ctx.Strict("DEFAULT_JACOCO_EXCLUDE_FILTER", strings.Join(DefaultMakeJacocoExcludeFilter, ",")) ctx.Strict("EXTRACT_JAR_PACKAGES", "${ExtractJarPackagesCmd}") + ctx.Strict("MANIFEST_CHECK", "${ManifestCheckCmd}") ctx.Strict("MANIFEST_FIXER", "${ManifestFixerCmd}") ctx.Strict("ANDROID_MANIFEST_MERGER", "${ManifestMergerCmd}") @@ -81,4 +77,17 @@ func makeVarsProvider(ctx android.MakeVarsContext) { ctx.Strict("HIDDENAPI", "${HiddenAPI}") ctx.Strict("DEX_FLAGS", "${DexFlags}") + + ctx.Strict("AIDL", "${AidlCmd}") + ctx.Strict("AAPT2", "${Aapt2Cmd}") + ctx.Strict("ZIPALIGN", "${ZipAlign}") + ctx.Strict("SIGNAPK_JAR", "${SignapkCmd}") + ctx.Strict("SIGNAPK_JNI_LIBRARY_PATH", "${SignapkJniLibrary}") + + ctx.Strict("SOONG_ZIP", "${SoongZipCmd}") + ctx.Strict("MERGE_ZIPS", "${MergeZipsCmd}") + ctx.Strict("ZIP2ZIP", "${Zip2ZipCmd}") + + ctx.Strict("ZIPTIME", "${Ziptime}") + } diff --git a/java/device_host_converter.go b/java/device_host_converter.go index 9f40a6c0d..11e68eb6c 100644 --- a/java/device_host_converter.go +++ b/java/device_host_converter.go @@ -15,9 +15,10 @@ package java import ( - "android/soong/android" + "fmt" + "io" - "github.com/google/blueprint" + "android/soong/android" ) type DeviceHostConverter struct { @@ -30,6 +31,12 @@ type DeviceHostConverter struct { implementationJars android.Paths implementationAndResourceJars android.Paths resourceJars android.Paths + + srcJarArgs []string + srcJarDeps android.Paths + + combinedHeaderJar android.Path + combinedImplementationJar android.Path } type DeviceHostConverterProperties struct { @@ -44,7 +51,7 @@ type DeviceForHost struct { // java_device_for_host makes the classes.jar output of a device java_library module available to host // java_library modules. // -// It is rarely necessary, and its used is restricted to a few whitelisted projects. +// It is rarely necessary, and its usage is restricted to a few allowed projects. func DeviceForHostFactory() android.Module { module := &DeviceForHost{} @@ -61,7 +68,7 @@ type HostForDevice struct { // java_host_for_device makes the classes.jar output of a host java_library module available to device // java_library modules. // -// It is rarely necessary, and its used is restricted to a few whitelisted projects. +// It is rarely necessary, and its usage is restricted to a few allowed projects. func HostForDeviceFactory() android.Module { module := &HostForDevice{} @@ -74,13 +81,13 @@ func HostForDeviceFactory() android.Module { var deviceHostConverterDepTag = dependencyTag{name: "device_host_converter"} func (d *DeviceForHost) DepsMutator(ctx android.BottomUpMutatorContext) { - variation := []blueprint.Variation{{Mutator: "arch", Variation: "android_common"}} - ctx.AddFarVariationDependencies(variation, deviceHostConverterDepTag, d.properties.Libs...) + ctx.AddFarVariationDependencies(ctx.Config().AndroidCommonTarget.Variations(), + deviceHostConverterDepTag, d.properties.Libs...) } func (d *HostForDevice) DepsMutator(ctx android.BottomUpMutatorContext) { - variation := []blueprint.Variation{{Mutator: "arch", Variation: ctx.Config().BuildOsCommonVariant}} - ctx.AddFarVariationDependencies(variation, deviceHostConverterDepTag, d.properties.Libs...) + ctx.AddFarVariationDependencies(ctx.Config().BuildOSCommonTarget.Variations(), + deviceHostConverterDepTag, d.properties.Libs...) } func (d *DeviceHostConverter) GenerateAndroidBuildActions(ctx android.ModuleContext) { @@ -94,10 +101,35 @@ func (d *DeviceHostConverter) GenerateAndroidBuildActions(ctx android.ModuleCont d.implementationJars = append(d.implementationJars, dep.ImplementationJars()...) d.implementationAndResourceJars = append(d.implementationAndResourceJars, dep.ImplementationAndResourcesJars()...) d.resourceJars = append(d.resourceJars, dep.ResourceJars()...) + + srcJarArgs, srcJarDeps := dep.SrcJarArgs() + d.srcJarArgs = append(d.srcJarArgs, srcJarArgs...) + d.srcJarDeps = append(d.srcJarDeps, srcJarDeps...) } else { ctx.PropertyErrorf("libs", "module %q cannot be used as a dependency", ctx.OtherModuleName(m)) } }) + + jarName := ctx.ModuleName() + ".jar" + + if len(d.implementationAndResourceJars) > 1 { + outputFile := android.PathForModuleOut(ctx, "combined", jarName) + TransformJarsToJar(ctx, outputFile, "combine", d.implementationAndResourceJars, + android.OptionalPath{}, false, nil, nil) + d.combinedImplementationJar = outputFile + } else { + d.combinedImplementationJar = d.implementationAndResourceJars[0] + } + + if len(d.headerJars) > 1 { + outputFile := android.PathForModuleOut(ctx, "turbine-combined", jarName) + TransformJarsToJar(ctx, outputFile, "turbine combine", d.headerJars, + android.OptionalPath{}, false, nil, []string{"META-INF/TRANSITIVE"}) + d.combinedHeaderJar = outputFile + } else { + d.combinedHeaderJar = d.headerJars[0] + } + } var _ Dependency = (*DeviceHostConverter)(nil) @@ -129,3 +161,30 @@ func (d *DeviceHostConverter) AidlIncludeDirs() android.Paths { func (d *DeviceHostConverter) ExportedSdkLibs() []string { return nil } + +func (d *DeviceHostConverter) ExportedPlugins() (android.Paths, []string) { + return nil, nil +} + +func (d *DeviceHostConverter) SrcJarArgs() ([]string, android.Paths) { + return d.srcJarArgs, d.srcJarDeps +} + +func (d *DeviceHostConverter) JacocoReportClassesFile() android.Path { + return nil +} + +func (d *DeviceHostConverter) AndroidMk() android.AndroidMkData { + return android.AndroidMkData{ + Class: "JAVA_LIBRARIES", + OutputFile: android.OptionalPathForPath(d.combinedImplementationJar), + Include: "$(BUILD_SYSTEM)/soong_java_prebuilt.mk", + Extra: []android.AndroidMkExtraFunc{ + func(w io.Writer, outputFile android.Path) { + fmt.Fprintln(w, "LOCAL_UNINSTALLABLE_MODULE := true") + fmt.Fprintln(w, "LOCAL_SOONG_HEADER_JAR :=", d.combinedHeaderJar.String()) + fmt.Fprintln(w, "LOCAL_SOONG_CLASSES_JAR :=", d.combinedImplementationJar.String()) + }, + }, + } +} diff --git a/java/device_host_converter_test.go b/java/device_host_converter_test.go index 146bf6f40..3c9a0f3f1 100644 --- a/java/device_host_converter_test.go +++ b/java/device_host_converter_test.go @@ -50,9 +50,7 @@ func TestDeviceForHost(t *testing.T) { } ` - config := testConfig(nil) - ctx := testContext(config, bp, nil) - run(t, ctx, config) + ctx, config := testJava(t, bp) deviceModule := ctx.ModuleForTests("device_module", "android_common") deviceTurbineCombined := deviceModule.Output("turbine-combined/device_module.jar") @@ -62,7 +60,7 @@ func TestDeviceForHost(t *testing.T) { deviceImportModule := ctx.ModuleForTests("device_import_module", "android_common") deviceImportCombined := deviceImportModule.Output("combined/device_import_module.jar") - hostModule := ctx.ModuleForTests("host_module", config.BuildOsCommonVariant) + hostModule := ctx.ModuleForTests("host_module", config.BuildOSCommonTarget.String()) hostJavac := hostModule.Output("javac/host_module.jar") hostRes := hostModule.Output("res/host_module.jar") combined := hostModule.Output("combined/host_module.jar") @@ -126,22 +124,20 @@ func TestHostForDevice(t *testing.T) { java_library { name: "device_module", - no_framework_libs: true, + sdk_version: "core_platform", srcs: ["b.java"], java_resources: ["java-res/b/b"], static_libs: ["host_for_device_module"], } ` - config := testConfig(nil) - ctx := testContext(config, bp, nil) - run(t, ctx, config) + ctx, config := testJava(t, bp) - hostModule := ctx.ModuleForTests("host_module", config.BuildOsCommonVariant) + hostModule := ctx.ModuleForTests("host_module", config.BuildOSCommonTarget.String()) hostJavac := hostModule.Output("javac/host_module.jar") hostRes := hostModule.Output("res/host_module.jar") - hostImportModule := ctx.ModuleForTests("host_import_module", config.BuildOsCommonVariant) + hostImportModule := ctx.ModuleForTests("host_import_module", config.BuildOSCommonTarget.String()) hostImportCombined := hostImportModule.Output("combined/host_import_module.jar") deviceModule := ctx.ModuleForTests("device_module", "android_common") diff --git a/java/dex.go b/java/dex.go index 45fa068d2..9e61e95ad 100644 --- a/java/dex.go +++ b/java/dex.go @@ -18,53 +18,72 @@ import ( "strings" "github.com/google/blueprint" + "github.com/google/blueprint/proptools" "android/soong/android" "android/soong/remoteexec" ) -var d8, d8RE = remoteexec.StaticRules(pctx, "d8", +var d8, d8RE = remoteexec.MultiCommandStaticRules(pctx, "d8", blueprint.RuleParams{ Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` + - `$reTemplate${config.D8Cmd} ${config.DexFlags} --output $outDir $d8Flags $in && ` + - `${config.SoongZipCmd} $zipFlags -o $outDir/classes.dex.jar -C $outDir -f "$outDir/classes*.dex" && ` + + `$d8Template${config.D8Cmd} ${config.DexFlags} --output $outDir $d8Flags $in && ` + + `$zipTemplate${config.SoongZipCmd} $zipFlags -o $outDir/classes.dex.jar -C $outDir -f "$outDir/classes*.dex" && ` + `${config.MergeZipsCmd} -D -stripFile "**/*.class" $out $outDir/classes.dex.jar $in`, CommandDeps: []string{ "${config.D8Cmd}", "${config.SoongZipCmd}", "${config.MergeZipsCmd}", }, - }, &remoteexec.REParams{ - Labels: map[string]string{"type": "compile", "compiler": "d8"}, - Inputs: []string{"${config.D8Jar}"}, - ExecStrategy: "${config.RED8ExecStrategy}", - ToolchainInputs: []string{"${config.JavaCmd}"}, - Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, + }, map[string]*remoteexec.REParams{ + "$d8Template": &remoteexec.REParams{ + Labels: map[string]string{"type": "compile", "compiler": "d8"}, + Inputs: []string{"${config.D8Jar}"}, + ExecStrategy: "${config.RED8ExecStrategy}", + ToolchainInputs: []string{"${config.JavaCmd}"}, + Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, + }, + "$zipTemplate": &remoteexec.REParams{ + Labels: map[string]string{"type": "tool", "name": "soong_zip"}, + Inputs: []string{"${config.SoongZipCmd}", "$outDir"}, + OutputFiles: []string{"$outDir/classes.dex.jar"}, + ExecStrategy: "${config.RED8ExecStrategy}", + Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, + }, }, []string{"outDir", "d8Flags", "zipFlags"}, nil) -var r8, r8RE = remoteexec.StaticRules(pctx, "r8", +var r8, r8RE = remoteexec.MultiCommandStaticRules(pctx, "r8", blueprint.RuleParams{ Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` + `rm -f "$outDict" && ` + - `$reTemplate${config.R8Cmd} ${config.DexFlags} -injars $in --output $outDir ` + + `$r8Template${config.R8Cmd} ${config.DexFlags} -injars $in --output $outDir ` + `--force-proguard-compatibility ` + `--no-data-resources ` + `-printmapping $outDict ` + `$r8Flags && ` + `touch "$outDict" && ` + - `${config.SoongZipCmd} $zipFlags -o $outDir/classes.dex.jar -C $outDir -f "$outDir/classes*.dex" && ` + + `$zipTemplate${config.SoongZipCmd} $zipFlags -o $outDir/classes.dex.jar -C $outDir -f "$outDir/classes*.dex" && ` + `${config.MergeZipsCmd} -D -stripFile "**/*.class" $out $outDir/classes.dex.jar $in`, CommandDeps: []string{ "${config.R8Cmd}", "${config.SoongZipCmd}", "${config.MergeZipsCmd}", }, - }, &remoteexec.REParams{ - Labels: map[string]string{"type": "compile", "compiler": "r8"}, - Inputs: []string{"$implicits", "${config.R8Jar}"}, - ExecStrategy: "${config.RER8ExecStrategy}", - ToolchainInputs: []string{"${config.JavaCmd}"}, - Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, + }, map[string]*remoteexec.REParams{ + "$r8Template": &remoteexec.REParams{ + Labels: map[string]string{"type": "compile", "compiler": "r8"}, + Inputs: []string{"$implicits", "${config.R8Jar}"}, + ExecStrategy: "${config.RER8ExecStrategy}", + ToolchainInputs: []string{"${config.JavaCmd}"}, + Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, + }, + "$zipTemplate": &remoteexec.REParams{ + Labels: map[string]string{"type": "tool", "name": "soong_zip"}, + Inputs: []string{"${config.SoongZipCmd}", "$outDir"}, + OutputFiles: []string{"$outDir/classes.dex.jar"}, + ExecStrategy: "${config.RER8ExecStrategy}", + Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, + }, }, []string{"outDir", "outDict", "r8Flags", "zipFlags"}, []string{"implicits"}) func (j *Module) dexCommonFlags(ctx android.ModuleContext) []string { @@ -84,20 +103,20 @@ func (j *Module) dexCommonFlags(ctx android.ModuleContext) []string { "--verbose") } - minSdkVersion, err := sdkVersionToNumberAsString(ctx, j.minSdkVersion()) + minSdkVersion, err := j.minSdkVersion().effectiveVersion(ctx) if err != nil { ctx.PropertyErrorf("min_sdk_version", "%s", err) } - flags = append(flags, "--min-api "+minSdkVersion) + flags = append(flags, "--min-api "+minSdkVersion.asNumberString()) return flags } func (j *Module) d8Flags(ctx android.ModuleContext, flags javaBuilderFlags) ([]string, android.Paths) { d8Flags := j.dexCommonFlags(ctx) - d8Flags = append(d8Flags, flags.bootClasspath.FormTurbineClasspath("--lib ")...) - d8Flags = append(d8Flags, flags.classpath.FormTurbineClasspath("--lib ")...) + d8Flags = append(d8Flags, flags.bootClasspath.FormRepeatedClassPath("--lib ")...) + d8Flags = append(d8Flags, flags.classpath.FormRepeatedClassPath("--lib ")...) var d8Deps android.Paths d8Deps = append(d8Deps, flags.bootClasspath...) @@ -126,7 +145,6 @@ func (j *Module) r8Flags(ctx android.ModuleContext, flags javaBuilderFlags) (r8F r8Flags = append(r8Flags, proguardRaiseDeps.FormJavaClassPath("-libraryjars")) r8Flags = append(r8Flags, flags.bootClasspath.FormJavaClassPath("-libraryjars")) r8Flags = append(r8Flags, flags.classpath.FormJavaClassPath("-libraryjars")) - r8Flags = append(r8Flags, "-forceprocessing") r8Deps = append(r8Deps, proguardRaiseDeps...) r8Deps = append(r8Deps, flags.bootClasspath...) @@ -189,7 +207,7 @@ func (j *Module) compileDex(ctx android.ModuleContext, flags javaBuilderFlags, outDir := android.PathForModuleOut(ctx, "dex") zipFlags := "--ignore_missing_files" - if j.deviceProperties.UncompressDex { + if proptools.Bool(j.deviceProperties.Uncompress_dex) { zipFlags += " -L 0" } @@ -236,7 +254,7 @@ func (j *Module) compileDex(ctx android.ModuleContext, flags javaBuilderFlags, }, }) } - if j.deviceProperties.UncompressDex { + if proptools.Bool(j.deviceProperties.Uncompress_dex) { alignedJavalibJar := android.PathForModuleOut(ctx, "aligned", jarName) TransformZipAlign(ctx, alignedJavalibJar, javalibJar) javalibJar = alignedJavalibJar diff --git a/java/dexpreopt.go b/java/dexpreopt.go index 9141f9ef9..28a2c8ae6 100644 --- a/java/dexpreopt.go +++ b/java/dexpreopt.go @@ -19,27 +19,34 @@ import ( "android/soong/dexpreopt" ) +type dexpreopterInterface interface { + IsInstallable() bool // Structs that embed dexpreopter must implement this. + dexpreoptDisabled(ctx android.BaseModuleContext) bool +} + type dexpreopter struct { dexpreoptProperties DexpreoptProperties - installPath android.OutputPath - uncompressedDex bool - isSDKLibrary bool - isTest bool - isInstallable bool + installPath android.InstallPath + uncompressedDex bool + isSDKLibrary bool + isTest bool + isPresignedPrebuilt bool + + manifestFile android.Path + usesLibs []string + optionalUsesLibs []string + enforceUsesLibs bool + libraryPaths map[string]android.Path builtInstalled string } type DexpreoptProperties struct { Dex_preopt struct { - // If false, prevent dexpreopting and stripping the dex file from the final jar. Defaults to - // true. + // If false, prevent dexpreopting. Defaults to true. Enabled *bool - // If true, never strip the dex files from the final jar when dexpreopting. Defaults to false. - No_stripping *bool - // If true, generate an app image (.art file) for this module. App_image *bool @@ -51,12 +58,16 @@ type DexpreoptProperties struct { // If set, provides the path to profile relative to the Android.bp file. If not set, // defaults to searching for a file that matches the name of this module in the default // profile location set by PRODUCT_DEX_PREOPT_PROFILE_DIR, or empty if not found. - Profile *string + Profile *string `android:"path"` } } -func (d *dexpreopter) dexpreoptDisabled(ctx android.ModuleContext) bool { - global := dexpreoptGlobalConfig(ctx) +func init() { + dexpreopt.DexpreoptRunningInSoong = true +} + +func (d *dexpreopter) dexpreoptDisabled(ctx android.BaseModuleContext) bool { + global := dexpreopt.GetGlobalConfig(ctx) if global.DisablePreopt { return true @@ -78,7 +89,16 @@ func (d *dexpreopter) dexpreoptDisabled(ctx android.ModuleContext) bool { return true } - if !d.isInstallable { + if !ctx.Module().(dexpreopterInterface).IsInstallable() { + return true + } + + if ctx.Host() { + return true + } + + // Don't preopt APEX variant module + if am, ok := ctx.Module().(android.ApexModule); ok && !am.IsForPlatform() { return true } @@ -87,51 +107,63 @@ func (d *dexpreopter) dexpreoptDisabled(ctx android.ModuleContext) bool { return false } -func odexOnSystemOther(ctx android.ModuleContext, installPath android.OutputPath) bool { - return dexpreopt.OdexOnSystemOtherByName(ctx.ModuleName(), android.InstallPathToOnDevicePath(ctx, installPath), dexpreoptGlobalConfig(ctx)) +func dexpreoptToolDepsMutator(ctx android.BottomUpMutatorContext) { + if d, ok := ctx.Module().(dexpreopterInterface); !ok || d.dexpreoptDisabled(ctx) { + return + } + dexpreopt.RegisterToolDeps(ctx) +} + +func odexOnSystemOther(ctx android.ModuleContext, installPath android.InstallPath) bool { + return dexpreopt.OdexOnSystemOtherByName(ctx.ModuleName(), android.InstallPathToOnDevicePath(ctx, installPath), dexpreopt.GetGlobalConfig(ctx)) } func (d *dexpreopter) dexpreopt(ctx android.ModuleContext, dexJarFile android.ModuleOutPath) android.ModuleOutPath { - if d.dexpreoptDisabled(ctx) { + // TODO(b/148690468): The check on d.installPath is to bail out in cases where + // the dexpreopter struct hasn't been fully initialized before we're called, + // e.g. in aar.go. This keeps the behaviour that dexpreopting is effectively + // disabled, even if installable is true. + if d.dexpreoptDisabled(ctx) || d.installPath.Base() == "." { return dexJarFile } - global := dexpreoptGlobalConfig(ctx) + globalSoong := dexpreopt.GetGlobalSoongConfig(ctx) + global := dexpreopt.GetGlobalConfig(ctx) bootImage := defaultBootImageConfig(ctx) - defaultBootImage := bootImage - if global.UseApexImage { - bootImage = apexBootImageConfig(ctx) + dexFiles := bootImage.dexPathsDeps.Paths() + dexLocations := bootImage.dexLocationsDeps + if global.UseArtImage { + bootImage = artBootImageConfig(ctx) } - var archs []android.ArchType - for _, a := range ctx.MultiTargets() { - archs = append(archs, a.Arch.ArchType) - } - if len(archs) == 0 { + targets := ctx.MultiTargets() + if len(targets) == 0 { // assume this is a java library, dexpreopt for all arches for now for _, target := range ctx.Config().Targets[android.Android] { - archs = append(archs, target.Arch.ArchType) + if target.NativeBridge == android.NativeBridgeDisabled { + targets = append(targets, target) + } } if inList(ctx.ModuleName(), global.SystemServerJars) && !d.isSDKLibrary { // If the module is not an SDK library and it's a system server jar, only preopt the primary arch. - archs = archs[:1] + targets = targets[:1] } } - if ctx.Config().SecondArchIsTranslated() { - // Only preopt primary arch for translated arch since there is only an image there. - archs = archs[:1] - } + var archs []android.ArchType var images android.Paths - for _, arch := range archs { - images = append(images, bootImage.images[arch]) + var imagesDeps []android.OutputPaths + for _, target := range targets { + archs = append(archs, target.Arch.ArchType) + variant := bootImage.getVariant(target) + images = append(images, variant.images) + imagesDeps = append(imagesDeps, variant.imagesDeps) } dexLocation := android.InstallPathToOnDevicePath(ctx, d.installPath) - strippedDexJarFile := android.PathForModuleOut(ctx, "dexpreopt", dexJarFile.Base()) - var profileClassListing android.OptionalPath + var profileBootListing android.OptionalPath profileIsTextListing := false if BoolDefault(d.dexpreoptProperties.Dex_preopt.Profile_guided, true) { // If dex_preopt.profile_guided is not set, default it based on the existence of the @@ -139,6 +171,8 @@ func (d *dexpreopter) dexpreopt(ctx android.ModuleContext, dexJarFile android.Mo if String(d.dexpreoptProperties.Dex_preopt.Profile) != "" { profileClassListing = android.OptionalPathForPath( android.PathForModuleSrc(ctx, String(d.dexpreoptProperties.Dex_preopt.Profile))) + profileBootListing = android.ExistentPathForSource(ctx, + ctx.ModuleDir(), String(d.dexpreoptProperties.Dex_preopt.Profile)+"-boot") profileIsTextListing = true } else { profileClassListing = android.ExistentPathForSource(ctx, @@ -146,43 +180,42 @@ func (d *dexpreopter) dexpreopt(ctx android.ModuleContext, dexJarFile android.Mo } } - dexpreoptConfig := dexpreopt.ModuleConfig{ + dexpreoptConfig := &dexpreopt.ModuleConfig{ Name: ctx.ModuleName(), DexLocation: dexLocation, BuildPath: android.PathForModuleOut(ctx, "dexpreopt", ctx.ModuleName()+".jar").OutputPath, DexPath: dexJarFile, + ManifestPath: d.manifestFile, UncompressedDex: d.uncompressedDex, HasApkLibraries: false, PreoptFlags: nil, ProfileClassListing: profileClassListing, ProfileIsTextListing: profileIsTextListing, + ProfileBootListing: profileBootListing, - EnforceUsesLibraries: false, - OptionalUsesLibraries: nil, - UsesLibraries: nil, - LibraryPaths: nil, + EnforceUsesLibraries: d.enforceUsesLibs, + PresentOptionalUsesLibraries: d.optionalUsesLibs, + UsesLibraries: d.usesLibs, + LibraryPaths: d.libraryPaths, - Archs: archs, - DexPreoptImages: images, + Archs: archs, + DexPreoptImages: images, + DexPreoptImagesDeps: imagesDeps, + DexPreoptImageLocations: bootImage.imageLocations, - // We use the dex paths and dex locations of the default boot image, as it - // contains the full dexpreopt boot classpath. Other images may just contain a subset of - // the dexpreopt boot classpath. - PreoptBootClassPathDexFiles: defaultBootImage.dexPaths.Paths(), - PreoptBootClassPathDexLocations: defaultBootImage.dexLocations, + PreoptBootClassPathDexFiles: dexFiles, + PreoptBootClassPathDexLocations: dexLocations, PreoptExtractedApk: false, NoCreateAppImage: !BoolDefault(d.dexpreoptProperties.Dex_preopt.App_image, true), ForceCreateAppImage: BoolDefault(d.dexpreoptProperties.Dex_preopt.App_image, false), - NoStripping: Bool(d.dexpreoptProperties.Dex_preopt.No_stripping), - StripInputPath: dexJarFile, - StripOutputPath: strippedDexJarFile.OutputPath, + PresignedPrebuilt: d.isPresignedPrebuilt, } - dexpreoptRule, err := dexpreopt.GenerateDexpreoptRule(ctx, global, dexpreoptConfig) + dexpreoptRule, err := dexpreopt.GenerateDexpreoptRule(ctx, globalSoong, global, dexpreoptConfig) if err != nil { ctx.ModuleErrorf("error generating dexpreopt rule: %s", err.Error()) return dexJarFile @@ -192,13 +225,5 @@ func (d *dexpreopter) dexpreopt(ctx android.ModuleContext, dexJarFile android.Mo d.builtInstalled = dexpreoptRule.Installs().String() - stripRule, err := dexpreopt.GenerateStripRule(global, dexpreoptConfig) - if err != nil { - ctx.ModuleErrorf("error generating dexpreopt strip rule: %s", err.Error()) - return dexJarFile - } - - stripRule.Build(pctx, ctx, "dexpreopt_strip", "dexpreopt strip") - - return strippedDexJarFile + return dexJarFile } diff --git a/java/dexpreopt_bootjars.go b/java/dexpreopt_bootjars.go index 4d87b2f77..2f0cbdb8c 100644 --- a/java/dexpreopt_bootjars.go +++ b/java/dexpreopt_bootjars.go @@ -22,12 +22,11 @@ import ( "android/soong/android" "android/soong/dexpreopt" - "github.com/google/blueprint/pathtools" "github.com/google/blueprint/proptools" ) func init() { - android.RegisterSingletonType("dex_bootjars", dexpreoptBootJarsFactory) + RegisterDexpreoptBootJarsComponents(android.InitRegistrationContext) } // The image "location" is a symbolic path that with multiarchitecture @@ -49,36 +48,108 @@ func init() { // The location is passed as an argument to the ART tools like dex2oat instead of the real path. The ART tools // will then reconstruct the real path, so the rules must have a dependency on the real path. +// Target-independent description of pre-compiled boot image. type bootImageConfig struct { - name string - modules []string - dexLocations []string - dexPaths android.WritablePaths - dir android.OutputPath - symbolsDir android.OutputPath - images map[android.ArchType]android.OutputPath -} + // Whether this image is an extension. + extension bool + + // Image name (used in directory names and ninja rule names). + name string + + // Basename of the image: the resulting filenames are <stem>[-<jar>].{art,oat,vdex}. + stem string + + // Output directory for the image files. + dir android.OutputPath + + // Output directory for the image files with debug symbols. + symbolsDir android.OutputPath + + // Subdirectory where the image files are installed. + installSubdir string + + // The names of jars that constitute this image. + modules []string + + // The "locations" of jars. + dexLocations []string // for this image + dexLocationsDeps []string // for the dependency images and in this image -type bootImage struct { - bootImageConfig + // File paths to jars. + dexPaths android.WritablePaths // for this image + dexPathsDeps android.WritablePaths // for the dependency images and in this image - installs map[android.ArchType]android.RuleBuilderInstalls - vdexInstalls map[android.ArchType]android.RuleBuilderInstalls - unstrippedInstalls map[android.ArchType]android.RuleBuilderInstalls + // The "locations" of the dependency images and in this image. + imageLocations []string + // File path to a zip archive with all image files (or nil, if not needed). + zip android.WritablePath + + // Rules which should be used in make to install the outputs. profileInstalls android.RuleBuilderInstalls + + // Target-dependent fields. + variants []*bootImageVariant +} + +// Target-dependent description of pre-compiled boot image. +type bootImageVariant struct { + *bootImageConfig + + // Target for which the image is generated. + target android.Target + + // Paths to image files. + images android.OutputPath // first image file + imagesDeps android.OutputPaths // all files + + // Only for extensions, paths to the primary boot images. + primaryImages android.OutputPath + + // Rules which should be used in make to install the outputs. + installs android.RuleBuilderInstalls + vdexInstalls android.RuleBuilderInstalls + unstrippedInstalls android.RuleBuilderInstalls } -func newBootImage(ctx android.PathContext, config bootImageConfig) *bootImage { - image := &bootImage{ - bootImageConfig: config, +func (image bootImageConfig) getVariant(target android.Target) *bootImageVariant { + for _, variant := range image.variants { + if variant.target.Os == target.Os && variant.target.Arch.ArchType == target.Arch.ArchType { + return variant + } + } + return nil +} - installs: make(map[android.ArchType]android.RuleBuilderInstalls), - vdexInstalls: make(map[android.ArchType]android.RuleBuilderInstalls), - unstrippedInstalls: make(map[android.ArchType]android.RuleBuilderInstalls), +func (image bootImageConfig) moduleName(idx int) string { + // Dexpreopt on the boot class path produces multiple files. The first dex file + // is converted into 'name'.art (to match the legacy assumption that 'name'.art + // exists), and the rest are converted to 'name'-<jar>.art. + m := image.modules[idx] + name := image.stem + if idx != 0 || image.extension { + name += "-" + stemOf(m) } + return name +} - return image +func (image bootImageConfig) firstModuleNameOrStem() string { + if len(image.modules) > 0 { + return image.moduleName(0) + } else { + return image.stem + } +} + +func (image bootImageConfig) moduleFiles(ctx android.PathContext, dir android.OutputPath, exts ...string) android.OutputPaths { + ret := make(android.OutputPaths, 0, len(image.modules)*len(exts)) + for i := range image.modules { + name := image.moduleName(i) + for _, ext := range exts { + ret = append(ret, dir.Join(ctx, name+ext)) + } + } + return ret } func concat(lists ...[]string) []string { @@ -97,7 +168,15 @@ func dexpreoptBootJarsFactory() android.Singleton { return &dexpreoptBootJars{} } +func RegisterDexpreoptBootJarsComponents(ctx android.RegistrationContext) { + ctx.RegisterSingletonType("dex_bootjars", dexpreoptBootJarsFactory) +} + func skipDexpreoptBootJars(ctx android.PathContext) bool { + if dexpreopt.GetGlobalConfig(ctx).DisablePreopt { + return true + } + if ctx.Config().UnbundledBuild() { return true } @@ -111,8 +190,23 @@ func skipDexpreoptBootJars(ctx android.PathContext) bool { } type dexpreoptBootJars struct { - defaultBootImage *bootImage - otherImages []*bootImage + defaultBootImage *bootImageConfig + otherImages []*bootImageConfig + + dexpreoptConfigForMake android.WritablePath +} + +// Accessor function for the apex package. Returns nil if dexpreopt is disabled. +func DexpreoptedArtApexJars(ctx android.BuilderContext) map[android.ArchType]android.OutputPaths { + if skipDexpreoptBootJars(ctx) { + return nil + } + // Include dexpreopt files for the primary boot image. + files := map[android.ArchType]android.OutputPaths{} + for _, variant := range artBootImageConfig(ctx).variants { + files[variant.target.Arch.ArchType] = variant.imagesDeps + } + return files } // dexpreoptBoot singleton rules @@ -120,8 +214,15 @@ func (d *dexpreoptBootJars) GenerateBuildActions(ctx android.SingletonContext) { if skipDexpreoptBootJars(ctx) { return } + if dexpreopt.GetCachedGlobalSoongConfig(ctx) == nil { + // No module has enabled dexpreopting, so we assume there will be no boot image to make. + return + } + + d.dexpreoptConfigForMake = android.PathForOutput(ctx, ctx.Config().DeviceName(), "dexpreopt.config") + writeGlobalConfigForMake(ctx, d.dexpreoptConfigForMake) - global := dexpreoptGlobalConfig(ctx) + global := dexpreopt.GetGlobalConfig(ctx) // Skip recompiling the boot image for the second sanitization phase. We'll get separate paths // and invalidate first-stage artifacts which are crucial to SANITIZE_LITE builds. @@ -135,28 +236,73 @@ func (d *dexpreoptBootJars) GenerateBuildActions(ctx android.SingletonContext) { // Always create the default boot image first, to get a unique profile rule for all images. d.defaultBootImage = buildBootImage(ctx, defaultBootImageConfig(ctx)) - if global.GenerateApexImage { - d.otherImages = append(d.otherImages, buildBootImage(ctx, apexBootImageConfig(ctx))) - } + // Create boot image for the ART apex (build artifacts are accessed via the global boot image config). + d.otherImages = append(d.otherImages, buildBootImage(ctx, artBootImageConfig(ctx))) dumpOatRules(ctx, d.defaultBootImage) } -// buildBootImage takes a bootImageConfig, creates rules to build it, and returns a *bootImage. -func buildBootImage(ctx android.SingletonContext, config bootImageConfig) *bootImage { - global := dexpreoptGlobalConfig(ctx) +// Inspect this module to see if it contains a bootclasspath dex jar. +// Note that the same jar may occur in multiple modules. +// This logic is tested in the apex package to avoid import cycle apex <-> java. +func getBootImageJar(ctx android.SingletonContext, image *bootImageConfig, module android.Module) (int, android.Path) { + // All apex Java libraries have non-installable platform variants, skip them. + if module.IsSkipInstall() { + return -1, nil + } - image := newBootImage(ctx, config) + jar, hasJar := module.(interface{ DexJar() android.Path }) + if !hasJar { + return -1, nil + } - bootDexJars := make(android.Paths, len(image.modules)) + name := ctx.ModuleName(module) + index := android.IndexList(name, image.modules) + if index == -1 { + return -1, nil + } + + // Check that this module satisfies constraints for a particular boot image. + apex, isApexModule := module.(android.ApexModule) + fromUpdatableApex := isApexModule && apex.Updatable() + if image.name == artBootImageName { + if isApexModule && strings.HasPrefix(apex.ApexName(), "com.android.art.") { + // ok: found the jar in the ART apex + } else if isApexModule && apex.IsForPlatform() && Bool(module.(*Library).deviceProperties.Hostdex) { + // exception (skip and continue): special "hostdex" platform variant + return -1, nil + } else if name == "jacocoagent" && ctx.Config().IsEnvTrue("EMMA_INSTRUMENT_FRAMEWORK") { + // exception (skip and continue): Jacoco platform variant for a coverage build + return -1, nil + } else if fromUpdatableApex { + // error: this jar is part of an updatable apex other than ART + ctx.Errorf("module '%s' from updatable apex '%s' is not allowed in the ART boot image", name, apex.ApexName()) + } else { + // error: this jar is part of the platform or a non-updatable apex + ctx.Errorf("module '%s' is not allowed in the ART boot image", name) + } + } else if image.name == frameworkBootImageName { + if !fromUpdatableApex { + // ok: this jar is part of the platform or a non-updatable apex + } else { + // error: this jar is part of an updatable apex + ctx.Errorf("module '%s' from updatable apex '%s' is not allowed in the framework boot image", name, apex.ApexName()) + } + } else { + panic("unknown boot image: " + image.name) + } + return index, jar.DexJar() +} + +// buildBootImage takes a bootImageConfig, creates rules to build it, and returns the image. +func buildBootImage(ctx android.SingletonContext, image *bootImageConfig) *bootImageConfig { + // Collect dex jar paths for the boot image modules. + // This logic is tested in the apex package to avoid import cycle apex <-> java. + bootDexJars := make(android.Paths, len(image.modules)) ctx.VisitAllModules(func(module android.Module) { - // Collect dex jar paths for the modules listed above. - if j, ok := module.(interface{ DexJar() android.Path }); ok { - name := ctx.ModuleName(module) - if i := android.IndexList(name, image.modules); i != -1 { - bootDexJars[i] = j.DexJar() - } + if i, j := getBootImageJar(ctx, image, module); i != -1 { + bootDexJars[i] = j } }) @@ -168,7 +314,8 @@ func buildBootImage(ctx android.SingletonContext, config bootImageConfig) *bootI missingDeps = append(missingDeps, image.modules[i]) bootDexJars[i] = android.PathForOutput(ctx, "missing") } else { - ctx.Errorf("failed to find dex jar path for module %q", + ctx.Errorf("failed to find a dex jar path for module '%s'"+ + ", note that some jars may be filtered out by module constraints", image.modules[i]) } } @@ -186,31 +333,42 @@ func buildBootImage(ctx android.SingletonContext, config bootImageConfig) *bootI } profile := bootImageProfileRule(ctx, image, missingDeps) + bootFrameworkProfileRule(ctx, image, missingDeps) + updatableBcpPackagesRule(ctx, image, missingDeps) - if !global.DisablePreopt { - targets := ctx.Config().Targets[android.Android] - if ctx.Config().SecondArchIsTranslated() { - targets = targets[:1] - } + var allFiles android.Paths + for _, variant := range image.variants { + files := buildBootImageVariant(ctx, variant, profile, missingDeps) + allFiles = append(allFiles, files.Paths()...) + } - for _, target := range targets { - buildBootImageRuleForArch(ctx, image, target.Arch.ArchType, profile, missingDeps) - } + if image.zip != nil { + rule := android.NewRuleBuilder() + rule.Command(). + BuiltTool(ctx, "soong_zip"). + FlagWithOutput("-o ", image.zip). + FlagWithArg("-C ", image.dir.String()). + FlagWithInputList("-f ", allFiles, " -f ") + + rule.Build(pctx, ctx, "zip_"+image.name, "zip "+image.name+" image") } return image } -func buildBootImageRuleForArch(ctx android.SingletonContext, image *bootImage, - arch android.ArchType, profile android.Path, missingDeps []string) { +func buildBootImageVariant(ctx android.SingletonContext, image *bootImageVariant, + profile android.Path, missingDeps []string) android.WritablePaths { - global := dexpreoptGlobalConfig(ctx) + globalSoong := dexpreopt.GetCachedGlobalSoongConfig(ctx) + global := dexpreopt.GetGlobalConfig(ctx) - symbolsDir := image.symbolsDir.Join(ctx, "system/framework", arch.String()) - symbolsFile := symbolsDir.Join(ctx, image.name+".oat") - outputDir := image.dir.Join(ctx, "system/framework", arch.String()) - outputPath := image.images[arch] - oatLocation := pathtools.ReplaceExtension(dexpreopt.PathToLocation(outputPath, arch), "oat") + arch := image.target.Arch.ArchType + symbolsDir := image.symbolsDir.Join(ctx, image.installSubdir, arch.String()) + symbolsFile := symbolsDir.Join(ctx, image.stem+".oat") + outputDir := image.dir.Join(ctx, image.installSubdir, arch.String()) + outputPath := outputDir.Join(ctx, image.stem+".oat") + oatLocation := dexpreopt.PathToLocation(outputPath, arch) + imagePath := outputPath.ReplaceExtension(ctx, "art") rule := android.NewRuleBuilder() rule.MissingDeps(missingDeps) @@ -238,7 +396,7 @@ func buildBootImageRuleForArch(ctx android.SingletonContext, image *bootImage, invocationPath := outputPath.ReplaceExtension(ctx, "invocation") - cmd.Tool(global.Tools.Dex2oat). + cmd.Tool(globalSoong.Dex2oat). Flag("--avoid-storing-invocation"). FlagWithOutput("--write-invocation-to=", invocationPath).ImplicitOutput(invocationPath). Flag("--runtime-arg").FlagWithArg("-Xms", global.Dex2oatImageXms). @@ -247,30 +405,39 @@ func buildBootImageRuleForArch(ctx android.SingletonContext, image *bootImage, if profile != nil { cmd.FlagWithArg("--compiler-filter=", "speed-profile") cmd.FlagWithInput("--profile-file=", profile) - } else if global.PreloadedClasses.Valid() { - cmd.FlagWithInput("--image-classes=", global.PreloadedClasses.Path()) } if global.DirtyImageObjects.Valid() { cmd.FlagWithInput("--dirty-image-objects=", global.DirtyImageObjects.Path()) } + if image.extension { + artImage := image.primaryImages + cmd. + Flag("--runtime-arg").FlagWithInputList("-Xbootclasspath:", image.dexPathsDeps.Paths(), ":"). + Flag("--runtime-arg").FlagWithList("-Xbootclasspath-locations:", image.dexLocationsDeps, ":"). + FlagWithArg("--boot-image=", dexpreopt.PathToLocation(artImage, arch)).Implicit(artImage) + } else { + cmd.FlagWithArg("--base=", ctx.Config().LibartImgDeviceBaseAddress()) + } + cmd. FlagForEachInput("--dex-file=", image.dexPaths.Paths()). FlagForEachArg("--dex-location=", image.dexLocations). Flag("--generate-debug-info"). Flag("--generate-build-id"). - FlagWithOutput("--oat-symbols=", symbolsFile). + Flag("--image-format=lz4hc"). + FlagWithArg("--oat-symbols=", symbolsFile.String()). Flag("--strip"). - FlagWithOutput("--oat-file=", outputPath.ReplaceExtension(ctx, "oat")). + FlagWithArg("--oat-file=", outputPath.String()). FlagWithArg("--oat-location=", oatLocation). - FlagWithOutput("--image=", outputPath). - FlagWithArg("--base=", ctx.Config().LibartImgDeviceBaseAddress()). + FlagWithArg("--image=", imagePath.String()). FlagWithArg("--instruction-set=", arch.String()). FlagWithArg("--instruction-set-variant=", global.CpuVariant[arch]). FlagWithArg("--instruction-set-features=", global.InstructionSetFeatures[arch]). FlagWithArg("--android-root=", global.EmptyDirectory). FlagWithArg("--no-inline-from=", "core-oj.jar"). + Flag("--force-determinism"). Flag("--abort-on-hard-verifier-error") if global.BootFlags != "" { @@ -283,68 +450,64 @@ func buildBootImageRuleForArch(ctx android.SingletonContext, image *bootImage, cmd.Textf(`|| ( echo %s ; false )`, proptools.ShellEscape(failureMessage)) - installDir := filepath.Join("/system/framework", arch.String()) - vdexInstallDir := filepath.Join("/system/framework") + installDir := filepath.Join("/", image.installSubdir, arch.String()) + vdexInstallDir := filepath.Join("/", image.installSubdir) - var extraFiles android.WritablePaths var vdexInstalls android.RuleBuilderInstalls var unstrippedInstalls android.RuleBuilderInstalls - // dex preopt on the bootclasspath produces multiple files. The first dex file - // is converted into to 'name'.art (to match the legacy assumption that 'name'.art - // exists), and the rest are converted to 'name'-<jar>.art. - // In addition, each .art file has an associated .oat and .vdex file, and an - // unstripped .oat file - for i, m := range image.modules { - name := image.name - if i != 0 { - name += "-" + m - } + var zipFiles android.WritablePaths - art := outputDir.Join(ctx, name+".art") - oat := outputDir.Join(ctx, name+".oat") - vdex := outputDir.Join(ctx, name+".vdex") - unstrippedOat := symbolsDir.Join(ctx, name+".oat") + for _, artOrOat := range image.moduleFiles(ctx, outputDir, ".art", ".oat") { + cmd.ImplicitOutput(artOrOat) + zipFiles = append(zipFiles, artOrOat) - extraFiles = append(extraFiles, art, oat, vdex, unstrippedOat) + // Install the .oat and .art files + rule.Install(artOrOat, filepath.Join(installDir, artOrOat.Base())) + } - // Install the .oat and .art files. - rule.Install(art, filepath.Join(installDir, art.Base())) - rule.Install(oat, filepath.Join(installDir, oat.Base())) + for _, vdex := range image.moduleFiles(ctx, outputDir, ".vdex") { + cmd.ImplicitOutput(vdex) + zipFiles = append(zipFiles, vdex) // The vdex files are identical between architectures, install them to a shared location. The Make rules will // only use the install rules for one architecture, and will create symlinks into the architecture-specific // directories. vdexInstalls = append(vdexInstalls, android.RuleBuilderInstall{vdex, filepath.Join(vdexInstallDir, vdex.Base())}) + } + + for _, unstrippedOat := range image.moduleFiles(ctx, symbolsDir, ".oat") { + cmd.ImplicitOutput(unstrippedOat) // Install the unstripped oat files. The Make rules will put these in $(TARGET_OUT_UNSTRIPPED) unstrippedInstalls = append(unstrippedInstalls, android.RuleBuilderInstall{unstrippedOat, filepath.Join(installDir, unstrippedOat.Base())}) } - cmd.ImplicitOutputs(extraFiles) - rule.Build(pctx, ctx, image.name+"JarsDexpreopt_"+arch.String(), "dexpreopt "+image.name+" jars "+arch.String()) // save output and installed files for makevars - image.installs[arch] = rule.Installs() - image.vdexInstalls[arch] = vdexInstalls - image.unstrippedInstalls[arch] = unstrippedInstalls + image.installs = rule.Installs() + image.vdexInstalls = vdexInstalls + image.unstrippedInstalls = unstrippedInstalls + + return zipFiles } const failureMessage = `ERROR: Dex2oat failed to compile a boot image. It is likely that the boot classpath is inconsistent. Rebuild with ART_BOOT_IMAGE_EXTRA_ARGS="--runtime-arg -verbose:verifier" to see verification errors.` -func bootImageProfileRule(ctx android.SingletonContext, image *bootImage, missingDeps []string) android.WritablePath { - global := dexpreoptGlobalConfig(ctx) +func bootImageProfileRule(ctx android.SingletonContext, image *bootImageConfig, missingDeps []string) android.WritablePath { + globalSoong := dexpreopt.GetCachedGlobalSoongConfig(ctx) + global := dexpreopt.GetGlobalConfig(ctx) - if !global.UseProfileForBootImage || ctx.Config().IsPdkBuild() || ctx.Config().UnbundledBuild() { + if global.DisableGenerateProfile || ctx.Config().IsPdkBuild() || ctx.Config().UnbundledBuild() { return nil } - return ctx.Config().Once(bootImageProfileRuleKey, func() interface{} { - tools := global.Tools + profile := ctx.Config().Once(bootImageProfileRuleKey, func() interface{} { + defaultProfile := "frameworks/base/config/boot-image-profile.txt" rule := android.NewRuleBuilder() rule.MissingDeps(missingDeps) @@ -356,28 +519,23 @@ func bootImageProfileRule(ctx android.SingletonContext, image *bootImage, missin bootImageProfile = combinedBootImageProfile } else if len(global.BootImageProfiles) == 1 { bootImageProfile = global.BootImageProfiles[0] + } else if path := android.ExistentPathForSource(ctx, defaultProfile); path.Valid() { + bootImageProfile = path.Path() } else { - // If not set, use the default. Some branches like master-art-host don't have frameworks/base, so manually - // handle the case that the default is missing. Those branches won't attempt to build the profile rule, - // and if they do they'll get a missing deps error. - defaultProfile := "frameworks/base/config/boot-image-profile.txt" - path := android.ExistentPathForSource(ctx, defaultProfile) - if path.Valid() { - bootImageProfile = path.Path() - } else { - missingDeps = append(missingDeps, defaultProfile) - bootImageProfile = android.PathForOutput(ctx, "missing") - } + // No profile (not even a default one, which is the case on some branches + // like master-art-host that don't have frameworks/base). + // Return nil and continue without profile. + return nil } profile := image.dir.Join(ctx, "boot.prof") rule.Command(). Text(`ANDROID_LOG_TAGS="*:e"`). - Tool(tools.Profman). + Tool(globalSoong.Profman). FlagWithInput("--create-profile-from=", bootImageProfile). - FlagForEachInput("--apk=", image.dexPaths.Paths()). - FlagForEachArg("--dex-location=", image.dexLocations). + FlagForEachInput("--apk=", image.dexPathsDeps.Paths()). + FlagForEachArg("--dex-location=", image.dexLocationsDeps). FlagWithOutput("--reference-profile-file=", profile) rule.Install(profile, "/system/etc/boot-image.prof") @@ -387,29 +545,128 @@ func bootImageProfileRule(ctx android.SingletonContext, image *bootImage, missin image.profileInstalls = rule.Installs() return profile - }).(android.WritablePath) + }) + if profile == nil { + return nil // wrap nil into a typed pointer with value nil + } + return profile.(android.WritablePath) } var bootImageProfileRuleKey = android.NewOnceKey("bootImageProfileRule") -func dumpOatRules(ctx android.SingletonContext, image *bootImage) { - var archs []android.ArchType - for arch := range image.images { - archs = append(archs, arch) +func bootFrameworkProfileRule(ctx android.SingletonContext, image *bootImageConfig, missingDeps []string) android.WritablePath { + globalSoong := dexpreopt.GetCachedGlobalSoongConfig(ctx) + global := dexpreopt.GetGlobalConfig(ctx) + + if global.DisableGenerateProfile || ctx.Config().IsPdkBuild() || ctx.Config().UnbundledBuild() { + return nil } - sort.Slice(archs, func(i, j int) bool { return archs[i].String() < archs[j].String() }) + return ctx.Config().Once(bootFrameworkProfileRuleKey, func() interface{} { + rule := android.NewRuleBuilder() + rule.MissingDeps(missingDeps) + + // Some branches like master-art-host don't have frameworks/base, so manually + // handle the case that the default is missing. Those branches won't attempt to build the profile rule, + // and if they do they'll get a missing deps error. + defaultProfile := "frameworks/base/config/boot-profile.txt" + path := android.ExistentPathForSource(ctx, defaultProfile) + var bootFrameworkProfile android.Path + if path.Valid() { + bootFrameworkProfile = path.Path() + } else { + missingDeps = append(missingDeps, defaultProfile) + bootFrameworkProfile = android.PathForOutput(ctx, "missing") + } + + profile := image.dir.Join(ctx, "boot.bprof") + + rule.Command(). + Text(`ANDROID_LOG_TAGS="*:e"`). + Tool(globalSoong.Profman). + Flag("--generate-boot-profile"). + FlagWithInput("--create-profile-from=", bootFrameworkProfile). + FlagForEachInput("--apk=", image.dexPathsDeps.Paths()). + FlagForEachArg("--dex-location=", image.dexLocationsDeps). + FlagWithOutput("--reference-profile-file=", profile) + rule.Install(profile, "/system/etc/boot-image.bprof") + rule.Build(pctx, ctx, "bootFrameworkProfile", "profile boot framework jars") + image.profileInstalls = append(image.profileInstalls, rule.Installs()...) + + return profile + }).(android.WritablePath) +} + +var bootFrameworkProfileRuleKey = android.NewOnceKey("bootFrameworkProfileRule") + +func updatableBcpPackagesRule(ctx android.SingletonContext, image *bootImageConfig, missingDeps []string) android.WritablePath { + if ctx.Config().IsPdkBuild() || ctx.Config().UnbundledBuild() { + return nil + } + + return ctx.Config().Once(updatableBcpPackagesRuleKey, func() interface{} { + global := dexpreopt.GetGlobalConfig(ctx) + updatableModules := dexpreopt.GetJarsFromApexJarPairs(global.UpdatableBootJars) + + // Collect `permitted_packages` for updatable boot jars. + var updatablePackages []string + ctx.VisitAllModules(func(module android.Module) { + if j, ok := module.(PermittedPackagesForUpdatableBootJars); ok { + name := ctx.ModuleName(module) + if i := android.IndexList(name, updatableModules); i != -1 { + pp := j.PermittedPackagesForUpdatableBootJars() + if len(pp) > 0 { + updatablePackages = append(updatablePackages, pp...) + } else { + ctx.Errorf("Missing permitted_packages for %s", name) + } + // Do not match the same library repeatedly. + updatableModules = append(updatableModules[:i], updatableModules[i+1:]...) + } + } + }) + + // Sort updatable packages to ensure deterministic ordering. + sort.Strings(updatablePackages) + + updatableBcpPackagesName := "updatable-bcp-packages.txt" + updatableBcpPackages := image.dir.Join(ctx, updatableBcpPackagesName) + + ctx.Build(pctx, android.BuildParams{ + Rule: android.WriteFile, + Output: updatableBcpPackages, + Args: map[string]string{ + // WriteFile automatically adds the last end-of-line. + "content": strings.Join(updatablePackages, "\\n"), + }, + }) + + rule := android.NewRuleBuilder() + rule.MissingDeps(missingDeps) + rule.Install(updatableBcpPackages, "/system/etc/"+updatableBcpPackagesName) + // TODO: Rename `profileInstalls` to `extraInstalls`? + // Maybe even move the field out of the bootImageConfig into some higher level type? + image.profileInstalls = append(image.profileInstalls, rule.Installs()...) + + return updatableBcpPackages + }).(android.WritablePath) +} + +var updatableBcpPackagesRuleKey = android.NewOnceKey("updatableBcpPackagesRule") + +func dumpOatRules(ctx android.SingletonContext, image *bootImageConfig) { var allPhonies android.Paths - for _, arch := range archs { + for _, image := range image.variants { + arch := image.target.Arch.ArchType // Create a rule to call oatdump. output := android.PathForOutput(ctx, "boot."+arch.String()+".oatdump.txt") rule := android.NewRuleBuilder() rule.Command(). // TODO: for now, use the debug version for better error reporting - Tool(ctx.Config().HostToolPath(ctx, "oatdumpd")). - FlagWithInputList("--runtime-arg -Xbootclasspath:", image.dexPaths.Paths(), ":"). - FlagWithList("--runtime-arg -Xbootclasspath-locations:", image.dexLocations, ":"). - FlagWithArg("--image=", dexpreopt.PathToLocation(image.images[arch], arch)).Implicit(image.images[arch]). + BuiltTool(ctx, "oatdumpd"). + FlagWithInputList("--runtime-arg -Xbootclasspath:", image.dexPathsDeps.Paths(), ":"). + FlagWithList("--runtime-arg -Xbootclasspath-locations:", image.dexLocationsDeps, ":"). + FlagWithArg("--image=", strings.Join(image.imageLocations, ":")).Implicits(image.imagesDeps.Paths()). FlagWithOutput("--output=", output). FlagWithArg("--instruction-set=", arch.String()) rule.Build(pctx, ctx, "dump-oat-boot-"+arch.String(), "dump oat boot "+arch.String()) @@ -436,30 +693,45 @@ func dumpOatRules(ctx android.SingletonContext, image *bootImage) { } +func writeGlobalConfigForMake(ctx android.SingletonContext, path android.WritablePath) { + data := dexpreopt.GetGlobalConfigRawData(ctx) + + ctx.Build(pctx, android.BuildParams{ + Rule: android.WriteFile, + Output: path, + Args: map[string]string{ + "content": string(data), + }, + }) +} + // Export paths for default boot image to Make func (d *dexpreoptBootJars) MakeVars(ctx android.MakeVarsContext) { + if d.dexpreoptConfigForMake != nil { + ctx.Strict("DEX_PREOPT_CONFIG_FOR_MAKE", d.dexpreoptConfigForMake.String()) + ctx.Strict("DEX_PREOPT_SOONG_CONFIG_FOR_MAKE", android.PathForOutput(ctx, "dexpreopt_soong.config").String()) + } + image := d.defaultBootImage if image != nil { ctx.Strict("DEXPREOPT_IMAGE_PROFILE_BUILT_INSTALLED", image.profileInstalls.String()) - ctx.Strict("DEXPREOPT_BOOTCLASSPATH_DEX_FILES", strings.Join(image.dexPaths.Strings(), " ")) - ctx.Strict("DEXPREOPT_BOOTCLASSPATH_DEX_LOCATIONS", strings.Join(image.dexLocations, " ")) + ctx.Strict("DEXPREOPT_BOOTCLASSPATH_DEX_FILES", strings.Join(image.dexPathsDeps.Strings(), " ")) + ctx.Strict("DEXPREOPT_BOOTCLASSPATH_DEX_LOCATIONS", strings.Join(image.dexLocationsDeps, " ")) var imageNames []string for _, current := range append(d.otherImages, image) { imageNames = append(imageNames, current.name) - var arches []android.ArchType - for arch, _ := range current.images { - arches = append(arches, arch) + for _, current := range current.variants { + sfx := current.name + "_" + current.target.Arch.ArchType.String() + ctx.Strict("DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_"+sfx, current.vdexInstalls.String()) + ctx.Strict("DEXPREOPT_IMAGE_"+sfx, current.images.String()) + ctx.Strict("DEXPREOPT_IMAGE_DEPS_"+sfx, strings.Join(current.imagesDeps.Strings(), " ")) + ctx.Strict("DEXPREOPT_IMAGE_BUILT_INSTALLED_"+sfx, current.installs.String()) + ctx.Strict("DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_"+sfx, current.unstrippedInstalls.String()) } - sort.Slice(arches, func(i, j int) bool { return arches[i].String() < arches[j].String() }) - - for _, arch := range arches { - ctx.Strict("DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_"+current.name+"_"+arch.String(), current.vdexInstalls[arch].String()) - ctx.Strict("DEXPREOPT_IMAGE_"+current.name+"_"+arch.String(), current.images[arch].String()) - ctx.Strict("DEXPREOPT_IMAGE_BUILT_INSTALLED_"+current.name+"_"+arch.String(), current.installs[arch].String()) - ctx.Strict("DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_"+current.name+"_"+arch.String(), current.unstrippedInstalls[arch].String()) - } + ctx.Strict("DEXPREOPT_IMAGE_LOCATIONS_"+current.name, strings.Join(current.imageLocations, ":")) + ctx.Strict("DEXPREOPT_IMAGE_ZIP_"+current.name, current.zip.String()) } ctx.Strict("DEXPREOPT_IMAGE_NAMES", strings.Join(imageNames, " ")) } diff --git a/java/dexpreopt_bootjars_test.go b/java/dexpreopt_bootjars_test.go index cbb52f15c..e7b3c3ba9 100644 --- a/java/dexpreopt_bootjars_test.go +++ b/java/dexpreopt_bootjars_test.go @@ -44,24 +44,25 @@ func TestDexpreoptBootJars(t *testing.T) { } ` - config := testConfig(nil) + config := testConfig(nil, bp, nil) - pathCtx := android.PathContextForTesting(config, nil) + pathCtx := android.PathContextForTesting(config) dexpreoptConfig := dexpreopt.GlobalConfigForTests(pathCtx) - dexpreoptConfig.RuntimeApexJars = []string{"foo", "bar", "baz"} - setDexpreoptTestGlobalConfig(config, dexpreoptConfig) + dexpreoptConfig.BootJars = []string{"foo", "bar", "baz"} + dexpreopt.SetTestGlobalConfig(config, dexpreoptConfig) - ctx := testContext(config, bp, nil) + ctx := testContext() - ctx.RegisterSingletonType("dex_bootjars", android.SingletonFactoryAdaptor(dexpreoptBootJarsFactory)) + RegisterDexpreoptBootJarsComponents(ctx) run(t, ctx, config) dexpreoptBootJars := ctx.SingletonForTests("dex_bootjars") - bootArt := dexpreoptBootJars.Output("boot.art") + bootArt := dexpreoptBootJars.Output("boot-foo.art") expectedInputs := []string{ + "dex_artjars/apex/com.android.art/javalib/arm64/boot.art", "dex_bootjars_input/foo.jar", "dex_bootjars_input/bar.jar", "dex_bootjars_input/baz.jar", @@ -82,19 +83,19 @@ func TestDexpreoptBootJars(t *testing.T) { expectedOutputs := []string{ "dex_bootjars/system/framework/arm64/boot.invocation", - "dex_bootjars/system/framework/arm64/boot.art", + "dex_bootjars/system/framework/arm64/boot-foo.art", "dex_bootjars/system/framework/arm64/boot-bar.art", "dex_bootjars/system/framework/arm64/boot-baz.art", - "dex_bootjars/system/framework/arm64/boot.oat", + "dex_bootjars/system/framework/arm64/boot-foo.oat", "dex_bootjars/system/framework/arm64/boot-bar.oat", "dex_bootjars/system/framework/arm64/boot-baz.oat", - "dex_bootjars/system/framework/arm64/boot.vdex", + "dex_bootjars/system/framework/arm64/boot-foo.vdex", "dex_bootjars/system/framework/arm64/boot-bar.vdex", "dex_bootjars/system/framework/arm64/boot-baz.vdex", - "dex_bootjars_unstripped/system/framework/arm64/boot.oat", + "dex_bootjars_unstripped/system/framework/arm64/boot-foo.oat", "dex_bootjars_unstripped/system/framework/arm64/boot-bar.oat", "dex_bootjars_unstripped/system/framework/arm64/boot-baz.oat", } diff --git a/java/dexpreopt_config.go b/java/dexpreopt_config.go index abc5fa18f..f8356d188 100644 --- a/java/dexpreopt_config.go +++ b/java/dexpreopt_config.go @@ -15,181 +15,194 @@ package java import ( - "android/soong/android" - "android/soong/dexpreopt" + "fmt" "path/filepath" "strings" -) - -// dexpreoptGlobalConfig returns the global dexpreopt.config. It is loaded once the first time it is called for any -// ctx.Config(), and returns the same data for all future calls with the same ctx.Config(). A value can be inserted -// for tests using setDexpreoptTestGlobalConfig. -func dexpreoptGlobalConfig(ctx android.PathContext) dexpreopt.GlobalConfig { - return ctx.Config().Once(dexpreoptGlobalConfigKey, func() interface{} { - if f := ctx.Config().DexpreoptGlobalConfig(); f != "" { - ctx.AddNinjaFileDeps(f) - globalConfig, err := dexpreopt.LoadGlobalConfig(ctx, f) - if err != nil { - panic(err) - } - return globalConfig - } - // No global config filename set, see if there is a test config set - return ctx.Config().Once(dexpreoptTestGlobalConfigKey, func() interface{} { - // Nope, return a config with preopting disabled - return dexpreopt.GlobalConfig{ - DisablePreopt: true, - } - }) - }).(dexpreopt.GlobalConfig) -} - -// setDexpreoptTestGlobalConfig sets a GlobalConfig that future calls to dexpreoptGlobalConfig will return. It must -// be called before the first call to dexpreoptGlobalConfig for the config. -func setDexpreoptTestGlobalConfig(config android.Config, globalConfig dexpreopt.GlobalConfig) { - config.Once(dexpreoptTestGlobalConfigKey, func() interface{} { return globalConfig }) -} - -var dexpreoptGlobalConfigKey = android.NewOnceKey("DexpreoptGlobalConfig") -var dexpreoptTestGlobalConfigKey = android.NewOnceKey("TestDexpreoptGlobalConfig") + "android/soong/android" + "android/soong/dexpreopt" +) // systemServerClasspath returns the on-device locations of the modules in the system server classpath. It is computed // once the first time it is called for any ctx.Config(), and returns the same slice for all future calls with the same // ctx.Config(). -func systemServerClasspath(ctx android.PathContext) []string { +func systemServerClasspath(ctx android.MakeVarsContext) []string { return ctx.Config().OnceStringSlice(systemServerClasspathKey, func() []string { - global := dexpreoptGlobalConfig(ctx) - + global := dexpreopt.GetGlobalConfig(ctx) var systemServerClasspathLocations []string - for _, m := range global.SystemServerJars { + nonUpdatable := dexpreopt.NonUpdatableSystemServerJars(ctx, global) + // 1) Non-updatable jars. + for _, m := range nonUpdatable { systemServerClasspathLocations = append(systemServerClasspathLocations, filepath.Join("/system/framework", m+".jar")) } + // 2) The jars that are from an updatable apex. + for _, m := range global.UpdatableSystemServerJars { + systemServerClasspathLocations = append(systemServerClasspathLocations, + dexpreopt.GetJarLocationFromApexJarPair(m)) + } + if len(systemServerClasspathLocations) != len(global.SystemServerJars)+len(global.UpdatableSystemServerJars) { + panic(fmt.Errorf("Wrong number of system server jars, got %d, expected %d", + len(systemServerClasspathLocations), + len(global.SystemServerJars)+len(global.UpdatableSystemServerJars))) + } return systemServerClasspathLocations }) } var systemServerClasspathKey = android.NewOnceKey("systemServerClasspath") -// defaultBootImageConfig returns the bootImageConfig that will be used to dexpreopt modules. It is computed once the -// first time it is called for any ctx.Config(), and returns the same slice for all future calls with the same -// ctx.Config(). -func defaultBootImageConfig(ctx android.PathContext) bootImageConfig { - return ctx.Config().Once(defaultBootImageConfigKey, func() interface{} { - global := dexpreoptGlobalConfig(ctx) +// dexpreoptTargets returns the list of targets that are relevant to dexpreopting, which excludes architectures +// supported through native bridge. +func dexpreoptTargets(ctx android.PathContext) []android.Target { + var targets []android.Target + for _, target := range ctx.Config().Targets[android.Android] { + if target.NativeBridge == android.NativeBridgeDisabled { + targets = append(targets, target) + } + } - runtimeModules := global.RuntimeApexJars - nonFrameworkModules := concat(runtimeModules, global.ProductUpdatableBootModules) - frameworkModules := android.RemoveListFromList(global.BootJars, nonFrameworkModules) + return targets +} - var nonUpdatableBootModules []string - var nonUpdatableBootLocations []string +func stemOf(moduleName string) string { + // b/139391334: the stem of framework-minus-apex is framework + // This is hard coded here until we find a good way to query the stem + // of a module before any other mutators are run + if moduleName == "framework-minus-apex" { + return "framework" + } + return moduleName +} - for _, m := range runtimeModules { - nonUpdatableBootModules = append(nonUpdatableBootModules, m) - nonUpdatableBootLocations = append(nonUpdatableBootLocations, - filepath.Join("/apex/com.android.runtime/javalib", m+".jar")) +var ( + bootImageConfigKey = android.NewOnceKey("bootImageConfig") + artBootImageName = "art" + frameworkBootImageName = "boot" +) + +// Construct the global boot image configs. +func genBootImageConfigs(ctx android.PathContext) map[string]*bootImageConfig { + return ctx.Config().Once(bootImageConfigKey, func() interface{} { + + global := dexpreopt.GetGlobalConfig(ctx) + targets := dexpreoptTargets(ctx) + deviceDir := android.PathForOutput(ctx, ctx.Config().DeviceName()) + + artModules := global.ArtApexJars + // With EMMA_INSTRUMENT_FRAMEWORK=true the Core libraries depend on jacoco. + if ctx.Config().IsEnvTrue("EMMA_INSTRUMENT_FRAMEWORK") { + artModules = append(artModules, "jacocoagent") } + frameworkModules := android.RemoveListFromList(global.BootJars, + concat(artModules, dexpreopt.GetJarsFromApexJarPairs(global.UpdatableBootJars))) + artSubdir := "apex/com.android.art/javalib" + frameworkSubdir := "system/framework" + + var artLocations, frameworkLocations []string + for _, m := range artModules { + artLocations = append(artLocations, filepath.Join("/"+artSubdir, stemOf(m)+".jar")) + } for _, m := range frameworkModules { - nonUpdatableBootModules = append(nonUpdatableBootModules, m) - nonUpdatableBootLocations = append(nonUpdatableBootLocations, - filepath.Join("/system/framework", m+".jar")) + frameworkLocations = append(frameworkLocations, filepath.Join("/"+frameworkSubdir, stemOf(m)+".jar")) } - // The path to bootclasspath dex files needs to be known at module GenerateAndroidBuildAction time, before - // the bootclasspath modules have been compiled. Set up known paths for them, the singleton rules will copy - // them there. - // TODO: use module dependencies instead - var nonUpdatableBootDexPaths android.WritablePaths - for _, m := range nonUpdatableBootModules { - nonUpdatableBootDexPaths = append(nonUpdatableBootDexPaths, - android.PathForOutput(ctx, ctx.Config().DeviceName(), "dex_bootjars_input", m+".jar")) + // ART config for the primary boot image in the ART apex. + // It includes the Core Libraries. + artCfg := bootImageConfig{ + extension: false, + name: artBootImageName, + stem: "boot", + installSubdir: artSubdir, + modules: artModules, + dexLocations: artLocations, + dexLocationsDeps: artLocations, } - dir := android.PathForOutput(ctx, ctx.Config().DeviceName(), "dex_bootjars") - symbolsDir := android.PathForOutput(ctx, ctx.Config().DeviceName(), "dex_bootjars_unstripped") - images := make(map[android.ArchType]android.OutputPath) - - for _, target := range ctx.Config().Targets[android.Android] { - images[target.Arch.ArchType] = dir.Join(ctx, - "system/framework", target.Arch.ArchType.String()).Join(ctx, "boot.art") + // Framework config for the boot image extension. + // It includes framework libraries and depends on the ART config. + frameworkCfg := bootImageConfig{ + extension: true, + name: frameworkBootImageName, + stem: "boot", + installSubdir: frameworkSubdir, + modules: frameworkModules, + dexLocations: frameworkLocations, + dexLocationsDeps: append(artLocations, frameworkLocations...), } - return bootImageConfig{ - name: "boot", - modules: nonUpdatableBootModules, - dexLocations: nonUpdatableBootLocations, - dexPaths: nonUpdatableBootDexPaths, - dir: dir, - symbolsDir: symbolsDir, - images: images, + configs := map[string]*bootImageConfig{ + artBootImageName: &artCfg, + frameworkBootImageName: &frameworkCfg, } - }).(bootImageConfig) -} - -var defaultBootImageConfigKey = android.NewOnceKey("defaultBootImageConfig") -func apexBootImageConfig(ctx android.PathContext) bootImageConfig { - return ctx.Config().Once(apexBootImageConfigKey, func() interface{} { - global := dexpreoptGlobalConfig(ctx) + // common to all configs + for _, c := range configs { + c.dir = deviceDir.Join(ctx, "dex_"+c.name+"jars") + c.symbolsDir = deviceDir.Join(ctx, "dex_"+c.name+"jars_unstripped") - runtimeModules := global.RuntimeApexJars - nonFrameworkModules := concat(runtimeModules, global.ProductUpdatableBootModules) - frameworkModules := android.RemoveListFromList(global.BootJars, nonFrameworkModules) - imageModules := concat(runtimeModules, frameworkModules) + // expands to <stem>.art for primary image and <stem>-<1st module>.art for extension + imageName := c.firstModuleNameOrStem() + ".art" - var bootLocations []string + c.imageLocations = []string{c.dir.Join(ctx, c.installSubdir, imageName).String()} - for _, m := range runtimeModules { - bootLocations = append(bootLocations, - filepath.Join("/apex/com.android.runtime/javalib", m+".jar")) - } + // The path to bootclasspath dex files needs to be known at module + // GenerateAndroidBuildAction time, before the bootclasspath modules have been compiled. + // Set up known paths for them, the singleton rules will copy them there. + // TODO(b/143682396): use module dependencies instead + inputDir := deviceDir.Join(ctx, "dex_"+c.name+"jars_input") + for _, m := range c.modules { + c.dexPaths = append(c.dexPaths, inputDir.Join(ctx, stemOf(m)+".jar")) + } + c.dexPathsDeps = c.dexPaths + + // Create target-specific variants. + for _, target := range targets { + arch := target.Arch.ArchType + imageDir := c.dir.Join(ctx, c.installSubdir, arch.String()) + variant := &bootImageVariant{ + bootImageConfig: c, + target: target, + images: imageDir.Join(ctx, imageName), + imagesDeps: c.moduleFiles(ctx, imageDir, ".art", ".oat", ".vdex"), + } + c.variants = append(c.variants, variant) + } - for _, m := range frameworkModules { - bootLocations = append(bootLocations, - filepath.Join("/system/framework", m+".jar")) + c.zip = c.dir.Join(ctx, c.name+".zip") } - // The path to bootclasspath dex files needs to be known at module GenerateAndroidBuildAction time, before - // the bootclasspath modules have been compiled. Set up known paths for them, the singleton rules will copy - // them there. - // TODO: use module dependencies instead - var bootDexPaths android.WritablePaths - for _, m := range imageModules { - bootDexPaths = append(bootDexPaths, - android.PathForOutput(ctx, ctx.Config().DeviceName(), "dex_apexjars_input", m+".jar")) + // specific to the framework config + frameworkCfg.dexPathsDeps = append(artCfg.dexPathsDeps, frameworkCfg.dexPathsDeps...) + for i := range targets { + frameworkCfg.variants[i].primaryImages = artCfg.variants[i].images } + frameworkCfg.imageLocations = append(artCfg.imageLocations, frameworkCfg.imageLocations...) - dir := android.PathForOutput(ctx, ctx.Config().DeviceName(), "dex_apexjars") - symbolsDir := android.PathForOutput(ctx, ctx.Config().DeviceName(), "dex_apexjars_unstripped") - images := make(map[android.ArchType]android.OutputPath) - - for _, target := range ctx.Config().Targets[android.Android] { - images[target.Arch.ArchType] = dir.Join(ctx, - "system/framework", target.Arch.ArchType.String(), "apex.art") - } + return configs + }).(map[string]*bootImageConfig) +} - return bootImageConfig{ - name: "apex", - modules: imageModules, - dexLocations: bootLocations, - dexPaths: bootDexPaths, - dir: dir, - symbolsDir: symbolsDir, - images: images, - } - }).(bootImageConfig) +func artBootImageConfig(ctx android.PathContext) *bootImageConfig { + return genBootImageConfigs(ctx)[artBootImageName] } -var apexBootImageConfigKey = android.NewOnceKey("apexBootImageConfig") +func defaultBootImageConfig(ctx android.PathContext) *bootImageConfig { + return genBootImageConfigs(ctx)[frameworkBootImageName] +} func defaultBootclasspath(ctx android.PathContext) []string { return ctx.Config().OnceStringSlice(defaultBootclasspathKey, func() []string { - global := dexpreoptGlobalConfig(ctx) + global := dexpreopt.GetGlobalConfig(ctx) image := defaultBootImageConfig(ctx) - bootclasspath := append(copyOf(image.dexLocations), global.ProductUpdatableBootLocations...) + + updatableBootclasspath := make([]string, len(global.UpdatableBootJars)) + for i, p := range global.UpdatableBootJars { + updatableBootclasspath[i] = dexpreopt.GetJarLocationFromApexJarPair(p) + } + + bootclasspath := append(copyOf(image.dexLocationsDeps), updatableBootclasspath...) return bootclasspath }) } @@ -204,7 +217,7 @@ func init() { func dexpreoptConfigMakevars(ctx android.MakeVarsContext) { ctx.Strict("PRODUCT_BOOTCLASSPATH", strings.Join(defaultBootclasspath(ctx), ":")) - ctx.Strict("PRODUCT_DEX2OAT_BOOTCLASSPATH", strings.Join(defaultBootImageConfig(ctx).dexLocations, ":")) + ctx.Strict("PRODUCT_DEX2OAT_BOOTCLASSPATH", strings.Join(defaultBootImageConfig(ctx).dexLocationsDeps, ":")) ctx.Strict("PRODUCT_SYSTEM_SERVER_CLASSPATH", strings.Join(systemServerClasspath(ctx), ":")) ctx.Strict("DEXPREOPT_BOOT_JARS_MODULES", strings.Join(defaultBootImageConfig(ctx).modules, ":")) diff --git a/java/dexpreopt_test.go b/java/dexpreopt_test.go index 4af2f5c38..5550a4c17 100644 --- a/java/dexpreopt_test.go +++ b/java/dexpreopt_test.go @@ -30,6 +30,7 @@ func TestDexpreoptEnabled(t *testing.T) { android_app { name: "foo", srcs: ["a.java"], + sdk_version: "current", }`, enabled: true, }, @@ -52,14 +53,29 @@ func TestDexpreoptEnabled(t *testing.T) { }`, enabled: true, }, - { name: "app without sources", bp: ` android_app { name: "foo", + sdk_version: "current", + }`, + enabled: false, + }, + { + name: "app with libraries", + bp: ` + android_app { + name: "foo", + static_libs: ["lib"], + sdk_version: "current", + } + + java_library { + name: "lib", + srcs: ["a.java"], + sdk_version: "current", }`, - // TODO(ccross): this should probably be false enabled: true, }, { @@ -69,10 +85,8 @@ func TestDexpreoptEnabled(t *testing.T) { name: "foo", installable: true, }`, - // TODO(ccross): this should probably be false - enabled: true, + enabled: false, }, - { name: "static java library", bp: ` @@ -132,7 +146,7 @@ func TestDexpreoptEnabled(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - ctx := testJava(t, test.bp) + ctx, _ := testJava(t, test.bp) dexpreopt := ctx.ModuleForTests("foo", "android_common").MaybeDescription("dexpreopt") enabled := dexpreopt.Rule != nil diff --git a/java/droiddoc.go b/java/droiddoc.go index f56cae826..b564fea01 100644 --- a/java/droiddoc.go +++ b/java/droiddoc.go @@ -15,141 +15,50 @@ package java import ( - "android/soong/android" - "android/soong/java/config" "fmt" "path/filepath" - "runtime" "strings" "github.com/google/blueprint" + "github.com/google/blueprint/proptools" + + "android/soong/android" + "android/soong/java/config" + "android/soong/remoteexec" ) -var ( - javadoc = pctx.AndroidStaticRule("javadoc", - blueprint.RuleParams{ - Command: `rm -rf "$outDir" "$srcJarDir" "$stubsDir" && mkdir -p "$outDir" "$srcJarDir" "$stubsDir" && ` + - `${config.ZipSyncCmd} -d $srcJarDir -l $srcJarDir/list -f "*.java" $srcJars && ` + - `${config.SoongJavacWrapper} ${config.JavadocCmd} -encoding UTF-8 @$out.rsp @$srcJarDir/list ` + - `$opts $bootclasspathArgs $classpathArgs $sourcepathArgs ` + - `-d $outDir -quiet && ` + - `${config.SoongZipCmd} -write_if_changed -d -o $docZip -C $outDir -D $outDir && ` + - `${config.SoongZipCmd} -write_if_changed -jar -o $out -C $stubsDir -D $stubsDir $postDoclavaCmds && ` + - `rm -rf "$srcJarDir"`, - - CommandDeps: []string{ - "${config.ZipSyncCmd}", - "${config.JavadocCmd}", - "${config.SoongZipCmd}", - }, - CommandOrderOnly: []string{"${config.SoongJavacWrapper}"}, - Rspfile: "$out.rsp", - RspfileContent: "$in", - Restat: true, - }, - "outDir", "srcJarDir", "stubsDir", "srcJars", "opts", - "bootclasspathArgs", "classpathArgs", "sourcepathArgs", "docZip", "postDoclavaCmds") - - apiCheck = pctx.AndroidStaticRule("apiCheck", - blueprint.RuleParams{ - Command: `( ${config.ApiCheckCmd} -JXmx1024m -J"classpath $classpath" $opts ` + - `$apiFile $apiFileToCheck $removedApiFile $removedApiFileToCheck ` + - `&& touch $out ) || (echo -e "$msg" ; exit 38)`, - CommandDeps: []string{ - "${config.ApiCheckCmd}", - }, +func init() { + RegisterDocsBuildComponents(android.InitRegistrationContext) + RegisterStubsBuildComponents(android.InitRegistrationContext) + + // Register sdk member type. + android.RegisterSdkMemberType(&droidStubsSdkMemberType{ + SdkMemberTypeBase: android.SdkMemberTypeBase{ + PropertyName: "stubs_sources", + // stubs_sources can be used with sdk to provide the source stubs for APIs provided by + // the APEX. + SupportsSdk: true, }, - "classpath", "opts", "apiFile", "apiFileToCheck", "removedApiFile", "removedApiFileToCheck", "msg") + }) +} - updateApi = pctx.AndroidStaticRule("updateApi", - blueprint.RuleParams{ - Command: `( ( cp -f $srcApiFile $destApiFile && cp -f $srcRemovedApiFile $destRemovedApiFile ) ` + - `&& touch $out ) || (echo failed to update public API ; exit 38)`, - }, - "srcApiFile", "destApiFile", "srcRemovedApiFile", "destRemovedApiFile") - - metalava = pctx.AndroidStaticRule("metalava", - blueprint.RuleParams{ - Command: `rm -rf "$outDir" "$srcJarDir" "$stubsDir" && ` + - `mkdir -p "$outDir" "$srcJarDir" "$stubsDir" && ` + - `${config.ZipSyncCmd} -d $srcJarDir -l $srcJarDir/list -f "*.java" $srcJars && ` + - `${config.JavaCmd} -jar ${config.MetalavaJar} -encoding UTF-8 -source $javaVersion @$out.rsp @$srcJarDir/list ` + - `$bootclasspathArgs $classpathArgs $sourcepathArgs --no-banner --color --quiet --format=v2 ` + - `$opts && ` + - `${config.SoongZipCmd} -write_if_changed -jar -o $out -C $stubsDir -D $stubsDir && ` + - `(if $writeSdkValues; then ${config.SoongZipCmd} -write_if_changed -d -o $metadataZip ` + - `-C $metadataDir -D $metadataDir; fi) && ` + - `rm -rf "$srcJarDir"`, - CommandDeps: []string{ - "${config.ZipSyncCmd}", - "${config.JavaCmd}", - "${config.MetalavaJar}", - "${config.SoongZipCmd}", - }, - Rspfile: "$out.rsp", - RspfileContent: "$in", - Restat: true, - }, - "outDir", "srcJarDir", "stubsDir", "srcJars", "javaVersion", "bootclasspathArgs", - "classpathArgs", "sourcepathArgs", "opts", "writeSdkValues", "metadataZip", "metadataDir") - - metalavaApiCheck = pctx.AndroidStaticRule("metalavaApiCheck", - blueprint.RuleParams{ - Command: `( rm -rf "$srcJarDir" && mkdir -p "$srcJarDir" && ` + - `${config.ZipSyncCmd} -d $srcJarDir -l $srcJarDir/list -f "*.java" $srcJars && ` + - `${config.JavaCmd} -jar ${config.MetalavaJar} -encoding UTF-8 -source $javaVersion @$out.rsp @$srcJarDir/list ` + - `$bootclasspathArgs $classpathArgs $sourcepathArgs --no-banner --color --quiet --format=v2 ` + - `$opts && touch $out && rm -rf "$srcJarDir") || ` + - `( echo -e "$msg" ; exit 38 )`, - CommandDeps: []string{ - "${config.ZipSyncCmd}", - "${config.JavaCmd}", - "${config.MetalavaJar}", - }, - Rspfile: "$out.rsp", - RspfileContent: "$in", - }, - "srcJarDir", "srcJars", "javaVersion", "bootclasspathArgs", "classpathArgs", "sourcepathArgs", "opts", "msg") +func RegisterDocsBuildComponents(ctx android.RegistrationContext) { + ctx.RegisterModuleType("doc_defaults", DocDefaultsFactory) - nullabilityWarningsCheck = pctx.AndroidStaticRule("nullabilityWarningsCheck", - blueprint.RuleParams{ - Command: `( diff $expected $actual && touch $out ) || ( echo -e "$msg" ; exit 38 )`, - }, - "expected", "actual", "msg") - - dokka = pctx.AndroidStaticRule("dokka", - blueprint.RuleParams{ - Command: `rm -rf "$outDir" "$srcJarDir" "$stubsDir" && ` + - `mkdir -p "$outDir" "$srcJarDir" "$stubsDir" && ` + - `${config.ZipSyncCmd} -d $srcJarDir -l $srcJarDir/list -f "*.java" $srcJars && ` + - `${config.JavaCmd} -jar ${config.DokkaJar} $srcJarDir ` + - `$classpathArgs -format dac -dacRoot /reference/kotlin -output $outDir $opts && ` + - `${config.SoongZipCmd} -write_if_changed -d -o $docZip -C $outDir -D $outDir && ` + - `${config.SoongZipCmd} -write_if_changed -jar -o $out -C $stubsDir -D $stubsDir && ` + - `rm -rf "$srcJarDir"`, - CommandDeps: []string{ - "${config.ZipSyncCmd}", - "${config.DokkaJar}", - "${config.MetalavaJar}", - "${config.SoongZipCmd}", - }, - Restat: true, - }, - "outDir", "srcJarDir", "stubsDir", "srcJars", "classpathArgs", "opts", "docZip") -) + ctx.RegisterModuleType("droiddoc", DroiddocFactory) + ctx.RegisterModuleType("droiddoc_host", DroiddocHostFactory) + ctx.RegisterModuleType("droiddoc_exported_dir", ExportedDroiddocDirFactory) + ctx.RegisterModuleType("javadoc", JavadocFactory) + ctx.RegisterModuleType("javadoc_host", JavadocHostFactory) +} -func init() { - android.RegisterModuleType("doc_defaults", DocDefaultsFactory) - android.RegisterModuleType("stubs_defaults", StubsDefaultsFactory) +func RegisterStubsBuildComponents(ctx android.RegistrationContext) { + ctx.RegisterModuleType("stubs_defaults", StubsDefaultsFactory) - android.RegisterModuleType("droiddoc", DroiddocFactory) - android.RegisterModuleType("droiddoc_host", DroiddocHostFactory) - android.RegisterModuleType("droiddoc_exported_dir", ExportedDroiddocDirFactory) - android.RegisterModuleType("javadoc", JavadocFactory) - android.RegisterModuleType("javadoc_host", JavadocHostFactory) + ctx.RegisterModuleType("droidstubs", DroidstubsFactory) + ctx.RegisterModuleType("droidstubs_host", DroidstubsHostFactory) - android.RegisterModuleType("droidstubs", DroidstubsFactory) - android.RegisterModuleType("droidstubs_host", DroidstubsHostFactory) + ctx.RegisterModuleType("prebuilt_stubs_sources", PrebuiltStubsSourcesFactory) } var ( @@ -170,31 +79,25 @@ type JavadocProperties struct { // filegroup or genrule can be included within this property. Exclude_srcs []string `android:"path,arch_variant"` + // list of package names that should actually be used. If this property is left unspecified, + // all the sources from the srcs property is used. + Filter_packages []string + // list of java libraries that will be in the classpath. Libs []string `android:"arch_variant"` - // don't build against the default libraries (bootclasspath, ext, and framework for device - // targets) - No_standard_libs *bool - - // don't build against the framework libraries (ext, and framework for device targets) - No_framework_libs *bool - - // the java library (in classpath) for documentation that provides java srcs and srcjars. - Srcs_lib *string - - // the base dirs under srcs_lib will be scanned for java srcs. - Srcs_lib_whitelist_dirs []string - - // the sub dirs under srcs_lib_whitelist_dirs will be scanned for java srcs. - Srcs_lib_whitelist_pkgs []string - // If set to false, don't allow this module(-docs.zip) to be exported. Defaults to true. Installable *bool - // if not blank, set to the version of the sdk to compile against + // if not blank, set to the version of the sdk to compile against. + // Defaults to compiling against the current platform. Sdk_version *string `android:"arch_variant"` + // When targeting 1.9 and above, override the modules to use with --system, + // otherwise provides defaults libraries to add to the bootclasspath. + // Defaults to "none" + System_modules *string + Aidl struct { // Top level directories to pass to aidl tool Include_dirs []string @@ -213,10 +116,15 @@ type JavadocProperties struct { // Available variables for substitution: // // $(location <label>): the path to the arg_files with name <label> + // $$: a literal $ Args *string // names of the output files used in args that will be generated Out []string + + // If set, metalava is sandboxed to only read files explicitly specified on the command + // line. Defaults to false. + Sandbox *bool } type ApiToCheck struct { @@ -248,7 +156,7 @@ type DroiddocProperties struct { // proofread file contains all of the text content of the javadocs concatenated into one file, // suitable for spell-checking and other goodness. - Proofread_file *string `android:"path"` + Proofread_file *string // a todo file lists the program elements that are missing documentation. // At some point, this might be improved to show more warnings. @@ -274,37 +182,15 @@ type DroiddocProperties struct { // filegroup or genrule can be included within this property. Knowntags []string `android:"path"` - // the tag name used to distinguish if the API files belong to public/system/test. - Api_tag_name *string - // the generated public API filename by Doclava. Api_filename *string - // the generated public Dex API filename by Doclava. - Dex_api_filename *string - - // the generated private API filename by Doclava. - Private_api_filename *string - - // the generated private Dex API filename by Doclava. - Private_dex_api_filename *string - // the generated removed API filename by Doclava. Removed_api_filename *string // the generated removed Dex API filename by Doclava. Removed_dex_api_filename *string - // mapping of dex signatures to source file and line number. This is a temporary property and - // will be deleted; you probably shouldn't be using it. - Dex_mapping_filename *string - - // the generated exact API filename by Doclava. - Exact_api_filename *string - - // the generated proguard filename by Doclava. - Proguard_filename *string - // if set to false, don't allow droiddoc to generate stubs source files. Defaults to true. Create_stubs *bool @@ -320,48 +206,53 @@ type DroiddocProperties struct { // if set to true, generate docs through Dokka instead of Doclava. Dokka_enabled *bool + + // Compat config XML. Generates compat change documentation if set. + Compat_config *string `android:"path"` } type DroidstubsProperties struct { - // the tag name used to distinguish if the API files belong to public/system/test. - Api_tag_name *string - // the generated public API filename by Metalava. Api_filename *string - // the generated public Dex API filename by Metalava. - Dex_api_filename *string - - // the generated private API filename by Metalava. - Private_api_filename *string - - // the generated private Dex API filename by Metalava. - Private_dex_api_filename *string - // the generated removed API filename by Metalava. Removed_api_filename *string // the generated removed Dex API filename by Metalava. Removed_dex_api_filename *string - // mapping of dex signatures to source file and line number. This is a temporary property and - // will be deleted; you probably shouldn't be using it. - Dex_mapping_filename *string - - // the generated exact API filename by Metalava. - Exact_api_filename *string - - // the generated proguard filename by Metalava. - Proguard_filename *string - Check_api struct { Last_released ApiToCheck Current ApiToCheck - // do not perform API check against Last_released, in the case that both two specified API - // files by Last_released are modules which don't exist. + // The java_sdk_library module generates references to modules (i.e. filegroups) + // from which information about the latest API version can be obtained. As those + // modules may not exist (e.g. because a previous version has not been released) it + // sets ignore_missing_latest_api=true on the droidstubs modules it creates so + // that droidstubs can ignore those references if the modules do not yet exist. + // + // If true then this will ignore module references for modules that do not exist + // in properties that supply the previous version of the API. + // + // There are two sets of those: + // * Api_file, Removed_api_file in check_api.last_released + // * New_since in check_api.api_lint.new_since + // + // The first two must be set as a pair, so either they should both exist or neither + // should exist - in which case when this property is true they are ignored. If one + // exists and the other does not then it is an error. Ignore_missing_latest_api *bool `blueprint:"mutated"` + + Api_lint struct { + Enabled *bool + + // If set, performs api_lint on any new APIs not found in the given signature file + New_since *string `android:"path"` + + // If not blank, path to the baseline txt file for approved API lint violations. + Baseline_file *string `android:"path"` + } } // user can specify the version of previous released API file in order to do compatibility check. @@ -385,12 +276,20 @@ type DroidstubsProperties struct { // if set to true, allow Metalava to generate doc_stubs source files. Defaults to false. Create_doc_stubs *bool + // if set to false then do not write out stubs. Defaults to true. + // + // TODO(b/146727827): Remove capability when we do not need to generate stubs and API separately. + Generate_stubs *bool + // is set to true, Metalava will allow framework SDK to contain API levels annotations. Api_levels_annotations_enabled *bool // the dirs which Metalava extracts API levels annotations from. Api_levels_annotations_dirs []string + // the filename which Metalava extracts API levels annotations from. Defaults to android.jar. + Api_levels_jar_filename *string + // if set to true, collect the values used by the Dev tools and // write them in files packaged with the SDK. Defaults to false. Write_sdk_values *bool @@ -414,14 +313,6 @@ type droiddocBuilderFlags struct { doclavaStubsFlags string doclavaDocsFlags string postDoclavaCmds string - - metalavaStubsFlags string - metalavaAnnotationsFlags string - metalavaMergeAnnoDirFlags string - metalavaInclusionAnnotationsFlags string - metalavaApiLevelsAnnotationsFlags string - - metalavaApiToXmlFlags string } func InitDroiddocModule(module android.DefaultableModule, hod android.HostOrDeviceSupported) { @@ -429,8 +320,10 @@ func InitDroiddocModule(module android.DefaultableModule, hod android.HostOrDevi android.InitDefaultableModule(module) } -func apiCheckEnabled(apiToCheck ApiToCheck, apiVersionTag string) bool { - if String(apiToCheck.Api_file) != "" && String(apiToCheck.Removed_api_file) != "" { +func apiCheckEnabled(ctx android.ModuleContext, apiToCheck ApiToCheck, apiVersionTag string) bool { + if ctx.Config().IsEnvTrue("WITHOUT_CHECK_API") { + return false + } else if String(apiToCheck.Api_file) != "" && String(apiToCheck.Removed_api_file) != "" { return true } else if String(apiToCheck.Api_file) != "" { panic("for " + apiVersionTag + " removed_api_file has to be non-empty!") @@ -460,25 +353,21 @@ func ignoreMissingModules(ctx android.BottomUpMutatorContext, apiToCheck *ApiToC apiToCheck.Removed_api_file = nil } +// Used by xsd_config type ApiFilePath interface { ApiFilePath() android.Path } -func transformUpdateApi(ctx android.ModuleContext, destApiFile, destRemovedApiFile, - srcApiFile, srcRemovedApiFile android.Path, output android.WritablePath) { - ctx.Build(pctx, android.BuildParams{ - Rule: updateApi, - Description: "Update API", - Output: output, - Implicits: append(android.Paths{}, srcApiFile, srcRemovedApiFile, - destApiFile, destRemovedApiFile), - Args: map[string]string{ - "destApiFile": destApiFile.String(), - "srcApiFile": srcApiFile.String(), - "destRemovedApiFile": destRemovedApiFile.String(), - "srcRemovedApiFile": srcRemovedApiFile.String(), - }, - }) +type ApiStubsSrcProvider interface { + StubsSrcJar() android.Path +} + +// Provider of information about API stubs, used by java_sdk_library. +type ApiStubsProvider interface { + ApiFilePath + RemovedApiFilePath() android.Path + + ApiStubsSrcProvider } // @@ -494,6 +383,7 @@ type Javadoc struct { srcFiles android.Paths sourcepaths android.Paths argFiles android.Paths + implicits android.Paths args string @@ -501,10 +391,18 @@ type Javadoc struct { stubsSrcJar android.WritablePath } -func (j *Javadoc) Srcs() android.Paths { - return android.Paths{j.stubsSrcJar} +func (j *Javadoc) OutputFiles(tag string) (android.Paths, error) { + switch tag { + case "": + return android.Paths{j.stubsSrcJar}, nil + case ".docs.zip": + return android.Paths{j.docZip}, nil + default: + return nil, fmt.Errorf("unsupported module reference tag %q", tag) + } } +// javadoc converts .java source files to documentation using javadoc. func JavadocFactory() android.Module { module := &Javadoc{} @@ -514,6 +412,7 @@ func JavadocFactory() android.Module { return module } +// javadoc_host converts .java source files to documentation using javadoc. func JavadocHostFactory() android.Module { module := &Javadoc{} @@ -523,58 +422,41 @@ func JavadocHostFactory() android.Module { return module } -var _ android.SourceFileProducer = (*Javadoc)(nil) +var _ android.OutputFileProducer = (*Javadoc)(nil) + +func (j *Javadoc) sdkVersion() sdkSpec { + return sdkSpecFrom(String(j.properties.Sdk_version)) +} -func (j *Javadoc) sdkVersion() string { - return String(j.properties.Sdk_version) +func (j *Javadoc) systemModules() string { + return proptools.String(j.properties.System_modules) } -func (j *Javadoc) minSdkVersion() string { +func (j *Javadoc) minSdkVersion() sdkSpec { return j.sdkVersion() } -func (j *Javadoc) targetSdkVersion() string { +func (j *Javadoc) targetSdkVersion() sdkSpec { return j.sdkVersion() } func (j *Javadoc) addDeps(ctx android.BottomUpMutatorContext) { if ctx.Device() { - if !Bool(j.properties.No_standard_libs) { - sdkDep := decodeSdkDep(ctx, sdkContext(j)) - if sdkDep.useDefaultLibs { - ctx.AddVariationDependencies(nil, bootClasspathTag, config.DefaultBootclasspathLibraries...) - if ctx.Config().TargetOpenJDK9() { - ctx.AddVariationDependencies(nil, systemModulesTag, config.DefaultSystemModules) - } - if !Bool(j.properties.No_framework_libs) { - ctx.AddVariationDependencies(nil, libTag, config.DefaultLibraries...) - } - } else if sdkDep.useModule { - if ctx.Config().TargetOpenJDK9() { - ctx.AddVariationDependencies(nil, systemModulesTag, sdkDep.systemModules) - } - ctx.AddVariationDependencies(nil, bootClasspathTag, sdkDep.modules...) + sdkDep := decodeSdkDep(ctx, sdkContext(j)) + if sdkDep.useDefaultLibs { + ctx.AddVariationDependencies(nil, bootClasspathTag, config.DefaultBootclasspathLibraries...) + ctx.AddVariationDependencies(nil, systemModulesTag, config.DefaultSystemModules) + if sdkDep.hasFrameworkLibs() { + ctx.AddVariationDependencies(nil, libTag, config.DefaultLibraries...) } + } else if sdkDep.useModule { + ctx.AddVariationDependencies(nil, bootClasspathTag, sdkDep.bootclasspath...) + ctx.AddVariationDependencies(nil, systemModulesTag, sdkDep.systemModules) + ctx.AddVariationDependencies(nil, java9LibTag, sdkDep.java9Classpath...) } } ctx.AddVariationDependencies(nil, libTag, j.properties.Libs...) - if j.properties.Srcs_lib != nil { - ctx.AddVariationDependencies(nil, srcsLibTag, *j.properties.Srcs_lib) - } -} - -func (j *Javadoc) genWhitelistPathPrefixes(whitelistPathPrefixes map[string]bool) { - for _, dir := range j.properties.Srcs_lib_whitelist_dirs { - for _, pkg := range j.properties.Srcs_lib_whitelist_pkgs { - // convert foo.bar.baz to foo/bar/baz - pkgAsPath := filepath.Join(strings.Split(pkg, ".")...) - prefix := filepath.Join(dir, pkgAsPath) - if _, found := whitelistPathPrefixes[prefix]; !found { - whitelistPathPrefixes[prefix] = true - } - } - } } func (j *Javadoc) collectAidlFlags(ctx android.ModuleContext, deps deps) droiddocBuilderFlags { @@ -610,24 +492,33 @@ func (j *Javadoc) aidlFlags(ctx android.ModuleContext, aidlPreprocess android.Op return strings.Join(flags, " "), deps } +// TODO: remove the duplication between this and the one in gen.go func (j *Javadoc) genSources(ctx android.ModuleContext, srcFiles android.Paths, flags droiddocBuilderFlags) android.Paths { outSrcFiles := make(android.Paths, 0, len(srcFiles)) + var aidlSrcs android.Paths + + aidlIncludeFlags := genAidlIncludeFlags(srcFiles) for _, srcFile := range srcFiles { switch srcFile.Ext() { case ".aidl": - javaFile := genAidl(ctx, srcFile, flags.aidlFlags, flags.aidlDeps) - outSrcFiles = append(outSrcFiles, javaFile) - case ".sysprop": - javaFile := genSysprop(ctx, srcFile) + aidlSrcs = append(aidlSrcs, srcFile) + case ".logtags": + javaFile := genLogtags(ctx, srcFile) outSrcFiles = append(outSrcFiles, javaFile) default: outSrcFiles = append(outSrcFiles, srcFile) } } + // Process all aidl files together to support sharding them into one or more rules that produce srcjars. + if len(aidlSrcs) > 0 { + srcJarFiles := genAidl(ctx, aidlSrcs, flags.aidlFlags+aidlIncludeFlags, flags.aidlDeps) + outSrcFiles = append(outSrcFiles, srcJarFiles...) + } + return outSrcFiles } @@ -636,9 +527,13 @@ func (j *Javadoc) collectDeps(ctx android.ModuleContext) deps { sdkDep := decodeSdkDep(ctx, sdkContext(j)) if sdkDep.invalidVersion { - ctx.AddMissingDependencies(sdkDep.modules) + ctx.AddMissingDependencies(sdkDep.bootclasspath) + ctx.AddMissingDependencies(sdkDep.java9Classpath) } else if sdkDep.useFiles { deps.bootClasspath = append(deps.bootClasspath, sdkDep.jars...) + deps.aidlPreprocess = sdkDep.aidl + } else { + deps.aidlPreprocess = sdkDep.aidl } ctx.VisitDirectDeps(func(module android.Module) { @@ -649,40 +544,30 @@ func (j *Javadoc) collectDeps(ctx android.ModuleContext) deps { case bootClasspathTag: if dep, ok := module.(Dependency); ok { deps.bootClasspath = append(deps.bootClasspath, dep.ImplementationJars()...) + } else if sm, ok := module.(SystemModulesProvider); ok { + // A system modules dependency has been added to the bootclasspath + // so add its libs to the bootclasspath. + deps.bootClasspath = append(deps.bootClasspath, sm.HeaderJars()...) } else { panic(fmt.Errorf("unknown dependency %q for %q", otherName, ctx.ModuleName())) } case libTag: switch dep := module.(type) { case SdkLibraryDependency: - deps.classpath = append(deps.classpath, dep.SdkImplementationJars(ctx, j.sdkVersion())...) + deps.classpath = append(deps.classpath, dep.SdkHeaderJars(ctx, j.sdkVersion())...) case Dependency: deps.classpath = append(deps.classpath, dep.HeaderJars()...) + deps.aidlIncludeDirs = append(deps.aidlIncludeDirs, dep.AidlIncludeDirs()...) case android.SourceFileProducer: checkProducesJars(ctx, dep) deps.classpath = append(deps.classpath, dep.Srcs()...) default: ctx.ModuleErrorf("depends on non-java module %q", otherName) } - case srcsLibTag: + case java9LibTag: switch dep := module.(type) { case Dependency: - srcs := dep.(SrcDependency).CompiledSrcs() - whitelistPathPrefixes := make(map[string]bool) - j.genWhitelistPathPrefixes(whitelistPathPrefixes) - for _, src := range srcs { - if _, ok := src.(android.WritablePath); ok { // generated sources - deps.srcs = append(deps.srcs, src) - } else { // select source path for documentation based on whitelist path prefixs. - for k := range whitelistPathPrefixes { - if strings.HasPrefix(src.Rel(), k) { - deps.srcs = append(deps.srcs, src) - break - } - } - } - } - deps.srcJars = append(deps.srcJars, dep.(SrcDependency).CompiledSrcJars()...) + deps.java9Classpath = append(deps.java9Classpath, dep.HeaderJars()...) default: ctx.ModuleErrorf("depends on non-java module %q", otherName) } @@ -690,16 +575,58 @@ func (j *Javadoc) collectDeps(ctx android.ModuleContext) deps { if deps.systemModules != nil { panic("Found two system module dependencies") } - sm := module.(*SystemModules) - if sm.outputFile == nil { - panic("Missing directory for system module dependency") - } - deps.systemModules = sm.outputFile + sm := module.(SystemModulesProvider) + outputDir, outputDeps := sm.OutputDirAndDeps() + deps.systemModules = &systemModules{outputDir, outputDeps} } }) // do not pass exclude_srcs directly when expanding srcFiles since exclude_srcs // may contain filegroup or genrule. srcFiles := android.PathsForModuleSrcExcludes(ctx, j.properties.Srcs, j.properties.Exclude_srcs) + j.implicits = append(j.implicits, srcFiles...) + + filterByPackage := func(srcs []android.Path, filterPackages []string) []android.Path { + if filterPackages == nil { + return srcs + } + filtered := []android.Path{} + for _, src := range srcs { + if src.Ext() != ".java" { + // Don't filter-out non-Java (=generated sources) by package names. This is not ideal, + // but otherwise metalava emits stub sources having references to the generated AIDL classes + // in filtered-out pacages (e.g. com.android.internal.*). + // TODO(b/141149570) We need to fix this by introducing default private constructors or + // fixing metalava to not emit constructors having references to unknown classes. + filtered = append(filtered, src) + continue + } + packageName := strings.ReplaceAll(filepath.Dir(src.Rel()), "/", ".") + if android.HasAnyPrefix(packageName, filterPackages) { + filtered = append(filtered, src) + } + } + return filtered + } + srcFiles = filterByPackage(srcFiles, j.properties.Filter_packages) + + // While metalava needs package html files, it does not need them to be explicit on the command + // line. More importantly, the metalava rsp file is also used by the subsequent jdiff action if + // jdiff_enabled=true. javadoc complains if it receives html files on the command line. The filter + // below excludes html files from the rsp file for both metalava and jdiff. Note that the html + // files are still included as implicit inputs for successful remote execution and correct + // incremental builds. + filterHtml := func(srcs []android.Path) []android.Path { + filtered := []android.Path{} + for _, src := range srcs { + if src.Ext() == ".html" { + continue + } + filtered = append(filtered, src) + } + return filtered + } + srcFiles = filterHtml(srcFiles) + flags := j.collectAidlFlags(ctx, deps) srcFiles = j.genSources(ctx, srcFiles, flags) @@ -710,9 +637,6 @@ func (j *Javadoc) collectDeps(ctx android.ModuleContext) deps { j.srcFiles = srcFiles.FilterOutByExt(".srcjar") j.srcFiles = append(j.srcFiles, deps.srcs...) - j.docZip = android.PathForModuleOut(ctx, ctx.ModuleName()+"-"+"docs.zip") - j.stubsSrcJar = android.PathForModuleOut(ctx, ctx.ModuleName()+"-"+"stubs.srcjar") - if j.properties.Local_sourcepaths == nil && len(j.srcFiles) > 0 { j.properties.Local_sourcepaths = append(j.properties.Local_sourcepaths, ".") } @@ -763,51 +687,43 @@ func (j *Javadoc) DepsMutator(ctx android.BottomUpMutatorContext) { func (j *Javadoc) GenerateAndroidBuildActions(ctx android.ModuleContext) { deps := j.collectDeps(ctx) - var implicits android.Paths - implicits = append(implicits, deps.bootClasspath...) - implicits = append(implicits, deps.classpath...) + j.docZip = android.PathForModuleOut(ctx, ctx.ModuleName()+"-"+"docs.zip") + + outDir := android.PathForModuleOut(ctx, "out") + srcJarDir := android.PathForModuleOut(ctx, "srcjars") + + j.stubsSrcJar = nil + + rule := android.NewRuleBuilder() + + rule.Command().Text("rm -rf").Text(outDir.String()) + rule.Command().Text("mkdir -p").Text(outDir.String()) - var bootClasspathArgs, classpathArgs, sourcepathArgs string + srcJarList := zipSyncCmd(ctx, rule, srcJarDir, j.srcJars) javaVersion := getJavaVersion(ctx, String(j.properties.Java_version), sdkContext(j)) - if len(deps.bootClasspath) > 0 { - var systemModules classpath - if deps.systemModules != nil { - systemModules = append(systemModules, deps.systemModules) - } - bootClasspathArgs = systemModules.FormJavaSystemModulesPath("--system ", ctx.Device()) - bootClasspathArgs = bootClasspathArgs + " --patch-module java.base=." - } - if len(deps.classpath.Strings()) > 0 { - classpathArgs = "-classpath " + strings.Join(deps.classpath.Strings(), ":") - } - - implicits = append(implicits, j.srcJars...) - implicits = append(implicits, j.argFiles...) - - opts := "-source " + javaVersion + " -J-Xmx1024m -XDignore.symbol.file -Xdoclint:none" - - sourcepathArgs = "-sourcepath " + strings.Join(j.sourcepaths.Strings(), ":") - - ctx.Build(pctx, android.BuildParams{ - Rule: javadoc, - Description: "Javadoc", - Output: j.stubsSrcJar, - ImplicitOutput: j.docZip, - Inputs: j.srcFiles, - Implicits: implicits, - Args: map[string]string{ - "outDir": android.PathForModuleOut(ctx, "out").String(), - "srcJarDir": android.PathForModuleOut(ctx, "srcjars").String(), - "stubsDir": android.PathForModuleOut(ctx, "stubsDir").String(), - "srcJars": strings.Join(j.srcJars.Strings(), " "), - "opts": opts, - "bootclasspathArgs": bootClasspathArgs, - "classpathArgs": classpathArgs, - "sourcepathArgs": sourcepathArgs, - "docZip": j.docZip.String(), - }, - }) + + cmd := javadocSystemModulesCmd(ctx, rule, j.srcFiles, outDir, srcJarDir, srcJarList, + deps.systemModules, deps.classpath, j.sourcepaths) + + cmd.FlagWithArg("-source ", javaVersion.String()). + Flag("-J-Xmx1024m"). + Flag("-XDignore.symbol.file"). + Flag("-Xdoclint:none") + + rule.Command(). + BuiltTool(ctx, "soong_zip"). + Flag("-write_if_changed"). + Flag("-d"). + FlagWithOutput("-o ", j.docZip). + FlagWithArg("-C ", outDir.String()). + FlagWithArg("-D ", outDir.String()) + + rule.Restat() + + zipSyncCleanupCmd(rule, srcJarDir) + + rule.Build(pctx, ctx, "javadoc", "javadoc") } // @@ -818,14 +734,9 @@ type Droiddoc struct { properties DroiddocProperties apiFile android.WritablePath - dexApiFile android.WritablePath privateApiFile android.WritablePath - privateDexApiFile android.WritablePath removedApiFile android.WritablePath removedDexApiFile android.WritablePath - exactApiFile android.WritablePath - apiMappingFile android.WritablePath - proguardFile android.WritablePath checkCurrentApiTimestamp android.WritablePath updateCurrentApiTimestamp android.WritablePath @@ -834,6 +745,7 @@ type Droiddoc struct { apiFilePath android.Path } +// droiddoc converts .java source files to documentation using doclava or dokka. func DroiddocFactory() android.Module { module := &Droiddoc{} @@ -844,6 +756,7 @@ func DroiddocFactory() android.Module { return module } +// droiddoc_host converts .java source files to documentation using doclava or dokka. func DroiddocHostFactory() android.Module { module := &Droiddoc{} @@ -870,58 +783,19 @@ func (d *Droiddoc) DepsMutator(ctx android.BottomUpMutatorContext) { } } -func (d *Droiddoc) initBuilderFlags(ctx android.ModuleContext, implicits *android.Paths, - deps deps) (droiddocBuilderFlags, error) { - var flags droiddocBuilderFlags - - *implicits = append(*implicits, deps.bootClasspath...) - *implicits = append(*implicits, deps.classpath...) - - if len(deps.bootClasspath.Strings()) > 0 { - // For OpenJDK 8 we can use -bootclasspath to define the core libraries code. - flags.bootClasspathArgs = deps.bootClasspath.FormJavaClassPath("-bootclasspath") - } - flags.classpathArgs = deps.classpath.FormJavaClassPath("-classpath") - // Dokka doesn't support bootClasspath, so combine these two classpath vars for Dokka. - dokkaClasspath := classpath{} - dokkaClasspath = append(dokkaClasspath, deps.bootClasspath...) - dokkaClasspath = append(dokkaClasspath, deps.classpath...) - flags.dokkaClasspathArgs = dokkaClasspath.FormJavaClassPath("-classpath") - - // TODO(nanzhang): Remove this if- statement once we finish migration for all Doclava - // based stubs generation. - // In the future, all the docs generation depends on Metalava stubs (droidstubs) srcjar - // dir. We need add the srcjar dir to -sourcepath arg, so that Javadoc can figure out - // the correct package name base path. - if len(d.Javadoc.properties.Local_sourcepaths) > 0 { - flags.sourcepathArgs = "-sourcepath " + strings.Join(d.Javadoc.sourcepaths.Strings(), ":") - } else { - flags.sourcepathArgs = "-sourcepath " + android.PathForModuleOut(ctx, "srcjars").String() - } - - return flags, nil -} - -func (d *Droiddoc) collectDoclavaDocsFlags(ctx android.ModuleContext, implicits *android.Paths, - jsilver, doclava android.Path) string { - - *implicits = append(*implicits, jsilver) - *implicits = append(*implicits, doclava) - - var date string - if runtime.GOOS == "darwin" { - date = `date -r` - } else { - date = `date -d` - } - +func (d *Droiddoc) doclavaDocsFlags(ctx android.ModuleContext, cmd *android.RuleBuilderCommand, docletPath classpath) { + buildNumberFile := ctx.Config().BuildNumberFile(ctx) // Droiddoc always gets "-source 1.8" because it doesn't support 1.9 sources. For modules with 1.9 // sources, droiddoc will get sources produced by metalava which will have already stripped out the // 1.9 language features. - args := " -source 1.8 -J-Xmx1600m -J-XX:-OmitStackTraceInFastThrow -XDignore.symbol.file " + - "-doclet com.google.doclava.Doclava -docletpath " + jsilver.String() + ":" + doclava.String() + " " + - "-hdf page.build " + ctx.Config().BuildId() + "-" + ctx.Config().BuildNumberFromFile() + " " + - `-hdf page.now "$$(` + date + ` @$$(cat ` + ctx.Config().Getenv("BUILD_DATETIME_FILE") + `) "+%d %b %Y %k:%M")" ` + cmd.FlagWithArg("-source ", "1.8"). + Flag("-J-Xmx1600m"). + Flag("-J-XX:-OmitStackTraceInFastThrow"). + Flag("-XDignore.symbol.file"). + FlagWithArg("-doclet ", "com.google.doclava.Doclava"). + FlagWithInputList("-docletpath ", docletPath.Paths(), ":"). + FlagWithArg("-hdf page.build ", ctx.Config().BuildId()+"-$(cat "+buildNumberFile.String()+")").OrderOnly(buildNumberFile). + FlagWithArg("-hdf page.now ", `"$(date -d @$(cat `+ctx.Config().Getenv("BUILD_DATETIME_FILE")+`) "+%d %b %Y %k:%M")" `) if String(d.properties.Custom_template) == "" { // TODO: This is almost always droiddoc-templates-sdk @@ -930,23 +804,22 @@ func (d *Droiddoc) collectDoclavaDocsFlags(ctx android.ModuleContext, implicits ctx.VisitDirectDepsWithTag(droiddocTemplateTag, func(m android.Module) { if t, ok := m.(*ExportedDroiddocDir); ok { - *implicits = append(*implicits, t.deps...) - args = args + " -templatedir " + t.dir.String() + cmd.FlagWithArg("-templatedir ", t.dir.String()).Implicits(t.deps) } else { - ctx.PropertyErrorf("custom_template", "module %q is not a droiddoc_template", ctx.OtherModuleName(m)) + ctx.PropertyErrorf("custom_template", "module %q is not a droiddoc_exported_dir", ctx.OtherModuleName(m)) } }) if len(d.properties.Html_dirs) > 0 { - htmlDir := d.properties.Html_dirs[0] - *implicits = append(*implicits, android.PathsForModuleSrc(ctx, []string{filepath.Join(d.properties.Html_dirs[0], "**/*")})...) - args = args + " -htmldir " + htmlDir + htmlDir := android.PathForModuleSrc(ctx, d.properties.Html_dirs[0]) + cmd.FlagWithArg("-htmldir ", htmlDir.String()). + Implicits(android.PathsForModuleSrc(ctx, []string{filepath.Join(d.properties.Html_dirs[0], "**/*")})) } if len(d.properties.Html_dirs) > 1 { - htmlDir2 := d.properties.Html_dirs[1] - *implicits = append(*implicits, android.PathsForModuleSrc(ctx, []string{filepath.Join(htmlDir2, "**/*")})...) - args = args + " -htmldir2 " + htmlDir2 + htmlDir2 := android.PathForModuleSrc(ctx, d.properties.Html_dirs[1]) + cmd.FlagWithArg("-htmldir2 ", htmlDir2.String()). + Implicits(android.PathsForModuleSrc(ctx, []string{filepath.Join(d.properties.Html_dirs[1], "**/*")})) } if len(d.properties.Html_dirs) > 2 { @@ -954,275 +827,343 @@ func (d *Droiddoc) collectDoclavaDocsFlags(ctx android.ModuleContext, implicits } knownTags := android.PathsForModuleSrc(ctx, d.properties.Knowntags) - *implicits = append(*implicits, knownTags...) + cmd.FlagForEachInput("-knowntags ", knownTags) - for _, kt := range knownTags { - args = args + " -knowntags " + kt.String() - } - - for _, hdf := range d.properties.Hdf { - args = args + " -hdf " + hdf - } + cmd.FlagForEachArg("-hdf ", d.properties.Hdf) if String(d.properties.Proofread_file) != "" { proofreadFile := android.PathForModuleOut(ctx, String(d.properties.Proofread_file)) - args = args + " -proofread " + proofreadFile.String() + cmd.FlagWithOutput("-proofread ", proofreadFile) } if String(d.properties.Todo_file) != "" { // tricky part: // we should not compute full path for todo_file through PathForModuleOut(). // the non-standard doclet will get the full path relative to "-o". - args = args + " -todo " + String(d.properties.Todo_file) + cmd.FlagWithArg("-todo ", String(d.properties.Todo_file)). + ImplicitOutput(android.PathForModuleOut(ctx, String(d.properties.Todo_file))) } if String(d.properties.Resourcesdir) != "" { // TODO: should we add files under resourcesDir to the implicits? It seems that // resourcesDir is one sub dir of htmlDir resourcesDir := android.PathForModuleSrc(ctx, String(d.properties.Resourcesdir)) - args = args + " -resourcesdir " + resourcesDir.String() + cmd.FlagWithArg("-resourcesdir ", resourcesDir.String()) } if String(d.properties.Resourcesoutdir) != "" { // TODO: it seems -resourceoutdir reference/android/images/ didn't get generated anywhere. - args = args + " -resourcesoutdir " + String(d.properties.Resourcesoutdir) + cmd.FlagWithArg("-resourcesoutdir ", String(d.properties.Resourcesoutdir)) } - return args } -func (d *Droiddoc) collectStubsFlags(ctx android.ModuleContext, - implicitOutputs *android.WritablePaths) string { - var doclavaFlags string - if apiCheckEnabled(d.properties.Check_api.Current, "current") || - apiCheckEnabled(d.properties.Check_api.Last_released, "last_released") || +func (d *Droiddoc) stubsFlags(ctx android.ModuleContext, cmd *android.RuleBuilderCommand, stubsDir android.WritablePath) { + if apiCheckEnabled(ctx, d.properties.Check_api.Current, "current") || + apiCheckEnabled(ctx, d.properties.Check_api.Last_released, "last_released") || String(d.properties.Api_filename) != "" { + d.apiFile = android.PathForModuleOut(ctx, ctx.ModuleName()+"_api.txt") - doclavaFlags += " -api " + d.apiFile.String() - *implicitOutputs = append(*implicitOutputs, d.apiFile) + cmd.FlagWithOutput("-api ", d.apiFile) d.apiFilePath = d.apiFile } - if apiCheckEnabled(d.properties.Check_api.Current, "current") || - apiCheckEnabled(d.properties.Check_api.Last_released, "last_released") || + if apiCheckEnabled(ctx, d.properties.Check_api.Current, "current") || + apiCheckEnabled(ctx, d.properties.Check_api.Last_released, "last_released") || String(d.properties.Removed_api_filename) != "" { d.removedApiFile = android.PathForModuleOut(ctx, ctx.ModuleName()+"_removed.txt") - doclavaFlags += " -removedApi " + d.removedApiFile.String() - *implicitOutputs = append(*implicitOutputs, d.removedApiFile) + cmd.FlagWithOutput("-removedApi ", d.removedApiFile) } - if String(d.properties.Private_api_filename) != "" { - d.privateApiFile = android.PathForModuleOut(ctx, String(d.properties.Private_api_filename)) - doclavaFlags += " -privateApi " + d.privateApiFile.String() - *implicitOutputs = append(*implicitOutputs, d.privateApiFile) + if String(d.properties.Removed_dex_api_filename) != "" { + d.removedDexApiFile = android.PathForModuleOut(ctx, String(d.properties.Removed_dex_api_filename)) + cmd.FlagWithOutput("-removedDexApi ", d.removedDexApiFile) } - if String(d.properties.Dex_api_filename) != "" { - d.dexApiFile = android.PathForModuleOut(ctx, String(d.properties.Dex_api_filename)) - doclavaFlags += " -dexApi " + d.dexApiFile.String() - *implicitOutputs = append(*implicitOutputs, d.dexApiFile) + if BoolDefault(d.properties.Create_stubs, true) { + cmd.FlagWithArg("-stubs ", stubsDir.String()) } - if String(d.properties.Private_dex_api_filename) != "" { - d.privateDexApiFile = android.PathForModuleOut(ctx, String(d.properties.Private_dex_api_filename)) - doclavaFlags += " -privateDexApi " + d.privateDexApiFile.String() - *implicitOutputs = append(*implicitOutputs, d.privateDexApiFile) + if Bool(d.properties.Write_sdk_values) { + cmd.FlagWithArg("-sdkvalues ", android.PathForModuleOut(ctx, "out").String()) } +} - if String(d.properties.Removed_dex_api_filename) != "" { - d.removedDexApiFile = android.PathForModuleOut(ctx, String(d.properties.Removed_dex_api_filename)) - doclavaFlags += " -removedDexApi " + d.removedDexApiFile.String() - *implicitOutputs = append(*implicitOutputs, d.removedDexApiFile) +func (d *Droiddoc) postDoclavaCmds(ctx android.ModuleContext, rule *android.RuleBuilder) { + if String(d.properties.Static_doc_index_redirect) != "" { + staticDocIndexRedirect := android.PathForModuleSrc(ctx, String(d.properties.Static_doc_index_redirect)) + rule.Command().Text("cp"). + Input(staticDocIndexRedirect). + Output(android.PathForModuleOut(ctx, "out", "index.html")) } - if String(d.properties.Exact_api_filename) != "" { - d.exactApiFile = android.PathForModuleOut(ctx, String(d.properties.Exact_api_filename)) - doclavaFlags += " -exactApi " + d.exactApiFile.String() - *implicitOutputs = append(*implicitOutputs, d.exactApiFile) + if String(d.properties.Static_doc_properties) != "" { + staticDocProperties := android.PathForModuleSrc(ctx, String(d.properties.Static_doc_properties)) + rule.Command().Text("cp"). + Input(staticDocProperties). + Output(android.PathForModuleOut(ctx, "out", "source.properties")) } +} - if String(d.properties.Dex_mapping_filename) != "" { - d.apiMappingFile = android.PathForModuleOut(ctx, String(d.properties.Dex_mapping_filename)) - doclavaFlags += " -apiMapping " + d.apiMappingFile.String() - *implicitOutputs = append(*implicitOutputs, d.apiMappingFile) - } +func javadocCmd(ctx android.ModuleContext, rule *android.RuleBuilder, srcs android.Paths, + outDir, srcJarDir, srcJarList android.Path, sourcepaths android.Paths) *android.RuleBuilderCommand { - if String(d.properties.Proguard_filename) != "" { - d.proguardFile = android.PathForModuleOut(ctx, String(d.properties.Proguard_filename)) - doclavaFlags += " -proguard " + d.proguardFile.String() - *implicitOutputs = append(*implicitOutputs, d.proguardFile) - } + cmd := rule.Command(). + BuiltTool(ctx, "soong_javac_wrapper").Tool(config.JavadocCmd(ctx)). + Flag(config.JavacVmFlags). + FlagWithArg("-encoding ", "UTF-8"). + FlagWithRspFileInputList("@", srcs). + FlagWithInput("@", srcJarList) - if BoolDefault(d.properties.Create_stubs, true) { - doclavaFlags += " -stubs " + android.PathForModuleOut(ctx, "stubsDir").String() + // TODO(ccross): Remove this if- statement once we finish migration for all Doclava + // based stubs generation. + // In the future, all the docs generation depends on Metalava stubs (droidstubs) srcjar + // dir. We need add the srcjar dir to -sourcepath arg, so that Javadoc can figure out + // the correct package name base path. + if len(sourcepaths) > 0 { + cmd.FlagWithList("-sourcepath ", sourcepaths.Strings(), ":") + } else { + cmd.FlagWithArg("-sourcepath ", srcJarDir.String()) } - if Bool(d.properties.Write_sdk_values) { - doclavaFlags += " -sdkvalues " + android.PathForModuleOut(ctx, "out").String() - } + cmd.FlagWithArg("-d ", outDir.String()). + Flag("-quiet") - return doclavaFlags + return cmd } -func (d *Droiddoc) getPostDoclavaCmds(ctx android.ModuleContext, implicits *android.Paths) string { - var cmds string - if String(d.properties.Static_doc_index_redirect) != "" { - static_doc_index_redirect := ctx.ExpandSource(String(d.properties.Static_doc_index_redirect), - "static_doc_index_redirect") - *implicits = append(*implicits, static_doc_index_redirect) - cmds = cmds + " && cp " + static_doc_index_redirect.String() + " " + - android.PathForModuleOut(ctx, "out", "index.html").String() +func javadocSystemModulesCmd(ctx android.ModuleContext, rule *android.RuleBuilder, srcs android.Paths, + outDir, srcJarDir, srcJarList android.Path, systemModules *systemModules, + classpath classpath, sourcepaths android.Paths) *android.RuleBuilderCommand { + + cmd := javadocCmd(ctx, rule, srcs, outDir, srcJarDir, srcJarList, sourcepaths) + + flag, deps := systemModules.FormJavaSystemModulesPath(ctx.Device()) + cmd.Flag(flag).Implicits(deps) + + cmd.FlagWithArg("--patch-module ", "java.base=.") + + if len(classpath) > 0 { + cmd.FlagWithInputList("-classpath ", classpath.Paths(), ":") } - if String(d.properties.Static_doc_properties) != "" { - static_doc_properties := ctx.ExpandSource(String(d.properties.Static_doc_properties), - "static_doc_properties") - *implicits = append(*implicits, static_doc_properties) - cmds = cmds + " && cp " + static_doc_properties.String() + " " + - android.PathForModuleOut(ctx, "out", "source.properties").String() - } - return cmds -} - -func (d *Droiddoc) transformDoclava(ctx android.ModuleContext, implicits android.Paths, - implicitOutputs android.WritablePaths, - bootclasspathArgs, classpathArgs, sourcepathArgs, opts, postDoclavaCmds string) { - ctx.Build(pctx, android.BuildParams{ - Rule: javadoc, - Description: "Doclava", - Output: d.Javadoc.stubsSrcJar, - Inputs: d.Javadoc.srcFiles, - Implicits: implicits, - ImplicitOutputs: implicitOutputs, - Args: map[string]string{ - "outDir": android.PathForModuleOut(ctx, "out").String(), - "srcJarDir": android.PathForModuleOut(ctx, "srcjars").String(), - "stubsDir": android.PathForModuleOut(ctx, "stubsDir").String(), - "srcJars": strings.Join(d.Javadoc.srcJars.Strings(), " "), - "opts": opts, - "bootclasspathArgs": bootclasspathArgs, - "classpathArgs": classpathArgs, - "sourcepathArgs": sourcepathArgs, - "docZip": d.Javadoc.docZip.String(), - "postDoclavaCmds": postDoclavaCmds, - }, - }) + return cmd } -func (d *Droiddoc) transformCheckApi(ctx android.ModuleContext, apiFile, removedApiFile android.Path, - checkApiClasspath classpath, msg, opts string, output android.WritablePath) { - ctx.Build(pctx, android.BuildParams{ - Rule: apiCheck, - Description: "Doclava Check API", - Output: output, - Inputs: nil, - Implicits: append(android.Paths{apiFile, removedApiFile, d.apiFile, d.removedApiFile}, - checkApiClasspath...), - Args: map[string]string{ - "msg": msg, - "classpath": checkApiClasspath.FormJavaClassPath(""), - "opts": opts, - "apiFile": apiFile.String(), - "apiFileToCheck": d.apiFile.String(), - "removedApiFile": removedApiFile.String(), - "removedApiFileToCheck": d.removedApiFile.String(), - }, - }) +func javadocBootclasspathCmd(ctx android.ModuleContext, rule *android.RuleBuilder, srcs android.Paths, + outDir, srcJarDir, srcJarList android.Path, bootclasspath, classpath classpath, + sourcepaths android.Paths) *android.RuleBuilderCommand { + + cmd := javadocCmd(ctx, rule, srcs, outDir, srcJarDir, srcJarList, sourcepaths) + + if len(bootclasspath) == 0 && ctx.Device() { + // explicitly specify -bootclasspath "" if the bootclasspath is empty to + // ensure java does not fall back to the default bootclasspath. + cmd.FlagWithArg("-bootclasspath ", `""`) + } else if len(bootclasspath) > 0 { + cmd.FlagWithInputList("-bootclasspath ", bootclasspath.Paths(), ":") + } + + if len(classpath) > 0 { + cmd.FlagWithInputList("-classpath ", classpath.Paths(), ":") + } + + return cmd } -func (d *Droiddoc) transformDokka(ctx android.ModuleContext, implicits android.Paths, - classpathArgs, opts string) { - ctx.Build(pctx, android.BuildParams{ - Rule: dokka, - Description: "Dokka", - Output: d.Javadoc.stubsSrcJar, - Inputs: d.Javadoc.srcFiles, - Implicits: implicits, - Args: map[string]string{ - "outDir": android.PathForModuleOut(ctx, "dokka-out").String(), - "srcJarDir": android.PathForModuleOut(ctx, "dokka-srcjars").String(), - "stubsDir": android.PathForModuleOut(ctx, "dokka-stubsDir").String(), - "srcJars": strings.Join(d.Javadoc.srcJars.Strings(), " "), - "classpathArgs": classpathArgs, - "opts": opts, - "docZip": d.Javadoc.docZip.String(), - }, - }) +func dokkaCmd(ctx android.ModuleContext, rule *android.RuleBuilder, + outDir, srcJarDir android.Path, bootclasspath, classpath classpath) *android.RuleBuilderCommand { + + // Dokka doesn't support bootClasspath, so combine these two classpath vars for Dokka. + dokkaClasspath := append(bootclasspath.Paths(), classpath.Paths()...) + + return rule.Command(). + BuiltTool(ctx, "dokka"). + Flag(config.JavacVmFlags). + Flag(srcJarDir.String()). + FlagWithInputList("-classpath ", dokkaClasspath, ":"). + FlagWithArg("-format ", "dac"). + FlagWithArg("-dacRoot ", "/reference/kotlin"). + FlagWithArg("-output ", outDir.String()) } func (d *Droiddoc) GenerateAndroidBuildActions(ctx android.ModuleContext) { deps := d.Javadoc.collectDeps(ctx) + d.Javadoc.docZip = android.PathForModuleOut(ctx, ctx.ModuleName()+"-"+"docs.zip") + d.Javadoc.stubsSrcJar = android.PathForModuleOut(ctx, ctx.ModuleName()+"-"+"stubs.srcjar") + jsilver := android.PathForOutput(ctx, "host", ctx.Config().PrebuiltOS(), "framework", "jsilver.jar") doclava := android.PathForOutput(ctx, "host", ctx.Config().PrebuiltOS(), "framework", "doclava.jar") java8Home := ctx.Config().Getenv("ANDROID_JAVA8_HOME") checkApiClasspath := classpath{jsilver, doclava, android.PathForSource(ctx, java8Home, "lib/tools.jar")} - var implicits android.Paths - implicits = append(implicits, d.Javadoc.srcJars...) - implicits = append(implicits, d.Javadoc.argFiles...) + outDir := android.PathForModuleOut(ctx, "out") + srcJarDir := android.PathForModuleOut(ctx, "srcjars") + stubsDir := android.PathForModuleOut(ctx, "stubsDir") - var implicitOutputs android.WritablePaths - implicitOutputs = append(implicitOutputs, d.Javadoc.docZip) - for _, o := range d.Javadoc.properties.Out { - implicitOutputs = append(implicitOutputs, android.PathForModuleGen(ctx, o)) + rule := android.NewRuleBuilder() + + rule.Command().Text("rm -rf").Text(outDir.String()).Text(stubsDir.String()) + rule.Command().Text("mkdir -p").Text(outDir.String()).Text(stubsDir.String()) + + srcJarList := zipSyncCmd(ctx, rule, srcJarDir, d.Javadoc.srcJars) + + var cmd *android.RuleBuilderCommand + if Bool(d.properties.Dokka_enabled) { + cmd = dokkaCmd(ctx, rule, outDir, srcJarDir, deps.bootClasspath, deps.classpath) + } else { + cmd = javadocBootclasspathCmd(ctx, rule, d.Javadoc.srcFiles, outDir, srcJarDir, srcJarList, + deps.bootClasspath, deps.classpath, d.Javadoc.sourcepaths) } - flags, err := d.initBuilderFlags(ctx, &implicits, deps) - if err != nil { - return + d.stubsFlags(ctx, cmd, stubsDir) + + cmd.Flag(d.Javadoc.args).Implicits(d.Javadoc.argFiles) + + if d.properties.Compat_config != nil { + compatConfig := android.PathForModuleSrc(ctx, String(d.properties.Compat_config)) + cmd.FlagWithInput("-compatconfig ", compatConfig) } - flags.doclavaStubsFlags = d.collectStubsFlags(ctx, &implicitOutputs) + var desc string if Bool(d.properties.Dokka_enabled) { - d.transformDokka(ctx, implicits, flags.classpathArgs, d.Javadoc.args) + desc = "dokka" } else { - flags.doclavaDocsFlags = d.collectDoclavaDocsFlags(ctx, &implicits, jsilver, doclava) - flags.postDoclavaCmds = d.getPostDoclavaCmds(ctx, &implicits) - d.transformDoclava(ctx, implicits, implicitOutputs, flags.bootClasspathArgs, flags.classpathArgs, - flags.sourcepathArgs, flags.doclavaDocsFlags+flags.doclavaStubsFlags+" "+d.Javadoc.args, - flags.postDoclavaCmds) + d.doclavaDocsFlags(ctx, cmd, classpath{jsilver, doclava}) + + for _, o := range d.Javadoc.properties.Out { + cmd.ImplicitOutput(android.PathForModuleGen(ctx, o)) + } + + d.postDoclavaCmds(ctx, rule) + desc = "doclava" } - if apiCheckEnabled(d.properties.Check_api.Current, "current") && + rule.Command(). + BuiltTool(ctx, "soong_zip"). + Flag("-write_if_changed"). + Flag("-d"). + FlagWithOutput("-o ", d.docZip). + FlagWithArg("-C ", outDir.String()). + FlagWithArg("-D ", outDir.String()) + + rule.Command(). + BuiltTool(ctx, "soong_zip"). + Flag("-write_if_changed"). + Flag("-jar"). + FlagWithOutput("-o ", d.stubsSrcJar). + FlagWithArg("-C ", stubsDir.String()). + FlagWithArg("-D ", stubsDir.String()) + + rule.Restat() + + zipSyncCleanupCmd(rule, srcJarDir) + + rule.Build(pctx, ctx, "javadoc", desc) + + if apiCheckEnabled(ctx, d.properties.Check_api.Current, "current") && !ctx.Config().IsPdkBuild() { - apiFile := ctx.ExpandSource(String(d.properties.Check_api.Current.Api_file), - "check_api.current.api_file") - removedApiFile := ctx.ExpandSource(String(d.properties.Check_api.Current.Removed_api_file), - "check_api.current_removed_api_file") + + apiFile := android.PathForModuleSrc(ctx, String(d.properties.Check_api.Current.Api_file)) + removedApiFile := android.PathForModuleSrc(ctx, String(d.properties.Check_api.Current.Removed_api_file)) d.checkCurrentApiTimestamp = android.PathForModuleOut(ctx, "check_current_api.timestamp") - d.transformCheckApi(ctx, apiFile, removedApiFile, checkApiClasspath, - fmt.Sprintf(`\n******************************\n`+ - `You have tried to change the API from what has been previously approved.\n\n`+ - `To make these errors go away, you have two choices:\n`+ - ` 1. You can add '@hide' javadoc comments to the methods, etc. listed in the\n`+ - ` errors above.\n\n`+ - ` 2. You can update current.txt by executing the following command:\n`+ - ` make %s-update-current-api\n\n`+ - ` To submit the revised current.txt to the main Android repository,\n`+ - ` you will need approval.\n`+ - `******************************\n`, ctx.ModuleName()), String(d.properties.Check_api.Current.Args), - d.checkCurrentApiTimestamp) + + rule := android.NewRuleBuilder() + + rule.Command().Text("( true") + + rule.Command(). + BuiltTool(ctx, "apicheck"). + Flag("-JXmx1024m"). + FlagWithInputList("-Jclasspath\\ ", checkApiClasspath.Paths(), ":"). + OptionalFlag(d.properties.Check_api.Current.Args). + Input(apiFile). + Input(d.apiFile). + Input(removedApiFile). + Input(d.removedApiFile) + + msg := fmt.Sprintf(`\n******************************\n`+ + `You have tried to change the API from what has been previously approved.\n\n`+ + `To make these errors go away, you have two choices:\n`+ + ` 1. You can add '@hide' javadoc comments to the methods, etc. listed in the\n`+ + ` errors above.\n\n`+ + ` 2. You can update current.txt by executing the following command:\n`+ + ` make %s-update-current-api\n\n`+ + ` To submit the revised current.txt to the main Android repository,\n`+ + ` you will need approval.\n`+ + `******************************\n`, ctx.ModuleName()) + + rule.Command(). + Text("touch").Output(d.checkCurrentApiTimestamp). + Text(") || ("). + Text("echo").Flag("-e").Flag(`"` + msg + `"`). + Text("; exit 38"). + Text(")") + + rule.Build(pctx, ctx, "doclavaCurrentApiCheck", "check current API") d.updateCurrentApiTimestamp = android.PathForModuleOut(ctx, "update_current_api.timestamp") - transformUpdateApi(ctx, apiFile, removedApiFile, d.apiFile, d.removedApiFile, - d.updateCurrentApiTimestamp) + + // update API rule + rule = android.NewRuleBuilder() + + rule.Command().Text("( true") + + rule.Command(). + Text("cp").Flag("-f"). + Input(d.apiFile).Flag(apiFile.String()) + + rule.Command(). + Text("cp").Flag("-f"). + Input(d.removedApiFile).Flag(removedApiFile.String()) + + msg = "failed to update public API" + + rule.Command(). + Text("touch").Output(d.updateCurrentApiTimestamp). + Text(") || ("). + Text("echo").Flag("-e").Flag(`"` + msg + `"`). + Text("; exit 38"). + Text(")") + + rule.Build(pctx, ctx, "doclavaCurrentApiUpdate", "update current API") } - if apiCheckEnabled(d.properties.Check_api.Last_released, "last_released") && + if apiCheckEnabled(ctx, d.properties.Check_api.Last_released, "last_released") && !ctx.Config().IsPdkBuild() { - apiFile := ctx.ExpandSource(String(d.properties.Check_api.Last_released.Api_file), - "check_api.last_released.api_file") - removedApiFile := ctx.ExpandSource(String(d.properties.Check_api.Last_released.Removed_api_file), - "check_api.last_released.removed_api_file") + + apiFile := android.PathForModuleSrc(ctx, String(d.properties.Check_api.Last_released.Api_file)) + removedApiFile := android.PathForModuleSrc(ctx, String(d.properties.Check_api.Last_released.Removed_api_file)) d.checkLastReleasedApiTimestamp = android.PathForModuleOut(ctx, "check_last_released_api.timestamp") - d.transformCheckApi(ctx, apiFile, removedApiFile, checkApiClasspath, - `\n******************************\n`+ - `You have tried to change the API from what has been previously released in\n`+ - `an SDK. Please fix the errors listed above.\n`+ - `******************************\n`, String(d.properties.Check_api.Last_released.Args), - d.checkLastReleasedApiTimestamp) + + rule := android.NewRuleBuilder() + + rule.Command(). + Text("("). + BuiltTool(ctx, "apicheck"). + Flag("-JXmx1024m"). + FlagWithInputList("-Jclasspath\\ ", checkApiClasspath.Paths(), ":"). + OptionalFlag(d.properties.Check_api.Last_released.Args). + Input(apiFile). + Input(d.apiFile). + Input(removedApiFile). + Input(d.removedApiFile) + + msg := `\n******************************\n` + + `You have tried to change the API from what has been previously released in\n` + + `an SDK. Please fix the errors listed above.\n` + + `******************************\n` + + rule.Command(). + Text("touch").Output(d.checkLastReleasedApiTimestamp). + Text(") || ("). + Text("echo").Flag("-e").Flag(`"` + msg + `"`). + Text("; exit 38"). + Text(")") + + rule.Build(pctx, ctx, "doclavaLastApiCheck", "check last API") } } @@ -1231,24 +1172,22 @@ func (d *Droiddoc) GenerateAndroidBuildActions(ctx android.ModuleContext) { // type Droidstubs struct { Javadoc + android.SdkBase properties DroidstubsProperties apiFile android.WritablePath apiXmlFile android.WritablePath lastReleasedApiXmlFile android.WritablePath - dexApiFile android.WritablePath privateApiFile android.WritablePath - privateDexApiFile android.WritablePath removedApiFile android.WritablePath removedDexApiFile android.WritablePath - apiMappingFile android.WritablePath - exactApiFile android.WritablePath - proguardFile android.WritablePath nullabilityWarningsFile android.WritablePath checkCurrentApiTimestamp android.WritablePath updateCurrentApiTimestamp android.WritablePath checkLastReleasedApiTimestamp android.WritablePath + apiLintTimestamp android.WritablePath + apiLintReport android.WritablePath checkNullabilityWarningsTimestamp android.WritablePath @@ -1264,6 +1203,9 @@ type Droidstubs struct { metadataDir android.WritablePath } +// droidstubs passes sources files through Metalava to generate stub .java files that only contain the API to be +// documented, filtering out hidden classes and methods. The resulting .java files are intended to be passed to +// a droiddoc module to generate documentation. func DroidstubsFactory() android.Module { module := &Droidstubs{} @@ -1271,9 +1213,14 @@ func DroidstubsFactory() android.Module { &module.Javadoc.properties) InitDroiddocModule(module, android.HostAndDeviceSupported) + android.InitSdkAwareModule(module) return module } +// droidstubs_host passes sources files through Metalava to generate stub .java files that only contain the API +// to be documented, filtering out hidden classes and methods. The resulting .java files are intended to be +// passed to a droiddoc_host module to generate documentation. Use a droidstubs_host instead of a droidstubs +// module when symbols needed by the source files are provided by java_library_host modules. func DroidstubsHostFactory() android.Module { module := &Droidstubs{} @@ -1284,15 +1231,48 @@ func DroidstubsHostFactory() android.Module { return module } +func (d *Droidstubs) OutputFiles(tag string) (android.Paths, error) { + switch tag { + case "": + return android.Paths{d.stubsSrcJar}, nil + case ".docs.zip": + return android.Paths{d.docZip}, nil + case ".annotations.zip": + return android.Paths{d.annotationsZip}, nil + case ".api_versions.xml": + return android.Paths{d.apiVersionsXml}, nil + default: + return nil, fmt.Errorf("unsupported module reference tag %q", tag) + } +} + func (d *Droidstubs) ApiFilePath() android.Path { return d.apiFilePath } +func (d *Droidstubs) RemovedApiFilePath() android.Path { + return d.removedApiFile +} + +func (d *Droidstubs) StubsSrcJar() android.Path { + return d.stubsSrcJar +} + func (d *Droidstubs) DepsMutator(ctx android.BottomUpMutatorContext) { d.Javadoc.addDeps(ctx) + // If requested clear any properties that provide information about the latest version + // of an API and which reference non-existent modules. if Bool(d.properties.Check_api.Ignore_missing_latest_api) { ignoreMissingModules(ctx, &d.properties.Check_api.Last_released) + + // If the new_since references a module, e.g. :module-latest-api and the module + // does not exist then clear it. + newSinceSrc := d.properties.Check_api.Api_lint.New_since + newSinceSrcModule := android.SrcIsModule(proptools.String(newSinceSrc)) + if newSinceSrcModule != "" && !ctx.OtherModuleExists(newSinceSrcModule) { + d.properties.Check_api.Api_lint.New_since = nil + } } if len(d.properties.Merge_annotations_dirs) != 0 { @@ -1314,333 +1294,225 @@ func (d *Droidstubs) DepsMutator(ctx android.BottomUpMutatorContext) { } } -func (d *Droidstubs) initBuilderFlags(ctx android.ModuleContext, implicits *android.Paths, - deps deps) (droiddocBuilderFlags, error) { - var flags droiddocBuilderFlags - - *implicits = append(*implicits, deps.bootClasspath...) - *implicits = append(*implicits, deps.classpath...) - - // continue to use -bootclasspath even if Metalava under -source 1.9 is enabled - // since it doesn't support system modules yet. - if len(deps.bootClasspath.Strings()) > 0 { - // For OpenJDK 8 we can use -bootclasspath to define the core libraries code. - flags.bootClasspathArgs = deps.bootClasspath.FormJavaClassPath("-bootclasspath") - } - flags.classpathArgs = deps.classpath.FormJavaClassPath("-classpath") - - flags.sourcepathArgs = "-sourcepath \"" + strings.Join(d.Javadoc.sourcepaths.Strings(), ":") + "\"" - return flags, nil -} - -func (d *Droidstubs) collectStubsFlags(ctx android.ModuleContext, - implicitOutputs *android.WritablePaths) string { - var metalavaFlags string - if apiCheckEnabled(d.properties.Check_api.Current, "current") || - apiCheckEnabled(d.properties.Check_api.Last_released, "last_released") || +func (d *Droidstubs) stubsFlags(ctx android.ModuleContext, cmd *android.RuleBuilderCommand, stubsDir android.OptionalPath) { + if apiCheckEnabled(ctx, d.properties.Check_api.Current, "current") || + apiCheckEnabled(ctx, d.properties.Check_api.Last_released, "last_released") || String(d.properties.Api_filename) != "" { d.apiFile = android.PathForModuleOut(ctx, ctx.ModuleName()+"_api.txt") - metalavaFlags = metalavaFlags + " --api " + d.apiFile.String() - *implicitOutputs = append(*implicitOutputs, d.apiFile) + cmd.FlagWithOutput("--api ", d.apiFile) d.apiFilePath = d.apiFile } - if apiCheckEnabled(d.properties.Check_api.Current, "current") || - apiCheckEnabled(d.properties.Check_api.Last_released, "last_released") || + if apiCheckEnabled(ctx, d.properties.Check_api.Current, "current") || + apiCheckEnabled(ctx, d.properties.Check_api.Last_released, "last_released") || String(d.properties.Removed_api_filename) != "" { d.removedApiFile = android.PathForModuleOut(ctx, ctx.ModuleName()+"_removed.txt") - metalavaFlags = metalavaFlags + " --removed-api " + d.removedApiFile.String() - *implicitOutputs = append(*implicitOutputs, d.removedApiFile) - } - - if String(d.properties.Private_api_filename) != "" { - d.privateApiFile = android.PathForModuleOut(ctx, String(d.properties.Private_api_filename)) - metalavaFlags = metalavaFlags + " --private-api " + d.privateApiFile.String() - *implicitOutputs = append(*implicitOutputs, d.privateApiFile) - } - - if String(d.properties.Dex_api_filename) != "" { - d.dexApiFile = android.PathForModuleOut(ctx, String(d.properties.Dex_api_filename)) - metalavaFlags += " --dex-api " + d.dexApiFile.String() - *implicitOutputs = append(*implicitOutputs, d.dexApiFile) - } - - if String(d.properties.Private_dex_api_filename) != "" { - d.privateDexApiFile = android.PathForModuleOut(ctx, String(d.properties.Private_dex_api_filename)) - metalavaFlags = metalavaFlags + " --private-dex-api " + d.privateDexApiFile.String() - *implicitOutputs = append(*implicitOutputs, d.privateDexApiFile) + cmd.FlagWithOutput("--removed-api ", d.removedApiFile) } if String(d.properties.Removed_dex_api_filename) != "" { d.removedDexApiFile = android.PathForModuleOut(ctx, String(d.properties.Removed_dex_api_filename)) - metalavaFlags = metalavaFlags + " --removed-dex-api " + d.removedDexApiFile.String() - *implicitOutputs = append(*implicitOutputs, d.removedDexApiFile) - } - - if String(d.properties.Exact_api_filename) != "" { - d.exactApiFile = android.PathForModuleOut(ctx, String(d.properties.Exact_api_filename)) - metalavaFlags = metalavaFlags + " --exact-api " + d.exactApiFile.String() - *implicitOutputs = append(*implicitOutputs, d.exactApiFile) - } - - if String(d.properties.Dex_mapping_filename) != "" { - d.apiMappingFile = android.PathForModuleOut(ctx, String(d.properties.Dex_mapping_filename)) - metalavaFlags = metalavaFlags + " --dex-api-mapping " + d.apiMappingFile.String() - *implicitOutputs = append(*implicitOutputs, d.apiMappingFile) - } - - if String(d.properties.Proguard_filename) != "" { - d.proguardFile = android.PathForModuleOut(ctx, String(d.properties.Proguard_filename)) - metalavaFlags += " --proguard " + d.proguardFile.String() - *implicitOutputs = append(*implicitOutputs, d.proguardFile) + cmd.FlagWithOutput("--removed-dex-api ", d.removedDexApiFile) } if Bool(d.properties.Write_sdk_values) { d.metadataDir = android.PathForModuleOut(ctx, "metadata") - metalavaFlags = metalavaFlags + " --sdk-values " + d.metadataDir.String() + cmd.FlagWithArg("--sdk-values ", d.metadataDir.String()) } - if Bool(d.properties.Create_doc_stubs) { - metalavaFlags += " --doc-stubs " + android.PathForModuleOut(ctx, "stubsDir").String() - } else { - metalavaFlags += " --stubs " + android.PathForModuleOut(ctx, "stubsDir").String() + if stubsDir.Valid() { + if Bool(d.properties.Create_doc_stubs) { + cmd.FlagWithArg("--doc-stubs ", stubsDir.String()) + } else { + cmd.FlagWithArg("--stubs ", stubsDir.String()) + cmd.Flag("--exclude-documentation-from-stubs") + } } - return metalavaFlags } -func (d *Droidstubs) collectAnnotationsFlags(ctx android.ModuleContext, - implicits *android.Paths, implicitOutputs *android.WritablePaths) (string, string) { - var flags, mergeAnnoDirFlags string +func (d *Droidstubs) annotationsFlags(ctx android.ModuleContext, cmd *android.RuleBuilderCommand) { if Bool(d.properties.Annotations_enabled) { - flags += " --include-annotations" + cmd.Flag("--include-annotations") + validatingNullability := strings.Contains(d.Javadoc.args, "--validate-nullability-from-merged-stubs") || String(d.properties.Validate_nullability_from_list) != "" + migratingNullability := String(d.properties.Previous_api) != "" - if !(migratingNullability || validatingNullability) { - ctx.PropertyErrorf("previous_api", - "has to be non-empty if annotations was enabled (unless validating nullability)") - } if migratingNullability { previousApi := android.PathForModuleSrc(ctx, String(d.properties.Previous_api)) - *implicits = append(*implicits, previousApi) - flags += " --migrate-nullness " + previousApi.String() + cmd.FlagWithInput("--migrate-nullness ", previousApi) } + if s := String(d.properties.Validate_nullability_from_list); s != "" { - flags += " --validate-nullability-from-list " + android.PathForModuleSrc(ctx, s).String() + cmd.FlagWithInput("--validate-nullability-from-list ", android.PathForModuleSrc(ctx, s)) } + if validatingNullability { d.nullabilityWarningsFile = android.PathForModuleOut(ctx, ctx.ModuleName()+"_nullability_warnings.txt") - *implicitOutputs = append(*implicitOutputs, d.nullabilityWarningsFile) - flags += " --nullability-warnings-txt " + d.nullabilityWarningsFile.String() + cmd.FlagWithOutput("--nullability-warnings-txt ", d.nullabilityWarningsFile) } d.annotationsZip = android.PathForModuleOut(ctx, ctx.ModuleName()+"_annotations.zip") - *implicitOutputs = append(*implicitOutputs, d.annotationsZip) - - flags += " --extract-annotations " + d.annotationsZip.String() + cmd.FlagWithOutput("--extract-annotations ", d.annotationsZip) - if len(d.properties.Merge_annotations_dirs) == 0 { - ctx.PropertyErrorf("merge_annotations_dirs", - "has to be non-empty if annotations was enabled!") + if len(d.properties.Merge_annotations_dirs) != 0 { + d.mergeAnnoDirFlags(ctx, cmd) } - ctx.VisitDirectDepsWithTag(metalavaMergeAnnotationsDirTag, func(m android.Module) { - if t, ok := m.(*ExportedDroiddocDir); ok { - *implicits = append(*implicits, t.deps...) - mergeAnnoDirFlags += " --merge-qualifier-annotations " + t.dir.String() - } else { - ctx.PropertyErrorf("merge_annotations_dirs", - "module %q is not a metalava merge-annotations dir", ctx.OtherModuleName(m)) - } - }) - flags += mergeAnnoDirFlags + // TODO(tnorbye): find owners to fix these warnings when annotation was enabled. - flags += " --hide HiddenTypedefConstant --hide SuperfluousPrefix --hide AnnotationExtraction" + cmd.FlagWithArg("--hide ", "HiddenTypedefConstant"). + FlagWithArg("--hide ", "SuperfluousPrefix"). + FlagWithArg("--hide ", "AnnotationExtraction") } +} - return flags, mergeAnnoDirFlags +func (d *Droidstubs) mergeAnnoDirFlags(ctx android.ModuleContext, cmd *android.RuleBuilderCommand) { + ctx.VisitDirectDepsWithTag(metalavaMergeAnnotationsDirTag, func(m android.Module) { + if t, ok := m.(*ExportedDroiddocDir); ok { + cmd.FlagWithArg("--merge-qualifier-annotations ", t.dir.String()).Implicits(t.deps) + } else { + ctx.PropertyErrorf("merge_annotations_dirs", + "module %q is not a metalava merge-annotations dir", ctx.OtherModuleName(m)) + } + }) } -func (d *Droidstubs) collectInclusionAnnotationsFlags(ctx android.ModuleContext, - implicits *android.Paths, implicitOutputs *android.WritablePaths) string { - var flags string +func (d *Droidstubs) inclusionAnnotationsFlags(ctx android.ModuleContext, cmd *android.RuleBuilderCommand) { ctx.VisitDirectDepsWithTag(metalavaMergeInclusionAnnotationsDirTag, func(m android.Module) { if t, ok := m.(*ExportedDroiddocDir); ok { - *implicits = append(*implicits, t.deps...) - flags += " --merge-inclusion-annotations " + t.dir.String() + cmd.FlagWithArg("--merge-inclusion-annotations ", t.dir.String()).Implicits(t.deps) } else { ctx.PropertyErrorf("merge_inclusion_annotations_dirs", "module %q is not a metalava merge-annotations dir", ctx.OtherModuleName(m)) } }) - - return flags } -func (d *Droidstubs) collectAPILevelsAnnotationsFlags(ctx android.ModuleContext, - implicits *android.Paths, implicitOutputs *android.WritablePaths) string { - var flags string - if Bool(d.properties.Api_levels_annotations_enabled) { - d.apiVersionsXml = android.PathForModuleOut(ctx, "api-versions.xml") - *implicitOutputs = append(*implicitOutputs, d.apiVersionsXml) +func (d *Droidstubs) apiLevelsAnnotationsFlags(ctx android.ModuleContext, cmd *android.RuleBuilderCommand) { + if !Bool(d.properties.Api_levels_annotations_enabled) { + return + } - if len(d.properties.Api_levels_annotations_dirs) == 0 { - ctx.PropertyErrorf("api_levels_annotations_dirs", - "has to be non-empty if api levels annotations was enabled!") - } - - flags = " --generate-api-levels " + d.apiVersionsXml.String() + " --apply-api-levels " + - d.apiVersionsXml.String() + " --current-version " + ctx.Config().PlatformSdkVersion() + - " --current-codename " + ctx.Config().PlatformSdkCodename() + " " - - ctx.VisitDirectDepsWithTag(metalavaAPILevelsAnnotationsDirTag, func(m android.Module) { - if t, ok := m.(*ExportedDroiddocDir); ok { - var androidJars android.Paths - for _, dep := range t.deps { - if strings.HasSuffix(dep.String(), "android.jar") { - androidJars = append(androidJars, dep) - } - } - *implicits = append(*implicits, androidJars...) - flags += " --android-jar-pattern " + t.dir.String() + "/%/public/android.jar " - } else { - ctx.PropertyErrorf("api_levels_annotations_dirs", - "module %q is not a metalava api-levels-annotations dir", ctx.OtherModuleName(m)) - } - }) + d.apiVersionsXml = android.PathForModuleOut(ctx, "api-versions.xml") + if len(d.properties.Api_levels_annotations_dirs) == 0 { + ctx.PropertyErrorf("api_levels_annotations_dirs", + "has to be non-empty if api levels annotations was enabled!") } - return flags + cmd.FlagWithOutput("--generate-api-levels ", d.apiVersionsXml) + cmd.FlagWithInput("--apply-api-levels ", d.apiVersionsXml) + cmd.FlagWithArg("--current-version ", ctx.Config().PlatformSdkVersion()) + cmd.FlagWithArg("--current-codename ", ctx.Config().PlatformSdkCodename()) + + filename := proptools.StringDefault(d.properties.Api_levels_jar_filename, "android.jar") + + ctx.VisitDirectDepsWithTag(metalavaAPILevelsAnnotationsDirTag, func(m android.Module) { + if t, ok := m.(*ExportedDroiddocDir); ok { + for _, dep := range t.deps { + if strings.HasSuffix(dep.String(), filename) { + cmd.Implicit(dep) + } + } + cmd.FlagWithArg("--android-jar-pattern ", t.dir.String()+"/%/public/"+filename) + } else { + ctx.PropertyErrorf("api_levels_annotations_dirs", + "module %q is not a metalava api-levels-annotations dir", ctx.OtherModuleName(m)) + } + }) } -func (d *Droidstubs) collectApiToXmlFlags(ctx android.ModuleContext, implicits *android.Paths, - implicitOutputs *android.WritablePaths) string { - var flags string - if Bool(d.properties.Jdiff_enabled) && !ctx.Config().IsPdkBuild() { +func (d *Droidstubs) apiToXmlFlags(ctx android.ModuleContext, cmd *android.RuleBuilderCommand) { + if Bool(d.properties.Jdiff_enabled) && !ctx.Config().IsPdkBuild() && d.apiFile != nil { if d.apiFile.String() == "" { ctx.ModuleErrorf("API signature file has to be specified in Metalava when jdiff is enabled.") } d.apiXmlFile = android.PathForModuleOut(ctx, ctx.ModuleName()+"_api.xml") - *implicitOutputs = append(*implicitOutputs, d.apiXmlFile) - - flags = " --api-xml " + d.apiXmlFile.String() + cmd.FlagWithOutput("--api-xml ", d.apiXmlFile) if String(d.properties.Check_api.Last_released.Api_file) == "" { ctx.PropertyErrorf("check_api.last_released.api_file", "has to be non-empty if jdiff was enabled!") } - lastReleasedApi := ctx.ExpandSource(String(d.properties.Check_api.Last_released.Api_file), - "check_api.last_released.api_file") - *implicits = append(*implicits, lastReleasedApi) + lastReleasedApi := android.PathForModuleSrc(ctx, String(d.properties.Check_api.Last_released.Api_file)) d.lastReleasedApiXmlFile = android.PathForModuleOut(ctx, ctx.ModuleName()+"_last_released_api.xml") - *implicitOutputs = append(*implicitOutputs, d.lastReleasedApiXmlFile) + cmd.FlagWithInput("--convert-to-jdiff ", lastReleasedApi).Output(d.lastReleasedApiXmlFile) + } +} - flags += " --convert-to-jdiff " + lastReleasedApi.String() + " " + - d.lastReleasedApiXmlFile.String() +func metalavaCmd(ctx android.ModuleContext, rule *android.RuleBuilder, javaVersion javaVersion, srcs android.Paths, + srcJarList android.Path, bootclasspath, classpath classpath, sourcepaths android.Paths, implicitsRsp android.WritablePath, sandbox bool) *android.RuleBuilderCommand { + // Metalava uses lots of memory, restrict the number of metalava jobs that can run in parallel. + rule.HighMem() + cmd := rule.Command() + if ctx.Config().IsEnvTrue("RBE_METALAVA") { + rule.Remoteable(android.RemoteRuleSupports{RBE: true}) + pool := ctx.Config().GetenvWithDefault("RBE_METALAVA_POOL", "metalava") + execStrategy := ctx.Config().GetenvWithDefault("RBE_METALAVA_EXEC_STRATEGY", remoteexec.LocalExecStrategy) + labels := map[string]string{"type": "compile", "lang": "java", "compiler": "metalava"} + if !sandbox { + execStrategy = remoteexec.LocalExecStrategy + labels["shallow"] = "true" + } + inputs := []string{android.PathForOutput(ctx, "host", ctx.Config().PrebuiltOS(), "framework", "metalava.jar").String()} + inputs = append(inputs, sourcepaths.Strings()...) + if v := ctx.Config().Getenv("RBE_METALAVA_INPUTS"); v != "" { + inputs = append(inputs, strings.Split(v, ",")...) + } + cmd.Text((&remoteexec.REParams{ + Labels: labels, + ExecStrategy: execStrategy, + Inputs: inputs, + RSPFile: implicitsRsp.String(), + ToolchainInputs: []string{config.JavaCmd(ctx).String()}, + Platform: map[string]string{remoteexec.PoolKey: pool}, + }).NoVarTemplate(ctx.Config())) } - return flags -} + cmd.BuiltTool(ctx, "metalava"). + Flag(config.JavacVmFlags). + FlagWithArg("-encoding ", "UTF-8"). + FlagWithArg("-source ", javaVersion.String()). + FlagWithRspFileInputList("@", srcs). + FlagWithInput("@", srcJarList) -func (d *Droidstubs) transformMetalava(ctx android.ModuleContext, implicits android.Paths, - implicitOutputs android.WritablePaths, javaVersion, - bootclasspathArgs, classpathArgs, sourcepathArgs, opts string) { + if javaHome := ctx.Config().Getenv("ANDROID_JAVA_HOME"); javaHome != "" { + cmd.Implicit(android.PathForSource(ctx, javaHome)) + } - var writeSdkValues, metadataZip, metadataDir string - if Bool(d.properties.Write_sdk_values) { - writeSdkValues = "true" - d.metadataZip = android.PathForModuleOut(ctx, ctx.ModuleName()+"-metadata.zip") - metadataZip = d.metadataZip.String() - metadataDir = d.metadataDir.String() - implicitOutputs = append(implicitOutputs, d.metadataZip) + if sandbox { + cmd.FlagWithOutput("--strict-input-files ", android.PathForModuleOut(ctx, ctx.ModuleName()+"-"+"violations.txt")) } else { - writeSdkValues = "false" - metadataZip = "" - metadataDir = "" - } - - ctx.Build(pctx, android.BuildParams{ - Rule: metalava, - Description: "Metalava", - Output: d.Javadoc.stubsSrcJar, - Inputs: d.Javadoc.srcFiles, - Implicits: implicits, - ImplicitOutputs: implicitOutputs, - Args: map[string]string{ - "outDir": android.PathForModuleOut(ctx, "out").String(), - "srcJarDir": android.PathForModuleOut(ctx, "srcjars").String(), - "stubsDir": android.PathForModuleOut(ctx, "stubsDir").String(), - "srcJars": strings.Join(d.Javadoc.srcJars.Strings(), " "), - "javaVersion": javaVersion, - "bootclasspathArgs": bootclasspathArgs, - "classpathArgs": classpathArgs, - "sourcepathArgs": sourcepathArgs, - "opts": opts, - "writeSdkValues": writeSdkValues, - "metadataZip": metadataZip, - "metadataDir": metadataDir, - }, - }) -} + cmd.FlagWithOutput("--strict-input-files:warn ", android.PathForModuleOut(ctx, ctx.ModuleName()+"-"+"violations.txt")) + } -func (d *Droidstubs) transformCheckApi(ctx android.ModuleContext, - apiFile, removedApiFile android.Path, baselineFile android.OptionalPath, updatedBaselineOut android.WritablePath, implicits android.Paths, - javaVersion, bootclasspathArgs, classpathArgs, sourcepathArgs, opts, subdir, msg string, - output android.WritablePath) { - - implicits = append(android.Paths{apiFile, removedApiFile, d.apiFile, d.removedApiFile}, implicits...) - var implicitOutputs android.WritablePaths - - if baselineFile.Valid() { - implicits = append(implicits, baselineFile.Path()) - implicitOutputs = append(implicitOutputs, updatedBaselineOut) - } - - ctx.Build(pctx, android.BuildParams{ - Rule: metalavaApiCheck, - Description: "Metalava Check API", - Output: output, - Inputs: d.Javadoc.srcFiles, - Implicits: implicits, - ImplicitOutputs: implicitOutputs, - Args: map[string]string{ - "srcJarDir": android.PathForModuleOut(ctx, subdir, "srcjars").String(), - "srcJars": strings.Join(d.Javadoc.srcJars.Strings(), " "), - "javaVersion": javaVersion, - "bootclasspathArgs": bootclasspathArgs, - "classpathArgs": classpathArgs, - "sourcepathArgs": sourcepathArgs, - "opts": opts, - "msg": msg, - }, - }) -} + if implicitsRsp != nil { + cmd.FlagWithArg("--strict-input-files-exempt ", "@"+implicitsRsp.String()) + } -func (d *Droidstubs) transformJdiff(ctx android.ModuleContext, implicits android.Paths, - implicitOutputs android.WritablePaths, - bootclasspathArgs, classpathArgs, sourcepathArgs, opts string) { - ctx.Build(pctx, android.BuildParams{ - Rule: javadoc, - Description: "Jdiff", - Output: d.jdiffStubsSrcJar, - Inputs: d.Javadoc.srcFiles, - Implicits: implicits, - ImplicitOutputs: implicitOutputs, - Args: map[string]string{ - "outDir": android.PathForModuleOut(ctx, "jdiff-out").String(), - "srcJarDir": android.PathForModuleOut(ctx, "jdiff-srcjars").String(), - "stubsDir": android.PathForModuleOut(ctx, "jdiff-stubsDir").String(), - "srcJars": strings.Join(d.Javadoc.srcJars.Strings(), " "), - "opts": opts, - "bootclasspathArgs": bootclasspathArgs, - "classpathArgs": classpathArgs, - "sourcepathArgs": sourcepathArgs, - "docZip": d.jdiffDocZip.String(), - }, - }) + if len(bootclasspath) > 0 { + cmd.FlagWithInputList("-bootclasspath ", bootclasspath.Paths(), ":") + } + + if len(classpath) > 0 { + cmd.FlagWithInputList("-classpath ", classpath.Paths(), ":") + } + + if len(sourcepaths) > 0 { + cmd.FlagWithList("-sourcepath ", sourcepaths.Strings(), ":") + } else { + cmd.FlagWithArg("-sourcepath ", `""`) + } + + cmd.Flag("--no-banner"). + Flag("--color"). + Flag("--quiet"). + Flag("--format=v2") + + return cmd } func (d *Droidstubs) GenerateAndroidBuildActions(ctx android.ModuleContext) { @@ -1648,29 +1520,36 @@ func (d *Droidstubs) GenerateAndroidBuildActions(ctx android.ModuleContext) { javaVersion := getJavaVersion(ctx, String(d.Javadoc.properties.Java_version), sdkContext(d)) - var implicits android.Paths - implicits = append(implicits, d.Javadoc.srcJars...) - implicits = append(implicits, d.Javadoc.argFiles...) + // Create rule for metalava - var implicitOutputs android.WritablePaths - for _, o := range d.Javadoc.properties.Out { - implicitOutputs = append(implicitOutputs, android.PathForModuleGen(ctx, o)) - } + srcJarDir := android.PathForModuleOut(ctx, "srcjars") - flags, err := d.initBuilderFlags(ctx, &implicits, deps) - metalavaCheckApiImplicits := implicits - jdiffImplicits := implicits + rule := android.NewRuleBuilder() - if err != nil { - return + generateStubs := BoolDefault(d.properties.Generate_stubs, true) + var stubsDir android.OptionalPath + if generateStubs { + d.Javadoc.stubsSrcJar = android.PathForModuleOut(ctx, ctx.ModuleName()+"-"+"stubs.srcjar") + stubsDir = android.OptionalPathForPath(android.PathForModuleOut(ctx, "stubsDir")) + rule.Command().Text("rm -rf").Text(stubsDir.String()) + rule.Command().Text("mkdir -p").Text(stubsDir.String()) } - flags.metalavaStubsFlags = d.collectStubsFlags(ctx, &implicitOutputs) - flags.metalavaAnnotationsFlags, flags.metalavaMergeAnnoDirFlags = - d.collectAnnotationsFlags(ctx, &implicits, &implicitOutputs) - flags.metalavaInclusionAnnotationsFlags = d.collectInclusionAnnotationsFlags(ctx, &implicits, &implicitOutputs) - flags.metalavaApiLevelsAnnotationsFlags = d.collectAPILevelsAnnotationsFlags(ctx, &implicits, &implicitOutputs) - flags.metalavaApiToXmlFlags = d.collectApiToXmlFlags(ctx, &implicits, &implicitOutputs) + srcJarList := zipSyncCmd(ctx, rule, srcJarDir, d.Javadoc.srcJars) + + implicitsRsp := android.PathForModuleOut(ctx, ctx.ModuleName()+"-"+"implicits.rsp") + + cmd := metalavaCmd(ctx, rule, javaVersion, d.Javadoc.srcFiles, srcJarList, + deps.bootClasspath, deps.classpath, d.Javadoc.sourcepaths, implicitsRsp, + Bool(d.Javadoc.properties.Sandbox)) + cmd.Implicits(d.Javadoc.implicits) + + d.stubsFlags(ctx, cmd, stubsDir) + + d.annotationsFlags(ctx, cmd) + d.inclusionAnnotationsFlags(ctx, cmd) + d.apiLevelsAnnotationsFlags(ctx, cmd) + d.apiToXmlFlags(ctx, cmd) if strings.Contains(d.Javadoc.args, "--generate-documentation") { // Currently Metalava have the ability to invoke Javadoc in a seperate process. @@ -1678,73 +1557,243 @@ func (d *Droidstubs) GenerateAndroidBuildActions(ctx android.ModuleContext) { // "--generate-documentation" arg. This is not needed when Metalava removes this feature. d.Javadoc.args = d.Javadoc.args + " -nodocs " } - d.transformMetalava(ctx, implicits, implicitOutputs, javaVersion, - flags.bootClasspathArgs, flags.classpathArgs, flags.sourcepathArgs, - flags.metalavaStubsFlags+flags.metalavaAnnotationsFlags+flags.metalavaInclusionAnnotationsFlags+ - flags.metalavaApiLevelsAnnotationsFlags+flags.metalavaApiToXmlFlags+" "+d.Javadoc.args) - if apiCheckEnabled(d.properties.Check_api.Current, "current") && + cmd.Flag(d.Javadoc.args).Implicits(d.Javadoc.argFiles) + for _, o := range d.Javadoc.properties.Out { + cmd.ImplicitOutput(android.PathForModuleGen(ctx, o)) + } + + // Add options for the other optional tasks: API-lint and check-released. + // We generate separate timestamp files for them. + + doApiLint := false + doCheckReleased := false + + // Add API lint options. + + if BoolDefault(d.properties.Check_api.Api_lint.Enabled, false) && !ctx.Config().IsPdkBuild() { + doApiLint = true + + newSince := android.OptionalPathForModuleSrc(ctx, d.properties.Check_api.Api_lint.New_since) + if newSince.Valid() { + cmd.FlagWithInput("--api-lint ", newSince.Path()) + } else { + cmd.Flag("--api-lint") + } + d.apiLintReport = android.PathForModuleOut(ctx, "api_lint_report.txt") + cmd.FlagWithOutput("--report-even-if-suppressed ", d.apiLintReport) // TODO: Change to ":api-lint" + + // TODO(b/154317059): Clean up this whitelist by baselining and/or checking in last-released. + if d.Name() != "android.car-system-stubs-docs" && + d.Name() != "android.car-stubs-docs" && + d.Name() != "system-api-stubs-docs" && + d.Name() != "test-api-stubs-docs" { + cmd.Flag("--lints-as-errors") + cmd.Flag("--warnings-as-errors") // Most lints are actually warnings. + } + + baselineFile := android.OptionalPathForModuleSrc(ctx, d.properties.Check_api.Api_lint.Baseline_file) + updatedBaselineOutput := android.PathForModuleOut(ctx, "api_lint_baseline.txt") + d.apiLintTimestamp = android.PathForModuleOut(ctx, "api_lint.timestamp") + + // Note this string includes a special shell quote $' ... ', which decodes the "\n"s. + // However, because $' ... ' doesn't expand environmental variables, we can't just embed + // $PWD, so we have to terminate $'...', use "$PWD", then start $' ... ' again, + // which is why we have '"$PWD"$' in it. + // + // TODO: metalava also has a slightly different message hardcoded. Should we unify this + // message and metalava's one? + msg := `$'` + // Enclose with $' ... ' + `************************************************************\n` + + `Your API changes are triggering API Lint warnings or errors.\n` + + `To make these errors go away, fix the code according to the\n` + + `error and/or warning messages above.\n` + + `\n` + + `If it is not possible to do so, there are workarounds:\n` + + `\n` + + `1. You can suppress the errors with @SuppressLint("<id>")\n` + + if baselineFile.Valid() { + cmd.FlagWithInput("--baseline:api-lint ", baselineFile.Path()) + cmd.FlagWithOutput("--update-baseline:api-lint ", updatedBaselineOutput) + + msg += fmt.Sprintf(``+ + `2. You can update the baseline by executing the following\n`+ + ` command:\n`+ + ` cp \\\n`+ + ` "'"$PWD"$'/%s" \\\n`+ + ` "'"$PWD"$'/%s"\n`+ + ` To submit the revised baseline.txt to the main Android\n`+ + ` repository, you will need approval.\n`, updatedBaselineOutput, baselineFile.Path()) + } else { + msg += fmt.Sprintf(``+ + `2. You can add a baseline file of existing lint failures\n`+ + ` to the build rule of %s.\n`, d.Name()) + } + // Note the message ends with a ' (single quote), to close the $' ... ' . + msg += `************************************************************\n'` + + cmd.FlagWithArg("--error-message:api-lint ", msg) + } + + // Add "check released" options. (Detect incompatible API changes from the last public release) + + if apiCheckEnabled(ctx, d.properties.Check_api.Last_released, "last_released") && !ctx.Config().IsPdkBuild() { - apiFile := ctx.ExpandSource(String(d.properties.Check_api.Current.Api_file), - "check_api.current.api_file") - removedApiFile := ctx.ExpandSource(String(d.properties.Check_api.Current.Removed_api_file), - "check_api.current_removed_api_file") - baselineFile := ctx.ExpandOptionalSource(d.properties.Check_api.Current.Baseline_file, - "check_api.current.baseline_file") + doCheckReleased = true + + if len(d.Javadoc.properties.Out) > 0 { + ctx.PropertyErrorf("out", "out property may not be combined with check_api") + } + + apiFile := android.PathForModuleSrc(ctx, String(d.properties.Check_api.Last_released.Api_file)) + removedApiFile := android.PathForModuleSrc(ctx, String(d.properties.Check_api.Last_released.Removed_api_file)) + baselineFile := android.OptionalPathForModuleSrc(ctx, d.properties.Check_api.Last_released.Baseline_file) + updatedBaselineOutput := android.PathForModuleOut(ctx, "last_released_baseline.txt") + + d.checkLastReleasedApiTimestamp = android.PathForModuleOut(ctx, "check_last_released_api.timestamp") + + cmd.FlagWithInput("--check-compatibility:api:released ", apiFile) + cmd.FlagWithInput("--check-compatibility:removed:released ", removedApiFile) - d.checkCurrentApiTimestamp = android.PathForModuleOut(ctx, "check_current_api.timestamp") - opts := " " + d.Javadoc.args + " --check-compatibility:api:current " + apiFile.String() + - " --check-compatibility:removed:current " + removedApiFile.String() + - flags.metalavaInclusionAnnotationsFlags + flags.metalavaMergeAnnoDirFlags + " " - baselineOut := android.PathForModuleOut(ctx, "current_baseline.txt") if baselineFile.Valid() { - opts = opts + "--baseline " + baselineFile.String() + " --update-baseline " + baselineOut.String() + " " + cmd.FlagWithInput("--baseline:compatibility:released ", baselineFile.Path()) + cmd.FlagWithOutput("--update-baseline:compatibility:released ", updatedBaselineOutput) } - d.transformCheckApi(ctx, apiFile, removedApiFile, baselineFile, baselineOut, metalavaCheckApiImplicits, - javaVersion, flags.bootClasspathArgs, flags.classpathArgs, flags.sourcepathArgs, opts, "current-apicheck", - fmt.Sprintf(`\n******************************\n`+ - `You have tried to change the API from what has been previously approved.\n\n`+ - `To make these errors go away, you have two choices:\n`+ - ` 1. You can add '@hide' javadoc comments to the methods, etc. listed in the\n`+ - ` errors above.\n\n`+ - ` 2. You can update current.txt by executing the following command:\n`+ - ` make %s-update-current-api\n\n`+ - ` To submit the revised current.txt to the main Android repository,\n`+ - ` you will need approval.\n`+ - `******************************\n`, ctx.ModuleName()), - d.checkCurrentApiTimestamp) + // Note this string includes quote ($' ... '), which decodes the "\n"s. + msg := `$'\n******************************\n` + + `You have tried to change the API from what has been previously released in\n` + + `an SDK. Please fix the errors listed above.\n` + + `******************************\n'` - d.updateCurrentApiTimestamp = android.PathForModuleOut(ctx, "update_current_api.timestamp") - transformUpdateApi(ctx, apiFile, removedApiFile, d.apiFile, d.removedApiFile, - d.updateCurrentApiTimestamp) + cmd.FlagWithArg("--error-message:compatibility:released ", msg) + } + + impRule := android.NewRuleBuilder() + impCmd := impRule.Command() + // A dummy action that copies the ninja generated rsp file to a new location. This allows us to + // add a large number of inputs to a file without exceeding bash command length limits (which + // would happen if we use the WriteFile rule). The cp is needed because RuleBuilder sets the + // rsp file to be ${output}.rsp. + impCmd.Text("cp").FlagWithRspFileInputList("", cmd.GetImplicits()).Output(implicitsRsp) + impRule.Build(pctx, ctx, "implicitsGen", "implicits generation") + cmd.Implicit(implicitsRsp) + + if generateStubs { + rule.Command(). + BuiltTool(ctx, "soong_zip"). + Flag("-write_if_changed"). + Flag("-jar"). + FlagWithOutput("-o ", d.Javadoc.stubsSrcJar). + FlagWithArg("-C ", stubsDir.String()). + FlagWithArg("-D ", stubsDir.String()) + } + + if Bool(d.properties.Write_sdk_values) { + d.metadataZip = android.PathForModuleOut(ctx, ctx.ModuleName()+"-metadata.zip") + rule.Command(). + BuiltTool(ctx, "soong_zip"). + Flag("-write_if_changed"). + Flag("-d"). + FlagWithOutput("-o ", d.metadataZip). + FlagWithArg("-C ", d.metadataDir.String()). + FlagWithArg("-D ", d.metadataDir.String()) + } + + // TODO: We don't really need two separate API files, but this is a reminiscence of how + // we used to run metalava separately for API lint and the "last_released" check. Unify them. + if doApiLint { + rule.Command().Text("touch").Output(d.apiLintTimestamp) + } + if doCheckReleased { + rule.Command().Text("touch").Output(d.checkLastReleasedApiTimestamp) } - if apiCheckEnabled(d.properties.Check_api.Last_released, "last_released") && + rule.Restat() + + zipSyncCleanupCmd(rule, srcJarDir) + + rule.Build(pctx, ctx, "metalava", "metalava merged") + + if apiCheckEnabled(ctx, d.properties.Check_api.Current, "current") && !ctx.Config().IsPdkBuild() { - apiFile := ctx.ExpandSource(String(d.properties.Check_api.Last_released.Api_file), - "check_api.last_released.api_file") - removedApiFile := ctx.ExpandSource(String(d.properties.Check_api.Last_released.Removed_api_file), - "check_api.last_released.removed_api_file") - baselineFile := ctx.ExpandOptionalSource(d.properties.Check_api.Last_released.Baseline_file, - "check_api.last_released.baseline_file") - d.checkLastReleasedApiTimestamp = android.PathForModuleOut(ctx, "check_last_released_api.timestamp") - opts := " " + d.Javadoc.args + " --check-compatibility:api:released " + apiFile.String() + - flags.metalavaInclusionAnnotationsFlags + " --check-compatibility:removed:released " + - removedApiFile.String() + flags.metalavaMergeAnnoDirFlags + " " - baselineOut := android.PathForModuleOut(ctx, "last_released_baseline.txt") + if len(d.Javadoc.properties.Out) > 0 { + ctx.PropertyErrorf("out", "out property may not be combined with check_api") + } + + apiFile := android.PathForModuleSrc(ctx, String(d.properties.Check_api.Current.Api_file)) + removedApiFile := android.PathForModuleSrc(ctx, String(d.properties.Check_api.Current.Removed_api_file)) + baselineFile := android.OptionalPathForModuleSrc(ctx, d.properties.Check_api.Current.Baseline_file) + if baselineFile.Valid() { - opts = opts + "--baseline " + baselineFile.String() + " --update-baseline " + baselineOut.String() + " " + ctx.PropertyErrorf("baseline_file", "current API check can't have a baseline file. (module %s)", ctx.ModuleName()) } - d.transformCheckApi(ctx, apiFile, removedApiFile, baselineFile, baselineOut, metalavaCheckApiImplicits, - javaVersion, flags.bootClasspathArgs, flags.classpathArgs, flags.sourcepathArgs, opts, "last-apicheck", - `\n******************************\n`+ - `You have tried to change the API from what has been previously released in\n`+ - `an SDK. Please fix the errors listed above.\n`+ - `******************************\n`, - d.checkLastReleasedApiTimestamp) + d.checkCurrentApiTimestamp = android.PathForModuleOut(ctx, "check_current_api.timestamp") + + rule := android.NewRuleBuilder() + + // Diff command line. + // -F matches the closest "opening" line, such as "package android {" + // and " public class Intent {". + diff := `diff -u -F '{ *$'` + + rule.Command().Text("( true") + rule.Command(). + Text(diff). + Input(apiFile).Input(d.apiFile) + + rule.Command(). + Text(diff). + Input(removedApiFile).Input(d.removedApiFile) + + msg := fmt.Sprintf(`\n******************************\n`+ + `You have tried to change the API from what has been previously approved.\n\n`+ + `To make these errors go away, you have two choices:\n`+ + ` 1. You can add '@hide' javadoc comments (and remove @SystemApi/@TestApi/etc)\n`+ + ` to the new methods, etc. shown in the above diff.\n\n`+ + ` 2. You can update current.txt and/or removed.txt by executing the following command:\n`+ + ` make %s-update-current-api\n\n`+ + ` To submit the revised current.txt to the main Android repository,\n`+ + ` you will need approval.\n`+ + `******************************\n`, ctx.ModuleName()) + + rule.Command(). + Text("touch").Output(d.checkCurrentApiTimestamp). + Text(") || ("). + Text("echo").Flag("-e").Flag(`"` + msg + `"`). + Text("; exit 38"). + Text(")") + + rule.Build(pctx, ctx, "metalavaCurrentApiCheck", "check current API") + + d.updateCurrentApiTimestamp = android.PathForModuleOut(ctx, "update_current_api.timestamp") + + // update API rule + rule = android.NewRuleBuilder() + + rule.Command().Text("( true") + + rule.Command(). + Text("cp").Flag("-f"). + Input(d.apiFile).Flag(apiFile.String()) + + rule.Command(). + Text("cp").Flag("-f"). + Input(d.removedApiFile).Flag(removedApiFile.String()) + + msg = "failed to update public API" + + rule.Command(). + Text("touch").Output(d.updateCurrentApiTimestamp). + Text(") || ("). + Text("echo").Flag("-e").Flag(`"` + msg + `"`). + Text("; exit 38"). + Text(")") + + rule.Build(pctx, ctx, "metalavaCurrentApiUpdate", "update current API") } if String(d.properties.Check_nullability_warnings) != "" { @@ -1752,9 +1801,11 @@ func (d *Droidstubs) GenerateAndroidBuildActions(ctx android.ModuleContext) { ctx.PropertyErrorf("check_nullability_warnings", "Cannot specify check_nullability_warnings unless validating nullability") } - checkNullabilityWarnings := ctx.ExpandSource(String(d.properties.Check_nullability_warnings), - "check_nullability_warnings") + + checkNullabilityWarnings := android.PathForModuleSrc(ctx, String(d.properties.Check_nullability_warnings)) + d.checkNullabilityWarningsTimestamp = android.PathForModuleOut(ctx, "check_nullability_warnings.timestamp") + msg := fmt.Sprintf(`\n******************************\n`+ `The warnings encountered during nullability annotation validation did\n`+ `not match the checked in file of expected warnings. The diffs are shown\n`+ @@ -1764,20 +1815,32 @@ func (d *Droidstubs) GenerateAndroidBuildActions(ctx android.ModuleContext) { ` cp %s %s\n`+ ` and submitting the updated file as part of your change.`, d.nullabilityWarningsFile, checkNullabilityWarnings) - ctx.Build(pctx, android.BuildParams{ - Rule: nullabilityWarningsCheck, - Description: "Nullability Warnings Check", - Output: d.checkNullabilityWarningsTimestamp, - Implicits: android.Paths{checkNullabilityWarnings, d.nullabilityWarningsFile}, - Args: map[string]string{ - "expected": checkNullabilityWarnings.String(), - "actual": d.nullabilityWarningsFile.String(), - "msg": msg, - }, - }) + + rule := android.NewRuleBuilder() + + rule.Command(). + Text("("). + Text("diff").Input(checkNullabilityWarnings).Input(d.nullabilityWarningsFile). + Text("&&"). + Text("touch").Output(d.checkNullabilityWarningsTimestamp). + Text(") || ("). + Text("echo").Flag("-e").Flag(`"` + msg + `"`). + Text("; exit 38"). + Text(")") + + rule.Build(pctx, ctx, "nullabilityWarningsCheck", "nullability warnings check") } if Bool(d.properties.Jdiff_enabled) && !ctx.Config().IsPdkBuild() { + if len(d.Javadoc.properties.Out) > 0 { + ctx.PropertyErrorf("out", "out property may not be combined with jdiff") + } + + outDir := android.PathForModuleOut(ctx, "jdiff-out") + srcJarDir := android.PathForModuleOut(ctx, "jdiff-srcjars") + stubsDir := android.PathForModuleOut(ctx, "jdiff-stubsDir") + + rule := android.NewRuleBuilder() // Please sync with android-api-council@ before making any changes for the name of jdiffDocZip below // since there's cron job downstream that fetch this .zip file periodically. @@ -1785,21 +1848,55 @@ func (d *Droidstubs) GenerateAndroidBuildActions(ctx android.ModuleContext) { d.jdiffDocZip = android.PathForModuleOut(ctx, ctx.ModuleName()+"-"+"jdiff-docs.zip") d.jdiffStubsSrcJar = android.PathForModuleOut(ctx, ctx.ModuleName()+"-"+"jdiff-stubs.srcjar") - var jdiffImplicitOutputs android.WritablePaths - jdiffImplicitOutputs = append(jdiffImplicitOutputs, d.jdiffDocZip) - jdiff := android.PathForOutput(ctx, "host", ctx.Config().PrebuiltOS(), "framework", "jdiff.jar") - jdiffImplicits = append(jdiffImplicits, android.Paths{jdiff, d.apiXmlFile, d.lastReleasedApiXmlFile}...) - opts := " -encoding UTF-8 -source 1.8 -J-Xmx1600m -XDignore.symbol.file " + - "-doclet jdiff.JDiff -docletpath " + jdiff.String() + " -quiet " + - "-newapi " + strings.TrimSuffix(d.apiXmlFile.Base(), d.apiXmlFile.Ext()) + - " -newapidir " + filepath.Dir(d.apiXmlFile.String()) + - " -oldapi " + strings.TrimSuffix(d.lastReleasedApiXmlFile.Base(), d.lastReleasedApiXmlFile.Ext()) + - " -oldapidir " + filepath.Dir(d.lastReleasedApiXmlFile.String()) + rule.Command().Text("rm -rf").Text(outDir.String()).Text(stubsDir.String()) + rule.Command().Text("mkdir -p").Text(outDir.String()).Text(stubsDir.String()) + + srcJarList := zipSyncCmd(ctx, rule, srcJarDir, d.Javadoc.srcJars) + + cmd := javadocBootclasspathCmd(ctx, rule, d.Javadoc.srcFiles, outDir, srcJarDir, srcJarList, + deps.bootClasspath, deps.classpath, d.sourcepaths) + + cmd.Flag("-J-Xmx1600m"). + Flag("-XDignore.symbol.file"). + FlagWithArg("-doclet ", "jdiff.JDiff"). + FlagWithInput("-docletpath ", jdiff). + Flag("-quiet") + + if d.apiXmlFile != nil { + cmd.FlagWithArg("-newapi ", strings.TrimSuffix(d.apiXmlFile.Base(), d.apiXmlFile.Ext())). + FlagWithArg("-newapidir ", filepath.Dir(d.apiXmlFile.String())). + Implicit(d.apiXmlFile) + } + + if d.lastReleasedApiXmlFile != nil { + cmd.FlagWithArg("-oldapi ", strings.TrimSuffix(d.lastReleasedApiXmlFile.Base(), d.lastReleasedApiXmlFile.Ext())). + FlagWithArg("-oldapidir ", filepath.Dir(d.lastReleasedApiXmlFile.String())). + Implicit(d.lastReleasedApiXmlFile) + } + + rule.Command(). + BuiltTool(ctx, "soong_zip"). + Flag("-write_if_changed"). + Flag("-d"). + FlagWithOutput("-o ", d.jdiffDocZip). + FlagWithArg("-C ", outDir.String()). + FlagWithArg("-D ", outDir.String()) - d.transformJdiff(ctx, jdiffImplicits, jdiffImplicitOutputs, flags.bootClasspathArgs, flags.classpathArgs, - flags.sourcepathArgs, opts) + rule.Command(). + BuiltTool(ctx, "soong_zip"). + Flag("-write_if_changed"). + Flag("-jar"). + FlagWithOutput("-o ", d.jdiffStubsSrcJar). + FlagWithArg("-C ", stubsDir.String()). + FlagWithArg("-D ", stubsDir.String()) + + rule.Restat() + + zipSyncCleanupCmd(rule, srcJarDir) + + rule.Build(pctx, ctx, "jdiff", "jdiff") } } @@ -1825,6 +1922,7 @@ type ExportedDroiddocDir struct { dir android.Path } +// droiddoc_exported_dir exports a directory of html templates or nullability annotations for use by doclava. func ExportedDroiddocDirFactory() android.Module { module := &ExportedDroiddocDir{} module.AddProperties(&module.properties) @@ -1848,9 +1946,6 @@ type DocDefaults struct { android.DefaultsModuleBase } -func (*DocDefaults) GenerateAndroidBuildActions(ctx android.ModuleContext) { -} - func DocDefaultsFactory() android.Module { module := &DocDefaults{} @@ -1876,3 +1971,154 @@ func StubsDefaultsFactory() android.Module { return module } + +func zipSyncCmd(ctx android.ModuleContext, rule *android.RuleBuilder, + srcJarDir android.ModuleOutPath, srcJars android.Paths) android.OutputPath { + + rule.Command().Text("rm -rf").Text(srcJarDir.String()) + rule.Command().Text("mkdir -p").Text(srcJarDir.String()) + srcJarList := srcJarDir.Join(ctx, "list") + + rule.Temporary(srcJarList) + + rule.Command().BuiltTool(ctx, "zipsync"). + FlagWithArg("-d ", srcJarDir.String()). + FlagWithOutput("-l ", srcJarList). + FlagWithArg("-f ", `"*.java"`). + Inputs(srcJars) + + return srcJarList +} + +func zipSyncCleanupCmd(rule *android.RuleBuilder, srcJarDir android.ModuleOutPath) { + rule.Command().Text("rm -rf").Text(srcJarDir.String()) +} + +var _ android.PrebuiltInterface = (*PrebuiltStubsSources)(nil) + +type PrebuiltStubsSourcesProperties struct { + Srcs []string `android:"path"` +} + +type PrebuiltStubsSources struct { + android.ModuleBase + android.DefaultableModuleBase + prebuilt android.Prebuilt + android.SdkBase + + properties PrebuiltStubsSourcesProperties + + // The source directories containing stubs source files. + srcDirs android.Paths + stubsSrcJar android.ModuleOutPath +} + +func (p *PrebuiltStubsSources) OutputFiles(tag string) (android.Paths, error) { + switch tag { + case "": + return android.Paths{p.stubsSrcJar}, nil + default: + return nil, fmt.Errorf("unsupported module reference tag %q", tag) + } +} + +func (d *PrebuiltStubsSources) StubsSrcJar() android.Path { + return d.stubsSrcJar +} + +func (p *PrebuiltStubsSources) GenerateAndroidBuildActions(ctx android.ModuleContext) { + p.stubsSrcJar = android.PathForModuleOut(ctx, ctx.ModuleName()+"-"+"stubs.srcjar") + + p.srcDirs = android.PathsForModuleSrc(ctx, p.properties.Srcs) + + rule := android.NewRuleBuilder() + command := rule.Command(). + BuiltTool(ctx, "soong_zip"). + Flag("-write_if_changed"). + Flag("-jar"). + FlagWithOutput("-o ", p.stubsSrcJar) + + for _, d := range p.srcDirs { + dir := d.String() + command. + FlagWithArg("-C ", dir). + FlagWithInput("-D ", d) + } + + rule.Restat() + + rule.Build(pctx, ctx, "zip src", "Create srcjar from prebuilt source") +} + +func (p *PrebuiltStubsSources) Prebuilt() *android.Prebuilt { + return &p.prebuilt +} + +func (p *PrebuiltStubsSources) Name() string { + return p.prebuilt.Name(p.ModuleBase.Name()) +} + +// prebuilt_stubs_sources imports a set of java source files as if they were +// generated by droidstubs. +// +// By default, a prebuilt_stubs_sources has a single variant that expects a +// set of `.java` files generated by droidstubs. +// +// Specifying `host_supported: true` will produce two variants, one for use as a dependency of device modules and one +// for host modules. +// +// Intended only for use by sdk snapshots. +func PrebuiltStubsSourcesFactory() android.Module { + module := &PrebuiltStubsSources{} + + module.AddProperties(&module.properties) + + android.InitPrebuiltModule(module, &module.properties.Srcs) + android.InitSdkAwareModule(module) + InitDroiddocModule(module, android.HostAndDeviceSupported) + return module +} + +type droidStubsSdkMemberType struct { + android.SdkMemberTypeBase +} + +func (mt *droidStubsSdkMemberType) AddDependencies(mctx android.BottomUpMutatorContext, dependencyTag blueprint.DependencyTag, names []string) { + mctx.AddVariationDependencies(nil, dependencyTag, names...) +} + +func (mt *droidStubsSdkMemberType) IsInstance(module android.Module) bool { + _, ok := module.(*Droidstubs) + return ok +} + +func (mt *droidStubsSdkMemberType) AddPrebuiltModule(ctx android.SdkMemberContext, member android.SdkMember) android.BpModule { + return ctx.SnapshotBuilder().AddPrebuiltModule(member, "prebuilt_stubs_sources") +} + +func (mt *droidStubsSdkMemberType) CreateVariantPropertiesStruct() android.SdkMemberProperties { + return &droidStubsInfoProperties{} +} + +type droidStubsInfoProperties struct { + android.SdkMemberPropertiesBase + + StubsSrcJar android.Path +} + +func (p *droidStubsInfoProperties) PopulateFromVariant(ctx android.SdkMemberContext, variant android.Module) { + droidstubs := variant.(*Droidstubs) + p.StubsSrcJar = droidstubs.stubsSrcJar +} + +func (p *droidStubsInfoProperties) AddToPropertySet(ctx android.SdkMemberContext, propertySet android.BpPropertySet) { + if p.StubsSrcJar != nil { + builder := ctx.SnapshotBuilder() + + snapshotRelativeDir := filepath.Join("java", ctx.Name()+"_stubs_sources") + + builder.UnzipToSnapshot(p.StubsSrcJar, snapshotRelativeDir) + + propertySet.AddProperty("srcs", []string{snapshotRelativeDir}) + } +} diff --git a/java/gen.go b/java/gen.go index 09776283f..d50a6653e 100644 --- a/java/gen.go +++ b/java/gen.go @@ -15,67 +15,93 @@ package java import ( + "strconv" + "strings" + "github.com/google/blueprint" + "github.com/google/blueprint/pathtools" "android/soong/android" ) func init() { - pctx.HostBinToolVariable("aidlCmd", "aidl") - pctx.HostBinToolVariable("syspropCmd", "sysprop_java") - pctx.SourcePathVariable("logtagsCmd", "build/tools/java-event-log-tags.py") - pctx.SourcePathVariable("mergeLogtagsCmd", "build/tools/merge-event-log-tags.py") + pctx.SourcePathVariable("logtagsCmd", "build/make/tools/java-event-log-tags.py") + pctx.SourcePathVariable("mergeLogtagsCmd", "build/make/tools/merge-event-log-tags.py") + pctx.SourcePathVariable("logtagsLib", "build/make/tools/event_log_tags.py") } var ( - aidl = pctx.AndroidStaticRule("aidl", - blueprint.RuleParams{ - Command: "$aidlCmd -d$depFile $aidlFlags $in $out", - CommandDeps: []string{"$aidlCmd"}, - }, - "depFile", "aidlFlags") - logtags = pctx.AndroidStaticRule("logtags", blueprint.RuleParams{ Command: "$logtagsCmd -o $out $in", - CommandDeps: []string{"$logtagsCmd"}, + CommandDeps: []string{"$logtagsCmd", "$logtagsLib"}, }) mergeLogtags = pctx.AndroidStaticRule("mergeLogtags", blueprint.RuleParams{ Command: "$mergeLogtagsCmd -o $out $in", - CommandDeps: []string{"$mergeLogtagsCmd"}, - }) - - sysprop = pctx.AndroidStaticRule("sysprop", - blueprint.RuleParams{ - Command: `rm -rf $out.tmp && mkdir -p $out.tmp && ` + - `$syspropCmd --java-output-dir $out.tmp $in && ` + - `${config.SoongZipCmd} -jar -o $out -C $out.tmp -D $out.tmp && rm -rf $out.tmp`, - CommandDeps: []string{ - "$syspropCmd", - "${config.SoongZipCmd}", - }, + CommandDeps: []string{"$mergeLogtagsCmd", "$logtagsLib"}, }) ) -func genAidl(ctx android.ModuleContext, aidlFile android.Path, aidlFlags string, deps android.Paths) android.Path { - javaFile := android.GenPathWithExt(ctx, "aidl", aidlFile, "java") - depFile := javaFile.String() + ".d" +func genAidl(ctx android.ModuleContext, aidlFiles android.Paths, aidlFlags string, deps android.Paths) android.Paths { + // Shard aidl files into groups of 50 to avoid having to recompile all of them if one changes and to avoid + // hitting command line length limits. + shards := android.ShardPaths(aidlFiles, 50) - ctx.Build(pctx, android.BuildParams{ - Rule: aidl, - Description: "aidl " + aidlFile.Rel(), - Output: javaFile, - Input: aidlFile, - Implicits: deps, - Args: map[string]string{ - "depFile": depFile, - "aidlFlags": aidlFlags, - }, - }) + srcJarFiles := make(android.Paths, 0, len(shards)) - return javaFile + for i, shard := range shards { + srcJarFile := android.PathForModuleGen(ctx, "aidl", "aidl"+strconv.Itoa(i)+".srcjar") + srcJarFiles = append(srcJarFiles, srcJarFile) + + outDir := srcJarFile.ReplaceExtension(ctx, "tmp") + + rule := android.NewRuleBuilder() + + rule.Command().Text("rm -rf").Flag(outDir.String()) + rule.Command().Text("mkdir -p").Flag(outDir.String()) + rule.Command().Text("FLAGS=' " + aidlFlags + "'") + + for _, aidlFile := range shard { + depFile := srcJarFile.InSameDir(ctx, aidlFile.String()+".d") + javaFile := outDir.Join(ctx, pathtools.ReplaceExtension(aidlFile.String(), "java")) + rule.Command(). + Tool(ctx.Config().HostToolPath(ctx, "aidl")). + FlagWithDepFile("-d", depFile). + Flag("$FLAGS"). + Input(aidlFile). + Output(javaFile). + Implicits(deps) + rule.Temporary(javaFile) + } + + rule.Command(). + Tool(ctx.Config().HostToolPath(ctx, "soong_zip")). + // TODO(b/124333557): this can't use -srcjar for now, aidl on parcelables generates java files + // without a package statement, which causes -srcjar to put them in the top level of the zip file. + // Once aidl skips parcelables we can use -srcjar. + //Flag("-srcjar"). + Flag("-write_if_changed"). + FlagWithOutput("-o ", srcJarFile). + FlagWithArg("-C ", outDir.String()). + FlagWithArg("-D ", outDir.String()) + + rule.Command().Text("rm -rf").Flag(outDir.String()) + + rule.Restat() + + ruleName := "aidl" + ruleDesc := "aidl" + if len(shards) > 1 { + ruleName += "_" + strconv.Itoa(i) + ruleDesc += " " + strconv.Itoa(i) + } + + rule.Build(pctx, ctx, ruleName, ruleDesc) + } + + return srcJarFiles } func genLogtags(ctx android.ModuleContext, logtagsFile android.Path) android.Path { @@ -91,44 +117,55 @@ func genLogtags(ctx android.ModuleContext, logtagsFile android.Path) android.Pat return javaFile } -func genSysprop(ctx android.ModuleContext, syspropFile android.Path) android.Path { - srcJarFile := android.GenPathWithExt(ctx, "sysprop", syspropFile, "srcjar") - - ctx.Build(pctx, android.BuildParams{ - Rule: sysprop, - Description: "sysprop_java " + syspropFile.Rel(), - Output: srcJarFile, - Input: syspropFile, - }) - - return srcJarFile +func genAidlIncludeFlags(srcFiles android.Paths) string { + var baseDirs []string + for _, srcFile := range srcFiles { + if srcFile.Ext() == ".aidl" { + baseDir := strings.TrimSuffix(srcFile.String(), srcFile.Rel()) + if baseDir != "" && !android.InList(baseDir, baseDirs) { + baseDirs = append(baseDirs, baseDir) + } + } + } + return android.JoinWithPrefix(baseDirs, " -I") } func (j *Module) genSources(ctx android.ModuleContext, srcFiles android.Paths, flags javaBuilderFlags) android.Paths { outSrcFiles := make(android.Paths, 0, len(srcFiles)) + var protoSrcs android.Paths + var aidlSrcs android.Paths + + aidlIncludeFlags := genAidlIncludeFlags(srcFiles) for _, srcFile := range srcFiles { switch srcFile.Ext() { case ".aidl": - javaFile := genAidl(ctx, srcFile, flags.aidlFlags, flags.aidlDeps) - outSrcFiles = append(outSrcFiles, javaFile) + aidlSrcs = append(aidlSrcs, srcFile) case ".logtags": j.logtagsSrcs = append(j.logtagsSrcs, srcFile) javaFile := genLogtags(ctx, srcFile) outSrcFiles = append(outSrcFiles, javaFile) case ".proto": - srcJarFile := genProto(ctx, srcFile, flags.proto) - outSrcFiles = append(outSrcFiles, srcJarFile) - case ".sysprop": - srcJarFile := genSysprop(ctx, srcFile) - outSrcFiles = append(outSrcFiles, srcJarFile) + protoSrcs = append(protoSrcs, srcFile) default: outSrcFiles = append(outSrcFiles, srcFile) } } + // Process all proto files together to support sharding them into one or more rules that produce srcjars. + if len(protoSrcs) > 0 { + srcJarFiles := genProto(ctx, protoSrcs, flags.proto) + outSrcFiles = append(outSrcFiles, srcJarFiles...) + } + + // Process all aidl files together to support sharding them into one or more rules that produce srcjars. + if len(aidlSrcs) > 0 { + srcJarFiles := genAidl(ctx, aidlSrcs, flags.aidlFlags+aidlIncludeFlags, flags.aidlDeps) + outSrcFiles = append(outSrcFiles, srcJarFiles...) + } + return outSrcFiles } diff --git a/java/genrule.go b/java/genrule.go index 25494ec8a..e0a9c8faf 100644 --- a/java/genrule.go +++ b/java/genrule.go @@ -20,8 +20,12 @@ import ( ) func init() { - android.RegisterModuleType("java_genrule", genRuleFactory) - android.RegisterModuleType("java_genrule_host", genRuleFactoryHost) + RegisterGenRuleBuildComponents(android.InitRegistrationContext) +} + +func RegisterGenRuleBuildComponents(ctx android.RegistrationContext) { + ctx.RegisterModuleType("java_genrule", genRuleFactory) + ctx.RegisterModuleType("java_genrule_host", genRuleFactoryHost) } // java_genrule is a genrule that can depend on other java_* objects. diff --git a/java/hiddenapi.go b/java/hiddenapi.go index 6020aba6e..b5a021785 100644 --- a/java/hiddenapi.go +++ b/java/hiddenapi.go @@ -28,9 +28,10 @@ var hiddenAPIGenerateCSVRule = pctx.AndroidStaticRule("hiddenAPIGenerateCSV", bl }, "outFlag", "stubAPIFlags") type hiddenAPI struct { + bootDexJarPath android.Path flagsCSVPath android.Path + indexCSVPath android.Path metadataCSVPath android.Path - bootDexJarPath android.Path } func (h *hiddenAPI) flagsCSV() android.Path { @@ -45,19 +46,22 @@ func (h *hiddenAPI) bootDexJar() android.Path { return h.bootDexJarPath } +func (h *hiddenAPI) indexCSV() android.Path { + return h.indexCSVPath +} + type hiddenAPIIntf interface { + bootDexJar() android.Path flagsCSV() android.Path + indexCSV() android.Path metadataCSV() android.Path - bootDexJar() android.Path } var _ hiddenAPIIntf = (*hiddenAPI)(nil) -func (h *hiddenAPI) hiddenAPI(ctx android.ModuleContext, dexJar android.ModuleOutPath, implementationJar android.Path, - uncompressDex bool) android.ModuleOutPath { - +func (h *hiddenAPI) hiddenAPI(ctx android.ModuleContext, name string, primary bool, dexJar android.ModuleOutPath, + implementationJar android.Path, uncompressDex bool) android.ModuleOutPath { if !ctx.Config().IsEnvTrue("UNSAFE_DISABLE_HIDDENAPI_FLAGS") { - name := ctx.ModuleName() // Modules whose names are of the format <x>-hiddenapi provide hiddenapi information // for the boot jar module <x>. Otherwise, the module provides information for itself. @@ -77,16 +81,22 @@ func (h *hiddenAPI) hiddenAPI(ctx android.ModuleContext, dexJar android.ModuleOu // Derive the greylist from classes jar. flagsCSV := android.PathForModuleOut(ctx, "hiddenapi", "flags.csv") metadataCSV := android.PathForModuleOut(ctx, "hiddenapi", "metadata.csv") - hiddenAPIGenerateCSV(ctx, flagsCSV, metadataCSV, implementationJar) - h.flagsCSVPath = flagsCSV - h.metadataCSVPath = metadataCSV + indexCSV := android.PathForModuleOut(ctx, "hiddenapi", "index.csv") + h.hiddenAPIGenerateCSV(ctx, flagsCSV, metadataCSV, indexCSV, implementationJar) // If this module is actually on the boot jars list and not providing // hiddenapi information for a module on the boot jars list then encode // the gathered information in the generated dex file. if name == bootJarName { hiddenAPIJar := android.PathForModuleOut(ctx, "hiddenapi", name+".jar") - h.bootDexJarPath = dexJar + + // More than one library with the same classes can be encoded but only one can + // be added to the global set of flags, otherwise it will result in duplicate + // classes which is an error. Therefore, only add the dex jar of one of them + // to the global set of flags. + if primary { + h.bootDexJarPath = dexJar + } hiddenAPIEncodeDex(ctx, hiddenAPIJar, dexJar, uncompressDex) dexJar = hiddenAPIJar } @@ -96,9 +106,7 @@ func (h *hiddenAPI) hiddenAPI(ctx android.ModuleContext, dexJar android.ModuleOu return dexJar } -func hiddenAPIGenerateCSV(ctx android.ModuleContext, flagsCSV, metadataCSV android.WritablePath, - classesJar android.Path) { - +func (h *hiddenAPI) hiddenAPIGenerateCSV(ctx android.ModuleContext, flagsCSV, metadataCSV, indexCSV android.WritablePath, classesJar android.Path) { stubFlagsCSV := hiddenAPISingletonPaths(ctx).stubFlags ctx.Build(pctx, android.BuildParams{ @@ -112,6 +120,7 @@ func hiddenAPIGenerateCSV(ctx android.ModuleContext, flagsCSV, metadataCSV andro "stubAPIFlags": stubFlagsCSV.String(), }, }) + h.flagsCSVPath = flagsCSV ctx.Build(pctx, android.BuildParams{ Rule: hiddenAPIGenerateCSVRule, @@ -124,18 +133,26 @@ func hiddenAPIGenerateCSV(ctx android.ModuleContext, flagsCSV, metadataCSV andro "stubAPIFlags": stubFlagsCSV.String(), }, }) - + h.metadataCSVPath = metadataCSV + + rule := android.NewRuleBuilder() + rule.Command(). + BuiltTool(ctx, "merge_csv"). + FlagWithInput("--zip_input=", classesJar). + FlagWithOutput("--output=", indexCSV) + rule.Build(pctx, ctx, "merged-hiddenapi-index", "Merged Hidden API index") + h.indexCSVPath = indexCSV } var hiddenAPIEncodeDexRule = pctx.AndroidStaticRule("hiddenAPIEncodeDex", blueprint.RuleParams{ - Command: `rm -rf $tmpDir && mkdir -p $tmpDir && mkdir $tmpDir/dex-input && mkdir $tmpDir/dex-output && ` + - `unzip -o -q $in 'classes*.dex' -d $tmpDir/dex-input && ` + - `for INPUT_DEX in $$(find $tmpDir/dex-input -maxdepth 1 -name 'classes*.dex' | sort); do ` + - ` echo "--input-dex=$${INPUT_DEX}"; ` + - ` echo "--output-dex=$tmpDir/dex-output/$$(basename $${INPUT_DEX})"; ` + - `done | xargs ${config.HiddenAPI} encode --api-flags=$flagsCsv $hiddenapiFlags && ` + - `${config.SoongZipCmd} $soongZipFlags -o $tmpDir/dex.jar -C $tmpDir/dex-output -f "$tmpDir/dex-output/classes*.dex" && ` + - `${config.MergeZipsCmd} -D -zipToNotStrip $tmpDir/dex.jar -stripFile "classes*.dex" $out $tmpDir/dex.jar $in`, + Command: `rm -rf $tmpDir && mkdir -p $tmpDir && mkdir $tmpDir/dex-input && mkdir $tmpDir/dex-output && + unzip -qoDD $in 'classes*.dex' -d $tmpDir/dex-input && + for INPUT_DEX in $$(find $tmpDir/dex-input -maxdepth 1 -name 'classes*.dex' | sort); do + echo "--input-dex=$${INPUT_DEX}"; + echo "--output-dex=$tmpDir/dex-output/$$(basename $${INPUT_DEX})"; + done | xargs ${config.HiddenAPI} encode --api-flags=$flagsCsv $hiddenapiFlags && + ${config.SoongZipCmd} $soongZipFlags -o $tmpDir/dex.jar -C $tmpDir/dex-output -f "$tmpDir/dex-output/classes*.dex" && + ${config.MergeZipsCmd} -D -zipToNotStrip $tmpDir/dex.jar -stripFile "classes*.dex" -stripFile "**/*.uau" $out $tmpDir/dex.jar $in`, CommandDeps: []string{ "${config.HiddenAPI}", "${config.SoongZipCmd}", @@ -159,9 +176,23 @@ func hiddenAPIEncodeDex(ctx android.ModuleContext, output android.WritablePath, tmpOutput = android.PathForModuleOut(ctx, "hiddenapi", "unaligned", "unaligned.jar") tmpDir = android.PathForModuleOut(ctx, "hiddenapi", "unaligned") } + + enforceHiddenApiFlagsToAllMembers := true // If frameworks/base doesn't exist we must be building with the 'master-art' manifest. // Disable assertion that all methods/fields have hidden API flags assigned. if !ctx.Config().FrameworksBaseDirExists(ctx) { + enforceHiddenApiFlagsToAllMembers = false + } + // b/149353192: when a module is instrumented, jacoco adds synthetic members + // $jacocoData and $jacocoInit. Since they don't exist when building the hidden API flags, + // don't complain when we don't find hidden API flags for the synthetic members. + if j, ok := ctx.Module().(interface { + shouldInstrument(android.BaseModuleContext) bool + }); ok && j.shouldInstrument(ctx) { + enforceHiddenApiFlagsToAllMembers = false + } + + if !enforceHiddenApiFlagsToAllMembers { hiddenapiFlags = "--no-force-assign-all" } diff --git a/java/hiddenapi_singleton.go b/java/hiddenapi_singleton.go index 09936ea78..95dd0bb09 100644 --- a/java/hiddenapi_singleton.go +++ b/java/hiddenapi_singleton.go @@ -15,17 +15,22 @@ package java import ( + "fmt" + "android/soong/android" ) func init() { android.RegisterSingletonType("hiddenapi", hiddenAPISingletonFactory) + android.RegisterSingletonType("hiddenapi_index", hiddenAPIIndexSingletonFactory) + android.RegisterModuleType("hiddenapi_flags", hiddenAPIFlagsFactory) } type hiddenAPISingletonPathsStruct struct { - stubFlags android.OutputPath flags android.OutputPath + index android.OutputPath metadata android.OutputPath + stubFlags android.OutputPath } var hiddenAPISingletonPathsKey = android.NewOnceKey("hiddenAPISingletonPathsKey") @@ -36,9 +41,10 @@ var hiddenAPISingletonPathsKey = android.NewOnceKey("hiddenAPISingletonPathsKey" func hiddenAPISingletonPaths(ctx android.PathContext) hiddenAPISingletonPathsStruct { return ctx.Config().Once(hiddenAPISingletonPathsKey, func() interface{} { return hiddenAPISingletonPathsStruct{ - stubFlags: android.PathForOutput(ctx, "hiddenapi", "hiddenapi-stub-flags.txt"), flags: android.PathForOutput(ctx, "hiddenapi", "hiddenapi-flags.csv"), + index: android.PathForOutput(ctx, "hiddenapi", "hiddenapi-index.csv"), metadata: android.PathForOutput(ctx, "hiddenapi", "hiddenapi-greylist.csv"), + stubFlags: android.PathForOutput(ctx, "hiddenapi", "hiddenapi-stub-flags.txt"), } }).(hiddenAPISingletonPathsStruct) } @@ -149,6 +155,14 @@ func stubFlagsRule(ctx android.SingletonContext) { // Collect dex jar paths for modules that had hiddenapi encode called on them. if h, ok := module.(hiddenAPIIntf); ok { if jar := h.bootDexJar(); jar != nil { + // For a java lib included in an APEX, only take the one built for + // the platform variant, and skip the variants for APEXes. + // Otherwise, the hiddenapi tool will complain about duplicated classes + if a, ok := module.(android.ApexModule); ok { + if android.InAnyApex(module.Name()) && !a.IsForPlatform() { + return + } + } bootDexJars = append(bootDexJars, jar) } } @@ -179,7 +193,7 @@ func stubFlagsRule(ctx android.SingletonContext) { rule.MissingDeps(missingDeps) rule.Command(). - Tool(pctx.HostBinToolPath(ctx, "hiddenapi")). + Tool(ctx.Config().HostToolPath(ctx, "hiddenapi")). Text("list"). FlagForEachInput("--boot-dex=", bootDexJars). FlagWithInputList("--public-stub-classpath=", publicStubPaths, ":"). @@ -193,27 +207,43 @@ func stubFlagsRule(ctx android.SingletonContext) { rule.Build(pctx, ctx, "hiddenAPIStubFlagsFile", "hiddenapi stub flags") } +func moduleForGreyListRemovedApis(ctx android.SingletonContext, module android.Module) bool { + switch ctx.ModuleName(module) { + case "api-stubs-docs", "system-api-stubs-docs", "android.car-stubs-docs", "android.car-system-stubs-docs": + return true + default: + return false + } +} + // flagsRule creates a rule to build hiddenapi-flags.csv out of flags.csv files generated for boot image modules and // the greylists. func flagsRule(ctx android.SingletonContext) android.Path { var flagsCSV android.Paths - - var greylistIgnoreConflicts android.Path + var greylistRemovedApis android.Paths ctx.VisitAllModules(func(module android.Module) { if h, ok := module.(hiddenAPIIntf); ok { if csv := h.flagsCSV(); csv != nil { flagsCSV = append(flagsCSV, csv) } - } else if ds, ok := module.(*Droidstubs); ok && ctx.ModuleName(module) == "hiddenapi-lists-docs" { - greylistIgnoreConflicts = ds.removedDexApiFile + } else if ds, ok := module.(*Droidstubs); ok { + // Track @removed public and system APIs via corresponding droidstubs targets. + // These APIs are not present in the stubs, however, we have to keep allowing access + // to them at runtime. + if moduleForGreyListRemovedApis(ctx, module) { + greylistRemovedApis = append(greylistRemovedApis, ds.removedDexApiFile) + } } }) - if greylistIgnoreConflicts == nil { - ctx.Errorf("failed to find removed_dex_api_filename from hiddenapi-lists-docs module") - return nil - } + combinedRemovedApis := android.PathForOutput(ctx, "hiddenapi", "combined-removed-dex.txt") + ctx.Build(pctx, android.BuildParams{ + Rule: android.Cat, + Inputs: greylistRemovedApis, + Output: combinedRemovedApis, + Description: "Combine removed apis for " + combinedRemovedApis.String(), + }) rule := android.NewRuleBuilder() @@ -228,8 +258,9 @@ func flagsRule(ctx android.SingletonContext) android.Path { Inputs(flagsCSV). FlagWithInput("--greylist ", android.PathForSource(ctx, "frameworks/base/config/hiddenapi-greylist.txt")). - FlagWithInput("--greylist-ignore-conflicts ", - greylistIgnoreConflicts). + FlagWithInput("--greylist-ignore-conflicts ", combinedRemovedApis). + FlagWithInput("--greylist-max-q ", + android.PathForSource(ctx, "frameworks/base/config/hiddenapi-greylist-max-q.txt")). FlagWithInput("--greylist-max-p ", android.PathForSource(ctx, "frameworks/base/config/hiddenapi-greylist-max-p.txt")). FlagWithInput("--greylist-max-o-ignore-conflicts ", @@ -280,10 +311,9 @@ func metadataRule(ctx android.SingletonContext) android.Path { outputPath := hiddenAPISingletonPaths(ctx).metadata rule.Command(). - Tool(android.PathForSource(ctx, "frameworks/base/tools/hiddenapi/merge_csv.py")). - Inputs(metadataCSV). - Text(">"). - Output(outputPath) + BuiltTool(ctx, "merge_csv"). + FlagWithOutput("--output=", outputPath). + Inputs(metadataCSV) rule.Build(pctx, ctx, "hiddenAPIGreylistMetadataFile", "hiddenapi greylist metadata") @@ -307,3 +337,90 @@ func commitChangeForRestat(rule *android.RuleBuilder, tempPath, outputPath andro Text("fi"). Text(")") } + +type hiddenAPIFlagsProperties struct { + // name of the file into which the flags will be copied. + Filename *string +} + +type hiddenAPIFlags struct { + android.ModuleBase + + properties hiddenAPIFlagsProperties + + outputFilePath android.OutputPath +} + +func (h *hiddenAPIFlags) GenerateAndroidBuildActions(ctx android.ModuleContext) { + filename := String(h.properties.Filename) + + inputPath := hiddenAPISingletonPaths(ctx).flags + h.outputFilePath = android.PathForModuleOut(ctx, filename).OutputPath + + // This ensures that outputFilePath has the correct name for others to + // use, as the source file may have a different name. + ctx.Build(pctx, android.BuildParams{ + Rule: android.Cp, + Output: h.outputFilePath, + Input: inputPath, + }) +} + +func (h *hiddenAPIFlags) OutputFiles(tag string) (android.Paths, error) { + switch tag { + case "": + return android.Paths{h.outputFilePath}, nil + default: + return nil, fmt.Errorf("unsupported module reference tag %q", tag) + } +} + +// hiddenapi-flags provides access to the hiddenapi-flags.csv file generated during the build. +func hiddenAPIFlagsFactory() android.Module { + module := &hiddenAPIFlags{} + module.AddProperties(&module.properties) + android.InitAndroidArchModule(module, android.HostAndDeviceSupported, android.MultilibCommon) + return module +} + +func hiddenAPIIndexSingletonFactory() android.Singleton { + return &hiddenAPIIndexSingleton{} +} + +type hiddenAPIIndexSingleton struct { + index android.Path +} + +func (h *hiddenAPIIndexSingleton) GenerateBuildActions(ctx android.SingletonContext) { + // Don't run any hiddenapi rules if UNSAFE_DISABLE_HIDDENAPI_FLAGS=true + if ctx.Config().IsEnvTrue("UNSAFE_DISABLE_HIDDENAPI_FLAGS") { + return + } + + indexes := android.Paths{} + ctx.VisitAllModules(func(module android.Module) { + if h, ok := module.(hiddenAPIIntf); ok { + if h.indexCSV() != nil { + indexes = append(indexes, h.indexCSV()) + } + } + }) + + rule := android.NewRuleBuilder() + rule.Command(). + BuiltTool(ctx, "merge_csv"). + FlagWithArg("--header=", "signature,file,startline,startcol,endline,endcol,properties"). + FlagWithOutput("--output=", hiddenAPISingletonPaths(ctx).index). + Inputs(indexes) + rule.Build(pctx, ctx, "singleton-merged-hiddenapi-index", "Singleton merged Hidden API index") + + h.index = hiddenAPISingletonPaths(ctx).index +} + +func (h *hiddenAPIIndexSingleton) MakeVars(ctx android.MakeVarsContext) { + if ctx.Config().IsEnvTrue("UNSAFE_DISABLE_HIDDENAPI_FLAGS") { + return + } + + ctx.Strict("INTERNAL_PLATFORM_HIDDENAPI_INDEX", h.index.String()) +} diff --git a/java/jacoco.go b/java/jacoco.go index 8b6d4ac87..9162161d3 100644 --- a/java/jacoco.go +++ b/java/jacoco.go @@ -25,13 +25,14 @@ import ( "github.com/google/blueprint/proptools" "android/soong/android" + "android/soong/java/config" ) var ( jacoco = pctx.AndroidStaticRule("jacoco", blueprint.RuleParams{ Command: `rm -rf $tmpDir && mkdir -p $tmpDir && ` + `${config.Zip2ZipCmd} -i $in -o $strippedJar $stripSpec && ` + - `${config.JavaCmd} -jar ${config.JacocoCLIJar} ` + + `${config.JavaCmd} ${config.JavaVmFlags} -jar ${config.JacocoCLIJar} ` + ` instrument --quiet --dest $tmpDir $strippedJar && ` + `${config.Ziptime} $tmpJar && ` + `${config.MergeZipsCmd} --ignore-duplicates -j $out $tmpJar $in`, @@ -76,7 +77,8 @@ func (j *Module) jacocoModuleToZipCommand(ctx android.ModuleContext) string { if err != nil { ctx.PropertyErrorf("jacoco.include_filter", "%s", err.Error()) } - excludes, err := jacocoFiltersToSpecs(j.properties.Jacoco.Exclude_filter) + // Also include the default list of classes to exclude from instrumentation. + excludes, err := jacocoFiltersToSpecs(append(j.properties.Jacoco.Exclude_filter, config.DefaultJacocoExcludeFilter...)) if err != nil { ctx.PropertyErrorf("jacoco.exclude_filter", "%s", err.Error()) } diff --git a/java/java.go b/java/java.go index 9ac38c92c..69826eec3 100644 --- a/java/java.go +++ b/java/java.go @@ -25,6 +25,7 @@ import ( "strings" "github.com/google/blueprint" + "github.com/google/blueprint/pathtools" "github.com/google/blueprint/proptools" "android/soong/android" @@ -33,23 +34,100 @@ import ( ) func init() { - android.RegisterModuleType("java_defaults", defaultsFactory) + RegisterJavaBuildComponents(android.InitRegistrationContext) + + // Register sdk member types. + android.RegisterSdkMemberType(javaHeaderLibsSdkMemberType) + + android.RegisterSdkMemberType(&librarySdkMemberType{ + android.SdkMemberTypeBase{ + PropertyName: "java_libs", + }, + func(j *Library) android.Path { + implementationJars := j.ImplementationJars() + if len(implementationJars) != 1 { + panic(fmt.Errorf("there must be only one implementation jar from %q", j.Name())) + } + + return implementationJars[0] + }, + }) + + android.RegisterSdkMemberType(&testSdkMemberType{ + SdkMemberTypeBase: android.SdkMemberTypeBase{ + PropertyName: "java_tests", + }, + }) +} + +func RegisterJavaBuildComponents(ctx android.RegistrationContext) { + ctx.RegisterModuleType("java_defaults", DefaultsFactory) + + ctx.RegisterModuleType("java_library", LibraryFactory) + ctx.RegisterModuleType("java_library_static", LibraryStaticFactory) + ctx.RegisterModuleType("java_library_host", LibraryHostFactory) + ctx.RegisterModuleType("java_binary", BinaryFactory) + ctx.RegisterModuleType("java_binary_host", BinaryHostFactory) + ctx.RegisterModuleType("java_test", TestFactory) + ctx.RegisterModuleType("java_test_helper_library", TestHelperLibraryFactory) + ctx.RegisterModuleType("java_test_host", TestHostFactory) + ctx.RegisterModuleType("java_test_import", JavaTestImportFactory) + ctx.RegisterModuleType("java_import", ImportFactory) + ctx.RegisterModuleType("java_import_host", ImportFactoryHost) + ctx.RegisterModuleType("java_device_for_host", DeviceForHostFactory) + ctx.RegisterModuleType("java_host_for_device", HostForDeviceFactory) + ctx.RegisterModuleType("dex_import", DexImportFactory) + + ctx.FinalDepsMutators(func(ctx android.RegisterMutatorsContext) { + ctx.BottomUp("dexpreopt_tool_deps", dexpreoptToolDepsMutator).Parallel() + }) + + ctx.RegisterSingletonType("logtags", LogtagsSingleton) + ctx.RegisterSingletonType("kythe_java_extract", kytheExtractJavaFactory) +} + +func (j *Module) CheckStableSdkVersion() error { + sdkVersion := j.sdkVersion() + if sdkVersion.stable() { + return nil + } + return fmt.Errorf("non stable SDK %v", sdkVersion) +} + +func (j *Module) checkSdkVersions(ctx android.ModuleContext) { + if j.RequiresStableAPIs(ctx) { + if sc, ok := ctx.Module().(sdkContext); ok { + if !sc.sdkVersion().specified() { + ctx.PropertyErrorf("sdk_version", + "sdk_version must have a value when the module is located at vendor or product(only if PRODUCT_ENFORCE_PRODUCT_PARTITION_INTERFACE is set).") + } + } + } + + ctx.VisitDirectDeps(func(module android.Module) { + tag := ctx.OtherModuleDependencyTag(module) + switch module.(type) { + // TODO(satayev): cover other types as well, e.g. imports + case *Library, *AndroidLibrary: + switch tag { + case bootClasspathTag, libTag, staticLibTag, java9LibTag: + checkLinkType(ctx, j, module.(linkTypeContext), tag.(dependencyTag)) + } + } + }) +} - android.RegisterModuleType("java_library", LibraryFactory) - android.RegisterModuleType("java_library_static", LibraryStaticFactory) - android.RegisterModuleType("java_library_host", LibraryHostFactory) - android.RegisterModuleType("java_binary", BinaryFactory) - android.RegisterModuleType("java_binary_host", BinaryHostFactory) - android.RegisterModuleType("java_test", TestFactory) - android.RegisterModuleType("java_test_helper_library", TestHelperLibraryFactory) - android.RegisterModuleType("java_test_host", TestHostFactory) - android.RegisterModuleType("java_import", ImportFactory) - android.RegisterModuleType("java_import_host", ImportFactoryHost) - android.RegisterModuleType("java_device_for_host", DeviceForHostFactory) - android.RegisterModuleType("java_host_for_device", HostForDeviceFactory) - android.RegisterModuleType("dex_import", DexImportFactory) +func (j *Module) checkPlatformAPI(ctx android.ModuleContext) { + if sc, ok := ctx.Module().(sdkContext); ok { + usePlatformAPI := proptools.Bool(j.deviceProperties.Platform_apis) + sdkVersionSpecified := sc.sdkVersion().specified() + if usePlatformAPI && sdkVersionSpecified { + ctx.PropertyErrorf("platform_apis", "platform_apis must be false when sdk_version is not empty.") + } else if !usePlatformAPI && !sdkVersionSpecified { + ctx.PropertyErrorf("platform_apis", "platform_apis must be true when sdk_version is empty.") + } - android.RegisterSingletonType("logtags", LogtagsSingleton) + } } // TODO: @@ -82,13 +160,6 @@ type CompilerProperties struct { // list of files that should be excluded from java_resources and java_resource_dirs Exclude_java_resources []string `android:"path,arch_variant"` - // don't build against the default libraries (bootclasspath, ext, and framework for device - // targets) - No_standard_libs *bool - - // don't build against the framework libraries (ext, and framework for device targets) - No_framework_libs *bool - // list of module-specific flags that will be used for javac compiles Javacflags []string `android:"arch_variant"` @@ -124,6 +195,9 @@ type CompilerProperties struct { // List of modules to use as annotation processors Plugins []string + // List of modules to export to libraries that directly depend on this library as annotation processors + Exported_plugins []string + // The number of Java source entries each Javac instance can process Javac_shard_size *int64 @@ -131,10 +205,10 @@ type CompilerProperties struct { Use_tools_jar *bool Openjdk9 struct { - // List of source files that should only be used when passing -source 1.9 + // List of source files that should only be used when passing -source 1.9 or higher Srcs []string `android:"path"` - // List of javac flags that should only be used when passing -source 1.9 + // List of javac flags that should only be used when passing -source 1.9 or higher Javacflags []string } @@ -179,14 +253,17 @@ type CompilerProperties struct { // List of files to include in the META-INF/services folder of the resulting jar. Services []string `android:"path,arch_variant"` + + // If true, package the kotlin stdlib into the jar. Defaults to true. + Static_kotlin_stdlib *bool `android:"arch_variant"` } type CompilerDeviceProperties struct { // list of module-specific flags that will be used for dex compiles Dxflags []string `android:"arch_variant"` - // if not blank, set to the version of the sdk to compile against. Defaults to compiling against the current - // sdk if platform_apis is not set. + // if not blank, set to the version of the sdk to compile against. + // Defaults to compiling against the current platform. Sdk_version *string // if not blank, set the minimum version of the sdk that the compiled artifacts will run against. @@ -197,7 +274,9 @@ type CompilerDeviceProperties struct { // Defaults to sdk_version if not set. Target_sdk_version *string - // if true, compile against the platform APIs instead of an SDK. + // Whether to compile against the platform APIs instead of an SDK. + // If true, then sdk_version must be empty. The value of this field + // is ignored when module's type isn't android_app. Platform_apis *bool Aidl struct { @@ -259,21 +338,71 @@ type CompilerDeviceProperties struct { Proguard_flags_files []string `android:"path"` } - // When targeting 1.9, override the modules to use with --system + // When targeting 1.9 and above, override the modules to use with --system, + // otherwise provides defaults libraries to add to the bootclasspath. System_modules *string - UncompressDex bool `blueprint:"mutated"` - IsSDKLibrary bool `blueprint:"mutated"` + // The name of the module as used in build configuration. + // + // Allows a library to separate its actual name from the name used in + // build configuration, e.g.ctx.Config().BootJars(). + ConfigurationName *string `blueprint:"mutated"` + + // set the name of the output + Stem *string + + // Keep the data uncompressed. We always need uncompressed dex for execution, + // so this might actually save space by avoiding storing the same data twice. + // This defaults to reasonable value based on module and should not be set. + // It exists only to support ART tests. + Uncompress_dex *bool + + IsSDKLibrary bool `blueprint:"mutated"` + + // If true, generate the signature file of APK Signing Scheme V4, along side the signed APK file. + // Defaults to false. + V4_signature *bool } func (me *CompilerDeviceProperties) EffectiveOptimizeEnabled() bool { return BoolDefault(me.Optimize.Enabled, me.Optimize.EnabledByDefault) } +// Functionality common to Module and Import +// +// It is embedded in Module so its functionality can be used by methods in Module +// but it is currently only initialized by Import and Library. +type embeddableInModuleAndImport struct { + + // Functionality related to this being used as a component of a java_sdk_library. + EmbeddableSdkLibraryComponent +} + +func (e *embeddableInModuleAndImport) initModuleAndImport(moduleBase *android.ModuleBase) { + e.initSdkLibraryComponent(moduleBase) +} + +// Module/Import's DepIsInSameApex(...) delegates to this method. +// +// This cannot implement DepIsInSameApex(...) directly as that leads to ambiguity with +// the one provided by ApexModuleBase. +func (e *embeddableInModuleAndImport) depIsInSameApex(ctx android.BaseModuleContext, dep android.Module) bool { + // dependencies other than the static linkage are all considered crossing APEX boundary + if staticLibTag == ctx.OtherModuleDependencyTag(dep) { + return true + } + return false +} + // Module contains the properties and members used by all java module types type Module struct { android.ModuleBase android.DefaultableModuleBase + android.ApexModuleBase + android.SdkBase + + // Functionality common to Module and Import. + embeddableInModuleAndImport properties CompilerProperties protoProperties android.ProtoProperties @@ -290,6 +419,10 @@ type Module struct { // jar file containing only resources including from static library dependencies resourceJar android.Path + // args and dependencies to package source files into a srcjar + srcJarArgs []string + srcJarDeps android.Paths + // jar file containing implementation classes and resources including static library // dependencies implementationAndResourcesJar android.Path @@ -327,11 +460,17 @@ type Module struct { // manifest file to use instead of properties.Manifest overrideManifest android.OptionalPath - // list of SDK lib names that this java moudule is exporting + // list of SDK lib names that this java module is exporting exportedSdkLibs []string - // list of source files, collected from compiledJavaSrcs and compiledSrcJars - // filter out Exclude_srcs, will be used by android.IDEInfo struct + // list of plugins that this java module is exporting + exportedPluginJars android.Paths + + // list of plugins that this java module is exporting + exportedPluginClasses []string + + // list of source files, collected from srcFiles with unique java and all kt files, + // will be used by android.IDEInfo struct expandIDEInfoCompiledSrcs []string // expanded Jarjar_rules @@ -340,50 +479,77 @@ type Module struct { // list of additional targets for checkbuild additionalCheckedModules android.Paths + // Extra files generated by the module type to be added as java resources. + extraResources android.Paths + hiddenAPI dexpreopter + linter + + // list of the xref extraction files + kytheFiles android.Paths + + distFile android.Path } -func (j *Module) Srcs() android.Paths { - return append(android.Paths{j.outputFile}, j.extraOutputFiles...) +func (j *Module) addHostProperties() { + j.AddProperties( + &j.properties, + &j.protoProperties, + ) } -func (j *Module) DexJarFile() android.Path { - return j.dexJarFile +func (j *Module) addHostAndDeviceProperties() { + j.addHostProperties() + j.AddProperties( + &j.deviceProperties, + &j.dexpreoptProperties, + &j.linter.properties, + ) +} + +func (j *Module) OutputFiles(tag string) (android.Paths, error) { + switch tag { + case "": + return append(android.Paths{j.outputFile}, j.extraOutputFiles...), nil + case ".jar": + return android.Paths{j.implementationAndResourcesJar}, nil + case ".proguard_map": + return android.Paths{j.proguardDictionary}, nil + default: + return nil, fmt.Errorf("unsupported module reference tag %q", tag) + } } -var _ android.SourceFileProducer = (*Module)(nil) +var _ android.OutputFileProducer = (*Module)(nil) -type Dependency interface { +// Methods that need to be implemented for a module that is added to apex java_libs property. +type ApexDependency interface { HeaderJars() android.Paths + ImplementationAndResourcesJars() android.Paths +} + +type Dependency interface { + ApexDependency ImplementationJars() android.Paths ResourceJars() android.Paths - ImplementationAndResourcesJars() android.Paths DexJar() android.Path AidlIncludeDirs() android.Paths ExportedSdkLibs() []string + ExportedPlugins() (android.Paths, []string) + SrcJarArgs() ([]string, android.Paths) + BaseModuleName() string + JacocoReportClassesFile() android.Path } -type SdkLibraryDependency interface { - SdkHeaderJars(ctx android.BaseContext, sdkVersion string) android.Paths - SdkImplementationJars(ctx android.BaseContext, sdkVersion string) android.Paths -} - -type SrcDependency interface { - CompiledSrcs() android.Paths - CompiledSrcJars() android.Paths -} - -func (j *Module) CompiledSrcs() android.Paths { - return j.compiledJavaSrcs +type xref interface { + XrefJavaFiles() android.Paths } -func (j *Module) CompiledSrcJars() android.Paths { - return j.compiledSrcJars +func (j *Module) XrefJavaFiles() android.Paths { + return j.kytheFiles } -var _ SrcDependency = (*Module)(nil) - func InitJavaModule(module android.DefaultableModule, hod android.HostOrDeviceSupported) { android.InitAndroidArchModule(module, hod, android.MultilibCommon) android.InitDefaultableModule(module) @@ -396,110 +562,178 @@ type dependencyTag struct { type jniDependencyTag struct { blueprint.BaseDependencyTag - target android.Target +} + +func IsJniDepTag(depTag blueprint.DependencyTag) bool { + _, ok := depTag.(*jniDependencyTag) + return ok } var ( staticLibTag = dependencyTag{name: "staticlib"} libTag = dependencyTag{name: "javalib"} + java9LibTag = dependencyTag{name: "java9lib"} pluginTag = dependencyTag{name: "plugin"} + exportedPluginTag = dependencyTag{name: "exported-plugin"} bootClasspathTag = dependencyTag{name: "bootclasspath"} systemModulesTag = dependencyTag{name: "system modules"} frameworkResTag = dependencyTag{name: "framework-res"} - frameworkApkTag = dependencyTag{name: "framework-apk"} kotlinStdlibTag = dependencyTag{name: "kotlin-stdlib"} kotlinAnnotationsTag = dependencyTag{name: "kotlin-annotations"} proguardRaiseTag = dependencyTag{name: "proguard-raise"} certificateTag = dependencyTag{name: "certificate"} instrumentationForTag = dependencyTag{name: "instrumentation_for"} + usesLibTag = dependencyTag{name: "uses-library"} + extraLintCheckTag = dependencyTag{name: "extra-lint-check"} ) +func IsLibDepTag(depTag blueprint.DependencyTag) bool { + return depTag == libTag +} + +func IsStaticLibDepTag(depTag blueprint.DependencyTag) bool { + return depTag == staticLibTag +} + type sdkDep struct { useModule, useFiles, useDefaultLibs, invalidVersion bool - modules []string + // The modules that will be added to the bootclasspath when targeting 1.8 or lower + bootclasspath []string + + // The default system modules to use. Will be an empty string if no system + // modules are to be used. systemModules string + // The modules that will be added ot the classpath when targeting 1.9 or higher + java9Classpath []string + frameworkResModule string jars android.Paths aidl android.OptionalPath + + noStandardLibs, noFrameworksLibs bool +} + +func (s sdkDep) hasStandardLibs() bool { + return !s.noStandardLibs +} + +func (s sdkDep) hasFrameworkLibs() bool { + return !s.noStandardLibs && !s.noFrameworksLibs } type jniLib struct { - name string - path android.Path - target android.Target + name string + path android.Path + target android.Target + coverageFile android.OptionalPath + unstrippedFile android.Path } -func (j *Module) shouldInstrument(ctx android.BaseContext) bool { - return j.properties.Instrument && ctx.Config().IsEnvTrue("EMMA_INSTRUMENT") +func (j *Module) shouldInstrument(ctx android.BaseModuleContext) bool { + return j.properties.Instrument && + ctx.Config().IsEnvTrue("EMMA_INSTRUMENT") && + ctx.DeviceConfig().JavaCoverageEnabledForPath(ctx.ModuleDir()) } -func (j *Module) shouldInstrumentStatic(ctx android.BaseContext) bool { +func (j *Module) shouldInstrumentStatic(ctx android.BaseModuleContext) bool { return j.shouldInstrument(ctx) && (ctx.Config().IsEnvTrue("EMMA_INSTRUMENT_STATIC") || ctx.Config().UnbundledBuild()) } -func (j *Module) sdkVersion() string { - return String(j.deviceProperties.Sdk_version) +func (j *Module) sdkVersion() sdkSpec { + return sdkSpecFrom(String(j.deviceProperties.Sdk_version)) +} + +func (j *Module) systemModules() string { + return proptools.String(j.deviceProperties.System_modules) } -func (j *Module) minSdkVersion() string { +func (j *Module) minSdkVersion() sdkSpec { if j.deviceProperties.Min_sdk_version != nil { - return *j.deviceProperties.Min_sdk_version + return sdkSpecFrom(*j.deviceProperties.Min_sdk_version) } return j.sdkVersion() } -func (j *Module) targetSdkVersion() string { +func (j *Module) targetSdkVersion() sdkSpec { if j.deviceProperties.Target_sdk_version != nil { - return *j.deviceProperties.Target_sdk_version + return sdkSpecFrom(*j.deviceProperties.Target_sdk_version) } return j.sdkVersion() } +func (j *Module) MinSdkVersion() string { + return j.minSdkVersion().version.String() +} + +func (j *Module) AvailableFor(what string) bool { + if what == android.AvailableToPlatform && Bool(j.deviceProperties.Hostdex) { + // Exception: for hostdex: true libraries, the platform variant is created + // even if it's not marked as available to platform. In that case, the platform + // variant is used only for the hostdex and not installed to the device. + return true + } + return j.ApexModuleBase.AvailableFor(what) +} + func (j *Module) deps(ctx android.BottomUpMutatorContext) { if ctx.Device() { - if !Bool(j.properties.No_standard_libs) { - sdkDep := decodeSdkDep(ctx, sdkContext(j)) - if sdkDep.useDefaultLibs { - ctx.AddVariationDependencies(nil, bootClasspathTag, config.DefaultBootclasspathLibraries...) - ctx.AddVariationDependencies(nil, systemModulesTag, config.DefaultSystemModules) - if !Bool(j.properties.No_framework_libs) { - ctx.AddVariationDependencies(nil, libTag, config.DefaultLibraries...) - } - } else if sdkDep.useModule { - ctx.AddVariationDependencies(nil, systemModulesTag, sdkDep.systemModules) - ctx.AddVariationDependencies(nil, bootClasspathTag, sdkDep.modules...) - if j.deviceProperties.EffectiveOptimizeEnabled() { - ctx.AddVariationDependencies(nil, proguardRaiseTag, config.DefaultBootclasspathLibraries...) - ctx.AddVariationDependencies(nil, proguardRaiseTag, config.DefaultLibraries...) - } + j.linter.deps(ctx) + + sdkDep := decodeSdkDep(ctx, sdkContext(j)) + if sdkDep.useDefaultLibs { + ctx.AddVariationDependencies(nil, bootClasspathTag, config.DefaultBootclasspathLibraries...) + ctx.AddVariationDependencies(nil, systemModulesTag, config.DefaultSystemModules) + if sdkDep.hasFrameworkLibs() { + ctx.AddVariationDependencies(nil, libTag, config.DefaultLibraries...) + } + } else if sdkDep.useModule { + ctx.AddVariationDependencies(nil, bootClasspathTag, sdkDep.bootclasspath...) + ctx.AddVariationDependencies(nil, java9LibTag, sdkDep.java9Classpath...) + if j.deviceProperties.EffectiveOptimizeEnabled() && sdkDep.hasStandardLibs() { + ctx.AddVariationDependencies(nil, proguardRaiseTag, config.DefaultBootclasspathLibraries...) + ctx.AddVariationDependencies(nil, proguardRaiseTag, config.DefaultLibraries...) } - } else if j.deviceProperties.System_modules == nil { - ctx.PropertyErrorf("no_standard_libs", - "system_modules is required to be set when no_standard_libs is true, did you mean no_framework_libs?") - } else if *j.deviceProperties.System_modules != "none" { - ctx.AddVariationDependencies(nil, systemModulesTag, *j.deviceProperties.System_modules) } - if (ctx.ModuleName() == "framework") || (ctx.ModuleName() == "framework-annotation-proc") { - ctx.AddVariationDependencies(nil, frameworkResTag, "framework-res") + if sdkDep.systemModules != "" { + ctx.AddVariationDependencies(nil, systemModulesTag, sdkDep.systemModules) } - if ctx.ModuleName() == "android_stubs_current" || - ctx.ModuleName() == "android_system_stubs_current" || - ctx.ModuleName() == "android_test_stubs_current" { - ctx.AddVariationDependencies(nil, frameworkApkTag, "framework-res") + } + + syspropPublicStubs := syspropPublicStubs(ctx.Config()) + + // rewriteSyspropLibs validates if a java module can link against platform's sysprop_library, + // and redirects dependency to public stub depending on the link type. + rewriteSyspropLibs := func(libs []string, prop string) []string { + // make a copy + ret := android.CopyOf(libs) + + for idx, lib := range libs { + stub, ok := syspropPublicStubs[lib] + + if !ok { + continue + } + + linkType, _ := j.getLinkType(ctx.ModuleName()) + // only platform modules can use internal props + if linkType != javaPlatform { + ret[idx] = stub + } } + + return ret } - ctx.AddVariationDependencies(nil, libTag, j.properties.Libs...) - ctx.AddVariationDependencies(nil, staticLibTag, j.properties.Static_libs...) + ctx.AddVariationDependencies(nil, libTag, rewriteSyspropLibs(j.properties.Libs, "libs")...) + ctx.AddVariationDependencies(nil, staticLibTag, rewriteSyspropLibs(j.properties.Static_libs, "static_libs")...) - ctx.AddFarVariationDependencies([]blueprint.Variation{ - {Mutator: "arch", Variation: ctx.Config().BuildOsCommonVariant}, - }, pluginTag, j.properties.Plugins...) + ctx.AddFarVariationDependencies(ctx.Config().BuildOSCommonTarget.Variations(), pluginTag, j.properties.Plugins...) + ctx.AddFarVariationDependencies(ctx.Config().BuildOSCommonTarget.Variations(), exportedPluginTag, j.properties.Exported_plugins...) android.ProtoDeps(ctx, &j.protoProperties) if j.hasSrcExt(".proto") { @@ -509,13 +743,21 @@ func (j *Module) deps(ctx android.BottomUpMutatorContext) { if j.hasSrcExt(".kt") { // TODO(ccross): move this to a mutator pass that can tell if generated sources contain // Kotlin files - ctx.AddVariationDependencies(nil, kotlinStdlibTag, "kotlin-stdlib") + ctx.AddVariationDependencies(nil, kotlinStdlibTag, + "kotlin-stdlib", "kotlin-stdlib-jdk7", "kotlin-stdlib-jdk8") if len(j.properties.Plugins) > 0 { ctx.AddVariationDependencies(nil, kotlinAnnotationsTag, "kotlin-annotations") } } - if j.shouldInstrumentStatic(ctx) { + // Framework libraries need special handling in static coverage builds: they should not have + // static dependency on jacoco, otherwise there would be multiple conflicting definitions of + // the same jacoco classes coming from different bootclasspath jars. + if inList(ctx.ModuleName(), config.InstrumentFrameworkModules) { + if ctx.Config().IsEnvTrue("EMMA_INSTRUMENT_FRAMEWORK") { + j.properties.Instrument = true + } + } else if j.shouldInstrumentStatic(ctx) { ctx.AddVariationDependencies(nil, staticLibTag, "jacocoagent") } } @@ -579,6 +821,7 @@ func (j *Module) aidlFlags(ctx android.ModuleContext, aidlPreprocess android.Opt type deps struct { classpath classpath + java9Classpath classpath bootClasspath classpath processorPath classpath processorClasses []string @@ -588,7 +831,7 @@ type deps struct { aidlIncludeDirs android.Paths srcs android.Paths srcJars android.Paths - systemModules android.Path + systemModules *systemModules aidlPreprocess android.OptionalPath kotlinStdlib android.Paths kotlinAnnotations android.Paths @@ -608,53 +851,74 @@ func checkProducesJars(ctx android.ModuleContext, dep android.SourceFileProducer type linkType int const ( + // TODO(jiyong) rename these for better readability. Make the allowed + // and disallowed link types explicit javaCore linkType = iota javaSdk javaSystem + javaModule + javaSystemServer javaPlatform ) -func getLinkType(m *Module, name string) (ret linkType, stubs bool) { - ver := m.sdkVersion() - switch { - case name == "core.current.stubs" || name == "core.platform.api.stubs" || - name == "stub-annotations" || name == "private-stub-annotations-jar" || - name == "core-lambda-stubs": +type linkTypeContext interface { + android.Module + getLinkType(name string) (ret linkType, stubs bool) +} + +func (m *Module) getLinkType(name string) (ret linkType, stubs bool) { + switch name { + case "core.current.stubs", "core.platform.api.stubs", "stub-annotations", + "private-stub-annotations-jar", "core-lambda-stubs", "core-generated-annotation-stubs": return javaCore, true - case ver == "core_current": - return javaCore, false - case name == "android_system_stubs_current": + case "android_stubs_current": + return javaSdk, true + case "android_system_stubs_current": return javaSystem, true - case strings.HasPrefix(ver, "system_"): - return javaSystem, false - case name == "android_test_stubs_current": + case "android_module_lib_stubs_current": + return javaModule, true + case "android_system_server_stubs_current": + return javaSystemServer, true + case "android_test_stubs_current": return javaSystem, true - case strings.HasPrefix(ver, "test_"): - return javaPlatform, false - case name == "android_stubs_current": - return javaSdk, true - case ver == "current": + } + + if stub, linkType := moduleStubLinkType(name); stub { + return linkType, true + } + + ver := m.sdkVersion() + switch ver.kind { + case sdkCore: + return javaCore, false + case sdkSystem: + return javaSystem, false + case sdkPublic: return javaSdk, false - case ver == "": + case sdkModule: + return javaModule, false + case sdkSystemServer: + return javaSystemServer, false + case sdkPrivate, sdkNone, sdkCorePlatform, sdkTest: return javaPlatform, false - default: - if _, err := strconv.Atoi(ver); err != nil { - panic(fmt.Errorf("expected sdk_version to be a number, got %q", ver)) - } - return javaSdk, false } + + if !ver.valid() { + panic(fmt.Errorf("sdk_version is invalid. got %q", ver.raw)) + } + return javaSdk, false } -func checkLinkType(ctx android.ModuleContext, from *Module, to *Library, tag dependencyTag) { +func checkLinkType(ctx android.ModuleContext, from *Module, to linkTypeContext, tag dependencyTag) { if ctx.Host() { return } - myLinkType, stubs := getLinkType(from, ctx.ModuleName()) + myLinkType, stubs := from.getLinkType(ctx.ModuleName()) if stubs { return } - otherLinkType, _ := getLinkType(&to.Module, ctx.OtherModuleName(to)) + otherLinkType, _ := to.getLinkType(ctx.OtherModuleName(to)) commonMessage := "Adjust sdk_version: property of the source or target module so that target module is built with the same or smaller API set than the source." switch myLinkType { @@ -671,11 +935,23 @@ func checkLinkType(ctx android.ModuleContext, from *Module, to *Library, tag dep } break case javaSystem: - if otherLinkType == javaPlatform { + if otherLinkType == javaPlatform || otherLinkType == javaModule || otherLinkType == javaSystemServer { ctx.ModuleErrorf("compiles against system API, but dependency %q is compiling against private API."+commonMessage, ctx.OtherModuleName(to)) } break + case javaModule: + if otherLinkType == javaPlatform || otherLinkType == javaSystemServer { + ctx.ModuleErrorf("compiles against module API, but dependency %q is compiling against private API."+commonMessage, + ctx.OtherModuleName(to)) + } + break + case javaSystemServer: + if otherLinkType == javaPlatform { + ctx.ModuleErrorf("compiles against system server API, but dependency %q is compiling against private API."+commonMessage, + ctx.OtherModuleName(to)) + } + break case javaPlatform: // no restriction on link-type break @@ -688,7 +964,8 @@ func (j *Module) collectDeps(ctx android.ModuleContext) deps { if ctx.Device() { sdkDep := decodeSdkDep(ctx, sdkContext(j)) if sdkDep.invalidVersion { - ctx.AddMissingDependencies(sdkDep.modules) + ctx.AddMissingDependencies(sdkDep.bootclasspath) + ctx.AddMissingDependencies(sdkDep.java9Classpath) } else if sdkDep.useFiles { // sdkDep.jar is actually equivalent to turbine header.jar. deps.classpath = append(deps.classpath, sdkDep.jars...) @@ -698,6 +975,12 @@ func (j *Module) collectDeps(ctx android.ModuleContext) deps { } } + // If this is a component library (stubs, etc.) for a java_sdk_library then + // add the name of that java_sdk_library to the exported sdk libs to make sure + // that, if necessary, a <uses-library> element for that java_sdk_library is + // added to the Android manifest. + j.exportedSdkLibs = append(j.exportedSdkLibs, j.OptionalImplicitSdkLibrary()...) + ctx.VisitDirectDeps(func(module android.Module) { otherName := ctx.OtherModuleName(module) tag := ctx.OtherModuleDependencyTag(module) @@ -711,20 +994,14 @@ func (j *Module) collectDeps(ctx android.ModuleContext) deps { return } - if to, ok := module.(*Library); ok { - switch tag { - case bootClasspathTag, libTag, staticLibTag: - checkLinkType(ctx, j, to, tag.(dependencyTag)) - } - } switch dep := module.(type) { case SdkLibraryDependency: switch tag { case libTag: deps.classpath = append(deps.classpath, dep.SdkHeaderJars(ctx, j.sdkVersion())...) // names of sdk libs that are directly depended are exported - j.exportedSdkLibs = append(j.exportedSdkLibs, otherName) - default: + j.exportedSdkLibs = append(j.exportedSdkLibs, dep.OptionalImplicitSdkLibrary()...) + case staticLibTag: ctx.ModuleErrorf("dependency on java_sdk_library %q can only be in libs", otherName) } case Dependency: @@ -736,6 +1013,10 @@ func (j *Module) collectDeps(ctx android.ModuleContext) deps { // sdk lib names from dependencies are re-exported j.exportedSdkLibs = append(j.exportedSdkLibs, dep.ExportedSdkLibs()...) deps.aidlIncludeDirs = append(deps.aidlIncludeDirs, dep.AidlIncludeDirs()...) + pluginJars, pluginClasses := dep.ExportedPlugins() + addPlugins(&deps, pluginJars, pluginClasses...) + case java9LibTag: + deps.java9Classpath = append(deps.java9Classpath, dep.HeaderJars()...) case staticLibTag: deps.classpath = append(deps.classpath, dep.HeaderJars()...) deps.staticJars = append(deps.staticJars, dep.ImplementationJars()...) @@ -744,36 +1025,33 @@ func (j *Module) collectDeps(ctx android.ModuleContext) deps { // sdk lib names from dependencies are re-exported j.exportedSdkLibs = append(j.exportedSdkLibs, dep.ExportedSdkLibs()...) deps.aidlIncludeDirs = append(deps.aidlIncludeDirs, dep.AidlIncludeDirs()...) + pluginJars, pluginClasses := dep.ExportedPlugins() + addPlugins(&deps, pluginJars, pluginClasses...) case pluginTag: if plugin, ok := dep.(*Plugin); ok { - deps.processorPath = append(deps.processorPath, dep.ImplementationAndResourcesJars()...) if plugin.pluginProperties.Processor_class != nil { - deps.processorClasses = append(deps.processorClasses, *plugin.pluginProperties.Processor_class) + addPlugins(&deps, plugin.ImplementationAndResourcesJars(), *plugin.pluginProperties.Processor_class) + } else { + addPlugins(&deps, plugin.ImplementationAndResourcesJars()) } deps.disableTurbine = deps.disableTurbine || Bool(plugin.pluginProperties.Generates_api) } else { ctx.PropertyErrorf("plugins", "%q is not a java_plugin module", otherName) } - case frameworkResTag: - if (ctx.ModuleName() == "framework") || (ctx.ModuleName() == "framework-annotation-proc") { - // framework.jar has a one-off dependency on the R.java and Manifest.java files - // generated by framework-res.apk - deps.srcJars = append(deps.srcJars, dep.(*AndroidApp).aaptSrcJar) - } - case frameworkApkTag: - if ctx.ModuleName() == "android_stubs_current" || - ctx.ModuleName() == "android_system_stubs_current" || - ctx.ModuleName() == "android_test_stubs_current" { - // framework stubs.jar need to depend on framework-res.apk, in order to pull the - // resource files out of there for aapt. - // - // Normally the package rule runs aapt, which includes the resource, - // but we're not running that in our package rule so just copy in the - // resource files here. - deps.staticResourceJars = append(deps.staticResourceJars, dep.(*AndroidApp).exportPackage) + case exportedPluginTag: + if plugin, ok := dep.(*Plugin); ok { + if plugin.pluginProperties.Generates_api != nil && *plugin.pluginProperties.Generates_api { + ctx.PropertyErrorf("exported_plugins", "Cannot export plugins with generates_api = true, found %v", otherName) + } + j.exportedPluginJars = append(j.exportedPluginJars, plugin.ImplementationAndResourcesJars()...) + if plugin.pluginProperties.Processor_class != nil { + j.exportedPluginClasses = append(j.exportedPluginClasses, *plugin.pluginProperties.Processor_class) + } + } else { + ctx.PropertyErrorf("exported_plugins", "%q is not a java_plugin module", otherName) } case kotlinStdlibTag: - deps.kotlinStdlib = dep.HeaderJars() + deps.kotlinStdlib = append(deps.kotlinStdlib, dep.HeaderJars()...) case kotlinAnnotationsTag: deps.kotlinAnnotations = dep.HeaderJars() } @@ -791,19 +1069,19 @@ func (j *Module) collectDeps(ctx android.ModuleContext) deps { } default: switch tag { - case android.DefaultsDepTag, android.SourceDepTag: - // Nothing to do + case bootClasspathTag: + // If a system modules dependency has been added to the bootclasspath + // then add its libs to the bootclasspath. + sm := module.(SystemModulesProvider) + deps.bootClasspath = append(deps.bootClasspath, sm.HeaderJars()...) + case systemModulesTag: if deps.systemModules != nil { panic("Found two system module dependencies") } - sm := module.(*SystemModules) - if sm.outputFile == nil { - panic("Missing directory for system module dependency") - } - deps.systemModules = sm.outputFile - default: - ctx.ModuleErrorf("depends on non-java module %q", otherName) + sm := module.(SystemModulesProvider) + outputDir, outputDeps := sm.OutputDirAndDeps() + deps.systemModules = &systemModules{outputDir, outputDeps} } } }) @@ -813,37 +1091,68 @@ func (j *Module) collectDeps(ctx android.ModuleContext) deps { return deps } -func getJavaVersion(ctx android.ModuleContext, javaVersion string, sdkContext sdkContext) string { - var ret string - v := sdkContext.sdkVersion() - // For PDK builds, use the latest SDK version instead of "current" - if ctx.Config().IsPdkBuild() && (v == "" || v == "current") { - sdkVersions := ctx.Config().Get(sdkVersionsKey).([]int) - latestSdkVersion := 0 - if len(sdkVersions) > 0 { - latestSdkVersion = sdkVersions[len(sdkVersions)-1] - } - v = strconv.Itoa(latestSdkVersion) - } +func addPlugins(deps *deps, pluginJars android.Paths, pluginClasses ...string) { + deps.processorPath = append(deps.processorPath, pluginJars...) + deps.processorClasses = append(deps.processorClasses, pluginClasses...) +} - sdk, err := sdkVersionToNumber(ctx, v) - if err != nil { - ctx.PropertyErrorf("sdk_version", "%s", err) - } +func getJavaVersion(ctx android.ModuleContext, javaVersion string, sdkContext sdkContext) javaVersion { if javaVersion != "" { - ret = javaVersion - } else if ctx.Device() && sdk <= 23 { - ret = "1.7" - } else if ctx.Device() && sdk <= 28 || !ctx.Config().TargetOpenJDK9() { - ret = "1.8" - } else if ctx.Device() && sdkContext.sdkVersion() != "" && sdk == android.FutureApiLevel { - // TODO(ccross): once we generate stubs we should be able to use 1.9 for sdk_version: "current" - ret = "1.8" + return normalizeJavaVersion(ctx, javaVersion) + } else if ctx.Device() { + return sdkContext.sdkVersion().defaultJavaLanguageVersion(ctx) } else { - ret = "1.9" + return JAVA_VERSION_9 } +} + +type javaVersion int + +const ( + JAVA_VERSION_UNSUPPORTED = 0 + JAVA_VERSION_6 = 6 + JAVA_VERSION_7 = 7 + JAVA_VERSION_8 = 8 + JAVA_VERSION_9 = 9 +) - return ret +func (v javaVersion) String() string { + switch v { + case JAVA_VERSION_6: + return "1.6" + case JAVA_VERSION_7: + return "1.7" + case JAVA_VERSION_8: + return "1.8" + case JAVA_VERSION_9: + return "1.9" + default: + return "unsupported" + } +} + +// Returns true if javac targeting this version uses system modules instead of a bootclasspath. +func (v javaVersion) usesJavaModules() bool { + return v >= 9 +} + +func normalizeJavaVersion(ctx android.BaseModuleContext, javaVersion string) javaVersion { + switch javaVersion { + case "1.6", "6": + return JAVA_VERSION_6 + case "1.7", "7": + return JAVA_VERSION_7 + case "1.8", "8": + return JAVA_VERSION_8 + case "1.9", "9": + return JAVA_VERSION_9 + case "10", "11": + ctx.PropertyErrorf("java_version", "Java language levels above 9 are not supported") + return JAVA_VERSION_UNSUPPORTED + default: + ctx.PropertyErrorf("java_version", "Unrecognized Java language level") + return JAVA_VERSION_UNSUPPORTED + } } func (j *Module) collectBuilderFlags(ctx android.ModuleContext, deps deps) javaBuilderFlags { @@ -855,7 +1164,7 @@ func (j *Module) collectBuilderFlags(ctx android.ModuleContext, deps deps) javaB // javac flags. javacFlags := j.properties.Javacflags - if flags.javaVersion == "1.9" { + if flags.javaVersion.usesJavaModules() { javacFlags = append(javacFlags, j.properties.Openjdk9.Javacflags...) } if ctx.Config().MinimizeJavaDebugInfo() { @@ -863,6 +1172,7 @@ func (j *Module) collectBuilderFlags(ctx android.ModuleContext, deps deps) javaB // disk and memory usage. javacFlags = append(javacFlags, "-g:source,lines") } + javacFlags = append(javacFlags, "-Xlint:-dep-ann") if ctx.Config().RunErrorProne() { if config.ErrorProneClasspath == nil { @@ -883,13 +1193,14 @@ func (j *Module) collectBuilderFlags(ctx android.ModuleContext, deps deps) javaB // classpath flags.bootClasspath = append(flags.bootClasspath, deps.bootClasspath...) flags.classpath = append(flags.classpath, deps.classpath...) + flags.java9Classpath = append(flags.java9Classpath, deps.java9Classpath...) flags.processorPath = append(flags.processorPath, deps.processorPath...) - flags.processor = strings.Join(deps.processorClasses, ",") + flags.processors = append(flags.processors, deps.processorClasses...) + flags.processors = android.FirstUniqueStrings(flags.processors) - if len(flags.bootClasspath) == 0 && ctx.Host() && flags.javaVersion != "1.9" && - !Bool(j.properties.No_standard_libs) && - inList(flags.javaVersion, []string{"1.6", "1.7", "1.8"}) { + if len(flags.bootClasspath) == 0 && ctx.Host() && !flags.javaVersion.usesJavaModules() && + decodeSdkDep(ctx, sdkContext(j)).hasStandardLibs() { // Give host-side tools a version of OpenJDK's standard libraries // close to what they're targeting. As of Dec 2017, AOSP is only // bundling OpenJDK 8 and 9, so nothing < 8 is available. @@ -913,7 +1224,7 @@ func (j *Module) collectBuilderFlags(ctx android.ModuleContext, deps deps) javaB } } - if j.properties.Patch_module != nil && flags.javaVersion == "1.9" { + if j.properties.Patch_module != nil && flags.javaVersion.usesJavaModules() { // Manually specify build directory in case it is not under the repo root. // (javac doesn't seem to expand into symbolc links when searching for patch-module targets, so // just adding a symlink under the root doesn't help.) @@ -926,9 +1237,7 @@ func (j *Module) collectBuilderFlags(ctx android.ModuleContext, deps deps) javaB } // systemModules - if deps.systemModules != nil { - flags.systemModules = append(flags.systemModules, deps.systemModules) - } + flags.systemModules = deps.systemModules // aidl flags. flags.aidlFlags, flags.aidlDeps = j.aidlFlags(ctx, deps.aidlPreprocess, deps.aidlIncludeDirs) @@ -942,14 +1251,13 @@ func (j *Module) collectBuilderFlags(ctx android.ModuleContext, deps deps) javaB return flags } -func (j *Module) compile(ctx android.ModuleContext, extraSrcJars ...android.Path) { - +func (j *Module) compile(ctx android.ModuleContext, aaptSrcJar android.Path) { j.exportAidlIncludeDirs = android.PathsForModuleSrc(ctx, j.deviceProperties.Aidl.Export_include_dirs) deps := j.collectDeps(ctx) flags := j.collectBuilderFlags(ctx, deps) - if flags.javaVersion == "1.9" { + if flags.javaVersion.usesJavaModules() { j.properties.Srcs = append(j.properties.Srcs, j.properties.Openjdk9.Srcs...) } srcFiles := android.PathsForModuleSrcExcludes(ctx, j.properties.Srcs, j.properties.Exclude_srcs) @@ -961,11 +1269,9 @@ func (j *Module) compile(ctx android.ModuleContext, extraSrcJars ...android.Path srcJars := srcFiles.FilterByExt(".srcjar") srcJars = append(srcJars, deps.srcJars...) - srcJars = append(srcJars, extraSrcJars...) - - // Collect source files from compiledJavaSrcs, compiledSrcJars and filter out Exclude_srcs - // that IDEInfo struct will use - j.expandIDEInfoCompiledSrcs = append(j.expandIDEInfoCompiledSrcs, srcFiles.Strings()...) + if aaptSrcJar != nil { + srcJars = append(srcJars, aaptSrcJar) + } if j.properties.Jarjar_rules != nil { j.expandJarjarRules = android.PathForModuleSrc(ctx, *j.properties.Jarjar_rules) @@ -983,6 +1289,9 @@ func (j *Module) compile(ctx android.ModuleContext, extraSrcJars ...android.Path } } + // Collect .java files for AIDEGen + j.expandIDEInfoCompiledSrcs = append(j.expandIDEInfoCompiledSrcs, uniqueSrcFiles.Strings()...) + var kotlinJars android.Paths if srcFiles.HasExt(".kt") { @@ -1007,6 +1316,9 @@ func (j *Module) compile(ctx android.ModuleContext, extraSrcJars ...android.Path kotlinSrcFiles = append(kotlinSrcFiles, uniqueSrcFiles...) kotlinSrcFiles = append(kotlinSrcFiles, srcFiles.FilterByExt(".kt")...) + // Collect .kt files for AIDEGen + j.expandIDEInfoCompiledSrcs = append(j.expandIDEInfoCompiledSrcs, srcFiles.FilterByExt(".kt").Strings()...) + flags.classpath = append(flags.classpath, deps.kotlinStdlib...) flags.classpath = append(flags.classpath, deps.kotlinAnnotations...) @@ -1016,11 +1328,13 @@ func (j *Module) compile(ctx android.ModuleContext, extraSrcJars ...android.Path if len(flags.processorPath) > 0 { // Use kapt for annotation processing kaptSrcJar := android.PathForModuleOut(ctx, "kapt", "kapt-sources.jar") - kotlinKapt(ctx, kaptSrcJar, kotlinSrcFiles, srcJars, flags) + kaptResJar := android.PathForModuleOut(ctx, "kapt", "kapt-res.jar") + kotlinKapt(ctx, kaptSrcJar, kaptResJar, kotlinSrcFiles, srcJars, flags) srcJars = append(srcJars, kaptSrcJar) + kotlinJars = append(kotlinJars, kaptResJar) // Disable annotation processing in javac, it's already been handled by kapt flags.processorPath = nil - flags.processor = "" + flags.processors = nil } kotlinJar := android.PathForModuleOut(ctx, "kotlin", jarName) @@ -1032,9 +1346,11 @@ func (j *Module) compile(ctx android.ModuleContext, extraSrcJars ...android.Path // Make javac rule depend on the kotlinc rule flags.classpath = append(flags.classpath, kotlinJar) - // Jar kotlin classes into the final jar after javac kotlinJars = append(kotlinJars, kotlinJar) - kotlinJars = append(kotlinJars, deps.kotlinStdlib...) + // Jar kotlin classes into the final jar after javac + if BoolDefault(j.properties.Static_kotlin_stdlib, true) { + kotlinJars = append(kotlinJars, deps.kotlinStdlib...) + } } jars := append(android.Paths(nil), kotlinJars...) @@ -1044,6 +1360,7 @@ func (j *Module) compile(ctx android.ModuleContext, extraSrcJars ...android.Path j.compiledSrcJars = srcJars enable_sharding := false + var headerJarFileWithoutJarjar android.Path if ctx.Device() && !ctx.Config().IsEnvFalse("TURBINE_ENABLED") && !deps.disableTurbine { if j.properties.Javac_shard_size != nil && *(j.properties.Javac_shard_size) > 0 { enable_sharding = true @@ -1053,7 +1370,8 @@ func (j *Module) compile(ctx android.ModuleContext, extraSrcJars ...android.Path // allow for the use of annotation processors that do function correctly // with sharding enabled. See: b/77284273. } - j.headerJarFile = j.compileJavaHeader(ctx, uniqueSrcFiles, srcJars, deps, flags, jarName, kotlinJars) + headerJarFileWithoutJarjar, j.headerJarFile = + j.compileJavaHeader(ctx, uniqueSrcFiles, srcJars, deps, flags, jarName, kotlinJars) if ctx.Failed() { return } @@ -1072,25 +1390,24 @@ func (j *Module) compile(ctx android.ModuleContext, extraSrcJars ...android.Path } if enable_sharding { - flags.classpath = append(flags.classpath, j.headerJarFile) + flags.classpath = append(flags.classpath, headerJarFileWithoutJarjar) shardSize := int(*(j.properties.Javac_shard_size)) var shardSrcs []android.Paths if len(uniqueSrcFiles) > 0 { shardSrcs = android.ShardPaths(uniqueSrcFiles, shardSize) for idx, shardSrc := range shardSrcs { - classes := android.PathForModuleOut(ctx, "javac", jarName+strconv.Itoa(idx)) - TransformJavaToClasses(ctx, classes, idx, shardSrc, nil, flags, extraJarDeps) + classes := j.compileJavaClasses(ctx, jarName, idx, shardSrc, + nil, flags, extraJarDeps) jars = append(jars, classes) } } if len(srcJars) > 0 { - classes := android.PathForModuleOut(ctx, "javac", jarName+strconv.Itoa(len(shardSrcs))) - TransformJavaToClasses(ctx, classes, len(shardSrcs), nil, srcJars, flags, extraJarDeps) + classes := j.compileJavaClasses(ctx, jarName, len(shardSrcs), + nil, srcJars, flags, extraJarDeps) jars = append(jars, classes) } } else { - classes := android.PathForModuleOut(ctx, "javac", jarName) - TransformJavaToClasses(ctx, classes, -1, uniqueSrcFiles, srcJars, flags, extraJarDeps) + classes := j.compileJavaClasses(ctx, jarName, -1, uniqueSrcFiles, srcJars, flags, extraJarDeps) jars = append(jars, classes) } if ctx.Failed() { @@ -1098,9 +1415,18 @@ func (j *Module) compile(ctx android.ModuleContext, extraSrcJars ...android.Path } } + j.srcJarArgs, j.srcJarDeps = resourcePathsToJarArgs(srcFiles), srcFiles + + var includeSrcJar android.WritablePath + if Bool(j.properties.Include_srcs) { + includeSrcJar = android.PathForModuleOut(ctx, ctx.ModuleName()+".srcjar") + TransformResourcesToJar(ctx, includeSrcJar, j.srcJarArgs, j.srcJarDeps) + } + dirArgs, dirDeps := ResourceDirsToJarArgs(ctx, j.properties.Java_resource_dirs, j.properties.Exclude_java_resource_dirs, j.properties.Exclude_java_resources) fileArgs, fileDeps := ResourceFilesToJarArgs(ctx, j.properties.Java_resources, j.properties.Exclude_java_resources) + extraArgs, extraDeps := resourcePathsToJarArgs(j.extraResources), j.extraResources var resArgs []string var resDeps android.Paths @@ -1111,11 +1437,8 @@ func (j *Module) compile(ctx android.ModuleContext, extraSrcJars ...android.Path resArgs = append(resArgs, fileArgs...) resDeps = append(resDeps, fileDeps...) - if Bool(j.properties.Include_srcs) { - srcArgs, srcDeps := SourceFilesToJarArgs(ctx, j.properties.Srcs, j.properties.Exclude_srcs) - resArgs = append(resArgs, srcArgs...) - resDeps = append(resDeps, srcDeps...) - } + resArgs = append(resArgs, extraArgs...) + resDeps = append(resDeps, extraDeps...) if len(resArgs) > 0 { resourceJar := android.PathForModuleOut(ctx, "res", jarName) @@ -1126,20 +1449,27 @@ func (j *Module) compile(ctx android.ModuleContext, extraSrcJars ...android.Path } } - if len(deps.staticResourceJars) > 0 { - var jars android.Paths - if j.resourceJar != nil { - jars = append(jars, j.resourceJar) - } - jars = append(jars, deps.staticResourceJars...) + var resourceJars android.Paths + if j.resourceJar != nil { + resourceJars = append(resourceJars, j.resourceJar) + } + if Bool(j.properties.Include_srcs) { + resourceJars = append(resourceJars, includeSrcJar) + } + resourceJars = append(resourceJars, deps.staticResourceJars...) + if len(resourceJars) > 1 { combinedJar := android.PathForModuleOut(ctx, "res-combined", jarName) - TransformJarsToJar(ctx, combinedJar, "for resources", jars, android.OptionalPath{}, + TransformJarsToJar(ctx, combinedJar, "for resources", resourceJars, android.OptionalPath{}, false, nil, nil) j.resourceJar = combinedJar + } else if len(resourceJars) == 1 { + j.resourceJar = resourceJars[0] } - jars = append(jars, deps.staticJars...) + if len(deps.staticJars) > 0 { + jars = append(jars, deps.staticJars...) + } manifest := j.overrideManifest if !manifest.Valid() && j.properties.Manifest != nil { @@ -1154,13 +1484,19 @@ func (j *Module) compile(ctx android.ModuleContext, extraSrcJars ...android.Path serviceFile := file.String() zipargs = append(zipargs, "-C", filepath.Dir(serviceFile), "-f", serviceFile) } + rule := zip + args := map[string]string{ + "jarArgs": "-P META-INF/services/ " + strings.Join(proptools.NinjaAndShellEscapeList(zipargs), " "), + } + if ctx.Config().IsEnvTrue("RBE_ZIP") { + rule = zipRE + args["implicits"] = strings.Join(services.Strings(), ",") + } ctx.Build(pctx, android.BuildParams{ - Rule: zip, + Rule: rule, Output: servicesJar, Implicits: services, - Args: map[string]string{ - "jarArgs": "-P META-INF/services/ " + strings.Join(proptools.NinjaAndShellEscapeList(zipargs), " "), - }, + Args: args, }) jars = append(jars, servicesJar) } @@ -1228,10 +1564,12 @@ func (j *Module) compile(ctx android.ModuleContext, extraSrcJars ...android.Path j.headerJarFile = j.implementationJarFile } - if ctx.Config().IsEnvTrue("EMMA_INSTRUMENT_FRAMEWORK") { - if inList(ctx.ModuleName(), config.InstrumentFrameworkModules) { - j.properties.Instrument = true - } + // Force enable the instrumentation for java code that is built for APEXes ... + // except for the jacocoagent itself (because instrumenting jacocoagent using jacocoagent + // doesn't make sense) + isJacocoAgent := ctx.ModuleName() == "jacocoagent" + if android.DirectlyInAnyApex(ctx, ctx.ModuleName()) && !isJacocoAgent && !j.IsForPlatform() { + j.properties.Instrument = true } if j.shouldInstrument(ctx) { @@ -1241,16 +1579,27 @@ func (j *Module) compile(ctx android.ModuleContext, extraSrcJars ...android.Path // merge implementation jar with resources if necessary implementationAndResourcesJar := outputFile if j.resourceJar != nil { - jars := android.Paths{implementationAndResourcesJar, j.resourceJar} + jars := android.Paths{j.resourceJar, implementationAndResourcesJar} combinedJar := android.PathForModuleOut(ctx, "withres", jarName) - TransformJarsToJar(ctx, combinedJar, "for resources", jars, android.OptionalPath{}, + TransformJarsToJar(ctx, combinedJar, "for resources", jars, manifest, false, nil, nil) implementationAndResourcesJar = combinedJar } j.implementationAndResourcesJar = implementationAndResourcesJar - if ctx.Device() && (Bool(j.properties.Installable) || Bool(j.deviceProperties.Compile_dex)) { + // Enable dex compilation for the APEX variants, unless it is disabled explicitly + if android.DirectlyInAnyApex(ctx, ctx.ModuleName()) && !j.IsForPlatform() { + if j.deviceProperties.Compile_dex == nil { + j.deviceProperties.Compile_dex = proptools.BoolPtr(true) + } + if j.deviceProperties.Hostdex == nil { + j.deviceProperties.Hostdex = proptools.BoolPtr(true) + } + } + + if ctx.Device() && j.hasCode(ctx) && + (Bool(j.properties.Installable) || Bool(j.deviceProperties.Compile_dex)) { // Dex compilation var dexOutputFile android.ModuleOutPath dexOutputFile = j.compileDex(ctx, flags, outputFile, jarName) @@ -1258,9 +1607,12 @@ func (j *Module) compile(ctx android.ModuleContext, extraSrcJars ...android.Path return } + configurationName := j.ConfigurationName() + primary := configurationName == ctx.ModuleName() + // Hidden API CSV generation and dex encoding - dexOutputFile = j.hiddenAPI.hiddenAPI(ctx, dexOutputFile, j.implementationJarFile, - j.deviceProperties.UncompressDex) + dexOutputFile = j.hiddenAPI.hiddenAPI(ctx, configurationName, primary, dexOutputFile, j.implementationJarFile, + proptools.Bool(j.deviceProperties.Uncompress_dex)) // merge dex jar with resources if necessary if j.resourceJar != nil { @@ -1268,7 +1620,7 @@ func (j *Module) compile(ctx android.ModuleContext, extraSrcJars ...android.Path combinedJar := android.PathForModuleOut(ctx, "dex-withres", jarName) TransformJarsToJar(ctx, combinedJar, "for dex resources", jars, android.OptionalPath{}, false, nil, nil) - if j.deviceProperties.UncompressDex { + if *j.deviceProperties.Uncompress_dex { combinedAlignedJar := android.PathForModuleOut(ctx, "dex-withres-aligned", jarName) TransformZipAlign(ctx, combinedAlignedJar, combinedJar) dexOutputFile = combinedAlignedJar @@ -1293,12 +1645,58 @@ func (j *Module) compile(ctx android.ModuleContext, extraSrcJars ...android.Path outputFile = implementationAndResourcesJar } + if ctx.Device() { + lintSDKVersionString := func(sdkSpec sdkSpec) string { + if v := sdkSpec.version; v.isNumbered() { + return v.String() + } else { + return ctx.Config().DefaultAppTargetSdk() + } + } + + j.linter.name = ctx.ModuleName() + j.linter.srcs = srcFiles + j.linter.srcJars = srcJars + j.linter.classpath = append(append(android.Paths(nil), flags.bootClasspath...), flags.classpath...) + j.linter.classes = j.implementationJarFile + j.linter.minSdkVersion = lintSDKVersionString(j.minSdkVersion()) + j.linter.targetSdkVersion = lintSDKVersionString(j.targetSdkVersion()) + j.linter.compileSdkVersion = lintSDKVersionString(j.sdkVersion()) + j.linter.javaLanguageLevel = flags.javaVersion.String() + j.linter.kotlinLanguageLevel = "1.3" + if j.ApexName() != "" && ctx.Config().UnbundledBuild() { + j.linter.buildModuleReportZip = true + } + j.linter.lint(ctx) + } + ctx.CheckbuildFile(outputFile) // Save the output file with no relative path so that it doesn't end up in a subdirectory when used as a resource j.outputFile = outputFile.WithoutRel() } +func (j *Module) compileJavaClasses(ctx android.ModuleContext, jarName string, idx int, + srcFiles, srcJars android.Paths, flags javaBuilderFlags, extraJarDeps android.Paths) android.WritablePath { + + kzipName := pathtools.ReplaceExtension(jarName, "kzip") + if idx >= 0 { + kzipName = strings.TrimSuffix(jarName, filepath.Ext(jarName)) + strconv.Itoa(idx) + ".kzip" + jarName += strconv.Itoa(idx) + } + + classes := android.PathForModuleOut(ctx, "javac", jarName) + TransformJavaToClasses(ctx, classes, idx, srcFiles, srcJars, flags, extraJarDeps) + + if ctx.Config().EmitXrefRules() { + extractionFile := android.PathForModuleOut(ctx, kzipName) + emitXrefRule(ctx, extractionFile, idx, srcFiles, srcJars, flags, extraJarDeps) + j.kytheFiles = append(j.kytheFiles, extractionFile) + } + + return classes +} + // Check for invalid kotlinc flags. Only use this for flags explicitly passed by the user, // since some of these flags may be used internally. func CheckKotlincFlags(ctx android.ModuleContext, flags []string) { @@ -1325,7 +1723,8 @@ func CheckKotlincFlags(ctx android.ModuleContext, flags []string) { } func (j *Module) compileJavaHeader(ctx android.ModuleContext, srcFiles, srcJars android.Paths, - deps deps, flags javaBuilderFlags, jarName string, extraJars android.Paths) android.Path { + deps deps, flags javaBuilderFlags, jarName string, + extraJars android.Paths) (headerJar, jarjarHeaderJar android.Path) { var jars android.Paths if len(srcFiles) > 0 || len(srcJars) > 0 { @@ -1333,7 +1732,7 @@ func (j *Module) compileJavaHeader(ctx android.ModuleContext, srcFiles, srcJars turbineJar := android.PathForModuleOut(ctx, "turbine", jarName) TransformJavaToHeaderClasses(ctx, turbineJar, srcFiles, srcJars, flags) if ctx.Failed() { - return nil + return nil, nil } jars = append(jars, turbineJar) } @@ -1342,27 +1741,27 @@ func (j *Module) compileJavaHeader(ctx android.ModuleContext, srcFiles, srcJars // Combine any static header libraries into classes-header.jar. If there is only // one input jar this step will be skipped. - var headerJar android.Path jars = append(jars, deps.staticHeaderJars...) // we cannot skip the combine step for now if there is only one jar // since we have to strip META-INF/TRANSITIVE dir from turbine.jar combinedJar := android.PathForModuleOut(ctx, "turbine-combined", jarName) TransformJarsToJar(ctx, combinedJar, "for turbine", jars, android.OptionalPath{}, - false, nil, []string{"META-INF"}) + false, nil, []string{"META-INF/TRANSITIVE"}) headerJar = combinedJar + jarjarHeaderJar = combinedJar if j.expandJarjarRules != nil { // Transform classes.jar into classes-jarjar.jar jarjarFile := android.PathForModuleOut(ctx, "turbine-jarjar", jarName) TransformJarJar(ctx, jarjarFile, headerJar, j.expandJarjarRules) - headerJar = jarjarFile + jarjarHeaderJar = jarjarFile if ctx.Failed() { - return nil + return nil, nil } } - return headerJar + return headerJar, jarjarHeaderJar } func (j *Module) instrument(ctx android.ModuleContext, flags javaBuilderFlags, @@ -1424,6 +1823,14 @@ func (j *Module) ExportedSdkLibs() []string { return j.exportedSdkLibs } +func (j *Module) ExportedPlugins() (android.Paths, []string) { + return j.exportedPluginJars, j.exportedPluginClasses +} + +func (j *Module) SrcJarArgs() ([]string, android.Paths) { + return j.srcJarArgs, j.srcJarDeps +} + var _ logtagsProducer = (*Module)(nil) func (j *Module) logtags() android.Paths { @@ -1434,6 +1841,7 @@ func (j *Module) logtags() android.Paths { func (j *Module) IDEInfo(dpInfo *android.IdeInfo) { dpInfo.Deps = append(dpInfo.Deps, j.CompilerDeps()...) dpInfo.Srcs = append(dpInfo.Srcs, j.expandIDEInfoCompiledSrcs...) + dpInfo.SrcJars = append(dpInfo.SrcJars, j.compiledSrcJars.Strings()...) dpInfo.Aidl_include_dirs = append(dpInfo.Aidl_include_dirs, j.deviceProperties.Aidl.Include_dirs...) if j.expandJarjarRules != nil { dpInfo.Jarjar_rules = append(dpInfo.Jarjar_rules, j.expandJarjarRules.String()) @@ -1447,15 +1855,67 @@ func (j *Module) CompilerDeps() []string { return jdeps } +func (j *Module) hasCode(ctx android.ModuleContext) bool { + srcFiles := android.PathsForModuleSrcExcludes(ctx, j.properties.Srcs, j.properties.Exclude_srcs) + return len(srcFiles) > 0 || len(ctx.GetDirectDepsWithTag(staticLibTag)) > 0 +} + +func (j *Module) DepIsInSameApex(ctx android.BaseModuleContext, dep android.Module) bool { + return j.depIsInSameApex(ctx, dep) +} + +func (j *Module) Stem() string { + return proptools.StringDefault(j.deviceProperties.Stem, j.Name()) +} + +func (j *Module) ConfigurationName() string { + return proptools.StringDefault(j.deviceProperties.ConfigurationName, j.BaseModuleName()) +} + +func (j *Module) JacocoReportClassesFile() android.Path { + return j.jacocoReportClassesFile +} + +func (j *Module) IsInstallable() bool { + return Bool(j.properties.Installable) +} + // // Java libraries (.jar file) // +type LibraryProperties struct { + Dist struct { + // The tag of the output of this module that should be output. + Tag *string `android:"arch_variant"` + } `android:"arch_variant"` +} + type Library struct { Module + + libraryProperties LibraryProperties + + InstallMixin func(ctx android.ModuleContext, installPath android.Path) (extraInstallDeps android.Paths) +} + +// Provides access to the list of permitted packages from updatable boot jars. +type PermittedPackagesForUpdatableBootJars interface { + PermittedPackagesForUpdatableBootJars() []string +} + +var _ PermittedPackagesForUpdatableBootJars = (*Library)(nil) + +func (j *Library) PermittedPackagesForUpdatableBootJars() []string { + return j.properties.Permitted_packages } func shouldUncompressDex(ctx android.ModuleContext, dexpreopter *dexpreopter) bool { + // Store uncompressed (and aligned) any dex files from jars in APEXes. + if am, ok := ctx.Module().(android.ApexModule); ok && !am.IsForPlatform() { + return true + } + // Store uncompressed (and do not strip) dex files from boot class path jars. if inList(ctx.ModuleName(), ctx.Config().BootJars()) { return true @@ -1474,16 +1934,33 @@ func shouldUncompressDex(ctx android.ModuleContext, dexpreopter *dexpreopter) bo } func (j *Library) GenerateAndroidBuildActions(ctx android.ModuleContext) { - j.dexpreopter.installPath = android.PathForModuleInstall(ctx, "framework", ctx.ModuleName()+".jar") + j.checkSdkVersions(ctx) + j.dexpreopter.installPath = android.PathForModuleInstall(ctx, "framework", j.Stem()+".jar") j.dexpreopter.isSDKLibrary = j.deviceProperties.IsSDKLibrary - j.dexpreopter.isInstallable = Bool(j.properties.Installable) - j.dexpreopter.uncompressedDex = shouldUncompressDex(ctx, &j.dexpreopter) - j.deviceProperties.UncompressDex = j.dexpreopter.uncompressedDex - j.compile(ctx) - - if (Bool(j.properties.Installable) || ctx.Host()) && !android.DirectlyInAnyApex(ctx, ctx.ModuleName()) { + if j.deviceProperties.Uncompress_dex == nil { + // If the value was not force-set by the user, use reasonable default based on the module. + j.deviceProperties.Uncompress_dex = proptools.BoolPtr(shouldUncompressDex(ctx, &j.dexpreopter)) + } + j.dexpreopter.uncompressedDex = *j.deviceProperties.Uncompress_dex + j.compile(ctx, nil) + + exclusivelyForApex := android.InAnyApex(ctx.ModuleName()) && !j.IsForPlatform() + if (Bool(j.properties.Installable) || ctx.Host()) && !exclusivelyForApex { + var extraInstallDeps android.Paths + if j.InstallMixin != nil { + extraInstallDeps = j.InstallMixin(ctx, j.outputFile) + } j.installFile = ctx.InstallFile(android.PathForModuleInstall(ctx, "framework"), - ctx.ModuleName()+".jar", j.outputFile) + j.Stem()+".jar", j.outputFile, extraInstallDeps...) + } + + // Verify Dist.Tag is set to a supported output + if j.libraryProperties.Dist.Tag != nil { + distFiles, err := j.OutputFiles(*j.libraryProperties.Dist.Tag) + if err != nil { + ctx.PropertyErrorf("dist.tag", "%s", err.Error()) + } + j.distFile = distFiles[0] } } @@ -1491,6 +1968,102 @@ func (j *Library) DepsMutator(ctx android.BottomUpMutatorContext) { j.deps(ctx) } +const ( + aidlIncludeDir = "aidl" + javaDir = "java" + jarFileSuffix = ".jar" + testConfigSuffix = "-AndroidTest.xml" +) + +// path to the jar file of a java library. Relative to <sdk_root>/<api_dir> +func sdkSnapshotFilePathForJar(osPrefix, name string) string { + return sdkSnapshotFilePathForMember(osPrefix, name, jarFileSuffix) +} + +func sdkSnapshotFilePathForMember(osPrefix, name string, suffix string) string { + return filepath.Join(javaDir, osPrefix, name+suffix) +} + +type librarySdkMemberType struct { + android.SdkMemberTypeBase + + // Function to retrieve the appropriate output jar (implementation or header) from + // the library. + jarToExportGetter func(j *Library) android.Path +} + +func (mt *librarySdkMemberType) AddDependencies(mctx android.BottomUpMutatorContext, dependencyTag blueprint.DependencyTag, names []string) { + mctx.AddVariationDependencies(nil, dependencyTag, names...) +} + +func (mt *librarySdkMemberType) IsInstance(module android.Module) bool { + _, ok := module.(*Library) + return ok +} + +func (mt *librarySdkMemberType) AddPrebuiltModule(ctx android.SdkMemberContext, member android.SdkMember) android.BpModule { + return ctx.SnapshotBuilder().AddPrebuiltModule(member, "java_import") +} + +func (mt *librarySdkMemberType) CreateVariantPropertiesStruct() android.SdkMemberProperties { + return &librarySdkMemberProperties{} +} + +type librarySdkMemberProperties struct { + android.SdkMemberPropertiesBase + + JarToExport android.Path `android:"arch_variant"` + AidlIncludeDirs android.Paths +} + +func (p *librarySdkMemberProperties) PopulateFromVariant(ctx android.SdkMemberContext, variant android.Module) { + j := variant.(*Library) + + p.JarToExport = ctx.MemberType().(*librarySdkMemberType).jarToExportGetter(j) + p.AidlIncludeDirs = j.AidlIncludeDirs() +} + +func (p *librarySdkMemberProperties) AddToPropertySet(ctx android.SdkMemberContext, propertySet android.BpPropertySet) { + builder := ctx.SnapshotBuilder() + + exportedJar := p.JarToExport + if exportedJar != nil { + snapshotRelativeJavaLibPath := sdkSnapshotFilePathForJar(p.OsPrefix(), ctx.Name()) + builder.CopyToSnapshot(exportedJar, snapshotRelativeJavaLibPath) + + propertySet.AddProperty("jars", []string{snapshotRelativeJavaLibPath}) + } + + aidlIncludeDirs := p.AidlIncludeDirs + if len(aidlIncludeDirs) != 0 { + sdkModuleContext := ctx.SdkModuleContext() + for _, dir := range aidlIncludeDirs { + // TODO(jiyong): copy parcelable declarations only + aidlFiles, _ := sdkModuleContext.GlobWithDeps(dir.String()+"/**/*.aidl", nil) + for _, file := range aidlFiles { + builder.CopyToSnapshot(android.PathForSource(sdkModuleContext, file), filepath.Join(aidlIncludeDir, file)) + } + } + + // TODO(b/151933053) - add aidl include dirs property + } +} + +var javaHeaderLibsSdkMemberType android.SdkMemberType = &librarySdkMemberType{ + android.SdkMemberTypeBase{ + PropertyName: "java_header_libs", + SupportsSdk: true, + }, + func(j *Library) android.Path { + headerJars := j.HeaderJars() + if len(headerJars) != 1 { + panic(fmt.Errorf("there must be only one header jar from %q", j.Name())) + } + + return headerJars[0] + }, +} + // java_library builds and links sources into a `.jar` file for the device, and possibly for the host as well. // // By default, a java_library has a single variant that produces a `.jar` file containing `.class` files that were @@ -1505,12 +2078,13 @@ func (j *Library) DepsMutator(ctx android.BottomUpMutatorContext) { func LibraryFactory() android.Module { module := &Library{} - module.AddProperties( - &module.Module.properties, - &module.Module.deviceProperties, - &module.Module.dexpreoptProperties, - &module.Module.protoProperties) + module.addHostAndDeviceProperties() + module.AddProperties(&module.libraryProperties) + + module.initModuleAndImport(&module.ModuleBase) + android.InitApexModule(module) + android.InitSdkAwareModule(module) InitJavaModule(module, android.HostAndDeviceSupported) return module } @@ -1527,12 +2101,11 @@ func LibraryStaticFactory() android.Module { func LibraryHostFactory() android.Module { module := &Library{} - module.AddProperties( - &module.Module.properties, - &module.Module.protoProperties) + module.addHostProperties() module.Module.properties.Installable = proptools.BoolPtr(true) + android.InitApexModule(module) InitJavaModule(module, android.HostSupported) return module } @@ -1557,6 +2130,15 @@ type testProperties struct { // list of files or filegroup modules that provide data that should be installed alongside // the test Data []string `android:"path"` + + // Flag to indicate whether or not to create test config automatically. If AndroidTest.xml + // doesn't exist next to the Android.bp, this attribute doesn't need to be set to true + // explicitly. + Auto_gen_config *bool + + // Add parameterized mainline modules to auto generated test config. The options will be + // handled by TradeFed to do downloading and installing the specified modules on the device. + Test_mainline_modules []string } type testHelperLibraryProperties struct { @@ -1565,6 +2147,16 @@ type testHelperLibraryProperties struct { Test_suites []string `android:"arch_variant"` } +type prebuiltTestProperties struct { + // list of compatibility suites (for example "cts", "vts") that the module should be + // installed into. + Test_suites []string `android:"arch_variant"` + + // the name of the test configuration (for example "AndroidTest.xml") that should be + // installed with the module. + Test_config *string `android:"path,arch_variant"` +} + type Test struct { Library @@ -1580,8 +2172,17 @@ type TestHelperLibrary struct { testHelperLibraryProperties testHelperLibraryProperties } +type JavaTestImport struct { + Import + + prebuiltTestProperties prebuiltTestProperties + + testConfig android.Path +} + func (j *Test) GenerateAndroidBuildActions(ctx android.ModuleContext) { - j.testConfig = tradefed.AutoGenJavaTestConfig(ctx, j.testProperties.Test_config, j.testProperties.Test_config_template, j.testProperties.Test_suites) + j.testConfig = tradefed.AutoGenJavaTestConfig(ctx, j.testProperties.Test_config, j.testProperties.Test_config_template, + j.testProperties.Test_suites, j.testProperties.Auto_gen_config) j.data = android.PathsForModuleSrc(ctx, j.testProperties.Data) j.Library.GenerateAndroidBuildActions(ctx) @@ -1591,6 +2192,72 @@ func (j *TestHelperLibrary) GenerateAndroidBuildActions(ctx android.ModuleContex j.Library.GenerateAndroidBuildActions(ctx) } +func (j *JavaTestImport) GenerateAndroidBuildActions(ctx android.ModuleContext) { + j.testConfig = tradefed.AutoGenJavaTestConfig(ctx, j.prebuiltTestProperties.Test_config, nil, + j.prebuiltTestProperties.Test_suites, nil) + + j.Import.GenerateAndroidBuildActions(ctx) +} + +type testSdkMemberType struct { + android.SdkMemberTypeBase +} + +func (mt *testSdkMemberType) AddDependencies(mctx android.BottomUpMutatorContext, dependencyTag blueprint.DependencyTag, names []string) { + mctx.AddVariationDependencies(nil, dependencyTag, names...) +} + +func (mt *testSdkMemberType) IsInstance(module android.Module) bool { + _, ok := module.(*Test) + return ok +} + +func (mt *testSdkMemberType) AddPrebuiltModule(ctx android.SdkMemberContext, member android.SdkMember) android.BpModule { + return ctx.SnapshotBuilder().AddPrebuiltModule(member, "java_test_import") +} + +func (mt *testSdkMemberType) CreateVariantPropertiesStruct() android.SdkMemberProperties { + return &testSdkMemberProperties{} +} + +type testSdkMemberProperties struct { + android.SdkMemberPropertiesBase + + JarToExport android.Path + TestConfig android.Path +} + +func (p *testSdkMemberProperties) PopulateFromVariant(ctx android.SdkMemberContext, variant android.Module) { + test := variant.(*Test) + + implementationJars := test.ImplementationJars() + if len(implementationJars) != 1 { + panic(fmt.Errorf("there must be only one implementation jar from %q", test.Name())) + } + + p.JarToExport = implementationJars[0] + p.TestConfig = test.testConfig +} + +func (p *testSdkMemberProperties) AddToPropertySet(ctx android.SdkMemberContext, propertySet android.BpPropertySet) { + builder := ctx.SnapshotBuilder() + + exportedJar := p.JarToExport + if exportedJar != nil { + snapshotRelativeJavaLibPath := sdkSnapshotFilePathForJar(p.OsPrefix(), ctx.Name()) + builder.CopyToSnapshot(exportedJar, snapshotRelativeJavaLibPath) + + propertySet.AddProperty("jars", []string{snapshotRelativeJavaLibPath}) + } + + testConfig := p.TestConfig + if testConfig != nil { + snapshotRelativeTestConfigPath := sdkSnapshotFilePathForMember(p.OsPrefix(), ctx.Name(), testConfigSuffix) + builder.CopyToSnapshot(testConfig, snapshotRelativeTestConfigPath) + propertySet.AddProperty("test_config", snapshotRelativeTestConfigPath) + } +} + // java_test builds a and links sources into a `.jar` file for the device, and possibly for the host as well, and // creates an `AndroidTest.xml` file to allow running the test with `atest` or a `TEST_MAPPING` file. // @@ -1602,15 +2269,12 @@ func (j *TestHelperLibrary) GenerateAndroidBuildActions(ctx android.ModuleContex func TestFactory() android.Module { module := &Test{} - module.AddProperties( - &module.Module.properties, - &module.Module.deviceProperties, - &module.Module.dexpreoptProperties, - &module.Module.protoProperties, - &module.testProperties) + module.addHostAndDeviceProperties() + module.AddProperties(&module.testProperties) module.Module.properties.Installable = proptools.BoolPtr(true) module.Module.dexpreopter.isTest = true + module.Module.linter.test = true InitJavaModule(module, android.HostAndDeviceSupported) return module @@ -1620,13 +2284,37 @@ func TestFactory() android.Module { func TestHelperLibraryFactory() android.Module { module := &TestHelperLibrary{} + module.addHostAndDeviceProperties() + module.AddProperties(&module.testHelperLibraryProperties) + + module.Module.properties.Installable = proptools.BoolPtr(true) + module.Module.dexpreopter.isTest = true + module.Module.linter.test = true + + InitJavaModule(module, android.HostAndDeviceSupported) + return module +} + +// java_test_import imports one or more `.jar` files into the build graph as if they were built by a java_test module +// and makes sure that it is added to the appropriate test suite. +// +// By default, a java_test_import has a single variant that expects a `.jar` file containing `.class` files that were +// compiled against an Android classpath. +// +// Specifying `host_supported: true` will produce two variants, one for use as a dependency of device modules and one +// for host modules. +func JavaTestImportFactory() android.Module { + module := &JavaTestImport{} + module.AddProperties( - &module.Module.properties, - &module.Module.deviceProperties, - &module.Module.dexpreoptProperties, - &module.Module.protoProperties, - &module.testHelperLibraryProperties) + &module.Import.properties, + &module.prebuiltTestProperties) + + module.Import.properties.Installable = proptools.BoolPtr(true) + android.InitPrebuiltModule(module, &module.properties.Jars) + android.InitApexModule(module) + android.InitSdkAwareModule(module) InitJavaModule(module, android.HostAndDeviceSupported) return module } @@ -1639,10 +2327,8 @@ func TestHelperLibraryFactory() android.Module { func TestHostFactory() android.Module { module := &Test{} - module.AddProperties( - &module.Module.properties, - &module.Module.protoProperties, - &module.testProperties) + module.addHostProperties() + module.AddProperties(&module.testProperties) module.Module.properties.Installable = proptools.BoolPtr(true) @@ -1670,7 +2356,7 @@ type Binary struct { isWrapperVariant bool wrapperFile android.Path - binaryFile android.OutputPath + binaryFile android.InstallPath } func (j *Binary) HostToolPath() android.OptionalPath { @@ -1726,12 +2412,8 @@ func (j *Binary) DepsMutator(ctx android.BottomUpMutatorContext) { func BinaryFactory() android.Module { module := &Binary{} - module.AddProperties( - &module.Module.properties, - &module.Module.deviceProperties, - &module.Module.dexpreoptProperties, - &module.Module.protoProperties, - &module.binaryProperties) + module.addHostAndDeviceProperties() + module.AddProperties(&module.binaryProperties) module.Module.properties.Installable = proptools.BoolPtr(true) @@ -1747,10 +2429,8 @@ func BinaryFactory() android.Module { func BinaryHostFactory() android.Module { module := &Binary{} - module.AddProperties( - &module.Module.properties, - &module.Module.protoProperties, - &module.binaryProperties) + module.addHostProperties() + module.AddProperties(&module.binaryProperties) module.Module.properties.Installable = proptools.BoolPtr(true) @@ -1764,7 +2444,7 @@ func BinaryHostFactory() android.Module { // type ImportProperties struct { - Jars []string `android:"path"` + Jars []string `android:"path,arch_variant"` Sdk_version *string @@ -1781,12 +2461,20 @@ type ImportProperties struct { // if set to true, run Jetifier against .jar file. Defaults to false. Jetifier *bool + + // set the name of the output + Stem *string } type Import struct { android.ModuleBase android.DefaultableModuleBase + android.ApexModuleBase prebuilt android.Prebuilt + android.SdkBase + + // Functionality common to Module and Import. + embeddableInModuleAndImport properties ImportProperties @@ -1794,14 +2482,18 @@ type Import struct { exportedSdkLibs []string } -func (j *Import) sdkVersion() string { - return String(j.properties.Sdk_version) +func (j *Import) sdkVersion() sdkSpec { + return sdkSpecFrom(String(j.properties.Sdk_version)) } -func (j *Import) minSdkVersion() string { +func (j *Import) minSdkVersion() sdkSpec { return j.sdkVersion() } +func (j *Import) MinSdkVersion() string { + return j.minSdkVersion().version.String() +} + func (j *Import) Prebuilt() *android.Prebuilt { return &j.prebuilt } @@ -1814,6 +2506,14 @@ func (j *Import) Name() string { return j.prebuilt.Name(j.ModuleBase.Name()) } +func (j *Import) Stem() string { + return proptools.StringDefault(j.properties.Stem, j.ModuleBase.Name()) +} + +func (a *Import) JacocoReportClassesFile() android.Path { + return nil +} + func (j *Import) DepsMutator(ctx android.BottomUpMutatorContext) { ctx.AddVariationDependencies(nil, libTag, j.properties.Libs...) } @@ -1821,7 +2521,7 @@ func (j *Import) DepsMutator(ctx android.BottomUpMutatorContext) { func (j *Import) GenerateAndroidBuildActions(ctx android.ModuleContext) { jars := android.PathsForModuleSrc(ctx, j.properties.Jars) - jarName := ctx.ModuleName() + ".jar" + jarName := j.Stem() + ".jar" outputFile := android.PathForModuleOut(ctx, "combined", jarName) TransformJarsToJar(ctx, outputFile, "for prebuilts", jars, android.OptionalPath{}, false, j.properties.Exclude_files, j.properties.Exclude_dirs) @@ -1832,6 +2532,12 @@ func (j *Import) GenerateAndroidBuildActions(ctx android.ModuleContext) { } j.combinedClasspathFile = outputFile + // If this is a component library (impl, stubs, etc.) for a java_sdk_library then + // add the name of that java_sdk_library to the exported sdk libs to make sure + // that, if necessary, a <uses-library> element for that java_sdk_library is + // added to the Android manifest. + j.exportedSdkLibs = append(j.exportedSdkLibs, j.OptionalImplicitSdkLibrary()...) + ctx.VisitDirectDeps(func(module android.Module) { otherName := ctx.OtherModuleName(module) tag := ctx.OtherModuleDependencyTag(module) @@ -1855,7 +2561,7 @@ func (j *Import) GenerateAndroidBuildActions(ctx android.ModuleContext) { j.exportedSdkLibs = android.FirstUniqueStrings(j.exportedSdkLibs) if Bool(j.properties.Installable) { ctx.InstallFile(android.PathForModuleInstall(ctx, "framework"), - ctx.ModuleName()+".jar", outputFile) + jarName, outputFile) } } @@ -1898,6 +2604,18 @@ func (j *Import) ExportedSdkLibs() []string { return j.exportedSdkLibs } +func (j *Import) ExportedPlugins() (android.Paths, []string) { + return nil, nil +} + +func (j *Import) SrcJarArgs() ([]string, android.Paths) { + return nil, nil +} + +func (j *Import) DepIsInSameApex(ctx android.BaseModuleContext, dep android.Module) bool { + return j.depIsInSameApex(ctx, dep) +} + // Add compile time check for interface implementation var _ android.IDEInfo = (*Import)(nil) var _ android.IDECustomizedModuleName = (*Import)(nil) @@ -1936,7 +2654,11 @@ func ImportFactory() android.Module { module.AddProperties(&module.properties) + module.initModuleAndImport(&module.ModuleBase) + android.InitPrebuiltModule(module, &module.properties.Jars) + android.InitApexModule(module) + android.InitSdkAwareModule(module) InitJavaModule(module, android.HostAndDeviceSupported) return module } @@ -1952,6 +2674,7 @@ func ImportFactoryHost() android.Module { module.AddProperties(&module.properties) android.InitPrebuiltModule(module, &module.properties.Jars) + android.InitApexModule(module) InitJavaModule(module, android.HostSupported) return module } @@ -1959,12 +2682,16 @@ func ImportFactoryHost() android.Module { // dex_import module type DexImportProperties struct { - Jars []string + Jars []string `android:"path"` + + // set the name of the output + Stem *string } type DexImport struct { android.ModuleBase android.DefaultableModuleBase + android.ApexModuleBase prebuilt android.Prebuilt properties DexImportProperties @@ -1987,8 +2714,20 @@ func (j *DexImport) Name() string { return j.prebuilt.Name(j.ModuleBase.Name()) } -func (j *DexImport) DepsMutator(ctx android.BottomUpMutatorContext) { - android.ExtractSourcesDeps(ctx, j.properties.Jars) +func (j *DexImport) Stem() string { + return proptools.StringDefault(j.properties.Stem, j.ModuleBase.Name()) +} + +func (a *DexImport) JacocoReportClassesFile() android.Path { + return nil +} + +func (a *DexImport) LintDepSets() LintDepSets { + return LintDepSets{} +} + +func (j *DexImport) IsInstallable() bool { + return true } func (j *DexImport) GenerateAndroidBuildActions(ctx android.ModuleContext) { @@ -1996,8 +2735,7 @@ func (j *DexImport) GenerateAndroidBuildActions(ctx android.ModuleContext) { ctx.PropertyErrorf("jars", "exactly one jar must be provided") } - j.dexpreopter.installPath = android.PathForModuleInstall(ctx, "framework", ctx.ModuleName()+".jar") - j.dexpreopter.isInstallable = true + j.dexpreopter.installPath = android.PathForModuleInstall(ctx, "framework", j.Stem()+".jar") j.dexpreopter.uncompressedDex = shouldUncompressDex(ctx, &j.dexpreopter) inputJar := ctx.ExpandSource(j.properties.Jars[0], "jars") @@ -2011,14 +2749,14 @@ func (j *DexImport) GenerateAndroidBuildActions(ctx android.ModuleContext) { // use zip2zip to uncompress classes*.dex files rule.Command(). - Tool(ctx.Config().HostToolPath(ctx, "zip2zip")). + BuiltTool(ctx, "zip2zip"). FlagWithInput("-i ", inputJar). FlagWithOutput("-o ", temporary). FlagWithArg("-0 ", "'classes*.dex'") // use zipalign to align uncompressed classes*.dex files rule.Command(). - Tool(ctx.Config().HostToolPath(ctx, "zipalign")). + BuiltTool(ctx, "zipalign"). Flag("-f"). Text("4"). Input(temporary). @@ -2041,8 +2779,10 @@ func (j *DexImport) GenerateAndroidBuildActions(ctx android.ModuleContext) { j.maybeStrippedDexJarFile = dexOutputFile - ctx.InstallFile(android.PathForModuleInstall(ctx, "framework"), - ctx.ModuleName()+".jar", dexOutputFile) + if j.IsForPlatform() { + ctx.InstallFile(android.PathForModuleInstall(ctx, "framework"), + j.Stem()+".jar", dexOutputFile) + } } func (j *DexImport) DexJar() android.Path { @@ -2059,6 +2799,7 @@ func DexImportFactory() android.Module { module.AddProperties(&module.properties) android.InitPrebuiltModule(module, &module.properties.Jars) + android.InitApexModule(module) InitJavaModule(module, android.DeviceSupported) return module } @@ -2069,9 +2810,7 @@ func DexImportFactory() android.Module { type Defaults struct { android.ModuleBase android.DefaultsModuleBase -} - -func (*Defaults) GenerateAndroidBuildActions(ctx android.ModuleContext) { + android.ApexModuleBase } // java_defaults provides a set of properties that can be inherited by other java or android modules. @@ -2109,10 +2848,9 @@ func defaultsFactory() android.Module { return DefaultsFactory() } -func DefaultsFactory(props ...interface{}) android.Module { +func DefaultsFactory() android.Module { module := &Defaults{} - module.AddProperties(props...) module.AddProperties( &CompilerProperties{}, &CompilerDeviceProperties{}, @@ -2126,14 +2864,37 @@ func DefaultsFactory(props ...interface{}) android.Module { &ImportProperties{}, &AARImportProperties{}, &sdkLibraryProperties{}, + &commonToSdkLibraryAndImportProperties{}, &DexImportProperties{}, + &android.ApexProperties{}, + &RuntimeResourceOverlayProperties{}, + &LintProperties{}, ) android.InitDefaultsModule(module) - return module } +func kytheExtractJavaFactory() android.Singleton { + return &kytheExtractJavaSingleton{} +} + +type kytheExtractJavaSingleton struct { +} + +func (ks *kytheExtractJavaSingleton) GenerateBuildActions(ctx android.SingletonContext) { + var xrefTargets android.Paths + ctx.VisitAllModules(func(module android.Module) { + if javaModule, ok := module.(xref); ok { + xrefTargets = append(xrefTargets, javaModule.XrefJavaFiles()...) + } + }) + // TODO(asmundak): perhaps emit a rule to output a warning if there were no xrefTargets + if len(xrefTargets) > 0 { + ctx.Phony("xref_java", xrefTargets...) + } +} + var Bool = proptools.Bool var BoolDefault = proptools.BoolDefault var String = proptools.String diff --git a/java/java_resources.go b/java/java_resources.go index 71611689a..787d74a0d 100644 --- a/java/java_resources.go +++ b/java/java_resources.go @@ -85,19 +85,19 @@ func ResourceFilesToJarArgs(ctx android.ModuleContext, return resourceFilesToJarArgs(ctx, res, exclude) } -// Convert java_resources properties to arguments to soong_zip -jar, keeping files that should -// normally not used as resources like *.java -func SourceFilesToJarArgs(ctx android.ModuleContext, - res, exclude []string) (args []string, deps android.Paths) { - - return resourceFilesToJarArgs(ctx, res, exclude) -} - func resourceFilesToJarArgs(ctx android.ModuleContext, res, exclude []string) (args []string, deps android.Paths) { files := android.PathsForModuleSrcExcludes(ctx, res, exclude) + args = resourcePathsToJarArgs(files) + + return args, files +} + +func resourcePathsToJarArgs(files android.Paths) []string { + var args []string + lastDir := "" for i, f := range files { rel := f.Rel() @@ -113,5 +113,5 @@ func resourceFilesToJarArgs(ctx android.ModuleContext, lastDir = dir } - return args, files + return args } diff --git a/java/java_test.go b/java/java_test.go index 50b2f69d9..8797119b5 100644 --- a/java/java_test.go +++ b/java/java_test.go @@ -18,10 +18,15 @@ import ( "io/ioutil" "os" "path/filepath" + "reflect" + "regexp" + "sort" "strconv" "strings" "testing" + "github.com/google/blueprint/proptools" + "android/soong/android" "android/soong/cc" "android/soong/dexpreopt" @@ -53,151 +58,44 @@ func TestMain(m *testing.M) { os.Exit(run()) } -func testConfig(env map[string]string) android.Config { - return TestConfig(buildDir, env) +func testConfig(env map[string]string, bp string, fs map[string][]byte) android.Config { + bp += dexpreopt.BpToolModulesForTest() + + config := TestConfig(buildDir, env, bp, fs) + + // Set up the global Once cache used for dexpreopt.GlobalSoongConfig, so that + // it doesn't create a real one, which would fail. + _ = dexpreopt.GlobalSoongConfigForTests(config) + + return config } -func testContext(config android.Config, bp string, - fs map[string][]byte) *android.TestContext { +func testContext() *android.TestContext { ctx := android.NewTestArchContext() - ctx.RegisterModuleType("android_app", android.ModuleFactoryAdaptor(AndroidAppFactory)) - ctx.RegisterModuleType("android_app_certificate", android.ModuleFactoryAdaptor(AndroidAppCertificateFactory)) - ctx.RegisterModuleType("android_library", android.ModuleFactoryAdaptor(AndroidLibraryFactory)) - ctx.RegisterModuleType("android_test", android.ModuleFactoryAdaptor(AndroidTestFactory)) - ctx.RegisterModuleType("android_test_helper_app", android.ModuleFactoryAdaptor(AndroidTestHelperAppFactory)) - ctx.RegisterModuleType("java_binary", android.ModuleFactoryAdaptor(BinaryFactory)) - ctx.RegisterModuleType("java_binary_host", android.ModuleFactoryAdaptor(BinaryHostFactory)) - ctx.RegisterModuleType("java_device_for_host", android.ModuleFactoryAdaptor(DeviceForHostFactory)) - ctx.RegisterModuleType("java_host_for_device", android.ModuleFactoryAdaptor(HostForDeviceFactory)) - ctx.RegisterModuleType("java_library", android.ModuleFactoryAdaptor(LibraryFactory)) - ctx.RegisterModuleType("java_library_host", android.ModuleFactoryAdaptor(LibraryHostFactory)) - ctx.RegisterModuleType("java_test", android.ModuleFactoryAdaptor(TestFactory)) - ctx.RegisterModuleType("java_import", android.ModuleFactoryAdaptor(ImportFactory)) - ctx.RegisterModuleType("java_import_host", android.ModuleFactoryAdaptor(ImportFactoryHost)) - ctx.RegisterModuleType("java_defaults", android.ModuleFactoryAdaptor(defaultsFactory)) - ctx.RegisterModuleType("java_system_modules", android.ModuleFactoryAdaptor(SystemModulesFactory)) - ctx.RegisterModuleType("java_genrule", android.ModuleFactoryAdaptor(genRuleFactory)) - ctx.RegisterModuleType("java_plugin", android.ModuleFactoryAdaptor(PluginFactory)) - ctx.RegisterModuleType("dex_import", android.ModuleFactoryAdaptor(DexImportFactory)) - ctx.RegisterModuleType("filegroup", android.ModuleFactoryAdaptor(android.FileGroupFactory)) - ctx.RegisterModuleType("genrule", android.ModuleFactoryAdaptor(genrule.GenRuleFactory)) - ctx.RegisterModuleType("droiddoc", android.ModuleFactoryAdaptor(DroiddocFactory)) - ctx.RegisterModuleType("droiddoc_host", android.ModuleFactoryAdaptor(DroiddocHostFactory)) - ctx.RegisterModuleType("droiddoc_template", android.ModuleFactoryAdaptor(ExportedDroiddocDirFactory)) - ctx.RegisterModuleType("java_sdk_library", android.ModuleFactoryAdaptor(SdkLibraryFactory)) - ctx.RegisterModuleType("override_android_app", android.ModuleFactoryAdaptor(OverrideAndroidAppModuleFactory)) - ctx.RegisterModuleType("prebuilt_apis", android.ModuleFactoryAdaptor(PrebuiltApisFactory)) - ctx.RegisterModuleType("android_app_set", android.ModuleFactoryAdaptor(AndroidApkSetFactory)) - ctx.PreArchMutators(android.RegisterPrebuiltsPreArchMutators) - ctx.PreArchMutators(android.RegisterPrebuiltsPostDepsMutators) + RegisterJavaBuildComponents(ctx) + RegisterAppBuildComponents(ctx) + RegisterAARBuildComponents(ctx) + RegisterGenRuleBuildComponents(ctx) + RegisterSystemModulesBuildComponents(ctx) + ctx.RegisterModuleType("java_plugin", PluginFactory) + ctx.RegisterModuleType("filegroup", android.FileGroupFactory) + ctx.RegisterModuleType("genrule", genrule.GenRuleFactory) + RegisterDocsBuildComponents(ctx) + RegisterStubsBuildComponents(ctx) + RegisterSdkLibraryBuildComponents(ctx) ctx.PreArchMutators(android.RegisterDefaultsPreArchMutators) - ctx.PreArchMutators(android.RegisterOverridePreArchMutators) - ctx.PreArchMutators(func(ctx android.RegisterMutatorsContext) { - ctx.TopDown("prebuilt_apis", PrebuiltApisMutator).Parallel() - ctx.TopDown("java_sdk_library", SdkLibraryMutator).Parallel() - }) + + RegisterPrebuiltApisBuildComponents(ctx) + + ctx.PostDepsMutators(android.RegisterOverridePostDepsMutators) ctx.RegisterPreSingletonType("overlay", android.SingletonFactoryAdaptor(OverlaySingletonFactory)) ctx.RegisterPreSingletonType("sdk_versions", android.SingletonFactoryAdaptor(sdkPreSingletonFactory)) // Register module types and mutators from cc needed for JNI testing - ctx.RegisterModuleType("cc_library", android.ModuleFactoryAdaptor(cc.LibraryFactory)) - ctx.RegisterModuleType("cc_object", android.ModuleFactoryAdaptor(cc.ObjectFactory)) - ctx.RegisterModuleType("toolchain_library", android.ModuleFactoryAdaptor(cc.ToolchainLibraryFactory)) - ctx.RegisterModuleType("llndk_library", android.ModuleFactoryAdaptor(cc.LlndkLibraryFactory)) - ctx.PreDepsMutators(func(ctx android.RegisterMutatorsContext) { - ctx.BottomUp("link", cc.LinkageMutator).Parallel() - ctx.BottomUp("begin", cc.BeginMutator).Parallel() - }) + cc.RegisterRequiredBuildComponentsForTest(ctx) - bp += GatherRequiredDepsForTest() - - mockFS := map[string][]byte{ - "Android.bp": []byte(bp), - "a.java": nil, - "b.java": nil, - "c.java": nil, - "b.kt": nil, - "a.jar": nil, - "b.jar": nil, - "APP_NOTICE": nil, - "GENRULE_NOTICE": nil, - "LIB_NOTICE": nil, - "TOOL_NOTICE": nil, - "java-res/a/a": nil, - "java-res/b/b": nil, - "java-res2/a": nil, - "java-fg/a.java": nil, - "java-fg/b.java": nil, - "java-fg/c.java": nil, - "api/current.txt": nil, - "api/removed.txt": nil, - "api/system-current.txt": nil, - "api/system-removed.txt": nil, - "api/test-current.txt": nil, - "api/test-removed.txt": nil, - "framework/aidl/a.aidl": nil, - - "prebuilts/sdk/14/public/android.jar": nil, - "prebuilts/sdk/14/public/framework.aidl": nil, - "prebuilts/sdk/14/system/android.jar": nil, - "prebuilts/sdk/17/public/android.jar": nil, - "prebuilts/sdk/17/public/framework.aidl": nil, - "prebuilts/sdk/17/system/android.jar": nil, - "prebuilts/sdk/25/public/android.jar": nil, - "prebuilts/sdk/25/public/framework.aidl": nil, - "prebuilts/sdk/25/system/android.jar": nil, - "prebuilts/sdk/current/core/android.jar": nil, - "prebuilts/sdk/current/public/android.jar": nil, - "prebuilts/sdk/current/public/framework.aidl": nil, - "prebuilts/sdk/current/public/core.jar": nil, - "prebuilts/sdk/current/system/android.jar": nil, - "prebuilts/sdk/current/test/android.jar": nil, - "prebuilts/sdk/28/public/api/foo.txt": nil, - "prebuilts/sdk/28/system/api/foo.txt": nil, - "prebuilts/sdk/28/test/api/foo.txt": nil, - "prebuilts/sdk/28/public/api/foo-removed.txt": nil, - "prebuilts/sdk/28/system/api/foo-removed.txt": nil, - "prebuilts/sdk/28/test/api/foo-removed.txt": nil, - "prebuilts/sdk/28/public/api/bar.txt": nil, - "prebuilts/sdk/28/system/api/bar.txt": nil, - "prebuilts/sdk/28/test/api/bar.txt": nil, - "prebuilts/sdk/28/public/api/bar-removed.txt": nil, - "prebuilts/sdk/28/system/api/bar-removed.txt": nil, - "prebuilts/sdk/28/test/api/bar-removed.txt": nil, - "prebuilts/sdk/tools/core-lambda-stubs.jar": nil, - "prebuilts/sdk/Android.bp": []byte(`prebuilt_apis { name: "sdk", api_dirs: ["14", "28", "current"],}`), - - "prebuilts/apks/app.apks": nil, - - // For framework-res, which is an implicit dependency for framework - "AndroidManifest.xml": nil, - "build/target/product/security/testkey": nil, - - "build/soong/scripts/jar-wrapper.sh": nil, - - "build/make/core/proguard.flags": nil, - "build/make/core/proguard_basic_keeps.flags": nil, - - "jdk8/jre/lib/jce.jar": nil, - "jdk8/jre/lib/rt.jar": nil, - "jdk8/lib/tools.jar": nil, - - "bar-doc/a.java": nil, - "bar-doc/b.java": nil, - "bar-doc/IFoo.aidl": nil, - "bar-doc/known_oj_tags.txt": nil, - "external/doclava/templates-sdk": nil, - - "cert/new_cert.x509.pem": nil, - "cert/new_cert.pk8": nil, - } - - for k, v := range fs { - mockFS[k] = v - } - - ctx.MockFileSystem(mockFS) + dexpreopt.RegisterToolModulesForTest(ctx) return ctx } @@ -205,11 +103,11 @@ func testContext(config android.Config, bp string, func run(t *testing.T, ctx *android.TestContext, config android.Config) { t.Helper() - pathCtx := android.PathContextForTesting(config, nil) - setDexpreoptTestGlobalConfig(config, dexpreopt.GlobalConfigForTests(pathCtx)) + pathCtx := android.PathContextForTesting(config) + dexpreopt.SetTestGlobalConfig(config, dexpreopt.GlobalConfigForTests(pathCtx)) - ctx.Register() - _, errs := ctx.ParseFileList(".", []string{"Android.bp", "prebuilts/sdk/Android.bp"}) + ctx.Register(config) + _, errs := ctx.ParseBlueprintsFiles("Android.bp") android.FailIfErrored(t, errs) _, errs = ctx.PrepareBuildActions(config) android.FailIfErrored(t, errs) @@ -217,9 +115,17 @@ func run(t *testing.T, ctx *android.TestContext, config android.Config) { func testJavaError(t *testing.T, pattern string, bp string) (*android.TestContext, android.Config) { t.Helper() - config := testConfig(nil) - ctx := testContext(config, bp, nil) + return testJavaErrorWithConfig(t, pattern, testConfig(nil, bp, nil)) +} + +func testJavaErrorWithConfig(t *testing.T, pattern string, config android.Config) (*android.TestContext, android.Config) { + t.Helper() + ctx := testContext() + pathCtx := android.PathContextForTesting(config) + dexpreopt.SetTestGlobalConfig(config, dexpreopt.GlobalConfigForTests(pathCtx)) + + ctx.Register(config) _, errs := ctx.ParseBlueprintsFiles("Android.bp") if len(errs) > 0 { android.FailIfNoMatchingErrors(t, pattern, errs) @@ -236,13 +142,22 @@ func testJavaError(t *testing.T, pattern string, bp string) (*android.TestContex return ctx, config } -func testJava(t *testing.T, bp string) *android.TestContext { +func testJavaWithFS(t *testing.T, bp string, fs map[string][]byte) (*android.TestContext, android.Config) { t.Helper() - config := testConfig(nil) - ctx := testContext(config, bp, nil) + return testJavaWithConfig(t, testConfig(nil, bp, fs)) +} + +func testJava(t *testing.T, bp string) (*android.TestContext, android.Config) { + t.Helper() + return testJavaWithFS(t, bp, nil) +} + +func testJavaWithConfig(t *testing.T, config android.Config) (*android.TestContext, android.Config) { + t.Helper() + ctx := testContext() run(t, ctx, config) - return ctx + return ctx, config } func moduleToPath(name string) string { @@ -256,8 +171,96 @@ func moduleToPath(name string) string { } } +func TestJavaLinkType(t *testing.T) { + testJava(t, ` + java_library { + name: "foo", + srcs: ["a.java"], + libs: ["bar"], + static_libs: ["baz"], + } + + java_library { + name: "bar", + sdk_version: "current", + srcs: ["b.java"], + } + + java_library { + name: "baz", + sdk_version: "system_current", + srcs: ["c.java"], + } + `) + + testJavaError(t, "Adjust sdk_version: property of the source or target module so that target module is built with the same or smaller API set than the source.", ` + java_library { + name: "foo", + srcs: ["a.java"], + libs: ["bar"], + sdk_version: "current", + static_libs: ["baz"], + } + + java_library { + name: "bar", + sdk_version: "current", + srcs: ["b.java"], + } + + java_library { + name: "baz", + sdk_version: "system_current", + srcs: ["c.java"], + } + `) + + testJava(t, ` + java_library { + name: "foo", + srcs: ["a.java"], + libs: ["bar"], + sdk_version: "system_current", + static_libs: ["baz"], + } + + java_library { + name: "bar", + sdk_version: "current", + srcs: ["b.java"], + } + + java_library { + name: "baz", + sdk_version: "system_current", + srcs: ["c.java"], + } + `) + + testJavaError(t, "Adjust sdk_version: property of the source or target module so that target module is built with the same or smaller API set than the source.", ` + java_library { + name: "foo", + srcs: ["a.java"], + libs: ["bar"], + sdk_version: "system_current", + static_libs: ["baz"], + } + + java_library { + name: "bar", + sdk_version: "current", + srcs: ["b.java"], + } + + java_library { + name: "baz", + srcs: ["c.java"], + } + `) +} + func TestSimple(t *testing.T) { - ctx := testJava(t, ` + ctx, _ := testJava(t, ` java_library { name: "foo", srcs: ["a.java"], @@ -300,8 +303,126 @@ func TestSimple(t *testing.T) { } } +func TestExportedPlugins(t *testing.T) { + type Result struct { + library string + processors string + } + var tests = []struct { + name string + extra string + results []Result + }{ + { + name: "Exported plugin is not a direct plugin", + extra: `java_library { name: "exports", srcs: ["a.java"], exported_plugins: ["plugin"] }`, + results: []Result{{library: "exports", processors: "-proc:none"}}, + }, + { + name: "Exports plugin to dependee", + extra: ` + java_library{name: "exports", exported_plugins: ["plugin"]} + java_library{name: "foo", srcs: ["a.java"], libs: ["exports"]} + java_library{name: "bar", srcs: ["a.java"], static_libs: ["exports"]} + `, + results: []Result{ + {library: "foo", processors: "-processor com.android.TestPlugin"}, + {library: "bar", processors: "-processor com.android.TestPlugin"}, + }, + }, + { + name: "Exports plugin to android_library", + extra: ` + java_library{name: "exports", exported_plugins: ["plugin"]} + android_library{name: "foo", srcs: ["a.java"], libs: ["exports"]} + android_library{name: "bar", srcs: ["a.java"], static_libs: ["exports"]} + `, + results: []Result{ + {library: "foo", processors: "-processor com.android.TestPlugin"}, + {library: "bar", processors: "-processor com.android.TestPlugin"}, + }, + }, + { + name: "Exports plugin is not propagated via transitive deps", + extra: ` + java_library{name: "exports", exported_plugins: ["plugin"]} + java_library{name: "foo", srcs: ["a.java"], libs: ["exports"]} + java_library{name: "bar", srcs: ["a.java"], static_libs: ["foo"]} + `, + results: []Result{ + {library: "foo", processors: "-processor com.android.TestPlugin"}, + {library: "bar", processors: "-proc:none"}, + }, + }, + { + name: "Exports plugin appends to plugins", + extra: ` + java_plugin{name: "plugin2", processor_class: "com.android.TestPlugin2"} + java_library{name: "exports", exported_plugins: ["plugin"]} + java_library{name: "foo", srcs: ["a.java"], libs: ["exports"], plugins: ["plugin2"]} + `, + results: []Result{ + {library: "foo", processors: "-processor com.android.TestPlugin,com.android.TestPlugin2"}, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctx, _ := testJava(t, ` + java_plugin { + name: "plugin", + processor_class: "com.android.TestPlugin", + } + `+test.extra) + + for _, want := range test.results { + javac := ctx.ModuleForTests(want.library, "android_common").Rule("javac") + if javac.Args["processor"] != want.processors { + t.Errorf("For library %v, expected %v, found %v", want.library, want.processors, javac.Args["processor"]) + } + } + }) + } +} + +func TestSdkVersionByPartition(t *testing.T) { + testJavaError(t, "sdk_version must have a value when the module is located at vendor or product", ` + java_library { + name: "foo", + srcs: ["a.java"], + vendor: true, + } + `) + + testJava(t, ` + java_library { + name: "bar", + srcs: ["b.java"], + } + `) + + for _, enforce := range []bool{true, false} { + bp := ` + java_library { + name: "foo", + srcs: ["a.java"], + product_specific: true, + } + ` + + config := testConfig(nil, bp, nil) + config.TestProductVariables.EnforceProductPartitionInterface = proptools.BoolPtr(enforce) + if enforce { + testJavaErrorWithConfig(t, "sdk_version must have a value when the module is located at vendor or product", config) + } else { + testJavaWithConfig(t, config) + } + } +} + func TestArchSpecific(t *testing.T) { - ctx := testJava(t, ` + ctx, _ := testJava(t, ` java_library { name: "foo", srcs: ["a.java"], @@ -320,7 +441,7 @@ func TestArchSpecific(t *testing.T) { } func TestBinary(t *testing.T) { - ctx := testJava(t, ` + ctx, _ := testJava(t, ` java_library_host { name: "foo", srcs: ["a.java"], @@ -349,11 +470,11 @@ func TestBinary(t *testing.T) { } func TestPrebuilts(t *testing.T) { - ctx := testJava(t, ` + ctx, _ := testJava(t, ` java_library { name: "foo", - srcs: ["a.java"], - libs: ["bar"], + srcs: ["a.java", ":stubs-source"], + libs: ["bar", "sdklib"], static_libs: ["baz"], } @@ -371,17 +492,50 @@ func TestPrebuilts(t *testing.T) { name: "qux", jars: ["b.jar"], } + + java_sdk_library_import { + name: "sdklib", + public: { + jars: ["c.jar"], + }, + } + + prebuilt_stubs_sources { + name: "stubs-source", + srcs: ["stubs/sources"], + } + + java_test_import { + name: "test", + jars: ["a.jar"], + test_suites: ["cts"], + test_config: "AndroidTest.xml", + } `) - javac := ctx.ModuleForTests("foo", "android_common").Rule("javac") + fooModule := ctx.ModuleForTests("foo", "android_common") + javac := fooModule.Rule("javac") combineJar := ctx.ModuleForTests("foo", "android_common").Description("for javac") barJar := ctx.ModuleForTests("bar", "android_common").Rule("combineJar").Output bazJar := ctx.ModuleForTests("baz", "android_common").Rule("combineJar").Output + sdklibStubsJar := ctx.ModuleForTests("sdklib.stubs", "android_common").Rule("combineJar").Output + + fooLibrary := fooModule.Module().(*Library) + assertDeepEquals(t, "foo java sources incorrect", + []string{"a.java"}, fooLibrary.compiledJavaSrcs.Strings()) + + assertDeepEquals(t, "foo java source jars incorrect", + []string{".intermediates/stubs-source/android_common/stubs-source-stubs.srcjar"}, + android.NormalizePathsForTesting(fooLibrary.compiledSrcJars)) if !strings.Contains(javac.Args["classpath"], barJar.String()) { t.Errorf("foo classpath %v does not contain %q", javac.Args["classpath"], barJar.String()) } + if !strings.Contains(javac.Args["classpath"], sdklibStubsJar.String()) { + t.Errorf("foo classpath %v does not contain %q", javac.Args["classpath"], sdklibStubsJar.String()) + } + if len(combineJar.Inputs) != 2 || combineJar.Inputs[1].String() != bazJar.String() { t.Errorf("foo combineJar inputs %v does not contain %q", combineJar.Inputs, bazJar.String()) } @@ -389,8 +543,63 @@ func TestPrebuilts(t *testing.T) { ctx.ModuleForTests("qux", "android_common").Rule("Cp") } +func assertDeepEquals(t *testing.T, message string, expected interface{}, actual interface{}) { + if !reflect.DeepEqual(expected, actual) { + t.Errorf("%s: expected %q, found %q", message, expected, actual) + } +} + +func TestJavaSdkLibraryImport(t *testing.T) { + ctx, _ := testJava(t, ` + java_library { + name: "foo", + srcs: ["a.java"], + libs: ["sdklib"], + sdk_version: "current", + } + + java_library { + name: "foo.system", + srcs: ["a.java"], + libs: ["sdklib"], + sdk_version: "system_current", + } + + java_library { + name: "foo.test", + srcs: ["a.java"], + libs: ["sdklib"], + sdk_version: "test_current", + } + + java_sdk_library_import { + name: "sdklib", + public: { + jars: ["a.jar"], + }, + system: { + jars: ["b.jar"], + }, + test: { + jars: ["c.jar"], + stub_srcs: ["c.java"], + }, + } + `) + + for _, scope := range []string{"", ".system", ".test"} { + fooModule := ctx.ModuleForTests("foo"+scope, "android_common") + javac := fooModule.Rule("javac") + + sdklibStubsJar := ctx.ModuleForTests("sdklib.stubs"+scope, "android_common").Rule("combineJar").Output + if !strings.Contains(javac.Args["classpath"], sdklibStubsJar.String()) { + t.Errorf("foo classpath %v does not contain %q", javac.Args["classpath"], sdklibStubsJar.String()) + } + } +} + func TestDefaults(t *testing.T) { - ctx := testJava(t, ` + ctx, _ := testJava(t, ` java_defaults { name: "defaults", srcs: ["a.java"], @@ -497,12 +706,6 @@ func TestResources(t *testing.T) { args: "-C java-res -f java-res/a/a -f java-res/b/b", }, { - // Test that a module with "include_srcs: true" includes its source files in the resources jar - name: "include sources", - prop: `include_srcs: true`, - args: "-C . -f a.java -f b.java -f c.java", - }, - { // Test that a module with wildcards in java_resource_dirs has the correct path prefixes name: "wildcard dirs", prop: `java_resource_dirs: ["java-res/*"]`, @@ -542,7 +745,7 @@ func TestResources(t *testing.T) { for _, test := range table { t.Run(test.name, func(t *testing.T) { - ctx := testJava(t, ` + ctx, _ := testJavaWithFS(t, ` java_library { name: "foo", srcs: [ @@ -552,7 +755,13 @@ func TestResources(t *testing.T) { ], `+test.prop+`, } - `+test.extra) + `+test.extra, + map[string][]byte{ + "java-res/a/a": nil, + "java-res/b/b": nil, + "java-res2/a": nil, + }, + ) foo := ctx.ModuleForTests("foo", "android_common").Output("withres/foo.jar") fooRes := ctx.ModuleForTests("foo", "android_common").Output("res/foo.jar") @@ -570,8 +779,75 @@ func TestResources(t *testing.T) { } } +func TestIncludeSrcs(t *testing.T) { + ctx, _ := testJavaWithFS(t, ` + java_library { + name: "foo", + srcs: [ + "a.java", + "b.java", + "c.java", + ], + include_srcs: true, + } + + java_library { + name: "bar", + srcs: [ + "a.java", + "b.java", + "c.java", + ], + java_resource_dirs: ["java-res"], + include_srcs: true, + } + `, map[string][]byte{ + "java-res/a/a": nil, + "java-res/b/b": nil, + "java-res2/a": nil, + }) + + // Test a library with include_srcs: true + foo := ctx.ModuleForTests("foo", "android_common").Output("withres/foo.jar") + fooSrcJar := ctx.ModuleForTests("foo", "android_common").Output("foo.srcjar") + + if g, w := fooSrcJar.Output.String(), foo.Inputs.Strings(); !inList(g, w) { + t.Errorf("foo combined jars %v does not contain %q", w, g) + } + + if g, w := fooSrcJar.Args["jarArgs"], "-C . -f a.java -f b.java -f c.java"; g != w { + t.Errorf("foo source jar args %q is not %q", w, g) + } + + // Test a library with include_srcs: true and resources + bar := ctx.ModuleForTests("bar", "android_common").Output("withres/bar.jar") + barResCombined := ctx.ModuleForTests("bar", "android_common").Output("res-combined/bar.jar") + barRes := ctx.ModuleForTests("bar", "android_common").Output("res/bar.jar") + barSrcJar := ctx.ModuleForTests("bar", "android_common").Output("bar.srcjar") + + if g, w := barSrcJar.Output.String(), barResCombined.Inputs.Strings(); !inList(g, w) { + t.Errorf("bar combined resource jars %v does not contain %q", w, g) + } + + if g, w := barRes.Output.String(), barResCombined.Inputs.Strings(); !inList(g, w) { + t.Errorf("bar combined resource jars %v does not contain %q", w, g) + } + + if g, w := barResCombined.Output.String(), bar.Inputs.Strings(); !inList(g, w) { + t.Errorf("bar combined jars %v does not contain %q", w, g) + } + + if g, w := barSrcJar.Args["jarArgs"], "-C . -f a.java -f b.java -f c.java"; g != w { + t.Errorf("bar source jar args %q is not %q", w, g) + } + + if g, w := barRes.Args["jarArgs"], "-C java-res -f java-res/a/a -f java-res/b/b"; g != w { + t.Errorf("bar resource jar args %q is not %q", w, g) + } +} + func TestGeneratedSources(t *testing.T) { - ctx := testJava(t, ` + ctx, _ := testJavaWithFS(t, ` java_library { name: "foo", srcs: [ @@ -586,7 +862,10 @@ func TestGeneratedSources(t *testing.T) { tool_files: ["java-res/a"], out: ["gen.java"], } - `) + `, map[string][]byte{ + "a.java": nil, + "b.java": nil, + }) javac := ctx.ModuleForTests("foo", "android_common").Rule("javac") genrule := ctx.ModuleForTests("gen", "").Rule("generator") @@ -604,7 +883,7 @@ func TestGeneratedSources(t *testing.T) { } func TestTurbine(t *testing.T) { - ctx := testJava(t, ` + ctx, _ := testJava(t, ` java_library { name: "foo", srcs: ["a.java"], @@ -653,7 +932,7 @@ func TestTurbine(t *testing.T) { } func TestSharding(t *testing.T) { - ctx := testJava(t, ` + ctx, _ := testJava(t, ` java_library { name: "bar", srcs: ["a.java","b.java","c.java"], @@ -671,16 +950,22 @@ func TestSharding(t *testing.T) { } func TestDroiddoc(t *testing.T) { - ctx := testJava(t, ` - droiddoc_template { + ctx, _ := testJavaWithFS(t, ` + droiddoc_exported_dir { name: "droiddoc-templates-sdk", path: ".", } + filegroup { + name: "bar-doc-aidl-srcs", + srcs: ["bar-doc/IBar.aidl"], + path: "bar-doc", + } droiddoc { name: "bar-doc", srcs: [ - "bar-doc/*.java", + "bar-doc/a.java", "bar-doc/IFoo.aidl", + ":bar-doc-aidl-srcs", ], exclude_srcs: [ "bar-doc/b.java" @@ -696,25 +981,148 @@ func TestDroiddoc(t *testing.T) { todo_file: "libcore-docs-todo.html", args: "-offlinemode -title \"libcore\"", } - `) + `, + map[string][]byte{ + "bar-doc/a.java": nil, + "bar-doc/b.java": nil, + }) - stubsJar := filepath.Join(buildDir, ".intermediates", "bar-doc", "android_common", "bar-doc-stubs.srcjar") - barDoc := ctx.ModuleForTests("bar-doc", "android_common").Output("bar-doc-stubs.srcjar") - if stubsJar != barDoc.Output.String() { - t.Errorf("expected stubs Jar [%q], got %q", stubsJar, barDoc.Output.String()) - } - inputs := ctx.ModuleForTests("bar-doc", "android_common").Rule("javadoc").Inputs + barDoc := ctx.ModuleForTests("bar-doc", "android_common").Rule("javadoc") var javaSrcs []string - for _, i := range inputs { + for _, i := range barDoc.Inputs { javaSrcs = append(javaSrcs, i.Base()) } - if len(javaSrcs) != 2 || javaSrcs[0] != "a.java" || javaSrcs[1] != "IFoo.java" { - t.Errorf("inputs of bar-doc must be []string{\"a.java\", \"IFoo.java\", but was %#v.", javaSrcs) + if len(javaSrcs) != 1 || javaSrcs[0] != "a.java" { + t.Errorf("inputs of bar-doc must be []string{\"a.java\"}, but was %#v.", javaSrcs) + } + + aidl := ctx.ModuleForTests("bar-doc", "android_common").Rule("aidl") + if g, w := barDoc.Implicits.Strings(), aidl.Output.String(); !inList(w, g) { + t.Errorf("implicits of bar-doc must contain %q, but was %q.", w, g) + } + + if g, w := aidl.Implicits.Strings(), []string{"bar-doc/IBar.aidl", "bar-doc/IFoo.aidl"}; !reflect.DeepEqual(w, g) { + t.Errorf("aidl inputs must be %q, but was %q", w, g) + } +} + +func TestDroidstubs(t *testing.T) { + ctx, _ := testJavaWithFS(t, ` + droiddoc_exported_dir { + name: "droiddoc-templates-sdk", + path: ".", + } + + droidstubs { + name: "bar-stubs", + srcs: [ + "bar-doc/a.java", + ], + api_levels_annotations_dirs: [ + "droiddoc-templates-sdk", + ], + api_levels_annotations_enabled: true, + } + + droidstubs { + name: "bar-stubs-other", + srcs: [ + "bar-doc/a.java", + ], + api_levels_annotations_dirs: [ + "droiddoc-templates-sdk", + ], + api_levels_annotations_enabled: true, + api_levels_jar_filename: "android.other.jar", + } + `, + map[string][]byte{ + "bar-doc/a.java": nil, + }) + testcases := []struct { + moduleName string + expectedJarFilename string + }{ + { + moduleName: "bar-stubs", + expectedJarFilename: "android.jar", + }, + { + moduleName: "bar-stubs-other", + expectedJarFilename: "android.other.jar", + }, + } + for _, c := range testcases { + m := ctx.ModuleForTests(c.moduleName, "android_common") + metalava := m.Rule("metalava") + expected := "--android-jar-pattern ./%/public/" + c.expectedJarFilename + if actual := metalava.RuleParams.Command; !strings.Contains(actual, expected) { + t.Errorf("For %q, expected metalava argument %q, but was not found %q", c.moduleName, expected, actual) + } + } +} + +func TestDroidstubsWithSystemModules(t *testing.T) { + ctx, _ := testJava(t, ` + droidstubs { + name: "stubs-source-system-modules", + srcs: [ + "bar-doc/a.java", + ], + sdk_version: "none", + system_modules: "source-system-modules", + } + + java_library { + name: "source-jar", + srcs: [ + "a.java", + ], + } + + java_system_modules { + name: "source-system-modules", + libs: ["source-jar"], + } + + droidstubs { + name: "stubs-prebuilt-system-modules", + srcs: [ + "bar-doc/a.java", + ], + sdk_version: "none", + system_modules: "prebuilt-system-modules", + } + + java_import { + name: "prebuilt-jar", + jars: ["a.jar"], + } + + java_system_modules_import { + name: "prebuilt-system-modules", + libs: ["prebuilt-jar"], + } + `) + + checkSystemModulesUseByDroidstubs(t, ctx, "stubs-source-system-modules", "source-jar.jar") + + checkSystemModulesUseByDroidstubs(t, ctx, "stubs-prebuilt-system-modules", "prebuilt-jar.jar") +} + +func checkSystemModulesUseByDroidstubs(t *testing.T, ctx *android.TestContext, moduleName string, systemJar string) { + metalavaRule := ctx.ModuleForTests(moduleName, "android_common").Rule("metalava") + var systemJars []string + for _, i := range metalavaRule.Implicits { + systemJars = append(systemJars, i.Base()) + } + if len(systemJars) < 1 || systemJars[0] != systemJar { + t.Errorf("inputs of %q must be []string{%q}, but was %#v.", moduleName, systemJar, systemJars) } } func TestJarGenrules(t *testing.T) { - ctx := testJava(t, ` + ctx, _ := testJava(t, ` java_library { name: "foo", srcs: ["a.java"], @@ -768,7 +1176,7 @@ func TestJarGenrules(t *testing.T) { } func TestExcludeFileGroupInSrcs(t *testing.T) { - ctx := testJava(t, ` + ctx, _ := testJava(t, ` java_library { name: "foo", srcs: ["a.java", ":foo-srcs"], @@ -793,9 +1201,22 @@ func TestExcludeFileGroupInSrcs(t *testing.T) { } } +func TestJavaLibrary(t *testing.T) { + config := testConfig(nil, "", map[string][]byte{ + "libcore/Android.bp": []byte(` + java_library { + name: "core", + sdk_version: "none", + system_modules: "none", + }`), + }) + ctx := testContext() + run(t, ctx, config) +} + func TestJavaSdkLibrary(t *testing.T) { - ctx := testJava(t, ` - droiddoc_template { + ctx, _ := testJava(t, ` + droiddoc_exported_dir { name: "droiddoc-templates-sdk", path: ".", } @@ -812,26 +1233,67 @@ func TestJavaSdkLibrary(t *testing.T) { java_library { name: "baz", srcs: ["c.java"], - libs: ["foo", "bar"], + libs: ["foo", "bar.stubs"], sdk_version: "system_current", } + java_sdk_library { + name: "barney", + srcs: ["c.java"], + api_only: true, + } + java_sdk_library { + name: "betty", + srcs: ["c.java"], + shared_library: false, + } + java_sdk_library_import { + name: "quuz", + public: { + jars: ["c.jar"], + }, + } + java_sdk_library_import { + name: "fred", + public: { + jars: ["b.jar"], + }, + } + java_sdk_library_import { + name: "wilma", + public: { + jars: ["b.jar"], + }, + shared_library: false, + } java_library { name: "qux", srcs: ["c.java"], - libs: ["baz"], + libs: ["baz", "fred", "quuz.stubs", "wilma", "barney", "betty"], sdk_version: "system_current", } + java_library { + name: "baz-test", + srcs: ["c.java"], + libs: ["foo"], + sdk_version: "test_current", + } + java_library { + name: "baz-29", + srcs: ["c.java"], + libs: ["foo"], + sdk_version: "system_29", + } `) // check the existence of the internal modules ctx.ModuleForTests("foo", "android_common") - ctx.ModuleForTests("foo"+sdkStubsLibrarySuffix, "android_common") - ctx.ModuleForTests("foo"+sdkStubsLibrarySuffix+sdkSystemApiSuffix, "android_common") - ctx.ModuleForTests("foo"+sdkStubsLibrarySuffix+sdkTestApiSuffix, "android_common") - ctx.ModuleForTests("foo"+sdkDocsSuffix, "android_common") - ctx.ModuleForTests("foo"+sdkDocsSuffix+sdkSystemApiSuffix, "android_common") - ctx.ModuleForTests("foo"+sdkDocsSuffix+sdkTestApiSuffix, "android_common") - ctx.ModuleForTests("foo"+sdkXmlFileSuffix, "android_arm64_armv8-a") + ctx.ModuleForTests(apiScopePublic.stubsLibraryModuleName("foo"), "android_common") + ctx.ModuleForTests(apiScopeSystem.stubsLibraryModuleName("foo"), "android_common") + ctx.ModuleForTests(apiScopeTest.stubsLibraryModuleName("foo"), "android_common") + ctx.ModuleForTests(apiScopePublic.stubsSourceModuleName("foo"), "android_common") + ctx.ModuleForTests(apiScopeSystem.stubsSourceModuleName("foo"), "android_common") + ctx.ModuleForTests(apiScopeTest.stubsSourceModuleName("foo"), "android_common") + ctx.ModuleForTests("foo"+sdkXmlFileSuffix, "android_common") ctx.ModuleForTests("foo.api.public.28", "") ctx.ModuleForTests("foo.api.system.28", "") ctx.ModuleForTests("foo.api.test.28", "") @@ -853,13 +1315,290 @@ func TestJavaSdkLibrary(t *testing.T) { "foo.stubs.jar") } + bazTestJavac := ctx.ModuleForTests("baz-test", "android_common").Rule("javac") + // tests if baz-test is actually linked to the test stubs lib + if !strings.Contains(bazTestJavac.Args["classpath"], "foo.stubs.test.jar") { + t.Errorf("baz-test javac classpath %v does not contain %q", bazTestJavac.Args["classpath"], + "foo.stubs.test.jar") + } + + baz29Javac := ctx.ModuleForTests("baz-29", "android_common").Rule("javac") + // tests if baz-29 is actually linked to the system 29 stubs lib + if !strings.Contains(baz29Javac.Args["classpath"], "prebuilts/sdk/29/system/foo.jar") { + t.Errorf("baz-29 javac classpath %v does not contain %q", baz29Javac.Args["classpath"], + "prebuilts/sdk/29/system/foo.jar") + } + // test if baz has exported SDK lib names foo and bar to qux qux := ctx.ModuleForTests("qux", "android_common") if quxLib, ok := qux.Module().(*Library); ok { sdkLibs := quxLib.ExportedSdkLibs() - if len(sdkLibs) != 2 || !android.InList("foo", sdkLibs) || !android.InList("bar", sdkLibs) { - t.Errorf("qux should export \"foo\" and \"bar\" but exports %v", sdkLibs) + sort.Strings(sdkLibs) + if w := []string{"bar", "foo", "fred", "quuz"}; !reflect.DeepEqual(w, sdkLibs) { + t.Errorf("qux should export %q but exports %q", w, sdkLibs) + } + } +} + +func TestJavaSdkLibrary_DoNotAccessImplWhenItIsNotBuilt(t *testing.T) { + ctx, _ := testJava(t, ` + java_sdk_library { + name: "foo", + srcs: ["a.java"], + api_only: true, + public: { + enabled: true, + }, + } + + java_library { + name: "bar", + srcs: ["b.java"], + libs: ["foo"], } + `) + + // The bar library should depend on the stubs jar. + barLibrary := ctx.ModuleForTests("bar", "android_common").Rule("javac") + if expected, actual := `^-classpath .*:/[^:]*/turbine-combined/foo\.stubs\.jar$`, barLibrary.Args["classpath"]; !regexp.MustCompile(expected).MatchString(actual) { + t.Errorf("expected %q, found %#q", expected, actual) + } +} + +func TestJavaSdkLibrary_UseSourcesFromAnotherSdkLibrary(t *testing.T) { + testJava(t, ` + java_sdk_library { + name: "foo", + srcs: ["a.java"], + api_packages: ["foo"], + public: { + enabled: true, + }, + } + + java_library { + name: "bar", + srcs: ["b.java", ":foo{.public.stubs.source}"], + } + `) +} + +func TestJavaSdkLibrary_AccessOutputFiles_MissingScope(t *testing.T) { + testJavaError(t, `"foo" does not provide api scope system`, ` + java_sdk_library { + name: "foo", + srcs: ["a.java"], + api_packages: ["foo"], + public: { + enabled: true, + }, + } + + java_library { + name: "bar", + srcs: ["b.java", ":foo{.system.stubs.source}"], + } + `) +} + +func TestJavaSdkLibraryImport_AccessOutputFiles(t *testing.T) { + testJava(t, ` + java_sdk_library_import { + name: "foo", + public: { + jars: ["a.jar"], + stub_srcs: ["a.java"], + current_api: "api/current.txt", + removed_api: "api/removed.txt", + }, + } + + java_library { + name: "bar", + srcs: [":foo{.public.stubs.source}"], + java_resources: [ + ":foo{.public.api.txt}", + ":foo{.public.removed-api.txt}", + ], + } + `) +} + +func TestJavaSdkLibraryImport_AccessOutputFiles_Invalid(t *testing.T) { + bp := ` + java_sdk_library_import { + name: "foo", + public: { + jars: ["a.jar"], + }, + } + ` + + t.Run("stubs.source", func(t *testing.T) { + testJavaError(t, `stubs.source not available for api scope public`, bp+` + java_library { + name: "bar", + srcs: [":foo{.public.stubs.source}"], + java_resources: [ + ":foo{.public.api.txt}", + ":foo{.public.removed-api.txt}", + ], + } + `) + }) + + t.Run("api.txt", func(t *testing.T) { + testJavaError(t, `api.txt not available for api scope public`, bp+` + java_library { + name: "bar", + srcs: ["a.java"], + java_resources: [ + ":foo{.public.api.txt}", + ], + } + `) + }) + + t.Run("removed-api.txt", func(t *testing.T) { + testJavaError(t, `removed-api.txt not available for api scope public`, bp+` + java_library { + name: "bar", + srcs: ["a.java"], + java_resources: [ + ":foo{.public.removed-api.txt}", + ], + } + `) + }) +} + +func TestJavaSdkLibrary_InvalidScopes(t *testing.T) { + testJavaError(t, `module "foo": enabled api scope "system" depends on disabled scope "public"`, ` + java_sdk_library { + name: "foo", + srcs: ["a.java", "b.java"], + api_packages: ["foo"], + // Explicitly disable public to test the check that ensures the set of enabled + // scopes is consistent. + public: { + enabled: false, + }, + system: { + enabled: true, + }, + } + `) +} + +func TestJavaSdkLibrary_SdkVersion_ForScope(t *testing.T) { + testJava(t, ` + java_sdk_library { + name: "foo", + srcs: ["a.java", "b.java"], + api_packages: ["foo"], + system: { + enabled: true, + sdk_version: "module_current", + }, + } + `) +} + +func TestJavaSdkLibrary_ModuleLib(t *testing.T) { + testJava(t, ` + java_sdk_library { + name: "foo", + srcs: ["a.java", "b.java"], + api_packages: ["foo"], + system: { + enabled: true, + }, + module_lib: { + enabled: true, + }, + } + `) +} + +func TestJavaSdkLibrary_SystemServer(t *testing.T) { + testJava(t, ` + java_sdk_library { + name: "foo", + srcs: ["a.java", "b.java"], + api_packages: ["foo"], + system: { + enabled: true, + }, + system_server: { + enabled: true, + }, + } + `) +} + +func TestJavaSdkLibrary_MissingScope(t *testing.T) { + testJavaError(t, `requires api scope module-lib from foo but it only has \[\] available`, ` + java_sdk_library { + name: "foo", + srcs: ["a.java"], + public: { + enabled: false, + }, + } + + java_library { + name: "baz", + srcs: ["a.java"], + libs: ["foo"], + sdk_version: "module_current", + } + `) +} + +func TestJavaSdkLibrary_FallbackScope(t *testing.T) { + testJava(t, ` + java_sdk_library { + name: "foo", + srcs: ["a.java"], + system: { + enabled: true, + }, + } + + java_library { + name: "baz", + srcs: ["a.java"], + libs: ["foo"], + // foo does not have module-lib scope so it should fallback to system + sdk_version: "module_current", + } + `) +} + +func TestJavaSdkLibrary_DefaultToStubs(t *testing.T) { + ctx, _ := testJava(t, ` + java_sdk_library { + name: "foo", + srcs: ["a.java"], + system: { + enabled: true, + }, + default_to_stubs: true, + } + + java_library { + name: "baz", + srcs: ["a.java"], + libs: ["foo"], + // does not have sdk_version set, should fallback to module, + // which will then fallback to system because the module scope + // is not enabled. + } + `) + // The baz library should depend on the system stubs jar. + bazLibrary := ctx.ModuleForTests("baz", "android_common").Rule("javac") + if expected, actual := `^-classpath .*:/[^:]*/turbine-combined/foo\.stubs.system\.jar$`, bazLibrary.Args["classpath"]; !regexp.MustCompile(expected).MatchString(actual) { + t.Errorf("expected %q, found %#q", expected, actual) } } @@ -942,46 +1681,186 @@ func checkPatchModuleFlag(t *testing.T, ctx *android.TestContext, moduleName str } func TestPatchModule(t *testing.T) { - bp := ` - java_library { - name: "foo", - srcs: ["a.java"], - } - - java_library { - name: "bar", - srcs: ["b.java"], - no_standard_libs: true, - system_modules: "none", - patch_module: "java.base", - } + t.Run("Java language level 8", func(t *testing.T) { + // Test with legacy javac -source 1.8 -target 1.8 + bp := ` + java_library { + name: "foo", + srcs: ["a.java"], + java_version: "1.8", + } - java_library { - name: "baz", - srcs: ["c.java"], - patch_module: "java.base", - } - ` + java_library { + name: "bar", + srcs: ["b.java"], + sdk_version: "none", + system_modules: "none", + patch_module: "java.base", + java_version: "1.8", + } - t.Run("1.8", func(t *testing.T) { - // Test default javac 1.8 - ctx := testJava(t, bp) + java_library { + name: "baz", + srcs: ["c.java"], + patch_module: "java.base", + java_version: "1.8", + } + ` + ctx, _ := testJava(t, bp) checkPatchModuleFlag(t, ctx, "foo", "") checkPatchModuleFlag(t, ctx, "bar", "") checkPatchModuleFlag(t, ctx, "baz", "") }) - t.Run("1.9", func(t *testing.T) { - // Test again with javac 1.9 - config := testConfig(map[string]string{"EXPERIMENTAL_USE_OPENJDK9": "true"}) - ctx := testContext(config, bp, nil) - run(t, ctx, config) + t.Run("Java language level 9", func(t *testing.T) { + // Test with default javac -source 9 -target 9 + bp := ` + java_library { + name: "foo", + srcs: ["a.java"], + } + + java_library { + name: "bar", + srcs: ["b.java"], + sdk_version: "none", + system_modules: "none", + patch_module: "java.base", + } + + java_library { + name: "baz", + srcs: ["c.java"], + patch_module: "java.base", + } + ` + ctx, _ := testJava(t, bp) checkPatchModuleFlag(t, ctx, "foo", "") expected := "java.base=.:" + buildDir checkPatchModuleFlag(t, ctx, "bar", expected) - expected = "java.base=" + strings.Join([]string{".", buildDir, moduleToPath("ext"), moduleToPath("framework"), moduleToPath("updatable_media_stubs")}, ":") + expected = "java.base=" + strings.Join([]string{".", buildDir, moduleToPath("ext"), moduleToPath("framework")}, ":") checkPatchModuleFlag(t, ctx, "baz", expected) }) } + +func TestJavaSystemModules(t *testing.T) { + ctx, _ := testJava(t, ` + java_system_modules { + name: "system-modules", + libs: ["system-module1", "system-module2"], + } + java_library { + name: "system-module1", + srcs: ["a.java"], + sdk_version: "none", + system_modules: "none", + } + java_library { + name: "system-module2", + srcs: ["b.java"], + sdk_version: "none", + system_modules: "none", + } + `) + + // check the existence of the module + systemModules := ctx.ModuleForTests("system-modules", "android_common") + + cmd := systemModules.Rule("jarsTosystemModules") + + // make sure the command compiles against the supplied modules. + for _, module := range []string{"system-module1.jar", "system-module2.jar"} { + if !strings.Contains(cmd.Args["classpath"], module) { + t.Errorf("system modules classpath %v does not contain %q", cmd.Args["classpath"], + module) + } + } +} + +func TestJavaSystemModulesImport(t *testing.T) { + ctx, _ := testJava(t, ` + java_system_modules_import { + name: "system-modules", + libs: ["system-module1", "system-module2"], + } + java_import { + name: "system-module1", + jars: ["a.jar"], + } + java_import { + name: "system-module2", + jars: ["b.jar"], + } + `) + + // check the existence of the module + systemModules := ctx.ModuleForTests("system-modules", "android_common") + + cmd := systemModules.Rule("jarsTosystemModules") + + // make sure the command compiles against the supplied modules. + for _, module := range []string{"system-module1.jar", "system-module2.jar"} { + if !strings.Contains(cmd.Args["classpath"], module) { + t.Errorf("system modules classpath %v does not contain %q", cmd.Args["classpath"], + module) + } + } +} + +func TestJavaLibraryWithSystemModules(t *testing.T) { + ctx, _ := testJava(t, ` + java_library { + name: "lib-with-source-system-modules", + srcs: [ + "a.java", + ], + sdk_version: "none", + system_modules: "source-system-modules", + } + + java_library { + name: "source-jar", + srcs: [ + "a.java", + ], + } + + java_system_modules { + name: "source-system-modules", + libs: ["source-jar"], + } + + java_library { + name: "lib-with-prebuilt-system-modules", + srcs: [ + "a.java", + ], + sdk_version: "none", + system_modules: "prebuilt-system-modules", + } + + java_import { + name: "prebuilt-jar", + jars: ["a.jar"], + } + + java_system_modules_import { + name: "prebuilt-system-modules", + libs: ["prebuilt-jar"], + } + `) + + checkBootClasspathForSystemModule(t, ctx, "lib-with-source-system-modules", "/source-jar.jar") + + checkBootClasspathForSystemModule(t, ctx, "lib-with-prebuilt-system-modules", "/prebuilt-jar.jar") +} + +func checkBootClasspathForSystemModule(t *testing.T, ctx *android.TestContext, moduleName string, expectedSuffix string) { + javacRule := ctx.ModuleForTests(moduleName, "android_common").Rule("javac") + bootClasspath := javacRule.Args["bootClasspath"] + if strings.HasPrefix(bootClasspath, "--system ") && strings.HasSuffix(bootClasspath, expectedSuffix) { + t.Errorf("bootclasspath of %q must start with --system and end with %q, but was %#v.", moduleName, expectedSuffix, bootClasspath) + } +} diff --git a/java/jdeps.go b/java/jdeps.go index 18498befc..4f636a59f 100644 --- a/java/jdeps.go +++ b/java/jdeps.go @@ -17,7 +17,6 @@ package java import ( "encoding/json" "fmt" - "os" "android/soong/android" ) @@ -35,8 +34,11 @@ func jDepsGeneratorSingleton() android.Singleton { } type jdepsGeneratorSingleton struct { + outputPath android.Path } +var _ android.SingletonMakeVarsProvider = (*jdepsGeneratorSingleton)(nil) + const ( // Environment variables used to modify behavior of this singleton. envVariableCollectJavaDeps = "SOONG_COLLECT_JAVA_DEPS" @@ -72,6 +74,7 @@ func (j *jdepsGeneratorSingleton) GenerateBuildActions(ctx android.SingletonCont dpInfo.Aidl_include_dirs = android.FirstUniqueStrings(dpInfo.Aidl_include_dirs) dpInfo.Jarjar_rules = android.FirstUniqueStrings(dpInfo.Jarjar_rules) dpInfo.Jars = android.FirstUniqueStrings(dpInfo.Jars) + dpInfo.SrcJars = android.FirstUniqueStrings(dpInfo.SrcJars) moduleInfos[name] = dpInfo mkProvider, ok := module.(android.AndroidMkDataProvider) @@ -91,23 +94,36 @@ func (j *jdepsGeneratorSingleton) GenerateBuildActions(ctx android.SingletonCont moduleInfos[name] = dpInfo }) - jfpath := android.PathForOutput(ctx, jdepsJsonFileName).String() + jfpath := android.PathForOutput(ctx, jdepsJsonFileName) err := createJsonFile(moduleInfos, jfpath) if err != nil { ctx.Errorf(err.Error()) } + j.outputPath = jfpath + + // This is necessary to satisfy the dangling rules check as this file is written by Soong rather than a rule. + ctx.Build(pctx, android.BuildParams{ + Rule: android.Touch, + Output: jfpath, + }) } -func createJsonFile(moduleInfos map[string]android.IdeInfo, jfpath string) error { - file, err := os.Create(jfpath) - if err != nil { - return fmt.Errorf("Failed to create file: %s, relative: %v", jdepsJsonFileName, err) +func (j *jdepsGeneratorSingleton) MakeVars(ctx android.MakeVarsContext) { + if j.outputPath == nil { + return } - defer file.Close() + + ctx.DistForGoal("general-tests", j.outputPath) +} + +func createJsonFile(moduleInfos map[string]android.IdeInfo, jfpath android.WritablePath) error { buf, err := json.MarshalIndent(moduleInfos, "", "\t") if err != nil { - return fmt.Errorf("Write file failed: %s, relative: %v", jdepsJsonFileName, err) + return fmt.Errorf("JSON marshal of java deps failed: %s", err) + } + err = android.WriteFileToOutputDir(jfpath, buf, 0666) + if err != nil { + return fmt.Errorf("Writing java deps to %s failed: %s", jfpath.String(), err) } - fmt.Fprintf(file, string(buf)) return nil } diff --git a/java/kotlin.go b/java/kotlin.go index 58dc64cd1..673970b9c 100644 --- a/java/kotlin.go +++ b/java/kotlin.go @@ -18,6 +18,7 @@ import ( "bytes" "encoding/base64" "encoding/binary" + "path/filepath" "strings" "android/soong/android" @@ -27,16 +28,23 @@ import ( var kotlinc = pctx.AndroidRemoteStaticRule("kotlinc", android.RemoteRuleSupports{Goma: true}, blueprint.RuleParams{ - Command: `rm -rf "$classesDir" "$srcJarDir" "$kotlinBuildFile" && mkdir -p "$classesDir" "$srcJarDir" && ` + + Command: `rm -rf "$classesDir" "$srcJarDir" "$kotlinBuildFile" "$emptyDir" && ` + + `mkdir -p "$classesDir" "$srcJarDir" "$emptyDir" && ` + `${config.ZipSyncCmd} -d $srcJarDir -l $srcJarDir/list -f "*.java" $srcJars && ` + - `${config.GenKotlinBuildFileCmd} $classpath $classesDir $out.rsp $srcJarDir/list > $kotlinBuildFile &&` + + `${config.GenKotlinBuildFileCmd} $classpath "$name" $classesDir $out.rsp $srcJarDir/list > $kotlinBuildFile &&` + `${config.KotlincCmd} ${config.JavacHeapFlags} $kotlincFlags ` + - `-jvm-target $kotlinJvmTarget -Xbuild-file=$kotlinBuildFile && ` + + `-jvm-target $kotlinJvmTarget -Xbuild-file=$kotlinBuildFile -kotlin-home $emptyDir && ` + `${config.SoongZipCmd} -jar -o $out -C $classesDir -D $classesDir && ` + `rm -rf "$srcJarDir"`, CommandDeps: []string{ "${config.KotlincCmd}", "${config.KotlinCompilerJar}", + "${config.KotlinPreloaderJar}", + "${config.KotlinReflectJar}", + "${config.KotlinScriptRuntimeJar}", + "${config.KotlinStdlibJar}", + "${config.KotlinTrove4jJar}", + "${config.KotlinAnnotationJar}", "${config.GenKotlinBuildFileCmd}", "${config.SoongZipCmd}", "${config.ZipSyncCmd}", @@ -44,7 +52,8 @@ var kotlinc = pctx.AndroidRemoteStaticRule("kotlinc", android.RemoteRuleSupports Rspfile: "$out.rsp", RspfileContent: `$in`, }, - "kotlincFlags", "classpath", "srcJars", "srcJarDir", "classesDir", "kotlinJvmTarget", "kotlinBuildFile") + "kotlincFlags", "classpath", "srcJars", "srcJarDir", "classesDir", "kotlinJvmTarget", "kotlinBuildFile", + "emptyDir", "name") // kotlinCompile takes .java and .kt sources and srcJars, and compiles the .kt sources into a classes jar in outputFile. func kotlinCompile(ctx android.ModuleContext, outputFile android.WritablePath, @@ -55,6 +64,9 @@ func kotlinCompile(ctx android.ModuleContext, outputFile android.WritablePath, deps = append(deps, flags.kotlincClasspath...) deps = append(deps, srcJars...) + kotlinName := filepath.Join(ctx.ModuleDir(), ctx.ModuleSubDir(), ctx.ModuleName()) + kotlinName = strings.ReplaceAll(kotlinName, "/", "__") + ctx.Build(pctx, android.BuildParams{ Rule: kotlinc, Description: "kotlinc", @@ -68,17 +80,20 @@ func kotlinCompile(ctx android.ModuleContext, outputFile android.WritablePath, "classesDir": android.PathForModuleOut(ctx, "kotlinc", "classes").String(), "srcJarDir": android.PathForModuleOut(ctx, "kotlinc", "srcJars").String(), "kotlinBuildFile": android.PathForModuleOut(ctx, "kotlinc-build.xml").String(), + "emptyDir": android.PathForModuleOut(ctx, "kotlinc", "empty").String(), // http://b/69160377 kotlinc only supports -jvm-target 1.6 and 1.8 "kotlinJvmTarget": "1.8", + "name": kotlinName, }, }) } var kapt = pctx.AndroidRemoteStaticRule("kapt", android.RemoteRuleSupports{Goma: true}, blueprint.RuleParams{ - Command: `rm -rf "$srcJarDir" "$kotlinBuildFile" "$kaptDir" && mkdir -p "$srcJarDir" "$kaptDir" && ` + + Command: `rm -rf "$srcJarDir" "$kotlinBuildFile" "$kaptDir" && ` + + `mkdir -p "$srcJarDir" "$kaptDir/sources" "$kaptDir/classes" && ` + `${config.ZipSyncCmd} -d $srcJarDir -l $srcJarDir/list -f "*.java" $srcJars && ` + - `${config.GenKotlinBuildFileCmd} $classpath "" $out.rsp $srcJarDir/list > $kotlinBuildFile &&` + + `${config.GenKotlinBuildFileCmd} $classpath "$name" "" $out.rsp $srcJarDir/list > $kotlinBuildFile &&` + `${config.KotlincCmd} ${config.KotlincSuppressJDK9Warnings} ${config.JavacHeapFlags} $kotlincFlags ` + `-Xplugin=${config.KotlinKaptJar} ` + `-P plugin:org.jetbrains.kotlin.kapt3:sources=$kaptDir/sources ` + @@ -91,6 +106,7 @@ var kapt = pctx.AndroidRemoteStaticRule("kapt", android.RemoteRuleSupports{Goma: `$kaptProcessor ` + `-Xbuild-file=$kotlinBuildFile && ` + `${config.SoongZipCmd} -jar -o $out -C $kaptDir/sources -D $kaptDir/sources && ` + + `${config.SoongZipCmd} -jar -o $classesJarOut -C $kaptDir/classes -D $kaptDir/classes && ` + `rm -rf "$srcJarDir"`, CommandDeps: []string{ "${config.KotlincCmd}", @@ -104,13 +120,14 @@ var kapt = pctx.AndroidRemoteStaticRule("kapt", android.RemoteRuleSupports{Goma: RspfileContent: `$in`, }, "kotlincFlags", "encodedJavacFlags", "kaptProcessorPath", "kaptProcessor", - "classpath", "srcJars", "srcJarDir", "kaptDir", "kotlinJvmTarget", "kotlinBuildFile") + "classpath", "srcJars", "srcJarDir", "kaptDir", "kotlinJvmTarget", "kotlinBuildFile", "name", + "classesJarOut") // kotlinKapt performs Kotlin-compatible annotation processing. It takes .kt and .java sources and srcjars, and runs // annotation processors over all of them, producing a srcjar of generated code in outputFile. The srcjar should be // added as an additional input to kotlinc and javac rules, and the javac rule should have annotation processing // disabled. -func kotlinKapt(ctx android.ModuleContext, outputFile android.WritablePath, +func kotlinKapt(ctx android.ModuleContext, srcJarOutputFile, resJarOutputFile android.WritablePath, srcFiles, srcJars android.Paths, flags javaBuilderFlags) { @@ -119,24 +136,31 @@ func kotlinKapt(ctx android.ModuleContext, outputFile android.WritablePath, deps = append(deps, srcJars...) deps = append(deps, flags.processorPath...) - kaptProcessorPath := flags.processorPath.FormTurbineClasspath("-P plugin:org.jetbrains.kotlin.kapt3:apclasspath=") + kaptProcessorPath := flags.processorPath.FormRepeatedClassPath("-P plugin:org.jetbrains.kotlin.kapt3:apclasspath=") kaptProcessor := "" - if flags.processor != "" { - kaptProcessor = "-P plugin:org.jetbrains.kotlin.kapt3:processors=" + flags.processor + for i, p := range flags.processors { + if i > 0 { + kaptProcessor += " " + } + kaptProcessor += "-P plugin:org.jetbrains.kotlin.kapt3:processors=" + p } encodedJavacFlags := kaptEncodeFlags([][2]string{ - {"-source", flags.javaVersion}, - {"-target", flags.javaVersion}, + {"-source", flags.javaVersion.String()}, + {"-target", flags.javaVersion.String()}, }) + kotlinName := filepath.Join(ctx.ModuleDir(), ctx.ModuleSubDir(), ctx.ModuleName()) + kotlinName = strings.ReplaceAll(kotlinName, "/", "__") + ctx.Build(pctx, android.BuildParams{ - Rule: kapt, - Description: "kapt", - Output: outputFile, - Inputs: srcFiles, - Implicits: deps, + Rule: kapt, + Description: "kapt", + Output: srcJarOutputFile, + ImplicitOutput: resJarOutputFile, + Inputs: srcFiles, + Implicits: deps, Args: map[string]string{ "classpath": flags.kotlincClasspath.FormJavaClassPath("-classpath"), "kotlincFlags": flags.kotlincFlags, @@ -147,6 +171,8 @@ func kotlinKapt(ctx android.ModuleContext, outputFile android.WritablePath, "kaptProcessor": kaptProcessor, "kaptDir": android.PathForModuleOut(ctx, "kapt/gen").String(), "encodedJavacFlags": encodedJavacFlags, + "name": kotlinName, + "classesJarOut": resJarOutputFile.String(), }, }) } diff --git a/java/kotlin_test.go b/java/kotlin_test.go index e0eb0c06d..60ca1c476 100644 --- a/java/kotlin_test.go +++ b/java/kotlin_test.go @@ -22,7 +22,7 @@ import ( ) func TestKotlin(t *testing.T) { - ctx := testJava(t, ` + ctx, _ := testJava(t, ` java_library { name: "foo", srcs: ["a.java", "b.kt"], @@ -84,11 +84,11 @@ func TestKotlin(t *testing.T) { } func TestKapt(t *testing.T) { - ctx := testJava(t, ` + ctx, _ := testJava(t, ` java_library { name: "foo", srcs: ["a.java", "b.kt"], - plugins: ["bar"], + plugins: ["bar", "baz"], } java_plugin { @@ -96,6 +96,12 @@ func TestKapt(t *testing.T) { processor_class: "com.bar", srcs: ["b.java"], } + + java_plugin { + name: "baz", + processor_class: "com.baz", + srcs: ["b.java"], + } `) buildOS := android.BuildOs.String() @@ -105,6 +111,7 @@ func TestKapt(t *testing.T) { javac := ctx.ModuleForTests("foo", "android_common").Rule("javac") bar := ctx.ModuleForTests("bar", buildOS+"_common").Rule("javac").Output.String() + baz := ctx.ModuleForTests("baz", buildOS+"_common").Rule("javac").Output.String() // Test that the kotlin and java sources are passed to kapt and kotlinc if len(kapt.Inputs) != 2 || kapt.Inputs[0].String() != "a.java" || kapt.Inputs[1].String() != "b.kt" { @@ -136,11 +143,12 @@ func TestKapt(t *testing.T) { } // Test that the processors are passed to kapt - expectedProcessorPath := "-P plugin:org.jetbrains.kotlin.kapt3:apclasspath=" + bar + expectedProcessorPath := "-P plugin:org.jetbrains.kotlin.kapt3:apclasspath=" + bar + + " -P plugin:org.jetbrains.kotlin.kapt3:apclasspath=" + baz if kapt.Args["kaptProcessorPath"] != expectedProcessorPath { t.Errorf("expected kaptProcessorPath %q, got %q", expectedProcessorPath, kapt.Args["kaptProcessorPath"]) } - expectedProcessor := "-P plugin:org.jetbrains.kotlin.kapt3:processors=com.bar" + expectedProcessor := "-P plugin:org.jetbrains.kotlin.kapt3:processors=com.bar -P plugin:org.jetbrains.kotlin.kapt3:processors=com.baz" if kapt.Args["kaptProcessor"] != expectedProcessor { t.Errorf("expected kaptProcessor %q, got %q", expectedProcessor, kapt.Args["kaptProcessor"]) } diff --git a/java/lint.go b/java/lint.go new file mode 100644 index 000000000..639106793 --- /dev/null +++ b/java/lint.go @@ -0,0 +1,519 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package java + +import ( + "fmt" + "sort" + "strings" + + "android/soong/android" +) + +type LintProperties struct { + // Controls for running Android Lint on the module. + Lint struct { + + // If true, run Android Lint on the module. Defaults to true. + Enabled *bool + + // Flags to pass to the Android Lint tool. + Flags []string + + // Checks that should be treated as fatal. + Fatal_checks []string + + // Checks that should be treated as errors. + Error_checks []string + + // Checks that should be treated as warnings. + Warning_checks []string + + // Checks that should be skipped. + Disabled_checks []string + + // Modules that provide extra lint checks + Extra_check_modules []string + } +} + +type linter struct { + name string + manifest android.Path + mergedManifest android.Path + srcs android.Paths + srcJars android.Paths + resources android.Paths + classpath android.Paths + classes android.Path + extraLintCheckJars android.Paths + test bool + library bool + minSdkVersion string + targetSdkVersion string + compileSdkVersion string + javaLanguageLevel string + kotlinLanguageLevel string + outputs lintOutputs + properties LintProperties + + reports android.Paths + + buildModuleReportZip bool +} + +type lintOutputs struct { + html android.Path + text android.Path + xml android.Path + + depSets LintDepSets +} + +type lintOutputsIntf interface { + lintOutputs() *lintOutputs +} + +type lintDepSetsIntf interface { + LintDepSets() LintDepSets +} + +type LintDepSets struct { + HTML, Text, XML *android.DepSet +} + +type LintDepSetsBuilder struct { + HTML, Text, XML *android.DepSetBuilder +} + +func NewLintDepSetBuilder() LintDepSetsBuilder { + return LintDepSetsBuilder{ + HTML: android.NewDepSetBuilder(android.POSTORDER), + Text: android.NewDepSetBuilder(android.POSTORDER), + XML: android.NewDepSetBuilder(android.POSTORDER), + } +} + +func (l LintDepSetsBuilder) Direct(html, text, xml android.Path) LintDepSetsBuilder { + l.HTML.Direct(html) + l.Text.Direct(text) + l.XML.Direct(xml) + return l +} + +func (l LintDepSetsBuilder) Transitive(depSets LintDepSets) LintDepSetsBuilder { + if depSets.HTML != nil { + l.HTML.Transitive(depSets.HTML) + } + if depSets.Text != nil { + l.Text.Transitive(depSets.Text) + } + if depSets.XML != nil { + l.XML.Transitive(depSets.XML) + } + return l +} + +func (l LintDepSetsBuilder) Build() LintDepSets { + return LintDepSets{ + HTML: l.HTML.Build(), + Text: l.Text.Build(), + XML: l.XML.Build(), + } +} + +func (l *linter) LintDepSets() LintDepSets { + return l.outputs.depSets +} + +var _ lintDepSetsIntf = (*linter)(nil) + +var _ lintOutputsIntf = (*linter)(nil) + +func (l *linter) lintOutputs() *lintOutputs { + return &l.outputs +} + +func (l *linter) enabled() bool { + return BoolDefault(l.properties.Lint.Enabled, true) +} + +func (l *linter) deps(ctx android.BottomUpMutatorContext) { + if !l.enabled() { + return + } + + extraCheckModules := l.properties.Lint.Extra_check_modules + + if checkOnly := ctx.Config().Getenv("ANDROID_LINT_CHECK"); checkOnly != "" { + if checkOnlyModules := ctx.Config().Getenv("ANDROID_LINT_CHECK_EXTRA_MODULES"); checkOnlyModules != "" { + extraCheckModules = strings.Split(checkOnlyModules, ",") + } + } + + ctx.AddFarVariationDependencies(ctx.Config().BuildOSCommonTarget.Variations(), + extraLintCheckTag, extraCheckModules...) +} + +func (l *linter) writeLintProjectXML(ctx android.ModuleContext, + rule *android.RuleBuilder) (projectXMLPath, configXMLPath, cacheDir, homeDir android.WritablePath, deps android.Paths) { + + var resourcesList android.WritablePath + if len(l.resources) > 0 { + // The list of resources may be too long to put on the command line, but + // we can't use the rsp file because it is already being used for srcs. + // Insert a second rule to write out the list of resources to a file. + resourcesList = android.PathForModuleOut(ctx, "lint", "resources.list") + resListRule := android.NewRuleBuilder() + resListRule.Command().Text("cp").FlagWithRspFileInputList("", l.resources).Output(resourcesList) + resListRule.Build(pctx, ctx, "lint_resources_list", "lint resources list") + deps = append(deps, l.resources...) + } + + projectXMLPath = android.PathForModuleOut(ctx, "lint", "project.xml") + // Lint looks for a lint.xml file next to the project.xml file, give it one. + configXMLPath = android.PathForModuleOut(ctx, "lint", "lint.xml") + cacheDir = android.PathForModuleOut(ctx, "lint", "cache") + homeDir = android.PathForModuleOut(ctx, "lint", "home") + + srcJarDir := android.PathForModuleOut(ctx, "lint-srcjars") + srcJarList := zipSyncCmd(ctx, rule, srcJarDir, l.srcJars) + + cmd := rule.Command(). + BuiltTool(ctx, "lint-project-xml"). + FlagWithOutput("--project_out ", projectXMLPath). + FlagWithOutput("--config_out ", configXMLPath). + FlagWithArg("--name ", ctx.ModuleName()) + + if l.library { + cmd.Flag("--library") + } + if l.test { + cmd.Flag("--test") + } + if l.manifest != nil { + deps = append(deps, l.manifest) + cmd.FlagWithArg("--manifest ", l.manifest.String()) + } + if l.mergedManifest != nil { + deps = append(deps, l.mergedManifest) + cmd.FlagWithArg("--merged_manifest ", l.mergedManifest.String()) + } + + // TODO(ccross): some of the files in l.srcs are generated sources and should be passed to + // lint separately. + cmd.FlagWithRspFileInputList("--srcs ", l.srcs) + deps = append(deps, l.srcs...) + + cmd.FlagWithInput("--generated_srcs ", srcJarList) + deps = append(deps, l.srcJars...) + + if resourcesList != nil { + cmd.FlagWithInput("--resources ", resourcesList) + } + + if l.classes != nil { + deps = append(deps, l.classes) + cmd.FlagWithArg("--classes ", l.classes.String()) + } + + cmd.FlagForEachArg("--classpath ", l.classpath.Strings()) + deps = append(deps, l.classpath...) + + cmd.FlagForEachArg("--extra_checks_jar ", l.extraLintCheckJars.Strings()) + deps = append(deps, l.extraLintCheckJars...) + + cmd.FlagWithArg("--root_dir ", "$PWD") + + // The cache tag in project.xml is relative to the root dir, or the project.xml file if + // the root dir is not set. + cmd.FlagWithArg("--cache_dir ", cacheDir.String()) + + cmd.FlagWithInput("@", + android.PathForSource(ctx, "build/soong/java/lint_defaults.txt")) + + cmd.FlagForEachArg("--disable_check ", l.properties.Lint.Disabled_checks) + cmd.FlagForEachArg("--warning_check ", l.properties.Lint.Warning_checks) + cmd.FlagForEachArg("--error_check ", l.properties.Lint.Error_checks) + cmd.FlagForEachArg("--fatal_check ", l.properties.Lint.Fatal_checks) + + return projectXMLPath, configXMLPath, cacheDir, homeDir, deps +} + +// generateManifest adds a command to the rule to write a dummy manifest cat contains the +// minSdkVersion and targetSdkVersion for modules (like java_library) that don't have a manifest. +func (l *linter) generateManifest(ctx android.ModuleContext, rule *android.RuleBuilder) android.Path { + manifestPath := android.PathForModuleOut(ctx, "lint", "AndroidManifest.xml") + + rule.Command().Text("("). + Text(`echo "<?xml version='1.0' encoding='utf-8'?>" &&`). + Text(`echo "<manifest xmlns:android='http://schemas.android.com/apk/res/android'" &&`). + Text(`echo " android:versionCode='1' android:versionName='1' >" &&`). + Textf(`echo " <uses-sdk android:minSdkVersion='%s' android:targetSdkVersion='%s'/>" &&`, + l.minSdkVersion, l.targetSdkVersion). + Text(`echo "</manifest>"`). + Text(") >").Output(manifestPath) + + return manifestPath +} + +func (l *linter) lint(ctx android.ModuleContext) { + if !l.enabled() { + return + } + + extraLintCheckModules := ctx.GetDirectDepsWithTag(extraLintCheckTag) + for _, extraLintCheckModule := range extraLintCheckModules { + if dep, ok := extraLintCheckModule.(Dependency); ok { + l.extraLintCheckJars = append(l.extraLintCheckJars, dep.ImplementationAndResourcesJars()...) + } else { + ctx.PropertyErrorf("lint.extra_check_modules", + "%s is not a java module", ctx.OtherModuleName(extraLintCheckModule)) + } + } + + rule := android.NewRuleBuilder() + + if l.manifest == nil { + manifest := l.generateManifest(ctx, rule) + l.manifest = manifest + } + + projectXML, lintXML, cacheDir, homeDir, deps := l.writeLintProjectXML(ctx, rule) + + html := android.PathForModuleOut(ctx, "lint-report.html") + text := android.PathForModuleOut(ctx, "lint-report.txt") + xml := android.PathForModuleOut(ctx, "lint-report.xml") + + depSetsBuilder := NewLintDepSetBuilder().Direct(html, text, xml) + + ctx.VisitDirectDepsWithTag(staticLibTag, func(dep android.Module) { + if depLint, ok := dep.(lintDepSetsIntf); ok { + depSetsBuilder.Transitive(depLint.LintDepSets()) + } + }) + + rule.Command().Text("rm -rf").Flag(cacheDir.String()).Flag(homeDir.String()) + rule.Command().Text("mkdir -p").Flag(cacheDir.String()).Flag(homeDir.String()) + + var annotationsZipPath, apiVersionsXMLPath android.Path + if ctx.Config().UnbundledBuildUsePrebuiltSdks() { + annotationsZipPath = android.PathForSource(ctx, "prebuilts/sdk/current/public/data/annotations.zip") + apiVersionsXMLPath = android.PathForSource(ctx, "prebuilts/sdk/current/public/data/api-versions.xml") + } else { + annotationsZipPath = copiedAnnotationsZipPath(ctx) + apiVersionsXMLPath = copiedAPIVersionsXmlPath(ctx) + } + + cmd := rule.Command(). + Text("("). + Flag("JAVA_OPTS=-Xmx2048m"). + FlagWithArg("ANDROID_SDK_HOME=", homeDir.String()). + FlagWithInput("SDK_ANNOTATIONS=", annotationsZipPath). + FlagWithInput("LINT_OPTS=-DLINT_API_DATABASE=", apiVersionsXMLPath). + Tool(android.PathForSource(ctx, "prebuilts/cmdline-tools/tools/bin/lint")). + Implicit(android.PathForSource(ctx, "prebuilts/cmdline-tools/tools/lib/lint-classpath.jar")). + Flag("--quiet"). + FlagWithInput("--project ", projectXML). + FlagWithInput("--config ", lintXML). + FlagWithOutput("--html ", html). + FlagWithOutput("--text ", text). + FlagWithOutput("--xml ", xml). + FlagWithArg("--compile-sdk-version ", l.compileSdkVersion). + FlagWithArg("--java-language-level ", l.javaLanguageLevel). + FlagWithArg("--kotlin-language-level ", l.kotlinLanguageLevel). + FlagWithArg("--url ", fmt.Sprintf(".=.,%s=out", android.PathForOutput(ctx).String())). + Flag("--exitcode"). + Flags(l.properties.Lint.Flags). + Implicits(deps) + + if checkOnly := ctx.Config().Getenv("ANDROID_LINT_CHECK"); checkOnly != "" { + cmd.FlagWithArg("--check ", checkOnly) + } + + cmd.Text("|| (").Text("cat").Input(text).Text("; exit 7)").Text(")") + + rule.Command().Text("rm -rf").Flag(cacheDir.String()).Flag(homeDir.String()) + + rule.Build(pctx, ctx, "lint", "lint") + + l.outputs = lintOutputs{ + html: html, + text: text, + xml: xml, + + depSets: depSetsBuilder.Build(), + } + + if l.buildModuleReportZip { + l.reports = BuildModuleLintReportZips(ctx, l.LintDepSets()) + } +} + +func BuildModuleLintReportZips(ctx android.ModuleContext, depSets LintDepSets) android.Paths { + htmlList := depSets.HTML.ToSortedList() + textList := depSets.Text.ToSortedList() + xmlList := depSets.XML.ToSortedList() + + if len(htmlList) == 0 && len(textList) == 0 && len(xmlList) == 0 { + return nil + } + + htmlZip := android.PathForModuleOut(ctx, "lint-report-html.zip") + lintZip(ctx, htmlList, htmlZip) + + textZip := android.PathForModuleOut(ctx, "lint-report-text.zip") + lintZip(ctx, textList, textZip) + + xmlZip := android.PathForModuleOut(ctx, "lint-report-xml.zip") + lintZip(ctx, xmlList, xmlZip) + + return android.Paths{htmlZip, textZip, xmlZip} +} + +type lintSingleton struct { + htmlZip android.WritablePath + textZip android.WritablePath + xmlZip android.WritablePath +} + +func (l *lintSingleton) GenerateBuildActions(ctx android.SingletonContext) { + l.generateLintReportZips(ctx) + l.copyLintDependencies(ctx) +} + +func (l *lintSingleton) copyLintDependencies(ctx android.SingletonContext) { + if ctx.Config().UnbundledBuildUsePrebuiltSdks() { + return + } + + var frameworkDocStubs android.Module + ctx.VisitAllModules(func(m android.Module) { + if ctx.ModuleName(m) == "framework-doc-stubs" { + if frameworkDocStubs == nil { + frameworkDocStubs = m + } else { + ctx.Errorf("lint: multiple framework-doc-stubs modules found: %s and %s", + ctx.ModuleSubDir(m), ctx.ModuleSubDir(frameworkDocStubs)) + } + } + }) + + if frameworkDocStubs == nil { + if !ctx.Config().AllowMissingDependencies() { + ctx.Errorf("lint: missing framework-doc-stubs") + } + return + } + + ctx.Build(pctx, android.BuildParams{ + Rule: android.Cp, + Input: android.OutputFileForModule(ctx, frameworkDocStubs, ".annotations.zip"), + Output: copiedAnnotationsZipPath(ctx), + }) + + ctx.Build(pctx, android.BuildParams{ + Rule: android.Cp, + Input: android.OutputFileForModule(ctx, frameworkDocStubs, ".api_versions.xml"), + Output: copiedAPIVersionsXmlPath(ctx), + }) +} + +func copiedAnnotationsZipPath(ctx android.PathContext) android.WritablePath { + return android.PathForOutput(ctx, "lint", "annotations.zip") +} + +func copiedAPIVersionsXmlPath(ctx android.PathContext) android.WritablePath { + return android.PathForOutput(ctx, "lint", "api_versions.xml") +} + +func (l *lintSingleton) generateLintReportZips(ctx android.SingletonContext) { + if ctx.Config().UnbundledBuild() { + return + } + + var outputs []*lintOutputs + var dirs []string + ctx.VisitAllModules(func(m android.Module) { + if ctx.Config().EmbeddedInMake() && !m.ExportedToMake() { + return + } + + if apex, ok := m.(android.ApexModule); ok && apex.NotAvailableForPlatform() && apex.IsForPlatform() { + // There are stray platform variants of modules in apexes that are not available for + // the platform, and they sometimes can't be built. Don't depend on them. + return + } + + if l, ok := m.(lintOutputsIntf); ok { + outputs = append(outputs, l.lintOutputs()) + } + }) + + dirs = android.SortedUniqueStrings(dirs) + + zip := func(outputPath android.WritablePath, get func(*lintOutputs) android.Path) { + var paths android.Paths + + for _, output := range outputs { + if p := get(output); p != nil { + paths = append(paths, p) + } + } + + lintZip(ctx, paths, outputPath) + } + + l.htmlZip = android.PathForOutput(ctx, "lint-report-html.zip") + zip(l.htmlZip, func(l *lintOutputs) android.Path { return l.html }) + + l.textZip = android.PathForOutput(ctx, "lint-report-text.zip") + zip(l.textZip, func(l *lintOutputs) android.Path { return l.text }) + + l.xmlZip = android.PathForOutput(ctx, "lint-report-xml.zip") + zip(l.xmlZip, func(l *lintOutputs) android.Path { return l.xml }) + + ctx.Phony("lint-check", l.htmlZip, l.textZip, l.xmlZip) +} + +func (l *lintSingleton) MakeVars(ctx android.MakeVarsContext) { + if !ctx.Config().UnbundledBuild() { + ctx.DistForGoal("lint-check", l.htmlZip, l.textZip, l.xmlZip) + } +} + +var _ android.SingletonMakeVarsProvider = (*lintSingleton)(nil) + +func init() { + android.RegisterSingletonType("lint", + func() android.Singleton { return &lintSingleton{} }) +} + +func lintZip(ctx android.BuilderContext, paths android.Paths, outputPath android.WritablePath) { + paths = android.SortedUniquePaths(android.CopyOfPaths(paths)) + + sort.Slice(paths, func(i, j int) bool { + return paths[i].String() < paths[j].String() + }) + + rule := android.NewRuleBuilder() + + rule.Command().BuiltTool(ctx, "soong_zip"). + FlagWithOutput("-o ", outputPath). + FlagWithArg("-C ", android.PathForIntermediates(ctx).String()). + FlagWithRspFileInputList("-l ", paths) + + rule.Build(pctx, ctx, outputPath.Base(), outputPath.Base()) +} diff --git a/java/lint_defaults.txt b/java/lint_defaults.txt new file mode 100644 index 000000000..0786b7c00 --- /dev/null +++ b/java/lint_defaults.txt @@ -0,0 +1,78 @@ +# Treat LintError as fatal to catch invocation errors +--fatal_check LintError + +# Downgrade existing errors to warnings +--warning_check AppCompatResource # 55 occurences in 10 modules +--warning_check AppLinkUrlError # 111 occurences in 53 modules +--warning_check BlockedPrivateApi # 2 occurences in 2 modules +--warning_check ByteOrderMark # 2 occurences in 2 modules +--warning_check DuplicateActivity # 3 occurences in 3 modules +--warning_check DuplicateDefinition # 3623 occurences in 48 modules +--warning_check DuplicateIds # 207 occurences in 22 modules +--warning_check EllipsizeMaxLines # 12 occurences in 7 modules +--warning_check ExtraTranslation # 21276 occurences in 27 modules +--warning_check FontValidationError # 4 occurences in 1 modules +--warning_check FullBackupContent # 16 occurences in 1 modules +--warning_check GetContentDescriptionOverride # 3 occurences in 2 modules +--warning_check HalfFloat # 31 occurences in 1 modules +--warning_check HardcodedDebugMode # 99 occurences in 95 modules +--warning_check ImpliedQuantity # 703 occurences in 27 modules +--warning_check ImpliedTouchscreenHardware # 4 occurences in 4 modules +--warning_check IncludeLayoutParam # 11 occurences in 6 modules +--warning_check Instantiatable # 145 occurences in 19 modules +--warning_check InvalidPermission # 6 occurences in 4 modules +--warning_check InvalidUsesTagAttribute # 6 occurences in 2 modules +--warning_check InvalidWakeLockTag # 111 occurences in 37 modules +--warning_check JavascriptInterface # 3 occurences in 2 modules +--warning_check LibraryCustomView # 9 occurences in 4 modules +--warning_check LogTagMismatch # 81 occurences in 13 modules +--warning_check LongLogTag # 249 occurences in 12 modules +--warning_check MenuTitle # 5 occurences in 4 modules +--warning_check MissingClass # 537 occurences in 141 modules +--warning_check MissingConstraints # 39 occurences in 10 modules +--warning_check MissingDefaultResource # 1257 occurences in 40 modules +--warning_check MissingIntentFilterForMediaSearch # 1 occurences in 1 modules +--warning_check MissingLeanbackLauncher # 3 occurences in 3 modules +--warning_check MissingLeanbackSupport # 2 occurences in 2 modules +--warning_check MissingOnPlayFromSearch # 1 occurences in 1 modules +--warning_check MissingPermission # 2071 occurences in 150 modules +--warning_check MissingPrefix # 46 occurences in 41 modules +--warning_check MissingQuantity # 100 occurences in 1 modules +--warning_check MissingSuperCall # 121 occurences in 36 modules +--warning_check MissingTvBanner # 3 occurences in 3 modules +--warning_check NamespaceTypo # 3 occurences in 3 modules +--warning_check NetworkSecurityConfig # 46 occurences in 12 modules +--warning_check NewApi # 1996 occurences in 122 modules +--warning_check NotSibling # 15 occurences in 10 modules +--warning_check ObjectAnimatorBinding # 14 occurences in 5 modules +--warning_check OnClick # 49 occurences in 21 modules +--warning_check Orientation # 77 occurences in 19 modules +--warning_check Override # 385 occurences in 36 modules +--warning_check ParcelCreator # 23 occurences in 2 modules +--warning_check ProtectedPermissions # 2413 occurences in 381 modules +--warning_check Range # 80 occurences in 28 modules +--warning_check RecyclerView # 1 occurences in 1 modules +--warning_check ReferenceType # 4 occurences in 1 modules +--warning_check ResourceAsColor # 19 occurences in 14 modules +--warning_check RequiredSize # 52 occurences in 13 modules +--warning_check ResAuto # 3 occurences in 1 modules +--warning_check ResourceCycle # 37 occurences in 10 modules +--warning_check ResourceType # 137 occurences in 36 modules +--warning_check RestrictedApi # 28 occurences in 5 modules +--warning_check RtlCompat # 9 occurences in 6 modules +--warning_check ServiceCast # 3 occurences in 1 modules +--warning_check SoonBlockedPrivateApi # 5 occurences in 3 modules +--warning_check StringFormatInvalid # 148 occurences in 11 modules +--warning_check StringFormatMatches # 4800 occurences in 30 modules +--warning_check UnknownId # 8 occurences in 7 modules +--warning_check ValidFragment # 12 occurences in 5 modules +--warning_check ValidRestrictions # 5 occurences in 1 modules +--warning_check WebViewLayout # 3 occurences in 1 modules +--warning_check WrongCall # 21 occurences in 3 modules +--warning_check WrongConstant # 894 occurences in 126 modules +--warning_check WrongManifestParent # 10 occurences in 4 modules +--warning_check WrongThread # 14 occurences in 6 modules +--warning_check WrongViewCast # 1 occurences in 1 modules + +# TODO(b/158390965): remove this when lint doesn't crash +--disable_check HardcodedDebugMode diff --git a/java/platform_compat_config.go b/java/platform_compat_config.go new file mode 100644 index 000000000..cb8e6841a --- /dev/null +++ b/java/platform_compat_config.go @@ -0,0 +1,197 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package java + +import ( + "android/soong/android" + "fmt" +) + +func init() { + android.RegisterSingletonType("platform_compat_config_singleton", platformCompatConfigSingletonFactory) + android.RegisterModuleType("platform_compat_config", PlatformCompatConfigFactory) + android.RegisterModuleType("global_compat_config", globalCompatConfigFactory) +} + +func platformCompatConfigPath(ctx android.PathContext) android.OutputPath { + return android.PathForOutput(ctx, "compat_config", "merged_compat_config.xml") +} + +type platformCompatConfigSingleton struct { + metadata android.Path +} + +type platformCompatConfigProperties struct { + Src *string `android:"path"` +} + +type platformCompatConfig struct { + android.ModuleBase + + properties platformCompatConfigProperties + installDirPath android.InstallPath + configFile android.OutputPath + metadataFile android.OutputPath +} + +func (p *platformCompatConfig) compatConfigMetadata() android.OutputPath { + return p.metadataFile +} + +func (p *platformCompatConfig) CompatConfig() android.OutputPath { + return p.configFile +} + +func (p *platformCompatConfig) SubDir() string { + return "compatconfig" +} + +type PlatformCompatConfigIntf interface { + android.Module + + compatConfigMetadata() android.OutputPath + CompatConfig() android.OutputPath + // Sub dir under etc dir. + SubDir() string +} + +var _ PlatformCompatConfigIntf = (*platformCompatConfig)(nil) + +// compat singleton rules +func (p *platformCompatConfigSingleton) GenerateBuildActions(ctx android.SingletonContext) { + + var compatConfigMetadata android.Paths + + ctx.VisitAllModules(func(module android.Module) { + if c, ok := module.(PlatformCompatConfigIntf); ok { + metadata := c.compatConfigMetadata() + compatConfigMetadata = append(compatConfigMetadata, metadata) + } + }) + + if compatConfigMetadata == nil { + // nothing to do. + return + } + + rule := android.NewRuleBuilder() + outputPath := platformCompatConfigPath(ctx) + + rule.Command(). + BuiltTool(ctx, "process-compat-config"). + FlagForEachInput("--xml ", compatConfigMetadata). + FlagWithOutput("--merged-config ", outputPath) + + rule.Build(pctx, ctx, "merged-compat-config", "Merge compat config") + + p.metadata = outputPath +} + +func (p *platformCompatConfigSingleton) MakeVars(ctx android.MakeVarsContext) { + if p.metadata != nil { + ctx.Strict("INTERNAL_PLATFORM_MERGED_COMPAT_CONFIG", p.metadata.String()) + } +} + +func (p *platformCompatConfig) GenerateAndroidBuildActions(ctx android.ModuleContext) { + rule := android.NewRuleBuilder() + + configFileName := p.Name() + ".xml" + metadataFileName := p.Name() + "_meta.xml" + p.configFile = android.PathForModuleOut(ctx, configFileName).OutputPath + p.metadataFile = android.PathForModuleOut(ctx, metadataFileName).OutputPath + path := android.PathForModuleSrc(ctx, String(p.properties.Src)) + + rule.Command(). + BuiltTool(ctx, "process-compat-config"). + FlagWithInput("--jar ", path). + FlagWithOutput("--device-config ", p.configFile). + FlagWithOutput("--merged-config ", p.metadataFile) + + p.installDirPath = android.PathForModuleInstall(ctx, "etc", "compatconfig") + rule.Build(pctx, ctx, configFileName, "Extract compat/compat_config.xml and install it") + +} + +func (p *platformCompatConfig) AndroidMkEntries() []android.AndroidMkEntries { + return []android.AndroidMkEntries{android.AndroidMkEntries{ + Class: "ETC", + OutputFile: android.OptionalPathForPath(p.configFile), + Include: "$(BUILD_PREBUILT)", + ExtraEntries: []android.AndroidMkExtraEntriesFunc{ + func(entries *android.AndroidMkEntries) { + entries.SetString("LOCAL_MODULE_PATH", p.installDirPath.ToMakePath().String()) + entries.SetString("LOCAL_INSTALLED_MODULE_STEM", p.configFile.Base()) + }, + }, + }} +} + +func platformCompatConfigSingletonFactory() android.Singleton { + return &platformCompatConfigSingleton{} +} + +func PlatformCompatConfigFactory() android.Module { + module := &platformCompatConfig{} + module.AddProperties(&module.properties) + android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibFirst) + return module +} + +//============== merged_compat_config ================= +type globalCompatConfigProperties struct { + // name of the file into which the metadata will be copied. + Filename *string +} + +type globalCompatConfig struct { + android.ModuleBase + + properties globalCompatConfigProperties + + outputFilePath android.OutputPath +} + +func (c *globalCompatConfig) GenerateAndroidBuildActions(ctx android.ModuleContext) { + filename := String(c.properties.Filename) + + inputPath := platformCompatConfigPath(ctx) + c.outputFilePath = android.PathForModuleOut(ctx, filename).OutputPath + + // This ensures that outputFilePath has the correct name for others to + // use, as the source file may have a different name. + ctx.Build(pctx, android.BuildParams{ + Rule: android.Cp, + Output: c.outputFilePath, + Input: inputPath, + }) +} + +func (h *globalCompatConfig) OutputFiles(tag string) (android.Paths, error) { + switch tag { + case "": + return android.Paths{h.outputFilePath}, nil + default: + return nil, fmt.Errorf("unsupported module reference tag %q", tag) + } +} + +// global_compat_config provides access to the merged compat config xml file generated by the build. +func globalCompatConfigFactory() android.Module { + module := &globalCompatConfig{} + module.AddProperties(&module.properties) + android.InitAndroidArchModule(module, android.HostAndDeviceSupported, android.MultilibCommon) + return module +} diff --git a/java/plugin.go b/java/plugin.go index a5e8292e7..947c28626 100644 --- a/java/plugin.go +++ b/java/plugin.go @@ -24,10 +24,8 @@ func init() { func PluginFactory() android.Module { module := &Plugin{} - module.AddProperties( - &module.Module.properties, - &module.Module.protoProperties, - &module.pluginProperties) + module.addHostProperties() + module.AddProperties(&module.pluginProperties) InitJavaModule(module, android.HostSupported) return module diff --git a/java/plugin_test.go b/java/plugin_test.go index d1aef2c19..c7913d3db 100644 --- a/java/plugin_test.go +++ b/java/plugin_test.go @@ -20,7 +20,7 @@ import ( ) func TestNoPlugin(t *testing.T) { - ctx := testJava(t, ` + ctx, _ := testJava(t, ` java_library { name: "foo", srcs: ["a.java"], @@ -44,7 +44,7 @@ func TestNoPlugin(t *testing.T) { } func TestPlugin(t *testing.T) { - ctx := testJava(t, ` + ctx, _ := testJava(t, ` java_library { name: "foo", srcs: ["a.java"], @@ -83,7 +83,7 @@ func TestPlugin(t *testing.T) { } func TestPluginGeneratesApi(t *testing.T) { - ctx := testJava(t, ` + ctx, _ := testJava(t, ` java_library { name: "foo", srcs: ["a.java"], diff --git a/java/prebuilt_apis.go b/java/prebuilt_apis.go index c37081130..999c72f3c 100644 --- a/java/prebuilt_apis.go +++ b/java/prebuilt_apis.go @@ -15,19 +15,20 @@ package java import ( - "android/soong/android" "sort" "strings" "github.com/google/blueprint/proptools" + + "android/soong/android" ) func init() { - android.RegisterModuleType("prebuilt_apis", PrebuiltApisFactory) + RegisterPrebuiltApisBuildComponents(android.InitRegistrationContext) +} - android.PreArchMutators(func(ctx android.RegisterMutatorsContext) { - ctx.TopDown("prebuilt_apis", PrebuiltApisMutator).Parallel() - }) +func RegisterPrebuiltApisBuildComponents(ctx android.RegistrationContext) { + ctx.RegisterModuleType("prebuilt_apis", PrebuiltApisFactory) } type prebuiltApisProperties struct { @@ -44,7 +45,7 @@ func (module *prebuiltApis) GenerateAndroidBuildActions(ctx android.ModuleContex // no need to implement } -func parseJarPath(ctx android.BaseModuleContext, path string) (module string, apiver string, scope string) { +func parseJarPath(path string) (module string, apiver string, scope string) { elements := strings.Split(path, "/") apiver = elements[0] @@ -54,12 +55,12 @@ func parseJarPath(ctx android.BaseModuleContext, path string) (module string, ap return } -func parseApiFilePath(ctx android.BaseModuleContext, path string) (module string, apiver string, scope string) { +func parseApiFilePath(ctx android.LoadHookContext, path string) (module string, apiver string, scope string) { elements := strings.Split(path, "/") apiver = elements[0] scope = elements[1] - if scope != "public" && scope != "system" && scope != "test" { + if scope != "public" && scope != "system" && scope != "test" && scope != "module-lib" && scope != "system-server" { ctx.ModuleErrorf("invalid scope %q found in path: %q", scope, path) return } @@ -69,23 +70,27 @@ func parseApiFilePath(ctx android.BaseModuleContext, path string) (module string return } -func createImport(mctx android.TopDownMutatorContext, module string, scope string, apiver string, path string) { +func prebuiltApiModuleName(mctx android.LoadHookContext, module string, scope string, apiver string) string { + return mctx.ModuleName() + "_" + scope + "_" + apiver + "_" + module +} + +func createImport(mctx android.LoadHookContext, module string, scope string, apiver string, path string) { props := struct { Name *string Jars []string Sdk_version *string Installable *bool }{} - props.Name = proptools.StringPtr(mctx.ModuleName() + "_" + scope + "_" + apiver + "_" + module) + props.Name = proptools.StringPtr(prebuiltApiModuleName(mctx, module, scope, apiver)) props.Jars = append(props.Jars, path) // TODO(hansson): change to scope after migration is done. props.Sdk_version = proptools.StringPtr("current") props.Installable = proptools.BoolPtr(false) - mctx.CreateModule(android.ModuleFactoryAdaptor(ImportFactory), &props) + mctx.CreateModule(ImportFactory, &props) } -func createFilegroup(mctx android.TopDownMutatorContext, module string, scope string, apiver string, path string) { +func createFilegroup(mctx android.LoadHookContext, module string, scope string, apiver string, path string) { fgName := module + ".api." + scope + "." + apiver filegroupProps := struct { Name *string @@ -93,14 +98,14 @@ func createFilegroup(mctx android.TopDownMutatorContext, module string, scope st }{} filegroupProps.Name = proptools.StringPtr(fgName) filegroupProps.Srcs = []string{path} - mctx.CreateModule(android.ModuleFactoryAdaptor(android.FileGroupFactory), &filegroupProps) + mctx.CreateModule(android.FileGroupFactory, &filegroupProps) } -func getPrebuiltFiles(mctx android.TopDownMutatorContext, name string) []string { +func getPrebuiltFiles(mctx android.LoadHookContext, name string) []string { mydir := mctx.ModuleDir() + "/" var files []string for _, apiver := range mctx.Module().(*prebuiltApis).properties.Api_dirs { - for _, scope := range []string{"public", "system", "test", "core"} { + for _, scope := range []string{"public", "system", "test", "core", "module-lib", "system-server"} { vfiles, err := mctx.GlobWithDeps(mydir+apiver+"/"+scope+"/"+name, nil) if err != nil { mctx.ModuleErrorf("failed to glob %s files under %q: %s", name, mydir+apiver+"/"+scope, err) @@ -111,7 +116,7 @@ func getPrebuiltFiles(mctx android.TopDownMutatorContext, name string) []string return files } -func prebuiltSdkStubs(mctx android.TopDownMutatorContext) { +func prebuiltSdkStubs(mctx android.LoadHookContext) { mydir := mctx.ModuleDir() + "/" // <apiver>/<scope>/<module>.jar files := getPrebuiltFiles(mctx, "*.jar") @@ -119,12 +124,33 @@ func prebuiltSdkStubs(mctx android.TopDownMutatorContext) { for _, f := range files { // create a Import module for each jar file localPath := strings.TrimPrefix(f, mydir) - module, apiver, scope := parseJarPath(mctx, localPath) + module, apiver, scope := parseJarPath(localPath) createImport(mctx, module, scope, apiver, localPath) } } -func prebuiltApiFiles(mctx android.TopDownMutatorContext) { +func createSystemModules(mctx android.LoadHookContext, apiver string) { + props := struct { + Name *string + Libs []string + }{} + props.Name = proptools.StringPtr(prebuiltApiModuleName(mctx, "system_modules", "public", apiver)) + props.Libs = append(props.Libs, prebuiltApiModuleName(mctx, "core-for-system-modules", "public", apiver)) + + mctx.CreateModule(SystemModulesFactory, &props) +} + +func prebuiltSdkSystemModules(mctx android.LoadHookContext) { + for _, apiver := range mctx.Module().(*prebuiltApis).properties.Api_dirs { + jar := android.ExistentPathForSource(mctx, + mctx.ModuleDir(), apiver, "public", "core-for-system-modules.jar") + if jar.Valid() { + createSystemModules(mctx, apiver) + } + } +} + +func prebuiltApiFiles(mctx android.LoadHookContext) { mydir := mctx.ModuleDir() + "/" // <apiver>/<scope>/api/<module>.txt files := getPrebuiltFiles(mctx, "api/*.txt") @@ -174,10 +200,11 @@ func prebuiltApiFiles(mctx android.TopDownMutatorContext) { } } -func PrebuiltApisMutator(mctx android.TopDownMutatorContext) { +func createPrebuiltApiModules(mctx android.LoadHookContext) { if _, ok := mctx.Module().(*prebuiltApis); ok { prebuiltApiFiles(mctx) prebuiltSdkStubs(mctx) + prebuiltSdkSystemModules(mctx) } } @@ -187,9 +214,17 @@ func PrebuiltApisMutator(mctx android.TopDownMutatorContext) { // generates a filegroup module named <module>-api.<scope>.<ver>. // // It also creates <module>-api.<scope>.latest for the latest <ver>. +// +// Similarly, it generates a java_import for all API .jar files found under the +// directory where the Android.bp is located. Specifically, an API file located +// at ./<ver>/<scope>/api/<module>.jar generates a java_import module named +// <prebuilt-api-module>_<scope>_<ver>_<module>, and for SDK versions >= 30 +// a java_system_modules module named +// <prebuilt-api-module>_public_<ver>_system_modules func PrebuiltApisFactory() android.Module { module := &prebuiltApis{} module.AddProperties(&module.properties) android.InitAndroidModule(module) + android.AddLoadHook(module, createPrebuiltApiModules) return module } diff --git a/java/proto.go b/java/proto.go index 37de1d283..4d735ebbe 100644 --- a/java/proto.go +++ b/java/proto.go @@ -15,36 +15,61 @@ package java import ( + "path/filepath" + "strconv" + "android/soong/android" ) -func genProto(ctx android.ModuleContext, protoFile android.Path, flags android.ProtoFlags) android.Path { - srcJarFile := android.GenPathWithExt(ctx, "proto", protoFile, "srcjar") +func genProto(ctx android.ModuleContext, protoFiles android.Paths, flags android.ProtoFlags) android.Paths { + // Shard proto files into groups of 100 to avoid having to recompile all of them if one changes and to avoid + // hitting command line length limits. + shards := android.ShardPaths(protoFiles, 100) - outDir := srcJarFile.ReplaceExtension(ctx, "tmp") - depFile := srcJarFile.ReplaceExtension(ctx, "srcjar.d") + srcJarFiles := make(android.Paths, 0, len(shards)) - rule := android.NewRuleBuilder() + for i, shard := range shards { + srcJarFile := android.PathForModuleGen(ctx, "proto", "proto"+strconv.Itoa(i)+".srcjar") + srcJarFiles = append(srcJarFiles, srcJarFile) - rule.Command().Text("rm -rf").Flag(outDir.String()) - rule.Command().Text("mkdir -p").Flag(outDir.String()) + outDir := srcJarFile.ReplaceExtension(ctx, "tmp") - android.ProtoRule(ctx, rule, protoFile, flags, flags.Deps, outDir, depFile, nil) + rule := android.NewRuleBuilder() - // Proto generated java files have an unknown package name in the path, so package the entire output directory - // into a srcjar. - rule.Command(). - Tool(ctx.Config().HostToolPath(ctx, "soong_zip")). - Flag("-jar"). - FlagWithOutput("-o ", srcJarFile). - FlagWithArg("-C ", outDir.String()). - FlagWithArg("-D ", outDir.String()) + rule.Command().Text("rm -rf").Flag(outDir.String()) + rule.Command().Text("mkdir -p").Flag(outDir.String()) - rule.Command().Text("rm -rf").Flag(outDir.String()) + for _, protoFile := range shard { + depFile := srcJarFile.InSameDir(ctx, protoFile.String()+".d") + rule.Command().Text("mkdir -p").Flag(filepath.Dir(depFile.String())) + android.ProtoRule(ctx, rule, protoFile, flags, flags.Deps, outDir, depFile, nil) + } - rule.Build(pctx, ctx, "protoc_"+protoFile.Rel(), "protoc "+protoFile.Rel()) + // Proto generated java files have an unknown package name in the path, so package the entire output directory + // into a srcjar. + rule.Command(). + BuiltTool(ctx, "soong_zip"). + Flag("-jar"). + Flag("-write_if_changed"). + FlagWithOutput("-o ", srcJarFile). + FlagWithArg("-C ", outDir.String()). + FlagWithArg("-D ", outDir.String()) + + rule.Command().Text("rm -rf").Flag(outDir.String()) + + rule.Restat() + + ruleName := "protoc" + ruleDesc := "protoc" + if len(shards) > 1 { + ruleName += "_" + strconv.Itoa(i) + ruleDesc += " " + strconv.Itoa(i) + } + + rule.Build(pctx, ctx, ruleName, ruleDesc) + } - return srcJarFile + return srcJarFiles } func protoDeps(ctx android.BottomUpMutatorContext, p *android.ProtoProperties) { @@ -75,20 +100,29 @@ func protoFlags(ctx android.ModuleContext, j *CompilerProperties, p *android.Pro flags.proto = android.GetProtoFlags(ctx, p) if String(p.Proto.Plugin) == "" { + var typeToPlugin string switch String(p.Proto.Type) { case "micro": flags.proto.OutTypeFlag = "--javamicro_out" + typeToPlugin = "javamicro" case "nano": flags.proto.OutTypeFlag = "--javanano_out" - case "lite": + typeToPlugin = "javanano" + case "lite", "": flags.proto.OutTypeFlag = "--java_out" flags.proto.OutParams = append(flags.proto.OutParams, "lite") - case "full", "": + case "full": flags.proto.OutTypeFlag = "--java_out" default: ctx.PropertyErrorf("proto.type", "unknown proto type %q", String(p.Proto.Type)) } + + if typeToPlugin != "" { + hostTool := ctx.Config().HostToolPath(ctx, "protoc-gen-"+typeToPlugin) + flags.proto.Deps = append(flags.proto.Deps, hostTool) + flags.proto.Flags = append(flags.proto.Flags, "--plugin=protoc-gen-"+typeToPlugin+"="+hostTool.String()) + } } flags.proto.OutParams = append(flags.proto.OutParams, j.Proto.Output_params...) diff --git a/java/robolectric.go b/java/robolectric.go new file mode 100644 index 000000000..c6b07a17e --- /dev/null +++ b/java/robolectric.go @@ -0,0 +1,228 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package java + +import ( + "fmt" + "io" + "strconv" + "strings" + + "android/soong/android" +) + +func init() { + android.RegisterModuleType("android_robolectric_test", RobolectricTestFactory) +} + +var robolectricDefaultLibs = []string{ + "robolectric_android-all-stub", + "Robolectric_all-target", + "mockito-robolectric-prebuilt", + "truth-prebuilt", +} + +var ( + roboCoverageLibsTag = dependencyTag{name: "roboSrcs"} +) + +type robolectricProperties struct { + // The name of the android_app module that the tests will run against. + Instrumentation_for *string + + // Additional libraries for which coverage data should be generated + Coverage_libs []string + + Test_options struct { + // Timeout in seconds when running the tests. + Timeout *int64 + + // Number of shards to use when running the tests. + Shards *int64 + } +} + +type robolectricTest struct { + Library + + robolectricProperties robolectricProperties + + libs []string + tests []string + + roboSrcJar android.Path +} + +func (r *robolectricTest) DepsMutator(ctx android.BottomUpMutatorContext) { + r.Library.DepsMutator(ctx) + + if r.robolectricProperties.Instrumentation_for != nil { + ctx.AddVariationDependencies(nil, instrumentationForTag, String(r.robolectricProperties.Instrumentation_for)) + } else { + ctx.PropertyErrorf("instrumentation_for", "missing required instrumented module") + } + + ctx.AddVariationDependencies(nil, libTag, robolectricDefaultLibs...) + + ctx.AddVariationDependencies(nil, roboCoverageLibsTag, r.robolectricProperties.Coverage_libs...) +} + +func (r *robolectricTest) GenerateAndroidBuildActions(ctx android.ModuleContext) { + roboTestConfig := android.PathForModuleGen(ctx, "robolectric"). + Join(ctx, "com/android/tools/test_config.properties") + + // TODO: this inserts paths to built files into the test, it should really be inserting the contents. + instrumented := ctx.GetDirectDepsWithTag(instrumentationForTag) + + if len(instrumented) != 1 { + panic(fmt.Errorf("expected exactly 1 instrumented dependency, got %d", len(instrumented))) + } + + instrumentedApp, ok := instrumented[0].(*AndroidApp) + if !ok { + ctx.PropertyErrorf("instrumentation_for", "dependency must be an android_app") + } + + generateRoboTestConfig(ctx, roboTestConfig, instrumentedApp) + r.extraResources = android.Paths{roboTestConfig} + + r.Library.GenerateAndroidBuildActions(ctx) + + roboSrcJar := android.PathForModuleGen(ctx, "robolectric", ctx.ModuleName()+".srcjar") + r.generateRoboSrcJar(ctx, roboSrcJar, instrumentedApp) + r.roboSrcJar = roboSrcJar + + for _, dep := range ctx.GetDirectDepsWithTag(libTag) { + r.libs = append(r.libs, dep.(Dependency).BaseModuleName()) + } + + // TODO: this could all be removed if tradefed was used as the test runner, it will find everything + // annotated as a test and run it. + for _, src := range r.compiledJavaSrcs { + s := src.Rel() + if !strings.HasSuffix(s, "Test.java") { + continue + } else if strings.HasSuffix(s, "/BaseRobolectricTest.java") { + continue + } else if strings.HasPrefix(s, "src/") { + s = strings.TrimPrefix(s, "src/") + } + r.tests = append(r.tests, s) + } +} + +func generateRoboTestConfig(ctx android.ModuleContext, outputFile android.WritablePath, instrumentedApp *AndroidApp) { + manifest := instrumentedApp.mergedManifestFile + resourceApk := instrumentedApp.outputFile + + rule := android.NewRuleBuilder() + + rule.Command().Text("rm -f").Output(outputFile) + rule.Command(). + Textf(`echo "android_merged_manifest=%s" >>`, manifest.String()).Output(outputFile).Text("&&"). + Textf(`echo "android_resource_apk=%s" >>`, resourceApk.String()).Output(outputFile). + // Make it depend on the files to which it points so the test file's timestamp is updated whenever the + // contents change + Implicit(manifest). + Implicit(resourceApk) + + rule.Build(pctx, ctx, "generate_test_config", "generate test_config.properties") +} + +func (r *robolectricTest) generateRoboSrcJar(ctx android.ModuleContext, outputFile android.WritablePath, + instrumentedApp *AndroidApp) { + + srcJarArgs := copyOf(instrumentedApp.srcJarArgs) + srcJarDeps := append(android.Paths(nil), instrumentedApp.srcJarDeps...) + + for _, m := range ctx.GetDirectDepsWithTag(roboCoverageLibsTag) { + if dep, ok := m.(Dependency); ok { + depSrcJarArgs, depSrcJarDeps := dep.SrcJarArgs() + srcJarArgs = append(srcJarArgs, depSrcJarArgs...) + srcJarDeps = append(srcJarDeps, depSrcJarDeps...) + } + } + + TransformResourcesToJar(ctx, outputFile, srcJarArgs, srcJarDeps) +} + +func (r *robolectricTest) AndroidMkEntries() []android.AndroidMkEntries { + entriesList := r.Library.AndroidMkEntries() + entries := &entriesList[0] + + entries.ExtraFooters = []android.AndroidMkExtraFootersFunc{ + func(w io.Writer, name, prefix, moduleDir string, entries *android.AndroidMkEntries) { + if s := r.robolectricProperties.Test_options.Shards; s != nil && *s > 1 { + numShards := int(*s) + shardSize := (len(r.tests) + numShards - 1) / numShards + shards := android.ShardStrings(r.tests, shardSize) + for i, shard := range shards { + r.writeTestRunner(w, name, "Run"+name+strconv.Itoa(i), shard) + } + + // TODO: add rules to dist the outputs of the individual tests, or combine them together? + fmt.Fprintln(w, "") + fmt.Fprintln(w, ".PHONY:", "Run"+name) + fmt.Fprintln(w, "Run"+name, ": \\") + for i := range shards { + fmt.Fprintln(w, " ", "Run"+name+strconv.Itoa(i), "\\") + } + fmt.Fprintln(w, "") + } else { + r.writeTestRunner(w, name, "Run"+name, r.tests) + } + }, + } + + return entriesList +} + +func (r *robolectricTest) writeTestRunner(w io.Writer, module, name string, tests []string) { + fmt.Fprintln(w, "") + fmt.Fprintln(w, "include $(CLEAR_VARS)") + fmt.Fprintln(w, "LOCAL_MODULE :=", name) + fmt.Fprintln(w, "LOCAL_JAVA_LIBRARIES :=", module) + fmt.Fprintln(w, "LOCAL_JAVA_LIBRARIES += ", strings.Join(r.libs, " ")) + fmt.Fprintln(w, "LOCAL_TEST_PACKAGE :=", String(r.robolectricProperties.Instrumentation_for)) + fmt.Fprintln(w, "LOCAL_INSTRUMENT_SRCJARS :=", r.roboSrcJar.String()) + fmt.Fprintln(w, "LOCAL_ROBOTEST_FILES :=", strings.Join(tests, " ")) + if t := r.robolectricProperties.Test_options.Timeout; t != nil { + fmt.Fprintln(w, "LOCAL_ROBOTEST_TIMEOUT :=", *t) + } + fmt.Fprintln(w, "-include external/robolectric-shadows/run_robotests.mk") + +} + +// An android_robolectric_test module compiles tests against the Robolectric framework that can run on the local host +// instead of on a device. It also generates a rule with the name of the module prefixed with "Run" that can be +// used to run the tests. Running the tests with build rule will eventually be deprecated and replaced with atest. +// +// The test runner considers any file listed in srcs whose name ends with Test.java to be a test class, unless +// it is named BaseRobolectricTest.java. The path to the each source file must exactly match the package +// name, or match the package name when the prefix "src/" is removed. +func RobolectricTestFactory() android.Module { + module := &robolectricTest{} + + module.addHostProperties() + module.AddProperties( + &module.Module.deviceProperties, + &module.robolectricProperties) + + module.Module.dexpreopter.isTest = true + module.Module.linter.test = true + + InitJavaModule(module, android.DeviceSupported) + return module +} diff --git a/java/sdk.go b/java/sdk.go index e93f8fb69..f96ecded4 100644 --- a/java/sdk.go +++ b/java/sdk.go @@ -15,15 +15,15 @@ package java import ( - "android/soong/android" - "android/soong/java/config" "fmt" "path/filepath" - "runtime" "sort" "strconv" "strings" + "android/soong/android" + "android/soong/java/config" + "github.com/google/blueprint/pathtools" ) @@ -35,83 +35,309 @@ func init() { var sdkVersionsKey = android.NewOnceKey("sdkVersionsKey") var sdkFrameworkAidlPathKey = android.NewOnceKey("sdkFrameworkAidlPathKey") +var nonUpdatableFrameworkAidlPathKey = android.NewOnceKey("nonUpdatableFrameworkAidlPathKey") var apiFingerprintPathKey = android.NewOnceKey("apiFingerprintPathKey") type sdkContext interface { - // sdkVersion eturns the sdk_version property of the current module, or an empty string if it is not set. - sdkVersion() string - // minSdkVersion returns the min_sdk_version property of the current module, or sdkVersion() if it is not set. - minSdkVersion() string - // targetSdkVersion returns the target_sdk_version property of the current module, or sdkVersion() if it is not set. - targetSdkVersion() string -} - -func sdkVersionOrDefault(ctx android.BaseContext, v string) string { - switch v { - case "", "current", "system_current", "test_current", "core_current": - return ctx.Config().DefaultAppTargetSdk() + // sdkVersion returns sdkSpec that corresponds to the sdk_version property of the current module + sdkVersion() sdkSpec + // systemModules returns the system_modules property of the current module, or an empty string if it is not set. + systemModules() string + // minSdkVersion returns sdkSpec that corresponds to the min_sdk_version property of the current module, + // or from sdk_version if it is not set. + minSdkVersion() sdkSpec + // targetSdkVersion returns the sdkSpec that corresponds to the target_sdk_version property of the current module, + // or from sdk_version if it is not set. + targetSdkVersion() sdkSpec +} + +func UseApiFingerprint(ctx android.BaseModuleContext) bool { + if ctx.Config().UnbundledBuild() && + !ctx.Config().UnbundledBuildUsePrebuiltSdks() && + ctx.Config().IsEnvTrue("UNBUNDLED_BUILD_TARGET_SDK_WITH_API_FINGERPRINT") { + return true + } + return false +} + +// sdkKind represents a particular category of an SDK spec like public, system, test, etc. +type sdkKind int + +const ( + sdkInvalid sdkKind = iota + sdkNone + sdkCore + sdkCorePlatform + sdkPublic + sdkSystem + sdkTest + sdkModule + sdkSystemServer + sdkPrivate +) + +// String returns the string representation of this sdkKind +func (k sdkKind) String() string { + switch k { + case sdkPrivate: + return "private" + case sdkNone: + return "none" + case sdkPublic: + return "public" + case sdkSystem: + return "system" + case sdkTest: + return "test" + case sdkCore: + return "core" + case sdkCorePlatform: + return "core_platform" + case sdkModule: + return "module" + case sdkSystemServer: + return "system_server" default: - return v + return "invalid" } } -// Returns a sdk version as a number. For modules targeting an unreleased SDK (meaning it does not yet have a number) -// it returns android.FutureApiLevel (10000). -func sdkVersionToNumber(ctx android.BaseContext, v string) (int, error) { - switch v { - case "", "current", "test_current", "system_current", "core_current": - return ctx.Config().DefaultAppTargetSdkInt(), nil +// sdkVersion represents a specific version number of an SDK spec of a particular kind +type sdkVersion int + +const ( + // special version number for a not-yet-frozen SDK + sdkVersionCurrent sdkVersion = sdkVersion(android.FutureApiLevel) + // special version number to be used for SDK specs where version number doesn't + // make sense, e.g. "none", "", etc. + sdkVersionNone sdkVersion = sdkVersion(0) +) + +// isCurrent checks if the sdkVersion refers to the not-yet-published version of an sdkKind +func (v sdkVersion) isCurrent() bool { + return v == sdkVersionCurrent +} + +// isNumbered checks if the sdkVersion refers to the published (a.k.a numbered) version of an sdkKind +func (v sdkVersion) isNumbered() bool { + return !v.isCurrent() && v != sdkVersionNone +} + +// String returns the string representation of this sdkVersion. +func (v sdkVersion) String() string { + if v.isCurrent() { + return "current" + } else if v.isNumbered() { + return strconv.Itoa(int(v)) + } + return "(no version)" +} + +// asNumberString directly converts the numeric value of this sdk version as a string. +// When isNumbered() is true, this method is the same as String(). However, for sdkVersionCurrent +// and sdkVersionNone, this returns 10000 and 0 while String() returns "current" and "(no version"), +// respectively. +func (v sdkVersion) asNumberString() string { + return strconv.Itoa(int(v)) +} + +// sdkSpec represents the kind and the version of an SDK for a module to build against +type sdkSpec struct { + kind sdkKind + version sdkVersion + raw string +} + +func (s sdkSpec) String() string { + return fmt.Sprintf("%s_%s", s.kind, s.version) +} + +// valid checks if this sdkSpec is well-formed. Note however that true doesn't mean that the +// specified SDK actually exists. +func (s sdkSpec) valid() bool { + return s.kind != sdkInvalid +} + +// specified checks if this sdkSpec is well-formed and is not "". +func (s sdkSpec) specified() bool { + return s.valid() && s.kind != sdkPrivate +} + +// whether the API surface is managed and versioned, i.e. has .txt file that +// get frozen on SDK freeze and changes get reviewed by API council. +func (s sdkSpec) stable() bool { + if !s.specified() { + return false + } + switch s.kind { + case sdkNone: + // there is nothing to manage and version in this case; de facto stable API. + return true + case sdkCore, sdkPublic, sdkSystem, sdkModule, sdkSystemServer: + return true + case sdkCorePlatform, sdkTest, sdkPrivate: + return false default: - n := android.GetNumericSdkVersion(v) - if i, err := strconv.Atoi(n); err != nil { - return -1, fmt.Errorf("invalid sdk version %q", n) - } else { - return i, nil - } + panic(fmt.Errorf("unknown sdkKind=%v", s.kind)) } + return false } -func sdkVersionToNumberAsString(ctx android.BaseContext, v string) (string, error) { - n, err := sdkVersionToNumber(ctx, v) - if err != nil { - return "", err +// prebuiltSdkAvailableForUnbundledBuilt tells whether this sdkSpec can have a prebuilt SDK +// that can be used for unbundled builds. +func (s sdkSpec) prebuiltSdkAvailableForUnbundledBuild() bool { + // "", "none", and "core_platform" are not available for unbundled build + // as we don't/can't have prebuilt stub for the versions + return s.kind != sdkPrivate && s.kind != sdkNone && s.kind != sdkCorePlatform +} + +// forPdkBuild converts this sdkSpec into another sdkSpec that is for the PDK builds. +func (s sdkSpec) forPdkBuild(ctx android.EarlyModuleContext) sdkSpec { + // For PDK builds, use the latest SDK version instead of "current" or "" + if s.kind == sdkPrivate || s.kind == sdkPublic { + kind := s.kind + if kind == sdkPrivate { + // We don't have prebuilt SDK for private APIs, so use the public SDK + // instead. This looks odd, but that's how it has been done. + // TODO(b/148271073): investigate the need for this. + kind = sdkPublic + } + version := sdkVersion(LatestSdkVersionInt(ctx)) + return sdkSpec{kind, version, s.raw} } - return strconv.Itoa(n), nil + return s } -func decodeSdkDep(ctx android.BaseContext, sdkContext sdkContext) sdkDep { - v := sdkContext.sdkVersion() - // For PDK builds, use the latest SDK version instead of "current" - if ctx.Config().IsPdkBuild() && (v == "" || v == "current") { - sdkVersions := ctx.Config().Get(sdkVersionsKey).([]int) - latestSdkVersion := 0 - if len(sdkVersions) > 0 { - latestSdkVersion = sdkVersions[len(sdkVersions)-1] +// usePrebuilt determines whether prebuilt SDK should be used for this sdkSpec with the given context. +func (s sdkSpec) usePrebuilt(ctx android.EarlyModuleContext) bool { + if s.version.isCurrent() { + // "current" can be built from source and be from prebuilt SDK + return ctx.Config().UnbundledBuildUsePrebuiltSdks() + } else if s.version.isNumbered() { + // sanity check + if s.kind != sdkPublic && s.kind != sdkSystem && s.kind != sdkTest { + panic(fmt.Errorf("prebuilt SDK is not not available for sdkKind=%q", s.kind)) + return false } - v = strconv.Itoa(latestSdkVersion) + // numbered SDKs are always from prebuilt + return true } + // "", "none", "core_platform" fall here + return false +} - numericSdkVersion, err := sdkVersionToNumber(ctx, v) +// effectiveVersion converts an sdkSpec into the concrete sdkVersion that the module +// should use. For modules targeting an unreleased SDK (meaning it does not yet have a number) +// it returns android.FutureApiLevel(10000). +func (s sdkSpec) effectiveVersion(ctx android.EarlyModuleContext) (sdkVersion, error) { + if !s.valid() { + return s.version, fmt.Errorf("invalid sdk version %q", s.raw) + } + if ctx.Config().IsPdkBuild() { + s = s.forPdkBuild(ctx) + } + if s.version.isNumbered() { + return s.version, nil + } + return sdkVersion(ctx.Config().DefaultAppTargetSdkInt()), nil +} + +// effectiveVersionString converts an sdkSpec into the concrete version string that the module +// should use. For modules targeting an unreleased SDK (meaning it does not yet have a number) +// it returns the codename (P, Q, R, etc.) +func (s sdkSpec) effectiveVersionString(ctx android.EarlyModuleContext) (string, error) { + ver, err := s.effectiveVersion(ctx) + if err == nil && int(ver) == ctx.Config().DefaultAppTargetSdkInt() { + return ctx.Config().DefaultAppTargetSdk(), nil + } + return ver.String(), err +} + +func (s sdkSpec) defaultJavaLanguageVersion(ctx android.EarlyModuleContext) javaVersion { + sdk, err := s.effectiveVersion(ctx) if err != nil { ctx.PropertyErrorf("sdk_version", "%s", err) - return sdkDep{} } + if sdk <= 23 { + return JAVA_VERSION_7 + } else if sdk <= 29 { + return JAVA_VERSION_8 + } else { + return JAVA_VERSION_9 + } +} + +func sdkSpecFrom(str string) sdkSpec { + switch str { + // special cases first + case "": + return sdkSpec{sdkPrivate, sdkVersionNone, str} + case "none": + return sdkSpec{sdkNone, sdkVersionNone, str} + case "core_platform": + return sdkSpec{sdkCorePlatform, sdkVersionNone, str} + default: + // the syntax is [kind_]version + sep := strings.LastIndex(str, "_") + + var kindString string + if sep == 0 { + return sdkSpec{sdkInvalid, sdkVersionNone, str} + } else if sep == -1 { + kindString = "" + } else { + kindString = str[0:sep] + } + versionString := str[sep+1 : len(str)] + + var kind sdkKind + switch kindString { + case "": + kind = sdkPublic + case "core": + kind = sdkCore + case "system": + kind = sdkSystem + case "test": + kind = sdkTest + case "module": + kind = sdkModule + case "system_server": + kind = sdkSystemServer + default: + return sdkSpec{sdkInvalid, sdkVersionNone, str} + } - toPrebuilt := func(sdk string) sdkDep { - var api, v string - if strings.Contains(sdk, "_") { - t := strings.Split(sdk, "_") - api = t[0] - v = t[1] + var version sdkVersion + if versionString == "current" { + version = sdkVersionCurrent + } else if i, err := strconv.Atoi(versionString); err == nil { + version = sdkVersion(i) } else { - api = "public" - v = sdk + return sdkSpec{sdkInvalid, sdkVersionNone, str} } - dir := filepath.Join("prebuilts", "sdk", v, api) + + return sdkSpec{kind, version, str} + } +} + +func decodeSdkDep(ctx android.EarlyModuleContext, sdkContext sdkContext) sdkDep { + sdkVersion := sdkContext.sdkVersion() + if !sdkVersion.valid() { + ctx.PropertyErrorf("sdk_version", "invalid version %q", sdkVersion.raw) + return sdkDep{} + } + + if ctx.Config().IsPdkBuild() { + sdkVersion = sdkVersion.forPdkBuild(ctx) + } + + if sdkVersion.usePrebuilt(ctx) { + dir := filepath.Join("prebuilts", "sdk", sdkVersion.version.String(), sdkVersion.kind.String()) jar := filepath.Join(dir, "android.jar") // There's no aidl for other SDKs yet. // TODO(77525052): Add aidl files for other SDKs too. - public_dir := filepath.Join("prebuilts", "sdk", v, "public") + public_dir := filepath.Join("prebuilts", "sdk", sdkVersion.version.String(), "public") aidl := filepath.Join(public_dir, "framework.aidl") jarPath := android.ExistentPathForSource(ctx, jar) aidlPath := android.ExistentPathForSource(ctx, aidl) @@ -120,79 +346,104 @@ func decodeSdkDep(ctx android.BaseContext, sdkContext sdkContext) sdkDep { if (!jarPath.Valid() || !aidlPath.Valid()) && ctx.Config().AllowMissingDependencies() { return sdkDep{ invalidVersion: true, - modules: []string{fmt.Sprintf("sdk_%s_%s_android", api, v)}, + bootclasspath: []string{fmt.Sprintf("sdk_%s_%s_android", sdkVersion.kind, sdkVersion.version.String())}, } } if !jarPath.Valid() { - ctx.PropertyErrorf("sdk_version", "invalid sdk version %q, %q does not exist", v, jar) + ctx.PropertyErrorf("sdk_version", "invalid sdk version %q, %q does not exist", sdkVersion.raw, jar) return sdkDep{} } if !aidlPath.Valid() { - ctx.PropertyErrorf("sdk_version", "invalid sdk version %q, %q does not exist", v, aidl) + ctx.PropertyErrorf("sdk_version", "invalid sdk version %q, %q does not exist", sdkVersion.raw, aidl) return sdkDep{} } + var systemModules string + if sdkVersion.defaultJavaLanguageVersion(ctx).usesJavaModules() { + systemModules = "sdk_public_" + sdkVersion.version.String() + "_system_modules" + } + return sdkDep{ - useFiles: true, - jars: android.Paths{jarPath.Path(), lambdaStubsPath}, - aidl: android.OptionalPathForPath(aidlPath.Path()), + useFiles: true, + jars: android.Paths{jarPath.Path(), lambdaStubsPath}, + aidl: android.OptionalPathForPath(aidlPath.Path()), + systemModules: systemModules, } } - toModule := func(m, r string, aidl android.Path) sdkDep { - ret := sdkDep{ + toModule := func(modules []string, res string, aidl android.Path) sdkDep { + return sdkDep{ useModule: true, - modules: []string{m, config.DefaultLambdaStubsLibrary}, - systemModules: m + "_system_modules", - frameworkResModule: r, + bootclasspath: append(modules, config.DefaultLambdaStubsLibrary), + systemModules: "core-current-stubs-system-modules", + java9Classpath: modules, + frameworkResModule: res, aidl: android.OptionalPathForPath(aidl), } - - if m == "core.current.stubs" { - ret.systemModules = "core-system-modules" - } else if m == "core.platform.api.stubs" { - ret.systemModules = "core-platform-api-stubs-system-modules" - } - return ret } // Ensures that the specificed system SDK version is one of BOARD_SYSTEMSDK_VERSIONS (for vendor apks) // or PRODUCT_SYSTEMSDK_VERSIONS (for other apks or when BOARD_SYSTEMSDK_VERSIONS is not set) - if strings.HasPrefix(v, "system_") && numericSdkVersion != android.FutureApiLevel { + if sdkVersion.kind == sdkSystem && sdkVersion.version.isNumbered() { allowed_versions := ctx.DeviceConfig().PlatformSystemSdkVersions() if ctx.DeviceSpecific() || ctx.SocSpecific() { if len(ctx.DeviceConfig().SystemSdkVersions()) > 0 { allowed_versions = ctx.DeviceConfig().SystemSdkVersions() } } - if len(allowed_versions) > 0 && !android.InList(strconv.Itoa(numericSdkVersion), allowed_versions) { + if len(allowed_versions) > 0 && !android.InList(sdkVersion.version.String(), allowed_versions) { ctx.PropertyErrorf("sdk_version", "incompatible sdk version %q. System SDK version should be one of %q", - v, allowed_versions) + sdkVersion.raw, allowed_versions) } } - if ctx.Config().UnbundledBuildUsePrebuiltSdks() && v != "" { - return toPrebuilt(v) - } + switch sdkVersion.kind { + case sdkPrivate: + return sdkDep{ + useDefaultLibs: true, + frameworkResModule: "framework-res", + } + case sdkNone: + systemModules := sdkContext.systemModules() + if systemModules == "" { + ctx.PropertyErrorf("sdk_version", + `system_modules is required to be set to a non-empty value when sdk_version is "none", did you mean sdk_version: "core_platform"?`) + } else if systemModules == "none" { + return sdkDep{ + noStandardLibs: true, + } + } - switch v { - case "": + return sdkDep{ + useModule: true, + noStandardLibs: true, + systemModules: systemModules, + bootclasspath: []string{systemModules}, + } + case sdkCorePlatform: return sdkDep{ useDefaultLibs: true, frameworkResModule: "framework-res", + noFrameworksLibs: true, } - case "current": - return toModule("android_stubs_current", "framework-res", sdkFrameworkAidlPath(ctx)) - case "system_current": - return toModule("android_system_stubs_current", "framework-res", sdkFrameworkAidlPath(ctx)) - case "test_current": - return toModule("android_test_stubs_current", "framework-res", sdkFrameworkAidlPath(ctx)) - case "core_current": - return toModule("core.current.stubs", "", nil) + case sdkPublic: + return toModule([]string{"android_stubs_current"}, "framework-res", sdkFrameworkAidlPath(ctx)) + case sdkSystem: + return toModule([]string{"android_system_stubs_current"}, "framework-res", sdkFrameworkAidlPath(ctx)) + case sdkTest: + return toModule([]string{"android_test_stubs_current"}, "framework-res", sdkFrameworkAidlPath(ctx)) + case sdkCore: + return toModule([]string{"core.current.stubs"}, "", nil) + case sdkModule: + // TODO(146757305): provide .apk and .aidl that have more APIs for modules + return toModule([]string{"android_module_lib_stubs_current"}, "framework-res", nonUpdatableFrameworkAidlPath(ctx)) + case sdkSystemServer: + // TODO(146757305): provide .apk and .aidl that have more APIs for modules + return toModule([]string{"android_system_server_stubs_current"}, "framework-res", sdkFrameworkAidlPath(ctx)) default: - return toPrebuilt(v) + panic(fmt.Errorf("invalid sdk %q", sdkVersion.raw)) } } @@ -225,6 +476,15 @@ func (sdkPreSingleton) GenerateBuildActions(ctx android.SingletonContext) { ctx.Config().Once(sdkVersionsKey, func() interface{} { return sdkVersions }) } +func LatestSdkVersionInt(ctx android.EarlyModuleContext) int { + sdkVersions := ctx.Config().Get(sdkVersionsKey).([]int) + latestSdkVersion := 0 + if len(sdkVersions) > 0 { + latestSdkVersion = sdkVersions[len(sdkVersions)-1] + } + return latestSdkVersion +} + func sdkSingletonFactory() android.Singleton { return sdkSingleton{} } @@ -237,6 +497,7 @@ func (sdkSingleton) GenerateBuildActions(ctx android.SingletonContext) { } createSdkFrameworkAidl(ctx) + createNonUpdatableFrameworkAidl(ctx) createAPIFingerprint(ctx) } @@ -248,6 +509,31 @@ func createSdkFrameworkAidl(ctx android.SingletonContext) { "android_system_stubs_current", } + combinedAidl := sdkFrameworkAidlPath(ctx) + tempPath := combinedAidl.ReplaceExtension(ctx, "aidl.tmp") + + rule := createFrameworkAidl(stubsModules, tempPath, ctx) + + commitChangeForRestat(rule, tempPath, combinedAidl) + + rule.Build(pctx, ctx, "framework_aidl", "generate framework.aidl") +} + +// Creates a version of framework.aidl for the non-updatable part of the platform. +func createNonUpdatableFrameworkAidl(ctx android.SingletonContext) { + stubsModules := []string{"android_module_lib_stubs_current"} + + combinedAidl := nonUpdatableFrameworkAidlPath(ctx) + tempPath := combinedAidl.ReplaceExtension(ctx, "aidl.tmp") + + rule := createFrameworkAidl(stubsModules, tempPath, ctx) + + commitChangeForRestat(rule, tempPath, combinedAidl) + + rule.Build(pctx, ctx, "framework_non_updatable_aidl", "generate framework_non_updatable.aidl") +} + +func createFrameworkAidl(stubsModules []string, path android.OutputPath, ctx android.SingletonContext) *android.RuleBuilder { stubsJars := make([]android.Paths, len(stubsModules)) ctx.VisitAllModules(func(module android.Module) { @@ -267,8 +553,7 @@ func createSdkFrameworkAidl(ctx android.SingletonContext) { if ctx.Config().AllowMissingDependencies() { missingDeps = append(missingDeps, stubsModules[i]) } else { - ctx.Errorf("failed to find dex jar path for module %q", - stubsModules[i]) + ctx.Errorf("failed to find dex jar path for module %q", stubsModules[i]) } } } @@ -284,7 +569,7 @@ func createSdkFrameworkAidl(ctx android.SingletonContext) { rule.Command(). Text("rm -f").Output(aidl) rule.Command(). - Tool(ctx.Config().HostToolPath(ctx, "sdkparcelables")). + BuiltTool(ctx, "sdkparcelables"). Input(jar). Output(aidl) @@ -292,20 +577,15 @@ func createSdkFrameworkAidl(ctx android.SingletonContext) { } } - combinedAidl := sdkFrameworkAidlPath(ctx) - tempPath := combinedAidl.ReplaceExtension(ctx, "aidl.tmp") - rule.Command(). - Text("rm -f").Output(tempPath) + Text("rm -f").Output(path) rule.Command(). Text("cat"). Inputs(aidls). Text("| sort -u >"). - Output(tempPath) + Output(path) - commitChangeForRestat(rule, tempPath, combinedAidl) - - rule.Build(pctx, ctx, "framework_aidl", "generate framework.aidl") + return rule } func sdkFrameworkAidlPath(ctx android.PathContext) android.OutputPath { @@ -314,6 +594,12 @@ func sdkFrameworkAidlPath(ctx android.PathContext) android.OutputPath { }).(android.OutputPath) } +func nonUpdatableFrameworkAidlPath(ctx android.PathContext) android.OutputPath { + return ctx.Config().Once(nonUpdatableFrameworkAidlPathKey, func() interface{} { + return android.PathForOutput(ctx, "framework_non_updatable.aidl") + }).(android.OutputPath) +} + // Create api_fingerprint.txt func createAPIFingerprint(ctx android.SingletonContext) { out := ApiFingerprintPath(ctx) @@ -337,15 +623,7 @@ func createAPIFingerprint(ctx android.SingletonContext) { cmd.Text("cat"). Inputs(android.PathsForSource(ctx, in)). - Text("|") - - if runtime.GOOS == "darwin" { - cmd.Text("md5") - } else { - cmd.Text("md5sum") - } - - cmd.Text("| cut -d' ' -f1 >"). + Text("| md5sum | cut -d' ' -f1 >"). Output(out) } else { // Unbundled build diff --git a/java/sdk_library.go b/java/sdk_library.go index 84be4dda9..f2a509ad2 100644 --- a/java/sdk_library.go +++ b/java/sdk_library.go @@ -15,95 +15,396 @@ package java import ( - "android/soong/android" - "android/soong/genrule" "fmt" - "io" "path" "path/filepath" + "reflect" + "regexp" "sort" "strings" "sync" "github.com/google/blueprint" "github.com/google/blueprint/proptools" + + "android/soong/android" ) -var ( - sdkStubsLibrarySuffix = ".stubs" - sdkSystemApiSuffix = ".system" - sdkTestApiSuffix = ".test" - sdkDocsSuffix = ".docs" - sdkXmlFileSuffix = ".xml" +const ( + sdkXmlFileSuffix = ".xml" + permissionsTemplate = `<?xml version=\"1.0\" encoding=\"utf-8\"?>\n` + + `<!-- Copyright (C) 2018 The Android Open Source Project\n` + + `\n` + + ` Licensed under the Apache License, Version 2.0 (the \"License\");\n` + + ` you may not use this file except in compliance with the License.\n` + + ` You may obtain a copy of the License at\n` + + `\n` + + ` http://www.apache.org/licenses/LICENSE-2.0\n` + + `\n` + + ` Unless required by applicable law or agreed to in writing, software\n` + + ` distributed under the License is distributed on an \"AS IS\" BASIS,\n` + + ` WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n` + + ` See the License for the specific language governing permissions and\n` + + ` limitations under the License.\n` + + `-->\n` + + `<permissions>\n` + + ` <library name=\"%s\" file=\"%s\"/>\n` + + `</permissions>\n` ) -type stubsLibraryDependencyTag struct { +// A tag to associated a dependency with a specific api scope. +type scopeDependencyTag struct { blueprint.BaseDependencyTag + name string + apiScope *apiScope + + // Function for extracting appropriate path information from the dependency. + depInfoExtractor func(paths *scopePaths, dep android.Module) error +} + +// Extract tag specific information from the dependency. +func (tag scopeDependencyTag) extractDepInfo(ctx android.ModuleContext, dep android.Module, paths *scopePaths) { + err := tag.depInfoExtractor(paths, dep) + if err != nil { + ctx.ModuleErrorf("has an invalid {scopeDependencyTag: %s} dependency on module %s: %s", tag.name, ctx.OtherModuleName(dep), err.Error()) + } +} + +// Provides information about an api scope, e.g. public, system, test. +type apiScope struct { + // The name of the api scope, e.g. public, system, test name string + + // The api scope that this scope extends. + extends *apiScope + + // The legacy enabled status for a specific scope can be dependent on other + // properties that have been specified on the library so it is provided by + // a function that can determine the status by examining those properties. + legacyEnabledStatus func(module *SdkLibrary) bool + + // The default enabled status for non-legacy behavior, which is triggered by + // explicitly enabling at least one api scope. + defaultEnabledStatus bool + + // Gets a pointer to the scope specific properties. + scopeSpecificProperties func(module *SdkLibrary) *ApiScopeProperties + + // The name of the field in the dynamically created structure. + fieldName string + + // The name of the property in the java_sdk_library_import + propertyName string + + // The tag to use to depend on the stubs library module. + stubsTag scopeDependencyTag + + // The tag to use to depend on the stubs source module (if separate from the API module). + stubsSourceTag scopeDependencyTag + + // The tag to use to depend on the API file generating module (if separate from the stubs source module). + apiFileTag scopeDependencyTag + + // The tag to use to depend on the stubs source and API module. + stubsSourceAndApiTag scopeDependencyTag + + // The scope specific prefix to add to the api file base of "current.txt" or "removed.txt". + apiFilePrefix string + + // The scope specific prefix to add to the sdk library module name to construct a scope specific + // module name. + moduleSuffix string + + // SDK version that the stubs library is built against. Note that this is always + // *current. Older stubs library built with a numbered SDK version is created from + // the prebuilt jar. + sdkVersion string + + // Extra arguments to pass to droidstubs for this scope. + droidstubsArgs []string + + // The args that must be passed to droidstubs to generate the stubs source + // for this scope. + // + // The stubs source must include the definitions of everything that is in this + // api scope and all the scopes that this one extends. + droidstubsArgsForGeneratingStubsSource []string + + // The args that must be passed to droidstubs to generate the API for this scope. + // + // The API only includes the additional members that this scope adds over the scope + // that it extends. + droidstubsArgsForGeneratingApi []string + + // True if the stubs source and api can be created by the same metalava invocation. + createStubsSourceAndApiTogether bool + + // Whether the api scope can be treated as unstable, and should skip compat checks. + unstable bool } -type syspropLibraryInterface interface { - SyspropJavaModule() *SdkLibrary +// Initialize a scope, creating and adding appropriate dependency tags +func initApiScope(scope *apiScope) *apiScope { + name := scope.name + scopeByName[name] = scope + allScopeNames = append(allScopeNames, name) + scope.propertyName = strings.ReplaceAll(name, "-", "_") + scope.fieldName = proptools.FieldNameForProperty(scope.propertyName) + scope.stubsTag = scopeDependencyTag{ + name: name + "-stubs", + apiScope: scope, + depInfoExtractor: (*scopePaths).extractStubsLibraryInfoFromDependency, + } + scope.stubsSourceTag = scopeDependencyTag{ + name: name + "-stubs-source", + apiScope: scope, + depInfoExtractor: (*scopePaths).extractStubsSourceInfoFromDep, + } + scope.apiFileTag = scopeDependencyTag{ + name: name + "-api", + apiScope: scope, + depInfoExtractor: (*scopePaths).extractApiInfoFromDep, + } + scope.stubsSourceAndApiTag = scopeDependencyTag{ + name: name + "-stubs-source-and-api", + apiScope: scope, + depInfoExtractor: (*scopePaths).extractStubsSourceAndApiInfoFromApiStubsProvider, + } + + // To get the args needed to generate the stubs source append all the args from + // this scope and all the scopes it extends as each set of args adds additional + // members to the stubs. + var stubsSourceArgs []string + for s := scope; s != nil; s = s.extends { + stubsSourceArgs = append(stubsSourceArgs, s.droidstubsArgs...) + } + scope.droidstubsArgsForGeneratingStubsSource = stubsSourceArgs + + // Currently the args needed to generate the API are the same as the args + // needed to add additional members. + apiArgs := scope.droidstubsArgs + scope.droidstubsArgsForGeneratingApi = apiArgs + + // If the args needed to generate the stubs and API are the same then they + // can be generated in a single invocation of metalava, otherwise they will + // need separate invocations. + scope.createStubsSourceAndApiTogether = reflect.DeepEqual(stubsSourceArgs, apiArgs) + + return scope } -var ( - publicApiStubsTag = dependencyTag{name: "public"} - systemApiStubsTag = dependencyTag{name: "system"} - testApiStubsTag = dependencyTag{name: "test"} - publicApiFileTag = dependencyTag{name: "publicApi"} - systemApiFileTag = dependencyTag{name: "systemApi"} - testApiFileTag = dependencyTag{name: "testApi"} -) +func (scope *apiScope) stubsLibraryModuleName(baseName string) string { + return baseName + ".stubs" + scope.moduleSuffix +} -type apiScope int +func (scope *apiScope) stubsSourceModuleName(baseName string) string { + return baseName + ".stubs.source" + scope.moduleSuffix +} -const ( - apiScopePublic apiScope = iota - apiScopeSystem - apiScopeTest +func (scope *apiScope) apiModuleName(baseName string) string { + return baseName + ".api" + scope.moduleSuffix +} + +func (scope *apiScope) String() string { + return scope.name +} + +type apiScopes []*apiScope + +func (scopes apiScopes) Strings(accessor func(*apiScope) string) []string { + var list []string + for _, scope := range scopes { + list = append(list, accessor(scope)) + } + return list +} + +var ( + scopeByName = make(map[string]*apiScope) + allScopeNames []string + apiScopePublic = initApiScope(&apiScope{ + name: "public", + + // Public scope is enabled by default for both legacy and non-legacy modes. + legacyEnabledStatus: func(module *SdkLibrary) bool { + return true + }, + defaultEnabledStatus: true, + + scopeSpecificProperties: func(module *SdkLibrary) *ApiScopeProperties { + return &module.sdkLibraryProperties.Public + }, + sdkVersion: "current", + }) + apiScopeSystem = initApiScope(&apiScope{ + name: "system", + extends: apiScopePublic, + legacyEnabledStatus: (*SdkLibrary).generateTestAndSystemScopesByDefault, + scopeSpecificProperties: func(module *SdkLibrary) *ApiScopeProperties { + return &module.sdkLibraryProperties.System + }, + apiFilePrefix: "system-", + moduleSuffix: ".system", + sdkVersion: "system_current", + droidstubsArgs: []string{"-showAnnotation android.annotation.SystemApi\\(client=android.annotation.SystemApi.Client.PRIVILEGED_APPS\\)"}, + }) + apiScopeTest = initApiScope(&apiScope{ + name: "test", + extends: apiScopePublic, + legacyEnabledStatus: (*SdkLibrary).generateTestAndSystemScopesByDefault, + scopeSpecificProperties: func(module *SdkLibrary) *ApiScopeProperties { + return &module.sdkLibraryProperties.Test + }, + apiFilePrefix: "test-", + moduleSuffix: ".test", + sdkVersion: "test_current", + droidstubsArgs: []string{"-showAnnotation android.annotation.TestApi"}, + unstable: true, + }) + apiScopeModuleLib = initApiScope(&apiScope{ + name: "module-lib", + extends: apiScopeSystem, + // The module-lib scope is disabled by default in legacy mode. + // + // Enabling this would break existing usages. + legacyEnabledStatus: func(module *SdkLibrary) bool { + return false + }, + scopeSpecificProperties: func(module *SdkLibrary) *ApiScopeProperties { + return &module.sdkLibraryProperties.Module_lib + }, + apiFilePrefix: "module-lib-", + moduleSuffix: ".module_lib", + sdkVersion: "module_current", + droidstubsArgs: []string{ + "--show-annotation android.annotation.SystemApi\\(client=android.annotation.SystemApi.Client.MODULE_LIBRARIES\\)", + }, + }) + apiScopeSystemServer = initApiScope(&apiScope{ + name: "system-server", + extends: apiScopePublic, + // The system-server scope is disabled by default in legacy mode. + // + // Enabling this would break existing usages. + legacyEnabledStatus: func(module *SdkLibrary) bool { + return false + }, + scopeSpecificProperties: func(module *SdkLibrary) *ApiScopeProperties { + return &module.sdkLibraryProperties.System_server + }, + apiFilePrefix: "system-server-", + moduleSuffix: ".system_server", + sdkVersion: "system_server_current", + droidstubsArgs: []string{ + "--show-annotation android.annotation.SystemApi\\(client=android.annotation.SystemApi.Client.SYSTEM_SERVER\\) ", + "--hide-annotation android.annotation.Hide", + // com.android.* classes are okay in this interface" + "--hide InternalClasses", + }, + }) + allApiScopes = apiScopes{ + apiScopePublic, + apiScopeSystem, + apiScopeTest, + apiScopeModuleLib, + apiScopeSystemServer, + } ) var ( javaSdkLibrariesLock sync.Mutex ) -// java_sdk_library is to make a Java library that implements optional platform APIs to apps. -// It is actually a wrapper of several modules: 1) stubs library that clients are linked against -// to, 2) droiddoc module that internally generates API stubs source files, 3) the real runtime -// shared library that implements the APIs, and 4) XML file for adding the runtime lib to the -// classpath at runtime if requested via <uses-library>. -// // TODO: these are big features that are currently missing // 1) disallowing linking to the runtime shared lib // 2) HTML generation func init() { - android.RegisterModuleType("java_sdk_library", SdkLibraryFactory) - - android.PreArchMutators(func(ctx android.RegisterMutatorsContext) { - ctx.TopDown("java_sdk_library", SdkLibraryMutator).Parallel() - }) + RegisterSdkLibraryBuildComponents(android.InitRegistrationContext) android.RegisterMakeVarsProvider(pctx, func(ctx android.MakeVarsContext) { javaSdkLibraries := javaSdkLibraries(ctx.Config()) sort.Strings(*javaSdkLibraries) ctx.Strict("JAVA_SDK_LIBRARIES", strings.Join(*javaSdkLibraries, " ")) }) + + // Register sdk member types. + android.RegisterSdkMemberType(&sdkLibrarySdkMemberType{ + android.SdkMemberTypeBase{ + PropertyName: "java_sdk_libs", + SupportsSdk: true, + }, + }) +} + +func RegisterSdkLibraryBuildComponents(ctx android.RegistrationContext) { + ctx.RegisterModuleType("java_sdk_library", SdkLibraryFactory) + ctx.RegisterModuleType("java_sdk_library_import", sdkLibraryImportFactory) +} + +// Properties associated with each api scope. +type ApiScopeProperties struct { + // Indicates whether the api surface is generated. + // + // If this is set for any scope then all scopes must explicitly specify if they + // are enabled. This is to prevent new usages from depending on legacy behavior. + // + // Otherwise, if this is not set for any scope then the default behavior is + // scope specific so please refer to the scope specific property documentation. + Enabled *bool + + // The sdk_version to use for building the stubs. + // + // If not specified then it will use an sdk_version determined as follows: + // 1) If the sdk_version specified on the java_sdk_library is none then this + // will be none. This is used for java_sdk_library instances that are used + // to create stubs that contribute to the core_current sdk version. + // 2) Otherwise, it is assumed that this library extends but does not contribute + // directly to a specific sdk_version and so this uses the sdk_version appropriate + // for the api scope. e.g. public will use sdk_version: current, system will use + // sdk_version: system_current, etc. + // + // This does not affect the sdk_version used for either generating the stubs source + // or the API file. They both have to use the same sdk_version as is used for + // compiling the implementation library. + Sdk_version *string } type sdkLibraryProperties struct { - // list of optional source files that are part of API but not part of runtime library. - Api_srcs []string `android:"arch_variant"` + // Visibility for impl library module. If not specified then defaults to the + // visibility property. + Impl_library_visibility []string + + // Visibility for stubs library modules. If not specified then defaults to the + // visibility property. + Stubs_library_visibility []string + + // Visibility for stubs source modules. If not specified then defaults to the + // visibility property. + Stubs_source_visibility []string // List of Java libraries that will be in the classpath when building stubs Stub_only_libs []string `android:"arch_variant"` - // list of package names that will be documented and publicized as API + // list of package names that will be documented and publicized as API. + // This allows the API to be restricted to a subset of the source files provided. + // If this is unspecified then all the source files will be treated as being part + // of the API. Api_packages []string // list of package names that must be hidden from the API Hidden_api_packages []string + // the relative path to the directory containing the api specification files. + // Defaults to "api". + Api_dir *string + + // Determines whether a runtime implementation library is built; defaults to false. + // + // If true then it also prevents the module from being used as a shared module, i.e. + // it is as is shared_library: false, was set. + Api_only *bool + // local files that are used within user customized droiddoc options. Droiddoc_option_files []string @@ -113,15 +414,8 @@ type sdkLibraryProperties struct { // $(location <label>): the path to the droiddoc_option_files with name <label> Droiddoc_options []string - // the java library (in classpath) for documentation that provides java srcs and srcjars. - Srcs_lib *string - - // the base dirs under srcs_lib will be scanned for java srcs. - Srcs_lib_whitelist_dirs []string - - // the sub dirs under srcs_lib_whitelist_dirs will be scanned for java srcs. - // Defaults to "android.annotation". - Srcs_lib_whitelist_pkgs []string + // is set to true, Metalava will allow framework SDK to contain annotations. + Annotations_enabled *bool // a list of top-level directories containing files to merge qualifier annotations // (i.e. those intended to be included in the stubs written) from. @@ -136,273 +430,708 @@ type sdkLibraryProperties struct { // don't create dist rules. No_dist *bool `blueprint:"mutated"` + // indicates whether system and test apis should be generated. + Generate_system_and_test_apis bool `blueprint:"mutated"` + + // The properties specific to the public api scope + // + // Unless explicitly specified by using public.enabled the public api scope is + // enabled by default in both legacy and non-legacy mode. + Public ApiScopeProperties + + // The properties specific to the system api scope + // + // In legacy mode the system api scope is enabled by default when sdk_version + // is set to something other than "none". + // + // In non-legacy mode the system api scope is disabled by default. + System ApiScopeProperties + + // The properties specific to the test api scope + // + // In legacy mode the test api scope is enabled by default when sdk_version + // is set to something other than "none". + // + // In non-legacy mode the test api scope is disabled by default. + Test ApiScopeProperties + + // The properties specific to the module-lib api scope + // + // Unless explicitly specified by using test.enabled the module-lib api scope is + // disabled by default. + Module_lib ApiScopeProperties + + // The properties specific to the system-server api scope + // + // Unless explicitly specified by using test.enabled the module-lib api scope is + // disabled by default. + System_server ApiScopeProperties + + // Determines if the stubs are preferred over the implementation library + // for linking, even when the client doesn't specify sdk_version. When this + // is set to true, such clients are provided with the widest API surface that + // this lib provides. Note however that this option doesn't affect the clients + // that are in the same APEX as this library. In that case, the clients are + // always linked with the implementation library. Default is false. + Default_to_stubs *bool + + // Properties related to api linting. + Api_lint struct { + // Enable api linting. + Enabled *bool + } + // TODO: determines whether to create HTML doc or not //Html_doc *bool } -type SdkLibrary struct { - Library +// Paths to outputs from java_sdk_library and java_sdk_library_import. +// +// Fields that are android.Paths are always set (during GenerateAndroidBuildActions). +// OptionalPaths are always set by java_sdk_library but may not be set by +// java_sdk_library_import as not all instances provide that information. +type scopePaths struct { + // The path (represented as Paths for convenience when returning) to the stubs header jar. + // + // That is the jar that is created by turbine. + stubsHeaderPath android.Paths - sdkLibraryProperties sdkLibraryProperties + // The path (represented as Paths for convenience when returning) to the stubs implementation jar. + // + // This is not the implementation jar, it still only contains stubs. + stubsImplPath android.Paths - publicApiStubsPath android.Paths - systemApiStubsPath android.Paths - testApiStubsPath android.Paths + // The API specification file, e.g. system_current.txt. + currentApiFilePath android.OptionalPath - publicApiStubsImplPath android.Paths - systemApiStubsImplPath android.Paths - testApiStubsImplPath android.Paths + // The specification of API elements removed since the last release. + removedApiFilePath android.OptionalPath - publicApiFilePath android.Path - systemApiFilePath android.Path - testApiFilePath android.Path + // The stubs source jar. + stubsSrcJar android.OptionalPath } -var _ Dependency = (*SdkLibrary)(nil) -var _ SdkLibraryDependency = (*SdkLibrary)(nil) +func (paths *scopePaths) extractStubsLibraryInfoFromDependency(dep android.Module) error { + if lib, ok := dep.(Dependency); ok { + paths.stubsHeaderPath = lib.HeaderJars() + paths.stubsImplPath = lib.ImplementationJars() + return nil + } else { + return fmt.Errorf("expected module that implements Dependency, e.g. java_library") + } +} -func (module *SdkLibrary) DepsMutator(ctx android.BottomUpMutatorContext) { - useBuiltStubs := !ctx.Config().UnbundledBuildUsePrebuiltSdks() - // Add dependencies to the stubs library - if useBuiltStubs { - ctx.AddVariationDependencies(nil, publicApiStubsTag, module.stubsName(apiScopePublic)) +func (paths *scopePaths) treatDepAsApiStubsProvider(dep android.Module, action func(provider ApiStubsProvider)) error { + if apiStubsProvider, ok := dep.(ApiStubsProvider); ok { + action(apiStubsProvider) + return nil + } else { + return fmt.Errorf("expected module that implements ApiStubsProvider, e.g. droidstubs") } - ctx.AddVariationDependencies(nil, publicApiFileTag, module.docsName(apiScopePublic)) +} - if !Bool(module.properties.No_standard_libs) { - if useBuiltStubs { - ctx.AddVariationDependencies(nil, systemApiStubsTag, module.stubsName(apiScopeSystem)) - ctx.AddVariationDependencies(nil, testApiStubsTag, module.stubsName(apiScopeTest)) - } - ctx.AddVariationDependencies(nil, systemApiFileTag, module.docsName(apiScopeSystem)) - ctx.AddVariationDependencies(nil, testApiFileTag, module.docsName(apiScopeTest)) +func (paths *scopePaths) treatDepAsApiStubsSrcProvider(dep android.Module, action func(provider ApiStubsSrcProvider)) error { + if apiStubsProvider, ok := dep.(ApiStubsSrcProvider); ok { + action(apiStubsProvider) + return nil + } else { + return fmt.Errorf("expected module that implements ApiStubsSrcProvider, e.g. droidstubs") } +} - module.Library.deps(ctx) +func (paths *scopePaths) extractApiInfoFromApiStubsProvider(provider ApiStubsProvider) { + paths.currentApiFilePath = android.OptionalPathForPath(provider.ApiFilePath()) + paths.removedApiFilePath = android.OptionalPathForPath(provider.RemovedApiFilePath()) } -func (module *SdkLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) { - module.Library.GenerateAndroidBuildActions(ctx) +func (paths *scopePaths) extractApiInfoFromDep(dep android.Module) error { + return paths.treatDepAsApiStubsProvider(dep, func(provider ApiStubsProvider) { + paths.extractApiInfoFromApiStubsProvider(provider) + }) +} - // Record the paths to the header jars of the library (stubs and impl). - // When this java_sdk_library is dependened from others via "libs" property, - // the recorded paths will be returned depending on the link type of the caller. - ctx.VisitDirectDeps(func(to android.Module) { - otherName := ctx.OtherModuleName(to) - tag := ctx.OtherModuleDependencyTag(to) +func (paths *scopePaths) extractStubsSourceInfoFromApiStubsProviders(provider ApiStubsSrcProvider) { + paths.stubsSrcJar = android.OptionalPathForPath(provider.StubsSrcJar()) +} + +func (paths *scopePaths) extractStubsSourceInfoFromDep(dep android.Module) error { + return paths.treatDepAsApiStubsSrcProvider(dep, func(provider ApiStubsSrcProvider) { + paths.extractStubsSourceInfoFromApiStubsProviders(provider) + }) +} + +func (paths *scopePaths) extractStubsSourceAndApiInfoFromApiStubsProvider(dep android.Module) error { + return paths.treatDepAsApiStubsProvider(dep, func(provider ApiStubsProvider) { + paths.extractApiInfoFromApiStubsProvider(provider) + paths.extractStubsSourceInfoFromApiStubsProviders(provider) + }) +} - if lib, ok := to.(Dependency); ok { - switch tag { - case publicApiStubsTag: - module.publicApiStubsPath = lib.HeaderJars() - module.publicApiStubsImplPath = lib.ImplementationJars() - case systemApiStubsTag: - module.systemApiStubsPath = lib.HeaderJars() - module.systemApiStubsImplPath = lib.ImplementationJars() - case testApiStubsTag: - module.testApiStubsPath = lib.HeaderJars() - module.testApiStubsImplPath = lib.ImplementationJars() +type commonToSdkLibraryAndImportProperties struct { + // The naming scheme to use for the components that this module creates. + // + // If not specified then it defaults to "default". The other allowable value is + // "framework-modules" which matches the scheme currently used by framework modules + // for the equivalent components represented as separate Soong modules. + // + // This is a temporary mechanism to simplify conversion from separate modules for each + // component that follow a different naming pattern to the default one. + // + // TODO(b/155480189) - Remove once naming inconsistencies have been resolved. + Naming_scheme *string + + // Specifies whether this module can be used as an Android shared library; defaults + // to true. + // + // An Android shared library is one that can be referenced in a <uses-library> element + // in an AndroidManifest.xml. + Shared_library *bool +} + +// Common code between sdk library and sdk library import +type commonToSdkLibraryAndImport struct { + moduleBase *android.ModuleBase + + scopePaths map[*apiScope]*scopePaths + + namingScheme sdkLibraryComponentNamingScheme + + commonSdkLibraryProperties commonToSdkLibraryAndImportProperties + + // Functionality related to this being used as a component of a java_sdk_library. + EmbeddableSdkLibraryComponent +} + +func (c *commonToSdkLibraryAndImport) initCommon(moduleBase *android.ModuleBase) { + c.moduleBase = moduleBase + + moduleBase.AddProperties(&c.commonSdkLibraryProperties) + + // Initialize this as an sdk library component. + c.initSdkLibraryComponent(moduleBase) +} + +func (c *commonToSdkLibraryAndImport) initCommonAfterDefaultsApplied(ctx android.DefaultableHookContext) bool { + schemeProperty := proptools.StringDefault(c.commonSdkLibraryProperties.Naming_scheme, "default") + switch schemeProperty { + case "default": + c.namingScheme = &defaultNamingScheme{} + case "framework-modules": + c.namingScheme = &frameworkModulesNamingScheme{} + default: + ctx.PropertyErrorf("naming_scheme", "expected 'default' but was %q", schemeProperty) + return false + } + + // Only track this sdk library if this can be used as a shared library. + if c.sharedLibrary() { + // Use the name specified in the module definition as the owner. + c.sdkLibraryComponentProperties.SdkLibraryToImplicitlyTrack = proptools.StringPtr(c.moduleBase.BaseModuleName()) + } + + return true +} + +// Module name of the runtime implementation library +func (c *commonToSdkLibraryAndImport) implLibraryModuleName() string { + return c.moduleBase.BaseModuleName() + ".impl" +} + +// Module name of the XML file for the lib +func (c *commonToSdkLibraryAndImport) xmlPermissionsModuleName() string { + return c.moduleBase.BaseModuleName() + sdkXmlFileSuffix +} + +// Name of the java_library module that compiles the stubs source. +func (c *commonToSdkLibraryAndImport) stubsLibraryModuleName(apiScope *apiScope) string { + return c.namingScheme.stubsLibraryModuleName(apiScope, c.moduleBase.BaseModuleName()) +} + +// Name of the droidstubs module that generates the stubs source and may also +// generate/check the API. +func (c *commonToSdkLibraryAndImport) stubsSourceModuleName(apiScope *apiScope) string { + return c.namingScheme.stubsSourceModuleName(apiScope, c.moduleBase.BaseModuleName()) +} + +// Name of the droidstubs module that generates/checks the API. Only used if it +// requires different arts to the stubs source generating module. +func (c *commonToSdkLibraryAndImport) apiModuleName(apiScope *apiScope) string { + return c.namingScheme.apiModuleName(apiScope, c.moduleBase.BaseModuleName()) +} + +// The component names for different outputs of the java_sdk_library. +// +// They are similar to the names used for the child modules it creates +const ( + stubsSourceComponentName = "stubs.source" + + apiTxtComponentName = "api.txt" + + removedApiTxtComponentName = "removed-api.txt" +) + +// A regular expression to match tags that reference a specific stubs component. +// +// It will only match if given a valid scope and a valid component. It is verfy strict +// to ensure it does not accidentally match a similar looking tag that should be processed +// by the embedded Library. +var tagSplitter = func() *regexp.Regexp { + // Given a list of literal string items returns a regular expression that will + // match any one of the items. + choice := func(items ...string) string { + return `\Q` + strings.Join(items, `\E|\Q`) + `\E` + } + + // Regular expression to match one of the scopes. + scopesRegexp := choice(allScopeNames...) + + // Regular expression to match one of the components. + componentsRegexp := choice(stubsSourceComponentName, apiTxtComponentName, removedApiTxtComponentName) + + // Regular expression to match any combination of one scope and one component. + return regexp.MustCompile(fmt.Sprintf(`^\.(%s)\.(%s)$`, scopesRegexp, componentsRegexp)) +}() + +// For OutputFileProducer interface +// +// .<scope>.stubs.source +// .<scope>.api.txt +// .<scope>.removed-api.txt +func (c *commonToSdkLibraryAndImport) commonOutputFiles(tag string) (android.Paths, error) { + if groups := tagSplitter.FindStringSubmatch(tag); groups != nil { + scopeName := groups[1] + component := groups[2] + + if scope, ok := scopeByName[scopeName]; ok { + paths := c.findScopePaths(scope) + if paths == nil { + return nil, fmt.Errorf("%q does not provide api scope %s", c.moduleBase.BaseModuleName(), scopeName) + } + + switch component { + case stubsSourceComponentName: + if paths.stubsSrcJar.Valid() { + return android.Paths{paths.stubsSrcJar.Path()}, nil + } + + case apiTxtComponentName: + if paths.currentApiFilePath.Valid() { + return android.Paths{paths.currentApiFilePath.Path()}, nil + } + + case removedApiTxtComponentName: + if paths.removedApiFilePath.Valid() { + return android.Paths{paths.removedApiFilePath.Path()}, nil + } } + + return nil, fmt.Errorf("%s not available for api scope %s", component, scopeName) + } else { + return nil, fmt.Errorf("unknown scope %s in %s", scope, tag) } - if doc, ok := to.(ApiFilePath); ok { - switch tag { - case publicApiFileTag: - module.publicApiFilePath = doc.ApiFilePath() - case systemApiFileTag: - module.systemApiFilePath = doc.ApiFilePath() - case testApiFileTag: - module.testApiFilePath = doc.ApiFilePath() - default: - ctx.ModuleErrorf("depends on module %q of unknown tag %q", otherName, tag) + + } else { + return nil, nil + } +} + +func (c *commonToSdkLibraryAndImport) getScopePathsCreateIfNeeded(scope *apiScope) *scopePaths { + if c.scopePaths == nil { + c.scopePaths = make(map[*apiScope]*scopePaths) + } + paths := c.scopePaths[scope] + if paths == nil { + paths = &scopePaths{} + c.scopePaths[scope] = paths + } + + return paths +} + +func (c *commonToSdkLibraryAndImport) findScopePaths(scope *apiScope) *scopePaths { + if c.scopePaths == nil { + return nil + } + + return c.scopePaths[scope] +} + +// If this does not support the requested api scope then find the closest available +// scope it does support. Returns nil if no such scope is available. +func (c *commonToSdkLibraryAndImport) findClosestScopePath(scope *apiScope) *scopePaths { + for s := scope; s != nil; s = s.extends { + if paths := c.findScopePaths(s); paths != nil { + return paths + } + } + + // This should never happen outside tests as public should be the base scope for every + // scope and is enabled by default. + return nil +} + +func (c *commonToSdkLibraryAndImport) selectHeaderJarsForSdkVersion(ctx android.BaseModuleContext, sdkVersion sdkSpec) android.Paths { + + // If a specific numeric version has been requested then use prebuilt versions of the sdk. + if sdkVersion.version.isNumbered() { + return PrebuiltJars(ctx, c.moduleBase.BaseModuleName(), sdkVersion) + } + + var apiScope *apiScope + switch sdkVersion.kind { + case sdkSystem: + apiScope = apiScopeSystem + case sdkModule: + apiScope = apiScopeModuleLib + case sdkTest: + apiScope = apiScopeTest + case sdkSystemServer: + apiScope = apiScopeSystemServer + default: + apiScope = apiScopePublic + } + + paths := c.findClosestScopePath(apiScope) + if paths == nil { + var scopes []string + for _, s := range allApiScopes { + if c.findScopePaths(s) != nil { + scopes = append(scopes, s.name) } } - }) + ctx.ModuleErrorf("requires api scope %s from %s but it only has %q available", apiScope.name, c.moduleBase.BaseModuleName(), scopes) + return nil + } + + return paths.stubsHeaderPath } -func (module *SdkLibrary) AndroidMk() android.AndroidMkData { - data := module.Library.AndroidMk() - data.Required = append(data.Required, module.xmlFileName()) +func (c *commonToSdkLibraryAndImport) sdkComponentPropertiesForChildLibrary() interface{} { + componentProps := &struct { + SdkLibraryToImplicitlyTrack *string + }{} - data.Custom = func(w io.Writer, name, prefix, moduleDir string, data android.AndroidMkData) { - android.WriteAndroidMkData(w, data) + if c.sharedLibrary() { + // Mark the stubs library as being components of this java_sdk_library so that + // any app that includes code which depends (directly or indirectly) on the stubs + // library will have the appropriate <uses-library> invocation inserted into its + // manifest if necessary. + componentProps.SdkLibraryToImplicitlyTrack = proptools.StringPtr(c.moduleBase.BaseModuleName()) + } - module.Library.AndroidMkHostDex(w, name, data) - if !Bool(module.sdkLibraryProperties.No_dist) { - // Create a phony module that installs the impl library, for the case when this lib is - // in PRODUCT_PACKAGES. - owner := module.ModuleBase.Owner() - if owner == "" { - if Bool(module.sdkLibraryProperties.Core_lib) { - owner = "core" - } else { - owner = "android" + return componentProps +} + +// Check if this can be used as a shared library. +func (c *commonToSdkLibraryAndImport) sharedLibrary() bool { + return proptools.BoolDefault(c.commonSdkLibraryProperties.Shared_library, true) +} + +// Properties related to the use of a module as an component of a java_sdk_library. +type SdkLibraryComponentProperties struct { + + // The name of the java_sdk_library/_import to add to a <uses-library> entry + // in the AndroidManifest.xml of any Android app that includes code that references + // this module. If not set then no java_sdk_library/_import is tracked. + SdkLibraryToImplicitlyTrack *string `blueprint:"mutated"` +} + +// Structure to be embedded in a module struct that needs to support the +// SdkLibraryComponentDependency interface. +type EmbeddableSdkLibraryComponent struct { + sdkLibraryComponentProperties SdkLibraryComponentProperties +} + +func (e *EmbeddableSdkLibraryComponent) initSdkLibraryComponent(moduleBase *android.ModuleBase) { + moduleBase.AddProperties(&e.sdkLibraryComponentProperties) +} + +// to satisfy SdkLibraryComponentDependency +func (e *EmbeddableSdkLibraryComponent) OptionalImplicitSdkLibrary() []string { + if e.sdkLibraryComponentProperties.SdkLibraryToImplicitlyTrack != nil { + return []string{*e.sdkLibraryComponentProperties.SdkLibraryToImplicitlyTrack} + } + return nil +} + +// Implemented by modules that are (or possibly could be) a component of a java_sdk_library +// (including the java_sdk_library) itself. +type SdkLibraryComponentDependency interface { + // The optional name of the sdk library that should be implicitly added to the + // AndroidManifest of an app that contains code which references the sdk library. + // + // Returns an array containing 0 or 1 items rather than a *string to make it easier + // to append this to the list of exported sdk libraries. + OptionalImplicitSdkLibrary() []string +} + +// Make sure that all the module types that are components of java_sdk_library/_import +// and which can be referenced (directly or indirectly) from an android app implement +// the SdkLibraryComponentDependency interface. +var _ SdkLibraryComponentDependency = (*Library)(nil) +var _ SdkLibraryComponentDependency = (*Import)(nil) +var _ SdkLibraryComponentDependency = (*SdkLibrary)(nil) +var _ SdkLibraryComponentDependency = (*SdkLibraryImport)(nil) + +// Provides access to sdk_version related header and implentation jars. +type SdkLibraryDependency interface { + SdkLibraryComponentDependency + + // Get the header jars appropriate for the supplied sdk_version. + // + // These are turbine generated jars so they only change if the externals of the + // class changes but it does not contain and implementation or JavaDoc. + SdkHeaderJars(ctx android.BaseModuleContext, sdkVersion sdkSpec) android.Paths + + // Get the implementation jars appropriate for the supplied sdk version. + // + // These are either the implementation jar for the whole sdk library or the implementation + // jars for the stubs. The latter should only be needed when generating JavaDoc as otherwise + // they are identical to the corresponding header jars. + SdkImplementationJars(ctx android.BaseModuleContext, sdkVersion sdkSpec) android.Paths +} + +type SdkLibrary struct { + Library + + sdkLibraryProperties sdkLibraryProperties + + // Map from api scope to the scope specific property structure. + scopeToProperties map[*apiScope]*ApiScopeProperties + + commonToSdkLibraryAndImport +} + +var _ Dependency = (*SdkLibrary)(nil) +var _ SdkLibraryDependency = (*SdkLibrary)(nil) + +func (module *SdkLibrary) generateTestAndSystemScopesByDefault() bool { + return module.sdkLibraryProperties.Generate_system_and_test_apis +} + +func (module *SdkLibrary) getGeneratedApiScopes(ctx android.EarlyModuleContext) apiScopes { + // Check to see if any scopes have been explicitly enabled. If any have then all + // must be. + anyScopesExplicitlyEnabled := false + for _, scope := range allApiScopes { + scopeProperties := module.scopeToProperties[scope] + if scopeProperties.Enabled != nil { + anyScopesExplicitlyEnabled = true + break + } + } + + var generatedScopes apiScopes + enabledScopes := make(map[*apiScope]struct{}) + for _, scope := range allApiScopes { + scopeProperties := module.scopeToProperties[scope] + // If any scopes are explicitly enabled then ignore the legacy enabled status. + // This is to ensure that any new usages of this module type do not rely on legacy + // behaviour. + defaultEnabledStatus := false + if anyScopesExplicitlyEnabled { + defaultEnabledStatus = scope.defaultEnabledStatus + } else { + defaultEnabledStatus = scope.legacyEnabledStatus(module) + } + enabled := proptools.BoolDefault(scopeProperties.Enabled, defaultEnabledStatus) + if enabled { + enabledScopes[scope] = struct{}{} + generatedScopes = append(generatedScopes, scope) + } + } + + // Now check to make sure that any scope that is extended by an enabled scope is also + // enabled. + for _, scope := range allApiScopes { + if _, ok := enabledScopes[scope]; ok { + extends := scope.extends + if extends != nil { + if _, ok := enabledScopes[extends]; !ok { + ctx.ModuleErrorf("enabled api scope %q depends on disabled scope %q", scope, extends) } } - // Create dist rules to install the stubs libs to the dist dir - if len(module.publicApiStubsPath) == 1 { - fmt.Fprintln(w, "$(call dist-for-goals,sdk win_sdk,"+ - module.publicApiStubsImplPath.Strings()[0]+ - ":"+path.Join("apistubs", owner, "public", - module.BaseModuleName()+".jar")+")") - } - if len(module.systemApiStubsPath) == 1 { - fmt.Fprintln(w, "$(call dist-for-goals,sdk win_sdk,"+ - module.systemApiStubsImplPath.Strings()[0]+ - ":"+path.Join("apistubs", owner, "system", - module.BaseModuleName()+".jar")+")") - } - if len(module.testApiStubsPath) == 1 { - fmt.Fprintln(w, "$(call dist-for-goals,sdk win_sdk,"+ - module.testApiStubsImplPath.Strings()[0]+ - ":"+path.Join("apistubs", owner, "test", - module.BaseModuleName()+".jar")+")") - } - if module.publicApiFilePath != nil { - fmt.Fprintln(w, "$(call dist-for-goals,sdk win_sdk,"+ - module.publicApiFilePath.String()+ - ":"+path.Join("apistubs", owner, "public", "api", - module.BaseModuleName()+".txt")+")") - } - if module.systemApiFilePath != nil { - fmt.Fprintln(w, "$(call dist-for-goals,sdk win_sdk,"+ - module.systemApiFilePath.String()+ - ":"+path.Join("apistubs", owner, "system", "api", - module.BaseModuleName()+".txt")+")") - } - if module.testApiFilePath != nil { - fmt.Fprintln(w, "$(call dist-for-goals,sdk win_sdk,"+ - module.testApiFilePath.String()+ - ":"+path.Join("apistubs", owner, "test", "api", - module.BaseModuleName()+".txt")+")") - } } } - return data + + return generatedScopes } -// Module name of the stubs library -func (module *SdkLibrary) stubsName(apiScope apiScope) string { - stubsName := module.BaseModuleName() + sdkStubsLibrarySuffix - switch apiScope { - case apiScopeSystem: - stubsName = stubsName + sdkSystemApiSuffix - case apiScopeTest: - stubsName = stubsName + sdkTestApiSuffix +type sdkLibraryComponentTag struct { + blueprint.BaseDependencyTag + name string +} + +// Mark this tag so dependencies that use it are excluded from visibility enforcement. +func (t sdkLibraryComponentTag) ExcludeFromVisibilityEnforcement() {} + +var xmlPermissionsFileTag = sdkLibraryComponentTag{name: "xml-permissions-file"} + +func IsXmlPermissionsFileDepTag(depTag blueprint.DependencyTag) bool { + if dt, ok := depTag.(sdkLibraryComponentTag); ok { + return dt == xmlPermissionsFileTag } - return stubsName + return false } -// Module name of the docs -func (module *SdkLibrary) docsName(apiScope apiScope) string { - docsName := module.BaseModuleName() + sdkDocsSuffix - switch apiScope { - case apiScopeSystem: - docsName = docsName + sdkSystemApiSuffix - case apiScopeTest: - docsName = docsName + sdkTestApiSuffix +var implLibraryTag = sdkLibraryComponentTag{name: "impl-library"} + +func (module *SdkLibrary) DepsMutator(ctx android.BottomUpMutatorContext) { + for _, apiScope := range module.getGeneratedApiScopes(ctx) { + // Add dependencies to the stubs library + ctx.AddVariationDependencies(nil, apiScope.stubsTag, module.stubsLibraryModuleName(apiScope)) + + // If the stubs source and API cannot be generated together then add an additional dependency on + // the API module. + if apiScope.createStubsSourceAndApiTogether { + // Add a dependency on the stubs source in order to access both stubs source and api information. + ctx.AddVariationDependencies(nil, apiScope.stubsSourceAndApiTag, module.stubsSourceModuleName(apiScope)) + } else { + // Add separate dependencies on the creators of the stubs source files and the API. + ctx.AddVariationDependencies(nil, apiScope.stubsSourceTag, module.stubsSourceModuleName(apiScope)) + ctx.AddVariationDependencies(nil, apiScope.apiFileTag, module.apiModuleName(apiScope)) + } + } + + if module.requiresRuntimeImplementationLibrary() { + // Add dependency to the rule for generating the implementation library. + ctx.AddDependency(module, implLibraryTag, module.implLibraryModuleName()) + + if module.sharedLibrary() { + // Add dependency to the rule for generating the xml permissions file + ctx.AddDependency(module, xmlPermissionsFileTag, module.xmlPermissionsModuleName()) + } + + // Only add the deps for the library if it is actually going to be built. + module.Library.deps(ctx) } - return docsName } -// Module name of the runtime implementation library -func (module *SdkLibrary) implName() string { - return module.BaseModuleName() +func (module *SdkLibrary) OutputFiles(tag string) (android.Paths, error) { + paths, err := module.commonOutputFiles(tag) + if paths == nil && err == nil { + return module.Library.OutputFiles(tag) + } else { + return paths, err + } } -// File path to the runtime implementation library -func (module *SdkLibrary) implPath() string { - partition := "system" - if module.SocSpecific() { - partition = "vendor" - } else if module.DeviceSpecific() { - partition = "odm" - } else if module.ProductSpecific() { - partition = "product" +func (module *SdkLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) { + // Only build an implementation library if required. + if module.requiresRuntimeImplementationLibrary() { + module.Library.GenerateAndroidBuildActions(ctx) } - return "/" + partition + "/framework/" + module.implName() + ".jar" + + // Record the paths to the header jars of the library (stubs and impl). + // When this java_sdk_library is depended upon from others via "libs" property, + // the recorded paths will be returned depending on the link type of the caller. + ctx.VisitDirectDeps(func(to android.Module) { + tag := ctx.OtherModuleDependencyTag(to) + + // Extract information from any of the scope specific dependencies. + if scopeTag, ok := tag.(scopeDependencyTag); ok { + apiScope := scopeTag.apiScope + scopePaths := module.getScopePathsCreateIfNeeded(apiScope) + + // Extract information from the dependency. The exact information extracted + // is determined by the nature of the dependency which is determined by the tag. + scopeTag.extractDepInfo(ctx, to, scopePaths) + } + }) } -// Module name of the XML file for the lib -func (module *SdkLibrary) xmlFileName() string { - return module.BaseModuleName() + sdkXmlFileSuffix -} - -// SDK version that the stubs library is built against. Note that this is always -// *current. Older stubs library built with a numberd SDK version is created from -// the prebuilt jar. -func (module *SdkLibrary) sdkVersion(apiScope apiScope) string { - switch apiScope { - case apiScopePublic: - return "current" - case apiScopeSystem: - return "system_current" - case apiScopeTest: - return "test_current" - default: - return "current" +func (module *SdkLibrary) AndroidMkEntries() []android.AndroidMkEntries { + if !module.requiresRuntimeImplementationLibrary() { + return nil } + entriesList := module.Library.AndroidMkEntries() + entries := &entriesList[0] + entries.Required = append(entries.Required, module.xmlPermissionsModuleName()) + return entriesList } -// $(INTERNAL_PLATFORM_<apiTagName>_API_FILE) points to the generated -// api file for the current source -// TODO: remove this when apicheck is done in soong -func (module *SdkLibrary) apiTagName(apiScope apiScope) string { - apiTagName := strings.Replace(strings.ToUpper(module.BaseModuleName()), ".", "_", -1) - switch apiScope { - case apiScopeSystem: - apiTagName = apiTagName + "_SYSTEM" - case apiScopeTest: - apiTagName = apiTagName + "_TEST" +// The dist path of the stub artifacts +func (module *SdkLibrary) apiDistPath(apiScope *apiScope) string { + if module.ModuleBase.Owner() != "" { + return path.Join("apistubs", module.ModuleBase.Owner(), apiScope.name) + } else if Bool(module.sdkLibraryProperties.Core_lib) { + return path.Join("apistubs", "core", apiScope.name) + } else { + return path.Join("apistubs", "android", apiScope.name) } - return apiTagName } -func (module *SdkLibrary) latestApiFilegroupName(apiScope apiScope) string { - name := ":" + module.BaseModuleName() + ".api." - switch apiScope { - case apiScopePublic: - name = name + "public" - case apiScopeSystem: - name = name + "system" - case apiScopeTest: - name = name + "test" +// Get the sdk version for use when compiling the stubs library. +func (module *SdkLibrary) sdkVersionForStubsLibrary(mctx android.EarlyModuleContext, apiScope *apiScope) string { + scopeProperties := module.scopeToProperties[apiScope] + if scopeProperties.Sdk_version != nil { + return proptools.String(scopeProperties.Sdk_version) } - name = name + ".latest" - return name + + sdkDep := decodeSdkDep(mctx, sdkContext(&module.Library)) + if sdkDep.hasStandardLibs() { + // If building against a standard sdk then use the sdk version appropriate for the scope. + return apiScope.sdkVersion + } else { + // Otherwise, use no system module. + return "none" + } +} + +func (module *SdkLibrary) latestApiFilegroupName(apiScope *apiScope) string { + return ":" + module.BaseModuleName() + ".api." + apiScope.name + ".latest" +} + +func (module *SdkLibrary) latestRemovedApiFilegroupName(apiScope *apiScope) string { + return ":" + module.BaseModuleName() + "-removed.api." + apiScope.name + ".latest" } -func (module *SdkLibrary) latestRemovedApiFilegroupName(apiScope apiScope) string { - name := ":" + module.BaseModuleName() + "-removed.api." - switch apiScope { - case apiScopePublic: - name = name + "public" - case apiScopeSystem: - name = name + "system" - case apiScopeTest: - name = name + "test" +// Creates the implementation java library +func (module *SdkLibrary) createImplLibrary(mctx android.DefaultableHookContext) { + + moduleNamePtr := proptools.StringPtr(module.BaseModuleName()) + + props := struct { + Name *string + Visibility []string + Instrument bool + ConfigurationName *string + }{ + Name: proptools.StringPtr(module.implLibraryModuleName()), + Visibility: module.sdkLibraryProperties.Impl_library_visibility, + // Set the instrument property to ensure it is instrumented when instrumentation is required. + Instrument: true, + + // Make the created library behave as if it had the same name as this module. + ConfigurationName: moduleNamePtr, + } + + properties := []interface{}{ + &module.properties, + &module.protoProperties, + &module.deviceProperties, + &module.dexpreoptProperties, + &module.linter.properties, + &props, + module.sdkComponentPropertiesForChildLibrary(), } - name = name + ".latest" - return name + mctx.CreateModule(LibraryFactory, properties...) } // Creates a static java library that has API stubs -func (module *SdkLibrary) createStubsLibrary(mctx android.TopDownMutatorContext, apiScope apiScope) { +func (module *SdkLibrary) createStubsLibrary(mctx android.DefaultableHookContext, apiScope *apiScope) { props := struct { Name *string + Visibility []string Srcs []string + Installable *bool Sdk_version *string + System_modules *string + Patch_module *string Libs []string - Soc_specific *bool - Device_specific *bool - Product_specific *bool Compile_dex *bool - No_standard_libs *bool - System_modules *string Java_version *string Product_variables struct { - Unbundled_build struct { - Enabled *bool - } Pdk struct { Enabled *bool } @@ -411,251 +1140,313 @@ func (module *SdkLibrary) createStubsLibrary(mctx android.TopDownMutatorContext, Srcs []string Javacflags []string } + Dist struct { + Targets []string + Dest *string + Dir *string + Tag *string + } }{} - props.Name = proptools.StringPtr(module.stubsName(apiScope)) + props.Name = proptools.StringPtr(module.stubsLibraryModuleName(apiScope)) + + // If stubs_library_visibility is not set then the created module will use the + // visibility of this module. + visibility := module.sdkLibraryProperties.Stubs_library_visibility + props.Visibility = visibility + // sources are generated from the droiddoc - props.Srcs = []string{":" + module.docsName(apiScope)} - props.Sdk_version = proptools.StringPtr(module.sdkVersion(apiScope)) + props.Srcs = []string{":" + module.stubsSourceModuleName(apiScope)} + sdkVersion := module.sdkVersionForStubsLibrary(mctx, apiScope) + props.Sdk_version = proptools.StringPtr(sdkVersion) + props.System_modules = module.deviceProperties.System_modules + props.Patch_module = module.properties.Patch_module + props.Installable = proptools.BoolPtr(false) props.Libs = module.sdkLibraryProperties.Stub_only_libs - // Unbundled apps will use the prebult one from /prebuilts/sdk - if mctx.Config().UnbundledBuildUsePrebuiltSdks() { - props.Product_variables.Unbundled_build.Enabled = proptools.BoolPtr(false) + // The stub-annotations library contains special versions of the annotations + // with CLASS retention policy, so that they're kept. + if proptools.Bool(module.sdkLibraryProperties.Annotations_enabled) { + props.Libs = append(props.Libs, "stub-annotations") } props.Product_variables.Pdk.Enabled = proptools.BoolPtr(false) - props.No_standard_libs = module.Library.Module.properties.No_standard_libs - props.System_modules = module.Library.Module.deviceProperties.System_modules - props.Openjdk9.Srcs = module.Library.Module.properties.Openjdk9.Srcs - props.Openjdk9.Javacflags = module.Library.Module.properties.Openjdk9.Javacflags - props.Java_version = module.Library.Module.properties.Java_version - if module.Library.Module.deviceProperties.Compile_dex != nil { - props.Compile_dex = module.Library.Module.deviceProperties.Compile_dex + props.Openjdk9.Srcs = module.properties.Openjdk9.Srcs + props.Openjdk9.Javacflags = module.properties.Openjdk9.Javacflags + // We compile the stubs for 1.8 in line with the main android.jar stubs, and potential + // interop with older developer tools that don't support 1.9. + props.Java_version = proptools.StringPtr("1.8") + if module.deviceProperties.Compile_dex != nil { + props.Compile_dex = module.deviceProperties.Compile_dex } - if module.SocSpecific() { - props.Soc_specific = proptools.BoolPtr(true) - } else if module.DeviceSpecific() { - props.Device_specific = proptools.BoolPtr(true) - } else if module.ProductSpecific() { - props.Product_specific = proptools.BoolPtr(true) + // Dist the class jar artifact for sdk builds. + if !Bool(module.sdkLibraryProperties.No_dist) { + props.Dist.Targets = []string{"sdk", "win_sdk"} + props.Dist.Dest = proptools.StringPtr(fmt.Sprintf("%v.jar", module.BaseModuleName())) + props.Dist.Dir = proptools.StringPtr(module.apiDistPath(apiScope)) + props.Dist.Tag = proptools.StringPtr(".jar") } - mctx.CreateModule(android.ModuleFactoryAdaptor(LibraryFactory), &props) + mctx.CreateModule(LibraryFactory, &props, module.sdkComponentPropertiesForChildLibrary()) } -// Creates a droiddoc module that creates stubs source files from the given full source -// files -func (module *SdkLibrary) createDocs(mctx android.TopDownMutatorContext, apiScope apiScope) { +// Creates a droidstubs module that creates stubs source files from the given full source +// files and also updates and checks the API specification files. +func (module *SdkLibrary) createStubsSourcesAndApi(mctx android.DefaultableHookContext, apiScope *apiScope, name string, createStubSources, createApi bool, scopeSpecificDroidstubsArgs []string) { props := struct { Name *string + Visibility []string Srcs []string Installable *bool - Srcs_lib *string - Srcs_lib_whitelist_dirs []string - Srcs_lib_whitelist_pkgs []string + Sdk_version *string + System_modules *string Libs []string Arg_files []string Args *string - Api_tag_name *string - Api_filename *string - Removed_api_filename *string - No_standard_libs *bool Java_version *string + Annotations_enabled *bool Merge_annotations_dirs []string Merge_inclusion_annotations_dirs []string + Generate_stubs *bool Check_api struct { Current ApiToCheck Last_released ApiToCheck Ignore_missing_latest_api *bool + + Api_lint struct { + Enabled *bool + New_since *string + Baseline_file *string + } } Aidl struct { Include_dirs []string Local_include_dirs []string } + Dist struct { + Targets []string + Dest *string + Dir *string + } }{} - props.Name = proptools.StringPtr(module.docsName(apiScope)) - props.Srcs = append(props.Srcs, module.Library.Module.properties.Srcs...) - props.Srcs = append(props.Srcs, module.sdkLibraryProperties.Api_srcs...) + // The stubs source processing uses the same compile time classpath when extracting the + // API from the implementation library as it does when compiling it. i.e. the same + // * sdk version + // * system_modules + // * libs (static_libs/libs) + + props.Name = proptools.StringPtr(name) + + // If stubs_source_visibility is not set then the created module will use the + // visibility of this module. + visibility := module.sdkLibraryProperties.Stubs_source_visibility + props.Visibility = visibility + + props.Srcs = append(props.Srcs, module.properties.Srcs...) + props.Sdk_version = module.deviceProperties.Sdk_version + props.System_modules = module.deviceProperties.System_modules props.Installable = proptools.BoolPtr(false) // A droiddoc module has only one Libs property and doesn't distinguish between // shared libs and static libs. So we need to add both of these libs to Libs property. - props.Libs = module.Library.Module.properties.Libs - props.Libs = append(props.Libs, module.Library.Module.properties.Static_libs...) - props.Aidl.Include_dirs = module.Library.Module.deviceProperties.Aidl.Include_dirs - props.Aidl.Local_include_dirs = module.Library.Module.deviceProperties.Aidl.Local_include_dirs - props.No_standard_libs = module.Library.Module.properties.No_standard_libs - props.Java_version = module.Library.Module.properties.Java_version + props.Libs = module.properties.Libs + props.Libs = append(props.Libs, module.properties.Static_libs...) + props.Aidl.Include_dirs = module.deviceProperties.Aidl.Include_dirs + props.Aidl.Local_include_dirs = module.deviceProperties.Aidl.Local_include_dirs + props.Java_version = module.properties.Java_version + props.Annotations_enabled = module.sdkLibraryProperties.Annotations_enabled props.Merge_annotations_dirs = module.sdkLibraryProperties.Merge_annotations_dirs props.Merge_inclusion_annotations_dirs = module.sdkLibraryProperties.Merge_inclusion_annotations_dirs - droiddocArgs := " --stub-packages " + strings.Join(module.sdkLibraryProperties.Api_packages, ":") + - " " + android.JoinWithPrefix(module.sdkLibraryProperties.Hidden_api_packages, " --hide-package ") + - " " + android.JoinWithPrefix(module.sdkLibraryProperties.Droiddoc_options, " ") + - " --hide MissingPermission --hide BroadcastBehavior " + - "--hide HiddenSuperclass --hide DeprecationMismatch --hide UnavailableSymbol " + - "--hide SdkConstant --hide HiddenTypeParameter --hide Todo --hide Typo" + droidstubsArgs := []string{} + if len(module.sdkLibraryProperties.Api_packages) != 0 { + droidstubsArgs = append(droidstubsArgs, "--stub-packages "+strings.Join(module.sdkLibraryProperties.Api_packages, ":")) + } + if len(module.sdkLibraryProperties.Hidden_api_packages) != 0 { + droidstubsArgs = append(droidstubsArgs, + android.JoinWithPrefix(module.sdkLibraryProperties.Hidden_api_packages, " --hide-package ")) + } + droidstubsArgs = append(droidstubsArgs, module.sdkLibraryProperties.Droiddoc_options...) + disabledWarnings := []string{ + "MissingPermission", + "BroadcastBehavior", + "HiddenSuperclass", + "DeprecationMismatch", + "UnavailableSymbol", + "SdkConstant", + "HiddenTypeParameter", + "Todo", + "Typo", + } + droidstubsArgs = append(droidstubsArgs, android.JoinWithPrefix(disabledWarnings, "--hide ")) - switch apiScope { - case apiScopeSystem: - droiddocArgs = droiddocArgs + " -showAnnotation android.annotation.SystemApi" - case apiScopeTest: - droiddocArgs = droiddocArgs + " -showAnnotation android.annotation.TestApi" + if !createStubSources { + // Stubs are not required. + props.Generate_stubs = proptools.BoolPtr(false) } + + // Add in scope specific arguments. + droidstubsArgs = append(droidstubsArgs, scopeSpecificDroidstubsArgs...) props.Arg_files = module.sdkLibraryProperties.Droiddoc_option_files - props.Args = proptools.StringPtr(droiddocArgs) - - // List of APIs identified from the provided source files are created. They are later - // compared against to the not-yet-released (a.k.a current) list of APIs and to the - // last-released (a.k.a numbered) list of API. - currentApiFileName := "current.txt" - removedApiFileName := "removed.txt" - switch apiScope { - case apiScopeSystem: - currentApiFileName = "system-" + currentApiFileName - removedApiFileName = "system-" + removedApiFileName - case apiScopeTest: - currentApiFileName = "test-" + currentApiFileName - removedApiFileName = "test-" + removedApiFileName - } - currentApiFileName = path.Join("api", currentApiFileName) - removedApiFileName = path.Join("api", removedApiFileName) - // TODO(jiyong): remove these three props - props.Api_tag_name = proptools.StringPtr(module.apiTagName(apiScope)) - props.Api_filename = proptools.StringPtr(currentApiFileName) - props.Removed_api_filename = proptools.StringPtr(removedApiFileName) - - // check against the not-yet-release API - props.Check_api.Current.Api_file = proptools.StringPtr(currentApiFileName) - props.Check_api.Current.Removed_api_file = proptools.StringPtr(removedApiFileName) - - // check against the latest released API - props.Check_api.Last_released.Api_file = proptools.StringPtr( - module.latestApiFilegroupName(apiScope)) - props.Check_api.Last_released.Removed_api_file = proptools.StringPtr( - module.latestRemovedApiFilegroupName(apiScope)) - props.Check_api.Ignore_missing_latest_api = proptools.BoolPtr(true) - props.Srcs_lib = module.sdkLibraryProperties.Srcs_lib - props.Srcs_lib_whitelist_dirs = module.sdkLibraryProperties.Srcs_lib_whitelist_dirs - props.Srcs_lib_whitelist_pkgs = module.sdkLibraryProperties.Srcs_lib_whitelist_pkgs - - mctx.CreateModule(android.ModuleFactoryAdaptor(DroidstubsFactory), &props) + props.Args = proptools.StringPtr(strings.Join(droidstubsArgs, " ")) + + if createApi { + // List of APIs identified from the provided source files are created. They are later + // compared against to the not-yet-released (a.k.a current) list of APIs and to the + // last-released (a.k.a numbered) list of API. + currentApiFileName := apiScope.apiFilePrefix + "current.txt" + removedApiFileName := apiScope.apiFilePrefix + "removed.txt" + apiDir := module.getApiDir() + currentApiFileName = path.Join(apiDir, currentApiFileName) + removedApiFileName = path.Join(apiDir, removedApiFileName) + + // check against the not-yet-release API + props.Check_api.Current.Api_file = proptools.StringPtr(currentApiFileName) + props.Check_api.Current.Removed_api_file = proptools.StringPtr(removedApiFileName) + + if !apiScope.unstable { + // check against the latest released API + latestApiFilegroupName := proptools.StringPtr(module.latestApiFilegroupName(apiScope)) + props.Check_api.Last_released.Api_file = latestApiFilegroupName + props.Check_api.Last_released.Removed_api_file = proptools.StringPtr( + module.latestRemovedApiFilegroupName(apiScope)) + props.Check_api.Ignore_missing_latest_api = proptools.BoolPtr(true) + + if proptools.Bool(module.sdkLibraryProperties.Api_lint.Enabled) { + // Enable api lint. + props.Check_api.Api_lint.Enabled = proptools.BoolPtr(true) + props.Check_api.Api_lint.New_since = latestApiFilegroupName + + // If it exists then pass a lint-baseline.txt through to droidstubs. + baselinePath := path.Join(apiDir, apiScope.apiFilePrefix+"lint-baseline.txt") + baselinePathRelativeToRoot := path.Join(mctx.ModuleDir(), baselinePath) + paths, err := mctx.GlobWithDeps(baselinePathRelativeToRoot, nil) + if err != nil { + mctx.ModuleErrorf("error checking for presence of %s: %s", baselinePathRelativeToRoot, err) + } + if len(paths) == 1 { + props.Check_api.Api_lint.Baseline_file = proptools.StringPtr(baselinePath) + } else if len(paths) != 0 { + mctx.ModuleErrorf("error checking for presence of %s: expected one path, found: %v", baselinePathRelativeToRoot, paths) + } + } + } + + // Dist the api txt artifact for sdk builds. + if !Bool(module.sdkLibraryProperties.No_dist) { + props.Dist.Targets = []string{"sdk", "win_sdk"} + props.Dist.Dest = proptools.StringPtr(fmt.Sprintf("%v.txt", module.BaseModuleName())) + props.Dist.Dir = proptools.StringPtr(path.Join(module.apiDistPath(apiScope), "api")) + } + } + + mctx.CreateModule(DroidstubsFactory, &props) +} + +func (module *SdkLibrary) DepIsInSameApex(mctx android.BaseModuleContext, dep android.Module) bool { + depTag := mctx.OtherModuleDependencyTag(dep) + if depTag == xmlPermissionsFileTag { + return true + } + return module.Library.DepIsInSameApex(mctx, dep) } // Creates the xml file that publicizes the runtime library -func (module *SdkLibrary) createXmlFile(mctx android.TopDownMutatorContext) { - template := ` -<?xml version="1.0" encoding="utf-8"?> -<!-- Copyright (C) 2018 The Android Open Source Project - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> - -<permissions> - <library name="%s" file="%s"/> -</permissions> -` - // genrule to generate the xml file content from the template above - // TODO: preserve newlines in the generate xml file. Newlines are being squashed - // in the ninja file. Do we need to have an external tool for this? - xmlContent := fmt.Sprintf(template, module.BaseModuleName(), module.implPath()) - genruleProps := struct { - Name *string - Cmd *string - Out []string - }{} - genruleProps.Name = proptools.StringPtr(module.xmlFileName() + "-gen") - genruleProps.Cmd = proptools.StringPtr("echo '" + xmlContent + "' > $(out)") - genruleProps.Out = []string{module.xmlFileName()} - mctx.CreateModule(android.ModuleFactoryAdaptor(genrule.GenRuleFactory), &genruleProps) - - // creates a prebuilt_etc module to actually place the xml file under - // <partition>/etc/permissions - etcProps := struct { - Name *string - Src *string - Sub_dir *string - Soc_specific *bool - Device_specific *bool - Product_specific *bool - }{} - etcProps.Name = proptools.StringPtr(module.xmlFileName()) - etcProps.Src = proptools.StringPtr(":" + module.xmlFileName() + "-gen") - etcProps.Sub_dir = proptools.StringPtr("permissions") - if module.SocSpecific() { - etcProps.Soc_specific = proptools.BoolPtr(true) - } else if module.DeviceSpecific() { - etcProps.Device_specific = proptools.BoolPtr(true) - } else if module.ProductSpecific() { - etcProps.Product_specific = proptools.BoolPtr(true) +func (module *SdkLibrary) createXmlFile(mctx android.DefaultableHookContext) { + props := struct { + Name *string + Lib_name *string + Apex_available []string + }{ + Name: proptools.StringPtr(module.xmlPermissionsModuleName()), + Lib_name: proptools.StringPtr(module.BaseModuleName()), + Apex_available: module.ApexProperties.Apex_available, } - mctx.CreateModule(android.ModuleFactoryAdaptor(android.PrebuiltEtcFactory), &etcProps) + + mctx.CreateModule(sdkLibraryXmlFactory, &props) } -func (module *SdkLibrary) PrebuiltJars(ctx android.BaseContext, sdkVersion string) android.Paths { - var api, v string - if sdkVersion == "" { - api = "system" - v = "current" - } else if strings.Contains(sdkVersion, "_") { - t := strings.Split(sdkVersion, "_") - api = t[0] - v = t[1] +func PrebuiltJars(ctx android.BaseModuleContext, baseName string, s sdkSpec) android.Paths { + var ver sdkVersion + var kind sdkKind + if s.usePrebuilt(ctx) { + ver = s.version + kind = s.kind } else { - api = "public" - v = sdkVersion + // We don't have prebuilt SDK for the specific sdkVersion. + // Instead of breaking the build, fallback to use "system_current" + ver = sdkVersionCurrent + kind = sdkSystem } - dir := filepath.Join("prebuilts", "sdk", v, api) - jar := filepath.Join(dir, module.BaseModuleName()+".jar") + + dir := filepath.Join("prebuilts", "sdk", ver.String(), kind.String()) + jar := filepath.Join(dir, baseName+".jar") jarPath := android.ExistentPathForSource(ctx, jar) if !jarPath.Valid() { - ctx.PropertyErrorf("sdk_library", "invalid sdk version %q, %q does not exist", v, jar) + if ctx.Config().AllowMissingDependencies() { + return android.Paths{android.PathForSource(ctx, jar)} + } else { + ctx.PropertyErrorf("sdk_library", "invalid sdk version %q, %q does not exist", s.raw, jar) + } return nil } return android.Paths{jarPath.Path()} } -// to satisfy SdkLibraryDependency interface -func (module *SdkLibrary) SdkHeaderJars(ctx android.BaseContext, sdkVersion string) android.Paths { - // This module is just a wrapper for the stubs. - if ctx.Config().UnbundledBuildUsePrebuiltSdks() { - return module.PrebuiltJars(ctx, sdkVersion) - } else { - if strings.HasPrefix(sdkVersion, "system_") { - return module.systemApiStubsPath - } else if sdkVersion == "" { - return module.Library.HeaderJars() - } else { - return module.publicApiStubsPath - } +// Get the apex name for module, "" if it is for platform. +func getApexNameForModule(module android.Module) string { + if apex, ok := module.(android.ApexModule); ok { + return apex.ApexName() } + + return "" } -// to satisfy SdkLibraryDependency interface -func (module *SdkLibrary) SdkImplementationJars(ctx android.BaseContext, sdkVersion string) android.Paths { - // This module is just a wrapper for the stubs. - if ctx.Config().UnbundledBuildUsePrebuiltSdks() { - return module.PrebuiltJars(ctx, sdkVersion) - } else { - if strings.HasPrefix(sdkVersion, "system_") { - return module.systemApiStubsImplPath - } else if sdkVersion == "" { - return module.Library.ImplementationJars() - } else { - return module.publicApiStubsImplPath +// Check to see if the other module is within the same named APEX as this module. +// +// If either this or the other module are on the platform then this will return +// false. +func withinSameApexAs(module android.ApexModule, other android.Module) bool { + name := module.ApexName() + return name != "" && getApexNameForModule(other) == name +} + +func (module *SdkLibrary) sdkJars(ctx android.BaseModuleContext, sdkVersion sdkSpec, headerJars bool) android.Paths { + // If the client doesn't set sdk_version, but if this library prefers stubs over + // the impl library, let's provide the widest API surface possible. To do so, + // force override sdk_version to module_current so that the closest possible API + // surface could be found in selectHeaderJarsForSdkVersion + if module.defaultsToStubs() && !sdkVersion.specified() { + sdkVersion = sdkSpecFrom("module_current") + } + + // Only provide access to the implementation library if it is actually built. + if module.requiresRuntimeImplementationLibrary() { + // Check any special cases for java_sdk_library. + // + // Only allow access to the implementation library in the following condition: + // * No sdk_version specified on the referencing module. + // * The referencing module is in the same apex as this. + if sdkVersion.kind == sdkPrivate || withinSameApexAs(module, ctx.Module()) { + if headerJars { + return module.HeaderJars() + } else { + return module.ImplementationJars() + } } } + + return module.selectHeaderJarsForSdkVersion(ctx, sdkVersion) +} + +// to satisfy SdkLibraryDependency interface +func (module *SdkLibrary) SdkHeaderJars(ctx android.BaseModuleContext, sdkVersion sdkSpec) android.Paths { + return module.sdkJars(ctx, sdkVersion, true /*headerJars*/) +} + +// to satisfy SdkLibraryDependency interface +func (module *SdkLibrary) SdkImplementationJars(ctx android.BaseModuleContext, sdkVersion sdkSpec) android.Paths { + return module.sdkJars(ctx, sdkVersion, false /*headerJars*/) } func (module *SdkLibrary) SetNoDist() { @@ -670,31 +1461,40 @@ func javaSdkLibraries(config android.Config) *[]string { }).(*[]string) } +func (module *SdkLibrary) getApiDir() string { + return proptools.StringDefault(module.sdkLibraryProperties.Api_dir, "api") +} + // For a java_sdk_library module, create internal modules for stubs, docs, // runtime libs and xml file. If requested, the stubs and docs are created twice // once for public API level and once for system API level -func SdkLibraryMutator(mctx android.TopDownMutatorContext) { - if module, ok := mctx.Module().(*SdkLibrary); ok { - module.createInternalModules(mctx) - } else if module, ok := mctx.Module().(syspropLibraryInterface); ok { - module.SyspropJavaModule().createInternalModules(mctx) +func (module *SdkLibrary) CreateInternalModules(mctx android.DefaultableHookContext) { + // If the module has been disabled then don't create any child modules. + if !module.Enabled() { + return } -} -func (module *SdkLibrary) createInternalModules(mctx android.TopDownMutatorContext) { - if len(module.Library.Module.properties.Srcs) == 0 { + if len(module.properties.Srcs) == 0 { mctx.PropertyErrorf("srcs", "java_sdk_library must specify srcs") + return } - if len(module.sdkLibraryProperties.Api_packages) == 0 { - mctx.PropertyErrorf("api_packages", "java_sdk_library must specify api_packages") - } + // If this builds against standard libraries (i.e. is not part of the core libraries) + // then assume it provides both system and test apis. Otherwise, assume it does not and + // also assume it does not contribute to the dist build. + sdkDep := decodeSdkDep(mctx, sdkContext(&module.Library)) + hasSystemAndTestApis := sdkDep.hasStandardLibs() + module.sdkLibraryProperties.Generate_system_and_test_apis = hasSystemAndTestApis + module.sdkLibraryProperties.No_dist = proptools.BoolPtr(!hasSystemAndTestApis) missing_current_api := false - for _, scope := range []string{"", "system-", "test-"} { + generatedScopes := module.getGeneratedApiScopes(mctx) + + apiDir := module.getApiDir() + for _, scope := range generatedScopes { for _, api := range []string{"current.txt", "removed.txt"} { - path := path.Join(mctx.ModuleDir(), "api", scope+api) + path := path.Join(mctx.ModuleDir(), apiDir, scope.apiFilePrefix+api) p := android.ExistentPathForSource(mctx, path) if !p.Valid() { mctx.ModuleErrorf("Current api file %#v doesn't exist", path) @@ -713,50 +1513,772 @@ func (module *SdkLibrary) createInternalModules(mctx android.TopDownMutatorConte mctx.ModuleErrorf("One or more current api files are missing. "+ "You can update them by:\n"+ - "%s %q && m update-api", script, mctx.ModuleDir()) + "%s %q %s && m update-api", + script, filepath.Join(mctx.ModuleDir(), apiDir), + strings.Join(generatedScopes.Strings(func(s *apiScope) string { return s.apiFilePrefix }), " ")) return } - // for public API stubs - module.createStubsLibrary(mctx, apiScopePublic) - module.createDocs(mctx, apiScopePublic) + for _, scope := range generatedScopes { + stubsSourceArgs := scope.droidstubsArgsForGeneratingStubsSource + stubsSourceModuleName := module.stubsSourceModuleName(scope) - if !Bool(module.properties.No_standard_libs) { - // for system API stubs - module.createStubsLibrary(mctx, apiScopeSystem) - module.createDocs(mctx, apiScopeSystem) + // If the args needed to generate the stubs and API are the same then they + // can be generated in a single invocation of metalava, otherwise they will + // need separate invocations. + if scope.createStubsSourceAndApiTogether { + // Use the stubs source name for legacy reasons. + module.createStubsSourcesAndApi(mctx, scope, stubsSourceModuleName, true, true, stubsSourceArgs) + } else { + module.createStubsSourcesAndApi(mctx, scope, stubsSourceModuleName, true, false, stubsSourceArgs) - // for test API stubs - module.createStubsLibrary(mctx, apiScopeTest) - module.createDocs(mctx, apiScopeTest) + apiArgs := scope.droidstubsArgsForGeneratingApi + apiName := module.apiModuleName(scope) + module.createStubsSourcesAndApi(mctx, scope, apiName, false, true, apiArgs) + } - // for runtime - module.createXmlFile(mctx) + module.createStubsLibrary(mctx, scope) } - // record java_sdk_library modules so that they are exported to make - javaSdkLibraries := javaSdkLibraries(mctx.Config()) - javaSdkLibrariesLock.Lock() - defer javaSdkLibrariesLock.Unlock() - *javaSdkLibraries = append(*javaSdkLibraries, module.BaseModuleName()) + if module.requiresRuntimeImplementationLibrary() { + // Create child module to create an implementation library. + // + // This temporarily creates a second implementation library that can be explicitly + // referenced. + // + // TODO(b/156618935) - update comment once only one implementation library is created. + module.createImplLibrary(mctx) + + // Only create an XML permissions file that declares the library as being usable + // as a shared library if required. + if module.sharedLibrary() { + module.createXmlFile(mctx) + } + + // record java_sdk_library modules so that they are exported to make + javaSdkLibraries := javaSdkLibraries(mctx.Config()) + javaSdkLibrariesLock.Lock() + defer javaSdkLibrariesLock.Unlock() + *javaSdkLibraries = append(*javaSdkLibraries, module.BaseModuleName()) + } } func (module *SdkLibrary) InitSdkLibraryProperties() { - module.AddProperties( - &module.sdkLibraryProperties, - &module.Library.Module.properties, - &module.Library.Module.dexpreoptProperties, - &module.Library.Module.deviceProperties, - &module.Library.Module.protoProperties, - ) + module.addHostAndDeviceProperties() + module.AddProperties(&module.sdkLibraryProperties) + + module.initSdkLibraryComponent(&module.ModuleBase) + + module.properties.Installable = proptools.BoolPtr(true) + module.deviceProperties.IsSDKLibrary = true +} + +func (module *SdkLibrary) requiresRuntimeImplementationLibrary() bool { + return !proptools.Bool(module.sdkLibraryProperties.Api_only) +} + +func (module *SdkLibrary) defaultsToStubs() bool { + return proptools.Bool(module.sdkLibraryProperties.Default_to_stubs) +} + +// Defines how to name the individual component modules the sdk library creates. +type sdkLibraryComponentNamingScheme interface { + stubsLibraryModuleName(scope *apiScope, baseName string) string + + stubsSourceModuleName(scope *apiScope, baseName string) string + + apiModuleName(scope *apiScope, baseName string) string +} + +type defaultNamingScheme struct { +} + +func (s *defaultNamingScheme) stubsLibraryModuleName(scope *apiScope, baseName string) string { + return scope.stubsLibraryModuleName(baseName) +} + +func (s *defaultNamingScheme) stubsSourceModuleName(scope *apiScope, baseName string) string { + return scope.stubsSourceModuleName(baseName) +} + +func (s *defaultNamingScheme) apiModuleName(scope *apiScope, baseName string) string { + return scope.apiModuleName(baseName) +} + +var _ sdkLibraryComponentNamingScheme = (*defaultNamingScheme)(nil) - module.Library.Module.properties.Installable = proptools.BoolPtr(true) - module.Library.Module.deviceProperties.IsSDKLibrary = true +type frameworkModulesNamingScheme struct { } +func (s *frameworkModulesNamingScheme) moduleSuffix(scope *apiScope) string { + suffix := scope.name + if scope == apiScopeModuleLib { + suffix = "module_libs_" + } + return suffix +} + +func (s *frameworkModulesNamingScheme) stubsLibraryModuleName(scope *apiScope, baseName string) string { + return fmt.Sprintf("%s-stubs-%sapi", baseName, s.moduleSuffix(scope)) +} + +func (s *frameworkModulesNamingScheme) stubsSourceModuleName(scope *apiScope, baseName string) string { + return fmt.Sprintf("%s-stubs-srcs-%sapi", baseName, s.moduleSuffix(scope)) +} + +func (s *frameworkModulesNamingScheme) apiModuleName(scope *apiScope, baseName string) string { + return fmt.Sprintf("%s-api-%sapi", baseName, s.moduleSuffix(scope)) +} + +var _ sdkLibraryComponentNamingScheme = (*frameworkModulesNamingScheme)(nil) + +func moduleStubLinkType(name string) (stub bool, ret linkType) { + // This suffix-based approach is fragile and could potentially mis-trigger. + // TODO(b/155164730): Clean this up when modules no longer reference sdk_lib stubs directly. + if strings.HasSuffix(name, ".stubs.public") || strings.HasSuffix(name, "-stubs-publicapi") { + return true, javaSdk + } + if strings.HasSuffix(name, ".stubs.system") || strings.HasSuffix(name, "-stubs-systemapi") { + return true, javaSystem + } + if strings.HasSuffix(name, ".stubs.module_lib") || strings.HasSuffix(name, "-stubs-module_libs_api") { + return true, javaModule + } + if strings.HasSuffix(name, ".stubs.test") { + return true, javaSystem + } + return false, javaPlatform +} + +// java_sdk_library is a special Java library that provides optional platform APIs to apps. +// In practice, it can be viewed as a combination of several modules: 1) stubs library that clients +// are linked against to, 2) droiddoc module that internally generates API stubs source files, +// 3) the real runtime shared library that implements the APIs, and 4) XML file for adding +// the runtime lib to the classpath at runtime if requested via <uses-library>. func SdkLibraryFactory() android.Module { module := &SdkLibrary{} + + // Initialize information common between source and prebuilt. + module.initCommon(&module.ModuleBase) + module.InitSdkLibraryProperties() + android.InitApexModule(module) InitJavaModule(module, android.HostAndDeviceSupported) + + // Initialize the map from scope to scope specific properties. + scopeToProperties := make(map[*apiScope]*ApiScopeProperties) + for _, scope := range allApiScopes { + scopeToProperties[scope] = scope.scopeSpecificProperties(module) + } + module.scopeToProperties = scopeToProperties + + // Add the properties containing visibility rules so that they are checked. + android.AddVisibilityProperty(module, "impl_library_visibility", &module.sdkLibraryProperties.Impl_library_visibility) + android.AddVisibilityProperty(module, "stubs_library_visibility", &module.sdkLibraryProperties.Stubs_library_visibility) + android.AddVisibilityProperty(module, "stubs_source_visibility", &module.sdkLibraryProperties.Stubs_source_visibility) + + module.SetDefaultableHook(func(ctx android.DefaultableHookContext) { + // If no implementation is required then it cannot be used as a shared library + // either. + if !module.requiresRuntimeImplementationLibrary() { + // If shared_library has been explicitly set to true then it is incompatible + // with api_only: true. + if proptools.Bool(module.commonSdkLibraryProperties.Shared_library) { + ctx.PropertyErrorf("api_only/shared_library", "inconsistent settings, shared_library and api_only cannot both be true") + } + // Set shared_library: false. + module.commonSdkLibraryProperties.Shared_library = proptools.BoolPtr(false) + } + + if module.initCommonAfterDefaultsApplied(ctx) { + module.CreateInternalModules(ctx) + } + }) + return module +} + +// +// SDK library prebuilts +// + +// Properties associated with each api scope. +type sdkLibraryScopeProperties struct { + Jars []string `android:"path"` + + Sdk_version *string + + // List of shared java libs that this module has dependencies to + Libs []string + + // The stubs source. + Stub_srcs []string `android:"path"` + + // The current.txt + Current_api *string `android:"path"` + + // The removed.txt + Removed_api *string `android:"path"` +} + +type sdkLibraryImportProperties struct { + // List of shared java libs, common to all scopes, that this module has + // dependencies to + Libs []string +} + +type SdkLibraryImport struct { + android.ModuleBase + android.DefaultableModuleBase + prebuilt android.Prebuilt + android.ApexModuleBase + android.SdkBase + + properties sdkLibraryImportProperties + + // Map from api scope to the scope specific property structure. + scopeProperties map[*apiScope]*sdkLibraryScopeProperties + + commonToSdkLibraryAndImport + + // The reference to the implementation library created by the source module. + // Is nil if the source module does not exist. + implLibraryModule *Library + + // The reference to the xml permissions module created by the source module. + // Is nil if the source module does not exist. + xmlPermissionsFileModule *sdkLibraryXml +} + +var _ SdkLibraryDependency = (*SdkLibraryImport)(nil) + +// The type of a structure that contains a field of type sdkLibraryScopeProperties +// for each apiscope in allApiScopes, e.g. something like: +// struct { +// Public sdkLibraryScopeProperties +// System sdkLibraryScopeProperties +// ... +// } +var allScopeStructType = createAllScopePropertiesStructType() + +// Dynamically create a structure type for each apiscope in allApiScopes. +func createAllScopePropertiesStructType() reflect.Type { + var fields []reflect.StructField + for _, apiScope := range allApiScopes { + field := reflect.StructField{ + Name: apiScope.fieldName, + Type: reflect.TypeOf(sdkLibraryScopeProperties{}), + } + fields = append(fields, field) + } + + return reflect.StructOf(fields) +} + +// Create an instance of the scope specific structure type and return a map +// from apiscope to a pointer to each scope specific field. +func createPropertiesInstance() (interface{}, map[*apiScope]*sdkLibraryScopeProperties) { + allScopePropertiesPtr := reflect.New(allScopeStructType) + allScopePropertiesStruct := allScopePropertiesPtr.Elem() + scopeProperties := make(map[*apiScope]*sdkLibraryScopeProperties) + + for _, apiScope := range allApiScopes { + field := allScopePropertiesStruct.FieldByName(apiScope.fieldName) + scopeProperties[apiScope] = field.Addr().Interface().(*sdkLibraryScopeProperties) + } + + return allScopePropertiesPtr.Interface(), scopeProperties +} + +// java_sdk_library_import imports a prebuilt java_sdk_library. +func sdkLibraryImportFactory() android.Module { + module := &SdkLibraryImport{} + + allScopeProperties, scopeToProperties := createPropertiesInstance() + module.scopeProperties = scopeToProperties + module.AddProperties(&module.properties, allScopeProperties) + + // Initialize information common between source and prebuilt. + module.initCommon(&module.ModuleBase) + + android.InitPrebuiltModule(module, &[]string{""}) + android.InitApexModule(module) + android.InitSdkAwareModule(module) + InitJavaModule(module, android.HostAndDeviceSupported) + + module.SetDefaultableHook(func(mctx android.DefaultableHookContext) { + if module.initCommonAfterDefaultsApplied(mctx) { + module.createInternalModules(mctx) + } + }) + return module +} + +func (module *SdkLibraryImport) Prebuilt() *android.Prebuilt { + return &module.prebuilt +} + +func (module *SdkLibraryImport) Name() string { + return module.prebuilt.Name(module.ModuleBase.Name()) +} + +func (module *SdkLibraryImport) createInternalModules(mctx android.DefaultableHookContext) { + + // If the build is configured to use prebuilts then force this to be preferred. + if mctx.Config().UnbundledBuildUsePrebuiltSdks() { + module.prebuilt.ForcePrefer() + } + + for apiScope, scopeProperties := range module.scopeProperties { + if len(scopeProperties.Jars) == 0 { + continue + } + + module.createJavaImportForStubs(mctx, apiScope, scopeProperties) + + if len(scopeProperties.Stub_srcs) > 0 { + module.createPrebuiltStubsSources(mctx, apiScope, scopeProperties) + } + } + + javaSdkLibraries := javaSdkLibraries(mctx.Config()) + javaSdkLibrariesLock.Lock() + defer javaSdkLibrariesLock.Unlock() + *javaSdkLibraries = append(*javaSdkLibraries, module.BaseModuleName()) +} + +func (module *SdkLibraryImport) createJavaImportForStubs(mctx android.DefaultableHookContext, apiScope *apiScope, scopeProperties *sdkLibraryScopeProperties) { + // Creates a java import for the jar with ".stubs" suffix + props := struct { + Name *string + Sdk_version *string + Libs []string + Jars []string + Prefer *bool + }{} + props.Name = proptools.StringPtr(module.stubsLibraryModuleName(apiScope)) + props.Sdk_version = scopeProperties.Sdk_version + // Prepend any of the libs from the legacy public properties to the libs for each of the + // scopes to avoid having to duplicate them in each scope. + props.Libs = append(module.properties.Libs, scopeProperties.Libs...) + props.Jars = scopeProperties.Jars + + // The imports are preferred if the java_sdk_library_import is preferred. + props.Prefer = proptools.BoolPtr(module.prebuilt.Prefer()) + + mctx.CreateModule(ImportFactory, &props, module.sdkComponentPropertiesForChildLibrary()) +} + +func (module *SdkLibraryImport) createPrebuiltStubsSources(mctx android.DefaultableHookContext, apiScope *apiScope, scopeProperties *sdkLibraryScopeProperties) { + props := struct { + Name *string + Srcs []string + Prefer *bool + }{} + props.Name = proptools.StringPtr(module.stubsSourceModuleName(apiScope)) + props.Srcs = scopeProperties.Stub_srcs + mctx.CreateModule(PrebuiltStubsSourcesFactory, &props) + + // The stubs source is preferred if the java_sdk_library_import is preferred. + props.Prefer = proptools.BoolPtr(module.prebuilt.Prefer()) +} + +func (module *SdkLibraryImport) DepsMutator(ctx android.BottomUpMutatorContext) { + for apiScope, scopeProperties := range module.scopeProperties { + if len(scopeProperties.Jars) == 0 { + continue + } + + // Add dependencies to the prebuilt stubs library + ctx.AddVariationDependencies(nil, apiScope.stubsTag, module.stubsLibraryModuleName(apiScope)) + + if len(scopeProperties.Stub_srcs) > 0 { + // Add dependencies to the prebuilt stubs source library + ctx.AddVariationDependencies(nil, apiScope.stubsSourceTag, module.stubsSourceModuleName(apiScope)) + } + } + + implName := module.implLibraryModuleName() + if ctx.OtherModuleExists(implName) { + ctx.AddVariationDependencies(nil, implLibraryTag, implName) + + xmlPermissionsModuleName := module.xmlPermissionsModuleName() + if module.sharedLibrary() && ctx.OtherModuleExists(xmlPermissionsModuleName) { + // Add dependency to the rule for generating the xml permissions file + ctx.AddDependency(module, xmlPermissionsFileTag, xmlPermissionsModuleName) + } + } +} + +func (module *SdkLibraryImport) DepIsInSameApex(mctx android.BaseModuleContext, dep android.Module) bool { + depTag := mctx.OtherModuleDependencyTag(dep) + if depTag == xmlPermissionsFileTag { + return true + } + + // None of the other dependencies of the java_sdk_library_import are in the same apex + // as the one that references this module. + return false +} + +func (module *SdkLibraryImport) OutputFiles(tag string) (android.Paths, error) { + return module.commonOutputFiles(tag) +} + +func (module *SdkLibraryImport) GenerateAndroidBuildActions(ctx android.ModuleContext) { + // Record the paths to the prebuilt stubs library and stubs source. + ctx.VisitDirectDeps(func(to android.Module) { + tag := ctx.OtherModuleDependencyTag(to) + + // Extract information from any of the scope specific dependencies. + if scopeTag, ok := tag.(scopeDependencyTag); ok { + apiScope := scopeTag.apiScope + scopePaths := module.getScopePathsCreateIfNeeded(apiScope) + + // Extract information from the dependency. The exact information extracted + // is determined by the nature of the dependency which is determined by the tag. + scopeTag.extractDepInfo(ctx, to, scopePaths) + } else if tag == implLibraryTag { + if implLibrary, ok := to.(*Library); ok { + module.implLibraryModule = implLibrary + } else { + ctx.ModuleErrorf("implementation library must be of type *java.Library but was %T", to) + } + } else if tag == xmlPermissionsFileTag { + if xmlPermissionsFileModule, ok := to.(*sdkLibraryXml); ok { + module.xmlPermissionsFileModule = xmlPermissionsFileModule + } else { + ctx.ModuleErrorf("xml permissions file module must be of type *sdkLibraryXml but was %T", to) + } + } + }) + + // Populate the scope paths with information from the properties. + for apiScope, scopeProperties := range module.scopeProperties { + if len(scopeProperties.Jars) == 0 { + continue + } + + paths := module.getScopePathsCreateIfNeeded(apiScope) + paths.currentApiFilePath = android.OptionalPathForModuleSrc(ctx, scopeProperties.Current_api) + paths.removedApiFilePath = android.OptionalPathForModuleSrc(ctx, scopeProperties.Removed_api) + } +} + +func (module *SdkLibraryImport) sdkJars(ctx android.BaseModuleContext, sdkVersion sdkSpec, headerJars bool) android.Paths { + + // For consistency with SdkLibrary make the implementation jar available to libraries that + // are within the same APEX. + implLibraryModule := module.implLibraryModule + if implLibraryModule != nil && withinSameApexAs(module, ctx.Module()) { + if headerJars { + return implLibraryModule.HeaderJars() + } else { + return implLibraryModule.ImplementationJars() + } + } + + return module.selectHeaderJarsForSdkVersion(ctx, sdkVersion) +} + +// to satisfy SdkLibraryDependency interface +func (module *SdkLibraryImport) SdkHeaderJars(ctx android.BaseModuleContext, sdkVersion sdkSpec) android.Paths { + // This module is just a wrapper for the prebuilt stubs. + return module.sdkJars(ctx, sdkVersion, true) +} + +// to satisfy SdkLibraryDependency interface +func (module *SdkLibraryImport) SdkImplementationJars(ctx android.BaseModuleContext, sdkVersion sdkSpec) android.Paths { + // This module is just a wrapper for the stubs. + return module.sdkJars(ctx, sdkVersion, false) +} + +// to satisfy apex.javaDependency interface +func (module *SdkLibraryImport) DexJar() android.Path { + if module.implLibraryModule == nil { + return nil + } else { + return module.implLibraryModule.DexJar() + } +} + +// to satisfy apex.javaDependency interface +func (module *SdkLibraryImport) JacocoReportClassesFile() android.Path { + if module.implLibraryModule == nil { + return nil + } else { + return module.implLibraryModule.JacocoReportClassesFile() + } +} + +// to satisfy apex.javaDependency interface +func (module *SdkLibraryImport) LintDepSets() LintDepSets { + if module.implLibraryModule == nil { + return LintDepSets{} + } else { + return module.implLibraryModule.LintDepSets() + } +} + +// to satisfy apex.javaDependency interface +func (module *SdkLibraryImport) Stem() string { + return module.BaseModuleName() +} + +var _ ApexDependency = (*SdkLibraryImport)(nil) + +// to satisfy java.ApexDependency interface +func (module *SdkLibraryImport) HeaderJars() android.Paths { + if module.implLibraryModule == nil { + return nil + } else { + return module.implLibraryModule.HeaderJars() + } +} + +// to satisfy java.ApexDependency interface +func (module *SdkLibraryImport) ImplementationAndResourcesJars() android.Paths { + if module.implLibraryModule == nil { + return nil + } else { + return module.implLibraryModule.ImplementationAndResourcesJars() + } +} + +// +// java_sdk_library_xml +// +type sdkLibraryXml struct { + android.ModuleBase + android.DefaultableModuleBase + android.ApexModuleBase + + properties sdkLibraryXmlProperties + + outputFilePath android.OutputPath + installDirPath android.InstallPath +} + +type sdkLibraryXmlProperties struct { + // canonical name of the lib + Lib_name *string +} + +// java_sdk_library_xml builds the permission xml file for a java_sdk_library. +// Not to be used directly by users. java_sdk_library internally uses this. +func sdkLibraryXmlFactory() android.Module { + module := &sdkLibraryXml{} + + module.AddProperties(&module.properties) + + android.InitApexModule(module) + android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibCommon) + return module } + +// from android.PrebuiltEtcModule +func (module *sdkLibraryXml) SubDir() string { + return "permissions" +} + +// from android.PrebuiltEtcModule +func (module *sdkLibraryXml) OutputFile() android.OutputPath { + return module.outputFilePath +} + +// from android.ApexModule +func (module *sdkLibraryXml) AvailableFor(what string) bool { + return true +} + +func (module *sdkLibraryXml) DepsMutator(ctx android.BottomUpMutatorContext) { + // do nothing +} + +// File path to the runtime implementation library +func (module *sdkLibraryXml) implPath() string { + implName := proptools.String(module.properties.Lib_name) + if apexName := module.ApexName(); apexName != "" { + // TODO(b/146468504): ApexName() is only a soong module name, not apex name. + // In most cases, this works fine. But when apex_name is set or override_apex is used + // this can be wrong. + return fmt.Sprintf("/apex/%s/javalib/%s.jar", apexName, implName) + } + partition := "system" + if module.SocSpecific() { + partition = "vendor" + } else if module.DeviceSpecific() { + partition = "odm" + } else if module.ProductSpecific() { + partition = "product" + } else if module.SystemExtSpecific() { + partition = "system_ext" + } + return "/" + partition + "/framework/" + implName + ".jar" +} + +func (module *sdkLibraryXml) GenerateAndroidBuildActions(ctx android.ModuleContext) { + libName := proptools.String(module.properties.Lib_name) + xmlContent := fmt.Sprintf(permissionsTemplate, libName, module.implPath()) + + module.outputFilePath = android.PathForModuleOut(ctx, libName+".xml").OutputPath + rule := android.NewRuleBuilder() + rule.Command(). + Text("/bin/bash -c \"echo -e '" + xmlContent + "'\" > "). + Output(module.outputFilePath) + + rule.Build(pctx, ctx, "java_sdk_xml", "Permission XML") + + module.installDirPath = android.PathForModuleInstall(ctx, "etc", module.SubDir()) +} + +func (module *sdkLibraryXml) AndroidMkEntries() []android.AndroidMkEntries { + if !module.IsForPlatform() { + return []android.AndroidMkEntries{android.AndroidMkEntries{ + Disabled: true, + }} + } + + return []android.AndroidMkEntries{android.AndroidMkEntries{ + Class: "ETC", + OutputFile: android.OptionalPathForPath(module.outputFilePath), + ExtraEntries: []android.AndroidMkExtraEntriesFunc{ + func(entries *android.AndroidMkEntries) { + entries.SetString("LOCAL_MODULE_TAGS", "optional") + entries.SetString("LOCAL_MODULE_PATH", module.installDirPath.ToMakePath().String()) + entries.SetString("LOCAL_INSTALLED_MODULE_STEM", module.outputFilePath.Base()) + }, + }, + }} +} + +type sdkLibrarySdkMemberType struct { + android.SdkMemberTypeBase +} + +func (s *sdkLibrarySdkMemberType) AddDependencies(mctx android.BottomUpMutatorContext, dependencyTag blueprint.DependencyTag, names []string) { + mctx.AddVariationDependencies(nil, dependencyTag, names...) +} + +func (s *sdkLibrarySdkMemberType) IsInstance(module android.Module) bool { + _, ok := module.(*SdkLibrary) + return ok +} + +func (s *sdkLibrarySdkMemberType) AddPrebuiltModule(ctx android.SdkMemberContext, member android.SdkMember) android.BpModule { + return ctx.SnapshotBuilder().AddPrebuiltModule(member, "java_sdk_library_import") +} + +func (s *sdkLibrarySdkMemberType) CreateVariantPropertiesStruct() android.SdkMemberProperties { + return &sdkLibrarySdkMemberProperties{} +} + +type sdkLibrarySdkMemberProperties struct { + android.SdkMemberPropertiesBase + + // Scope to per scope properties. + Scopes map[*apiScope]scopeProperties + + // Additional libraries that the exported stubs libraries depend upon. + Libs []string + + // The Java stubs source files. + Stub_srcs []string + + // The naming scheme. + Naming_scheme *string + + // True if the java_sdk_library_import is for a shared library, false + // otherwise. + Shared_library *bool +} + +type scopeProperties struct { + Jars android.Paths + StubsSrcJar android.Path + CurrentApiFile android.Path + RemovedApiFile android.Path + SdkVersion string +} + +func (s *sdkLibrarySdkMemberProperties) PopulateFromVariant(ctx android.SdkMemberContext, variant android.Module) { + sdk := variant.(*SdkLibrary) + + s.Scopes = make(map[*apiScope]scopeProperties) + for _, apiScope := range allApiScopes { + paths := sdk.findScopePaths(apiScope) + if paths == nil { + continue + } + + jars := paths.stubsImplPath + if len(jars) > 0 { + properties := scopeProperties{} + properties.Jars = jars + properties.SdkVersion = sdk.sdkVersionForStubsLibrary(ctx.SdkModuleContext(), apiScope) + properties.StubsSrcJar = paths.stubsSrcJar.Path() + if paths.currentApiFilePath.Valid() { + properties.CurrentApiFile = paths.currentApiFilePath.Path() + } + if paths.removedApiFilePath.Valid() { + properties.RemovedApiFile = paths.removedApiFilePath.Path() + } + s.Scopes[apiScope] = properties + } + } + + s.Libs = sdk.properties.Libs + s.Naming_scheme = sdk.commonSdkLibraryProperties.Naming_scheme + s.Shared_library = proptools.BoolPtr(sdk.sharedLibrary()) +} + +func (s *sdkLibrarySdkMemberProperties) AddToPropertySet(ctx android.SdkMemberContext, propertySet android.BpPropertySet) { + if s.Naming_scheme != nil { + propertySet.AddProperty("naming_scheme", proptools.String(s.Naming_scheme)) + } + if s.Shared_library != nil { + propertySet.AddProperty("shared_library", *s.Shared_library) + } + + for _, apiScope := range allApiScopes { + if properties, ok := s.Scopes[apiScope]; ok { + scopeSet := propertySet.AddPropertySet(apiScope.propertyName) + + scopeDir := filepath.Join("sdk_library", s.OsPrefix(), apiScope.name) + + var jars []string + for _, p := range properties.Jars { + dest := filepath.Join(scopeDir, ctx.Name()+"-stubs.jar") + ctx.SnapshotBuilder().CopyToSnapshot(p, dest) + jars = append(jars, dest) + } + scopeSet.AddProperty("jars", jars) + + // Merge the stubs source jar into the snapshot zip so that when it is unpacked + // the source files are also unpacked. + snapshotRelativeDir := filepath.Join(scopeDir, ctx.Name()+"_stub_sources") + ctx.SnapshotBuilder().UnzipToSnapshot(properties.StubsSrcJar, snapshotRelativeDir) + scopeSet.AddProperty("stub_srcs", []string{snapshotRelativeDir}) + + if properties.CurrentApiFile != nil { + currentApiSnapshotPath := filepath.Join(scopeDir, ctx.Name()+".txt") + ctx.SnapshotBuilder().CopyToSnapshot(properties.CurrentApiFile, currentApiSnapshotPath) + scopeSet.AddProperty("current_api", currentApiSnapshotPath) + } + + if properties.RemovedApiFile != nil { + removedApiSnapshotPath := filepath.Join(scopeDir, ctx.Name()+"-removed.txt") + ctx.SnapshotBuilder().CopyToSnapshot(properties.RemovedApiFile, removedApiSnapshotPath) + scopeSet.AddProperty("removed_api", removedApiSnapshotPath) + } + + if properties.SdkVersion != "" { + scopeSet.AddProperty("sdk_version", properties.SdkVersion) + } + } + } + + if len(s.Libs) > 0 { + propertySet.AddPropertyWithTag("libs", s.Libs, ctx.SnapshotBuilder().SdkMemberReferencePropertyTag(false)) + } +} diff --git a/java/sdk_test.go b/java/sdk_test.go index e446129a5..52d2df552 100644 --- a/java/sdk_test.go +++ b/java/sdk_test.go @@ -28,173 +28,239 @@ import ( func TestClasspath(t *testing.T) { var classpathTestcases = []struct { - name string - unbundled bool - pdk bool - moduleType string - host android.OsClass - properties string - bootclasspath []string - system string - classpath []string - aidl string + name string + unbundled bool + pdk bool + moduleType string + host android.OsClass + properties string + + // for java 8 + bootclasspath []string + java8classpath []string + + // for java 9 + system string + java9classpath []string + + forces8 bool // if set, javac will always be called with java 8 arguments + + aidl string }{ { - name: "default", - bootclasspath: config.DefaultBootclasspathLibraries, - system: config.DefaultSystemModules, - classpath: config.DefaultLibraries, - aidl: "-Iframework/aidl", + name: "default", + bootclasspath: config.DefaultBootclasspathLibraries, + system: config.DefaultSystemModules, + java8classpath: config.DefaultLibraries, + java9classpath: config.DefaultLibraries, + aidl: "-Iframework/aidl", }, { - name: "blank sdk version", - properties: `sdk_version: "",`, - bootclasspath: config.DefaultBootclasspathLibraries, - system: config.DefaultSystemModules, - classpath: config.DefaultLibraries, - aidl: "-Iframework/aidl", + name: `sdk_version:"core_platform"`, + properties: `sdk_version:"core_platform"`, + bootclasspath: config.DefaultBootclasspathLibraries, + system: config.DefaultSystemModules, + java8classpath: []string{}, + aidl: "", + }, + { + name: "blank sdk version", + properties: `sdk_version: "",`, + bootclasspath: config.DefaultBootclasspathLibraries, + system: config.DefaultSystemModules, + java8classpath: config.DefaultLibraries, + java9classpath: config.DefaultLibraries, + aidl: "-Iframework/aidl", }, { - name: "sdk v25", - properties: `sdk_version: "25",`, - bootclasspath: []string{`""`}, - system: "bootclasspath", // special value to tell 1.9 test to expect bootclasspath - classpath: []string{"prebuilts/sdk/25/public/android.jar", "prebuilts/sdk/tools/core-lambda-stubs.jar"}, - aidl: "-pprebuilts/sdk/25/public/framework.aidl", + name: "sdk v29", + properties: `sdk_version: "29",`, + bootclasspath: []string{`""`}, + forces8: true, + java8classpath: []string{"prebuilts/sdk/29/public/android.jar", "prebuilts/sdk/tools/core-lambda-stubs.jar"}, + aidl: "-pprebuilts/sdk/29/public/framework.aidl", }, { - name: "current", - properties: `sdk_version: "current",`, - bootclasspath: []string{"android_stubs_current", "core-lambda-stubs"}, - system: "bootclasspath", // special value to tell 1.9 test to expect bootclasspath - aidl: "-p" + buildDir + "/framework.aidl", + name: "sdk v30", + properties: `sdk_version: "30",`, + bootclasspath: []string{`""`}, + system: "sdk_public_30_system_modules", + java8classpath: []string{"prebuilts/sdk/30/public/android.jar", "prebuilts/sdk/tools/core-lambda-stubs.jar"}, + java9classpath: []string{"prebuilts/sdk/30/public/android.jar", "prebuilts/sdk/tools/core-lambda-stubs.jar"}, + aidl: "-pprebuilts/sdk/30/public/framework.aidl", }, { - name: "system_current", - properties: `sdk_version: "system_current",`, - bootclasspath: []string{"android_system_stubs_current", "core-lambda-stubs"}, - system: "bootclasspath", // special value to tell 1.9 test to expect bootclasspath - aidl: "-p" + buildDir + "/framework.aidl", + name: "current", + properties: `sdk_version: "current",`, + bootclasspath: []string{"android_stubs_current", "core-lambda-stubs"}, + system: "core-current-stubs-system-modules", + java9classpath: []string{"android_stubs_current"}, + aidl: "-p" + buildDir + "/framework.aidl", }, { - name: "system_25", - properties: `sdk_version: "system_25",`, - bootclasspath: []string{`""`}, - system: "bootclasspath", // special value to tell 1.9 test to expect bootclasspath - classpath: []string{"prebuilts/sdk/25/system/android.jar", "prebuilts/sdk/tools/core-lambda-stubs.jar"}, - aidl: "-pprebuilts/sdk/25/public/framework.aidl", + name: "system_current", + properties: `sdk_version: "system_current",`, + bootclasspath: []string{"android_system_stubs_current", "core-lambda-stubs"}, + system: "core-current-stubs-system-modules", + java9classpath: []string{"android_system_stubs_current"}, + aidl: "-p" + buildDir + "/framework.aidl", }, { - name: "test_current", - properties: `sdk_version: "test_current",`, - bootclasspath: []string{"android_test_stubs_current", "core-lambda-stubs"}, - system: "bootclasspath", // special value to tell 1.9 test to expect bootclasspath - aidl: "-p" + buildDir + "/framework.aidl", + name: "system_29", + properties: `sdk_version: "system_29",`, + bootclasspath: []string{`""`}, + forces8: true, + java8classpath: []string{"prebuilts/sdk/29/system/android.jar", "prebuilts/sdk/tools/core-lambda-stubs.jar"}, + aidl: "-pprebuilts/sdk/29/public/framework.aidl", }, { - name: "core_current", - properties: `sdk_version: "core_current",`, - bootclasspath: []string{"core.current.stubs", "core-lambda-stubs"}, - system: "bootclasspath", // special value to tell 1.9 test to expect bootclasspath + name: "system_30", + properties: `sdk_version: "system_30",`, + bootclasspath: []string{`""`}, + system: "sdk_public_30_system_modules", + java8classpath: []string{"prebuilts/sdk/30/system/android.jar", "prebuilts/sdk/tools/core-lambda-stubs.jar"}, + java9classpath: []string{"prebuilts/sdk/30/system/android.jar", "prebuilts/sdk/tools/core-lambda-stubs.jar"}, + aidl: "-pprebuilts/sdk/30/public/framework.aidl", }, { - name: "nostdlib", - properties: `no_standard_libs: true, system_modules: "none"`, - system: "none", - bootclasspath: []string{`""`}, - classpath: []string{}, + name: "test_current", + properties: `sdk_version: "test_current",`, + bootclasspath: []string{"android_test_stubs_current", "core-lambda-stubs"}, + system: "core-current-stubs-system-modules", + java9classpath: []string{"android_test_stubs_current"}, + aidl: "-p" + buildDir + "/framework.aidl", }, { - name: "nostdlib system_modules", - properties: `no_standard_libs: true, system_modules: "core-platform-api-stubs-system-modules"`, - system: "core-platform-api-stubs-system-modules", - bootclasspath: []string{`""`}, - classpath: []string{}, + name: "core_current", + properties: `sdk_version: "core_current",`, + bootclasspath: []string{"core.current.stubs", "core-lambda-stubs"}, + system: "core-current-stubs-system-modules", + java9classpath: []string{"core.current.stubs"}, }, { - name: "host default", - moduleType: "java_library_host", - properties: ``, - host: android.Host, - bootclasspath: []string{"jdk8/jre/lib/jce.jar", "jdk8/jre/lib/rt.jar"}, - classpath: []string{}, + name: "nostdlib", + properties: `sdk_version: "none", system_modules: "none"`, + system: "none", + bootclasspath: []string{`""`}, + java8classpath: []string{}, }, { - name: "host nostdlib", - moduleType: "java_library_host", - host: android.Host, - properties: `no_standard_libs: true`, - classpath: []string{}, + + name: "nostdlib system_modules", + properties: `sdk_version: "none", system_modules: "core-platform-api-stubs-system-modules"`, + system: "core-platform-api-stubs-system-modules", + bootclasspath: []string{"core-platform-api-stubs-system-modules-lib"}, + java8classpath: []string{}, }, { - name: "host supported default", - host: android.Host, - properties: `host_supported: true,`, - classpath: []string{}, - bootclasspath: []string{"jdk8/jre/lib/jce.jar", "jdk8/jre/lib/rt.jar"}, + name: "host default", + moduleType: "java_library_host", + properties: ``, + host: android.Host, + bootclasspath: []string{"jdk8/jre/lib/jce.jar", "jdk8/jre/lib/rt.jar"}, + java8classpath: []string{}, }, { - name: "host supported nostdlib", - host: android.Host, - properties: `host_supported: true, no_standard_libs: true, system_modules: "none"`, - classpath: []string{}, + + name: "host supported default", + host: android.Host, + properties: `host_supported: true,`, + java8classpath: []string{}, + bootclasspath: []string{"jdk8/jre/lib/jce.jar", "jdk8/jre/lib/rt.jar"}, + }, + { + name: "host supported nostdlib", + host: android.Host, + properties: `host_supported: true, sdk_version: "none", system_modules: "none"`, + java8classpath: []string{}, }, { - name: "unbundled sdk v25", - unbundled: true, - properties: `sdk_version: "25",`, - bootclasspath: []string{`""`}, - system: "bootclasspath", // special value to tell 1.9 test to expect bootclasspath - classpath: []string{"prebuilts/sdk/25/public/android.jar", "prebuilts/sdk/tools/core-lambda-stubs.jar"}, - aidl: "-pprebuilts/sdk/25/public/framework.aidl", + name: "unbundled sdk v29", + unbundled: true, + properties: `sdk_version: "29",`, + bootclasspath: []string{`""`}, + forces8: true, + java8classpath: []string{"prebuilts/sdk/29/public/android.jar", "prebuilts/sdk/tools/core-lambda-stubs.jar"}, + aidl: "-pprebuilts/sdk/29/public/framework.aidl", }, { - name: "unbundled current", - unbundled: true, - properties: `sdk_version: "current",`, - bootclasspath: []string{`""`}, - system: "bootclasspath", // special value to tell 1.9 test to expect bootclasspath - classpath: []string{"prebuilts/sdk/current/public/android.jar", "prebuilts/sdk/tools/core-lambda-stubs.jar"}, - aidl: "-pprebuilts/sdk/current/public/framework.aidl", + name: "unbundled sdk v30", + unbundled: true, + properties: `sdk_version: "30",`, + bootclasspath: []string{`""`}, + system: "sdk_public_30_system_modules", + java8classpath: []string{"prebuilts/sdk/30/public/android.jar", "prebuilts/sdk/tools/core-lambda-stubs.jar"}, + java9classpath: []string{"prebuilts/sdk/30/public/android.jar", "prebuilts/sdk/tools/core-lambda-stubs.jar"}, + aidl: "-pprebuilts/sdk/30/public/framework.aidl", }, + { + name: "unbundled current", + unbundled: true, + properties: `sdk_version: "current",`, + bootclasspath: []string{`""`}, + system: "sdk_public_current_system_modules", + java8classpath: []string{"prebuilts/sdk/current/public/android.jar", "prebuilts/sdk/tools/core-lambda-stubs.jar"}, + java9classpath: []string{"prebuilts/sdk/current/public/android.jar", "prebuilts/sdk/tools/core-lambda-stubs.jar"}, + aidl: "-pprebuilts/sdk/current/public/framework.aidl", + }, + + { + name: "pdk default", + pdk: true, + bootclasspath: []string{`""`}, + system: "sdk_public_30_system_modules", + java8classpath: []string{"prebuilts/sdk/30/public/android.jar", "prebuilts/sdk/tools/core-lambda-stubs.jar"}, + java9classpath: []string{"prebuilts/sdk/30/public/android.jar", "prebuilts/sdk/tools/core-lambda-stubs.jar"}, + aidl: "-pprebuilts/sdk/30/public/framework.aidl", + }, { - name: "pdk default", - pdk: true, - bootclasspath: []string{`""`}, - system: "bootclasspath", // special value to tell 1.9 test to expect bootclasspath - classpath: []string{"prebuilts/sdk/25/public/android.jar", "prebuilts/sdk/tools/core-lambda-stubs.jar"}, - aidl: "-pprebuilts/sdk/25/public/framework.aidl", + name: "pdk current", + pdk: true, + properties: `sdk_version: "current",`, + bootclasspath: []string{`""`}, + system: "sdk_public_30_system_modules", + java8classpath: []string{"prebuilts/sdk/30/public/android.jar", "prebuilts/sdk/tools/core-lambda-stubs.jar"}, + java9classpath: []string{"prebuilts/sdk/30/public/android.jar", "prebuilts/sdk/tools/core-lambda-stubs.jar"}, + aidl: "-pprebuilts/sdk/30/public/framework.aidl", }, { - name: "pdk current", - pdk: true, - properties: `sdk_version: "current",`, - bootclasspath: []string{`""`}, - system: "bootclasspath", // special value to tell 1.9 test to expect bootclasspath - classpath: []string{"prebuilts/sdk/25/public/android.jar", "prebuilts/sdk/tools/core-lambda-stubs.jar"}, - aidl: "-pprebuilts/sdk/25/public/framework.aidl", + name: "pdk 29", + pdk: true, + properties: `sdk_version: "29",`, + bootclasspath: []string{`""`}, + system: "sdk_public_30_system_modules", + java8classpath: []string{"prebuilts/sdk/30/public/android.jar", "prebuilts/sdk/tools/core-lambda-stubs.jar"}, + java9classpath: []string{"prebuilts/sdk/30/public/android.jar", "prebuilts/sdk/tools/core-lambda-stubs.jar"}, + aidl: "-pprebuilts/sdk/30/public/framework.aidl", }, { - name: "pdk 25", - pdk: true, - properties: `sdk_version: "25",`, - bootclasspath: []string{`""`}, - system: "bootclasspath", // special value to tell 1.9 test to expect bootclasspath - classpath: []string{"prebuilts/sdk/25/public/android.jar", "prebuilts/sdk/tools/core-lambda-stubs.jar"}, - aidl: "-pprebuilts/sdk/25/public/framework.aidl", + name: "module_current", + properties: `sdk_version: "module_current",`, + bootclasspath: []string{"android_module_lib_stubs_current", "core-lambda-stubs"}, + system: "core-current-stubs-system-modules", + java9classpath: []string{"android_module_lib_stubs_current"}, + aidl: "-p" + buildDir + "/framework_non_updatable.aidl", + }, + { + name: "system_server_current", + properties: `sdk_version: "system_server_current",`, + bootclasspath: []string{"android_system_server_stubs_current", "core-lambda-stubs"}, + system: "core-current-stubs-system-modules", + java9classpath: []string{"android_system_server_stubs_current"}, + aidl: "-p" + buildDir + "/framework.aidl", }, } @@ -205,7 +271,7 @@ func TestClasspath(t *testing.T) { moduleType = testcase.moduleType } - bp := moduleType + ` { + props := ` name: "foo", srcs: ["a.java"], target: { @@ -213,6 +279,10 @@ func TestClasspath(t *testing.T) { srcs: ["bar-doc/IFoo.aidl"], }, }, + ` + bp := moduleType + " {" + props + testcase.properties + ` + }` + bpJava8 := moduleType + " {" + props + `java_version: "1.8", ` + testcase.properties + ` }` @@ -230,101 +300,153 @@ func TestClasspath(t *testing.T) { } bootclasspath := convertModulesToPaths(testcase.bootclasspath) - classpath := convertModulesToPaths(testcase.classpath) + java8classpath := convertModulesToPaths(testcase.java8classpath) + java9classpath := convertModulesToPaths(testcase.java9classpath) + + bc := "" + var bcDeps []string + if len(bootclasspath) > 0 { + bc = "-bootclasspath " + strings.Join(bootclasspath, ":") + if bootclasspath[0] != `""` { + bcDeps = bootclasspath + } + } - bc := strings.Join(bootclasspath, ":") - if bc != "" { - bc = "-bootclasspath " + bc + j8c := "" + if len(java8classpath) > 0 { + j8c = "-classpath " + strings.Join(java8classpath, ":") } - c := strings.Join(classpath, ":") - if c != "" { - c = "-classpath " + c + j9c := "" + if len(java9classpath) > 0 { + j9c = "-classpath " + strings.Join(java9classpath, ":") } + system := "" + var systemDeps []string if testcase.system == "none" { system = "--system=none" } else if testcase.system != "" { - system = "--system=" + filepath.Join(buildDir, ".intermediates", testcase.system, "android_common", "system") + "/" + dir := "" + if strings.HasPrefix(testcase.system, "sdk_public_") { + dir = "prebuilts/sdk" + } + system = "--system=" + filepath.Join(buildDir, ".intermediates", dir, testcase.system, "android_common", "system") + // The module-relative parts of these paths are hardcoded in system_modules.go: + systemDeps = []string{ + filepath.Join(buildDir, ".intermediates", dir, testcase.system, "android_common", "system", "lib", "modules"), + filepath.Join(buildDir, ".intermediates", dir, testcase.system, "android_common", "system", "lib", "jrt-fs.jar"), + filepath.Join(buildDir, ".intermediates", dir, testcase.system, "android_common", "system", "release"), + } } - checkClasspath := func(t *testing.T, ctx *android.TestContext) { - javac := ctx.ModuleForTests("foo", variant).Rule("javac") + checkClasspath := func(t *testing.T, ctx *android.TestContext, isJava8 bool) { + foo := ctx.ModuleForTests("foo", variant) + javac := foo.Rule("javac") + var deps []string - got := javac.Args["bootClasspath"] - if got != bc { - t.Errorf("bootclasspath expected %q != got %q", bc, got) + aidl := foo.MaybeRule("aidl") + if aidl.Rule != nil { + deps = append(deps, aidl.Output.String()) } - got = javac.Args["classpath"] - if got != c { - t.Errorf("classpath expected %q != got %q", c, got) + got := javac.Args["bootClasspath"] + expected := "" + if isJava8 || testcase.forces8 { + expected = bc + deps = append(deps, bcDeps...) + } else { + expected = system + deps = append(deps, systemDeps...) + } + if got != expected { + t.Errorf("bootclasspath expected %q != got %q", expected, got) } - var deps []string - if len(bootclasspath) > 0 && bootclasspath[0] != `""` { - deps = append(deps, bootclasspath...) + if isJava8 || testcase.forces8 { + expected = j8c + deps = append(deps, java8classpath...) + } else { + expected = j9c + deps = append(deps, java9classpath...) + } + got = javac.Args["classpath"] + if got != expected { + t.Errorf("classpath expected %q != got %q", expected, got) } - deps = append(deps, classpath...) if !reflect.DeepEqual(javac.Implicits.Strings(), deps) { t.Errorf("implicits expected %q != got %q", deps, javac.Implicits.Strings()) } } - t.Run("1.8", func(t *testing.T) { - // Test default javac 1.8 - config := testConfig(nil) + // Test with legacy javac -source 1.8 -target 1.8 + t.Run("Java language level 8", func(t *testing.T) { + config := testConfig(nil, bpJava8, nil) if testcase.unbundled { config.TestProductVariables.Unbundled_build = proptools.BoolPtr(true) } if testcase.pdk { config.TestProductVariables.Pdk = proptools.BoolPtr(true) } - ctx := testContext(config, bp, nil) + ctx := testContext() run(t, ctx, config) - checkClasspath(t, ctx) + checkClasspath(t, ctx, true /* isJava8 */) if testcase.host != android.Host { aidl := ctx.ModuleForTests("foo", variant).Rule("aidl") - aidlFlags := aidl.Args["aidlFlags"] - // Trim trailing "-I." to avoid having to specify it in every test - aidlFlags = strings.TrimSpace(strings.TrimSuffix(aidlFlags, "-I.")) - - if g, w := aidlFlags, testcase.aidl; g != w { - t.Errorf("want aidl flags %q, got %q", w, g) + if g, w := aidl.RuleParams.Command, testcase.aidl+" -I."; !strings.Contains(g, w) { + t.Errorf("want aidl command to contain %q, got %q", w, g) } } }) - // Test again with javac 1.9 - t.Run("1.9", func(t *testing.T) { - config := testConfig(map[string]string{"EXPERIMENTAL_USE_OPENJDK9": "true"}) + // Test with default javac -source 9 -target 9 + t.Run("Java language level 9", func(t *testing.T) { + config := testConfig(nil, bp, nil) if testcase.unbundled { config.TestProductVariables.Unbundled_build = proptools.BoolPtr(true) } if testcase.pdk { config.TestProductVariables.Pdk = proptools.BoolPtr(true) } - ctx := testContext(config, bp, nil) + ctx := testContext() run(t, ctx, config) - javac := ctx.ModuleForTests("foo", variant).Rule("javac") - got := javac.Args["bootClasspath"] - expected := system - if testcase.system == "bootclasspath" { - expected = bc + checkClasspath(t, ctx, false /* isJava8 */) + + if testcase.host != android.Host { + aidl := ctx.ModuleForTests("foo", variant).Rule("aidl") + + if g, w := aidl.RuleParams.Command, testcase.aidl+" -I."; !strings.Contains(g, w) { + t.Errorf("want aidl command to contain %q, got %q", w, g) + } } - if got != expected { - t.Errorf("bootclasspath expected %q != got %q", expected, got) + }) + + // Test again with PLATFORM_VERSION_CODENAME=REL, javac -source 8 -target 8 + t.Run("REL + Java language level 8", func(t *testing.T) { + config := testConfig(nil, bpJava8, nil) + config.TestProductVariables.Platform_sdk_codename = proptools.StringPtr("REL") + config.TestProductVariables.Platform_sdk_final = proptools.BoolPtr(true) + + if testcase.unbundled { + config.TestProductVariables.Unbundled_build = proptools.BoolPtr(true) + } + if testcase.pdk { + config.TestProductVariables.Pdk = proptools.BoolPtr(true) } + ctx := testContext() + run(t, ctx, config) + + checkClasspath(t, ctx, true /* isJava8 */) }) - // Test again with PLATFORM_VERSION_CODENAME=REL - t.Run("REL", func(t *testing.T) { - config := testConfig(nil) + // Test again with PLATFORM_VERSION_CODENAME=REL, javac -source 9 -target 9 + t.Run("REL + Java language level 9", func(t *testing.T) { + config := testConfig(nil, bp, nil) config.TestProductVariables.Platform_sdk_codename = proptools.StringPtr("REL") config.TestProductVariables.Platform_sdk_final = proptools.BoolPtr(true) @@ -334,12 +456,11 @@ func TestClasspath(t *testing.T) { if testcase.pdk { config.TestProductVariables.Pdk = proptools.BoolPtr(true) } - ctx := testContext(config, bp, nil) + ctx := testContext() run(t, ctx, config) - checkClasspath(t, ctx) + checkClasspath(t, ctx, false /* isJava8 */) }) }) } - } diff --git a/java/support_libraries.go b/java/support_libraries.go index 5a72f41a9..af7c3c2a0 100644 --- a/java/support_libraries.go +++ b/java/support_libraries.go @@ -52,8 +52,6 @@ func supportLibrariesMakeVarsProvider(ctx android.MakeVarsContext) { supportAars = append(supportAars, name) case *Library, *Import: supportJars = append(supportJars, name) - default: - ctx.ModuleErrorf(module, "unknown module type %t", module) } }) diff --git a/java/sysprop.go b/java/sysprop.go new file mode 100644 index 000000000..1a70499b8 --- /dev/null +++ b/java/sysprop.go @@ -0,0 +1,60 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package java + +import ( + "sync" + + "android/soong/android" +) + +type syspropLibraryInterface interface { + BaseModuleName() string + Owner() string + HasPublicStub() bool + JavaPublicStubName() string +} + +var ( + syspropPublicStubsKey = android.NewOnceKey("syspropPublicStubsJava") + syspropPublicStubsLock sync.Mutex +) + +func init() { + android.PreDepsMutators(func(ctx android.RegisterMutatorsContext) { + ctx.BottomUp("sysprop_java", SyspropMutator).Parallel() + }) +} + +func syspropPublicStubs(config android.Config) map[string]string { + return config.Once(syspropPublicStubsKey, func() interface{} { + return make(map[string]string) + }).(map[string]string) +} + +// gather list of sysprop libraries owned by platform. +func SyspropMutator(mctx android.BottomUpMutatorContext) { + if m, ok := mctx.Module().(syspropLibraryInterface); ok { + if m.Owner() != "Platform" || !m.HasPublicStub() { + return + } + + syspropPublicStubs := syspropPublicStubs(mctx.Config()) + syspropPublicStubsLock.Lock() + defer syspropPublicStubsLock.Unlock() + + syspropPublicStubs[m.BaseModuleName()] = m.JavaPublicStubName() + } +} diff --git a/java/system_modules.go b/java/system_modules.go index 9ee0307ca..7394fd547 100644 --- a/java/system_modules.go +++ b/java/system_modules.go @@ -28,21 +28,41 @@ import ( // system modules in a runtime image using the jmod and jlink tools. func init() { - android.RegisterModuleType("java_system_modules", SystemModulesFactory) + RegisterSystemModulesBuildComponents(android.InitRegistrationContext) pctx.SourcePathVariable("moduleInfoJavaPath", "build/soong/scripts/jars-to-module-info-java.sh") + + // Register sdk member types. + android.RegisterSdkMemberType(&systemModulesSdkMemberType{ + android.SdkMemberTypeBase{ + PropertyName: "java_system_modules", + SupportsSdk: true, + TransitiveSdkMembers: true, + }, + }) +} + +func RegisterSystemModulesBuildComponents(ctx android.RegistrationContext) { + ctx.RegisterModuleType("java_system_modules", SystemModulesFactory) + ctx.RegisterModuleType("java_system_modules_import", systemModulesImportFactory) } var ( jarsTosystemModules = pctx.AndroidStaticRule("jarsTosystemModules", blueprint.RuleParams{ Command: `rm -rf ${outDir} ${workDir} && mkdir -p ${workDir}/jmod && ` + - `${moduleInfoJavaPath} ${moduleName} $in > ${workDir}/module-info.java && ` + + `${moduleInfoJavaPath} java.base $in > ${workDir}/module-info.java && ` + `${config.JavacCmd} --system=none --patch-module=java.base=${classpath} ${workDir}/module-info.java && ` + `${config.SoongZipCmd} -jar -o ${workDir}/classes.jar -C ${workDir} -f ${workDir}/module-info.class && ` + `${config.MergeZipsCmd} -j ${workDir}/module.jar ${workDir}/classes.jar $in && ` + - `${config.JmodCmd} create --module-version 9 --target-platform android ` + - ` --class-path ${workDir}/module.jar ${workDir}/jmod/${moduleName}.jmod && ` + - `${config.JlinkCmd} --module-path ${workDir}/jmod --add-modules ${moduleName} --output ${outDir} && ` + + // Note: The version of the java.base module created must match the version + // of the jlink tool which consumes it. + `${config.JmodCmd} create --module-version ${config.JlinkVersion} --target-platform android ` + + ` --class-path ${workDir}/module.jar ${workDir}/jmod/java.base.jmod && ` + + `${config.JlinkCmd} --module-path ${workDir}/jmod --add-modules java.base --output ${outDir} ` + + // Note: The system-modules jlink plugin is disabled because (a) it is not + // useful on Android, and (b) it causes errors with later versions of jlink + // when the jdk.internal.module is absent from java.base (as it is here). + ` --disable-plugin system-modules && ` + `cp ${config.JrtFsJar} ${outDir}/lib/`, CommandDeps: []string{ "${moduleInfoJavaPath}", @@ -54,10 +74,14 @@ var ( "${config.JrtFsJar}", }, }, - "moduleName", "classpath", "outDir", "workDir") + "classpath", "outDir", "workDir") + + // Dependency tag that causes the added dependencies to be added as java_header_libs + // to the sdk/module_exports/snapshot. + systemModulesLibsTag = android.DependencyTagForSdkMemberType(javaHeaderLibsSdkMemberType) ) -func TransformJarsToSystemModules(ctx android.ModuleContext, moduleName string, jars android.Paths) android.WritablePath { +func TransformJarsToSystemModules(ctx android.ModuleContext, jars android.Paths) (android.Path, android.Paths) { outDir := android.PathForModuleOut(ctx, "system") workDir := android.PathForModuleOut(ctx, "modules") outputFile := android.PathForModuleOut(ctx, "system/lib/modules") @@ -73,72 +97,173 @@ func TransformJarsToSystemModules(ctx android.ModuleContext, moduleName string, Outputs: outputs, Inputs: jars, Args: map[string]string{ - "moduleName": moduleName, - "classpath": strings.Join(jars.Strings(), ":"), - "workDir": workDir.String(), - "outDir": outDir.String(), + "classpath": strings.Join(jars.Strings(), ":"), + "workDir": workDir.String(), + "outDir": outDir.String(), }, }) - return outputFile + return outDir, outputs.Paths() } +// java_system_modules creates a system module from a set of java libraries that can +// be referenced from the system_modules property. It must contain at a minimum the +// java.base module which must include classes from java.lang amongst other java packages. func SystemModulesFactory() android.Module { module := &SystemModules{} module.AddProperties(&module.properties) android.InitAndroidArchModule(module, android.HostAndDeviceSupported, android.MultilibCommon) + android.InitDefaultableModule(module) return module } +type SystemModulesProvider interface { + HeaderJars() android.Paths + OutputDirAndDeps() (android.Path, android.Paths) +} + +var _ SystemModulesProvider = (*SystemModules)(nil) + +var _ SystemModulesProvider = (*systemModulesImport)(nil) + type SystemModules struct { android.ModuleBase + android.DefaultableModuleBase + android.SdkBase properties SystemModulesProperties - outputFile android.Path + // The aggregated header jars from all jars specified in the libs property. + // Used when system module is added as a dependency to bootclasspath. + headerJars android.Paths + outputDir android.Path + outputDeps android.Paths } type SystemModulesProperties struct { // List of java library modules that should be included in the system modules Libs []string +} - // List of prebuilt jars that should be included in the system modules - Jars []string +func (system *SystemModules) HeaderJars() android.Paths { + return system.headerJars +} - // Sdk version that should be included in the system modules - Sdk_version *string +func (system *SystemModules) OutputDirAndDeps() (android.Path, android.Paths) { + if system.outputDir == nil || len(system.outputDeps) == 0 { + panic("Missing directory for system module dependency") + } + return system.outputDir, system.outputDeps } func (system *SystemModules) GenerateAndroidBuildActions(ctx android.ModuleContext) { var jars android.Paths - ctx.VisitDirectDepsWithTag(libTag, func(module android.Module) { + ctx.VisitDirectDepsWithTag(systemModulesLibsTag, func(module android.Module) { dep, _ := module.(Dependency) jars = append(jars, dep.HeaderJars()...) }) - jars = append(jars, android.PathsForModuleSrc(ctx, system.properties.Jars)...) + system.headerJars = jars - system.outputFile = TransformJarsToSystemModules(ctx, "java.base", jars) + system.outputDir, system.outputDeps = TransformJarsToSystemModules(ctx, jars) } func (system *SystemModules) DepsMutator(ctx android.BottomUpMutatorContext) { - ctx.AddVariationDependencies(nil, libTag, system.properties.Libs...) + ctx.AddVariationDependencies(nil, systemModulesLibsTag, system.properties.Libs...) } func (system *SystemModules) AndroidMk() android.AndroidMkData { return android.AndroidMkData{ Custom: func(w io.Writer, name, prefix, moduleDir string, data android.AndroidMkData) { + fmt.Fprintln(w) + makevar := "SOONG_SYSTEM_MODULES_" + name + fmt.Fprintln(w, makevar, ":=$=", system.outputDir.String()) + fmt.Fprintln(w) + + makevar = "SOONG_SYSTEM_MODULES_LIBS_" + name + fmt.Fprintln(w, makevar, ":=$=", strings.Join(system.properties.Libs, " ")) fmt.Fprintln(w) - fmt.Fprintln(w, makevar, ":=", system.outputFile.String()) - fmt.Fprintln(w, ".KATI_READONLY", ":=", makevar) + + makevar = "SOONG_SYSTEM_MODULES_DEPS_" + name + fmt.Fprintln(w, makevar, ":=$=", strings.Join(system.outputDeps.Strings(), " ")) + fmt.Fprintln(w) + fmt.Fprintln(w, name+":", "$("+makevar+")") fmt.Fprintln(w, ".PHONY:", name) - fmt.Fprintln(w) - makevar = "SOONG_SYSTEM_MODULES_LIBS_" + name - fmt.Fprintln(w, makevar, ":=", strings.Join(system.properties.Libs, " ")) - fmt.Fprintln(w, ".KATI_READONLY :=", makevar) }, } } + +// A prebuilt version of java_system_modules. It does not import the +// generated system module, it generates the system module from imported +// java libraries in the same way that java_system_modules does. It just +// acts as a prebuilt, i.e. can have the same base name as another module +// type and the one to use is selected at runtime. +func systemModulesImportFactory() android.Module { + module := &systemModulesImport{} + module.AddProperties(&module.properties) + android.InitPrebuiltModule(module, &module.properties.Libs) + android.InitAndroidArchModule(module, android.HostAndDeviceSupported, android.MultilibCommon) + android.InitDefaultableModule(module) + android.InitSdkAwareModule(module) + return module +} + +type systemModulesImport struct { + SystemModules + prebuilt android.Prebuilt +} + +func (system *systemModulesImport) Name() string { + return system.prebuilt.Name(system.ModuleBase.Name()) +} + +func (system *systemModulesImport) Prebuilt() *android.Prebuilt { + return &system.prebuilt +} + +type systemModulesSdkMemberType struct { + android.SdkMemberTypeBase +} + +func (mt *systemModulesSdkMemberType) AddDependencies(mctx android.BottomUpMutatorContext, dependencyTag blueprint.DependencyTag, names []string) { + mctx.AddVariationDependencies(nil, dependencyTag, names...) +} + +func (mt *systemModulesSdkMemberType) IsInstance(module android.Module) bool { + if _, ok := module.(*SystemModules); ok { + // A prebuilt system module cannot be added as a member of an sdk because the source and + // snapshot instances would conflict. + _, ok := module.(*systemModulesImport) + return !ok + } + return false +} + +func (mt *systemModulesSdkMemberType) AddPrebuiltModule(ctx android.SdkMemberContext, member android.SdkMember) android.BpModule { + return ctx.SnapshotBuilder().AddPrebuiltModule(member, "java_system_modules_import") +} + +type systemModulesInfoProperties struct { + android.SdkMemberPropertiesBase + + Libs []string +} + +func (mt *systemModulesSdkMemberType) CreateVariantPropertiesStruct() android.SdkMemberProperties { + return &systemModulesInfoProperties{} +} + +func (p *systemModulesInfoProperties) PopulateFromVariant(ctx android.SdkMemberContext, variant android.Module) { + systemModule := variant.(*SystemModules) + p.Libs = systemModule.properties.Libs +} + +func (p *systemModulesInfoProperties) AddToPropertySet(ctx android.SdkMemberContext, propertySet android.BpPropertySet) { + if len(p.Libs) > 0 { + // Add the references to the libraries that form the system module. + propertySet.AddPropertyWithTag("libs", p.Libs, ctx.SnapshotBuilder().SdkMemberReferencePropertyTag(true)) + } +} diff --git a/java/testing.go b/java/testing.go index 6b35bd040..48e449f34 100644 --- a/java/testing.go +++ b/java/testing.go @@ -18,16 +18,90 @@ import ( "fmt" "android/soong/android" + "android/soong/cc" ) -func TestConfig(buildDir string, env map[string]string) android.Config { +func TestConfig(buildDir string, env map[string]string, bp string, fs map[string][]byte) android.Config { + bp += GatherRequiredDepsForTest() + + mockFS := map[string][]byte{ + "api/current.txt": nil, + "api/removed.txt": nil, + "api/system-current.txt": nil, + "api/system-removed.txt": nil, + "api/test-current.txt": nil, + "api/test-removed.txt": nil, + + "prebuilts/sdk/14/public/android.jar": nil, + "prebuilts/sdk/14/public/framework.aidl": nil, + "prebuilts/sdk/14/system/android.jar": nil, + "prebuilts/sdk/17/public/android.jar": nil, + "prebuilts/sdk/17/public/framework.aidl": nil, + "prebuilts/sdk/17/system/android.jar": nil, + "prebuilts/sdk/29/public/android.jar": nil, + "prebuilts/sdk/29/public/framework.aidl": nil, + "prebuilts/sdk/29/system/android.jar": nil, + "prebuilts/sdk/29/system/foo.jar": nil, + "prebuilts/sdk/30/public/android.jar": nil, + "prebuilts/sdk/30/public/framework.aidl": nil, + "prebuilts/sdk/30/system/android.jar": nil, + "prebuilts/sdk/30/system/foo.jar": nil, + "prebuilts/sdk/30/public/core-for-system-modules.jar": nil, + "prebuilts/sdk/current/core/android.jar": nil, + "prebuilts/sdk/current/public/android.jar": nil, + "prebuilts/sdk/current/public/framework.aidl": nil, + "prebuilts/sdk/current/public/core.jar": nil, + "prebuilts/sdk/current/public/core-for-system-modules.jar": nil, + "prebuilts/sdk/current/system/android.jar": nil, + "prebuilts/sdk/current/test/android.jar": nil, + "prebuilts/sdk/28/public/api/foo.txt": nil, + "prebuilts/sdk/28/system/api/foo.txt": nil, + "prebuilts/sdk/28/test/api/foo.txt": nil, + "prebuilts/sdk/28/public/api/foo-removed.txt": nil, + "prebuilts/sdk/28/system/api/foo-removed.txt": nil, + "prebuilts/sdk/28/test/api/foo-removed.txt": nil, + "prebuilts/sdk/28/public/api/bar.txt": nil, + "prebuilts/sdk/28/system/api/bar.txt": nil, + "prebuilts/sdk/28/test/api/bar.txt": nil, + "prebuilts/sdk/28/public/api/bar-removed.txt": nil, + "prebuilts/sdk/28/system/api/bar-removed.txt": nil, + "prebuilts/sdk/28/test/api/bar-removed.txt": nil, + "prebuilts/sdk/30/public/api/foo.txt": nil, + "prebuilts/sdk/30/system/api/foo.txt": nil, + "prebuilts/sdk/30/test/api/foo.txt": nil, + "prebuilts/sdk/30/public/api/foo-removed.txt": nil, + "prebuilts/sdk/30/system/api/foo-removed.txt": nil, + "prebuilts/sdk/30/test/api/foo-removed.txt": nil, + "prebuilts/sdk/30/public/api/bar.txt": nil, + "prebuilts/sdk/30/system/api/bar.txt": nil, + "prebuilts/sdk/30/test/api/bar.txt": nil, + "prebuilts/sdk/30/public/api/bar-removed.txt": nil, + "prebuilts/sdk/30/system/api/bar-removed.txt": nil, + "prebuilts/sdk/30/test/api/bar-removed.txt": nil, + "prebuilts/sdk/tools/core-lambda-stubs.jar": nil, + "prebuilts/sdk/Android.bp": []byte(`prebuilt_apis { name: "sdk", api_dirs: ["14", "28", "30", "current"],}`), + + // For java_sdk_library + "api/module-lib-current.txt": nil, + "api/module-lib-removed.txt": nil, + "api/system-server-current.txt": nil, + "api/system-server-removed.txt": nil, + "build/soong/scripts/gen-java-current-api-files.sh": nil, + } + + cc.GatherRequiredFilesForTest(mockFS) + + for k, v := range fs { + mockFS[k] = v + } + if env == nil { env = make(map[string]string) } if env["ANDROID_JAVA8_HOME"] == "" { env["ANDROID_JAVA8_HOME"] = "jdk8" } - config := android.TestArchConfig(buildDir, env) + config := android.TestArchConfig(buildDir, env, bp, mockFS) return config } @@ -38,13 +112,16 @@ func GatherRequiredDepsForTest() string { extraModules := []string{ "core-lambda-stubs", "ext", - "updatable_media_stubs", "android_stubs_current", "android_system_stubs_current", "android_test_stubs_current", + "android_module_lib_stubs_current", + "android_system_server_stubs_current", "core.current.stubs", "core.platform.api.stubs", "kotlin-stdlib", + "kotlin-stdlib-jdk7", + "kotlin-stdlib-jdk8", "kotlin-annotations", } @@ -53,8 +130,7 @@ func GatherRequiredDepsForTest() string { java_library { name: "%s", srcs: ["a.java"], - no_standard_libs: true, - sdk_version: "core_current", + sdk_version: "none", system_modules: "core-platform-api-stubs-system-modules", } `, extra) @@ -64,8 +140,7 @@ func GatherRequiredDepsForTest() string { java_library { name: "framework", srcs: ["a.java"], - no_standard_libs: true, - sdk_version: "core_current", + sdk_version: "none", system_modules: "core-platform-api-stubs-system-modules", aidl: { export_include_dirs: ["framework/aidl"], @@ -74,22 +149,49 @@ func GatherRequiredDepsForTest() string { android_app { name: "framework-res", - no_framework_libs: true, + sdk_version: "core_platform", + } + + java_library { + name: "android.hidl.base-V1.0-java", + srcs: ["a.java"], + sdk_version: "none", + system_modules: "core-platform-api-stubs-system-modules", + installable: true, + } + + java_library { + name: "android.hidl.manager-V1.0-java", + srcs: ["a.java"], + sdk_version: "none", + system_modules: "core-platform-api-stubs-system-modules", + installable: true, + } + + java_library { + name: "org.apache.http.legacy", + srcs: ["a.java"], + sdk_version: "none", + system_modules: "core-platform-api-stubs-system-modules", + installable: true, } ` systemModules := []string{ - "core-system-modules", + "core-current-stubs-system-modules", "core-platform-api-stubs-system-modules", - "android_stubs_current_system_modules", - "android_system_stubs_current_system_modules", - "android_test_stubs_current_system_modules", } for _, extra := range systemModules { bp += fmt.Sprintf(` java_system_modules { - name: "%s", + name: "%[1]s", + libs: ["%[1]s-lib"], + } + java_library { + name: "%[1]s-lib", + sdk_version: "none", + system_modules: "none", } `, extra) } diff --git a/java/tradefed.go b/java/tradefed.go new file mode 100644 index 000000000..ebbdec13d --- /dev/null +++ b/java/tradefed.go @@ -0,0 +1,37 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package java + +import ( + "android/soong/android" +) + +func init() { + android.RegisterModuleType("tradefed_java_library_host", tradefedJavaLibraryFactory) +} + +// tradefed_java_library_factory wraps java_library and installs an additional +// copy of the output jar to $HOST_OUT/tradefed. +func tradefedJavaLibraryFactory() android.Module { + module := LibraryHostFactory().(*Library) + module.InstallMixin = tradefedJavaLibraryInstall + return module +} + +func tradefedJavaLibraryInstall(ctx android.ModuleContext, path android.Path) android.Paths { + installedPath := ctx.InstallFile(android.PathForModuleInstall(ctx, "tradefed"), + ctx.ModuleName()+".jar", path) + return android.Paths{installedPath} +} diff --git a/makedeps/deps.go b/makedeps/deps.go index e64e6f788..db4953259 100644 --- a/makedeps/deps.go +++ b/makedeps/deps.go @@ -57,10 +57,12 @@ func Parse(filename string, r io.Reader) (*Deps, error) { return nil, fmt.Errorf("%sunsupported variable expansion: %v", pos(node), x.Target.Dump()) } outputs := x.Target.Words() - if len(outputs) == 0 { - return nil, fmt.Errorf("%smissing output: %v", pos(node), x) + if len(outputs) > 0 { + ret.Output = outputs[0].Value(nil) + } else { + // TODO(b/141372861): put this back + //return nil, fmt.Errorf("%smissing output: %v", pos(node), x) } - ret.Output = outputs[0].Value(nil) if !x.Prerequisites.Const() { return nil, fmt.Errorf("%sunsupported variable expansion: %v", pos(node), x.Prerequisites.Dump()) diff --git a/makedeps/deps_test.go b/makedeps/deps_test.go index a32df650e..ac2f6996a 100644 --- a/makedeps/deps_test.go +++ b/makedeps/deps_test.go @@ -147,6 +147,20 @@ b: e`, }, }, }, + { + // TODO(b/141372861): remove this + // AIDL produces a dep file with no output file for a parcelable (b/ + name: "AIDL parcelable", + input: ` : \ + frameworks/base/tests/net/integration/src/com/android/server/net/integrationtests/HttpResponse.aidl +`, + output: Deps{ + Output: "", + Inputs: []string{ + "frameworks/base/tests/net/integration/src/com/android/server/net/integrationtests/HttpResponse.aidl", + }, + }, + }, } for _, tc := range testCases { diff --git a/partner/Android.bp b/partner/Android.bp new file mode 100644 index 000000000..f2ced8dd3 --- /dev/null +++ b/partner/Android.bp @@ -0,0 +1,49 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +// Sample project for creating an extended androidmk +// + +blueprint_go_binary { + name: "partner_androidmk", + srcs: [ + "androidmk/androidmk.go", + ], + testSrcs: [ + "androidmk/androidmk_test.go", + ], + deps: [ + "androidmk-lib", + "partner-bpfix-extensions", + ], +} + +blueprint_go_binary { + name: "partner_bpfix", + srcs: [ + "bpfix/bpfix.go", + ], + deps: [ + "bpfix-cmd", + "partner-bpfix-extensions", + ], +} + +bootstrap_go_package { + name: "partner-bpfix-extensions", + pkgPath: "android/soong/partner/bpfix/extensions", + srcs: ["bpfix/extensions/headers.go"], + deps: ["bpfix-lib"], +} diff --git a/partner/androidmk/androidmk.go b/partner/androidmk/androidmk.go new file mode 100644 index 000000000..f49981b09 --- /dev/null +++ b/partner/androidmk/androidmk.go @@ -0,0 +1,58 @@ +// Copyright 2017 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "flag" + "fmt" + "io/ioutil" + "os" + + "android/soong/androidmk/androidmk" + + _ "android/soong/partner/bpfix/extensions" +) + +var usage = func() { + fmt.Fprintf(os.Stderr, "usage: %s [flags] <inputFile>\n"+ + "\n%s parses <inputFile> as an Android.mk file and attempts to output an analogous Android.bp file (to standard out)\n", os.Args[0], os.Args[0]) + flag.PrintDefaults() + os.Exit(1) +} + +func main() { + flag.Usage = usage + flag.Parse() + if len(flag.Args()) != 1 { + usage() + } + filePathToRead := flag.Arg(0) + b, err := ioutil.ReadFile(filePathToRead) + if err != nil { + fmt.Println(err.Error()) + return + } + + output, errs := androidmk.ConvertFile(os.Args[1], bytes.NewBuffer(b)) + if len(errs) > 0 { + for _, err := range errs { + fmt.Fprintln(os.Stderr, "ERROR: ", err) + } + os.Exit(1) + } + + fmt.Print(output) +} diff --git a/partner/androidmk/androidmk_test.go b/partner/androidmk/androidmk_test.go new file mode 100644 index 000000000..6bae836d9 --- /dev/null +++ b/partner/androidmk/androidmk_test.go @@ -0,0 +1,73 @@ +// Copyright 2016 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "fmt" + "strings" + "testing" + + "android/soong/androidmk/androidmk" + "android/soong/bpfix/bpfix" + + _ "android/soong/partner/bpfix/extensions" +) + +var testCases = []struct { + desc string + in string + expected string +}{ + { + desc: "headers replacement", + in: ` +include $(CLEAR_VARS) +LOCAL_MODULE := test +LOCAL_SRC_FILES := a.c +LOCAL_C_INCLUDES := test1 $(TARGET_OUT_HEADERS)/my_headers test2 +include $(BUILD_SHARED_LIBRARY)`, + expected: ` +cc_library_shared { + name: "test", + srcs: ["a.c"], + include_dirs: [ + "test1", + + "test2", + ], + header_libs: ["my_header_lib"] +}`, + }, +} + +func TestEndToEnd(t *testing.T) { + for i, test := range testCases { + expected, err := bpfix.Reformat(test.expected) + if err != nil { + t.Error(err) + } + + got, errs := androidmk.ConvertFile(fmt.Sprintf("<testcase %d>", i), bytes.NewBufferString(test.in)) + if len(errs) > 0 { + t.Errorf("Unexpected errors: %q", errs) + continue + } + + if got != expected { + t.Errorf("failed testcase '%s'\ninput:\n%s\n\nexpected:\n%s\ngot:\n%s\n", test.desc, strings.TrimSpace(test.in), expected, got) + } + } +} diff --git a/partner/bpfix/bpfix.go b/partner/bpfix/bpfix.go new file mode 100644 index 000000000..687fe1c04 --- /dev/null +++ b/partner/bpfix/bpfix.go @@ -0,0 +1,27 @@ +// Copyright 2017 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file provides a command-line interface to bpfix + +package main + +import ( + "android/soong/bpfix/cmd_lib" + + _ "android/soong/partner/bpfix/extensions" +) + +func main() { + cmd_lib.Run() +} diff --git a/partner/bpfix/extensions/headers.go b/partner/bpfix/extensions/headers.go new file mode 100644 index 000000000..169dab667 --- /dev/null +++ b/partner/bpfix/extensions/headers.go @@ -0,0 +1,120 @@ +package extensions + +import ( + "strings" + + "github.com/google/blueprint/parser" + + "android/soong/bpfix/bpfix" +) + +var fixSteps = bpfix.FixStepsExtension{ + Name: "partner-include-dirs", + Steps: []bpfix.FixStep{ + { + Name: "fixIncludeDirs", + Fix: fixIncludeDirs, + }, + }, +} + +func init() { + bpfix.RegisterFixStepExtension(&fixSteps) +} + +type includeDirFix struct { + libName string + libType string + variable string + subdir string +} + +var commonIncludeDirs = []includeDirFix{ + { + libName: "my_header_lib", + libType: "header_libs", + variable: "TARGET_OUT_HEADERS", + subdir: "/my_headers", + }, +} + +func findHeaderLib(e parser.Expression) (*includeDirFix, bool) { + if op, ok := e.(*parser.Operator); ok { + if op.Operator != '+' { + return nil, false + } + arg0, ok := op.Args[0].(*parser.Variable) + arg1, ok1 := op.Args[1].(*parser.String) + if !ok || !ok1 { + return nil, false + } + for _, lib := range commonIncludeDirs { + if arg0.Name == lib.variable && arg1.Value == lib.subdir { + return &lib, true + } + } + } + return nil, false +} +func searchThroughOperatorList(mod *parser.Module, e parser.Expression) { + if list, ok := e.(*parser.List); ok { + newList := make([]parser.Expression, 0, len(list.Values)) + for _, item := range list.Values { + if lib, found := findHeaderLib(item); found { + if lib.libName != "" { + addLibrary(mod, lib.libType, lib.libName) + } + } else { + newList = append(newList, item) + } + } + list.Values = newList + } + if op, ok := e.(*parser.Operator); ok { + searchThroughOperatorList(mod, op.Args[0]) + searchThroughOperatorList(mod, op.Args[1]) + } +} +func getLiteralListProperty(mod *parser.Module, name string) (list *parser.List, found bool) { + prop, ok := mod.GetProperty(name) + if !ok { + return nil, false + } + list, ok = prop.Value.(*parser.List) + return list, ok +} +func addLibrary(mod *parser.Module, libType string, libName string) { + var list, ok = getLiteralListProperty(mod, libType) + if !ok { + list = new(parser.List) + prop := new(parser.Property) + prop.Name = libType + prop.Value = list + mod.Properties = append(mod.Properties, prop) + } else { + for _, v := range list.Values { + if stringValue, ok := v.(*parser.String); ok && stringValue.Value == libName { + return + } + } + } + lib := new(parser.String) + lib.Value = libName + list.Values = append(list.Values, lib) +} +func fixIncludeDirs(f *bpfix.Fixer) error { + tree := f.Tree() + for _, def := range tree.Defs { + mod, ok := def.(*parser.Module) + if !ok { + continue + } + if !strings.HasPrefix(mod.Type, "cc_") { + continue + } + if prop, ok := mod.GetProperty("include_dirs"); ok { + searchThroughOperatorList(mod, prop.Value) + } + } + return nil +} diff --git a/phony/Android.bp b/phony/Android.bp new file mode 100644 index 000000000..2c423ef75 --- /dev/null +++ b/phony/Android.bp @@ -0,0 +1,12 @@ +bootstrap_go_package { + name: "soong-phony", + pkgPath: "android/soong/phony", + deps: [ + "blueprint", + "soong-android", + ], + srcs: [ + "phony.go", + ], + pluginFor: ["soong_build"], +} diff --git a/phony/phony.go b/phony/phony.go index ed6a2fef1..305a434f4 100644 --- a/phony/phony.go +++ b/phony/phony.go @@ -28,7 +28,9 @@ func init() { type phony struct { android.ModuleBase - requiredModuleNames []string + requiredModuleNames []string + hostRequiredModuleNames []string + targetRequiredModuleNames []string } func PhonyFactory() android.Module { @@ -40,8 +42,12 @@ func PhonyFactory() android.Module { func (p *phony) GenerateAndroidBuildActions(ctx android.ModuleContext) { p.requiredModuleNames = ctx.RequiredModuleNames() - if len(p.requiredModuleNames) == 0 { - ctx.PropertyErrorf("required", "phony must not have empty required dependencies in order to be useful(and therefore permitted).") + p.hostRequiredModuleNames = ctx.HostRequiredModuleNames() + p.targetRequiredModuleNames = ctx.TargetRequiredModuleNames() + if len(p.requiredModuleNames) == 0 && + len(p.hostRequiredModuleNames) == 0 && len(p.targetRequiredModuleNames) == 0 { + ctx.PropertyErrorf("required", "phony must not have empty required dependencies "+ + "in order to be useful(and therefore permitted).") } } @@ -54,7 +60,18 @@ func (p *phony) AndroidMk() android.AndroidMkData { if p.Host() { fmt.Fprintln(w, "LOCAL_IS_HOST_MODULE := true") } - fmt.Fprintln(w, "LOCAL_REQUIRED_MODULES := "+strings.Join(p.requiredModuleNames, " ")) + if len(p.requiredModuleNames) > 0 { + fmt.Fprintln(w, "LOCAL_REQUIRED_MODULES :=", + strings.Join(p.requiredModuleNames, " ")) + } + if len(p.hostRequiredModuleNames) > 0 { + fmt.Fprintln(w, "LOCAL_HOST_REQUIRED_MODULES :=", + strings.Join(p.hostRequiredModuleNames, " ")) + } + if len(p.targetRequiredModuleNames) > 0 { + fmt.Fprintln(w, "LOCAL_TARGET_REQUIRED_MODULES :=", + strings.Join(p.targetRequiredModuleNames, " ")) + } fmt.Fprintln(w, "include $(BUILD_PHONY_PACKAGE)") }, } diff --git a/python/Android.bp b/python/Android.bp new file mode 100644 index 000000000..ffd03fe89 --- /dev/null +++ b/python/Android.bp @@ -0,0 +1,24 @@ +bootstrap_go_package { + name: "soong-python", + pkgPath: "android/soong/python", + deps: [ + "blueprint", + "soong-android", + "soong-tradefed", + ], + srcs: [ + "androidmk.go", + "binary.go", + "builder.go", + "defaults.go", + "installer.go", + "library.go", + "proto.go", + "python.go", + "test.go", + ], + testSrcs: [ + "python_test.go", + ], + pluginFor: ["soong_build"], +} diff --git a/python/androidmk.go b/python/androidmk.go index 1e51e7b8a..247b80dc0 100644 --- a/python/androidmk.go +++ b/python/androidmk.go @@ -66,15 +66,13 @@ func (p *testDecorator) AndroidMk(base *Module, ret *android.AndroidMkData) { fmt.Fprintln(w, "LOCAL_COMPATIBILITY_SUITE :=", strings.Join(p.binaryDecorator.binaryProperties.Test_suites, " ")) } - // If the test config has an explicit config specified use it. - if p.testProperties.Test_config != nil { - fmt.Fprintln(w, "LOCAL_TEST_CONFIG :=", - *p.testProperties.Test_config) - } else { - if p.testConfig != nil { - fmt.Fprintln(w, "LOCAL_FULL_TEST_CONFIG :=", - p.testConfig.String()) - } + if p.testConfig != nil { + fmt.Fprintln(w, "LOCAL_FULL_TEST_CONFIG :=", + p.testConfig.String()) + } + + if !BoolDefault(p.binaryProperties.Auto_gen_config, true) { + fmt.Fprintln(w, "LOCAL_DISABLE_AUTO_GENERATE_TEST_CONFIG := true") } }) base.subAndroidMk(ret, p.binaryDecorator.pythonInstaller) @@ -89,13 +87,13 @@ func (installer *pythonInstaller) AndroidMk(base *Module, ret *android.AndroidMk ret.Required = append(ret.Required, "libc++") ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) { - path := installer.path.RelPathString() - dir, file := filepath.Split(path) + path, file := filepath.Split(installer.path.ToMakePath().String()) stem := strings.TrimSuffix(file, filepath.Ext(file)) fmt.Fprintln(w, "LOCAL_MODULE_SUFFIX := "+filepath.Ext(file)) - fmt.Fprintln(w, "LOCAL_MODULE_PATH := $(OUT_DIR)/"+filepath.Clean(dir)) + fmt.Fprintln(w, "LOCAL_MODULE_PATH := "+path) fmt.Fprintln(w, "LOCAL_MODULE_STEM := "+stem) fmt.Fprintln(w, "LOCAL_SHARED_LIBRARIES := "+strings.Join(installer.androidMkSharedLibs, " ")) + fmt.Fprintln(w, "LOCAL_CHECK_ELF_FILES := false") }) } diff --git a/python/binary.go b/python/binary.go index 140f07af9..695fa123b 100644 --- a/python/binary.go +++ b/python/binary.go @@ -47,6 +47,11 @@ type BinaryProperties struct { // false it will act much like the normal `python` executable, but with the sources and // libraries automatically included in the PYTHONPATH. Autorun *bool `android:"arch_variant"` + + // Flag to indicate whether or not to create test config automatically. If AndroidTest.xml + // doesn't exist next to the Android.bp, this attribute doesn't need to be set to true + // explicitly. + Auto_gen_config *bool } type binaryDecorator struct { diff --git a/python/installer.go b/python/installer.go index 62f36f4b5..396f03667 100644 --- a/python/installer.go +++ b/python/installer.go @@ -33,7 +33,7 @@ type pythonInstaller struct { dir64 string relative string - path android.OutputPath + path android.InstallPath androidMkSharedLibs []string } @@ -47,12 +47,12 @@ func NewPythonInstaller(dir, dir64 string) *pythonInstaller { var _ installer = (*pythonInstaller)(nil) -func (installer *pythonInstaller) installDir(ctx android.ModuleContext) android.OutputPath { +func (installer *pythonInstaller) installDir(ctx android.ModuleContext) android.InstallPath { dir := installer.dir if ctx.Arch().ArchType.Multilib == "lib64" && installer.dir64 != "" { dir = installer.dir64 } - if !ctx.Host() && !ctx.Arch().Native { + if !ctx.Host() && ctx.Config().HasMultilibConflict(ctx.Arch().ArchType) { dir = filepath.Join(dir, ctx.Arch().ArchType.String()) } return android.PathForModuleInstall(ctx, dir, installer.relative) diff --git a/python/proto.go b/python/proto.go index 85ed1a517..b71e047a5 100644 --- a/python/proto.go +++ b/python/proto.go @@ -34,7 +34,7 @@ func genProto(ctx android.ModuleContext, protoFile android.Path, flags android.P // Proto generated python files have an unknown package name in the path, so package the entire output directory // into a srcszip. zipCmd := rule.Command(). - Tool(ctx.Config().HostToolPath(ctx, "soong_zip")). + BuiltTool(ctx, "soong_zip"). FlagWithOutput("-o ", srcsZipFile) if pkgPath != "" { zipCmd.FlagWithArg("-P ", pkgPath) diff --git a/python/python.go b/python/python.go index ad089090b..8b912be17 100644 --- a/python/python.go +++ b/python/python.go @@ -306,22 +306,17 @@ func (p *Module) DepsMutator(ctx android.BottomUpMutatorContext) { if p.bootstrapper.autorun() { launcherModule = "py2-launcher-autorun" } - ctx.AddFarVariationDependencies([]blueprint.Variation{ - {Mutator: "arch", Variation: ctx.Target().String()}, - }, launcherTag, launcherModule) + ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherTag, launcherModule) // Add py2-launcher shared lib dependencies. Ideally, these should be // derived from the `shared_libs` property of "py2-launcher". However, we // cannot read the property at this stage and it will be too late to add // dependencies later. - ctx.AddFarVariationDependencies([]blueprint.Variation{ - {Mutator: "arch", Variation: ctx.Target().String()}, - }, launcherSharedLibTag, "libsqlite") + ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherSharedLibTag, "libsqlite") if ctx.Target().Os.Bionic() { - ctx.AddFarVariationDependencies([]blueprint.Variation{ - {Mutator: "arch", Variation: ctx.Target().String()}, - }, launcherSharedLibTag, "libc", "libdl", "libm") + ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherSharedLibTag, + "libc", "libdl", "libm") } } @@ -331,9 +326,29 @@ func (p *Module) DepsMutator(ctx android.BottomUpMutatorContext) { p.properties.Version.Py3.Libs)...) if p.bootstrapper != nil && p.isEmbeddedLauncherEnabled(pyVersion3) { - //TODO(nanzhang): Add embedded launcher for Python3. - ctx.PropertyErrorf("version.py3.embedded_launcher", - "is not supported yet for Python3.") + ctx.AddVariationDependencies(nil, pythonLibTag, "py3-stdlib") + + launcherModule := "py3-launcher" + if p.bootstrapper.autorun() { + launcherModule = "py3-launcher-autorun" + } + ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherTag, launcherModule) + + // Add py3-launcher shared lib dependencies. Ideally, these should be + // derived from the `shared_libs` property of "py3-launcher". However, we + // cannot read the property at this stage and it will be too late to add + // dependencies later. + ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherSharedLibTag, "libsqlite") + + if ctx.Device() { + ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherSharedLibTag, + "liblog") + } + + if ctx.Target().Os.Bionic() { + ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherSharedLibTag, + "libc", "libdl", "libm") + } } default: panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.", @@ -375,11 +390,11 @@ func (p *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) { // Only Python binaries and test has non-empty bootstrapper. if p.bootstrapper != nil { p.walkTransitiveDeps(ctx) - // TODO(nanzhang): Since embedded launcher is not supported for Python3 for now, - // so we initialize "embedded_launcher" to false. embeddedLauncher := false if p.properties.Actual_version == pyVersion2 { embeddedLauncher = p.isEmbeddedLauncherEnabled(pyVersion2) + } else { + embeddedLauncher = p.isEmbeddedLauncherEnabled(pyVersion3) } p.installSource = p.bootstrapper.bootstrap(ctx, p.properties.Actual_version, embeddedLauncher, p.srcsPathMappings, p.srcsZip, p.depsSrcsZips) diff --git a/python/python_test.go b/python/python_test.go index e5fe126c1..1245ca184 100644 --- a/python/python_test.go +++ b/python/python_test.go @@ -28,6 +28,8 @@ import ( "android/soong/android" ) +var buildDir string + type pyModule struct { name string actualVersion string @@ -50,7 +52,7 @@ var ( noSrcFileErr = moduleVariantErrTemplate + "doesn't have any source files!" badSrcFileExtErr = moduleVariantErrTemplate + "srcs: found non (.py|.proto) file: %q!" badDataFileExtErr = moduleVariantErrTemplate + "data: found (.py|.proto) file: %q!" - bpFile = "Blueprints" + bpFile = "Android.bp" data = []struct { desc string @@ -71,7 +73,7 @@ var ( }, errors: []string{ fmt.Sprintf(noSrcFileErr, - "dir/Blueprints:1:1", "lib1", "PY3"), + "dir/Android.bp:1:1", "lib1", "PY3"), }, }, { @@ -90,7 +92,7 @@ var ( }, errors: []string{ fmt.Sprintf(badSrcFileExtErr, - "dir/Blueprints:3:11", "lib1", "PY3", "dir/file1.exe"), + "dir/Android.bp:3:11", "lib1", "PY3", "dir/file1.exe"), }, }, { @@ -113,7 +115,7 @@ var ( }, errors: []string{ fmt.Sprintf(badDataFileExtErr, - "dir/Blueprints:6:11", "lib1", "PY3", "dir/file2.py"), + "dir/Android.bp:6:11", "lib1", "PY3", "dir/file2.py"), }, }, { @@ -149,9 +151,9 @@ var ( }, errors: []string{ fmt.Sprintf(pkgPathErrTemplate, - "dir/Blueprints:11:15", "lib2", "PY3", "a/c/../../../"), + "dir/Android.bp:11:15", "lib2", "PY3", "a/c/../../../"), fmt.Sprintf(pkgPathErrTemplate, - "dir/Blueprints:19:15", "lib3", "PY3", "/a/c/../../"), + "dir/Android.bp:19:15", "lib3", "PY3", "/a/c/../../"), }, }, { @@ -174,11 +176,11 @@ var ( "dir/-e/f/file1.py": nil, }, errors: []string{ - fmt.Sprintf(badIdentifierErrTemplate, "dir/Blueprints:4:11", + fmt.Sprintf(badIdentifierErrTemplate, "dir/Android.bp:4:11", "lib1", "PY3", "a/b/c/-e/f/file1.py", "-e"), - fmt.Sprintf(badIdentifierErrTemplate, "dir/Blueprints:4:11", + fmt.Sprintf(badIdentifierErrTemplate, "dir/Android.bp:4:11", "lib1", "PY3", "a/b/c/.file1.py", ".file1"), - fmt.Sprintf(badIdentifierErrTemplate, "dir/Blueprints:4:11", + fmt.Sprintf(badIdentifierErrTemplate, "dir/Android.bp:4:11", "lib1", "PY3", "a/b/c/123/file1.py", "123"), }, }, @@ -211,7 +213,7 @@ var ( "dir/file1.py": nil, }, errors: []string{ - fmt.Sprintf(dupRunfileErrTemplate, "dir/Blueprints:9:6", + fmt.Sprintf(dupRunfileErrTemplate, "dir/Android.bp:9:6", "lib2", "PY3", "a/b/c/file1.py", "lib2", "dir/file1.py", "lib1", "dir/c/file1.py"), }, @@ -324,23 +326,18 @@ var ( ) func TestPythonModule(t *testing.T) { - config, buildDir := setupBuildEnv(t) - defer tearDownBuildEnv(buildDir) for _, d := range data { t.Run(d.desc, func(t *testing.T) { + config := android.TestConfig(buildDir, nil, "", d.mockFiles) ctx := android.NewTestContext() ctx.PreDepsMutators(func(ctx android.RegisterMutatorsContext) { ctx.BottomUp("version_split", versionSplitMutator()).Parallel() }) - ctx.RegisterModuleType("python_library_host", - android.ModuleFactoryAdaptor(PythonLibraryHostFactory)) - ctx.RegisterModuleType("python_binary_host", - android.ModuleFactoryAdaptor(PythonBinaryHostFactory)) - ctx.RegisterModuleType("python_defaults", - android.ModuleFactoryAdaptor(defaultsFactory)) + ctx.RegisterModuleType("python_library_host", PythonLibraryHostFactory) + ctx.RegisterModuleType("python_binary_host", PythonBinaryHostFactory) + ctx.RegisterModuleType("python_defaults", defaultsFactory) ctx.PreArchMutators(android.RegisterDefaultsPreArchMutators) - ctx.Register() - ctx.MockFileSystem(d.mockFiles) + ctx.Register(config) _, testErrs := ctx.ParseBlueprintsFiles(bpFile) android.FailIfErrored(t, testErrs) _, actErrs := ctx.PrepareBuildActions(config) @@ -428,17 +425,25 @@ func expectModule(t *testing.T, ctx *android.TestContext, buildDir, name, varian return } -func setupBuildEnv(t *testing.T) (config android.Config, buildDir string) { - buildDir, err := ioutil.TempDir("", buildNamePrefix) +func setUp() { + var err error + buildDir, err = ioutil.TempDir("", "soong_python_test") if err != nil { - t.Fatal(err) + panic(err) } - - config = android.TestConfig(buildDir, nil) - - return } -func tearDownBuildEnv(buildDir string) { +func tearDown() { os.RemoveAll(buildDir) } + +func TestMain(m *testing.M) { + run := func() int { + setUp() + defer tearDown() + + return m.Run() + } + + os.Exit(run()) +} diff --git a/python/test.go b/python/test.go index 55b0ab53a..a669c73a6 100644 --- a/python/test.go +++ b/python/test.go @@ -29,11 +29,11 @@ func init() { type TestProperties struct { // the name of the test configuration (for example "AndroidTest.xml") that should be // installed with the module. - Test_config *string `android:"arch_variant"` + Test_config *string `android:"path,arch_variant"` // the name of the test configuration template (for example "AndroidTestTemplate.xml") that // should be installed with the module. - Test_config_template *string `android:"arch_variant"` + Test_config_template *string `android:"path,arch_variant"` } type testDecorator struct { @@ -50,7 +50,8 @@ func (test *testDecorator) bootstrapperProps() []interface{} { func (test *testDecorator) install(ctx android.ModuleContext, file android.Path) { test.testConfig = tradefed.AutoGenPythonBinaryHostTestConfig(ctx, test.testProperties.Test_config, - test.testProperties.Test_config_template, test.binaryDecorator.binaryProperties.Test_suites) + test.testProperties.Test_config_template, test.binaryDecorator.binaryProperties.Test_suites, + test.binaryDecorator.binaryProperties.Auto_gen_config) test.binaryDecorator.pythonInstaller.dir = "nativetest" test.binaryDecorator.pythonInstaller.dir64 = "nativetest64" diff --git a/python/tests/Android.bp b/python/tests/Android.bp index 1f4305c41..c8bf42023 100644 --- a/python/tests/Android.bp +++ b/python/tests/Android.bp @@ -27,6 +27,22 @@ python_test_host { }, py3: { enabled: false, + embedded_launcher: true, + }, + }, +} + +python_test_host { + name: "par_test3", + main: "par_test.py", + srcs: [ + "par_test.py", + "testpkg/par_test.py", + ], + + version: { + py3: { + embedded_launcher: true, }, }, } diff --git a/python/tests/par_test.py b/python/tests/par_test.py index 1fafe0f82..56a5063c4 100644 --- a/python/tests/par_test.py +++ b/python/tests/par_test.py @@ -44,6 +44,13 @@ assert_equal("sys.path[0]", sys.path[0], archive) assert_equal("sys.path[1]", sys.path[1], os.path.join(archive, "internal")) assert_equal("sys.path[2]", sys.path[2], os.path.join(archive, "internal", "stdlib")) +if os.getenv('ARGTEST', False): + assert_equal("len(sys.argv)", len(sys.argv), 3) + assert_equal("sys.argv[1]", sys.argv[1], "--arg1") + assert_equal("sys.argv[2]", sys.argv[2], "arg2") +else: + assert_equal("len(sys.argv)", len(sys.argv), 1) + if failed: sys.exit(1) diff --git a/python/tests/py-cmd_test.py b/python/tests/py-cmd_test.py new file mode 100644 index 000000000..acda2d742 --- /dev/null +++ b/python/tests/py-cmd_test.py @@ -0,0 +1,78 @@ +# Copyright 2020 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import site +import sys + +# This file checks the visible python state against expected values when run +# using a prebuilt python. + +failed = False +def assert_equal(what, a, b): + global failed + if a != b: + print("Expected %s('%s') == '%s'" % (what, a, b)) + failed = True + +assert_equal("__name__", __name__, "__main__") +assert_equal("os.path.basename(__file__)", os.path.basename(__file__), "py-cmd_test.py") + +if os.getenv('ARGTEST', False): + assert_equal("len(sys.argv)", len(sys.argv), 3) + assert_equal("sys.argv[1]", sys.argv[1], "arg1") + assert_equal("sys.argv[2]", sys.argv[2], "arg2") +elif os.getenv('ARGTEST2', False): + assert_equal("len(sys.argv)", len(sys.argv), 3) + assert_equal("sys.argv[1]", sys.argv[1], "--arg1") + assert_equal("sys.argv[2]", sys.argv[2], "arg2") +else: + assert_equal("len(sys.argv)", len(sys.argv), 1) + +if os.getenv('ARGTEST_ONLY', False): + if failed: + sys.exit(1) + sys.exit(0) + +assert_equal("__package__", __package__, None) +assert_equal("sys.argv[0]", sys.argv[0], 'py-cmd_test.py') +if sys.version_info[0] == 2: + assert_equal("basename(sys.executable)", os.path.basename(sys.executable), 'py2-cmd') +else: + assert_equal("basename(sys.executable)", os.path.basename(sys.executable), 'py3-cmd') +assert_equal("sys.exec_prefix", sys.exec_prefix, sys.executable) +assert_equal("sys.prefix", sys.prefix, sys.executable) +assert_equal("site.ENABLE_USER_SITE", site.ENABLE_USER_SITE, None) + +if sys.version_info[0] == 2: + assert_equal("len(sys.path)", len(sys.path), 4) + assert_equal("sys.path[0]", sys.path[0], os.path.dirname(__file__)) + assert_equal("sys.path[1]", sys.path[1], "/extra") + assert_equal("sys.path[2]", sys.path[2], os.path.join(sys.executable, "internal")) + assert_equal("sys.path[3]", sys.path[3], os.path.join(sys.executable, "internal", "stdlib")) +else: + assert_equal("len(sys.path)", len(sys.path), 8) + assert_equal("sys.path[0]", sys.path[0], os.path.abspath(os.path.dirname(__file__))) + assert_equal("sys.path[1]", sys.path[1], "/extra") + assert_equal("sys.path[2]", sys.path[2], os.path.join(sys.executable, 'lib', 'python' + str(sys.version_info[0]) + str(sys.version_info[1]) + '.zip')) + assert_equal("sys.path[3]", sys.path[3], os.path.join(sys.executable, 'lib', 'python' + str(sys.version_info[0]) + '.' + str(sys.version_info[1]), '..')) + assert_equal("sys.path[4]", sys.path[4], os.path.join(sys.executable, 'lib', 'python' + str(sys.version_info[0]) + '.' + str(sys.version_info[1]))) + assert_equal("sys.path[5]", sys.path[5], os.path.join(sys.executable, 'lib', 'python' + str(sys.version_info[0]) + '.' + str(sys.version_info[1]), 'lib-dynload')) + assert_equal("sys.path[6]", sys.path[6], os.path.join(sys.executable, "internal")) + assert_equal("sys.path[7]", sys.path[7], os.path.join(sys.executable, "internal", "stdlib")) + +if failed: + sys.exit(1) + +import testpkg.pycmd_test diff --git a/python/tests/runtest.sh b/python/tests/runtest.sh index a319558ff..35941dc88 100755 --- a/python/tests/runtest.sh +++ b/python/tests/runtest.sh @@ -23,8 +23,11 @@ if [ -z $ANDROID_HOST_OUT ]; then exit 1 fi -if [ ! -f $ANDROID_HOST_OUT/nativetest64/par_test/par_test ]; then - echo "Run 'm par_test' first" +if [[ ( ! -f $ANDROID_HOST_OUT/nativetest64/par_test/par_test ) || + ( ! -f $ANDROID_HOST_OUT/nativetest64/par_test3/par_test3 ) || + ( ! -f $ANDROID_HOST_OUT/bin/py2-cmd ) || + ( ! -f $ANDROID_HOST_OUT/bin/py3-cmd )]]; then + echo "Run 'm par_test par_test3 py2-cmd py3-cmd' first" exit 1 fi @@ -36,4 +39,23 @@ PYTHONHOME= PYTHONPATH= $ANDROID_HOST_OUT/nativetest64/par_test/par_test PYTHONHOME=/usr $ANDROID_HOST_OUT/nativetest64/par_test/par_test PYTHONPATH=/usr $ANDROID_HOST_OUT/nativetest64/par_test/par_test +ARGTEST=true $ANDROID_HOST_OUT/nativetest64/par_test/par_test --arg1 arg2 + +PYTHONHOME= PYTHONPATH= $ANDROID_HOST_OUT/nativetest64/par_test3/par_test3 +PYTHONHOME=/usr $ANDROID_HOST_OUT/nativetest64/par_test3/par_test3 +PYTHONPATH=/usr $ANDROID_HOST_OUT/nativetest64/par_test3/par_test3 + +ARGTEST=true $ANDROID_HOST_OUT/nativetest64/par_test3/par_test3 --arg1 arg2 + +cd $(dirname ${BASH_SOURCE[0]}) + +PYTHONPATH=/extra $ANDROID_HOST_OUT/bin/py2-cmd py-cmd_test.py +PYTHONPATH=/extra $ANDROID_HOST_OUT/bin/py3-cmd py-cmd_test.py + +ARGTEST=true PYTHONPATH=/extra $ANDROID_HOST_OUT/bin/py2-cmd py-cmd_test.py arg1 arg2 +ARGTEST2=true PYTHONPATH=/extra $ANDROID_HOST_OUT/bin/py2-cmd py-cmd_test.py --arg1 arg2 + +ARGTEST=true PYTHONPATH=/extra $ANDROID_HOST_OUT/bin/py3-cmd py-cmd_test.py arg1 arg2 +ARGTEST2=true PYTHONPATH=/extra $ANDROID_HOST_OUT/bin/py3-cmd py-cmd_test.py --arg1 arg2 + echo "Passed!" diff --git a/python/tests/testpkg/__init__.py b/python/tests/testpkg/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/python/tests/testpkg/__init__.py diff --git a/python/tests/testpkg/par_test.py b/python/tests/testpkg/par_test.py index 22dd09564..ffad430e4 100644 --- a/python/tests/testpkg/par_test.py +++ b/python/tests/testpkg/par_test.py @@ -29,7 +29,13 @@ archive = sys.modules["__main__"].__loader__.archive assert_equal("__name__", __name__, "testpkg.par_test") assert_equal("__file__", __file__, os.path.join(archive, "testpkg/par_test.py")) -assert_equal("__package__", __package__, "testpkg") + +# Python3 is returning None here for me, and I haven't found any problems caused by this. +if sys.version_info[0] == 2: + assert_equal("__package__", __package__, "testpkg") +else: + assert_equal("__package__", __package__, None) + assert_equal("__loader__.archive", __loader__.archive, archive) assert_equal("__loader__.prefix", __loader__.prefix, "testpkg/") diff --git a/python/tests/testpkg/pycmd_test.py b/python/tests/testpkg/pycmd_test.py new file mode 100644 index 000000000..6b8a26301 --- /dev/null +++ b/python/tests/testpkg/pycmd_test.py @@ -0,0 +1,33 @@ +# Copyright 2018 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +# This file checks the visible python state against expected values when run +# via the py*-cmd prebuilts + +failed = False +def assert_equal(what, a, b): + global failed + if a != b: + print("Expected %s('%s') == '%s'" % (what, a, b)) + failed = True + +assert_equal("__name__", __name__, "testpkg.pycmd_test") +assert_equal("basename(__file__)", os.path.basename(__file__), "pycmd_test.py") +assert_equal("__package__", __package__, "testpkg") + +if failed: + sys.exit(1) diff --git a/remoteexec/Android.bp b/remoteexec/Android.bp new file mode 100644 index 000000000..fc2c0e324 --- /dev/null +++ b/remoteexec/Android.bp @@ -0,0 +1,15 @@ +bootstrap_go_package { + name: "soong-remoteexec", + pkgPath: "android/soong/remoteexec", + deps: [ + "blueprint", + "soong-android", + ], + srcs: [ + "remoteexec.go", + ], + testSrcs: [ + "remoteexec_test.go", + ], + pluginFor: ["soong_build"], +} diff --git a/remoteexec/remoteexec.go b/remoteexec/remoteexec.go index 860db5568..d6e2c0a75 100644 --- a/remoteexec/remoteexec.go +++ b/remoteexec/remoteexec.go @@ -75,6 +75,9 @@ type REParams struct { // OutputFiles is a list of output file paths or ninja variables as placeholders for rule // outputs. OutputFiles []string + // OutputDirectories is a list of output directories or ninja variables as placeholders for + // rule output directories. + OutputDirectories []string // ToolchainInputs is a list of paths or ninja variables pointing to the location of // toolchain binaries used by the rule. ToolchainInputs []string @@ -82,17 +85,31 @@ type REParams struct { func init() { pctx.VariableFunc("Wrapper", func(ctx android.PackageVarContext) string { - if override := ctx.Config().Getenv("RBE_WRAPPER"); override != "" { - return override - } - return DefaultWrapperPath + return wrapper(ctx.Config()) }) } -// Generate the remote execution wrapper template to be added as a prefix to the rule's command. +func wrapper(cfg android.Config) string { + if override := cfg.Getenv("RBE_WRAPPER"); override != "" { + return override + } + return DefaultWrapperPath +} + +// Template generates the remote execution wrapper template to be added as a prefix to the rule's +// command. func (r *REParams) Template() string { - template := "${remoteexec.Wrapper}" + return "${remoteexec.Wrapper}" + r.wrapperArgs() +} + +// NoVarTemplate generates the remote execution wrapper template without variables, to be used in +// RuleBuilder. +func (r *REParams) NoVarTemplate(cfg android.Config) string { + return wrapper(cfg) + r.wrapperArgs() +} +func (r *REParams) wrapperArgs() string { + args := "" var kvs []string labels := r.Labels if len(labels) == 0 { @@ -102,7 +119,7 @@ func (r *REParams) Template() string { kvs = append(kvs, k+"="+v) } sort.Strings(kvs) - template += " --labels=" + strings.Join(kvs, ",") + args += " --labels=" + strings.Join(kvs, ",") var platform []string for k, v := range r.Platform { @@ -116,36 +133,42 @@ func (r *REParams) Template() string { } if platform != nil { sort.Strings(platform) - template += " --platform=\"" + strings.Join(platform, ",") + "\"" + args += " --platform=\"" + strings.Join(platform, ",") + "\"" } strategy := r.ExecStrategy if strategy == "" { strategy = defaultExecStrategy } - template += " --exec_strategy=" + strategy + args += " --exec_strategy=" + strategy if len(r.Inputs) > 0 { - template += " --inputs=" + strings.Join(r.Inputs, ",") + args += " --inputs=" + strings.Join(r.Inputs, ",") } if r.RSPFile != "" { - template += " --input_list_paths=" + r.RSPFile + args += " --input_list_paths=" + r.RSPFile } if len(r.OutputFiles) > 0 { - template += " --output_files=" + strings.Join(r.OutputFiles, ",") + args += " --output_files=" + strings.Join(r.OutputFiles, ",") + } + + if len(r.OutputDirectories) > 0 { + args += " --output_directories=" + strings.Join(r.OutputDirectories, ",") } if len(r.ToolchainInputs) > 0 { - template += " --toolchain_inputs=" + strings.Join(r.ToolchainInputs, ",") + args += " --toolchain_inputs=" + strings.Join(r.ToolchainInputs, ",") } - return template + " -- " + return args + " -- " } // StaticRules returns a pair of rules based on the given RuleParams, where the first rule is a -// locally executable rule and the second rule is a remotely executable rule. +// locally executable rule and the second rule is a remotely executable rule. commonArgs are args +// used for both the local and remotely executable rules. reArgs are used only for remote +// execution. func StaticRules(ctx android.PackageContext, name string, ruleParams blueprint.RuleParams, reParams *REParams, commonArgs []string, reArgs []string) (blueprint.Rule, blueprint.Rule) { ruleParamsRE := ruleParams ruleParams.Command = strings.ReplaceAll(ruleParams.Command, "$reTemplate", "") @@ -155,6 +178,22 @@ func StaticRules(ctx android.PackageContext, name string, ruleParams blueprint.R ctx.AndroidRemoteStaticRule(name+"RE", android.RemoteRuleSupports{RBE: true}, ruleParamsRE, append(commonArgs, reArgs...)...) } +// MultiCommandStaticRules returns a pair of rules based on the given RuleParams, where the first +// rule is a locally executable rule and the second rule is a remotely executable rule. This +// function supports multiple remote execution wrappers placed in the template when commands are +// chained together with &&. commonArgs are args used for both the local and remotely executable +// rules. reArgs are args used only for remote execution. +func MultiCommandStaticRules(ctx android.PackageContext, name string, ruleParams blueprint.RuleParams, reParams map[string]*REParams, commonArgs []string, reArgs []string) (blueprint.Rule, blueprint.Rule) { + ruleParamsRE := ruleParams + for k, v := range reParams { + ruleParams.Command = strings.ReplaceAll(ruleParams.Command, k, "") + ruleParamsRE.Command = strings.ReplaceAll(ruleParamsRE.Command, k, v.Template()) + } + + return ctx.AndroidStaticRule(name, ruleParams, commonArgs...), + ctx.AndroidRemoteStaticRule(name+"RE", android.RemoteRuleSupports{RBE: true}, ruleParamsRE, append(commonArgs, reArgs...)...) +} + // EnvOverrideFunc retrieves a variable func that evaluates to the value of the given environment // variable if set, otherwise the given default. func EnvOverrideFunc(envVar, defaultVal string) func(ctx android.PackageVarContext) string { diff --git a/remoteexec/remoteexec_test.go b/remoteexec/remoteexec_test.go index 30e891ced..56985d356 100644 --- a/remoteexec/remoteexec_test.go +++ b/remoteexec/remoteexec_test.go @@ -17,6 +17,8 @@ package remoteexec import ( "fmt" "testing" + + "android/soong/android" ) func TestTemplate(t *testing.T) { @@ -64,6 +66,22 @@ func TestTemplate(t *testing.T) { } } +func TestNoVarTemplate(t *testing.T) { + params := &REParams{ + Labels: map[string]string{"type": "compile", "lang": "cpp", "compiler": "clang"}, + Inputs: []string{"$in"}, + OutputFiles: []string{"$out"}, + Platform: map[string]string{ + ContainerImageKey: DefaultImage, + PoolKey: "default", + }, + } + want := fmt.Sprintf("prebuilts/remoteexecution-client/live/rewrapper --labels=compiler=clang,lang=cpp,type=compile --platform=\"Pool=default,container-image=%s\" --exec_strategy=local --inputs=$in --output_files=$out -- ", DefaultImage) + if got := params.NoVarTemplate(android.NullConfig("")); got != want { + t.Errorf("NoVarTemplate() returned\n%s\nwant\n%s", got, want) + } +} + func TestTemplateDeterminism(t *testing.T) { r := &REParams{ Labels: map[string]string{"type": "compile", "lang": "cpp", "compiler": "clang"}, diff --git a/rust/Android.bp b/rust/Android.bp new file mode 100644 index 000000000..24fd83015 --- /dev/null +++ b/rust/Android.bp @@ -0,0 +1,30 @@ +bootstrap_go_package { + name: "soong-rust", + pkgPath: "android/soong/rust", + deps: [ + "soong", + "soong-android", + "soong-cc", + "soong-rust-config", + ], + srcs: [ + "androidmk.go", + "compiler.go", + "binary.go", + "builder.go", + "library.go", + "prebuilt.go", + "proc_macro.go", + "rust.go", + "test.go", + "testing.go", + ], + testSrcs: [ + "binary_test.go", + "compiler_test.go", + "library_test.go", + "rust_test.go", + "test_test.go", + ], + pluginFor: ["soong_build"], +} diff --git a/rust/OWNERS b/rust/OWNERS new file mode 100644 index 000000000..afd06e422 --- /dev/null +++ b/rust/OWNERS @@ -0,0 +1,5 @@ +# Additional owner/reviewers for rust rules, including parent directory owners. +per-file * = chh@google.com, ivanlozano@google.com, jeffv@google.com, srhines@google.com + +# Limited owners/reviewers of the allowed list. +per-file allowed_list.go = chh@google.com, ivanlozano@google.com, jeffv@google.com, jgalenson@google.com, srhines@google.com diff --git a/rust/androidmk.go b/rust/androidmk.go new file mode 100644 index 000000000..0fba739a2 --- /dev/null +++ b/rust/androidmk.go @@ -0,0 +1,151 @@ +// Copyright 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rust + +import ( + "fmt" + "io" + "path/filepath" + "strings" + + "android/soong/android" +) + +type AndroidMkContext interface { + Name() string + Target() android.Target + subAndroidMk(*android.AndroidMkData, interface{}) +} + +type subAndroidMkProvider interface { + AndroidMk(AndroidMkContext, *android.AndroidMkData) +} + +func (mod *Module) subAndroidMk(data *android.AndroidMkData, obj interface{}) { + if mod.subAndroidMkOnce == nil { + mod.subAndroidMkOnce = make(map[subAndroidMkProvider]bool) + } + if androidmk, ok := obj.(subAndroidMkProvider); ok { + if !mod.subAndroidMkOnce[androidmk] { + mod.subAndroidMkOnce[androidmk] = true + androidmk.AndroidMk(mod, data) + } + } +} + +func (mod *Module) AndroidMk() android.AndroidMkData { + ret := android.AndroidMkData{ + OutputFile: mod.outputFile, + Include: "$(BUILD_SYSTEM)/soong_rust_prebuilt.mk", + Extra: []android.AndroidMkExtraFunc{ + func(w io.Writer, outputFile android.Path) { + if len(mod.Properties.AndroidMkRlibs) > 0 { + fmt.Fprintln(w, "LOCAL_RLIB_LIBRARIES := "+strings.Join(mod.Properties.AndroidMkRlibs, " ")) + } + if len(mod.Properties.AndroidMkDylibs) > 0 { + fmt.Fprintln(w, "LOCAL_DYLIB_LIBRARIES := "+strings.Join(mod.Properties.AndroidMkDylibs, " ")) + } + if len(mod.Properties.AndroidMkProcMacroLibs) > 0 { + fmt.Fprintln(w, "LOCAL_PROC_MACRO_LIBRARIES := "+strings.Join(mod.Properties.AndroidMkProcMacroLibs, " ")) + } + if len(mod.Properties.AndroidMkSharedLibs) > 0 { + fmt.Fprintln(w, "LOCAL_SHARED_LIBRARIES := "+strings.Join(mod.Properties.AndroidMkSharedLibs, " ")) + } + if len(mod.Properties.AndroidMkStaticLibs) > 0 { + fmt.Fprintln(w, "LOCAL_STATIC_LIBRARIES := "+strings.Join(mod.Properties.AndroidMkStaticLibs, " ")) + } + }, + }, + } + + mod.subAndroidMk(&ret, mod.compiler) + + ret.SubName += mod.Properties.SubName + + return ret +} + +func (binary *binaryDecorator) AndroidMk(ctx AndroidMkContext, ret *android.AndroidMkData) { + ctx.subAndroidMk(ret, binary.baseCompiler) + + ret.Class = "EXECUTABLES" + ret.DistFile = binary.distFile + ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) { + fmt.Fprintln(w, "LOCAL_SOONG_UNSTRIPPED_BINARY :=", binary.unstrippedOutputFile.String()) + }) +} + +func (test *testDecorator) AndroidMk(ctx AndroidMkContext, ret *android.AndroidMkData) { + test.binaryDecorator.AndroidMk(ctx, ret) + ret.Class = "NATIVE_TESTS" + ret.SubName = test.getMutatedModuleSubName(ctx.Name()) + ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) { + if len(test.Properties.Test_suites) > 0 { + fmt.Fprintln(w, "LOCAL_COMPATIBILITY_SUITE :=", + strings.Join(test.Properties.Test_suites, " ")) + } + if test.testConfig != nil { + fmt.Fprintln(w, "LOCAL_FULL_TEST_CONFIG :=", test.testConfig.String()) + } + if !BoolDefault(test.Properties.Auto_gen_config, true) { + fmt.Fprintln(w, "LOCAL_DISABLE_AUTO_GENERATE_TEST_CONFIG := true") + } + }) + // TODO(chh): add test data with androidMkWriteTestData(test.data, ctx, ret) +} + +func (library *libraryDecorator) AndroidMk(ctx AndroidMkContext, ret *android.AndroidMkData) { + ctx.subAndroidMk(ret, library.baseCompiler) + + if library.rlib() { + ret.Class = "RLIB_LIBRARIES" + } else if library.dylib() { + ret.Class = "DYLIB_LIBRARIES" + } else if library.static() { + ret.Class = "STATIC_LIBRARIES" + } else if library.shared() { + ret.Class = "SHARED_LIBRARIES" + } + + ret.DistFile = library.distFile + ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) { + if !library.rlib() { + fmt.Fprintln(w, "LOCAL_SOONG_UNSTRIPPED_BINARY :=", library.unstrippedOutputFile.String()) + } + }) +} + +func (procMacro *procMacroDecorator) AndroidMk(ctx AndroidMkContext, ret *android.AndroidMkData) { + ctx.subAndroidMk(ret, procMacro.baseCompiler) + + ret.Class = "PROC_MACRO_LIBRARIES" + ret.DistFile = procMacro.distFile + +} + +func (compiler *baseCompiler) AndroidMk(ctx AndroidMkContext, ret *android.AndroidMkData) { + // Soong installation is only supported for host modules. Have Make + // installation trigger Soong installation. + if ctx.Target().Os.Class == android.Host { + ret.OutputFile = android.OptionalPathForPath(compiler.path) + } + ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) { + path, file := filepath.Split(compiler.path.ToMakePath().String()) + stem, suffix, _ := android.SplitFileExt(file) + fmt.Fprintln(w, "LOCAL_MODULE_SUFFIX := "+suffix) + fmt.Fprintln(w, "LOCAL_MODULE_PATH := "+path) + fmt.Fprintln(w, "LOCAL_MODULE_STEM := "+stem) + }) +} diff --git a/rust/binary.go b/rust/binary.go new file mode 100644 index 000000000..fda056e43 --- /dev/null +++ b/rust/binary.go @@ -0,0 +1,120 @@ +// Copyright 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rust + +import ( + "android/soong/android" +) + +func init() { + android.RegisterModuleType("rust_binary", RustBinaryFactory) + android.RegisterModuleType("rust_binary_host", RustBinaryHostFactory) +} + +type BinaryCompilerProperties struct { + // path to the main source file that contains the program entry point (e.g. src/main.rs) + Srcs []string `android:"path,arch_variant"` + + // passes -C prefer-dynamic to rustc, which tells it to dynamically link the stdlib + // (assuming it has no dylib dependencies already) + Prefer_dynamic *bool +} + +type binaryDecorator struct { + *baseCompiler + + Properties BinaryCompilerProperties + distFile android.OptionalPath + unstrippedOutputFile android.Path +} + +var _ compiler = (*binaryDecorator)(nil) + +// rust_binary produces a binary that is runnable on a device. +func RustBinaryFactory() android.Module { + module, _ := NewRustBinary(android.HostAndDeviceSupported) + return module.Init() +} + +func RustBinaryHostFactory() android.Module { + module, _ := NewRustBinary(android.HostSupported) + return module.Init() +} + +func NewRustBinary(hod android.HostOrDeviceSupported) (*Module, *binaryDecorator) { + module := newModule(hod, android.MultilibFirst) + + binary := &binaryDecorator{ + baseCompiler: NewBaseCompiler("bin", "", InstallInSystem), + } + + module.compiler = binary + + return module, binary +} + +func (binary *binaryDecorator) preferDynamic() bool { + return Bool(binary.Properties.Prefer_dynamic) +} + +func (binary *binaryDecorator) compilerFlags(ctx ModuleContext, flags Flags) Flags { + flags = binary.baseCompiler.compilerFlags(ctx, flags) + + if ctx.toolchain().Bionic() { + // no-undefined-version breaks dylib compilation since __rust_*alloc* functions aren't defined, + // but we can apply this to binaries. + flags.LinkFlags = append(flags.LinkFlags, + "-Wl,--gc-sections", + "-Wl,-z,nocopyreloc", + "-Wl,--no-undefined-version") + } + + if binary.preferDynamic() { + flags.RustFlags = append(flags.RustFlags, "-C prefer-dynamic") + } + return flags +} + +func (binary *binaryDecorator) compilerDeps(ctx DepsContext, deps Deps) Deps { + deps = binary.baseCompiler.compilerDeps(ctx, deps) + + if ctx.toolchain().Bionic() { + deps = binary.baseCompiler.bionicDeps(ctx, deps) + deps.CrtBegin = "crtbegin_dynamic" + deps.CrtEnd = "crtend_android" + } + + return deps +} + +func (binary *binaryDecorator) compilerProps() []interface{} { + return append(binary.baseCompiler.compilerProps(), + &binary.Properties) +} + +func (binary *binaryDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Path { + fileName := binary.getStem(ctx) + ctx.toolchain().ExecutableSuffix() + + srcPath := srcPathFromModuleSrcs(ctx, binary.Properties.Srcs) + + outputFile := android.PathForModuleOut(ctx, fileName) + binary.unstrippedOutputFile = outputFile + + flags.RustFlags = append(flags.RustFlags, deps.depFlags...) + + TransformSrcToBinary(ctx, srcPath, deps, flags, outputFile, deps.linkDirs) + + return outputFile +} diff --git a/rust/binary_test.go b/rust/binary_test.go new file mode 100644 index 000000000..ab2dae153 --- /dev/null +++ b/rust/binary_test.go @@ -0,0 +1,55 @@ +// Copyright 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rust + +import ( + "strings" + "testing" +) + +// Test that the prefer_dynamic property is handled correctly. +func TestPreferDynamicBinary(t *testing.T) { + ctx := testRust(t, ` + rust_binary_host { + name: "fizz-buzz-dynamic", + srcs: ["foo.rs"], + prefer_dynamic: true, + } + + rust_binary_host { + name: "fizz-buzz", + srcs: ["foo.rs"], + }`) + + fizzBuzz := ctx.ModuleForTests("fizz-buzz", "linux_glibc_x86_64").Output("fizz-buzz") + fizzBuzzDynamic := ctx.ModuleForTests("fizz-buzz-dynamic", "linux_glibc_x86_64").Output("fizz-buzz-dynamic") + + // Do not compile binary modules with the --test flag. + flags := fizzBuzzDynamic.Args["rustcFlags"] + if strings.Contains(flags, "--test") { + t.Errorf("extra --test flag, rustcFlags: %#v", flags) + } + if !strings.Contains(flags, "prefer-dynamic") { + t.Errorf("missing prefer-dynamic flag, rustcFlags: %#v", flags) + } + + flags = fizzBuzz.Args["rustcFlags"] + if strings.Contains(flags, "--test") { + t.Errorf("extra --test flag, rustcFlags: %#v", flags) + } + if strings.Contains(flags, "prefer-dynamic") { + t.Errorf("unexpected prefer-dynamic flag, rustcFlags: %#v", flags) + } +} diff --git a/rust/builder.go b/rust/builder.go new file mode 100644 index 000000000..27eeec23d --- /dev/null +++ b/rust/builder.go @@ -0,0 +1,159 @@ +// Copyright 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rust + +import ( + "strings" + + "github.com/google/blueprint" + + "android/soong/android" +) + +var ( + _ = pctx.SourcePathVariable("rustcCmd", "${config.RustBin}/rustc") + rustc = pctx.AndroidStaticRule("rustc", + blueprint.RuleParams{ + Command: "$rustcCmd " + + "-C linker=${config.RustLinker} " + + "-C link-args=\"${crtBegin} ${config.RustLinkerArgs} ${linkFlags} ${crtEnd}\" " + + "--emit link -o $out --emit dep-info=$out.d $in ${libFlags} $rustcFlags", + CommandDeps: []string{"$rustcCmd"}, + // Rustc deps-info writes out make compatible dep files: https://github.com/rust-lang/rust/issues/7633 + Deps: blueprint.DepsGCC, + Depfile: "$out.d", + }, + "rustcFlags", "linkFlags", "libFlags", "crtBegin", "crtEnd") +) + +func init() { + +} + +func TransformSrcToBinary(ctx android.ModuleContext, mainSrc android.Path, deps PathDeps, flags Flags, + outputFile android.WritablePath, includeDirs []string) { + flags.RustFlags = append(flags.RustFlags, "-C lto") + + transformSrctoCrate(ctx, mainSrc, deps, flags, outputFile, "bin", includeDirs) +} + +func TransformSrctoRlib(ctx android.ModuleContext, mainSrc android.Path, deps PathDeps, flags Flags, + outputFile android.WritablePath, includeDirs []string) { + transformSrctoCrate(ctx, mainSrc, deps, flags, outputFile, "rlib", includeDirs) +} + +func TransformSrctoDylib(ctx android.ModuleContext, mainSrc android.Path, deps PathDeps, flags Flags, + outputFile android.WritablePath, includeDirs []string) { + transformSrctoCrate(ctx, mainSrc, deps, flags, outputFile, "dylib", includeDirs) +} + +func TransformSrctoStatic(ctx android.ModuleContext, mainSrc android.Path, deps PathDeps, flags Flags, + outputFile android.WritablePath, includeDirs []string) { + flags.RustFlags = append(flags.RustFlags, "-C lto") + transformSrctoCrate(ctx, mainSrc, deps, flags, outputFile, "staticlib", includeDirs) +} + +func TransformSrctoShared(ctx android.ModuleContext, mainSrc android.Path, deps PathDeps, flags Flags, + outputFile android.WritablePath, includeDirs []string) { + flags.RustFlags = append(flags.RustFlags, "-C lto") + transformSrctoCrate(ctx, mainSrc, deps, flags, outputFile, "cdylib", includeDirs) +} + +func TransformSrctoProcMacro(ctx android.ModuleContext, mainSrc android.Path, deps PathDeps, + flags Flags, outputFile android.WritablePath, includeDirs []string) { + transformSrctoCrate(ctx, mainSrc, deps, flags, outputFile, "proc-macro", includeDirs) +} + +func rustLibsToPaths(libs RustLibraries) android.Paths { + var paths android.Paths + for _, lib := range libs { + paths = append(paths, lib.Path) + } + return paths +} + +func transformSrctoCrate(ctx android.ModuleContext, main android.Path, deps PathDeps, flags Flags, + outputFile android.WritablePath, crate_type string, includeDirs []string) { + + var inputs android.Paths + var implicits android.Paths + var libFlags, rustcFlags, linkFlags []string + crate_name := ctx.(ModuleContext).CrateName() + targetTriple := ctx.(ModuleContext).toolchain().RustTriple() + + inputs = append(inputs, main) + + // Collect rustc flags + rustcFlags = append(rustcFlags, flags.GlobalRustFlags...) + rustcFlags = append(rustcFlags, flags.RustFlags...) + rustcFlags = append(rustcFlags, "--crate-type="+crate_type) + if crate_name != "" { + rustcFlags = append(rustcFlags, "--crate-name="+crate_name) + } + if targetTriple != "" { + rustcFlags = append(rustcFlags, "--target="+targetTriple) + linkFlags = append(linkFlags, "-target "+targetTriple) + } + // TODO once we have static libraries in the host prebuilt .bp, this + // should be unconditionally added. + if !ctx.Host() { + // If we're on a device build, do not use an implicit sysroot + rustcFlags = append(rustcFlags, "--sysroot=/dev/null") + } + // Collect linker flags + linkFlags = append(linkFlags, flags.GlobalLinkFlags...) + linkFlags = append(linkFlags, flags.LinkFlags...) + + // Collect library/crate flags + for _, lib := range deps.RLibs { + libFlags = append(libFlags, "--extern "+lib.CrateName+"="+lib.Path.String()) + } + for _, lib := range deps.DyLibs { + libFlags = append(libFlags, "--extern "+lib.CrateName+"="+lib.Path.String()) + } + for _, proc_macro := range deps.ProcMacros { + libFlags = append(libFlags, "--extern "+proc_macro.CrateName+"="+proc_macro.Path.String()) + } + + for _, path := range includeDirs { + libFlags = append(libFlags, "-L "+path) + } + + // Collect dependencies + implicits = append(implicits, rustLibsToPaths(deps.RLibs)...) + implicits = append(implicits, rustLibsToPaths(deps.DyLibs)...) + implicits = append(implicits, rustLibsToPaths(deps.ProcMacros)...) + implicits = append(implicits, deps.StaticLibs...) + implicits = append(implicits, deps.SharedLibs...) + if deps.CrtBegin.Valid() { + implicits = append(implicits, deps.CrtBegin.Path(), deps.CrtEnd.Path()) + } + + ctx.Build(pctx, android.BuildParams{ + Rule: rustc, + Description: "rustc " + main.Rel(), + Output: outputFile, + Inputs: inputs, + Implicits: implicits, + Args: map[string]string{ + "rustcFlags": strings.Join(rustcFlags, " "), + "linkFlags": strings.Join(linkFlags, " "), + "libFlags": strings.Join(libFlags, " "), + "crtBegin": deps.CrtBegin.String(), + "crtEnd": deps.CrtEnd.String(), + }, + }) + +} diff --git a/rust/compiler.go b/rust/compiler.go new file mode 100644 index 000000000..4593165f4 --- /dev/null +++ b/rust/compiler.go @@ -0,0 +1,259 @@ +// Copyright 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rust + +import ( + "fmt" + "path/filepath" + + "github.com/google/blueprint/proptools" + + "android/soong/android" + "android/soong/rust/config" +) + +func getEdition(compiler *baseCompiler) string { + return proptools.StringDefault(compiler.Properties.Edition, config.DefaultEdition) +} + +func getDenyWarnings(compiler *baseCompiler) bool { + return BoolDefault(compiler.Properties.Deny_warnings, config.DefaultDenyWarnings) +} + +func (compiler *baseCompiler) setNoStdlibs() { + compiler.Properties.No_stdlibs = proptools.BoolPtr(true) +} + +func NewBaseCompiler(dir, dir64 string, location installLocation) *baseCompiler { + return &baseCompiler{ + Properties: BaseCompilerProperties{}, + dir: dir, + dir64: dir64, + location: location, + } +} + +type installLocation int + +const ( + InstallInSystem installLocation = 0 + InstallInData = iota +) + +type BaseCompilerProperties struct { + // whether to pass "-D warnings" to rustc. Defaults to true. + Deny_warnings *bool + + // flags to pass to rustc + Flags []string `android:"path,arch_variant"` + + // flags to pass to the linker + Ld_flags []string `android:"path,arch_variant"` + + // list of rust rlib crate dependencies + Rlibs []string `android:"arch_variant"` + + // list of rust dylib crate dependencies + Dylibs []string `android:"arch_variant"` + + // list of rust proc_macro crate dependencies + Proc_macros []string `android:"arch_variant"` + + // list of C shared library dependencies + Shared_libs []string `android:"arch_variant"` + + // list of C static library dependencies + Static_libs []string `android:"arch_variant"` + + // crate name, required for libraries. This must be the expected extern crate name used in source + Crate_name string `android:"arch_variant"` + + // list of features to enable for this crate + Features []string `android:"arch_variant"` + + // specific rust edition that should be used if the default version is not desired + Edition *string `android:"arch_variant"` + + // sets name of the output + Stem *string `android:"arch_variant"` + + // append to name of output + Suffix *string `android:"arch_variant"` + + // install to a subdirectory of the default install path for the module + Relative_install_path *string `android:"arch_variant"` + + // whether to suppress inclusion of standard crates - defaults to false + No_stdlibs *bool +} + +type baseCompiler struct { + Properties BaseCompilerProperties + pathDeps android.Paths + rustFlagsDeps android.Paths + linkFlagsDeps android.Paths + flags string + linkFlags string + depFlags []string + linkDirs []string + edition string + src android.Path //rustc takes a single src file + + // Install related + dir string + dir64 string + subDir string + relative string + path android.InstallPath + location installLocation +} + +var _ compiler = (*baseCompiler)(nil) + +func (compiler *baseCompiler) inData() bool { + return compiler.location == InstallInData +} + +func (compiler *baseCompiler) compilerProps() []interface{} { + return []interface{}{&compiler.Properties} +} + +func (compiler *baseCompiler) featuresToFlags(features []string) []string { + flags := []string{} + for _, feature := range features { + flags = append(flags, "--cfg 'feature=\""+feature+"\"'") + } + return flags +} + +func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags) Flags { + + if getDenyWarnings(compiler) { + flags.RustFlags = append(flags.RustFlags, "-D warnings") + } + flags.RustFlags = append(flags.RustFlags, compiler.Properties.Flags...) + flags.RustFlags = append(flags.RustFlags, compiler.featuresToFlags(compiler.Properties.Features)...) + flags.RustFlags = append(flags.RustFlags, "--edition="+getEdition(compiler)) + flags.LinkFlags = append(flags.LinkFlags, compiler.Properties.Ld_flags...) + flags.GlobalRustFlags = append(flags.GlobalRustFlags, config.GlobalRustFlags...) + flags.GlobalRustFlags = append(flags.GlobalRustFlags, ctx.toolchain().ToolchainRustFlags()) + flags.GlobalLinkFlags = append(flags.GlobalLinkFlags, ctx.toolchain().ToolchainLinkFlags()) + + if ctx.Host() && !ctx.Windows() { + rpath_prefix := `\$$ORIGIN/` + if ctx.Darwin() { + rpath_prefix = "@loader_path/" + } + + var rpath string + if ctx.toolchain().Is64Bit() { + rpath = "lib64" + } else { + rpath = "lib" + } + flags.LinkFlags = append(flags.LinkFlags, "-Wl,-rpath,"+rpath_prefix+rpath) + flags.LinkFlags = append(flags.LinkFlags, "-Wl,-rpath,"+rpath_prefix+"../"+rpath) + } + + return flags +} + +func (compiler *baseCompiler) compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Path { + panic(fmt.Errorf("baseCrater doesn't know how to crate things!")) +} + +func (compiler *baseCompiler) compilerDeps(ctx DepsContext, deps Deps) Deps { + deps.Rlibs = append(deps.Rlibs, compiler.Properties.Rlibs...) + deps.Dylibs = append(deps.Dylibs, compiler.Properties.Dylibs...) + deps.ProcMacros = append(deps.ProcMacros, compiler.Properties.Proc_macros...) + deps.StaticLibs = append(deps.StaticLibs, compiler.Properties.Static_libs...) + deps.SharedLibs = append(deps.SharedLibs, compiler.Properties.Shared_libs...) + + if !Bool(compiler.Properties.No_stdlibs) { + for _, stdlib := range config.Stdlibs { + // If we're building for host, use the compiler's stdlibs + if ctx.Host() { + stdlib = stdlib + "_" + ctx.toolchain().RustTriple() + } + + // This check is technically insufficient - on the host, where + // static linking is the default, if one of our static + // dependencies uses a dynamic library, we need to dynamically + // link the stdlib as well. + if (len(deps.Dylibs) > 0) || (!ctx.Host()) { + // Dynamically linked stdlib + deps.Dylibs = append(deps.Dylibs, stdlib) + } + } + } + return deps +} + +func (compiler *baseCompiler) bionicDeps(ctx DepsContext, deps Deps) Deps { + deps.SharedLibs = append(deps.SharedLibs, "liblog") + deps.SharedLibs = append(deps.SharedLibs, "libc") + deps.SharedLibs = append(deps.SharedLibs, "libm") + deps.SharedLibs = append(deps.SharedLibs, "libdl") + + //TODO(b/141331117) libstd requires libgcc on Android + deps.StaticLibs = append(deps.StaticLibs, "libgcc") + + return deps +} + +func (compiler *baseCompiler) crateName() string { + return compiler.Properties.Crate_name +} + +func (compiler *baseCompiler) installDir(ctx ModuleContext) android.InstallPath { + dir := compiler.dir + if ctx.toolchain().Is64Bit() && compiler.dir64 != "" { + dir = compiler.dir64 + } + if !ctx.Host() || ctx.Target().NativeBridge == android.NativeBridgeEnabled { + dir = filepath.Join(dir, ctx.Arch().ArchType.String()) + } + return android.PathForModuleInstall(ctx, dir, compiler.subDir, + compiler.relativeInstallPath(), compiler.relative) +} + +func (compiler *baseCompiler) install(ctx ModuleContext, file android.Path) { + compiler.path = ctx.InstallFile(compiler.installDir(ctx), file.Base(), file) +} + +func (compiler *baseCompiler) getStem(ctx ModuleContext) string { + return compiler.getStemWithoutSuffix(ctx) + String(compiler.Properties.Suffix) +} + +func (compiler *baseCompiler) getStemWithoutSuffix(ctx BaseModuleContext) string { + stem := ctx.baseModuleName() + if String(compiler.Properties.Stem) != "" { + stem = String(compiler.Properties.Stem) + } + + return stem +} + +func (compiler *baseCompiler) relativeInstallPath() string { + return String(compiler.Properties.Relative_install_path) +} + +func srcPathFromModuleSrcs(ctx ModuleContext, srcs []string) android.Path { + srcPaths := android.PathsForModuleSrc(ctx, srcs) + if len(srcPaths) != 1 { + ctx.PropertyErrorf("srcs", "srcs can only contain one path for rust modules") + } + return srcPaths[0] +} diff --git a/rust/compiler_test.go b/rust/compiler_test.go new file mode 100644 index 000000000..bbf9f8d11 --- /dev/null +++ b/rust/compiler_test.go @@ -0,0 +1,76 @@ +// Copyright 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rust + +import ( + "strings" + "testing" +) + +// Test that feature flags are being correctly generated. +func TestFeaturesToFlags(t *testing.T) { + ctx := testRust(t, ` + rust_library_host_dylib { + name: "libfoo", + srcs: ["foo.rs"], + crate_name: "foo", + features: [ + "fizz", + "buzz" + ], + }`) + + libfooDylib := ctx.ModuleForTests("libfoo", "linux_glibc_x86_64_dylib").Rule("rustc") + + if !strings.Contains(libfooDylib.Args["rustcFlags"], "cfg 'feature=\"fizz\"'") || + !strings.Contains(libfooDylib.Args["rustcFlags"], "cfg 'feature=\"buzz\"'") { + t.Fatalf("missing fizz and buzz feature flags for libfoo dylib, rustcFlags: %#v", libfooDylib.Args["rustcFlags"]) + } +} + +// Test that we reject multiple source files. +func TestEnforceSingleSourceFile(t *testing.T) { + + singleSrcError := "srcs can only contain one path for rust modules" + + // Test libraries + testRustError(t, singleSrcError, ` + rust_library_host { + name: "foo-bar-library", + srcs: ["foo.rs", "src/bar.rs"], + }`) + + // Test binaries + testRustError(t, singleSrcError, ` + rust_binary_host { + name: "foo-bar-binary", + srcs: ["foo.rs", "src/bar.rs"], + }`) + + // Test proc_macros + testRustError(t, singleSrcError, ` + rust_proc_macro { + name: "foo-bar-proc-macro", + srcs: ["foo.rs", "src/bar.rs"], + }`) + + // Test prebuilts + testRustError(t, singleSrcError, ` + rust_prebuilt_dylib { + name: "foo-bar-prebuilt", + srcs: ["liby.so", "libz.so"], + host_supported: true, + }`) +} diff --git a/rust/config/Android.bp b/rust/config/Android.bp new file mode 100644 index 000000000..5026da39e --- /dev/null +++ b/rust/config/Android.bp @@ -0,0 +1,19 @@ +bootstrap_go_package { + name: "soong-rust-config", + pkgPath: "android/soong/rust/config", + deps: [ + "soong-android", + "soong-cc-config", + ], + srcs: [ + "arm_device.go", + "arm64_device.go", + "global.go", + "toolchain.go", + "allowed_list.go", + "x86_darwin_host.go", + "x86_linux_host.go", + "x86_device.go", + "x86_64_device.go", + ], +} diff --git a/rust/config/allowed_list.go b/rust/config/allowed_list.go new file mode 100644 index 000000000..a339050f8 --- /dev/null +++ b/rust/config/allowed_list.go @@ -0,0 +1,29 @@ +package config + +var ( + RustAllowedPaths = []string{ + "external/minijail", + "external/rust", + "external/crosvm", + "external/adhd", + "prebuilts/rust", + } + + RustModuleTypes = []string{ + "rust_binary", + "rust_binary_host", + "rust_library", + "rust_library_dylib", + "rust_library_rlib", + "rust_library_shared", + "rust_library_static", + "rust_library_host", + "rust_library_host_dylib", + "rust_library_host_rlib", + "rust_library_host_shared", + "rust_library_host_static", + "rust_proc_macro", + "rust_test", + "rust_test_host", + } +) diff --git a/rust/config/arm64_device.go b/rust/config/arm64_device.go new file mode 100644 index 000000000..60796d8d8 --- /dev/null +++ b/rust/config/arm64_device.go @@ -0,0 +1,93 @@ +// Copyright 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "strings" + + "android/soong/android" +) + +var ( + Arm64RustFlags = []string{} + Arm64ArchFeatureRustFlags = map[string][]string{} + Arm64LinkFlags = []string{ + "-Wl,--icf=safe", + "-Wl,-z,max-page-size=4096", + + "-Wl,--execute-only", + "-Wl,-z,separate-code", + } + + Arm64ArchVariantRustFlags = map[string][]string{ + "armv8-a": []string{}, + "armv8-2a": []string{}, + } +) + +func init() { + registerToolchainFactory(android.Android, android.Arm64, Arm64ToolchainFactory) + + pctx.StaticVariable("Arm64ToolchainRustFlags", strings.Join(Arm64RustFlags, " ")) + pctx.StaticVariable("Arm64ToolchainLinkFlags", strings.Join(Arm64LinkFlags, " ")) + + for variant, rustFlags := range Arm64ArchVariantRustFlags { + pctx.StaticVariable("Arm64"+variant+"VariantRustFlags", + strings.Join(rustFlags, " ")) + } + +} + +type toolchainArm64 struct { + toolchain64Bit + toolchainRustFlags string +} + +func (t *toolchainArm64) RustTriple() string { + return "aarch64-linux-android" +} + +func (t *toolchainArm64) ToolchainLinkFlags() string { + return "${config.DeviceGlobalLinkFlags} ${config.Arm64ToolchainLinkFlags}" +} + +func (t *toolchainArm64) ToolchainRustFlags() string { + return t.toolchainRustFlags +} + +func (t *toolchainArm64) RustFlags() string { + return "${config.Arm64ToolchainRustFlags}" +} + +func (t *toolchainArm64) Supported() bool { + return true +} + +func Arm64ToolchainFactory(arch android.Arch) Toolchain { + toolchainRustFlags := []string{ + "${config.Arm64ToolchainRustFlags}", + "${config.Arm64" + arch.ArchVariant + "VariantRustFlags}", + } + + toolchainRustFlags = append(toolchainRustFlags, deviceGlobalRustFlags...) + + for _, feature := range arch.ArchFeatures { + toolchainRustFlags = append(toolchainRustFlags, Arm64ArchFeatureRustFlags[feature]...) + } + + return &toolchainArm64{ + toolchainRustFlags: strings.Join(toolchainRustFlags, " "), + } +} diff --git a/rust/config/arm_device.go b/rust/config/arm_device.go new file mode 100644 index 000000000..aedb42b31 --- /dev/null +++ b/rust/config/arm_device.go @@ -0,0 +1,92 @@ +// Copyright 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "strings" + + "android/soong/android" +) + +var ( + ArmRustFlags = []string{} + ArmArchFeatureRustFlags = map[string][]string{} + ArmLinkFlags = []string{ + "-Wl,--icf=safe", + "-Wl,-m,armelf", + } + + ArmArchVariantRustFlags = map[string][]string{ + "armv7-a": []string{}, + "armv7-a-neon": []string{}, + "armv8-a": []string{}, + "armv8-2a": []string{}, + } +) + +func init() { + registerToolchainFactory(android.Android, android.Arm, ArmToolchainFactory) + + pctx.StaticVariable("ArmToolchainRustFlags", strings.Join(ArmRustFlags, " ")) + pctx.StaticVariable("ArmToolchainLinkFlags", strings.Join(ArmLinkFlags, " ")) + + for variant, rustFlags := range ArmArchVariantRustFlags { + pctx.StaticVariable("Arm"+variant+"VariantRustFlags", + strings.Join(rustFlags, " ")) + } + +} + +type toolchainArm struct { + toolchain64Bit + toolchainRustFlags string +} + +func (t *toolchainArm) RustTriple() string { + return "arm-linux-androideabi" +} + +func (t *toolchainArm) ToolchainLinkFlags() string { + return "${config.DeviceGlobalLinkFlags} ${config.ArmToolchainLinkFlags}" +} + +func (t *toolchainArm) ToolchainRustFlags() string { + return t.toolchainRustFlags +} + +func (t *toolchainArm) RustFlags() string { + return "${config.ArmToolchainRustFlags}" +} + +func (t *toolchainArm) Supported() bool { + return true +} + +func ArmToolchainFactory(arch android.Arch) Toolchain { + toolchainRustFlags := []string{ + "${config.ArmToolchainRustFlags}", + "${config.Arm" + arch.ArchVariant + "VariantRustFlags}", + } + + toolchainRustFlags = append(toolchainRustFlags, deviceGlobalRustFlags...) + + for _, feature := range arch.ArchFeatures { + toolchainRustFlags = append(toolchainRustFlags, ArmArchFeatureRustFlags[feature]...) + } + + return &toolchainArm{ + toolchainRustFlags: strings.Join(toolchainRustFlags, " "), + } +} diff --git a/rust/config/global.go b/rust/config/global.go new file mode 100644 index 000000000..690d83e80 --- /dev/null +++ b/rust/config/global.go @@ -0,0 +1,89 @@ +// Copyright 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "strings" + + "android/soong/android" + _ "android/soong/cc/config" +) + +var pctx = android.NewPackageContext("android/soong/rust/config") + +var ( + RustDefaultVersion = "1.40.0" + RustDefaultBase = "prebuilts/rust/" + DefaultEdition = "2018" + Stdlibs = []string{ + "libstd", + "libtest", + } + + DefaultDenyWarnings = true + + GlobalRustFlags = []string{ + "--remap-path-prefix $$(pwd)=", + "-C codegen-units=1", + "-C opt-level=3", + "-C relocation-model=pic", + } + + deviceGlobalRustFlags = []string{} + + deviceGlobalLinkFlags = []string{ + "-Bdynamic", + "-nostdlib", + "-Wl,-z,noexecstack", + "-Wl,-z,relro", + "-Wl,-z,now", + "-Wl,--build-id=md5", + "-Wl,--warn-shared-textrel", + "-Wl,--fatal-warnings", + + "-Wl,--pack-dyn-relocs=android+relr", + "-Wl,--no-undefined", + "-Wl,--hash-style=gnu", + } +) + +func init() { + pctx.SourcePathVariable("RustDefaultBase", RustDefaultBase) + pctx.VariableConfigMethod("HostPrebuiltTag", android.Config.PrebuiltOS) + + pctx.VariableFunc("RustBase", func(ctx android.PackageVarContext) string { + if override := ctx.Config().Getenv("RUST_PREBUILTS_BASE"); override != "" { + return override + } + return "${RustDefaultBase}" + }) + + pctx.VariableFunc("RustVersion", func(ctx android.PackageVarContext) string { + if override := ctx.Config().Getenv("RUST_PREBUILTS_VERSION"); override != "" { + return override + } + return RustDefaultVersion + }) + + pctx.StaticVariable("RustPath", "${RustBase}/${HostPrebuiltTag}/${RustVersion}") + pctx.StaticVariable("RustBin", "${RustPath}/bin") + + pctx.ImportAs("ccConfig", "android/soong/cc/config") + pctx.StaticVariable("RustLinker", "${ccConfig.ClangBin}/clang++") + pctx.StaticVariable("RustLinkerArgs", "-B ${ccConfig.ClangBin} -fuse-ld=lld") + + pctx.StaticVariable("DeviceGlobalLinkFlags", strings.Join(deviceGlobalLinkFlags, " ")) + +} diff --git a/rust/config/toolchain.go b/rust/config/toolchain.go new file mode 100644 index 000000000..616d88b89 --- /dev/null +++ b/rust/config/toolchain.go @@ -0,0 +1,130 @@ +// Copyright 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "android/soong/android" +) + +type Toolchain interface { + RustTriple() string + ToolchainRustFlags() string + ToolchainLinkFlags() string + + SharedLibSuffix() string + StaticLibSuffix() string + RlibSuffix() string + DylibSuffix() string + ProcMacroSuffix() string + ExecutableSuffix() string + + Is64Bit() bool + Supported() bool + + Bionic() bool +} + +type toolchainBase struct { +} + +func (toolchainBase) RustTriple() string { + panic("toolchainBase does not define a triple.") +} + +func (toolchainBase) ToolchainRustFlags() string { + panic("toolchainBase does not provide rust flags.") +} + +func (toolchainBase) ToolchainLinkFlags() string { + panic("toolchainBase does not provide link flags.") +} + +func (toolchainBase) Is64Bit() bool { + panic("toolchainBase cannot determine datapath width.") +} + +func (toolchainBase) Bionic() bool { + return true +} + +type toolchain64Bit struct { + toolchainBase +} + +func (toolchain64Bit) Is64Bit() bool { + return true +} + +type toolchain32Bit struct { + toolchainBase +} + +func (toolchain32Bit) Is64Bit() bool { + return false +} + +func (toolchain32Bit) Bionic() bool { + return true +} + +func (toolchainBase) ExecutableSuffix() string { + return "" +} + +func (toolchainBase) SharedLibSuffix() string { + return ".so" +} + +func (toolchainBase) StaticLibSuffix() string { + return ".a" +} + +func (toolchainBase) RlibSuffix() string { + return ".rlib" +} +func (toolchainBase) DylibSuffix() string { + return ".dylib.so" +} + +func (toolchainBase) ProcMacroSuffix() string { + return ".so" +} + +func (toolchainBase) Supported() bool { + return false +} + +func toolchainBaseFactory() Toolchain { + return &toolchainBase{} +} + +type toolchainFactory func(arch android.Arch) Toolchain + +var toolchainFactories = make(map[android.OsType]map[android.ArchType]toolchainFactory) + +func registerToolchainFactory(os android.OsType, arch android.ArchType, factory toolchainFactory) { + if toolchainFactories[os] == nil { + toolchainFactories[os] = make(map[android.ArchType]toolchainFactory) + } + toolchainFactories[os][arch] = factory +} + +func FindToolchain(os android.OsType, arch android.Arch) Toolchain { + factory := toolchainFactories[os][arch.ArchType] + if factory == nil { + return toolchainBaseFactory() + } + return factory(arch) +} diff --git a/rust/config/x86_64_device.go b/rust/config/x86_64_device.go new file mode 100644 index 000000000..9a6c00b19 --- /dev/null +++ b/rust/config/x86_64_device.go @@ -0,0 +1,94 @@ +// Copyright 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "strings" + + "android/soong/android" +) + +var ( + x86_64RustFlags = []string{} + x86_64ArchFeatureRustFlags = map[string][]string{} + x86_64LinkFlags = []string{} + + x86_64ArchVariantRustFlags = map[string][]string{ + "": []string{}, + "broadwell": []string{"-C target-cpu=broadwell"}, + "haswell": []string{"-C target-cpu=haswell"}, + "ivybridge": []string{"-C target-cpu=ivybridge"}, + "sandybridge": []string{"-C target-cpu=sandybridge"}, + "silvermont": []string{"-C target-cpu=silvermont"}, + "skylake": []string{"-C target-cpu=skylake"}, + //TODO: Add target-cpu=stoneyridge when rustc supports it. + "stoneyridge": []string{""}, + } +) + +func init() { + registerToolchainFactory(android.Android, android.X86_64, x86_64ToolchainFactory) + + pctx.StaticVariable("X86_64ToolchainRustFlags", strings.Join(x86_64RustFlags, " ")) + pctx.StaticVariable("X86_64ToolchainLinkFlags", strings.Join(x86_64LinkFlags, " ")) + + for variant, rustFlags := range x86_64ArchVariantRustFlags { + pctx.StaticVariable("X86_64"+variant+"VariantRustFlags", + strings.Join(rustFlags, " ")) + } + +} + +type toolchainX86_64 struct { + toolchain64Bit + toolchainRustFlags string +} + +func (t *toolchainX86_64) RustTriple() string { + return "x86_64-linux-android" +} + +func (t *toolchainX86_64) ToolchainLinkFlags() string { + return "${config.DeviceGlobalLinkFlags} ${config.X86_64ToolchainLinkFlags}" +} + +func (t *toolchainX86_64) ToolchainRustFlags() string { + return t.toolchainRustFlags +} + +func (t *toolchainX86_64) RustFlags() string { + return "${config.X86_64ToolchainRustFlags}" +} + +func (t *toolchainX86_64) Supported() bool { + return true +} + +func x86_64ToolchainFactory(arch android.Arch) Toolchain { + toolchainRustFlags := []string{ + "${config.X86_64ToolchainRustFlags}", + "${config.X86_64" + arch.ArchVariant + "VariantRustFlags}", + } + + toolchainRustFlags = append(toolchainRustFlags, deviceGlobalRustFlags...) + + for _, feature := range arch.ArchFeatures { + toolchainRustFlags = append(toolchainRustFlags, x86_64ArchFeatureRustFlags[feature]...) + } + + return &toolchainX86_64{ + toolchainRustFlags: strings.Join(toolchainRustFlags, " "), + } +} diff --git a/rust/config/x86_darwin_host.go b/rust/config/x86_darwin_host.go new file mode 100644 index 000000000..7cfc59cec --- /dev/null +++ b/rust/config/x86_darwin_host.go @@ -0,0 +1,81 @@ +// Copyright 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "strings" + + "android/soong/android" +) + +var ( + DarwinRustFlags = []string{} + DarwinRustLinkFlags = []string{} + darwinX8664Rustflags = []string{} + darwinX8664Linkflags = []string{} +) + +func init() { + registerToolchainFactory(android.Darwin, android.X86_64, darwinX8664ToolchainFactory) + pctx.StaticVariable("DarwinToolchainRustFlags", strings.Join(DarwinRustFlags, " ")) + pctx.StaticVariable("DarwinToolchainLinkFlags", strings.Join(DarwinRustLinkFlags, " ")) + pctx.StaticVariable("DarwinToolchainX8664RustFlags", strings.Join(darwinX8664Rustflags, " ")) + pctx.StaticVariable("DarwinToolchainX8664LinkFlags", strings.Join(darwinX8664Linkflags, " ")) + +} + +type toolchainDarwin struct { + toolchainRustFlags string + toolchainLinkFlags string +} + +type toolchainDarwinX8664 struct { + toolchain64Bit + toolchainDarwin +} + +func (toolchainDarwinX8664) Supported() bool { + return true +} + +func (toolchainDarwinX8664) Bionic() bool { + return false +} + +func (t *toolchainDarwinX8664) Name() string { + return "x86_64" +} + +func (t *toolchainDarwinX8664) RustTriple() string { + return "x86_64-apple-darwin" +} + +func (t *toolchainDarwin) ShlibSuffix() string { + return ".dylib" +} + +func (t *toolchainDarwinX8664) ToolchainLinkFlags() string { + return "${config.DarwinToolchainLinkFlags} ${config.DarwinToolchainX8664LinkFlags}" +} + +func (t *toolchainDarwinX8664) ToolchainRustFlags() string { + return "${config.DarwinToolchainRustFlags} ${config.DarwinToolchainX8664RustFlags}" +} + +func darwinX8664ToolchainFactory(arch android.Arch) Toolchain { + return toolchainDarwinX8664Singleton +} + +var toolchainDarwinX8664Singleton Toolchain = &toolchainDarwinX8664{} diff --git a/rust/config/x86_device.go b/rust/config/x86_device.go new file mode 100644 index 000000000..ec19b3c30 --- /dev/null +++ b/rust/config/x86_device.go @@ -0,0 +1,97 @@ +// Copyright 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "strings" + + "android/soong/android" +) + +var ( + x86RustFlags = []string{} + x86ArchFeatureRustFlags = map[string][]string{} + x86LinkFlags = []string{} + + x86ArchVariantRustFlags = map[string][]string{ + "": []string{}, + "atom": []string{"-C target-cpu=atom"}, + "broadwell": []string{"-C target-cpu=broadwell"}, + "haswell": []string{"-C target-cpu=haswell"}, + "ivybridge": []string{"-C target-cpu=ivybridge"}, + "sandybridge": []string{"-C target-cpu=sandybridge"}, + "silvermont": []string{"-C target-cpu=silvermont"}, + "skylake": []string{"-C target-cpu=skylake"}, + //TODO: Add target-cpu=stoneyridge when rustc supports it. + "stoneyridge": []string{""}, + // use prescott for x86_64, like cc/config/x86_device.go + "x86_64": []string{"-C target-cpu=prescott"}, + } +) + +func init() { + registerToolchainFactory(android.Android, android.X86, x86ToolchainFactory) + + pctx.StaticVariable("X86ToolchainRustFlags", strings.Join(x86RustFlags, " ")) + pctx.StaticVariable("X86ToolchainLinkFlags", strings.Join(x86LinkFlags, " ")) + + for variant, rustFlags := range x86ArchVariantRustFlags { + pctx.StaticVariable("X86"+variant+"VariantRustFlags", + strings.Join(rustFlags, " ")) + } + +} + +type toolchainX86 struct { + toolchain32Bit + toolchainRustFlags string +} + +func (t *toolchainX86) RustTriple() string { + return "i686-linux-android" +} + +func (t *toolchainX86) ToolchainLinkFlags() string { + return "${config.DeviceGlobalLinkFlags} ${config.X86ToolchainLinkFlags}" +} + +func (t *toolchainX86) ToolchainRustFlags() string { + return t.toolchainRustFlags +} + +func (t *toolchainX86) RustFlags() string { + return "${config.X86ToolchainRustFlags}" +} + +func (t *toolchainX86) Supported() bool { + return true +} + +func x86ToolchainFactory(arch android.Arch) Toolchain { + toolchainRustFlags := []string{ + "${config.X86ToolchainRustFlags}", + "${config.X86" + arch.ArchVariant + "VariantRustFlags}", + } + + toolchainRustFlags = append(toolchainRustFlags, deviceGlobalRustFlags...) + + for _, feature := range arch.ArchFeatures { + toolchainRustFlags = append(toolchainRustFlags, x86ArchFeatureRustFlags[feature]...) + } + + return &toolchainX86{ + toolchainRustFlags: strings.Join(toolchainRustFlags, " "), + } +} diff --git a/rust/config/x86_linux_host.go b/rust/config/x86_linux_host.go new file mode 100644 index 000000000..5376e5ba5 --- /dev/null +++ b/rust/config/x86_linux_host.go @@ -0,0 +1,117 @@ +// Copyright 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "strings" + + "android/soong/android" +) + +var ( + LinuxRustFlags = []string{} + LinuxRustLinkFlags = []string{} + linuxX86Rustflags = []string{} + linuxX86Linkflags = []string{} + linuxX8664Rustflags = []string{} + linuxX8664Linkflags = []string{} +) + +func init() { + registerToolchainFactory(android.Linux, android.X86_64, linuxX8664ToolchainFactory) + registerToolchainFactory(android.Linux, android.X86, linuxX86ToolchainFactory) + + pctx.StaticVariable("LinuxToolchainRustFlags", strings.Join(LinuxRustFlags, " ")) + pctx.StaticVariable("LinuxToolchainLinkFlags", strings.Join(LinuxRustLinkFlags, " ")) + pctx.StaticVariable("LinuxToolchainX86RustFlags", strings.Join(linuxX86Rustflags, " ")) + pctx.StaticVariable("LinuxToolchainX86LinkFlags", strings.Join(linuxX86Linkflags, " ")) + pctx.StaticVariable("LinuxToolchainX8664RustFlags", strings.Join(linuxX8664Rustflags, " ")) + pctx.StaticVariable("LinuxToolchainX8664LinkFlags", strings.Join(linuxX8664Linkflags, " ")) + +} + +type toolchainLinux struct { + toolchainRustFlags string + toolchainLinkFlags string +} + +type toolchainLinuxX86 struct { + toolchain32Bit + toolchainLinux +} + +type toolchainLinuxX8664 struct { + toolchain64Bit + toolchainLinux +} + +func (toolchainLinuxX8664) Supported() bool { + return true +} + +func (toolchainLinuxX8664) Bionic() bool { + return false +} + +func (t *toolchainLinuxX8664) Name() string { + return "x86_64" +} + +func (t *toolchainLinuxX8664) RustTriple() string { + return "x86_64-unknown-linux-gnu" +} + +func (t *toolchainLinuxX8664) ToolchainLinkFlags() string { + return "${config.LinuxToolchainLinkFlags} ${config.LinuxToolchainX8664LinkFlags}" +} + +func (t *toolchainLinuxX8664) ToolchainRustFlags() string { + return "${config.LinuxToolchainRustFlags} ${config.LinuxToolchainX8664RustFlags}" +} + +func linuxX8664ToolchainFactory(arch android.Arch) Toolchain { + return toolchainLinuxX8664Singleton +} + +func (toolchainLinuxX86) Supported() bool { + return true +} + +func (toolchainLinuxX86) Bionic() bool { + return false +} + +func (t *toolchainLinuxX86) Name() string { + return "x86" +} + +func (t *toolchainLinuxX86) RustTriple() string { + return "i686-unknown-linux-gnu" +} + +func (t *toolchainLinuxX86) ToolchainLinkFlags() string { + return "${config.LinuxToolchainLinkFlags} ${config.LinuxToolchainX86LinkFlags}" +} + +func (t *toolchainLinuxX86) ToolchainRustFlags() string { + return "${config.LinuxToolchainRustFlags} ${config.LinuxToolchainX86RustFlags}" +} + +func linuxX86ToolchainFactory(arch android.Arch) Toolchain { + return toolchainLinuxX86Singleton +} + +var toolchainLinuxX8664Singleton Toolchain = &toolchainLinuxX8664{} +var toolchainLinuxX86Singleton Toolchain = &toolchainLinuxX86{} diff --git a/rust/library.go b/rust/library.go new file mode 100644 index 000000000..0cf2dd045 --- /dev/null +++ b/rust/library.go @@ -0,0 +1,432 @@ +// Copyright 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rust + +import ( + "regexp" + "strings" + + "android/soong/android" + "android/soong/rust/config" +) + +func init() { + android.RegisterModuleType("rust_library", RustLibraryFactory) + android.RegisterModuleType("rust_library_dylib", RustLibraryDylibFactory) + android.RegisterModuleType("rust_library_rlib", RustLibraryRlibFactory) + android.RegisterModuleType("rust_library_host", RustLibraryHostFactory) + android.RegisterModuleType("rust_library_host_dylib", RustLibraryDylibHostFactory) + android.RegisterModuleType("rust_library_host_rlib", RustLibraryRlibHostFactory) + android.RegisterModuleType("rust_library_shared", RustLibrarySharedFactory) + android.RegisterModuleType("rust_library_static", RustLibraryStaticFactory) + android.RegisterModuleType("rust_library_host_shared", RustLibrarySharedHostFactory) + android.RegisterModuleType("rust_library_host_static", RustLibraryStaticHostFactory) +} + +type VariantLibraryProperties struct { + Enabled *bool `android:"arch_variant"` +} + +type LibraryCompilerProperties struct { + Rlib VariantLibraryProperties `android:"arch_variant"` + Dylib VariantLibraryProperties `android:"arch_variant"` + Shared VariantLibraryProperties `android:"arch_variant"` + Static VariantLibraryProperties `android:"arch_variant"` + + // path to the source file that is the main entry point of the program (e.g. src/lib.rs) + Srcs []string `android:"path,arch_variant"` + + // path to include directories to pass to cc_* modules, only relevant for static/shared variants. + Include_dirs []string `android:"path,arch_variant"` +} + +type LibraryMutatedProperties struct { + // Build a dylib variant + BuildDylib bool `blueprint:"mutated"` + // Build an rlib variant + BuildRlib bool `blueprint:"mutated"` + // Build a shared library variant + BuildShared bool `blueprint:"mutated"` + // Build a static library variant + BuildStatic bool `blueprint:"mutated"` + + // This variant is a dylib + VariantIsDylib bool `blueprint:"mutated"` + // This variant is an rlib + VariantIsRlib bool `blueprint:"mutated"` + // This variant is a shared library + VariantIsShared bool `blueprint:"mutated"` + // This variant is a static library + VariantIsStatic bool `blueprint:"mutated"` +} + +type libraryDecorator struct { + *baseCompiler + + Properties LibraryCompilerProperties + MutatedProperties LibraryMutatedProperties + distFile android.OptionalPath + unstrippedOutputFile android.Path + includeDirs android.Paths +} + +type libraryInterface interface { + rlib() bool + dylib() bool + static() bool + shared() bool + + // Returns true if the build options for the module have selected a particular build type + buildRlib() bool + buildDylib() bool + buildShared() bool + buildStatic() bool + + // Sets a particular variant type + setRlib() + setDylib() + setShared() + setStatic() + + // Build a specific library variant + BuildOnlyRlib() + BuildOnlyDylib() + BuildOnlyStatic() + BuildOnlyShared() +} + +func (library *libraryDecorator) exportedDirs() []string { + return library.linkDirs +} + +func (library *libraryDecorator) exportedDepFlags() []string { + return library.depFlags +} + +func (library *libraryDecorator) reexportDirs(dirs ...string) { + library.linkDirs = android.FirstUniqueStrings(append(library.linkDirs, dirs...)) +} + +func (library *libraryDecorator) reexportDepFlags(flags ...string) { + library.depFlags = android.FirstUniqueStrings(append(library.depFlags, flags...)) +} + +func (library *libraryDecorator) rlib() bool { + return library.MutatedProperties.VariantIsRlib +} + +func (library *libraryDecorator) dylib() bool { + return library.MutatedProperties.VariantIsDylib +} + +func (library *libraryDecorator) shared() bool { + return library.MutatedProperties.VariantIsShared +} + +func (library *libraryDecorator) static() bool { + return library.MutatedProperties.VariantIsStatic +} + +func (library *libraryDecorator) buildRlib() bool { + return library.MutatedProperties.BuildRlib && BoolDefault(library.Properties.Rlib.Enabled, true) +} + +func (library *libraryDecorator) buildDylib() bool { + return library.MutatedProperties.BuildDylib && BoolDefault(library.Properties.Dylib.Enabled, true) +} + +func (library *libraryDecorator) buildShared() bool { + return library.MutatedProperties.BuildShared && BoolDefault(library.Properties.Shared.Enabled, true) +} + +func (library *libraryDecorator) buildStatic() bool { + return library.MutatedProperties.BuildStatic && BoolDefault(library.Properties.Static.Enabled, true) +} + +func (library *libraryDecorator) setRlib() { + library.MutatedProperties.VariantIsRlib = true + library.MutatedProperties.VariantIsDylib = false + library.MutatedProperties.VariantIsStatic = false + library.MutatedProperties.VariantIsShared = false +} + +func (library *libraryDecorator) setDylib() { + library.MutatedProperties.VariantIsRlib = false + library.MutatedProperties.VariantIsDylib = true + library.MutatedProperties.VariantIsStatic = false + library.MutatedProperties.VariantIsShared = false +} + +func (library *libraryDecorator) setShared() { + library.MutatedProperties.VariantIsStatic = false + library.MutatedProperties.VariantIsShared = true + library.MutatedProperties.VariantIsRlib = false + library.MutatedProperties.VariantIsDylib = false +} + +func (library *libraryDecorator) setStatic() { + library.MutatedProperties.VariantIsStatic = true + library.MutatedProperties.VariantIsShared = false + library.MutatedProperties.VariantIsRlib = false + library.MutatedProperties.VariantIsDylib = false +} + +var _ compiler = (*libraryDecorator)(nil) +var _ libraryInterface = (*libraryDecorator)(nil) + +// rust_library produces all variants. +func RustLibraryFactory() android.Module { + module, _ := NewRustLibrary(android.HostAndDeviceSupported) + return module.Init() +} + +// rust_library_dylib produces a dylib. +func RustLibraryDylibFactory() android.Module { + module, library := NewRustLibrary(android.HostAndDeviceSupported) + library.BuildOnlyDylib() + return module.Init() +} + +// rust_library_rlib produces an rlib. +func RustLibraryRlibFactory() android.Module { + module, library := NewRustLibrary(android.HostAndDeviceSupported) + library.BuildOnlyRlib() + return module.Init() +} + +// rust_library_shared produces a shared library. +func RustLibrarySharedFactory() android.Module { + module, library := NewRustLibrary(android.HostAndDeviceSupported) + library.BuildOnlyShared() + return module.Init() +} + +// rust_library_static produces a static library. +func RustLibraryStaticFactory() android.Module { + module, library := NewRustLibrary(android.HostAndDeviceSupported) + library.BuildOnlyStatic() + return module.Init() +} + +// rust_library_host produces all variants. +func RustLibraryHostFactory() android.Module { + module, _ := NewRustLibrary(android.HostSupported) + return module.Init() +} + +// rust_library_dylib_host produces a dylib. +func RustLibraryDylibHostFactory() android.Module { + module, library := NewRustLibrary(android.HostSupported) + library.BuildOnlyDylib() + return module.Init() +} + +// rust_library_rlib_host produces an rlib. +func RustLibraryRlibHostFactory() android.Module { + module, library := NewRustLibrary(android.HostSupported) + library.BuildOnlyRlib() + return module.Init() +} + +// rust_library_static_host produces a static library. +func RustLibraryStaticHostFactory() android.Module { + module, library := NewRustLibrary(android.HostSupported) + library.BuildOnlyStatic() + return module.Init() +} + +// rust_library_shared_host produces an shared library. +func RustLibrarySharedHostFactory() android.Module { + module, library := NewRustLibrary(android.HostSupported) + library.BuildOnlyShared() + return module.Init() +} + +func (library *libraryDecorator) BuildOnlyDylib() { + library.MutatedProperties.BuildRlib = false + library.MutatedProperties.BuildShared = false + library.MutatedProperties.BuildStatic = false + +} + +func (library *libraryDecorator) BuildOnlyRlib() { + library.MutatedProperties.BuildDylib = false + library.MutatedProperties.BuildShared = false + library.MutatedProperties.BuildStatic = false +} + +func (library *libraryDecorator) BuildOnlyStatic() { + library.MutatedProperties.BuildShared = false + library.MutatedProperties.BuildRlib = false + library.MutatedProperties.BuildDylib = false + +} + +func (library *libraryDecorator) BuildOnlyShared() { + library.MutatedProperties.BuildStatic = false + library.MutatedProperties.BuildRlib = false + library.MutatedProperties.BuildDylib = false +} + +func NewRustLibrary(hod android.HostOrDeviceSupported) (*Module, *libraryDecorator) { + module := newModule(hod, android.MultilibFirst) + + library := &libraryDecorator{ + MutatedProperties: LibraryMutatedProperties{ + BuildDylib: true, + BuildRlib: true, + BuildShared: true, + BuildStatic: true, + }, + baseCompiler: NewBaseCompiler("lib", "lib64", InstallInSystem), + } + + module.compiler = library + + return module, library +} + +func (library *libraryDecorator) compilerProps() []interface{} { + return append(library.baseCompiler.compilerProps(), + &library.Properties, + &library.MutatedProperties) +} + +func (library *libraryDecorator) compilerDeps(ctx DepsContext, deps Deps) Deps { + + // TODO(b/144861059) Remove if C libraries support dylib linkage in the future. + if !ctx.Host() && (library.static() || library.shared()) { + library.setNoStdlibs() + for _, stdlib := range config.Stdlibs { + deps.Rlibs = append(deps.Rlibs, stdlib+".static") + } + } + + deps = library.baseCompiler.compilerDeps(ctx, deps) + + if ctx.toolchain().Bionic() && (library.dylib() || library.shared()) { + deps = library.baseCompiler.bionicDeps(ctx, deps) + } + + return deps +} +func (library *libraryDecorator) compilerFlags(ctx ModuleContext, flags Flags) Flags { + flags.RustFlags = append(flags.RustFlags, "-C metadata="+ctx.baseModuleName()) + flags = library.baseCompiler.compilerFlags(ctx, flags) + if library.shared() || library.static() { + library.includeDirs = append(library.includeDirs, android.PathsForModuleSrc(ctx, library.Properties.Include_dirs)...) + } + return flags +} + +func (library *libraryDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Path { + var outputFile android.WritablePath + + srcPath := srcPathFromModuleSrcs(ctx, library.Properties.Srcs) + + flags.RustFlags = append(flags.RustFlags, deps.depFlags...) + + if library.dylib() { + // We need prefer-dynamic for now to avoid linking in the static stdlib. See: + // https://github.com/rust-lang/rust/issues/19680 + // https://github.com/rust-lang/rust/issues/34909 + flags.RustFlags = append(flags.RustFlags, "-C prefer-dynamic") + } + + if library.rlib() { + fileName := library.getStem(ctx) + ctx.toolchain().RlibSuffix() + outputFile = android.PathForModuleOut(ctx, fileName) + + TransformSrctoRlib(ctx, srcPath, deps, flags, outputFile, deps.linkDirs) + } else if library.dylib() { + fileName := library.getStem(ctx) + ctx.toolchain().DylibSuffix() + outputFile = android.PathForModuleOut(ctx, fileName) + + TransformSrctoDylib(ctx, srcPath, deps, flags, outputFile, deps.linkDirs) + } else if library.static() { + fileName := library.getStem(ctx) + ctx.toolchain().StaticLibSuffix() + outputFile = android.PathForModuleOut(ctx, fileName) + + TransformSrctoStatic(ctx, srcPath, deps, flags, outputFile, deps.linkDirs) + } else if library.shared() { + fileName := library.getStem(ctx) + ctx.toolchain().SharedLibSuffix() + outputFile = android.PathForModuleOut(ctx, fileName) + + TransformSrctoShared(ctx, srcPath, deps, flags, outputFile, deps.linkDirs) + } + + if library.rlib() || library.dylib() { + library.reexportDirs(deps.linkDirs...) + library.reexportDepFlags(deps.depFlags...) + } + library.unstrippedOutputFile = outputFile + + return outputFile +} + +func (library *libraryDecorator) getStem(ctx ModuleContext) string { + stem := library.baseCompiler.getStemWithoutSuffix(ctx) + validateLibraryStem(ctx, stem, library.crateName()) + + return stem + String(library.baseCompiler.Properties.Suffix) +} + +var validCrateName = regexp.MustCompile("[^a-zA-Z0-9_]+") + +func validateLibraryStem(ctx BaseModuleContext, filename string, crate_name string) { + if crate_name == "" { + ctx.PropertyErrorf("crate_name", "crate_name must be defined.") + } + + // crate_names are used for the library output file, and rustc expects these + // to be alphanumeric with underscores allowed. + if validCrateName.MatchString(crate_name) { + ctx.PropertyErrorf("crate_name", + "library crate_names must be alphanumeric with underscores allowed") + } + + // Libraries are expected to begin with "lib" followed by the crate_name + if !strings.HasPrefix(filename, "lib"+crate_name) { + ctx.ModuleErrorf("Invalid name or stem property; library filenames must start with lib<crate_name>") + } +} + +func LibraryMutator(mctx android.BottomUpMutatorContext) { + if m, ok := mctx.Module().(*Module); ok && m.compiler != nil { + switch library := m.compiler.(type) { + case libraryInterface: + + // We only build the rust library variants here. This assumes that + // LinkageMutator runs first and there's an empty variant + // if rust variants are required. + if !library.static() && !library.shared() { + if library.buildRlib() && library.buildDylib() { + modules := mctx.CreateLocalVariations("rlib", "dylib") + rlib := modules[0].(*Module) + dylib := modules[1].(*Module) + + rlib.compiler.(libraryInterface).setRlib() + dylib.compiler.(libraryInterface).setDylib() + } else if library.buildRlib() { + modules := mctx.CreateLocalVariations("rlib") + modules[0].(*Module).compiler.(libraryInterface).setRlib() + } else if library.buildDylib() { + modules := mctx.CreateLocalVariations("dylib") + modules[0].(*Module).compiler.(libraryInterface).setDylib() + } + } + } + } +} diff --git a/rust/library_test.go b/rust/library_test.go new file mode 100644 index 000000000..9f9f374b9 --- /dev/null +++ b/rust/library_test.go @@ -0,0 +1,116 @@ +// Copyright 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rust + +import ( + "strings" + "testing" +) + +// Test that variants are being generated correctly, and that crate-types are correct. +func TestLibraryVariants(t *testing.T) { + + ctx := testRust(t, ` + rust_library_host { + name: "libfoo", + srcs: ["foo.rs"], + crate_name: "foo", + }`) + + // Test all variants are being built. + libfooRlib := ctx.ModuleForTests("libfoo", "linux_glibc_x86_64_rlib").Output("libfoo.rlib") + libfooDylib := ctx.ModuleForTests("libfoo", "linux_glibc_x86_64_dylib").Output("libfoo.dylib.so") + libfooStatic := ctx.ModuleForTests("libfoo", "linux_glibc_x86_64_static").Output("libfoo.a") + libfooShared := ctx.ModuleForTests("libfoo", "linux_glibc_x86_64_shared").Output("libfoo.so") + + rlibCrateType := "rlib" + dylibCrateType := "dylib" + sharedCrateType := "cdylib" + staticCrateType := "static" + + // Test crate type for rlib is correct. + if !strings.Contains(libfooRlib.Args["rustcFlags"], "crate-type="+rlibCrateType) { + t.Errorf("missing crate-type for static variant, expecting %#v, rustcFlags: %#v", rlibCrateType, libfooRlib.Args["rustcFlags"]) + } + + // Test crate type for dylib is correct. + if !strings.Contains(libfooDylib.Args["rustcFlags"], "crate-type="+dylibCrateType) { + t.Errorf("missing crate-type for static variant, expecting %#v, rustcFlags: %#v", dylibCrateType, libfooDylib.Args["rustcFlags"]) + } + + // Test crate type for C static libraries is correct. + if !strings.Contains(libfooStatic.Args["rustcFlags"], "crate-type="+staticCrateType) { + t.Errorf("missing crate-type for static variant, expecting %#v, rustcFlags: %#v", staticCrateType, libfooStatic.Args["rustcFlags"]) + } + + // Test crate type for C shared libraries is correct. + if !strings.Contains(libfooShared.Args["rustcFlags"], "crate-type="+sharedCrateType) { + t.Errorf("missing crate-type for shared variant, expecting %#v, got rustcFlags: %#v", sharedCrateType, libfooShared.Args["rustcFlags"]) + } + +} + +// Test that dylibs are not statically linking the standard library. +func TestDylibPreferDynamic(t *testing.T) { + ctx := testRust(t, ` + rust_library_host_dylib { + name: "libfoo", + srcs: ["foo.rs"], + crate_name: "foo", + }`) + + libfooDylib := ctx.ModuleForTests("libfoo", "linux_glibc_x86_64_dylib").Output("libfoo.dylib.so") + + if !strings.Contains(libfooDylib.Args["rustcFlags"], "prefer-dynamic") { + t.Errorf("missing prefer-dynamic flag for libfoo dylib, rustcFlags: %#v", libfooDylib.Args["rustcFlags"]) + } +} + +func TestValidateLibraryStem(t *testing.T) { + testRustError(t, "crate_name must be defined.", ` + rust_library_host { + name: "libfoo", + srcs: ["foo.rs"], + }`) + + testRustError(t, "library crate_names must be alphanumeric with underscores allowed", ` + rust_library_host { + name: "libfoo-bar", + srcs: ["foo.rs"], + crate_name: "foo-bar" + }`) + + testRustError(t, "Invalid name or stem property; library filenames must start with lib<crate_name>", ` + rust_library_host { + name: "foobar", + srcs: ["foo.rs"], + crate_name: "foo_bar" + }`) + testRustError(t, "Invalid name or stem property; library filenames must start with lib<crate_name>", ` + rust_library_host { + name: "foobar", + stem: "libfoo", + srcs: ["foo.rs"], + crate_name: "foo_bar" + }`) + testRustError(t, "Invalid name or stem property; library filenames must start with lib<crate_name>", ` + rust_library_host { + name: "foobar", + stem: "foo_bar", + srcs: ["foo.rs"], + crate_name: "foo_bar" + }`) + +} diff --git a/rust/prebuilt.go b/rust/prebuilt.go new file mode 100644 index 000000000..45bef9ea3 --- /dev/null +++ b/rust/prebuilt.go @@ -0,0 +1,71 @@ +// Copyright 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rust + +import ( + "android/soong/android" +) + +func init() { + android.RegisterModuleType("rust_prebuilt_dylib", PrebuiltDylibFactory) +} + +type PrebuiltProperties struct { + // path to the prebuilt file + Srcs []string `android:"path,arch_variant"` +} + +type prebuiltLibraryDecorator struct { + *libraryDecorator + Properties PrebuiltProperties +} + +var _ compiler = (*prebuiltLibraryDecorator)(nil) + +func PrebuiltDylibFactory() android.Module { + module, _ := NewPrebuiltDylib(android.HostAndDeviceSupported) + return module.Init() +} + +func NewPrebuiltDylib(hod android.HostOrDeviceSupported) (*Module, *prebuiltLibraryDecorator) { + module, library := NewRustLibrary(hod) + library.BuildOnlyDylib() + library.setNoStdlibs() + library.setDylib() + prebuilt := &prebuiltLibraryDecorator{ + libraryDecorator: library, + } + module.compiler = prebuilt + module.AddProperties(&library.Properties) + return module, prebuilt +} + +func (prebuilt *prebuiltLibraryDecorator) compilerProps() []interface{} { + return append(prebuilt.baseCompiler.compilerProps(), + &prebuilt.Properties) +} + +func (prebuilt *prebuiltLibraryDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Path { + srcPath := srcPathFromModuleSrcs(ctx, prebuilt.Properties.Srcs) + + prebuilt.unstrippedOutputFile = srcPath + + return srcPath +} + +func (prebuilt *prebuiltLibraryDecorator) compilerDeps(ctx DepsContext, deps Deps) Deps { + deps = prebuilt.baseCompiler.compilerDeps(ctx, deps) + return deps +} diff --git a/rust/proc_macro.go b/rust/proc_macro.go new file mode 100644 index 000000000..10ea1e380 --- /dev/null +++ b/rust/proc_macro.go @@ -0,0 +1,86 @@ +// Copyright 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rust + +import ( + "android/soong/android" +) + +func init() { + android.RegisterModuleType("rust_proc_macro", ProcMacroFactory) +} + +type ProcMacroCompilerProperties struct { + // path to the source file that is the main entry point of the program (e.g. src/lib.rs) + Srcs []string `android:"path,arch_variant"` + + // set name of the procMacro + Stem *string `android:"arch_variant"` + Suffix *string `android:"arch_variant"` +} + +type procMacroDecorator struct { + *baseCompiler + + Properties ProcMacroCompilerProperties + distFile android.OptionalPath + unstrippedOutputFile android.Path +} + +type procMacroInterface interface { +} + +var _ compiler = (*procMacroDecorator)(nil) + +func ProcMacroFactory() android.Module { + module, _ := NewProcMacro(android.HostSupportedNoCross) + return module.Init() +} + +func NewProcMacro(hod android.HostOrDeviceSupported) (*Module, *procMacroDecorator) { + module := newModule(hod, android.MultilibFirst) + + procMacro := &procMacroDecorator{ + baseCompiler: NewBaseCompiler("lib", "lib64", InstallInSystem), + } + + module.compiler = procMacro + + return module, procMacro +} + +func (procMacro *procMacroDecorator) compilerProps() []interface{} { + return append(procMacro.baseCompiler.compilerProps(), + &procMacro.Properties) +} + +func (procMacro *procMacroDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Path { + fileName := procMacro.getStem(ctx) + ctx.toolchain().ProcMacroSuffix() + outputFile := android.PathForModuleOut(ctx, fileName) + + srcPath := srcPathFromModuleSrcs(ctx, procMacro.Properties.Srcs) + + procMacro.unstrippedOutputFile = outputFile + + TransformSrctoProcMacro(ctx, srcPath, deps, flags, outputFile, deps.linkDirs) + return outputFile +} + +func (procMacro *procMacroDecorator) getStem(ctx ModuleContext) string { + stem := procMacro.baseCompiler.getStemWithoutSuffix(ctx) + validateLibraryStem(ctx, stem, procMacro.crateName()) + + return stem + String(procMacro.baseCompiler.Properties.Suffix) +} diff --git a/rust/rust.go b/rust/rust.go new file mode 100644 index 000000000..5cc884572 --- /dev/null +++ b/rust/rust.go @@ -0,0 +1,788 @@ +// Copyright 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rust + +import ( + "fmt" + "strings" + + "github.com/google/blueprint" + "github.com/google/blueprint/proptools" + + "android/soong/android" + "android/soong/cc" + "android/soong/rust/config" +) + +var pctx = android.NewPackageContext("android/soong/rust") + +func init() { + // Only allow rust modules to be defined for certain projects + + android.AddNeverAllowRules( + android.NeverAllow(). + NotIn(config.RustAllowedPaths...). + ModuleType(config.RustModuleTypes...)) + + android.RegisterModuleType("rust_defaults", defaultsFactory) + android.PreDepsMutators(func(ctx android.RegisterMutatorsContext) { + ctx.BottomUp("rust_libraries", LibraryMutator).Parallel() + ctx.BottomUp("rust_unit_tests", TestPerSrcMutator).Parallel() + }) + pctx.Import("android/soong/rust/config") +} + +type Flags struct { + GlobalRustFlags []string // Flags that apply globally to rust + GlobalLinkFlags []string // Flags that apply globally to linker + RustFlags []string // Flags that apply to rust + LinkFlags []string // Flags that apply to linker + RustFlagsDeps android.Paths // Files depended on by compiler flags + Toolchain config.Toolchain +} + +type BaseProperties struct { + AndroidMkRlibs []string + AndroidMkDylibs []string + AndroidMkProcMacroLibs []string + AndroidMkSharedLibs []string + AndroidMkStaticLibs []string + SubName string `blueprint:"mutated"` +} + +type Module struct { + android.ModuleBase + android.DefaultableModuleBase + + Properties BaseProperties + + hod android.HostOrDeviceSupported + multilib android.Multilib + + compiler compiler + cachedToolchain config.Toolchain + subAndroidMkOnce map[subAndroidMkProvider]bool + outputFile android.OptionalPath +} + +var _ android.ImageInterface = (*Module)(nil) + +func (mod *Module) ImageMutatorBegin(ctx android.BaseModuleContext) {} + +func (mod *Module) CoreVariantNeeded(ctx android.BaseModuleContext) bool { + return true +} + +func (mod *Module) RamdiskVariantNeeded(android.BaseModuleContext) bool { + return mod.InRamdisk() +} + +func (mod *Module) RecoveryVariantNeeded(android.BaseModuleContext) bool { + return mod.InRecovery() +} + +func (mod *Module) ExtraImageVariations(android.BaseModuleContext) []string { + return nil +} + +func (c *Module) SetImageVariation(ctx android.BaseModuleContext, variant string, module android.Module) { +} + +func (mod *Module) BuildStubs() bool { + return false +} + +func (mod *Module) HasStubsVariants() bool { + return false +} + +func (mod *Module) SelectedStl() string { + return "" +} + +func (mod *Module) NonCcVariants() bool { + if mod.compiler != nil { + if library, ok := mod.compiler.(libraryInterface); ok { + if library.buildRlib() || library.buildDylib() { + return true + } else { + return false + } + } + } + panic(fmt.Errorf("NonCcVariants called on non-library module: %q", mod.BaseModuleName())) +} + +func (mod *Module) ApiLevel() string { + panic(fmt.Errorf("Called ApiLevel on Rust module %q; stubs libraries are not yet supported.", mod.BaseModuleName())) +} + +func (mod *Module) Static() bool { + if mod.compiler != nil { + if library, ok := mod.compiler.(libraryInterface); ok { + return library.static() + } + } + panic(fmt.Errorf("Static called on non-library module: %q", mod.BaseModuleName())) +} + +func (mod *Module) Shared() bool { + if mod.compiler != nil { + if library, ok := mod.compiler.(libraryInterface); ok { + return library.static() + } + } + panic(fmt.Errorf("Shared called on non-library module: %q", mod.BaseModuleName())) +} + +func (mod *Module) Toc() android.OptionalPath { + if mod.compiler != nil { + if _, ok := mod.compiler.(libraryInterface); ok { + return android.OptionalPath{} + } + } + panic(fmt.Errorf("Toc() called on non-library module: %q", mod.BaseModuleName())) +} + +func (mod *Module) OnlyInRamdisk() bool { + return false +} + +func (mod *Module) OnlyInRecovery() bool { + return false +} + +func (mod *Module) UseSdk() bool { + return false +} + +func (mod *Module) UseVndk() bool { + return false +} + +func (mod *Module) MustUseVendorVariant() bool { + return false +} + +func (mod *Module) IsVndk() bool { + return false +} + +func (mod *Module) HasVendorVariant() bool { + return false +} + +func (mod *Module) SdkVersion() string { + return "" +} + +func (mod *Module) AlwaysSdk() bool { + return false +} + +func (mod *Module) ToolchainLibrary() bool { + return false +} + +func (mod *Module) NdkPrebuiltStl() bool { + return false +} + +func (mod *Module) StubDecorator() bool { + return false +} + +type Deps struct { + Dylibs []string + Rlibs []string + ProcMacros []string + SharedLibs []string + StaticLibs []string + + CrtBegin, CrtEnd string +} + +type PathDeps struct { + DyLibs RustLibraries + RLibs RustLibraries + SharedLibs android.Paths + StaticLibs android.Paths + ProcMacros RustLibraries + linkDirs []string + depFlags []string + //ReexportedDeps android.Paths + + CrtBegin android.OptionalPath + CrtEnd android.OptionalPath +} + +type RustLibraries []RustLibrary + +type RustLibrary struct { + Path android.Path + CrateName string +} + +type compiler interface { + compilerFlags(ctx ModuleContext, flags Flags) Flags + compilerProps() []interface{} + compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Path + compilerDeps(ctx DepsContext, deps Deps) Deps + crateName() string + + inData() bool + install(ctx ModuleContext, path android.Path) + relativeInstallPath() string +} + +func defaultsFactory() android.Module { + return DefaultsFactory() +} + +type Defaults struct { + android.ModuleBase + android.DefaultsModuleBase +} + +func DefaultsFactory(props ...interface{}) android.Module { + module := &Defaults{} + + module.AddProperties(props...) + module.AddProperties( + &BaseProperties{}, + &BaseCompilerProperties{}, + &BinaryCompilerProperties{}, + &LibraryCompilerProperties{}, + &ProcMacroCompilerProperties{}, + &PrebuiltProperties{}, + &TestProperties{}, + ) + + android.InitDefaultsModule(module) + return module +} + +func (mod *Module) CrateName() string { + return mod.compiler.crateName() +} + +func (mod *Module) CcLibrary() bool { + if mod.compiler != nil { + if _, ok := mod.compiler.(*libraryDecorator); ok { + return true + } + } + return false +} + +func (mod *Module) CcLibraryInterface() bool { + if mod.compiler != nil { + if _, ok := mod.compiler.(libraryInterface); ok { + return true + } + } + return false +} + +func (mod *Module) IncludeDirs() android.Paths { + if mod.compiler != nil { + if library, ok := mod.compiler.(*libraryDecorator); ok { + return library.includeDirs + } + } + panic(fmt.Errorf("IncludeDirs called on non-library module: %q", mod.BaseModuleName())) +} + +func (mod *Module) SetStatic() { + if mod.compiler != nil { + if library, ok := mod.compiler.(libraryInterface); ok { + library.setStatic() + return + } + } + panic(fmt.Errorf("SetStatic called on non-library module: %q", mod.BaseModuleName())) +} + +func (mod *Module) SetShared() { + if mod.compiler != nil { + if library, ok := mod.compiler.(libraryInterface); ok { + library.setShared() + return + } + } + panic(fmt.Errorf("SetShared called on non-library module: %q", mod.BaseModuleName())) +} + +func (mod *Module) SetBuildStubs() { + panic("SetBuildStubs not yet implemented for rust modules") +} + +func (mod *Module) SetStubsVersions(string) { + panic("SetStubsVersions not yet implemented for rust modules") +} + +func (mod *Module) StubsVersion() string { + panic("SetStubsVersions not yet implemented for rust modules") +} + +func (mod *Module) BuildStaticVariant() bool { + if mod.compiler != nil { + if library, ok := mod.compiler.(libraryInterface); ok { + return library.buildStatic() + } + } + panic(fmt.Errorf("BuildStaticVariant called on non-library module: %q", mod.BaseModuleName())) +} + +func (mod *Module) BuildSharedVariant() bool { + if mod.compiler != nil { + if library, ok := mod.compiler.(libraryInterface); ok { + return library.buildShared() + } + } + panic(fmt.Errorf("BuildSharedVariant called on non-library module: %q", mod.BaseModuleName())) +} + +// Rust module deps don't have a link order (?) +func (mod *Module) SetDepsInLinkOrder([]android.Path) {} + +func (mod *Module) GetDepsInLinkOrder() []android.Path { + return []android.Path{} +} + +func (mod *Module) GetStaticVariant() cc.LinkableInterface { + return nil +} + +func (mod *Module) Module() android.Module { + return mod +} + +func (mod *Module) StubsVersions() []string { + // For now, Rust has no stubs versions. + if mod.compiler != nil { + if _, ok := mod.compiler.(*libraryDecorator); ok { + return []string{} + } + } + panic(fmt.Errorf("StubsVersions called on non-library module: %q", mod.BaseModuleName())) +} + +func (mod *Module) OutputFile() android.OptionalPath { + return mod.outputFile +} + +func (mod *Module) InRecovery() bool { + // For now, Rust has no notion of the recovery image + return false +} +func (mod *Module) HasStaticVariant() bool { + if mod.GetStaticVariant() != nil { + return true + } + return false +} + +var _ cc.LinkableInterface = (*Module)(nil) + +func (mod *Module) Init() android.Module { + mod.AddProperties(&mod.Properties) + + if mod.compiler != nil { + mod.AddProperties(mod.compiler.compilerProps()...) + } + android.InitAndroidArchModule(mod, mod.hod, mod.multilib) + + android.InitDefaultableModule(mod) + + // Explicitly disable unsupported targets. + android.AddLoadHook(mod, func(ctx android.LoadHookContext) { + disableTargets := struct { + Target struct { + Linux_bionic struct { + Enabled *bool + } + } + }{} + disableTargets.Target.Linux_bionic.Enabled = proptools.BoolPtr(false) + + ctx.AppendProperties(&disableTargets) + }) + + return mod +} + +func newBaseModule(hod android.HostOrDeviceSupported, multilib android.Multilib) *Module { + return &Module{ + hod: hod, + multilib: multilib, + } +} +func newModule(hod android.HostOrDeviceSupported, multilib android.Multilib) *Module { + module := newBaseModule(hod, multilib) + return module +} + +type ModuleContext interface { + android.ModuleContext + ModuleContextIntf +} + +type BaseModuleContext interface { + android.BaseModuleContext + ModuleContextIntf +} + +type DepsContext interface { + android.BottomUpMutatorContext + ModuleContextIntf +} + +type ModuleContextIntf interface { + toolchain() config.Toolchain + baseModuleName() string + CrateName() string +} + +type depsContext struct { + android.BottomUpMutatorContext + moduleContextImpl +} + +type moduleContext struct { + android.ModuleContext + moduleContextImpl +} + +type moduleContextImpl struct { + mod *Module + ctx BaseModuleContext +} + +func (ctx *moduleContextImpl) toolchain() config.Toolchain { + return ctx.mod.toolchain(ctx.ctx) +} + +func (mod *Module) toolchain(ctx android.BaseModuleContext) config.Toolchain { + if mod.cachedToolchain == nil { + mod.cachedToolchain = config.FindToolchain(ctx.Os(), ctx.Arch()) + } + return mod.cachedToolchain +} + +func (d *Defaults) GenerateAndroidBuildActions(ctx android.ModuleContext) { +} + +func (mod *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { + ctx := &moduleContext{ + ModuleContext: actx, + moduleContextImpl: moduleContextImpl{ + mod: mod, + }, + } + ctx.ctx = ctx + + toolchain := mod.toolchain(ctx) + + if !toolchain.Supported() { + // This toolchain's unsupported, there's nothing to do for this mod. + return + } + + deps := mod.depsToPaths(ctx) + flags := Flags{ + Toolchain: toolchain, + } + + if mod.compiler != nil { + flags = mod.compiler.compilerFlags(ctx, flags) + outputFile := mod.compiler.compile(ctx, flags, deps) + mod.outputFile = android.OptionalPathForPath(outputFile) + mod.compiler.install(ctx, mod.outputFile.Path()) + } +} + +func (mod *Module) deps(ctx DepsContext) Deps { + deps := Deps{} + + if mod.compiler != nil { + deps = mod.compiler.compilerDeps(ctx, deps) + } + + deps.Rlibs = android.LastUniqueStrings(deps.Rlibs) + deps.Dylibs = android.LastUniqueStrings(deps.Dylibs) + deps.ProcMacros = android.LastUniqueStrings(deps.ProcMacros) + deps.SharedLibs = android.LastUniqueStrings(deps.SharedLibs) + deps.StaticLibs = android.LastUniqueStrings(deps.StaticLibs) + + return deps + +} + +func (ctx *moduleContextImpl) baseModuleName() string { + return ctx.mod.ModuleBase.BaseModuleName() +} + +func (ctx *moduleContextImpl) CrateName() string { + return ctx.mod.CrateName() +} + +type dependencyTag struct { + blueprint.BaseDependencyTag + name string + library bool + proc_macro bool +} + +var ( + rlibDepTag = dependencyTag{name: "rlibTag", library: true} + dylibDepTag = dependencyTag{name: "dylib", library: true} + procMacroDepTag = dependencyTag{name: "procMacro", proc_macro: true} + testPerSrcDepTag = dependencyTag{name: "rust_unit_tests"} +) + +func (mod *Module) depsToPaths(ctx android.ModuleContext) PathDeps { + var depPaths PathDeps + + directRlibDeps := []*Module{} + directDylibDeps := []*Module{} + directProcMacroDeps := []*Module{} + directSharedLibDeps := [](cc.LinkableInterface){} + directStaticLibDeps := [](cc.LinkableInterface){} + + ctx.VisitDirectDeps(func(dep android.Module) { + depName := ctx.OtherModuleName(dep) + depTag := ctx.OtherModuleDependencyTag(dep) + if rustDep, ok := dep.(*Module); ok { + //Handle Rust Modules + + linkFile := rustDep.outputFile + if !linkFile.Valid() { + ctx.ModuleErrorf("Invalid output file when adding dep %q to %q", depName, ctx.ModuleName()) + } + + switch depTag { + case dylibDepTag: + dylib, ok := rustDep.compiler.(libraryInterface) + if !ok || !dylib.dylib() { + ctx.ModuleErrorf("mod %q not an dylib library", depName) + return + } + directDylibDeps = append(directDylibDeps, rustDep) + mod.Properties.AndroidMkDylibs = append(mod.Properties.AndroidMkDylibs, depName) + case rlibDepTag: + rlib, ok := rustDep.compiler.(libraryInterface) + if !ok || !rlib.rlib() { + ctx.ModuleErrorf("mod %q not an rlib library", depName) + return + } + directRlibDeps = append(directRlibDeps, rustDep) + mod.Properties.AndroidMkRlibs = append(mod.Properties.AndroidMkRlibs, depName) + case procMacroDepTag: + directProcMacroDeps = append(directProcMacroDeps, rustDep) + mod.Properties.AndroidMkProcMacroLibs = append(mod.Properties.AndroidMkProcMacroLibs, depName) + } + + //Append the dependencies exportedDirs + if lib, ok := rustDep.compiler.(*libraryDecorator); ok { + depPaths.linkDirs = append(depPaths.linkDirs, lib.exportedDirs()...) + depPaths.depFlags = append(depPaths.depFlags, lib.exportedDepFlags()...) + } + + // Append this dependencies output to this mod's linkDirs so they can be exported to dependencies + // This can be probably be refactored by defining a common exporter interface similar to cc's + if depTag == dylibDepTag || depTag == rlibDepTag || depTag == procMacroDepTag { + linkDir := linkPathFromFilePath(linkFile.Path()) + if lib, ok := mod.compiler.(*libraryDecorator); ok { + lib.linkDirs = append(lib.linkDirs, linkDir) + } else if procMacro, ok := mod.compiler.(*procMacroDecorator); ok { + procMacro.linkDirs = append(procMacro.linkDirs, linkDir) + } + } + + } + + if ccDep, ok := dep.(cc.LinkableInterface); ok { + //Handle C dependencies + if _, ok := ccDep.(*Module); !ok { + if ccDep.Module().Target().Os != ctx.Os() { + ctx.ModuleErrorf("OS mismatch between %q and %q", ctx.ModuleName(), depName) + return + } + if ccDep.Module().Target().Arch.ArchType != ctx.Arch().ArchType { + ctx.ModuleErrorf("Arch mismatch between %q and %q", ctx.ModuleName(), depName) + return + } + } + + linkFile := ccDep.OutputFile() + linkPath := linkPathFromFilePath(linkFile.Path()) + libName := libNameFromFilePath(linkFile.Path()) + depFlag := "-l" + libName + + if !linkFile.Valid() { + ctx.ModuleErrorf("Invalid output file when adding dep %q to %q", depName, ctx.ModuleName()) + } + + exportDep := false + switch depTag { + case cc.StaticDepTag: + depFlag = "-lstatic=" + libName + depPaths.linkDirs = append(depPaths.linkDirs, linkPath) + depPaths.depFlags = append(depPaths.depFlags, depFlag) + directStaticLibDeps = append(directStaticLibDeps, ccDep) + mod.Properties.AndroidMkStaticLibs = append(mod.Properties.AndroidMkStaticLibs, depName) + case cc.SharedDepTag: + depFlag = "-ldylib=" + libName + depPaths.linkDirs = append(depPaths.linkDirs, linkPath) + depPaths.depFlags = append(depPaths.depFlags, depFlag) + directSharedLibDeps = append(directSharedLibDeps, ccDep) + mod.Properties.AndroidMkSharedLibs = append(mod.Properties.AndroidMkSharedLibs, depName) + exportDep = true + case cc.CrtBeginDepTag: + depPaths.CrtBegin = linkFile + case cc.CrtEndDepTag: + depPaths.CrtEnd = linkFile + } + + // Make sure these dependencies are propagated + if lib, ok := mod.compiler.(*libraryDecorator); ok && exportDep { + lib.linkDirs = append(lib.linkDirs, linkPath) + lib.depFlags = append(lib.depFlags, depFlag) + } else if procMacro, ok := mod.compiler.(*procMacroDecorator); ok && exportDep { + procMacro.linkDirs = append(procMacro.linkDirs, linkPath) + procMacro.depFlags = append(procMacro.depFlags, depFlag) + } + + } + }) + + var rlibDepFiles RustLibraries + for _, dep := range directRlibDeps { + rlibDepFiles = append(rlibDepFiles, RustLibrary{Path: dep.outputFile.Path(), CrateName: dep.CrateName()}) + } + var dylibDepFiles RustLibraries + for _, dep := range directDylibDeps { + dylibDepFiles = append(dylibDepFiles, RustLibrary{Path: dep.outputFile.Path(), CrateName: dep.CrateName()}) + } + var procMacroDepFiles RustLibraries + for _, dep := range directProcMacroDeps { + procMacroDepFiles = append(procMacroDepFiles, RustLibrary{Path: dep.outputFile.Path(), CrateName: dep.CrateName()}) + } + + var staticLibDepFiles android.Paths + for _, dep := range directStaticLibDeps { + staticLibDepFiles = append(staticLibDepFiles, dep.OutputFile().Path()) + } + + var sharedLibDepFiles android.Paths + for _, dep := range directSharedLibDeps { + sharedLibDepFiles = append(sharedLibDepFiles, dep.OutputFile().Path()) + } + + depPaths.RLibs = append(depPaths.RLibs, rlibDepFiles...) + depPaths.DyLibs = append(depPaths.DyLibs, dylibDepFiles...) + depPaths.SharedLibs = append(depPaths.SharedLibs, sharedLibDepFiles...) + depPaths.StaticLibs = append(depPaths.StaticLibs, staticLibDepFiles...) + depPaths.ProcMacros = append(depPaths.ProcMacros, procMacroDepFiles...) + + // Dedup exported flags from dependencies + depPaths.linkDirs = android.FirstUniqueStrings(depPaths.linkDirs) + depPaths.depFlags = android.FirstUniqueStrings(depPaths.depFlags) + + return depPaths +} + +func (mod *Module) InstallInData() bool { + if mod.compiler == nil { + return false + } + return mod.compiler.inData() +} + +func linkPathFromFilePath(filepath android.Path) string { + return strings.Split(filepath.String(), filepath.Base())[0] +} + +func libNameFromFilePath(filepath android.Path) string { + libName := strings.TrimSuffix(filepath.Base(), filepath.Ext()) + if strings.HasPrefix(libName, "lib") { + libName = libName[3:] + } + return libName +} + +func (mod *Module) DepsMutator(actx android.BottomUpMutatorContext) { + ctx := &depsContext{ + BottomUpMutatorContext: actx, + moduleContextImpl: moduleContextImpl{ + mod: mod, + }, + } + ctx.ctx = ctx + + deps := mod.deps(ctx) + commonDepVariations := []blueprint.Variation{} + if cc.VersionVariantAvailable(mod) { + commonDepVariations = append(commonDepVariations, + blueprint.Variation{Mutator: "version", Variation: ""}) + } + if !mod.Host() { + commonDepVariations = append(commonDepVariations, + blueprint.Variation{Mutator: "image", Variation: android.CoreVariation}) + } + actx.AddVariationDependencies( + append(commonDepVariations, []blueprint.Variation{ + {Mutator: "rust_libraries", Variation: "rlib"}, + {Mutator: "link", Variation: ""}}...), + rlibDepTag, deps.Rlibs...) + actx.AddVariationDependencies( + append(commonDepVariations, []blueprint.Variation{ + {Mutator: "rust_libraries", Variation: "dylib"}, + {Mutator: "link", Variation: ""}}...), + dylibDepTag, deps.Dylibs...) + + actx.AddVariationDependencies(append(commonDepVariations, + blueprint.Variation{Mutator: "link", Variation: "shared"}), + cc.SharedDepTag, deps.SharedLibs...) + actx.AddVariationDependencies(append(commonDepVariations, + blueprint.Variation{Mutator: "link", Variation: "static"}), + cc.StaticDepTag, deps.StaticLibs...) + + if deps.CrtBegin != "" { + actx.AddVariationDependencies(commonDepVariations, cc.CrtBeginDepTag, deps.CrtBegin) + } + if deps.CrtEnd != "" { + actx.AddVariationDependencies(commonDepVariations, cc.CrtEndDepTag, deps.CrtEnd) + } + + // proc_macros are compiler plugins, and so we need the host arch variant as a dependendcy. + actx.AddFarVariationDependencies(ctx.Config().BuildOSTarget.Variations(), procMacroDepTag, deps.ProcMacros...) +} + +func (mod *Module) Name() string { + name := mod.ModuleBase.Name() + if p, ok := mod.compiler.(interface { + Name(string) string + }); ok { + name = p.Name(name) + } + return name +} + +var Bool = proptools.Bool +var BoolDefault = proptools.BoolDefault +var String = proptools.String +var StringPtr = proptools.StringPtr diff --git a/rust/rust_test.go b/rust/rust_test.go new file mode 100644 index 000000000..020581dfc --- /dev/null +++ b/rust/rust_test.go @@ -0,0 +1,269 @@ +// Copyright 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rust + +import ( + "io/ioutil" + "os" + "runtime" + "strings" + "testing" + + "android/soong/android" + "android/soong/cc" +) + +var ( + buildDir string +) + +func setUp() { + var err error + buildDir, err = ioutil.TempDir("", "soong_rust_test") + if err != nil { + panic(err) + } +} + +func tearDown() { + os.RemoveAll(buildDir) +} + +func TestMain(m *testing.M) { + run := func() int { + setUp() + defer tearDown() + + return m.Run() + } + + os.Exit(run()) +} + +func testConfig(bp string) android.Config { + bp = bp + GatherRequiredDepsForTest() + + fs := map[string][]byte{ + "foo.rs": nil, + "src/bar.rs": nil, + "liby.so": nil, + "libz.so": nil, + } + + cc.GatherRequiredFilesForTest(fs) + + return android.TestArchConfig(buildDir, nil, bp, fs) +} + +func testRust(t *testing.T, bp string) *android.TestContext { + // TODO (b/140435149) + if runtime.GOOS != "linux" { + t.Skip("Only the Linux toolchain is supported for Rust") + } + + t.Helper() + config := testConfig(bp) + + t.Helper() + ctx := CreateTestContext() + ctx.Register(config) + + _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) + android.FailIfErrored(t, errs) + _, errs = ctx.PrepareBuildActions(config) + android.FailIfErrored(t, errs) + + return ctx +} + +func testRustError(t *testing.T, pattern string, bp string) { + // TODO (b/140435149) + if runtime.GOOS != "linux" { + t.Skip("Only the Linux toolchain is supported for Rust") + } + + t.Helper() + config := testConfig(bp) + + ctx := CreateTestContext() + ctx.Register(config) + + _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) + if len(errs) > 0 { + android.FailIfNoMatchingErrors(t, pattern, errs) + return + } + + _, errs = ctx.PrepareBuildActions(config) + if len(errs) > 0 { + android.FailIfNoMatchingErrors(t, pattern, errs) + return + } + + t.Fatalf("missing expected error %q (0 errors are returned)", pattern) +} + +// Test that we can extract the lib name from a lib path. +func TestLibNameFromFilePath(t *testing.T) { + libBarPath := android.PathForTesting("out/soong/.intermediates/external/libbar/libbar/linux_glibc_x86_64_shared/libbar.so.so") + libLibPath := android.PathForTesting("out/soong/.intermediates/external/libbar/libbar/linux_glibc_x86_64_shared/liblib.dylib.so") + + libBarName := libNameFromFilePath(libBarPath) + libLibName := libNameFromFilePath(libLibPath) + + expectedResult := "bar.so" + if libBarName != expectedResult { + t.Errorf("libNameFromFilePath returned the wrong name; expected '%#v', got '%#v'", expectedResult, libBarName) + } + + expectedResult = "lib.dylib" + if libLibName != expectedResult { + t.Errorf("libNameFromFilePath returned the wrong name; expected '%#v', got '%#v'", expectedResult, libLibPath) + } +} + +// Test that we can extract the link path from a lib path. +func TestLinkPathFromFilePath(t *testing.T) { + barPath := android.PathForTesting("out/soong/.intermediates/external/libbar/libbar/linux_glibc_x86_64_shared/libbar.so") + libName := linkPathFromFilePath(barPath) + expectedResult := "out/soong/.intermediates/external/libbar/libbar/linux_glibc_x86_64_shared/" + + if libName != expectedResult { + t.Errorf("libNameFromFilePath returned the wrong name; expected '%#v', got '%#v'", expectedResult, libName) + } +} + +// Test to make sure dependencies are being picked up correctly. +func TestDepsTracking(t *testing.T) { + ctx := testRust(t, ` + rust_library_host_static { + name: "libstatic", + srcs: ["foo.rs"], + crate_name: "static", + } + rust_library_host_shared { + name: "libshared", + srcs: ["foo.rs"], + crate_name: "shared", + } + rust_library_host_dylib { + name: "libdylib", + srcs: ["foo.rs"], + crate_name: "dylib", + } + rust_library_host_rlib { + name: "librlib", + srcs: ["foo.rs"], + crate_name: "rlib", + } + rust_proc_macro { + name: "libpm", + srcs: ["foo.rs"], + crate_name: "pm", + } + rust_binary_host { + name: "fizz-buzz", + dylibs: ["libdylib"], + rlibs: ["librlib"], + proc_macros: ["libpm"], + static_libs: ["libstatic"], + shared_libs: ["libshared"], + srcs: ["foo.rs"], + } + `) + module := ctx.ModuleForTests("fizz-buzz", "linux_glibc_x86_64").Module().(*Module) + + // Since dependencies are added to AndroidMk* properties, we can check these to see if they've been picked up. + if !android.InList("libdylib", module.Properties.AndroidMkDylibs) { + t.Errorf("Dylib dependency not detected (dependency missing from AndroidMkDylibs)") + } + + if !android.InList("librlib", module.Properties.AndroidMkRlibs) { + t.Errorf("Rlib dependency not detected (dependency missing from AndroidMkRlibs)") + } + + if !android.InList("libpm", module.Properties.AndroidMkProcMacroLibs) { + t.Errorf("Proc_macro dependency not detected (dependency missing from AndroidMkProcMacroLibs)") + } + + if !android.InList("libshared", module.Properties.AndroidMkSharedLibs) { + t.Errorf("Shared library dependency not detected (dependency missing from AndroidMkSharedLibs)") + } + + if !android.InList("libstatic", module.Properties.AndroidMkStaticLibs) { + t.Errorf("Static library dependency not detected (dependency missing from AndroidMkStaticLibs)") + } +} + +// Test to make sure proc_macros use host variants when building device modules. +func TestProcMacroDeviceDeps(t *testing.T) { + ctx := testRust(t, ` + rust_library_host_rlib { + name: "libbar", + srcs: ["foo.rs"], + crate_name: "bar", + } + // Make a dummy libstd to let resolution go through + rust_library_dylib { + name: "libstd", + crate_name: "std", + srcs: ["foo.rs"], + no_stdlibs: true, + } + rust_library_dylib { + name: "libterm", + crate_name: "term", + srcs: ["foo.rs"], + no_stdlibs: true, + } + rust_library_dylib { + name: "libtest", + crate_name: "test", + srcs: ["foo.rs"], + no_stdlibs: true, + } + rust_proc_macro { + name: "libpm", + rlibs: ["libbar"], + srcs: ["foo.rs"], + crate_name: "pm", + } + rust_binary { + name: "fizz-buzz", + proc_macros: ["libpm"], + srcs: ["foo.rs"], + } + `) + rustc := ctx.ModuleForTests("libpm", "linux_glibc_x86_64").Rule("rustc") + + if !strings.Contains(rustc.Args["libFlags"], "libbar/linux_glibc_x86_64") { + t.Errorf("Proc_macro is not using host variant of dependent modules.") + } +} + +// Test that no_stdlibs suppresses dependencies on rust standard libraries +func TestNoStdlibs(t *testing.T) { + ctx := testRust(t, ` + rust_binary { + name: "fizz-buzz", + srcs: ["foo.rs"], + no_stdlibs: true, + }`) + module := ctx.ModuleForTests("fizz-buzz", "android_arm64_armv8-a").Module().(*Module) + + if android.InList("libstd", module.Properties.AndroidMkDylibs) { + t.Errorf("no_stdlibs did not suppress dependency on libstd") + } +} diff --git a/rust/test.go b/rust/test.go new file mode 100644 index 000000000..469bec652 --- /dev/null +++ b/rust/test.go @@ -0,0 +1,185 @@ +// Copyright 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rust + +import ( + "path/filepath" + "strings" + + "android/soong/android" + "android/soong/tradefed" +) + +type TestProperties struct { + // the name of the test configuration (for example "AndroidTest.xml") that should be + // installed with the module. + Test_config *string `android:"path,arch_variant"` + + // the name of the test configuration template (for example "AndroidTestTemplate.xml") that + // should be installed with the module. + Test_config_template *string `android:"path,arch_variant"` + + // list of compatibility suites (for example "cts", "vts") that the module should be + // installed into. + Test_suites []string `android:"arch_variant"` + + // Flag to indicate whether or not to create test config automatically. If AndroidTest.xml + // doesn't exist next to the Android.bp, this attribute doesn't need to be set to true + // explicitly. + Auto_gen_config *bool +} + +// A test module is a binary module with extra --test compiler flag +// and different default installation directory. +// In golang, inheriance is written as a component. +type testDecorator struct { + *binaryDecorator + Properties TestProperties + testConfig android.Path +} + +func NewRustTest(hod android.HostOrDeviceSupported) (*Module, *testDecorator) { + module := newModule(hod, android.MultilibFirst) + + test := &testDecorator{ + binaryDecorator: &binaryDecorator{ + baseCompiler: NewBaseCompiler("nativetest", "nativetest64", InstallInData), + }, + } + + module.compiler = test + + return module, test +} + +func (test *testDecorator) compilerProps() []interface{} { + return append(test.binaryDecorator.compilerProps(), &test.Properties) +} + +func (test *testDecorator) getMutatedModuleSubName(moduleName string) string { + stem := String(test.baseCompiler.Properties.Stem) + if stem != "" && !strings.HasSuffix(moduleName, "_"+stem) { + // Avoid repeated suffix in the module name. + return "_" + stem + } + return "" +} + +func (test *testDecorator) install(ctx ModuleContext, file android.Path) { + name := ctx.ModuleName() + path := test.baseCompiler.relativeInstallPath() + // on device, use mutated module name + name = name + test.getMutatedModuleSubName(name) + if !ctx.Device() { // on host, use mutated module name + arch type + stem name + stem := String(test.baseCompiler.Properties.Stem) + if stem == "" { + stem = name + } + name = filepath.Join(name, ctx.Arch().ArchType.String(), stem) + } + test.testConfig = tradefed.AutoGenRustTestConfig(ctx, name, + test.Properties.Test_config, + test.Properties.Test_config_template, + test.Properties.Test_suites, + test.Properties.Auto_gen_config) + // default relative install path is module name + if path == "" { + test.baseCompiler.relative = ctx.ModuleName() + } + test.binaryDecorator.install(ctx, file) +} + +func (test *testDecorator) compilerFlags(ctx ModuleContext, flags Flags) Flags { + flags = test.binaryDecorator.compilerFlags(ctx, flags) + flags.RustFlags = append(flags.RustFlags, "--test") + return flags +} + +func init() { + // Rust tests are binary files built with --test. + android.RegisterModuleType("rust_test", RustTestFactory) + android.RegisterModuleType("rust_test_host", RustTestHostFactory) +} + +func RustTestFactory() android.Module { + module, _ := NewRustTest(android.HostAndDeviceSupported) + return module.Init() +} + +func RustTestHostFactory() android.Module { + module, _ := NewRustTest(android.HostSupported) + return module.Init() +} + +func (test *testDecorator) testPerSrc() bool { + return true +} + +func (test *testDecorator) srcs() []string { + return test.binaryDecorator.Properties.Srcs +} + +func (test *testDecorator) setSrc(name, src string) { + test.binaryDecorator.Properties.Srcs = []string{src} + test.baseCompiler.Properties.Stem = StringPtr(name) +} + +func (test *testDecorator) unsetSrc() { + test.binaryDecorator.Properties.Srcs = nil + test.baseCompiler.Properties.Stem = StringPtr("") +} + +type testPerSrc interface { + testPerSrc() bool + srcs() []string + setSrc(string, string) + unsetSrc() +} + +var _ testPerSrc = (*testDecorator)(nil) + +func TestPerSrcMutator(mctx android.BottomUpMutatorContext) { + if m, ok := mctx.Module().(*Module); ok { + if test, ok := m.compiler.(testPerSrc); ok { + numTests := len(test.srcs()) + if test.testPerSrc() && numTests > 0 { + if duplicate, found := android.CheckDuplicate(test.srcs()); found { + mctx.PropertyErrorf("srcs", "found a duplicate entry %q", duplicate) + return + } + // Rust compiler always compiles one source file at a time and + // uses the crate name as output file name. + // Cargo uses the test source file name as default crate name, + // but that can be redefined. + // So when there are multiple source files, the source file names will + // be the output file names, but when there is only one test file, + // use the crate name. + testNames := make([]string, numTests) + for i, src := range test.srcs() { + testNames[i] = strings.TrimSuffix(filepath.Base(src), filepath.Ext(src)) + } + crateName := m.compiler.crateName() + if numTests == 1 && crateName != "" { + testNames[0] = crateName + } + // TODO(chh): Add an "all tests" variation like cc/test.go? + tests := mctx.CreateLocalVariations(testNames...) + for i, src := range test.srcs() { + tests[i].(*Module).compiler.(testPerSrc).setSrc(testNames[i], src) + } + } + } + } +} diff --git a/rust/test_test.go b/rust/test_test.go new file mode 100644 index 000000000..f131c6ebe --- /dev/null +++ b/rust/test_test.go @@ -0,0 +1,63 @@ +// Copyright 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rust + +import ( + "strings" + "testing" +) + +// Check if rust_test_host accepts multiple source files and applies --test flag. +func TestRustTest(t *testing.T) { + ctx := testRust(t, ` + rust_test_host { + name: "my_test", + srcs: ["foo.rs", "src/bar.rs"], + crate_name: "new_test", // not used for multiple source files + relative_install_path: "rust/my-test", + }`) + + for _, name := range []string{"foo", "bar"} { + testingModule := ctx.ModuleForTests("my_test", "linux_glibc_x86_64_"+name) + testingBuildParams := testingModule.Output(name) + rustcFlags := testingBuildParams.Args["rustcFlags"] + if !strings.Contains(rustcFlags, "--test") { + t.Errorf("%v missing --test flag, rustcFlags: %#v", name, rustcFlags) + } + outPath := "/my_test/linux_glibc_x86_64_" + name + "/" + name + if !strings.Contains(testingBuildParams.Output.String(), outPath) { + t.Errorf("wrong output: %v expect: %v", testingBuildParams.Output, outPath) + } + } +} + +// crate_name is output file name, when there is only one source file. +func TestRustTestSingleFile(t *testing.T) { + ctx := testRust(t, ` + rust_test_host { + name: "my-test", + srcs: ["foo.rs"], + crate_name: "new_test", + relative_install_path: "my-pkg", + }`) + + name := "new_test" + testingModule := ctx.ModuleForTests("my-test", "linux_glibc_x86_64_"+name) + outPath := "/my-test/linux_glibc_x86_64_" + name + "/" + name + testingBuildParams := testingModule.Output(name) + if !strings.Contains(testingBuildParams.Output.String(), outPath) { + t.Errorf("wrong output: %v expect: %v", testingBuildParams.Output, outPath) + } +} diff --git a/rust/testing.go b/rust/testing.go new file mode 100644 index 000000000..f9adec828 --- /dev/null +++ b/rust/testing.go @@ -0,0 +1,114 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rust + +import ( + "android/soong/android" + "android/soong/cc" +) + +func GatherRequiredDepsForTest() string { + bp := ` + rust_prebuilt_dylib { + name: "libarena_x86_64-unknown-linux-gnu", + srcs: [""], + host_supported: true, + } + rust_prebuilt_dylib { + name: "libfmt_macros_x86_64-unknown-linux-gnu", + srcs: [""], + host_supported: true, + } + rust_prebuilt_dylib { + name: "libgraphviz_x86_64-unknown-linux-gnu", + srcs: [""], + host_supported: true, + } + rust_prebuilt_dylib { + name: "libserialize_x86_64-unknown-linux-gnu", + srcs: [""], + host_supported: true, + } + rust_prebuilt_dylib { + name: "libstd_x86_64-unknown-linux-gnu", + srcs: [""], + host_supported: true, + } + rust_prebuilt_dylib { + name: "libsyntax_x86_64-unknown-linux-gnu", + srcs: [""], + host_supported: true, + } + rust_prebuilt_dylib { + name: "libsyntax_ext_x86_64-unknown-linux-gnu", + srcs: [""], + host_supported: true, + } + rust_prebuilt_dylib { + name: "libsyntax_pos_x86_64-unknown-linux-gnu", + srcs: [""], + host_supported: true, + } + rust_prebuilt_dylib { + name: "libterm_x86_64-unknown-linux-gnu", + srcs: [""], + host_supported: true, + } + rust_prebuilt_dylib { + name: "libtest_x86_64-unknown-linux-gnu", + srcs: [""], + host_supported: true, + } + + ////////////////////////////// + // Device module requirements + + cc_library { + name: "liblog", + no_libcrt: true, + nocrt: true, + system_shared_libs: [], + } +` + cc.GatherRequiredDepsForTest(android.NoOsType) + return bp +} + +func CreateTestContext() *android.TestContext { + ctx := android.NewTestArchContext() + cc.RegisterRequiredBuildComponentsForTest(ctx) + ctx.RegisterModuleType("rust_binary", RustBinaryFactory) + ctx.RegisterModuleType("rust_binary_host", RustBinaryHostFactory) + ctx.RegisterModuleType("rust_test", RustTestFactory) + ctx.RegisterModuleType("rust_test_host", RustTestHostFactory) + ctx.RegisterModuleType("rust_library", RustLibraryFactory) + ctx.RegisterModuleType("rust_library_host", RustLibraryHostFactory) + ctx.RegisterModuleType("rust_library_host_rlib", RustLibraryRlibHostFactory) + ctx.RegisterModuleType("rust_library_host_dylib", RustLibraryDylibHostFactory) + ctx.RegisterModuleType("rust_library_rlib", RustLibraryRlibFactory) + ctx.RegisterModuleType("rust_library_dylib", RustLibraryDylibFactory) + ctx.RegisterModuleType("rust_library_shared", RustLibrarySharedFactory) + ctx.RegisterModuleType("rust_library_static", RustLibraryStaticFactory) + ctx.RegisterModuleType("rust_library_host_shared", RustLibrarySharedHostFactory) + ctx.RegisterModuleType("rust_library_host_static", RustLibraryStaticHostFactory) + ctx.RegisterModuleType("rust_proc_macro", ProcMacroFactory) + ctx.RegisterModuleType("rust_prebuilt_dylib", PrebuiltDylibFactory) + ctx.PreDepsMutators(func(ctx android.RegisterMutatorsContext) { + // rust mutators + ctx.BottomUp("rust_libraries", LibraryMutator).Parallel() + ctx.BottomUp("rust_unit_tests", TestPerSrcMutator).Parallel() + }) + + return ctx +} diff --git a/scripts/Android.bp b/scripts/Android.bp new file mode 100644 index 000000000..1f5503051 --- /dev/null +++ b/scripts/Android.bp @@ -0,0 +1,156 @@ +python_binary_host { + name: "manifest_fixer", + main: "manifest_fixer.py", + srcs: [ + "manifest_fixer.py", + ], + version: { + py2: { + enabled: true, + }, + py3: { + enabled: false, + }, + }, + libs: [ + "manifest_utils", + ], +} + +python_test_host { + name: "manifest_fixer_test", + main: "manifest_fixer_test.py", + srcs: [ + "manifest_fixer_test.py", + "manifest_fixer.py", + ], + version: { + py2: { + enabled: true, + }, + py3: { + enabled: false, + }, + }, + libs: [ + "manifest_utils", + ], + test_suites: ["general-tests"], +} + +python_library_host { + name: "manifest_utils", + srcs: [ + "manifest.py", + ], + version: { + py2: { + enabled: true, + }, + py3: { + enabled: false, + }, + }, +} + +python_binary_host { + name: "manifest_check", + main: "manifest_check.py", + srcs: [ + "manifest_check.py", + ], + version: { + py2: { + enabled: true, + }, + py3: { + enabled: false, + }, + }, + libs: [ + "manifest_utils", + ], +} + +python_test_host { + name: "manifest_check_test", + main: "manifest_check_test.py", + srcs: [ + "manifest_check_test.py", + "manifest_check.py", + ], + version: { + py2: { + enabled: true, + }, + py3: { + enabled: false, + }, + }, + libs: [ + "manifest_utils", + ], + test_suites: ["general-tests"], +} + +python_binary_host { + name: "jsonmodify", + main: "jsonmodify.py", + srcs: [ + "jsonmodify.py", + ], + version: { + py2: { + enabled: true, + }, + py3: { + enabled: false, + }, + } +} + +python_binary_host { + name: "test_config_fixer", + main: "test_config_fixer.py", + srcs: [ + "test_config_fixer.py", + ], + version: { + py2: { + enabled: true, + }, + py3: { + enabled: false, + }, + }, + libs: [ + "manifest_utils", + ], +} + +python_test_host { + name: "test_config_fixer_test", + main: "test_config_fixer_test.py", + srcs: [ + "test_config_fixer_test.py", + "test_config_fixer.py", + ], + version: { + py2: { + enabled: true, + }, + py3: { + enabled: false, + }, + }, + libs: [ + "manifest_utils", + ], + test_suites: ["general-tests"], +} + +python_binary_host { + name: "lint-project-xml", + main: "lint-project-xml.py", + srcs: ["lint-project-xml.py"], +} diff --git a/scripts/OWNERS b/scripts/OWNERS index 076b3f5c1..9e97a6011 100644 --- a/scripts/OWNERS +++ b/scripts/OWNERS @@ -1 +1,2 @@ per-file system-clang-format,system-clang-format-2 = enh@google.com,smoreland@google.com +per-file build-mainline-modules.sh = ngeoffray@google.com,paulduffin@google.com,mast@google.com diff --git a/scripts/TEST_MAPPING b/scripts/TEST_MAPPING new file mode 100644 index 000000000..1b0a2298f --- /dev/null +++ b/scripts/TEST_MAPPING @@ -0,0 +1,12 @@ +{ + "presubmit" : [ + { + "name": "manifest_check_test", + "host": true + }, + { + "name": "manifest_fixer_test", + "host": true + } + ] +} diff --git a/scripts/archive_repack.sh b/scripts/archive_repack.sh new file mode 100755 index 000000000..f09372dd9 --- /dev/null +++ b/scripts/archive_repack.sh @@ -0,0 +1,87 @@ +#!/bin/bash -e + +# Copyright 2019 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Script to extract and repack an archive with specified object files. +# Inputs: +# Environment: +# CLANG_BIN: path to the clang bin directory +# Arguments: +# -i ${file}: input file +# -o ${file}: output file +# -d ${file}: deps file + +set -o pipefail + +OPTSTRING=d:i:o: + +usage() { + cat <<EOF +Usage: archive_repack.sh [options] <objects to repack> + +OPTIONS: + -i <file>: input file + -o <file>: output file + -d <file>: deps file +EOF + exit 1 +} + +while getopts $OPTSTRING opt; do + case "$opt" in + d) depsfile="${OPTARG}" ;; + i) infile="${OPTARG}" ;; + o) outfile="${OPTARG}" ;; + ?) usage ;; + esac +done +shift "$(($OPTIND -1))" + +if [ -z "${infile}" ]; then + echo "-i argument is required" + usage +fi + +if [ -z "${outfile}" ]; then + echo "-o argument is required" + usage +fi + +# Produce deps file +if [ ! -z "${depsfile}" ]; then + cat <<EOF > "${depsfile}" +${outfile}: ${infile} ${CLANG_BIN}/llvm-ar +EOF +fi + +# Get absolute path for outfile and llvm-ar. +LLVM_AR="${PWD}/${CLANG_BIN}/llvm-ar" +if [[ "$outfile" != /* ]]; then + outfile="${PWD}/${outfile}" +fi + +tempdir="${outfile}.tmp" + +# Clean up any previous temporary files. +rm -f "${outfile}" +rm -rf "${tempdir}" + +# Do repack +# We have to change working directory since ar only allows extracting to CWD. +mkdir "${tempdir}" +cp "${infile}" "${tempdir}/archive" +cd "${tempdir}" +"${LLVM_AR}" x "archive" +"${LLVM_AR}" --format=gnu qc "${outfile}" "$@" diff --git a/scripts/build-aml-prebuilts.sh b/scripts/build-aml-prebuilts.sh new file mode 100755 index 000000000..c60eaa1f6 --- /dev/null +++ b/scripts/build-aml-prebuilts.sh @@ -0,0 +1,104 @@ +#!/bin/bash -e + +# This is a wrapper around "m" that builds the given modules in multi-arch mode +# for all architectures supported by Mainline modules. The make (kati) stage is +# skipped, so the build targets in the arguments can only be Soong modules or +# intermediate output files - make targets and normal installed paths are not +# supported. +# +# This script is typically used with "sdk" or "module_export" modules, which +# Soong will install in $OUT_DIR/soong/mainline-sdks (cf +# PathForMainlineSdksInstall in android/paths.go). + +export OUT_DIR=${OUT_DIR:-out} + +if [ -e ${OUT_DIR}/soong/.soong.in_make ]; then + # If ${OUT_DIR} has been created without --skip-make, Soong will create an + # ${OUT_DIR}/soong/build.ninja that leaves out many targets which are + # expected to be supplied by the .mk files, and that might cause errors in + # "m --skip-make" below. We therefore default to a different out dir + # location in that case. + AML_OUT_DIR=out/aml + echo "Avoiding in-make OUT_DIR '${OUT_DIR}' - building in '${AML_OUT_DIR}' instead" + OUT_DIR=${AML_OUT_DIR} +fi + +if [ ! -e "build/envsetup.sh" ]; then + echo "$0 must be run from the top of the tree" + exit 1 +fi + +source build/envsetup.sh + +my_get_build_var() { + # get_build_var will run Soong in normal in-make mode where it creates + # .soong.in_make. That would clobber our real out directory, so we need to + # run it in a different one. + OUT_DIR=${OUT_DIR}/get_build_var get_build_var "$@" +} + +readonly PLATFORM_SDK_VERSION="$(my_get_build_var PLATFORM_SDK_VERSION)" +readonly PLATFORM_VERSION="$(my_get_build_var PLATFORM_VERSION)" +PLATFORM_VERSION_ALL_CODENAMES="$(my_get_build_var PLATFORM_VERSION_ALL_CODENAMES)" + +# PLATFORM_VERSION_ALL_CODENAMES is a comma separated list like O,P. We need to +# turn this into ["O","P"]. +PLATFORM_VERSION_ALL_CODENAMES="${PLATFORM_VERSION_ALL_CODENAMES/,/'","'}" +PLATFORM_VERSION_ALL_CODENAMES="[\"${PLATFORM_VERSION_ALL_CODENAMES}\"]" + +# Logic from build/make/core/goma.mk +if [ "${USE_GOMA}" = true ]; then + if [ -n "${GOMA_DIR}" ]; then + goma_dir="${GOMA_DIR}" + else + goma_dir="${HOME}/goma" + fi + GOMA_CC="${goma_dir}/gomacc" + export CC_WRAPPER="${CC_WRAPPER}${CC_WRAPPER:+ }${GOMA_CC}" + export CXX_WRAPPER="${CXX_WRAPPER}${CXX_WRAPPER:+ }${GOMA_CC}" + export JAVAC_WRAPPER="${JAVAC_WRAPPER}${JAVAC_WRAPPER:+ }${GOMA_CC}" +else + USE_GOMA=false +fi + +readonly SOONG_OUT=${OUT_DIR}/soong +mkdir -p ${SOONG_OUT} +readonly SOONG_VARS=${SOONG_OUT}/soong.variables + +# Aml_abis: true +# - This flag configures Soong to compile for all architectures required for +# Mainline modules. +# CrossHost: linux_bionic +# CrossHostArch: x86_64 +# - Enable Bionic on host as ART needs prebuilts for it. +cat > ${SOONG_VARS}.new << EOF +{ + "Platform_sdk_version": ${PLATFORM_SDK_VERSION}, + "Platform_sdk_codename": "${PLATFORM_VERSION}", + "Platform_version_active_codenames": ${PLATFORM_VERSION_ALL_CODENAMES}, + + "DeviceName": "generic_arm64", + "HostArch": "x86_64", + "HostSecondaryArch": "x86", + "CrossHost": "linux_bionic", + "CrossHostArch": "x86_64", + "Aml_abis": true, + + "UseGoma": ${USE_GOMA} +} +EOF + +if [ -f ${SOONG_VARS} ] && cmp -s ${SOONG_VARS} ${SOONG_VARS}.new; then + # Don't touch soong.variables if we don't have to, to avoid Soong rebuilding + # the ninja file when it isn't necessary. + rm ${SOONG_VARS}.new +else + mv ${SOONG_VARS}.new ${SOONG_VARS} +fi + +# We use force building LLVM components flag (even though we actually don't +# compile them) because we don't have bionic host prebuilts +# for them. +export FORCE_BUILD_LLVM_COMPONENTS=true + +m --skip-make "$@" diff --git a/scripts/build-mainline-modules.sh b/scripts/build-mainline-modules.sh new file mode 100755 index 000000000..f836ea9fa --- /dev/null +++ b/scripts/build-mainline-modules.sh @@ -0,0 +1,68 @@ +#!/bin/bash -e + +# Non exhaustive list of modules where we want prebuilts. More can be added as +# needed. +MAINLINE_MODULES=( + com.android.art.debug + com.android.art.release + com.android.art.testing + com.android.conscrypt + com.android.runtime + com.android.tzdata + com.android.i18n +) + +# List of SDKs and module exports we know of. +MODULES_SDK_AND_EXPORTS=( + art-module-sdk + art-module-test-exports + conscrypt-module-sdk + conscrypt-module-test-exports + conscrypt-module-host-exports + runtime-module-sdk +) + +# We want to create apex modules for all supported architectures. +PRODUCTS=( + aosp_arm + aosp_arm64 + aosp_x86 + aosp_x86_64 +) + +if [ ! -e "build/make/core/Makefile" ]; then + echo "$0 must be run from the top of the tree" + exit 1 +fi + +echo_and_run() { + echo "$*" + "$@" +} + +OUT_DIR=$(source build/envsetup.sh > /dev/null; TARGET_PRODUCT= get_build_var OUT_DIR) +DIST_DIR=$(source build/envsetup.sh > /dev/null; TARGET_PRODUCT= get_build_var DIST_DIR) + +for product in "${PRODUCTS[@]}"; do + echo_and_run build/soong/soong_ui.bash --make-mode $@ \ + TARGET_PRODUCT=${product} \ + ${MAINLINE_MODULES[@]} + + PRODUCT_OUT=$(source build/envsetup.sh > /dev/null; TARGET_PRODUCT=${product} get_build_var PRODUCT_OUT) + TARGET_ARCH=$(source build/envsetup.sh > /dev/null; TARGET_PRODUCT=${product} get_build_var TARGET_ARCH) + rm -rf ${DIST_DIR}/${TARGET_ARCH}/ + mkdir -p ${DIST_DIR}/${TARGET_ARCH}/ + for module in "${MAINLINE_MODULES[@]}"; do + echo_and_run cp ${PWD}/${PRODUCT_OUT}/system/apex/${module}.apex ${DIST_DIR}/${TARGET_ARCH}/ + done +done + + +# Create multi-archs SDKs in a different out directory. The multi-arch script +# uses Soong in --skip-make mode which cannot use the same directory as normal +# mode with make. +export OUT_DIR=${OUT_DIR}/aml +echo_and_run build/soong/scripts/build-aml-prebuilts.sh ${MODULES_SDK_AND_EXPORTS[@]} + +rm -rf ${DIST_DIR}/mainline-sdks +echo_and_run cp -R ${OUT_DIR}/soong/mainline-sdks ${DIST_DIR} diff --git a/scripts/build-ndk-prebuilts.sh b/scripts/build-ndk-prebuilts.sh index 947458a62..b6ed65940 100755 --- a/scripts/build-ndk-prebuilts.sh +++ b/scripts/build-ndk-prebuilts.sh @@ -25,7 +25,7 @@ source build/envsetup.sh PLATFORM_SDK_VERSION=$(get_build_var PLATFORM_SDK_VERSION) PLATFORM_VERSION_ALL_CODENAMES=$(get_build_var PLATFORM_VERSION_ALL_CODENAMES) -# PLATFORM_VERSION_ALL_CODESNAMES is a comma separated list like O,P. We need to +# PLATFORM_VERSION_ALL_CODENAMES is a comma separated list like O,P. We need to # turn this into ["O","P"]. PLATFORM_VERSION_ALL_CODENAMES=${PLATFORM_VERSION_ALL_CODENAMES/,/'","'} PLATFORM_VERSION_ALL_CODENAMES="[\"${PLATFORM_VERSION_ALL_CODENAMES}\"]" diff --git a/scripts/build_broken_logs.go b/scripts/build_broken_logs.go index f081f262d..8021e55d0 100644 --- a/scripts/build_broken_logs.go +++ b/scripts/build_broken_logs.go @@ -60,37 +60,11 @@ var buildBrokenSettings = []struct { warnings []string }{ { - name: "BUILD_BROKEN_DUP_COPY_HEADERS", - behavior: DefaultDeprecated, - warnings: []string{"Duplicate header copy:"}, - }, - { name: "BUILD_BROKEN_DUP_RULES", behavior: DefaultFalse, warnings: []string{"overriding commands for target"}, }, { - name: "BUILD_BROKEN_ANDROIDMK_EXPORTS", - behavior: DefaultFalse, - warnings: []string{"export_keyword"}, - }, - { - name: "BUILD_BROKEN_PHONY_TARGETS", - behavior: DefaultFalse, - warnings: []string{ - "depends on PHONY target", - "looks like a real file", - "writing to readonly directory", - }, - }, - { - name: "BUILD_BROKEN_ENG_DEBUG_TAGS", - behavior: DefaultTrue, - warnings: []string{ - "Changes.md#LOCAL_MODULE_TAGS", - }, - }, - { name: "BUILD_BROKEN_USES_NETWORK", behavior: DefaultDeprecated, }, diff --git a/scripts/clang-tidy.sh b/scripts/clang-tidy.sh deleted file mode 100755 index 04d0bdd9a..000000000 --- a/scripts/clang-tidy.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -e - -# Copyright 2018 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Wrapper script to remove clang compiler flags rejected by clang-tidy. -# Inputs: -# Environment: -# CLANG_TIDY: path to the real clang-tidy program - -# clang-tidy doesn't recognize every flag that clang compiler does. -# It gives clang-diagnostic-unused-command-line-argument warnings -# to -Wa,* flags. -# The -flto flags caused clang-tidy to ignore the -I flags, -# see https://bugs.llvm.org/show_bug.cgi?id=38332. -# -fsanitize and -fwhole-program-vtables need -flto. -args=("${@}") -n=${#args[@]} -for ((i=0; i<$n; ++i)); do - case ${args[i]} in - -Wa,*|-flto|-flto=*|-fsanitize=*|-fsanitize-*|-fwhole-program-vtables) - unset args[i] - ;; - esac -done -${CLANG_TIDY} "${args[@]}" diff --git a/scripts/freeze-sysprop-api-files.sh b/scripts/freeze-sysprop-api-files.sh new file mode 100755 index 000000000..1b2ff7c72 --- /dev/null +++ b/scripts/freeze-sysprop-api-files.sh @@ -0,0 +1,39 @@ +#!/bin/bash -e + +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script freezes APIs of a sysprop_library after checking compatibility +# between latest API and current API. +# +# Usage: freeze-sysprop-api-files.sh <modulePath> <moduleName> +# +# <modulePath>: the directory, either relative or absolute, which holds the +# Android.bp file defining sysprop_library. +# +# <moduleName>: the name of sysprop_library to freeze API. +# +# Example: +# $ . build/envsetup.sh && lunch aosp_arm64-user +# $ . build/soong/scripts/freeze-sysprop-api-files.sh \ +# system/libsysprop/srcs PlatformProperties + +if [[ -z "$1" || -z "$2" ]]; then + echo "usage: $0 <modulePath> <moduleName>" >&2 + exit 1 +fi + +api_dir=$1/api + +m "$2-check-api" && cp -f "${api_dir}/$2-current.txt" "${api_dir}/$2-latest.txt" diff --git a/scripts/gen-java-current-api-files.sh b/scripts/gen-java-current-api-files.sh index 517d3916e..547387a46 100755 --- a/scripts/gen-java-current-api-files.sh +++ b/scripts/gen-java-current-api-files.sh @@ -15,15 +15,16 @@ # limitations under the License. if [[ -z "$1" ]]; then - echo "usage: $0 <modulePath>" >&2 + echo "usage: $0 <modulePath> scopes..." >&2 exit 1 fi -api_dir=$1/api +api_dir=$1 +shift mkdir -p "$api_dir" -scopes=("" system- test-) +scopes=("" "$@") apis=(current removed) for scope in "${scopes[@]}"; do @@ -31,3 +32,4 @@ for scope in "${scopes[@]}"; do touch "${api_dir}/${scope}${api}.txt" done done + diff --git a/scripts/gen-kotlin-build-file.sh b/scripts/gen-kotlin-build-file.sh index 1e03f72e0..177ca1b04 100755 --- a/scripts/gen-kotlin-build-file.sh +++ b/scripts/gen-kotlin-build-file.sh @@ -17,7 +17,7 @@ # Generates kotlinc module xml file to standard output based on rsp files if [[ -z "$1" ]]; then - echo "usage: $0 <classpath> <outDir> <rspFiles>..." >&2 + echo "usage: $0 <classpath> <name> <outDir> <rspFiles>..." >&2 exit 1 fi @@ -27,8 +27,9 @@ if [[ $1 == "-classpath" ]]; then fi; classpath=$1 -out_dir=$2 -shift 2 +name=$2 +out_dir=$3 +shift 3 # Path in the build file may be relative to the build file, we need to make them # absolute @@ -44,7 +45,7 @@ get_abs_path () { } # Print preamble -echo "<modules><module name=\"name\" type=\"java-production\" outputDir=\"${out_dir}\">" +echo "<modules><module name=\"${name}\" type=\"java-production\" outputDir=\"${out_dir}\">" # Print classpath entries for file in $(echo "$classpath" | tr ":" "\n"); do diff --git a/scripts/gen-sysprop-api-files.sh b/scripts/gen-sysprop-api-files.sh new file mode 100755 index 000000000..a4cb50649 --- /dev/null +++ b/scripts/gen-sysprop-api-files.sh @@ -0,0 +1,26 @@ +#!/bin/bash -e + +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [[ -z "$1" || -z "$2" ]]; then + echo "usage: $0 <modulePath> <moduleName>" >&2 + exit 1 +fi + +api_dir=$1/api + +mkdir -p "$api_dir" +touch "${api_dir}/$2-current.txt" +touch "${api_dir}/$2-latest.txt" diff --git a/scripts/gen_sorted_bss_symbols.sh b/scripts/gen_sorted_bss_symbols.sh new file mode 100755 index 000000000..244ed0dea --- /dev/null +++ b/scripts/gen_sorted_bss_symbols.sh @@ -0,0 +1,28 @@ +#!/bin/bash -e + +# Copyright 2019 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Script to generate a symbol ordering file that sorts bss section symbols by +# their sizes. +# Inputs: +# Environment: +# CROSS_COMPILE: prefix added to nm tools +# Arguments: +# $1: Input ELF file +# $2: Output symbol ordering file + +set -o pipefail + +${CROSS_COMPILE}nm --size-sort $1 | awk '{if ($2 == "b" || $2 == "B") print $3}' > $2 diff --git a/scripts/jar-wrapper.sh b/scripts/jar-wrapper.sh index 71c1d9067..b46804157 100644 --- a/scripts/jar-wrapper.sh +++ b/scripts/jar-wrapper.sh @@ -48,11 +48,11 @@ if [ ! -r "${jardir}/${jarfile}" ]; then exit 1 fi -javaOpts="" +declare -a javaOpts=() while expr "x$1" : 'x-J' >/dev/null; do - opt=`expr "$1" : '-J\(.*\)'` - javaOpts="${javaOpts} -${opt}" + opt=`expr "$1" : '-J-\{0,1\}\(.*\)'` + javaOpts+=("-${opt}") shift done -exec java ${javaOpts} -jar ${jardir}/${jarfile} "$@" +exec java "${javaOpts[@]}" -jar ${jardir}/${jarfile} "$@" diff --git a/scripts/jsonmodify.py b/scripts/jsonmodify.py new file mode 100755 index 000000000..4b2c3c250 --- /dev/null +++ b/scripts/jsonmodify.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python +# +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import argparse +import collections +import json +import sys + +def follow_path(obj, path): + cur = obj + last_key = None + for key in path.split('.'): + if last_key: + if last_key not in cur: + return None,None + cur = cur[last_key] + last_key = key + if last_key not in cur: + return None,None + return cur, last_key + + +def ensure_path(obj, path): + cur = obj + last_key = None + for key in path.split('.'): + if last_key: + if last_key not in cur: + cur[last_key] = dict() + cur = cur[last_key] + last_key = key + return cur, last_key + + +class SetValue(str): + def apply(self, obj, val): + cur, key = ensure_path(obj, self) + cur[key] = val + + +class Replace(str): + def apply(self, obj, val): + cur, key = follow_path(obj, self) + if cur: + cur[key] = val + + +class Remove(str): + def apply(self, obj): + cur, key = follow_path(obj, self) + if cur: + del cur[key] + + +class AppendList(str): + def apply(self, obj, *args): + cur, key = ensure_path(obj, self) + if key not in cur: + cur[key] = list() + if not isinstance(cur[key], list): + raise ValueError(self + " should be a array.") + cur[key].extend(args) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('-o', '--out', + help='write result to a file. If omitted, print to stdout', + metavar='output', + action='store') + parser.add_argument('input', nargs='?', help='JSON file') + parser.add_argument("-v", "--value", type=SetValue, + help='set value of the key specified by path. If path doesn\'t exist, creates new one.', + metavar=('path', 'value'), + nargs=2, dest='patch', default=[], action='append') + parser.add_argument("-s", "--replace", type=Replace, + help='replace value of the key specified by path. If path doesn\'t exist, no op.', + metavar=('path', 'value'), + nargs=2, dest='patch', action='append') + parser.add_argument("-r", "--remove", type=Remove, + help='remove the key specified by path. If path doesn\'t exist, no op.', + metavar='path', + nargs=1, dest='patch', action='append') + parser.add_argument("-a", "--append_list", type=AppendList, + help='append values to the list specified by path. If path doesn\'t exist, creates new list for it.', + metavar=('path', 'value'), + nargs='+', dest='patch', default=[], action='append') + args = parser.parse_args() + + if args.input: + with open(args.input) as f: + obj = json.load(f, object_pairs_hook=collections.OrderedDict) + else: + obj = json.load(sys.stdin, object_pairs_hook=collections.OrderedDict) + + for p in args.patch: + p[0].apply(obj, *p[1:]) + + if args.out: + with open(args.out, "w") as f: + json.dump(obj, f, indent=2) + else: + print(json.dumps(obj, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/scripts/lint-project-xml.py b/scripts/lint-project-xml.py new file mode 100755 index 000000000..38c57cadf --- /dev/null +++ b/scripts/lint-project-xml.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""This file generates project.xml and lint.xml files used to drive the Android Lint CLI tool.""" + +import argparse + + +def check_action(check_type): + """ + Returns an action that appends a tuple of check_type and the argument to the dest. + """ + class CheckAction(argparse.Action): + def __init__(self, option_strings, dest, nargs=None, **kwargs): + if nargs is not None: + raise ValueError("nargs must be None, was %s" % nargs) + super(CheckAction, self).__init__(option_strings, dest, **kwargs) + def __call__(self, parser, namespace, values, option_string=None): + checks = getattr(namespace, self.dest, []) + checks.append((check_type, values)) + setattr(namespace, self.dest, checks) + return CheckAction + + +def parse_args(): + """Parse commandline arguments.""" + + def convert_arg_line_to_args(arg_line): + for arg in arg_line.split(): + if arg.startswith('#'): + return + if not arg.strip(): + continue + yield arg + + parser = argparse.ArgumentParser(fromfile_prefix_chars='@') + parser.convert_arg_line_to_args = convert_arg_line_to_args + parser.add_argument('--project_out', dest='project_out', + help='file to which the project.xml contents will be written.') + parser.add_argument('--config_out', dest='config_out', + help='file to which the lint.xml contents will be written.') + parser.add_argument('--name', dest='name', + help='name of the module.') + parser.add_argument('--srcs', dest='srcs', action='append', default=[], + help='file containing whitespace separated list of source files.') + parser.add_argument('--generated_srcs', dest='generated_srcs', action='append', default=[], + help='file containing whitespace separated list of generated source files.') + parser.add_argument('--resources', dest='resources', action='append', default=[], + help='file containing whitespace separated list of resource files.') + parser.add_argument('--classes', dest='classes', action='append', default=[], + help='file containing the module\'s classes.') + parser.add_argument('--classpath', dest='classpath', action='append', default=[], + help='file containing classes from dependencies.') + parser.add_argument('--extra_checks_jar', dest='extra_checks_jars', action='append', default=[], + help='file containing extra lint checks.') + parser.add_argument('--manifest', dest='manifest', + help='file containing the module\'s manifest.') + parser.add_argument('--merged_manifest', dest='merged_manifest', + help='file containing merged manifest for the module and its dependencies.') + parser.add_argument('--library', dest='library', action='store_true', + help='mark the module as a library.') + parser.add_argument('--test', dest='test', action='store_true', + help='mark the module as a test.') + parser.add_argument('--cache_dir', dest='cache_dir', + help='directory to use for cached file.') + parser.add_argument('--root_dir', dest='root_dir', + help='directory to use for root dir.') + group = parser.add_argument_group('check arguments', 'later arguments override earlier ones.') + group.add_argument('--fatal_check', dest='checks', action=check_action('fatal'), default=[], + help='treat a lint issue as a fatal error.') + group.add_argument('--error_check', dest='checks', action=check_action('error'), default=[], + help='treat a lint issue as an error.') + group.add_argument('--warning_check', dest='checks', action=check_action('warning'), default=[], + help='treat a lint issue as a warning.') + group.add_argument('--disable_check', dest='checks', action=check_action('ignore'), default=[], + help='disable a lint issue.') + return parser.parse_args() + + +class NinjaRspFileReader: + """ + Reads entries from a Ninja rsp file. Ninja escapes any entries in the file that contain a + non-standard character by surrounding the whole entry with single quotes, and then replacing + any single quotes in the entry with the escape sequence '\''. + """ + + def __init__(self, filename): + self.f = open(filename, 'r') + self.r = self.character_reader(self.f) + + def __iter__(self): + return self + + def character_reader(self, f): + """Turns a file into a generator that returns one character at a time.""" + while True: + c = f.read(1) + if c: + yield c + else: + return + + def __next__(self): + entry = self.read_entry() + if entry: + return entry + else: + raise StopIteration + + def read_entry(self): + c = next(self.r, "") + if not c: + return "" + elif c == "'": + return self.read_quoted_entry() + else: + entry = c + for c in self.r: + if c == " " or c == "\n": + break + entry += c + return entry + + def read_quoted_entry(self): + entry = "" + for c in self.r: + if c == "'": + # Either the end of the quoted entry, or the beginning of an escape sequence, read the next + # character to find out. + c = next(self.r) + if not c or c == " " or c == "\n": + # End of the item + return entry + elif c == "\\": + # Escape sequence, expect a ' + c = next(self.r) + if c != "'": + # Malformed escape sequence + raise "malformed escape sequence %s'\\%s" % (entry, c) + entry += "'" + else: + raise "malformed escape sequence %s'%s" % (entry, c) + else: + entry += c + raise "unterminated quoted entry %s" % entry + + +def write_project_xml(f, args): + test_attr = "test='true' " if args.test else "" + + f.write("<?xml version='1.0' encoding='utf-8'?>\n") + f.write("<project>\n") + if args.root_dir: + f.write(" <root dir='%s' />\n" % args.root_dir) + f.write(" <module name='%s' android='true' %sdesugar='full' >\n" % (args.name, "library='true' " if args.library else "")) + if args.manifest: + f.write(" <manifest file='%s' %s/>\n" % (args.manifest, test_attr)) + if args.merged_manifest: + f.write(" <merged-manifest file='%s' %s/>\n" % (args.merged_manifest, test_attr)) + for src_file in args.srcs: + for src in NinjaRspFileReader(src_file): + f.write(" <src file='%s' %s/>\n" % (src, test_attr)) + for src_file in args.generated_srcs: + for src in NinjaRspFileReader(src_file): + f.write(" <src file='%s' generated='true' %s/>\n" % (src, test_attr)) + for res_file in args.resources: + for res in NinjaRspFileReader(res_file): + f.write(" <resource file='%s' %s/>\n" % (res, test_attr)) + for classes in args.classes: + f.write(" <classes jar='%s' />\n" % classes) + for classpath in args.classpath: + f.write(" <classpath jar='%s' />\n" % classpath) + for extra in args.extra_checks_jars: + f.write(" <lint-checks jar='%s' />\n" % extra) + f.write(" </module>\n") + if args.cache_dir: + f.write(" <cache dir='%s'/>\n" % args.cache_dir) + f.write("</project>\n") + + +def write_config_xml(f, args): + f.write("<?xml version='1.0' encoding='utf-8'?>\n") + f.write("<lint>\n") + for check in args.checks: + f.write(" <issue id='%s' severity='%s' />\n" % (check[1], check[0])) + f.write("</lint>\n") + + +def main(): + """Program entry point.""" + args = parse_args() + + if args.project_out: + with open(args.project_out, 'w') as f: + write_project_xml(f, args) + + if args.config_out: + with open(args.config_out, 'w') as f: + write_config_xml(f, args) + + +if __name__ == '__main__': + main() diff --git a/scripts/manifest.py b/scripts/manifest.py new file mode 100755 index 000000000..04f7405df --- /dev/null +++ b/scripts/manifest.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python +# +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""A tool for inserting values from the build system into a manifest or a test config.""" + +from __future__ import print_function +from xml.dom import minidom + + +android_ns = 'http://schemas.android.com/apk/res/android' + + +def get_children_with_tag(parent, tag_name): + children = [] + for child in parent.childNodes: + if child.nodeType == minidom.Node.ELEMENT_NODE and \ + child.tagName == tag_name: + children.append(child) + return children + + +def find_child_with_attribute(element, tag_name, namespace_uri, + attr_name, value): + for child in get_children_with_tag(element, tag_name): + attr = child.getAttributeNodeNS(namespace_uri, attr_name) + if attr is not None and attr.value == value: + return child + return None + + +def parse_manifest(doc): + """Get the manifest element.""" + + manifest = doc.documentElement + if manifest.tagName != 'manifest': + raise RuntimeError('expected manifest tag at root') + return manifest + + +def ensure_manifest_android_ns(doc): + """Make sure the manifest tag defines the android namespace.""" + + manifest = parse_manifest(doc) + + ns = manifest.getAttributeNodeNS(minidom.XMLNS_NAMESPACE, 'android') + if ns is None: + attr = doc.createAttributeNS(minidom.XMLNS_NAMESPACE, 'xmlns:android') + attr.value = android_ns + manifest.setAttributeNode(attr) + elif ns.value != android_ns: + raise RuntimeError('manifest tag has incorrect android namespace ' + + ns.value) + + +def parse_test_config(doc): + """ Get the configuration element. """ + + test_config = doc.documentElement + if test_config.tagName != 'configuration': + raise RuntimeError('expected configuration tag at root') + return test_config + + +def as_int(s): + try: + i = int(s) + except ValueError: + return s, False + return i, True + + +def compare_version_gt(a, b): + """Compare two SDK versions. + + Compares a and b, treating codenames like 'Q' as higher + than numerical versions like '28'. + + Returns True if a > b + + Args: + a: value to compare + b: value to compare + Returns: + True if a is a higher version than b + """ + + a, a_is_int = as_int(a.upper()) + b, b_is_int = as_int(b.upper()) + + if a_is_int == b_is_int: + # Both are codenames or both are versions, compare directly + return a > b + else: + # One is a codename, the other is not. Return true if + # b is an integer version + return b_is_int + + +def get_indent(element, default_level): + indent = '' + if element is not None and element.nodeType == minidom.Node.TEXT_NODE: + text = element.nodeValue + indent = text[:len(text)-len(text.lstrip())] + if not indent or indent == '\n': + # 1 indent = 4 space + indent = '\n' + (' ' * default_level * 4) + return indent + + +def write_xml(f, doc): + f.write('<?xml version="1.0" encoding="utf-8"?>\n') + for node in doc.childNodes: + f.write(node.toxml(encoding='utf-8') + '\n') diff --git a/scripts/manifest_check.py b/scripts/manifest_check.py new file mode 100755 index 000000000..9122da1fb --- /dev/null +++ b/scripts/manifest_check.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python +# +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""A tool for checking that a manifest agrees with the build system.""" + +from __future__ import print_function + +import argparse +import sys +from xml.dom import minidom + + +from manifest import android_ns +from manifest import get_children_with_tag +from manifest import parse_manifest +from manifest import write_xml + + +class ManifestMismatchError(Exception): + pass + + +def parse_args(): + """Parse commandline arguments.""" + + parser = argparse.ArgumentParser() + parser.add_argument('--uses-library', dest='uses_libraries', + action='append', + help='specify uses-library entries known to the build system') + parser.add_argument('--optional-uses-library', + dest='optional_uses_libraries', + action='append', + help='specify uses-library entries known to the build system with required:false') + parser.add_argument('--enforce-uses-libraries', + dest='enforce_uses_libraries', + action='store_true', + help='check the uses-library entries known to the build system against the manifest') + parser.add_argument('--extract-target-sdk-version', + dest='extract_target_sdk_version', + action='store_true', + help='print the targetSdkVersion from the manifest') + parser.add_argument('--output', '-o', dest='output', help='output AndroidManifest.xml file') + parser.add_argument('input', help='input AndroidManifest.xml file') + return parser.parse_args() + + +def enforce_uses_libraries(doc, uses_libraries, optional_uses_libraries): + """Verify that the <uses-library> tags in the manifest match those provided by the build system. + + Args: + doc: The XML document. + uses_libraries: The names of <uses-library> tags known to the build system + optional_uses_libraries: The names of <uses-library> tags with required:fals + known to the build system + Raises: + RuntimeError: Invalid manifest + ManifestMismatchError: Manifest does not match + """ + + manifest = parse_manifest(doc) + elems = get_children_with_tag(manifest, 'application') + application = elems[0] if len(elems) == 1 else None + if len(elems) > 1: + raise RuntimeError('found multiple <application> tags') + elif not elems: + if uses_libraries or optional_uses_libraries: + raise ManifestMismatchError('no <application> tag found') + return + + verify_uses_library(application, uses_libraries, optional_uses_libraries) + + +def verify_uses_library(application, uses_libraries, optional_uses_libraries): + """Verify that the uses-library values known to the build system match the manifest. + + Args: + application: the <application> tag in the manifest. + uses_libraries: the names of expected <uses-library> tags. + optional_uses_libraries: the names of expected <uses-library> tags with required="false". + Raises: + ManifestMismatchError: Manifest does not match + """ + + if uses_libraries is None: + uses_libraries = [] + + if optional_uses_libraries is None: + optional_uses_libraries = [] + + manifest_uses_libraries, manifest_optional_uses_libraries = parse_uses_library(application) + + err = [] + if manifest_uses_libraries != uses_libraries: + err.append('Expected required <uses-library> tags "%s", got "%s"' % + (', '.join(uses_libraries), ', '.join(manifest_uses_libraries))) + + if manifest_optional_uses_libraries != optional_uses_libraries: + err.append('Expected optional <uses-library> tags "%s", got "%s"' % + (', '.join(optional_uses_libraries), ', '.join(manifest_optional_uses_libraries))) + + if err: + raise ManifestMismatchError('\n'.join(err)) + + +def parse_uses_library(application): + """Extract uses-library tags from the manifest. + + Args: + application: the <application> tag in the manifest. + """ + + libs = get_children_with_tag(application, 'uses-library') + + uses_libraries = [uses_library_name(x) for x in libs if uses_library_required(x)] + optional_uses_libraries = [uses_library_name(x) for x in libs if not uses_library_required(x)] + + return first_unique_elements(uses_libraries), first_unique_elements(optional_uses_libraries) + + +def first_unique_elements(l): + result = [] + [result.append(x) for x in l if x not in result] + return result + + +def uses_library_name(lib): + """Extract the name attribute of a uses-library tag. + + Args: + lib: a <uses-library> tag. + """ + name = lib.getAttributeNodeNS(android_ns, 'name') + return name.value if name is not None else "" + + +def uses_library_required(lib): + """Extract the required attribute of a uses-library tag. + + Args: + lib: a <uses-library> tag. + """ + required = lib.getAttributeNodeNS(android_ns, 'required') + return (required.value == 'true') if required is not None else True + + +def extract_target_sdk_version(doc): + """Returns the targetSdkVersion from the manifest. + + Args: + doc: The XML document. + Raises: + RuntimeError: invalid manifest + """ + + manifest = parse_manifest(doc) + + # Get or insert the uses-sdk element + uses_sdk = get_children_with_tag(manifest, 'uses-sdk') + if len(uses_sdk) > 1: + raise RuntimeError('found multiple uses-sdk elements') + elif len(uses_sdk) == 0: + raise RuntimeError('missing uses-sdk element') + + uses_sdk = uses_sdk[0] + + min_attr = uses_sdk.getAttributeNodeNS(android_ns, 'minSdkVersion') + if min_attr is None: + raise RuntimeError('minSdkVersion is not specified') + + target_attr = uses_sdk.getAttributeNodeNS(android_ns, 'targetSdkVersion') + if target_attr is None: + target_attr = min_attr + + return target_attr.value + + +def main(): + """Program entry point.""" + try: + args = parse_args() + + doc = minidom.parse(args.input) + + if args.enforce_uses_libraries: + enforce_uses_libraries(doc, + args.uses_libraries, + args.optional_uses_libraries) + + if args.extract_target_sdk_version: + print(extract_target_sdk_version(doc)) + + if args.output: + with open(args.output, 'wb') as f: + write_xml(f, doc) + + # pylint: disable=broad-except + except Exception as err: + print('error: ' + str(err), file=sys.stderr) + sys.exit(-1) + +if __name__ == '__main__': + main() diff --git a/scripts/manifest_check_test.py b/scripts/manifest_check_test.py new file mode 100755 index 000000000..7baad5d38 --- /dev/null +++ b/scripts/manifest_check_test.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python +# +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Unit tests for manifest_fixer.py.""" + +import sys +import unittest +from xml.dom import minidom + +import manifest_check + +sys.dont_write_bytecode = True + + +def uses_library(name, attr=''): + return '<uses-library android:name="%s"%s />' % (name, attr) + + +def required(value): + return ' android:required="%s"' % ('true' if value else 'false') + + +class EnforceUsesLibrariesTest(unittest.TestCase): + """Unit tests for add_extract_native_libs function.""" + + def run_test(self, input_manifest, uses_libraries=None, optional_uses_libraries=None): + doc = minidom.parseString(input_manifest) + try: + manifest_check.enforce_uses_libraries(doc, uses_libraries, optional_uses_libraries) + return True + except manifest_check.ManifestMismatchError: + return False + + manifest_tmpl = ( + '<?xml version="1.0" encoding="utf-8"?>\n' + '<manifest xmlns:android="http://schemas.android.com/apk/res/android">\n' + ' <application>\n' + ' %s\n' + ' </application>\n' + '</manifest>\n') + + def test_uses_library(self): + manifest_input = self.manifest_tmpl % (uses_library('foo')) + matches = self.run_test(manifest_input, uses_libraries=['foo']) + self.assertTrue(matches) + + def test_uses_library_required(self): + manifest_input = self.manifest_tmpl % (uses_library('foo', required(True))) + matches = self.run_test(manifest_input, uses_libraries=['foo']) + self.assertTrue(matches) + + def test_optional_uses_library(self): + manifest_input = self.manifest_tmpl % (uses_library('foo', required(False))) + matches = self.run_test(manifest_input, optional_uses_libraries=['foo']) + self.assertTrue(matches) + + def test_expected_uses_library(self): + manifest_input = self.manifest_tmpl % (uses_library('foo', required(False))) + matches = self.run_test(manifest_input, uses_libraries=['foo']) + self.assertFalse(matches) + + def test_expected_optional_uses_library(self): + manifest_input = self.manifest_tmpl % (uses_library('foo')) + matches = self.run_test(manifest_input, optional_uses_libraries=['foo']) + self.assertFalse(matches) + + def test_missing_uses_library(self): + manifest_input = self.manifest_tmpl % ('') + matches = self.run_test(manifest_input, uses_libraries=['foo']) + self.assertFalse(matches) + + def test_missing_optional_uses_library(self): + manifest_input = self.manifest_tmpl % ('') + matches = self.run_test(manifest_input, optional_uses_libraries=['foo']) + self.assertFalse(matches) + + def test_extra_uses_library(self): + manifest_input = self.manifest_tmpl % (uses_library('foo')) + matches = self.run_test(manifest_input) + self.assertFalse(matches) + + def test_extra_optional_uses_library(self): + manifest_input = self.manifest_tmpl % (uses_library('foo', required(False))) + matches = self.run_test(manifest_input) + self.assertFalse(matches) + + def test_multiple_uses_library(self): + manifest_input = self.manifest_tmpl % ('\n'.join([uses_library('foo'), + uses_library('bar')])) + matches = self.run_test(manifest_input, uses_libraries=['foo', 'bar']) + self.assertTrue(matches) + + def test_multiple_optional_uses_library(self): + manifest_input = self.manifest_tmpl % ('\n'.join([uses_library('foo', required(False)), + uses_library('bar', required(False))])) + matches = self.run_test(manifest_input, optional_uses_libraries=['foo', 'bar']) + self.assertTrue(matches) + + def test_order_uses_library(self): + manifest_input = self.manifest_tmpl % ('\n'.join([uses_library('foo'), + uses_library('bar')])) + matches = self.run_test(manifest_input, uses_libraries=['bar', 'foo']) + self.assertFalse(matches) + + def test_order_optional_uses_library(self): + manifest_input = self.manifest_tmpl % ('\n'.join([uses_library('foo', required(False)), + uses_library('bar', required(False))])) + matches = self.run_test(manifest_input, optional_uses_libraries=['bar', 'foo']) + self.assertFalse(matches) + + def test_duplicate_uses_library(self): + manifest_input = self.manifest_tmpl % ('\n'.join([uses_library('foo'), + uses_library('foo')])) + matches = self.run_test(manifest_input, uses_libraries=['foo']) + self.assertTrue(matches) + + def test_duplicate_optional_uses_library(self): + manifest_input = self.manifest_tmpl % ('\n'.join([uses_library('foo', required(False)), + uses_library('foo', required(False))])) + matches = self.run_test(manifest_input, optional_uses_libraries=['foo']) + self.assertTrue(matches) + + def test_mixed(self): + manifest_input = self.manifest_tmpl % ('\n'.join([uses_library('foo'), + uses_library('bar', required(False))])) + matches = self.run_test(manifest_input, uses_libraries=['foo'], + optional_uses_libraries=['bar']) + self.assertTrue(matches) + + +class ExtractTargetSdkVersionTest(unittest.TestCase): + def test_target_sdk_version(self): + manifest = ( + '<?xml version="1.0" encoding="utf-8"?>\n' + '<manifest xmlns:android="http://schemas.android.com/apk/res/android">\n' + ' <uses-sdk android:minSdkVersion="28" android:targetSdkVersion="29" />\n' + '</manifest>\n') + doc = minidom.parseString(manifest) + target_sdk_version = manifest_check.extract_target_sdk_version(doc) + self.assertEqual(target_sdk_version, '29') + + def test_min_sdk_version(self): + manifest = ( + '<?xml version="1.0" encoding="utf-8"?>\n' + '<manifest xmlns:android="http://schemas.android.com/apk/res/android">\n' + ' <uses-sdk android:minSdkVersion="28" />\n' + '</manifest>\n') + doc = minidom.parseString(manifest) + target_sdk_version = manifest_check.extract_target_sdk_version(doc) + self.assertEqual(target_sdk_version, '28') + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/scripts/manifest_fixer.py b/scripts/manifest_fixer.py index 83868e603..c59732bb9 100755 --- a/scripts/manifest_fixer.py +++ b/scripts/manifest_fixer.py @@ -17,30 +17,20 @@ """A tool for inserting values from the build system into a manifest.""" from __future__ import print_function + import argparse import sys from xml.dom import minidom -android_ns = 'http://schemas.android.com/apk/res/android' - - -def get_children_with_tag(parent, tag_name): - children = [] - for child in parent.childNodes: - if child.nodeType == minidom.Node.ELEMENT_NODE and \ - child.tagName == tag_name: - children.append(child) - return children - - -def find_child_with_attribute(element, tag_name, namespace_uri, - attr_name, value): - for child in get_children_with_tag(element, tag_name): - attr = child.getAttributeNodeNS(namespace_uri, attr_name) - if attr is not None and attr.value == value: - return child - return None +from manifest import android_ns +from manifest import compare_version_gt +from manifest import ensure_manifest_android_ns +from manifest import find_child_with_attribute +from manifest import get_children_with_tag +from manifest import get_indent +from manifest import parse_manifest +from manifest import write_xml def parse_args(): @@ -61,6 +51,9 @@ def parse_args(): help='specify additional <uses-library> tag to add. android:requred is set to false') parser.add_argument('--uses-non-sdk-api', dest='uses_non_sdk_api', action='store_true', help='manifest is for a package built against the platform') + parser.add_argument('--logging-parent', dest='logging_parent', default='', + help=('specify logging parent as an additional <meta-data> tag. ' + 'This value is ignored if the logging_parent meta-data tag is present.')) parser.add_argument('--use-embedded-dex', dest='use_embedded_dex', action='store_true', help=('specify if the app wants to use embedded dex and avoid extracted,' 'locally compiled code. Must not conflict if already declared ' @@ -69,81 +62,14 @@ def parse_args(): default=None, type=lambda x: (str(x).lower() == 'true'), help=('specify if the app wants to use embedded native libraries. Must not conflict ' 'if already declared in the manifest.')) + parser.add_argument('--has-no-code', dest='has_no_code', action='store_true', + help=('adds hasCode="false" attribute to application. Ignored if application elem ' + 'already has a hasCode attribute.')) parser.add_argument('input', help='input AndroidManifest.xml file') parser.add_argument('output', help='output AndroidManifest.xml file') return parser.parse_args() -def parse_manifest(doc): - """Get the manifest element.""" - - manifest = doc.documentElement - if manifest.tagName != 'manifest': - raise RuntimeError('expected manifest tag at root') - return manifest - - -def ensure_manifest_android_ns(doc): - """Make sure the manifest tag defines the android namespace.""" - - manifest = parse_manifest(doc) - - ns = manifest.getAttributeNodeNS(minidom.XMLNS_NAMESPACE, 'android') - if ns is None: - attr = doc.createAttributeNS(minidom.XMLNS_NAMESPACE, 'xmlns:android') - attr.value = android_ns - manifest.setAttributeNode(attr) - elif ns.value != android_ns: - raise RuntimeError('manifest tag has incorrect android namespace ' + - ns.value) - - -def as_int(s): - try: - i = int(s) - except ValueError: - return s, False - return i, True - - -def compare_version_gt(a, b): - """Compare two SDK versions. - - Compares a and b, treating codenames like 'Q' as higher - than numerical versions like '28'. - - Returns True if a > b - - Args: - a: value to compare - b: value to compare - Returns: - True if a is a higher version than b - """ - - a, a_is_int = as_int(a.upper()) - b, b_is_int = as_int(b.upper()) - - if a_is_int == b_is_int: - # Both are codenames or both are versions, compare directly - return a > b - else: - # One is a codename, the other is not. Return true if - # b is an integer version - return b_is_int - - -def get_indent(element, default_level): - indent = '' - if element is not None and element.nodeType == minidom.Node.TEXT_NODE: - text = element.nodeValue - indent = text[:len(text)-len(text.lstrip())] - if not indent or indent == '\n': - # 1 indent = 4 space - indent = '\n' + (' ' * default_level * 4) - return indent - - def raise_min_sdk_version(doc, min_sdk_version, target_sdk_version, library): """Ensure the manifest contains a <uses-sdk> tag with a minSdkVersion. @@ -151,6 +77,7 @@ def raise_min_sdk_version(doc, min_sdk_version, target_sdk_version, library): doc: The XML document. May be modified by this function. min_sdk_version: The requested minSdkVersion attribute. target_sdk_version: The requested targetSdkVersion attribute. + library: True if the manifest is for a library. Raises: RuntimeError: invalid manifest """ @@ -200,6 +127,52 @@ def raise_min_sdk_version(doc, min_sdk_version, target_sdk_version, library): element.setAttributeNode(target_attr) +def add_logging_parent(doc, logging_parent_value): + """Add logging parent as an additional <meta-data> tag. + + Args: + doc: The XML document. May be modified by this function. + logging_parent_value: A string representing the logging + parent value. + Raises: + RuntimeError: Invalid manifest + """ + manifest = parse_manifest(doc) + + logging_parent_key = 'android.content.pm.LOGGING_PARENT' + elems = get_children_with_tag(manifest, 'application') + application = elems[0] if len(elems) == 1 else None + if len(elems) > 1: + raise RuntimeError('found multiple <application> tags') + elif not elems: + application = doc.createElement('application') + indent = get_indent(manifest.firstChild, 1) + first = manifest.firstChild + manifest.insertBefore(doc.createTextNode(indent), first) + manifest.insertBefore(application, first) + + indent = get_indent(application.firstChild, 2) + + last = application.lastChild + if last is not None and last.nodeType != minidom.Node.TEXT_NODE: + last = None + + if not find_child_with_attribute(application, 'meta-data', android_ns, + 'name', logging_parent_key): + ul = doc.createElement('meta-data') + ul.setAttributeNS(android_ns, 'android:name', logging_parent_key) + ul.setAttributeNS(android_ns, 'android:value', logging_parent_value) + application.insertBefore(doc.createTextNode(indent), last) + application.insertBefore(ul, last) + last = application.lastChild + + # align the closing tag with the opening tag if it's not + # indented + if last and last.nodeType != minidom.Node.TEXT_NODE: + indent = get_indent(application.previousSibling, 1) + application.appendChild(doc.createTextNode(indent)) + + def add_uses_libraries(doc, new_uses_libraries, required): """Add additional <uses-library> tags @@ -249,6 +222,7 @@ def add_uses_libraries(doc, new_uses_libraries, required): indent = get_indent(application.previousSibling, 1) application.appendChild(doc.createTextNode(indent)) + def add_uses_non_sdk_api(doc): """Add android:usesNonSdkApi=true attribute to <application>. @@ -323,10 +297,26 @@ def add_extract_native_libs(doc, extract_native_libs): (attr.value, value)) -def write_xml(f, doc): - f.write('<?xml version="1.0" encoding="utf-8"?>\n') - for node in doc.childNodes: - f.write(node.toxml(encoding='utf-8') + '\n') +def set_has_code_to_false(doc): + manifest = parse_manifest(doc) + elems = get_children_with_tag(manifest, 'application') + application = elems[0] if len(elems) == 1 else None + if len(elems) > 1: + raise RuntimeError('found multiple <application> tags') + elif not elems: + application = doc.createElement('application') + indent = get_indent(manifest.firstChild, 1) + first = manifest.firstChild + manifest.insertBefore(doc.createTextNode(indent), first) + manifest.insertBefore(application, first) + + attr = application.getAttributeNodeNS(android_ns, 'hasCode') + if attr is not None: + # Do nothing if the application already has a hasCode attribute. + return + attr = doc.createAttributeNS(android_ns, 'android:hasCode') + attr.value = 'false' + application.setAttributeNode(attr) def main(): @@ -350,9 +340,15 @@ def main(): if args.uses_non_sdk_api: add_uses_non_sdk_api(doc) + if args.logging_parent: + add_logging_parent(doc, args.logging_parent) + if args.use_embedded_dex: add_use_embedded_dex(doc) + if args.has_no_code: + set_has_code_to_false(doc) + if args.extract_native_libs is not None: add_extract_native_libs(doc, args.extract_native_libs) diff --git a/scripts/manifest_fixer_test.py b/scripts/manifest_fixer_test.py index 4ad9afaf3..d6e7f2674 100755 --- a/scripts/manifest_fixer_test.py +++ b/scripts/manifest_fixer_test.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -"""Unit tests for manifest_fixer_test.py.""" +"""Unit tests for manifest_fixer.py.""" import StringIO import sys @@ -226,12 +226,53 @@ class RaiseMinSdkVersionTest(unittest.TestCase): self.assertEqual(output, expected) +class AddLoggingParentTest(unittest.TestCase): + """Unit tests for add_logging_parent function.""" + + def add_logging_parent_test(self, input_manifest, logging_parent=None): + doc = minidom.parseString(input_manifest) + if logging_parent: + manifest_fixer.add_logging_parent(doc, logging_parent) + output = StringIO.StringIO() + manifest_fixer.write_xml(output, doc) + return output.getvalue() + + manifest_tmpl = ( + '<?xml version="1.0" encoding="utf-8"?>\n' + '<manifest xmlns:android="http://schemas.android.com/apk/res/android">\n' + '%s' + '</manifest>\n') + + def uses_logging_parent(self, logging_parent=None): + attrs = '' + if logging_parent: + meta_text = ('<meta-data android:name="android.content.pm.LOGGING_PARENT" ' + 'android:value="%s"/>\n') % (logging_parent) + attrs += ' <application>\n %s </application>\n' % (meta_text) + + return attrs + + def test_no_logging_parent(self): + """Tests manifest_fixer with no logging_parent.""" + manifest_input = self.manifest_tmpl % '' + expected = self.manifest_tmpl % self.uses_logging_parent() + output = self.add_logging_parent_test(manifest_input) + self.assertEqual(output, expected) + + def test_logging_parent(self): + """Tests manifest_fixer with no logging_parent.""" + manifest_input = self.manifest_tmpl % '' + expected = self.manifest_tmpl % self.uses_logging_parent('FOO') + output = self.add_logging_parent_test(manifest_input, 'FOO') + self.assertEqual(output, expected) + + class AddUsesLibrariesTest(unittest.TestCase): """Unit tests for add_uses_libraries function.""" def run_test(self, input_manifest, new_uses_libraries): doc = minidom.parseString(input_manifest) - manifest_fixer.add_uses_libraries(doc, new_uses_libraries) + manifest_fixer.add_uses_libraries(doc, new_uses_libraries, True) output = StringIO.StringIO() manifest_fixer.write_xml(output, doc) return output.getvalue() @@ -393,10 +434,10 @@ class AddExtractNativeLibsTest(unittest.TestCase): return output.getvalue() manifest_tmpl = ( - '<?xml version="1.0" encoding="utf-8"?>\n' - '<manifest xmlns:android="http://schemas.android.com/apk/res/android">\n' - ' <application%s/>\n' - '</manifest>\n') + '<?xml version="1.0" encoding="utf-8"?>\n' + '<manifest xmlns:android="http://schemas.android.com/apk/res/android">\n' + ' <application%s/>\n' + '</manifest>\n') def extract_native_libs(self, value): return ' android:extractNativeLibs="%s"' % value @@ -424,5 +465,47 @@ class AddExtractNativeLibsTest(unittest.TestCase): self.assertRaises(RuntimeError, self.run_test, manifest_input, False) +class AddNoCodeApplicationTest(unittest.TestCase): + """Unit tests for set_has_code_to_false function.""" + + def run_test(self, input_manifest): + doc = minidom.parseString(input_manifest) + manifest_fixer.set_has_code_to_false(doc) + output = StringIO.StringIO() + manifest_fixer.write_xml(output, doc) + return output.getvalue() + + manifest_tmpl = ( + '<?xml version="1.0" encoding="utf-8"?>\n' + '<manifest xmlns:android="http://schemas.android.com/apk/res/android">\n' + '%s' + '</manifest>\n') + + def test_no_application(self): + manifest_input = self.manifest_tmpl % '' + expected = self.manifest_tmpl % ' <application android:hasCode="false"/>\n' + output = self.run_test(manifest_input) + self.assertEqual(output, expected) + + def test_has_application_no_has_code(self): + manifest_input = self.manifest_tmpl % ' <application/>\n' + expected = self.manifest_tmpl % ' <application android:hasCode="false"/>\n' + output = self.run_test(manifest_input) + self.assertEqual(output, expected) + + def test_has_application_has_code_false(self): + """ Do nothing if there's already an application elemeent. """ + manifest_input = self.manifest_tmpl % ' <application android:hasCode="false"/>\n' + output = self.run_test(manifest_input) + self.assertEqual(output, manifest_input) + + def test_has_application_has_code_true(self): + """ Do nothing if there's already an application elemeent even if its + hasCode attribute is true. """ + manifest_input = self.manifest_tmpl % ' <application android:hasCode="true"/>\n' + output = self.run_test(manifest_input) + self.assertEqual(output, manifest_input) + + if __name__ == '__main__': - unittest.main() + unittest.main(verbosity=2) diff --git a/scripts/package-check.sh b/scripts/package-check.sh index f982e8244..d7e602f31 100755 --- a/scripts/package-check.sh +++ b/scripts/package-check.sh @@ -52,6 +52,7 @@ zip_contents=`zipinfo -1 $jar_file` # Check all class file names against the expected prefixes. old_ifs=${IFS} IFS=$'\n' +failed=false for zip_entry in ${zip_contents}; do # Check the suffix. if [[ "${zip_entry}" = *.class ]]; then @@ -65,8 +66,11 @@ for zip_entry in ${zip_contents}; do done if [[ "${found}" == "false" ]]; then echo "Class file ${zip_entry} is outside specified packages." - exit 1 + failed=true fi fi done +if [[ "${failed}" == "true" ]]; then + exit 1 +fi IFS=${old_ifs} diff --git a/scripts/setup-android-build.sh b/scripts/setup-android-build.sh new file mode 100755 index 000000000..dbb66c3d9 --- /dev/null +++ b/scripts/setup-android-build.sh @@ -0,0 +1,93 @@ +#! /bin/bash +# +# Sets the current directory as Android build output directory for a +# given target by writing the "prefix script" to it. Commands prefixed +# by this script are executed in the Android build environment. E.g., +# running +# ./run <command> +# runs <command> as if we issued +# cd <source> +# mount --bind <build dir> out +# . build/envsetup.sh +# lunch <config> +# <command> +# exit +# +# This arrangement eliminates the need to issue envsetup/lunch commands +# manually, and allows to run multiple builds from the same shell. +# Thus, if your source tree is in ~/aosp and you are building for +# 'blueline' and 'cuttlefish', issuing +# cd /sdx/blueline && \ +# ~/aosp/build/soong/scripts/setup-android-build.sh aosp_blueline-userdebug +# cd /sdx/cuttlefish && \ +# ~/aosp/build/soong/scripts/setup-android-build.sh aosp_cf_arm64_phone-userdebug +# sets up build directories in /sdx/blueline and /sdx/cuttlefish respectively. +# After that, issue +# /sdx/blueline/run m +# to build blueline image, and issue +# /sdx/cuttlefish atest CtsSecurityBulletinHostTestCases +# to run CTS tests. Notice there is no need to change to a specific directory for that. +# +# Argument: +# * configuration (one of those shown by `lunch` command). +# +set -e +function die() { printf "$@"; exit 1; } + +# Find out where the source tree using the fact that we are in its +# build/ subdirectory. +[[ "$(uname)" == Linux ]] || die "This setup runs only on Linux\n" +declare -r mydir="${0%/*}" +declare -r source="${mydir%/build/soong/scripts}" +[[ "/${mydir}/" =~ '/build/soong/scripts/' ]] || \ + die "$0 should be in build/soong/scripts/ subdirectory of the source tree\n" +[[ ! -e .repo && ! -e .git ]] || \ + die "Current directory looks like source. You should be in the _target_ directory.\n" +# Do not override old run script. +if [[ -x ./run ]]; then + # Set variables from config=xxx and source=xxx comments in the existing script. + . <(sed -nr 's/^# *source=(.*)/oldsource=\1/p;s/^# *config=(.*)/oldconfig=\1/p' run) + die "This directory has been already set up to build Android for %s from %s.\n\ +Remove 'run' file if you want to set it up afresh\n" "$oldconfig" "$oldsource" +fi + +(($#<2)) || die "usage: %s [<config>]\n" $0 + +if (($#==1)); then + # Configuration is provided, emit run script. + declare -r config="$1" + declare -r target="$PWD" + cat >./run <<EOF +#! /bin/bash +# source=$source +# config=$config +declare -r cmd=\$(printf ' %q' "\$@") +"$source/prebuilts/build-tools/linux-x86/bin/nsjail"\ + -Mo -q -e -t 0\ + -EANDROID_QUIET_BUILD=true \ + -B / -B "$target:$source/out"\ + --cwd "$source"\ + --skip_setsid \ + --keep_caps\ + --disable_clone_newcgroup\ + --disable_clone_newnet\ + --rlimit_as soft\ + --rlimit_core soft\ + --rlimit_cpu soft\ + --rlimit_fsize soft\ + --rlimit_nofile soft\ + --proc_rw\ + --hostname $(hostname) \ + --\ + /bin/bash -i -c ". build/envsetup.sh && lunch "$config" &&\$cmd" +EOF + chmod +x ./run +else + # No configuration, show available ones. + printf "Please specify build target. Common values:\n" + (cd "$source" + . build/envsetup.sh + get_build_var COMMON_LUNCH_CHOICES | tr ' ' '\n' | pr -c4 -tT -W"$(tput cols)" + ) + exit 1 +fi diff --git a/scripts/setup_go_workspace_for_soong.sh b/scripts/setup_go_workspace_for_soong.sh index 6374aae7f..479d09c0f 100755 --- a/scripts/setup_go_workspace_for_soong.sh +++ b/scripts/setup_go_workspace_for_soong.sh @@ -349,6 +349,7 @@ readonly BIND_PATHS=( "${ANDROID_PATH}/external/golang-protobuf|${OUTPUT_PATH}/src/github.com/golang/protobuf" "${ANDROID_PATH}/external/llvm/soong|${OUTPUT_PATH}/src/android/soong/llvm" "${ANDROID_PATH}/external/clang/soong|${OUTPUT_PATH}/src/android/soong/clang" + "${ANDROID_PATH}/external/robolectric-shadows/soong|${OUTPUT_PATH}/src/android/soong/robolectric" ) main "$@" diff --git a/scripts/strip.sh b/scripts/strip.sh index 0f77da8a9..40f018425 100755 --- a/scripts/strip.sh +++ b/scripts/strip.sh @@ -28,7 +28,7 @@ # --add-gnu-debuglink # --keep-mini-debug-info # --keep-symbols -# --use-gnu-strip +# --keep-symbols-and-debug-frame # --remove-build-id set -o pipefail @@ -39,80 +39,59 @@ usage() { cat <<EOF Usage: strip.sh [options] -k symbols -i in-file -o out-file -d deps-file Options: - --add-gnu-debuglink Add a gnu-debuglink section to out-file - --keep-mini-debug-info Keep compressed debug info in out-file - --keep-symbols Keep symbols in out-file - --use-gnu-strip Use strip/objcopy instead of llvm-{strip,objcopy} - --remove-build-id Remove the gnu build-id section in out-file + --add-gnu-debuglink Add a gnu-debuglink section to out-file + --keep-mini-debug-info Keep compressed debug info in out-file + --keep-symbols Keep symbols in out-file + --keep-symbols-and-debug-frame Keep symbols and .debug_frame in out-file + --remove-build-id Remove the gnu build-id section in out-file EOF exit 1 } -# Without --use-gnu-strip, GNU strip is replaced with llvm-strip to work around -# old GNU strip bug on lld output files, b/80093681. -# Similary, calls to objcopy are replaced with llvm-objcopy, -# with some exceptions. - do_strip() { - # ${CROSS_COMPILE}strip --strip-all does not strip .ARM.attributes, + # GNU strip --strip-all does not strip .ARM.attributes, # so we tell llvm-strip to keep it too. - if [ -z "${use_gnu_strip}" ]; then - "${CLANG_BIN}/llvm-strip" --strip-all -keep-section=.ARM.attributes "${infile}" -o "${outfile}.tmp" - else - "${CROSS_COMPILE}strip" --strip-all "${infile}" -o "${outfile}.tmp" - fi + "${CLANG_BIN}/llvm-strip" --strip-all --keep-section=.ARM.attributes "${infile}" -o "${outfile}.tmp" +} + +do_strip_keep_symbols_and_debug_frame() { + REMOVE_SECTIONS=`"${CLANG_BIN}/llvm-readelf" -S "${infile}" | awk '/.debug_/ {if ($2 != ".debug_frame") {print "--remove-section " $2}}' | xargs` + "${CLANG_BIN}/llvm-objcopy" "${infile}" "${outfile}.tmp" ${REMOVE_SECTIONS} } do_strip_keep_symbols() { - REMOVE_SECTIONS=`"${CROSS_COMPILE}readelf" -S "${infile}" | awk '/.debug_/ {print "--remove-section " $2}' | xargs` - if [ -z "${use_gnu_strip}" ]; then - "${CLANG_BIN}/llvm-objcopy" "${infile}" "${outfile}.tmp" ${REMOVE_SECTIONS} - else - "${CROSS_COMPILE}objcopy" "${infile}" "${outfile}.tmp" ${REMOVE_SECTIONS} - fi + REMOVE_SECTIONS=`"${CLANG_BIN}/llvm-readelf" -S "${infile}" | awk '/.debug_/ {print "--remove-section " $2}' | xargs` + "${CLANG_BIN}/llvm-objcopy" "${infile}" "${outfile}.tmp" ${REMOVE_SECTIONS} } do_strip_keep_symbol_list() { - if [ -z "${use_gnu_strip}" ]; then - echo "do_strip_keep_symbol_list does not work with llvm-objcopy" - echo "http://b/131631155" - usage - fi - echo "${symbols_to_keep}" | tr ',' '\n' > "${outfile}.symbolList" - KEEP_SYMBOLS="-w --strip-unneeded-symbol=* --keep-symbols=" - KEEP_SYMBOLS+="${outfile}.symbolList" - "${CROSS_COMPILE}objcopy" "${infile}" "${outfile}.tmp" ${KEEP_SYMBOLS} + KEEP_SYMBOLS="--strip-unneeded-symbol=* --keep-symbols=" + KEEP_SYMBOLS+="${outfile}.symbolList" + "${CROSS_COMPILE}objcopy" -w "${infile}" "${outfile}.tmp" ${KEEP_SYMBOLS} } do_strip_keep_mini_debug_info() { rm -f "${outfile}.dynsyms" "${outfile}.funcsyms" "${outfile}.keep_symbols" "${outfile}.debug" "${outfile}.mini_debuginfo" "${outfile}.mini_debuginfo.xz" local fail= - if [ -z "${use_gnu_strip}" ]; then - "${CLANG_BIN}/llvm-strip" --strip-all -keep-section=.ARM.attributes -remove-section=.comment "${infile}" -o "${outfile}.tmp" || fail=true - else - "${CROSS_COMPILE}strip" --strip-all -R .comment "${infile}" -o "${outfile}.tmp" || fail=true - fi + "${CLANG_BIN}/llvm-strip" --strip-all --keep-section=.ARM.attributes --remove-section=.comment "${infile}" -o "${outfile}.tmp" || fail=true + if [ -z $fail ]; then - # Current prebult llvm-objcopy does not support the following flags: - # --only-keep-debug --rename-section --keep-symbols - # For the following use cases, ${CROSS_COMPILE}objcopy does fine with lld linked files, - # except the --add-section flag. + # Current prebult llvm-objcopy does not support --only-keep-debug flag, + # and cannot process object files that are produced with the flag. Use + # GNU objcopy instead for now. (b/141010852) "${CROSS_COMPILE}objcopy" --only-keep-debug "${infile}" "${outfile}.debug" - "${CROSS_COMPILE}nm" -D "${infile}" --format=posix --defined-only 2> /dev/null | awk '{ print $1 }' | sort >"${outfile}.dynsyms" - "${CROSS_COMPILE}nm" "${infile}" --format=posix --defined-only | awk '{ if ($2 == "T" || $2 == "t" || $2 == "D") print $1 }' | sort > "${outfile}.funcsyms" + "${CLANG_BIN}/llvm-nm" -D "${infile}" --format=posix --defined-only 2> /dev/null | awk '{ print $1 }' | sort >"${outfile}.dynsyms" + "${CLANG_BIN}/llvm-nm" "${infile}" --format=posix --defined-only | awk '{ if ($2 == "T" || $2 == "t" || $2 == "D") print $1 }' | sort > "${outfile}.funcsyms" comm -13 "${outfile}.dynsyms" "${outfile}.funcsyms" > "${outfile}.keep_symbols" echo >> "${outfile}.keep_symbols" # Ensure that the keep_symbols file is not empty. "${CROSS_COMPILE}objcopy" --rename-section .debug_frame=saved_debug_frame "${outfile}.debug" "${outfile}.mini_debuginfo" "${CROSS_COMPILE}objcopy" -S --remove-section .gdb_index --remove-section .comment --keep-symbols="${outfile}.keep_symbols" "${outfile}.mini_debuginfo" "${CROSS_COMPILE}objcopy" --rename-section saved_debug_frame=.debug_frame "${outfile}.mini_debuginfo" "${XZ}" "${outfile}.mini_debuginfo" - if [ -z "${use_gnu_strip}" ]; then - "${CLANG_BIN}/llvm-objcopy" --add-section .gnu_debugdata="${outfile}.mini_debuginfo.xz" "${outfile}.tmp" - else - "${CROSS_COMPILE}objcopy" --add-section .gnu_debugdata="${outfile}.mini_debuginfo.xz" "${outfile}.tmp" - fi + + "${CLANG_BIN}/llvm-objcopy" --add-section .gnu_debugdata="${outfile}.mini_debuginfo.xz" "${outfile}.tmp" rm -f "${outfile}.dynsyms" "${outfile}.funcsyms" "${outfile}.keep_symbols" "${outfile}.debug" "${outfile}.mini_debuginfo" "${outfile}.mini_debuginfo.xz" else cp -f "${infile}" "${outfile}.tmp" @@ -120,19 +99,11 @@ do_strip_keep_mini_debug_info() { } do_add_gnu_debuglink() { - if [ -z "${use_gnu_strip}" ]; then - "${CLANG_BIN}/llvm-objcopy" --add-gnu-debuglink="${infile}" "${outfile}.tmp" - else - "${CROSS_COMPILE}objcopy" --add-gnu-debuglink="${infile}" "${outfile}.tmp" - fi + "${CLANG_BIN}/llvm-objcopy" --add-gnu-debuglink="${infile}" "${outfile}.tmp" } do_remove_build_id() { - if [ -z "${use_gnu_strip}" ]; then - "${CLANG_BIN}/llvm-strip" -remove-section=.note.gnu.build-id "${outfile}.tmp" -o "${outfile}.tmp.no-build-id" - else - "${CROSS_COMPILE}strip" --remove-section=.note.gnu.build-id "${outfile}.tmp" -o "${outfile}.tmp.no-build-id" - fi + "${CLANG_BIN}/llvm-strip" --remove-section=.note.gnu.build-id "${outfile}.tmp" -o "${outfile}.tmp.no-build-id" rm -f "${outfile}.tmp" mv "${outfile}.tmp.no-build-id" "${outfile}.tmp" } @@ -148,8 +119,8 @@ while getopts $OPTSTRING opt; do add-gnu-debuglink) add_gnu_debuglink=true ;; keep-mini-debug-info) keep_mini_debug_info=true ;; keep-symbols) keep_symbols=true ;; + keep-symbols-and-debug-frame) keep_symbols_and_debug_frame=true ;; remove-build-id) remove_build_id=true ;; - use-gnu-strip) use_gnu_strip=true ;; *) echo "Unknown option --${OPTARG}"; usage ;; esac;; ?) usage ;; @@ -177,6 +148,16 @@ if [ ! -z "${keep_symbols}" -a ! -z "${keep_mini_debug_info}" ]; then usage fi +if [ ! -z "${keep_symbols}" -a ! -z "${keep_symbols_and_debug_frame}" ]; then + echo "--keep-symbols and --keep-symbols-and-debug-frame cannot be used together" + usage +fi + +if [ ! -z "${keep_mini_debug_info}" -a ! -z "${keep_symbols_and_debug_frame}" ]; then + echo "--keep-symbols-mini-debug-info and --keep-symbols-and-debug-frame cannot be used together" + usage +fi + if [ ! -z "${symbols_to_keep}" -a ! -z "${keep_symbols}" ]; then echo "--keep-symbols and -k cannot be used together" usage @@ -195,6 +176,8 @@ elif [ ! -z "${symbols_to_keep}" ]; then do_strip_keep_symbol_list elif [ ! -z "${keep_mini_debug_info}" ]; then do_strip_keep_mini_debug_info +elif [ ! -z "${keep_symbols_and_debug_frame}" ]; then + do_strip_keep_symbols_and_debug_frame else do_strip fi @@ -210,18 +193,13 @@ fi rm -f "${outfile}" mv "${outfile}.tmp" "${outfile}" -if [ -z "${use_gnu_strip}" ]; then - USED_STRIP_OBJCOPY="${CLANG_BIN}/llvm-strip ${CLANG_BIN}/llvm-objcopy" -else - USED_STRIP_OBJCOPY="${CROSS_COMPILE}strip" -fi - cat <<EOF > "${depsfile}" ${outfile}: \ ${infile} \ - ${CROSS_COMPILE}nm \ ${CROSS_COMPILE}objcopy \ - ${CROSS_COMPILE}readelf \ - ${USED_STRIP_OBJCOPY} + ${CLANG_BIN}/llvm-nm \ + ${CLANG_BIN}/llvm-objcopy \ + ${CLANG_BIN}/llvm-readelf \ + ${CLANG_BIN}/llvm-strip EOF diff --git a/scripts/system-clang-format b/scripts/system-clang-format index 55773a29f..a7614d29f 100644 --- a/scripts/system-clang-format +++ b/scripts/system-clang-format @@ -1,9 +1,11 @@ BasedOnStyle: Google +Standard: Cpp11 AccessModifierOffset: -2 AllowShortFunctionsOnASingleLine: Inline ColumnLimit: 100 CommentPragmas: NOLINT:.* DerivePointerAlignment: false +IncludeBlocks: Preserve IndentWidth: 4 ContinuationIndentWidth: 8 PointerAlignment: Left diff --git a/scripts/system-clang-format-2 b/scripts/system-clang-format-2 index ede5d7e18..a4e23f860 100644 --- a/scripts/system-clang-format-2 +++ b/scripts/system-clang-format-2 @@ -1,8 +1,10 @@ BasedOnStyle: Google +Standard: Cpp11 AllowShortFunctionsOnASingleLine: Inline ColumnLimit: 100 CommentPragmas: NOLINT:.* DerivePointerAlignment: false +IncludeBlocks: Preserve IndentWidth: 2 PointerAlignment: Left TabWidth: 2 diff --git a/scripts/test_config_fixer.py b/scripts/test_config_fixer.py new file mode 100644 index 000000000..32d5b175e --- /dev/null +++ b/scripts/test_config_fixer.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python +# +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""A tool for modifying values in a test config.""" + +from __future__ import print_function + +import argparse +import sys +from xml.dom import minidom + + +from manifest import get_children_with_tag +from manifest import parse_manifest +from manifest import parse_test_config +from manifest import write_xml + + +def parse_args(): + """Parse commandline arguments.""" + + parser = argparse.ArgumentParser() + parser.add_argument('--manifest', default='', dest='manifest', + help=('AndroidManifest.xml that contains the original package name')) + parser.add_argument('--package-name', default='', dest='package_name', + help=('overwrite package fields in the test config')) + parser.add_argument('--test-file-name', default='', dest='test_file_name', + help=('overwrite test file name in the test config')) + parser.add_argument('input', help='input test config file') + parser.add_argument('output', help='output test config file') + return parser.parse_args() + + +def overwrite_package_name(test_config_doc, manifest_doc, package_name): + + manifest = parse_manifest(manifest_doc) + original_package = manifest.getAttribute('package') + + test_config = parse_test_config(test_config_doc) + tests = get_children_with_tag(test_config, 'test') + + for test in tests: + options = get_children_with_tag(test, 'option') + for option in options: + if option.getAttribute('name') == "package" and option.getAttribute('value') == original_package: + option.setAttribute('value', package_name) + +def overwrite_test_file_name(test_config_doc, test_file_name): + + test_config = parse_test_config(test_config_doc) + tests = get_children_with_tag(test_config, 'target_preparer') + + for test in tests: + if test.getAttribute('class') == "com.android.tradefed.targetprep.TestAppInstallSetup": + options = get_children_with_tag(test, 'option') + for option in options: + if option.getAttribute('name') == "test-file-name": + option.setAttribute('value', test_file_name) + +def main(): + """Program entry point.""" + try: + args = parse_args() + + doc = minidom.parse(args.input) + + if args.package_name: + if not args.manifest: + raise RuntimeError('--manifest flag required for --package-name') + manifest_doc = minidom.parse(args.manifest) + overwrite_package_name(doc, manifest_doc, args.package_name) + + if args.test_file_name: + overwrite_test_file_name(doc, args.test_file_name) + + with open(args.output, 'wb') as f: + write_xml(f, doc) + + # pylint: disable=broad-except + except Exception as err: + print('error: ' + str(err), file=sys.stderr) + sys.exit(-1) + +if __name__ == '__main__': + main() diff --git a/scripts/test_config_fixer_test.py b/scripts/test_config_fixer_test.py new file mode 100644 index 000000000..1272c6b33 --- /dev/null +++ b/scripts/test_config_fixer_test.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python +# +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Unit tests for test_config_fixer.py.""" + +import StringIO +import sys +import unittest +from xml.dom import minidom + +import test_config_fixer + +sys.dont_write_bytecode = True + + +class OverwritePackageNameTest(unittest.TestCase): + """ Unit tests for overwrite_package_name function """ + + manifest = ( + '<?xml version="1.0" encoding="utf-8"?>\n' + '<manifest xmlns:android="http://schemas.android.com/apk/res/android"\n' + ' package="com.android.foo">\n' + ' <application>\n' + ' </application>\n' + '</manifest>\n') + + test_config = ( + '<?xml version="1.0" encoding="utf-8"?>\n' + '<configuration description="Runs some tests.">\n' + ' <option name="test-suite-tag" value="apct"/>\n' + ' <target_preparer class="com.android.tradefed.targetprep.suite.SuiteApkInstaller">\n' + ' <option name="package" value="%s"/>\n' + ' </target_preparer>\n' + ' <test class="com.android.tradefed.testtype.AndroidJUnitTest">\n' + ' <option name="package" value="%s"/>\n' + ' <option name="runtime-hint" value="20s"/>\n' + ' </test>\n' + ' <test class="com.android.tradefed.testtype.AndroidJUnitTest">\n' + ' <option name="package" value="%s"/>\n' + ' <option name="runtime-hint" value="15s"/>\n' + ' </test>\n' + '</configuration>\n') + + def test_all(self): + doc = minidom.parseString(self.test_config % ("com.android.foo", "com.android.foo", "com.android.bar")) + manifest = minidom.parseString(self.manifest) + + test_config_fixer.overwrite_package_name(doc, manifest, "com.soong.foo") + output = StringIO.StringIO() + test_config_fixer.write_xml(output, doc) + + # Only the matching package name in a test node should be updated. + expected = self.test_config % ("com.android.foo", "com.soong.foo", "com.android.bar") + self.assertEqual(expected, output.getvalue()) + + +class OverwriteTestFileNameTest(unittest.TestCase): + """ Unit tests for overwrite_test_file_name function """ + + test_config = ( + '<?xml version="1.0" encoding="utf-8"?>\n' + '<configuration description="Runs some tests.">\n' + ' <target_preparer class="com.android.tradefed.targetprep.TestAppInstallSetup">\n' + ' <option name="test-file-name" value="%s"/>\n' + ' </target_preparer>\n' + ' <test class="com.android.tradefed.testtype.AndroidJUnitTest">\n' + ' <option name="package" value="com.android.foo"/>\n' + ' <option name="runtime-hint" value="20s"/>\n' + ' </test>\n' + '</configuration>\n') + + def test_all(self): + doc = minidom.parseString(self.test_config % ("foo.apk")) + + test_config_fixer.overwrite_test_file_name(doc, "bar.apk") + output = StringIO.StringIO() + test_config_fixer.write_xml(output, doc) + + # Only the matching package name in a test node should be updated. + expected = self.test_config % ("bar.apk") + self.assertEqual(expected, output.getvalue()) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/sdk/Android.bp b/sdk/Android.bp new file mode 100644 index 000000000..cb93351d9 --- /dev/null +++ b/sdk/Android.bp @@ -0,0 +1,27 @@ +bootstrap_go_package { + name: "soong-sdk", + pkgPath: "android/soong/sdk", + deps: [ + "blueprint", + "soong", + "soong-android", + "soong-apex", + "soong-cc", + "soong-java", + ], + srcs: [ + "bp.go", + "exports.go", + "sdk.go", + "update.go", + ], + testSrcs: [ + "bp_test.go", + "cc_sdk_test.go", + "exports_test.go", + "java_sdk_test.go", + "sdk_test.go", + "testing.go", + ], + pluginFor: ["soong_build"], +} diff --git a/sdk/bp.go b/sdk/bp.go new file mode 100644 index 000000000..68fe7ab4e --- /dev/null +++ b/sdk/bp.go @@ -0,0 +1,301 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sdk + +import ( + "fmt" + + "android/soong/android" +) + +type bpPropertySet struct { + properties map[string]interface{} + tags map[string]android.BpPropertyTag + order []string +} + +var _ android.BpPropertySet = (*bpPropertySet)(nil) + +func (s *bpPropertySet) init() { + s.properties = make(map[string]interface{}) + s.tags = make(map[string]android.BpPropertyTag) +} + +func (s *bpPropertySet) AddProperty(name string, value interface{}) { + if s.properties[name] != nil { + panic(fmt.Sprintf("Property %q already exists in property set", name)) + } + + s.properties[name] = value + s.order = append(s.order, name) +} + +func (s *bpPropertySet) AddPropertyWithTag(name string, value interface{}, tag android.BpPropertyTag) { + s.AddProperty(name, value) + s.tags[name] = tag +} + +func (s *bpPropertySet) AddPropertySet(name string) android.BpPropertySet { + set := newPropertySet() + s.AddProperty(name, set) + return set +} + +func (s *bpPropertySet) getValue(name string) interface{} { + return s.properties[name] +} + +func (s *bpPropertySet) getTag(name string) interface{} { + return s.tags[name] +} + +func (s *bpPropertySet) transformContents(transformer bpPropertyTransformer) { + var newOrder []string + for _, name := range s.order { + value := s.properties[name] + tag := s.tags[name] + var newValue interface{} + var newTag android.BpPropertyTag + if propertySet, ok := value.(*bpPropertySet); ok { + var newPropertySet *bpPropertySet + newPropertySet, newTag = transformPropertySet(transformer, name, propertySet, tag) + if newPropertySet == nil { + newValue = nil + } else { + newValue = newPropertySet + } + } else { + newValue, newTag = transformer.transformProperty(name, value, tag) + } + + if newValue == nil { + // Delete the property from the map and exclude it from the new order. + delete(s.properties, name) + } else { + // Update the property in the map and add the name to the new order list. + s.properties[name] = newValue + s.tags[name] = newTag + newOrder = append(newOrder, name) + } + } + s.order = newOrder +} + +func transformPropertySet(transformer bpPropertyTransformer, name string, propertySet *bpPropertySet, tag android.BpPropertyTag) (*bpPropertySet, android.BpPropertyTag) { + newPropertySet, newTag := transformer.transformPropertySetBeforeContents(name, propertySet, tag) + if newPropertySet != nil { + newPropertySet.transformContents(transformer) + + newPropertySet, newTag = transformer.transformPropertySetAfterContents(name, newPropertySet, newTag) + } + return newPropertySet, newTag +} + +func (s *bpPropertySet) setProperty(name string, value interface{}) { + if s.properties[name] == nil { + s.AddProperty(name, value) + } else { + s.properties[name] = value + s.tags[name] = nil + } +} + +func (s *bpPropertySet) insertAfter(position string, name string, value interface{}) { + if s.properties[name] != nil { + panic("Property %q already exists in property set") + } + + // Add the name to the end of the order, to ensure it has necessary capacity + // and to handle the case when the position does not exist. + s.order = append(s.order, name) + + // Search through the order for the item that matches supplied position. If + // found then insert the name of the new property after it. + for i, v := range s.order { + if v == position { + // Copy the items after the one where the new property should be inserted. + copy(s.order[i+2:], s.order[i+1:]) + // Insert the item in the list. + s.order[i+1] = name + } + } + + s.properties[name] = value +} + +type bpModule struct { + *bpPropertySet + moduleType string +} + +var _ android.BpModule = (*bpModule)(nil) + +type bpPropertyTransformer interface { + // Transform the property set, returning the new property set/tag to insert back into the + // parent property set (or module if this is the top level property set). + // + // This will be called before transforming the properties in the supplied set. + // + // The name will be "" for the top level property set. + // + // Returning (nil, ...) will cause the property set to be removed. + transformPropertySetBeforeContents(name string, propertySet *bpPropertySet, tag android.BpPropertyTag) (*bpPropertySet, android.BpPropertyTag) + + // Transform the property set, returning the new property set/tag to insert back into the + // parent property set (or module if this is the top level property set). + // + // This will be called after transforming the properties in the supplied set. + // + // The name will be "" for the top level property set. + // + // Returning (nil, ...) will cause the property set to be removed. + transformPropertySetAfterContents(name string, propertySet *bpPropertySet, tag android.BpPropertyTag) (*bpPropertySet, android.BpPropertyTag) + + // Transform a property, return the new value/tag to insert back into the property set. + // + // Returning (nil, ...) will cause the property to be removed. + transformProperty(name string, value interface{}, tag android.BpPropertyTag) (interface{}, android.BpPropertyTag) +} + +// Interface for transforming bpModule objects. +type bpTransformer interface { + // Transform the module, returning the result. + // + // The method can either create a new module and return that, or modify the supplied module + // in place and return that. + // + // After this returns the transformer is applied to the contents of the returned module. + transformModule(module *bpModule) *bpModule + + bpPropertyTransformer +} + +type identityTransformation struct{} + +var _ bpTransformer = (*identityTransformation)(nil) + +func (t identityTransformation) transformModule(module *bpModule) *bpModule { + return module +} + +func (t identityTransformation) transformPropertySetBeforeContents(name string, propertySet *bpPropertySet, tag android.BpPropertyTag) (*bpPropertySet, android.BpPropertyTag) { + return propertySet, tag +} + +func (t identityTransformation) transformPropertySetAfterContents(name string, propertySet *bpPropertySet, tag android.BpPropertyTag) (*bpPropertySet, android.BpPropertyTag) { + return propertySet, tag +} + +func (t identityTransformation) transformProperty(name string, value interface{}, tag android.BpPropertyTag) (interface{}, android.BpPropertyTag) { + return value, tag +} + +func (m *bpModule) deepCopy() *bpModule { + return m.transform(deepCopyTransformer) +} + +func (m *bpModule) transform(transformer bpTransformer) *bpModule { + transformedModule := transformer.transformModule(m) + // Copy the contents of the returned property set into the module and then transform that. + transformedModule.bpPropertySet, _ = transformPropertySet(transformer, "", transformedModule.bpPropertySet, nil) + return transformedModule +} + +type deepCopyTransformation struct { + identityTransformation +} + +func (t deepCopyTransformation) transformModule(module *bpModule) *bpModule { + // Take a shallow copy of the module. Any mutable property values will be copied by the + // transformer. + moduleCopy := *module + return &moduleCopy +} + +func (t deepCopyTransformation) transformPropertySetBeforeContents(name string, propertySet *bpPropertySet, tag android.BpPropertyTag) (*bpPropertySet, android.BpPropertyTag) { + // Create a shallow copy of the properties map. Any mutable property values will be copied by the + // transformer. + propertiesCopy := make(map[string]interface{}) + for propertyName, value := range propertySet.properties { + propertiesCopy[propertyName] = value + } + + // Ditto for tags map. + tagsCopy := make(map[string]android.BpPropertyTag) + for propertyName, propertyTag := range propertySet.tags { + tagsCopy[propertyName] = propertyTag + } + + // Create a new property set. + return &bpPropertySet{ + properties: propertiesCopy, + tags: tagsCopy, + order: append([]string(nil), propertySet.order...), + }, tag +} + +func (t deepCopyTransformation) transformProperty(name string, value interface{}, tag android.BpPropertyTag) (interface{}, android.BpPropertyTag) { + // Copy string slice, otherwise return value. + if values, ok := value.([]string); ok { + valuesCopy := make([]string, len(values)) + copy(valuesCopy, values) + return valuesCopy, tag + } + return value, tag +} + +var deepCopyTransformer bpTransformer = deepCopyTransformation{} + +// A .bp file +type bpFile struct { + modules map[string]*bpModule + order []*bpModule +} + +// Add a module. +// +// The module must have had its "name" property set to a string value that +// is unique within this file. +func (f *bpFile) AddModule(module android.BpModule) { + m := module.(*bpModule) + if name, ok := m.getValue("name").(string); ok { + if f.modules[name] != nil { + panic(fmt.Sprintf("Module %q already exists in bp file", name)) + } + + f.modules[name] = m + f.order = append(f.order, m) + } else { + panic("Module does not have a name property, or it is not a string") + } +} + +func (f *bpFile) newModule(moduleType string) *bpModule { + return newModule(moduleType) +} + +func newModule(moduleType string) *bpModule { + module := &bpModule{ + moduleType: moduleType, + bpPropertySet: newPropertySet(), + } + return module +} + +func newPropertySet() *bpPropertySet { + set := &bpPropertySet{} + set.init() + return set +} diff --git a/sdk/bp_test.go b/sdk/bp_test.go new file mode 100644 index 000000000..c630c2524 --- /dev/null +++ b/sdk/bp_test.go @@ -0,0 +1,76 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sdk + +import ( + "testing" + + "android/soong/android" +) + +type removeFredTransformation struct { + identityTransformation +} + +func (t removeFredTransformation) transformProperty(name string, value interface{}, tag android.BpPropertyTag) (interface{}, android.BpPropertyTag) { + if name == "fred" { + return nil, nil + } + return value, tag +} + +func (t removeFredTransformation) transformPropertySetBeforeContents(name string, propertySet *bpPropertySet, tag android.BpPropertyTag) (*bpPropertySet, android.BpPropertyTag) { + if name == "fred" { + return nil, nil + } + return propertySet, tag +} + +func (t removeFredTransformation) transformPropertySetAfterContents(name string, propertySet *bpPropertySet, tag android.BpPropertyTag) (*bpPropertySet, android.BpPropertyTag) { + if len(propertySet.properties) == 0 { + return nil, nil + } + return propertySet, tag +} + +func TestTransformRemoveProperty(t *testing.T) { + + helper := &TestHelper{t} + + set := newPropertySet() + set.AddProperty("name", "name") + set.AddProperty("fred", "12") + + set.transformContents(removeFredTransformation{}) + + contents := &generatedContents{} + outputPropertySet(contents, set) + helper.AssertTrimmedStringEquals("removing property failed", "name: \"name\",\n", contents.content.String()) +} + +func TestTransformRemovePropertySet(t *testing.T) { + + helper := &TestHelper{t} + + set := newPropertySet() + set.AddProperty("name", "name") + set.AddPropertySet("fred") + + set.transformContents(removeFredTransformation{}) + + contents := &generatedContents{} + outputPropertySet(contents, set) + helper.AssertTrimmedStringEquals("removing property set failed", "name: \"name\",\n", contents.content.String()) +} diff --git a/sdk/cc_sdk_test.go b/sdk/cc_sdk_test.go new file mode 100644 index 000000000..dded15360 --- /dev/null +++ b/sdk/cc_sdk_test.go @@ -0,0 +1,1893 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sdk + +import ( + "testing" + + "android/soong/android" + "android/soong/cc" +) + +func testSdkWithCc(t *testing.T, bp string) *testSdkResult { + t.Helper() + + fs := map[string][]byte{ + "Test.cpp": nil, + "include/Test.h": nil, + "include-android/AndroidTest.h": nil, + "include-host/HostTest.h": nil, + "arm64/include/Arm64Test.h": nil, + "libfoo.so": nil, + "aidl/foo/bar/Test.aidl": nil, + "some/where/stubslib.map.txt": nil, + } + return testSdkWithFs(t, bp, fs) +} + +// Contains tests for SDK members provided by the cc package. + +func TestSdkIsCompileMultilibBoth(t *testing.T) { + result := testSdkWithCc(t, ` + sdk { + name: "mysdk", + native_shared_libs: ["sdkmember"], + } + + cc_library_shared { + name: "sdkmember", + srcs: ["Test.cpp"], + stl: "none", + } + `) + + armOutput := result.Module("sdkmember", "android_arm_armv7-a-neon_shared").(*cc.Module).OutputFile() + arm64Output := result.Module("sdkmember", "android_arm64_armv8-a_shared").(*cc.Module).OutputFile() + + var inputs []string + buildParams := result.Module("mysdk", android.CommonOS.Name).BuildParamsForTests() + for _, bp := range buildParams { + if bp.Input != nil { + inputs = append(inputs, bp.Input.String()) + } + } + + // ensure that both 32/64 outputs are inputs of the sdk snapshot + ensureListContains(t, inputs, armOutput.String()) + ensureListContains(t, inputs, arm64Output.String()) +} + +func TestBasicSdkWithCc(t *testing.T) { + result := testSdkWithCc(t, ` + sdk { + name: "mysdk", + native_shared_libs: ["sdkmember"], + } + + cc_library_shared { + name: "sdkmember", + system_shared_libs: [], + } + + sdk_snapshot { + name: "mysdk@1", + native_shared_libs: ["sdkmember_mysdk_1"], + } + + sdk_snapshot { + name: "mysdk@2", + native_shared_libs: ["sdkmember_mysdk_2"], + } + + cc_prebuilt_library_shared { + name: "sdkmember", + srcs: ["libfoo.so"], + prefer: false, + system_shared_libs: [], + stl: "none", + } + + cc_prebuilt_library_shared { + name: "sdkmember_mysdk_1", + sdk_member_name: "sdkmember", + srcs: ["libfoo.so"], + system_shared_libs: [], + stl: "none", + // TODO: remove //apex_available:platform + apex_available: [ + "//apex_available:platform", + "myapex", + ], + } + + cc_prebuilt_library_shared { + name: "sdkmember_mysdk_2", + sdk_member_name: "sdkmember", + srcs: ["libfoo.so"], + system_shared_libs: [], + stl: "none", + // TODO: remove //apex_available:platform + apex_available: [ + "//apex_available:platform", + "myapex2", + ], + } + + cc_library_shared { + name: "mycpplib", + srcs: ["Test.cpp"], + shared_libs: ["sdkmember"], + system_shared_libs: [], + stl: "none", + apex_available: [ + "myapex", + "myapex2", + ], + } + + apex { + name: "myapex", + native_shared_libs: ["mycpplib"], + uses_sdks: ["mysdk@1"], + key: "myapex.key", + certificate: ":myapex.cert", + } + + apex { + name: "myapex2", + native_shared_libs: ["mycpplib"], + uses_sdks: ["mysdk@2"], + key: "myapex.key", + certificate: ":myapex.cert", + } + `) + + sdkMemberV1 := result.ModuleForTests("sdkmember_mysdk_1", "android_arm64_armv8-a_shared_myapex").Rule("toc").Output + sdkMemberV2 := result.ModuleForTests("sdkmember_mysdk_2", "android_arm64_armv8-a_shared_myapex2").Rule("toc").Output + + cpplibForMyApex := result.ModuleForTests("mycpplib", "android_arm64_armv8-a_shared_myapex") + cpplibForMyApex2 := result.ModuleForTests("mycpplib", "android_arm64_armv8-a_shared_myapex2") + + // Depending on the uses_sdks value, different libs are linked + ensureListContains(t, pathsToStrings(cpplibForMyApex.Rule("ld").Implicits), sdkMemberV1.String()) + ensureListContains(t, pathsToStrings(cpplibForMyApex2.Rule("ld").Implicits), sdkMemberV2.String()) +} + +// Make sure the sdk can use host specific cc libraries static/shared and both. +func TestHostSdkWithCc(t *testing.T) { + testSdkWithCc(t, ` + sdk { + name: "mysdk", + device_supported: false, + host_supported: true, + native_shared_libs: ["sdkshared"], + native_static_libs: ["sdkstatic"], + } + + cc_library_host_shared { + name: "sdkshared", + stl: "none", + } + + cc_library_host_static { + name: "sdkstatic", + stl: "none", + } + `) +} + +// Make sure the sdk can use cc libraries static/shared and both. +func TestSdkWithCc(t *testing.T) { + testSdkWithCc(t, ` + sdk { + name: "mysdk", + native_shared_libs: ["sdkshared", "sdkboth1"], + native_static_libs: ["sdkstatic", "sdkboth2"], + } + + cc_library_shared { + name: "sdkshared", + stl: "none", + } + + cc_library_static { + name: "sdkstatic", + stl: "none", + } + + cc_library { + name: "sdkboth1", + stl: "none", + } + + cc_library { + name: "sdkboth2", + stl: "none", + } + `) +} + +func TestSnapshotWithObject(t *testing.T) { + result := testSdkWithCc(t, ` + sdk { + name: "mysdk", + native_objects: ["crtobj"], + } + + cc_object { + name: "crtobj", + stl: "none", + } + `) + + result.CheckSnapshot("mysdk", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +cc_prebuilt_object { + name: "mysdk_crtobj@current", + sdk_member_name: "crtobj", + stl: "none", + arch: { + arm64: { + srcs: ["arm64/lib/crtobj.o"], + }, + arm: { + srcs: ["arm/lib/crtobj.o"], + }, + }, +} + +cc_prebuilt_object { + name: "crtobj", + prefer: false, + stl: "none", + arch: { + arm64: { + srcs: ["arm64/lib/crtobj.o"], + }, + arm: { + srcs: ["arm/lib/crtobj.o"], + }, + }, +} + +sdk_snapshot { + name: "mysdk@current", + native_objects: ["mysdk_crtobj@current"], +} +`), + checkAllCopyRules(` +.intermediates/crtobj/android_arm64_armv8-a/crtobj.o -> arm64/lib/crtobj.o +.intermediates/crtobj/android_arm_armv7-a-neon/crtobj.o -> arm/lib/crtobj.o +`), + ) +} + +func TestSnapshotWithCcDuplicateHeaders(t *testing.T) { + result := testSdkWithCc(t, ` + sdk { + name: "mysdk", + native_shared_libs: ["mynativelib1", "mynativelib2"], + } + + cc_library_shared { + name: "mynativelib1", + srcs: [ + "Test.cpp", + ], + export_include_dirs: ["include"], + stl: "none", + } + + cc_library_shared { + name: "mynativelib2", + srcs: [ + "Test.cpp", + ], + export_include_dirs: ["include"], + stl: "none", + } + `) + + result.CheckSnapshot("mysdk", "", + checkAllCopyRules(` +include/Test.h -> include/include/Test.h +.intermediates/mynativelib1/android_arm64_armv8-a_shared/mynativelib1.so -> arm64/lib/mynativelib1.so +.intermediates/mynativelib1/android_arm_armv7-a-neon_shared/mynativelib1.so -> arm/lib/mynativelib1.so +.intermediates/mynativelib2/android_arm64_armv8-a_shared/mynativelib2.so -> arm64/lib/mynativelib2.so +.intermediates/mynativelib2/android_arm_armv7-a-neon_shared/mynativelib2.so -> arm/lib/mynativelib2.so +`), + ) +} + +// Verify that when the shared library has some common and some arch specific properties that the generated +// snapshot is optimized properly. +func TestSnapshotWithCcSharedLibraryCommonProperties(t *testing.T) { + result := testSdkWithCc(t, ` + sdk { + name: "mysdk", + native_shared_libs: ["mynativelib"], + } + + cc_library_shared { + name: "mynativelib", + srcs: [ + "Test.cpp", + "aidl/foo/bar/Test.aidl", + ], + export_include_dirs: ["include"], + arch: { + arm64: { + export_system_include_dirs: ["arm64/include"], + }, + }, + stl: "none", + } + `) + + result.CheckSnapshot("mysdk", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +cc_prebuilt_library_shared { + name: "mysdk_mynativelib@current", + sdk_member_name: "mynativelib", + installable: false, + stl: "none", + export_include_dirs: ["include/include"], + arch: { + arm64: { + srcs: ["arm64/lib/mynativelib.so"], + export_system_include_dirs: ["arm64/include/arm64/include"], + }, + arm: { + srcs: ["arm/lib/mynativelib.so"], + }, + }, +} + +cc_prebuilt_library_shared { + name: "mynativelib", + prefer: false, + stl: "none", + export_include_dirs: ["include/include"], + arch: { + arm64: { + srcs: ["arm64/lib/mynativelib.so"], + export_system_include_dirs: ["arm64/include/arm64/include"], + }, + arm: { + srcs: ["arm/lib/mynativelib.so"], + }, + }, +} + +sdk_snapshot { + name: "mysdk@current", + native_shared_libs: ["mysdk_mynativelib@current"], +} +`), + checkAllCopyRules(` +include/Test.h -> include/include/Test.h +.intermediates/mynativelib/android_arm64_armv8-a_shared/mynativelib.so -> arm64/lib/mynativelib.so +arm64/include/Arm64Test.h -> arm64/include/arm64/include/Arm64Test.h +.intermediates/mynativelib/android_arm_armv7-a-neon_shared/mynativelib.so -> arm/lib/mynativelib.so`), + ) +} + +func TestSnapshotWithCcBinary(t *testing.T) { + result := testSdkWithCc(t, ` + module_exports { + name: "mymodule_exports", + native_binaries: ["mynativebinary"], + } + + cc_binary { + name: "mynativebinary", + srcs: [ + "Test.cpp", + ], + compile_multilib: "both", + stl: "none", + } + `) + + result.CheckSnapshot("mymodule_exports", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +cc_prebuilt_binary { + name: "mymodule_exports_mynativebinary@current", + sdk_member_name: "mynativebinary", + installable: false, + compile_multilib: "both", + arch: { + arm64: { + srcs: ["arm64/bin/mynativebinary"], + }, + arm: { + srcs: ["arm/bin/mynativebinary"], + }, + }, +} + +cc_prebuilt_binary { + name: "mynativebinary", + prefer: false, + compile_multilib: "both", + arch: { + arm64: { + srcs: ["arm64/bin/mynativebinary"], + }, + arm: { + srcs: ["arm/bin/mynativebinary"], + }, + }, +} + +module_exports_snapshot { + name: "mymodule_exports@current", + native_binaries: ["mymodule_exports_mynativebinary@current"], +} +`), + checkAllCopyRules(` +.intermediates/mynativebinary/android_arm64_armv8-a/mynativebinary -> arm64/bin/mynativebinary +.intermediates/mynativebinary/android_arm_armv7-a-neon/mynativebinary -> arm/bin/mynativebinary +`), + ) +} + +func TestMultipleHostOsTypesSnapshotWithCcBinary(t *testing.T) { + // b/145598135 - Generating host snapshots for anything other than linux is not supported. + SkipIfNotLinux(t) + + result := testSdkWithCc(t, ` + module_exports { + name: "myexports", + device_supported: false, + host_supported: true, + native_binaries: ["mynativebinary"], + target: { + windows: { + enabled: true, + }, + }, + } + + cc_binary { + name: "mynativebinary", + device_supported: false, + host_supported: true, + srcs: [ + "Test.cpp", + ], + compile_multilib: "both", + stl: "none", + target: { + windows: { + enabled: true, + }, + }, + } + `) + + result.CheckSnapshot("myexports", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +cc_prebuilt_binary { + name: "myexports_mynativebinary@current", + sdk_member_name: "mynativebinary", + device_supported: false, + host_supported: true, + installable: false, + target: { + linux_glibc: { + compile_multilib: "both", + }, + linux_glibc_x86_64: { + srcs: ["linux_glibc/x86_64/bin/mynativebinary"], + }, + linux_glibc_x86: { + srcs: ["linux_glibc/x86/bin/mynativebinary"], + }, + windows: { + compile_multilib: "64", + }, + windows_x86_64: { + srcs: ["windows/x86_64/bin/mynativebinary.exe"], + }, + }, +} + +cc_prebuilt_binary { + name: "mynativebinary", + prefer: false, + device_supported: false, + host_supported: true, + target: { + linux_glibc: { + compile_multilib: "both", + }, + linux_glibc_x86_64: { + srcs: ["linux_glibc/x86_64/bin/mynativebinary"], + }, + linux_glibc_x86: { + srcs: ["linux_glibc/x86/bin/mynativebinary"], + }, + windows: { + compile_multilib: "64", + }, + windows_x86_64: { + srcs: ["windows/x86_64/bin/mynativebinary.exe"], + }, + }, +} + +module_exports_snapshot { + name: "myexports@current", + device_supported: false, + host_supported: true, + native_binaries: ["myexports_mynativebinary@current"], + target: { + windows: { + compile_multilib: "64", + }, + }, +} +`), + checkAllCopyRules(` +.intermediates/mynativebinary/linux_glibc_x86_64/mynativebinary -> linux_glibc/x86_64/bin/mynativebinary +.intermediates/mynativebinary/linux_glibc_x86/mynativebinary -> linux_glibc/x86/bin/mynativebinary +.intermediates/mynativebinary/windows_x86_64/mynativebinary.exe -> windows/x86_64/bin/mynativebinary.exe +`), + ) +} + +func TestSnapshotWithCcSharedLibrary(t *testing.T) { + result := testSdkWithCc(t, ` + sdk { + name: "mysdk", + native_shared_libs: ["mynativelib"], + } + + cc_library_shared { + name: "mynativelib", + srcs: [ + "Test.cpp", + "aidl/foo/bar/Test.aidl", + ], + apex_available: ["apex1", "apex2"], + export_include_dirs: ["include"], + aidl: { + export_aidl_headers: true, + }, + stl: "none", + } + `) + + result.CheckSnapshot("mysdk", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +cc_prebuilt_library_shared { + name: "mysdk_mynativelib@current", + sdk_member_name: "mynativelib", + apex_available: [ + "apex1", + "apex2", + ], + installable: false, + stl: "none", + export_include_dirs: ["include/include"], + arch: { + arm64: { + srcs: ["arm64/lib/mynativelib.so"], + export_include_dirs: ["arm64/include_gen/mynativelib"], + }, + arm: { + srcs: ["arm/lib/mynativelib.so"], + export_include_dirs: ["arm/include_gen/mynativelib"], + }, + }, +} + +cc_prebuilt_library_shared { + name: "mynativelib", + prefer: false, + apex_available: [ + "apex1", + "apex2", + ], + stl: "none", + export_include_dirs: ["include/include"], + arch: { + arm64: { + srcs: ["arm64/lib/mynativelib.so"], + export_include_dirs: ["arm64/include_gen/mynativelib"], + }, + arm: { + srcs: ["arm/lib/mynativelib.so"], + export_include_dirs: ["arm/include_gen/mynativelib"], + }, + }, +} + +sdk_snapshot { + name: "mysdk@current", + native_shared_libs: ["mysdk_mynativelib@current"], +} +`), + checkAllCopyRules(` +include/Test.h -> include/include/Test.h +.intermediates/mynativelib/android_arm64_armv8-a_shared/mynativelib.so -> arm64/lib/mynativelib.so +.intermediates/mynativelib/android_arm64_armv8-a_shared/gen/aidl/aidl/foo/bar/Test.h -> arm64/include_gen/mynativelib/aidl/foo/bar/Test.h +.intermediates/mynativelib/android_arm64_armv8-a_shared/gen/aidl/aidl/foo/bar/BnTest.h -> arm64/include_gen/mynativelib/aidl/foo/bar/BnTest.h +.intermediates/mynativelib/android_arm64_armv8-a_shared/gen/aidl/aidl/foo/bar/BpTest.h -> arm64/include_gen/mynativelib/aidl/foo/bar/BpTest.h +.intermediates/mynativelib/android_arm_armv7-a-neon_shared/mynativelib.so -> arm/lib/mynativelib.so +.intermediates/mynativelib/android_arm_armv7-a-neon_shared/gen/aidl/aidl/foo/bar/Test.h -> arm/include_gen/mynativelib/aidl/foo/bar/Test.h +.intermediates/mynativelib/android_arm_armv7-a-neon_shared/gen/aidl/aidl/foo/bar/BnTest.h -> arm/include_gen/mynativelib/aidl/foo/bar/BnTest.h +.intermediates/mynativelib/android_arm_armv7-a-neon_shared/gen/aidl/aidl/foo/bar/BpTest.h -> arm/include_gen/mynativelib/aidl/foo/bar/BpTest.h +`), + ) +} + +func TestSnapshotWithCcSharedLibrarySharedLibs(t *testing.T) { + result := testSdkWithCc(t, ` + sdk { + name: "mysdk", + native_shared_libs: [ + "mynativelib", + "myothernativelib", + "mysystemnativelib", + ], + } + + cc_library { + name: "mysystemnativelib", + srcs: [ + "Test.cpp", + ], + stl: "none", + } + + cc_library_shared { + name: "myothernativelib", + srcs: [ + "Test.cpp", + ], + system_shared_libs: [ + // A reference to a library that is not an sdk member. Uses libm as that + // is in the default set of modules available to this test and so is available + // both here and also when the generated Android.bp file is tested in + // CheckSnapshot(). This ensures that the system_shared_libs property correctly + // handles references to modules that are not sdk members. + "libm", + ], + stl: "none", + } + + cc_library { + name: "mynativelib", + srcs: [ + "Test.cpp", + ], + shared_libs: [ + // A reference to another sdk member. + "myothernativelib", + ], + target: { + android: { + shared: { + shared_libs: [ + // A reference to a library that is not an sdk member. The libc library + // is used here to check that the shared_libs property is handled correctly + // in a similar way to how libm is used to check system_shared_libs above. + "libc", + ], + }, + }, + }, + stl: "none", + } + `) + + result.CheckSnapshot("mysdk", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +cc_prebuilt_library_shared { + name: "mysdk_mynativelib@current", + sdk_member_name: "mynativelib", + installable: false, + stl: "none", + shared_libs: [ + "mysdk_myothernativelib@current", + "libc", + ], + arch: { + arm64: { + srcs: ["arm64/lib/mynativelib.so"], + }, + arm: { + srcs: ["arm/lib/mynativelib.so"], + }, + }, +} + +cc_prebuilt_library_shared { + name: "mynativelib", + prefer: false, + stl: "none", + shared_libs: [ + "myothernativelib", + "libc", + ], + arch: { + arm64: { + srcs: ["arm64/lib/mynativelib.so"], + }, + arm: { + srcs: ["arm/lib/mynativelib.so"], + }, + }, +} + +cc_prebuilt_library_shared { + name: "mysdk_myothernativelib@current", + sdk_member_name: "myothernativelib", + installable: false, + stl: "none", + system_shared_libs: ["libm"], + arch: { + arm64: { + srcs: ["arm64/lib/myothernativelib.so"], + }, + arm: { + srcs: ["arm/lib/myothernativelib.so"], + }, + }, +} + +cc_prebuilt_library_shared { + name: "myothernativelib", + prefer: false, + stl: "none", + system_shared_libs: ["libm"], + arch: { + arm64: { + srcs: ["arm64/lib/myothernativelib.so"], + }, + arm: { + srcs: ["arm/lib/myothernativelib.so"], + }, + }, +} + +cc_prebuilt_library_shared { + name: "mysdk_mysystemnativelib@current", + sdk_member_name: "mysystemnativelib", + installable: false, + stl: "none", + arch: { + arm64: { + srcs: ["arm64/lib/mysystemnativelib.so"], + }, + arm: { + srcs: ["arm/lib/mysystemnativelib.so"], + }, + }, +} + +cc_prebuilt_library_shared { + name: "mysystemnativelib", + prefer: false, + stl: "none", + arch: { + arm64: { + srcs: ["arm64/lib/mysystemnativelib.so"], + }, + arm: { + srcs: ["arm/lib/mysystemnativelib.so"], + }, + }, +} + +sdk_snapshot { + name: "mysdk@current", + native_shared_libs: [ + "mysdk_mynativelib@current", + "mysdk_myothernativelib@current", + "mysdk_mysystemnativelib@current", + ], +} +`), + checkAllCopyRules(` +.intermediates/mynativelib/android_arm64_armv8-a_shared/mynativelib.so -> arm64/lib/mynativelib.so +.intermediates/mynativelib/android_arm_armv7-a-neon_shared/mynativelib.so -> arm/lib/mynativelib.so +.intermediates/myothernativelib/android_arm64_armv8-a_shared/myothernativelib.so -> arm64/lib/myothernativelib.so +.intermediates/myothernativelib/android_arm_armv7-a-neon_shared/myothernativelib.so -> arm/lib/myothernativelib.so +.intermediates/mysystemnativelib/android_arm64_armv8-a_shared/mysystemnativelib.so -> arm64/lib/mysystemnativelib.so +.intermediates/mysystemnativelib/android_arm_armv7-a-neon_shared/mysystemnativelib.so -> arm/lib/mysystemnativelib.so +`), + ) +} + +func TestHostSnapshotWithCcSharedLibrary(t *testing.T) { + // b/145598135 - Generating host snapshots for anything other than linux is not supported. + SkipIfNotLinux(t) + + result := testSdkWithCc(t, ` + sdk { + name: "mysdk", + device_supported: false, + host_supported: true, + native_shared_libs: ["mynativelib"], + } + + cc_library_shared { + name: "mynativelib", + device_supported: false, + host_supported: true, + srcs: [ + "Test.cpp", + "aidl/foo/bar/Test.aidl", + ], + export_include_dirs: ["include"], + aidl: { + export_aidl_headers: true, + }, + stl: "none", + sdk_version: "minimum", + } + `) + + result.CheckSnapshot("mysdk", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +cc_prebuilt_library_shared { + name: "mysdk_mynativelib@current", + sdk_member_name: "mynativelib", + device_supported: false, + host_supported: true, + installable: false, + sdk_version: "minimum", + stl: "none", + export_include_dirs: ["include/include"], + arch: { + x86_64: { + srcs: ["x86_64/lib/mynativelib.so"], + export_include_dirs: ["x86_64/include_gen/mynativelib"], + }, + x86: { + srcs: ["x86/lib/mynativelib.so"], + export_include_dirs: ["x86/include_gen/mynativelib"], + }, + }, +} + +cc_prebuilt_library_shared { + name: "mynativelib", + prefer: false, + device_supported: false, + host_supported: true, + sdk_version: "minimum", + stl: "none", + export_include_dirs: ["include/include"], + arch: { + x86_64: { + srcs: ["x86_64/lib/mynativelib.so"], + export_include_dirs: ["x86_64/include_gen/mynativelib"], + }, + x86: { + srcs: ["x86/lib/mynativelib.so"], + export_include_dirs: ["x86/include_gen/mynativelib"], + }, + }, +} + +sdk_snapshot { + name: "mysdk@current", + device_supported: false, + host_supported: true, + native_shared_libs: ["mysdk_mynativelib@current"], +} +`), + checkAllCopyRules(` +include/Test.h -> include/include/Test.h +.intermediates/mynativelib/linux_glibc_x86_64_shared/mynativelib.so -> x86_64/lib/mynativelib.so +.intermediates/mynativelib/linux_glibc_x86_64_shared/gen/aidl/aidl/foo/bar/Test.h -> x86_64/include_gen/mynativelib/aidl/foo/bar/Test.h +.intermediates/mynativelib/linux_glibc_x86_64_shared/gen/aidl/aidl/foo/bar/BnTest.h -> x86_64/include_gen/mynativelib/aidl/foo/bar/BnTest.h +.intermediates/mynativelib/linux_glibc_x86_64_shared/gen/aidl/aidl/foo/bar/BpTest.h -> x86_64/include_gen/mynativelib/aidl/foo/bar/BpTest.h +.intermediates/mynativelib/linux_glibc_x86_shared/mynativelib.so -> x86/lib/mynativelib.so +.intermediates/mynativelib/linux_glibc_x86_shared/gen/aidl/aidl/foo/bar/Test.h -> x86/include_gen/mynativelib/aidl/foo/bar/Test.h +.intermediates/mynativelib/linux_glibc_x86_shared/gen/aidl/aidl/foo/bar/BnTest.h -> x86/include_gen/mynativelib/aidl/foo/bar/BnTest.h +.intermediates/mynativelib/linux_glibc_x86_shared/gen/aidl/aidl/foo/bar/BpTest.h -> x86/include_gen/mynativelib/aidl/foo/bar/BpTest.h +`), + ) +} + +func TestMultipleHostOsTypesSnapshotWithCcSharedLibrary(t *testing.T) { + // b/145598135 - Generating host snapshots for anything other than linux is not supported. + SkipIfNotLinux(t) + + result := testSdkWithCc(t, ` + sdk { + name: "mysdk", + device_supported: false, + host_supported: true, + native_shared_libs: ["mynativelib"], + target: { + windows: { + enabled: true, + }, + }, + } + + cc_library_shared { + name: "mynativelib", + device_supported: false, + host_supported: true, + srcs: [ + "Test.cpp", + ], + stl: "none", + target: { + windows: { + enabled: true, + }, + }, + } + `) + + result.CheckSnapshot("mysdk", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +cc_prebuilt_library_shared { + name: "mysdk_mynativelib@current", + sdk_member_name: "mynativelib", + device_supported: false, + host_supported: true, + installable: false, + stl: "none", + target: { + linux_glibc_x86_64: { + srcs: ["linux_glibc/x86_64/lib/mynativelib.so"], + }, + linux_glibc_x86: { + srcs: ["linux_glibc/x86/lib/mynativelib.so"], + }, + windows_x86_64: { + srcs: ["windows/x86_64/lib/mynativelib.dll"], + }, + }, +} + +cc_prebuilt_library_shared { + name: "mynativelib", + prefer: false, + device_supported: false, + host_supported: true, + stl: "none", + target: { + linux_glibc_x86_64: { + srcs: ["linux_glibc/x86_64/lib/mynativelib.so"], + }, + linux_glibc_x86: { + srcs: ["linux_glibc/x86/lib/mynativelib.so"], + }, + windows_x86_64: { + srcs: ["windows/x86_64/lib/mynativelib.dll"], + }, + }, +} + +sdk_snapshot { + name: "mysdk@current", + device_supported: false, + host_supported: true, + native_shared_libs: ["mysdk_mynativelib@current"], + target: { + windows: { + compile_multilib: "64", + }, + }, +} +`), + checkAllCopyRules(` +.intermediates/mynativelib/linux_glibc_x86_64_shared/mynativelib.so -> linux_glibc/x86_64/lib/mynativelib.so +.intermediates/mynativelib/linux_glibc_x86_shared/mynativelib.so -> linux_glibc/x86/lib/mynativelib.so +.intermediates/mynativelib/windows_x86_64_shared/mynativelib.dll -> windows/x86_64/lib/mynativelib.dll +`), + ) +} + +func TestSnapshotWithCcStaticLibrary(t *testing.T) { + result := testSdkWithCc(t, ` + module_exports { + name: "myexports", + native_static_libs: ["mynativelib"], + } + + cc_library_static { + name: "mynativelib", + srcs: [ + "Test.cpp", + "aidl/foo/bar/Test.aidl", + ], + export_include_dirs: ["include"], + aidl: { + export_aidl_headers: true, + }, + stl: "none", + } + `) + + result.CheckSnapshot("myexports", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +cc_prebuilt_library_static { + name: "myexports_mynativelib@current", + sdk_member_name: "mynativelib", + installable: false, + stl: "none", + export_include_dirs: ["include/include"], + arch: { + arm64: { + srcs: ["arm64/lib/mynativelib.a"], + export_include_dirs: ["arm64/include_gen/mynativelib"], + }, + arm: { + srcs: ["arm/lib/mynativelib.a"], + export_include_dirs: ["arm/include_gen/mynativelib"], + }, + }, +} + +cc_prebuilt_library_static { + name: "mynativelib", + prefer: false, + stl: "none", + export_include_dirs: ["include/include"], + arch: { + arm64: { + srcs: ["arm64/lib/mynativelib.a"], + export_include_dirs: ["arm64/include_gen/mynativelib"], + }, + arm: { + srcs: ["arm/lib/mynativelib.a"], + export_include_dirs: ["arm/include_gen/mynativelib"], + }, + }, +} + +module_exports_snapshot { + name: "myexports@current", + native_static_libs: ["myexports_mynativelib@current"], +} +`), + checkAllCopyRules(` +include/Test.h -> include/include/Test.h +.intermediates/mynativelib/android_arm64_armv8-a_static/mynativelib.a -> arm64/lib/mynativelib.a +.intermediates/mynativelib/android_arm64_armv8-a_static/gen/aidl/aidl/foo/bar/Test.h -> arm64/include_gen/mynativelib/aidl/foo/bar/Test.h +.intermediates/mynativelib/android_arm64_armv8-a_static/gen/aidl/aidl/foo/bar/BnTest.h -> arm64/include_gen/mynativelib/aidl/foo/bar/BnTest.h +.intermediates/mynativelib/android_arm64_armv8-a_static/gen/aidl/aidl/foo/bar/BpTest.h -> arm64/include_gen/mynativelib/aidl/foo/bar/BpTest.h +.intermediates/mynativelib/android_arm_armv7-a-neon_static/mynativelib.a -> arm/lib/mynativelib.a +.intermediates/mynativelib/android_arm_armv7-a-neon_static/gen/aidl/aidl/foo/bar/Test.h -> arm/include_gen/mynativelib/aidl/foo/bar/Test.h +.intermediates/mynativelib/android_arm_armv7-a-neon_static/gen/aidl/aidl/foo/bar/BnTest.h -> arm/include_gen/mynativelib/aidl/foo/bar/BnTest.h +.intermediates/mynativelib/android_arm_armv7-a-neon_static/gen/aidl/aidl/foo/bar/BpTest.h -> arm/include_gen/mynativelib/aidl/foo/bar/BpTest.h +`), + ) +} + +func TestHostSnapshotWithCcStaticLibrary(t *testing.T) { + // b/145598135 - Generating host snapshots for anything other than linux is not supported. + SkipIfNotLinux(t) + + result := testSdkWithCc(t, ` + module_exports { + name: "myexports", + device_supported: false, + host_supported: true, + native_static_libs: ["mynativelib"], + } + + cc_library_static { + name: "mynativelib", + device_supported: false, + host_supported: true, + srcs: [ + "Test.cpp", + "aidl/foo/bar/Test.aidl", + ], + export_include_dirs: ["include"], + aidl: { + export_aidl_headers: true, + }, + stl: "none", + } + `) + + result.CheckSnapshot("myexports", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +cc_prebuilt_library_static { + name: "myexports_mynativelib@current", + sdk_member_name: "mynativelib", + device_supported: false, + host_supported: true, + installable: false, + stl: "none", + export_include_dirs: ["include/include"], + arch: { + x86_64: { + srcs: ["x86_64/lib/mynativelib.a"], + export_include_dirs: ["x86_64/include_gen/mynativelib"], + }, + x86: { + srcs: ["x86/lib/mynativelib.a"], + export_include_dirs: ["x86/include_gen/mynativelib"], + }, + }, +} + +cc_prebuilt_library_static { + name: "mynativelib", + prefer: false, + device_supported: false, + host_supported: true, + stl: "none", + export_include_dirs: ["include/include"], + arch: { + x86_64: { + srcs: ["x86_64/lib/mynativelib.a"], + export_include_dirs: ["x86_64/include_gen/mynativelib"], + }, + x86: { + srcs: ["x86/lib/mynativelib.a"], + export_include_dirs: ["x86/include_gen/mynativelib"], + }, + }, +} + +module_exports_snapshot { + name: "myexports@current", + device_supported: false, + host_supported: true, + native_static_libs: ["myexports_mynativelib@current"], +} +`), + checkAllCopyRules(` +include/Test.h -> include/include/Test.h +.intermediates/mynativelib/linux_glibc_x86_64_static/mynativelib.a -> x86_64/lib/mynativelib.a +.intermediates/mynativelib/linux_glibc_x86_64_static/gen/aidl/aidl/foo/bar/Test.h -> x86_64/include_gen/mynativelib/aidl/foo/bar/Test.h +.intermediates/mynativelib/linux_glibc_x86_64_static/gen/aidl/aidl/foo/bar/BnTest.h -> x86_64/include_gen/mynativelib/aidl/foo/bar/BnTest.h +.intermediates/mynativelib/linux_glibc_x86_64_static/gen/aidl/aidl/foo/bar/BpTest.h -> x86_64/include_gen/mynativelib/aidl/foo/bar/BpTest.h +.intermediates/mynativelib/linux_glibc_x86_static/mynativelib.a -> x86/lib/mynativelib.a +.intermediates/mynativelib/linux_glibc_x86_static/gen/aidl/aidl/foo/bar/Test.h -> x86/include_gen/mynativelib/aidl/foo/bar/Test.h +.intermediates/mynativelib/linux_glibc_x86_static/gen/aidl/aidl/foo/bar/BnTest.h -> x86/include_gen/mynativelib/aidl/foo/bar/BnTest.h +.intermediates/mynativelib/linux_glibc_x86_static/gen/aidl/aidl/foo/bar/BpTest.h -> x86/include_gen/mynativelib/aidl/foo/bar/BpTest.h +`), + ) +} + +func TestSnapshotWithCcLibrary(t *testing.T) { + result := testSdkWithCc(t, ` + module_exports { + name: "myexports", + native_libs: ["mynativelib"], + } + + cc_library { + name: "mynativelib", + srcs: [ + "Test.cpp", + ], + export_include_dirs: ["include"], + stl: "none", + } + `) + + result.CheckSnapshot("myexports", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +cc_prebuilt_library { + name: "myexports_mynativelib@current", + sdk_member_name: "mynativelib", + installable: false, + stl: "none", + export_include_dirs: ["include/include"], + arch: { + arm64: { + static: { + srcs: ["arm64/lib/mynativelib.a"], + }, + shared: { + srcs: ["arm64/lib/mynativelib.so"], + }, + }, + arm: { + static: { + srcs: ["arm/lib/mynativelib.a"], + }, + shared: { + srcs: ["arm/lib/mynativelib.so"], + }, + }, + }, +} + +cc_prebuilt_library { + name: "mynativelib", + prefer: false, + stl: "none", + export_include_dirs: ["include/include"], + arch: { + arm64: { + static: { + srcs: ["arm64/lib/mynativelib.a"], + }, + shared: { + srcs: ["arm64/lib/mynativelib.so"], + }, + }, + arm: { + static: { + srcs: ["arm/lib/mynativelib.a"], + }, + shared: { + srcs: ["arm/lib/mynativelib.so"], + }, + }, + }, +} + +module_exports_snapshot { + name: "myexports@current", + native_libs: ["myexports_mynativelib@current"], +} +`), + checkAllCopyRules(` +include/Test.h -> include/include/Test.h +.intermediates/mynativelib/android_arm64_armv8-a_static/mynativelib.a -> arm64/lib/mynativelib.a +.intermediates/mynativelib/android_arm64_armv8-a_shared/mynativelib.so -> arm64/lib/mynativelib.so +.intermediates/mynativelib/android_arm_armv7-a-neon_static/mynativelib.a -> arm/lib/mynativelib.a +.intermediates/mynativelib/android_arm_armv7-a-neon_shared/mynativelib.so -> arm/lib/mynativelib.so`), + ) +} + +func TestHostSnapshotWithMultiLib64(t *testing.T) { + // b/145598135 - Generating host snapshots for anything other than linux is not supported. + SkipIfNotLinux(t) + + result := testSdkWithCc(t, ` + module_exports { + name: "myexports", + device_supported: false, + host_supported: true, + target: { + host: { + compile_multilib: "64", + }, + }, + native_static_libs: ["mynativelib"], + } + + cc_library_static { + name: "mynativelib", + device_supported: false, + host_supported: true, + srcs: [ + "Test.cpp", + "aidl/foo/bar/Test.aidl", + ], + export_include_dirs: ["include"], + aidl: { + export_aidl_headers: true, + }, + stl: "none", + } + `) + + result.CheckSnapshot("myexports", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +cc_prebuilt_library_static { + name: "myexports_mynativelib@current", + sdk_member_name: "mynativelib", + device_supported: false, + host_supported: true, + installable: false, + stl: "none", + export_include_dirs: ["include/include"], + arch: { + x86_64: { + srcs: ["x86_64/lib/mynativelib.a"], + export_include_dirs: ["x86_64/include_gen/mynativelib"], + }, + }, +} + +cc_prebuilt_library_static { + name: "mynativelib", + prefer: false, + device_supported: false, + host_supported: true, + stl: "none", + export_include_dirs: ["include/include"], + arch: { + x86_64: { + srcs: ["x86_64/lib/mynativelib.a"], + export_include_dirs: ["x86_64/include_gen/mynativelib"], + }, + }, +} + +module_exports_snapshot { + name: "myexports@current", + device_supported: false, + host_supported: true, + native_static_libs: ["myexports_mynativelib@current"], + target: { + linux_glibc: { + compile_multilib: "64", + }, + }, +}`), + checkAllCopyRules(` +include/Test.h -> include/include/Test.h +.intermediates/mynativelib/linux_glibc_x86_64_static/mynativelib.a -> x86_64/lib/mynativelib.a +.intermediates/mynativelib/linux_glibc_x86_64_static/gen/aidl/aidl/foo/bar/Test.h -> x86_64/include_gen/mynativelib/aidl/foo/bar/Test.h +.intermediates/mynativelib/linux_glibc_x86_64_static/gen/aidl/aidl/foo/bar/BnTest.h -> x86_64/include_gen/mynativelib/aidl/foo/bar/BnTest.h +.intermediates/mynativelib/linux_glibc_x86_64_static/gen/aidl/aidl/foo/bar/BpTest.h -> x86_64/include_gen/mynativelib/aidl/foo/bar/BpTest.h +`), + ) +} + +func TestSnapshotWithCcHeadersLibrary(t *testing.T) { + result := testSdkWithCc(t, ` + sdk { + name: "mysdk", + native_header_libs: ["mynativeheaders"], + } + + cc_library_headers { + name: "mynativeheaders", + export_include_dirs: ["include"], + stl: "none", + } + `) + + result.CheckSnapshot("mysdk", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +cc_prebuilt_library_headers { + name: "mysdk_mynativeheaders@current", + sdk_member_name: "mynativeheaders", + stl: "none", + export_include_dirs: ["include/include"], +} + +cc_prebuilt_library_headers { + name: "mynativeheaders", + prefer: false, + stl: "none", + export_include_dirs: ["include/include"], +} + +sdk_snapshot { + name: "mysdk@current", + native_header_libs: ["mysdk_mynativeheaders@current"], +} +`), + checkAllCopyRules(` +include/Test.h -> include/include/Test.h +`), + ) +} + +func TestHostSnapshotWithCcHeadersLibrary(t *testing.T) { + // b/145598135 - Generating host snapshots for anything other than linux is not supported. + SkipIfNotLinux(t) + + result := testSdkWithCc(t, ` + sdk { + name: "mysdk", + device_supported: false, + host_supported: true, + native_header_libs: ["mynativeheaders"], + } + + cc_library_headers { + name: "mynativeheaders", + device_supported: false, + host_supported: true, + export_include_dirs: ["include"], + stl: "none", + } + `) + + result.CheckSnapshot("mysdk", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +cc_prebuilt_library_headers { + name: "mysdk_mynativeheaders@current", + sdk_member_name: "mynativeheaders", + device_supported: false, + host_supported: true, + stl: "none", + export_include_dirs: ["include/include"], +} + +cc_prebuilt_library_headers { + name: "mynativeheaders", + prefer: false, + device_supported: false, + host_supported: true, + stl: "none", + export_include_dirs: ["include/include"], +} + +sdk_snapshot { + name: "mysdk@current", + device_supported: false, + host_supported: true, + native_header_libs: ["mysdk_mynativeheaders@current"], +} +`), + checkAllCopyRules(` +include/Test.h -> include/include/Test.h +`), + ) +} + +func TestDeviceAndHostSnapshotWithCcHeadersLibrary(t *testing.T) { + // b/145598135 - Generating host snapshots for anything other than linux is not supported. + SkipIfNotLinux(t) + + result := testSdkWithCc(t, ` + sdk { + name: "mysdk", + host_supported: true, + native_header_libs: ["mynativeheaders"], + } + + cc_library_headers { + name: "mynativeheaders", + host_supported: true, + stl: "none", + export_system_include_dirs: ["include"], + target: { + android: { + export_include_dirs: ["include-android"], + }, + host: { + export_include_dirs: ["include-host"], + }, + }, + } + `) + + result.CheckSnapshot("mysdk", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +cc_prebuilt_library_headers { + name: "mysdk_mynativeheaders@current", + sdk_member_name: "mynativeheaders", + host_supported: true, + stl: "none", + export_system_include_dirs: ["include/include"], + target: { + android: { + export_include_dirs: ["include/include-android"], + }, + linux_glibc: { + export_include_dirs: ["include/include-host"], + }, + }, +} + +cc_prebuilt_library_headers { + name: "mynativeheaders", + prefer: false, + host_supported: true, + stl: "none", + export_system_include_dirs: ["include/include"], + target: { + android: { + export_include_dirs: ["include/include-android"], + }, + linux_glibc: { + export_include_dirs: ["include/include-host"], + }, + }, +} + +sdk_snapshot { + name: "mysdk@current", + host_supported: true, + native_header_libs: ["mysdk_mynativeheaders@current"], +} +`), + checkAllCopyRules(` +include/Test.h -> include/include/Test.h +include-android/AndroidTest.h -> include/include-android/AndroidTest.h +include-host/HostTest.h -> include/include-host/HostTest.h +`), + ) +} + +func TestSystemSharedLibPropagation(t *testing.T) { + // b/145598135 - Generating host snapshots for anything other than linux is not supported. + SkipIfNotLinux(t) + + result := testSdkWithCc(t, ` + sdk { + name: "mysdk", + native_shared_libs: ["sslnil", "sslempty", "sslnonempty"], + } + + cc_library { + name: "sslnil", + host_supported: true, + } + + cc_library { + name: "sslempty", + system_shared_libs: [], + } + + cc_library { + name: "sslnonempty", + system_shared_libs: ["sslnil"], + } + `) + + result.CheckSnapshot("mysdk", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +cc_prebuilt_library_shared { + name: "mysdk_sslnil@current", + sdk_member_name: "sslnil", + installable: false, + arch: { + arm64: { + srcs: ["arm64/lib/sslnil.so"], + }, + arm: { + srcs: ["arm/lib/sslnil.so"], + }, + }, +} + +cc_prebuilt_library_shared { + name: "sslnil", + prefer: false, + arch: { + arm64: { + srcs: ["arm64/lib/sslnil.so"], + }, + arm: { + srcs: ["arm/lib/sslnil.so"], + }, + }, +} + +cc_prebuilt_library_shared { + name: "mysdk_sslempty@current", + sdk_member_name: "sslempty", + installable: false, + system_shared_libs: [], + arch: { + arm64: { + srcs: ["arm64/lib/sslempty.so"], + }, + arm: { + srcs: ["arm/lib/sslempty.so"], + }, + }, +} + +cc_prebuilt_library_shared { + name: "sslempty", + prefer: false, + system_shared_libs: [], + arch: { + arm64: { + srcs: ["arm64/lib/sslempty.so"], + }, + arm: { + srcs: ["arm/lib/sslempty.so"], + }, + }, +} + +cc_prebuilt_library_shared { + name: "mysdk_sslnonempty@current", + sdk_member_name: "sslnonempty", + installable: false, + system_shared_libs: ["mysdk_sslnil@current"], + arch: { + arm64: { + srcs: ["arm64/lib/sslnonempty.so"], + }, + arm: { + srcs: ["arm/lib/sslnonempty.so"], + }, + }, +} + +cc_prebuilt_library_shared { + name: "sslnonempty", + prefer: false, + system_shared_libs: ["sslnil"], + arch: { + arm64: { + srcs: ["arm64/lib/sslnonempty.so"], + }, + arm: { + srcs: ["arm/lib/sslnonempty.so"], + }, + }, +} + +sdk_snapshot { + name: "mysdk@current", + native_shared_libs: [ + "mysdk_sslnil@current", + "mysdk_sslempty@current", + "mysdk_sslnonempty@current", + ], +} +`)) + + result = testSdkWithCc(t, ` + sdk { + name: "mysdk", + host_supported: true, + native_shared_libs: ["sslvariants"], + } + + cc_library { + name: "sslvariants", + host_supported: true, + target: { + android: { + system_shared_libs: [], + }, + }, + } + `) + + result.CheckSnapshot("mysdk", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +cc_prebuilt_library_shared { + name: "mysdk_sslvariants@current", + sdk_member_name: "sslvariants", + host_supported: true, + installable: false, + target: { + android: { + system_shared_libs: [], + }, + android_arm64: { + srcs: ["android/arm64/lib/sslvariants.so"], + }, + android_arm: { + srcs: ["android/arm/lib/sslvariants.so"], + }, + linux_glibc_x86_64: { + srcs: ["linux_glibc/x86_64/lib/sslvariants.so"], + }, + linux_glibc_x86: { + srcs: ["linux_glibc/x86/lib/sslvariants.so"], + }, + }, +} + +cc_prebuilt_library_shared { + name: "sslvariants", + prefer: false, + host_supported: true, + target: { + android: { + system_shared_libs: [], + }, + android_arm64: { + srcs: ["android/arm64/lib/sslvariants.so"], + }, + android_arm: { + srcs: ["android/arm/lib/sslvariants.so"], + }, + linux_glibc_x86_64: { + srcs: ["linux_glibc/x86_64/lib/sslvariants.so"], + }, + linux_glibc_x86: { + srcs: ["linux_glibc/x86/lib/sslvariants.so"], + }, + }, +} + +sdk_snapshot { + name: "mysdk@current", + host_supported: true, + native_shared_libs: ["mysdk_sslvariants@current"], +} +`)) +} + +func TestStubsLibrary(t *testing.T) { + result := testSdkWithCc(t, ` + sdk { + name: "mysdk", + native_shared_libs: ["stubslib"], + } + + cc_library { + name: "internaldep", + } + + cc_library { + name: "stubslib", + shared_libs: ["internaldep"], + stubs: { + symbol_file: "some/where/stubslib.map.txt", + versions: ["1", "2", "3"], + }, + } + `) + + result.CheckSnapshot("mysdk", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +cc_prebuilt_library_shared { + name: "mysdk_stubslib@current", + sdk_member_name: "stubslib", + installable: false, + stubs: { + versions: ["3"], + }, + arch: { + arm64: { + srcs: ["arm64/lib/stubslib.so"], + }, + arm: { + srcs: ["arm/lib/stubslib.so"], + }, + }, +} + +cc_prebuilt_library_shared { + name: "stubslib", + prefer: false, + stubs: { + versions: ["3"], + }, + arch: { + arm64: { + srcs: ["arm64/lib/stubslib.so"], + }, + arm: { + srcs: ["arm/lib/stubslib.so"], + }, + }, +} + +sdk_snapshot { + name: "mysdk@current", + native_shared_libs: ["mysdk_stubslib@current"], +} +`)) +} + +func TestDeviceAndHostSnapshotWithStubsLibrary(t *testing.T) { + // b/145598135 - Generating host snapshots for anything other than linux is not supported. + SkipIfNotLinux(t) + + result := testSdkWithCc(t, ` + sdk { + name: "mysdk", + host_supported: true, + native_shared_libs: ["stubslib"], + } + + cc_library { + name: "internaldep", + host_supported: true, + } + + cc_library { + name: "stubslib", + host_supported: true, + shared_libs: ["internaldep"], + stubs: { + symbol_file: "some/where/stubslib.map.txt", + versions: ["1", "2", "3"], + }, + } + `) + + result.CheckSnapshot("mysdk", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +cc_prebuilt_library_shared { + name: "mysdk_stubslib@current", + sdk_member_name: "stubslib", + host_supported: true, + installable: false, + stubs: { + versions: ["3"], + }, + target: { + android_arm64: { + srcs: ["android/arm64/lib/stubslib.so"], + }, + android_arm: { + srcs: ["android/arm/lib/stubslib.so"], + }, + linux_glibc_x86_64: { + srcs: ["linux_glibc/x86_64/lib/stubslib.so"], + }, + linux_glibc_x86: { + srcs: ["linux_glibc/x86/lib/stubslib.so"], + }, + }, +} + +cc_prebuilt_library_shared { + name: "stubslib", + prefer: false, + host_supported: true, + stubs: { + versions: ["3"], + }, + target: { + android_arm64: { + srcs: ["android/arm64/lib/stubslib.so"], + }, + android_arm: { + srcs: ["android/arm/lib/stubslib.so"], + }, + linux_glibc_x86_64: { + srcs: ["linux_glibc/x86_64/lib/stubslib.so"], + }, + linux_glibc_x86: { + srcs: ["linux_glibc/x86/lib/stubslib.so"], + }, + }, +} + +sdk_snapshot { + name: "mysdk@current", + host_supported: true, + native_shared_libs: ["mysdk_stubslib@current"], +} +`)) +} diff --git a/sdk/exports.go b/sdk/exports.go new file mode 100644 index 000000000..d3130574e --- /dev/null +++ b/sdk/exports.go @@ -0,0 +1,36 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sdk + +import "android/soong/android" + +func init() { + android.RegisterModuleType("module_exports", ModuleExportsFactory) + android.RegisterModuleType("module_exports_snapshot", ModuleExportsSnapshotsFactory) +} + +// module_exports defines the exports of a mainline module. The exports are Soong modules +// which are required by Soong modules that are not part of the mainline module. +func ModuleExportsFactory() android.Module { + return newSdkModule(true) +} + +// module_exports_snapshot is a versioned snapshot of prebuilt versions of all the exports +// of a mainline module. +func ModuleExportsSnapshotsFactory() android.Module { + s := newSdkModule(true) + s.properties.Snapshot = true + return s +} diff --git a/sdk/exports_test.go b/sdk/exports_test.go new file mode 100644 index 000000000..20e25212c --- /dev/null +++ b/sdk/exports_test.go @@ -0,0 +1,66 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sdk + +import ( + "testing" +) + +// Ensure that module_exports generates a module_exports_snapshot module. +func TestModuleExportsSnapshot(t *testing.T) { + packageBp := ` + module_exports { + name: "myexports", + java_libs: [ + "myjavalib", + ], + } + + java_library { + name: "myjavalib", + srcs: ["Test.java"], + system_modules: "none", + sdk_version: "none", + } + ` + + result := testSdkWithFs(t, ``, + map[string][]byte{ + "package/Test.java": nil, + "package/Android.bp": []byte(packageBp), + }) + + result.CheckSnapshot("myexports", "package", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +java_import { + name: "myexports_myjavalib@current", + sdk_member_name: "myjavalib", + jars: ["java/myjavalib.jar"], +} + +java_import { + name: "myjavalib", + prefer: false, + jars: ["java/myjavalib.jar"], +} + +module_exports_snapshot { + name: "myexports@current", + java_libs: ["myexports_myjavalib@current"], +} +`)) +} diff --git a/sdk/java_sdk_test.go b/sdk/java_sdk_test.go new file mode 100644 index 000000000..db395c582 --- /dev/null +++ b/sdk/java_sdk_test.go @@ -0,0 +1,1560 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sdk + +import ( + "testing" +) + +func testSdkWithJava(t *testing.T, bp string) *testSdkResult { + t.Helper() + + fs := map[string][]byte{ + "Test.java": nil, + "aidl/foo/bar/Test.aidl": nil, + + // For java_sdk_library + "api/current.txt": nil, + "api/removed.txt": nil, + "api/system-current.txt": nil, + "api/system-removed.txt": nil, + "api/test-current.txt": nil, + "api/test-removed.txt": nil, + "api/module-lib-current.txt": nil, + "api/module-lib-removed.txt": nil, + "api/system-server-current.txt": nil, + "api/system-server-removed.txt": nil, + "build/soong/scripts/gen-java-current-api-files.sh": nil, + } + + // for java_sdk_library tests + bp = ` +java_system_modules_import { + name: "core-current-stubs-system-modules", +} +java_system_modules_import { + name: "core-platform-api-stubs-system-modules", +} +java_import { + name: "core.platform.api.stubs", +} +java_import { + name: "android_stubs_current", +} +java_import { + name: "android_system_stubs_current", +} +java_import { + name: "android_test_stubs_current", +} +java_import { + name: "android_module_lib_stubs_current", +} +java_import { + name: "android_system_server_stubs_current", +} +java_import { + name: "core-lambda-stubs", + sdk_version: "none", +} +java_import { + name: "ext", + sdk_version: "none", +} +java_import { + name: "framework", + sdk_version: "none", +} +` + bp + + return testSdkWithFs(t, bp, fs) +} + +// Contains tests for SDK members provided by the java package. + +func TestBasicSdkWithJavaLibrary(t *testing.T) { + result := testSdkWithJava(t, ` + sdk { + name: "mysdk", + java_header_libs: ["sdkmember"], + } + + sdk_snapshot { + name: "mysdk@1", + java_header_libs: ["sdkmember_mysdk_1"], + } + + sdk_snapshot { + name: "mysdk@2", + java_header_libs: ["sdkmember_mysdk_2"], + } + + java_library { + name: "sdkmember", + srcs: ["Test.java"], + system_modules: "none", + sdk_version: "none", + host_supported: true, + } + + java_import { + name: "sdkmember_mysdk_1", + sdk_member_name: "sdkmember", + host_supported: true, + } + + java_import { + name: "sdkmember_mysdk_2", + sdk_member_name: "sdkmember", + host_supported: true, + } + + java_library { + name: "myjavalib", + srcs: ["Test.java"], + libs: ["sdkmember"], + system_modules: "none", + sdk_version: "none", + compile_dex: true, + host_supported: true, + apex_available: [ + "myapex", + "myapex2", + ], + } + + apex { + name: "myapex", + java_libs: ["myjavalib"], + uses_sdks: ["mysdk@1"], + key: "myapex.key", + certificate: ":myapex.cert", + } + + apex { + name: "myapex2", + java_libs: ["myjavalib"], + uses_sdks: ["mysdk@2"], + key: "myapex.key", + certificate: ":myapex.cert", + } + `) + + sdkMemberV1 := result.ctx.ModuleForTests("sdkmember_mysdk_1", "android_common_myapex").Rule("combineJar").Output + sdkMemberV2 := result.ctx.ModuleForTests("sdkmember_mysdk_2", "android_common_myapex2").Rule("combineJar").Output + + javalibForMyApex := result.ctx.ModuleForTests("myjavalib", "android_common_myapex") + javalibForMyApex2 := result.ctx.ModuleForTests("myjavalib", "android_common_myapex2") + + // Depending on the uses_sdks value, different libs are linked + ensureListContains(t, pathsToStrings(javalibForMyApex.Rule("javac").Implicits), sdkMemberV1.String()) + ensureListContains(t, pathsToStrings(javalibForMyApex2.Rule("javac").Implicits), sdkMemberV2.String()) +} + +func TestSnapshotWithJavaHeaderLibrary(t *testing.T) { + result := testSdkWithJava(t, ` + sdk { + name: "mysdk", + java_header_libs: ["myjavalib"], + } + + java_library { + name: "myjavalib", + srcs: ["Test.java"], + aidl: { + export_include_dirs: ["aidl"], + }, + system_modules: "none", + sdk_version: "none", + compile_dex: true, + host_supported: true, + } + `) + + result.CheckSnapshot("mysdk", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +java_import { + name: "mysdk_myjavalib@current", + sdk_member_name: "myjavalib", + jars: ["java/myjavalib.jar"], +} + +java_import { + name: "myjavalib", + prefer: false, + jars: ["java/myjavalib.jar"], +} + +sdk_snapshot { + name: "mysdk@current", + java_header_libs: ["mysdk_myjavalib@current"], +} + +`), + checkAllCopyRules(` +.intermediates/myjavalib/android_common/turbine-combined/myjavalib.jar -> java/myjavalib.jar +aidl/foo/bar/Test.aidl -> aidl/aidl/foo/bar/Test.aidl +`), + ) +} + +func TestHostSnapshotWithJavaHeaderLibrary(t *testing.T) { + // b/145598135 - Generating host snapshots for anything other than linux is not supported. + SkipIfNotLinux(t) + + result := testSdkWithJava(t, ` + sdk { + name: "mysdk", + device_supported: false, + host_supported: true, + java_header_libs: ["myjavalib"], + } + + java_library { + name: "myjavalib", + device_supported: false, + host_supported: true, + srcs: ["Test.java"], + aidl: { + export_include_dirs: ["aidl"], + }, + system_modules: "none", + sdk_version: "none", + compile_dex: true, + } + `) + + result.CheckSnapshot("mysdk", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +java_import { + name: "mysdk_myjavalib@current", + sdk_member_name: "myjavalib", + device_supported: false, + host_supported: true, + jars: ["java/myjavalib.jar"], +} + +java_import { + name: "myjavalib", + prefer: false, + device_supported: false, + host_supported: true, + jars: ["java/myjavalib.jar"], +} + +sdk_snapshot { + name: "mysdk@current", + device_supported: false, + host_supported: true, + java_header_libs: ["mysdk_myjavalib@current"], +} +`), + checkAllCopyRules(` +.intermediates/myjavalib/linux_glibc_common/javac/myjavalib.jar -> java/myjavalib.jar +aidl/foo/bar/Test.aidl -> aidl/aidl/foo/bar/Test.aidl +`), + ) +} + +func TestDeviceAndHostSnapshotWithJavaHeaderLibrary(t *testing.T) { + // b/145598135 - Generating host snapshots for anything other than linux is not supported. + SkipIfNotLinux(t) + + result := testSdkWithJava(t, ` + sdk { + name: "mysdk", + host_supported: true, + java_header_libs: ["myjavalib"], + } + + java_library { + name: "myjavalib", + host_supported: true, + srcs: ["Test.java"], + system_modules: "none", + sdk_version: "none", + compile_dex: true, + } + `) + + result.CheckSnapshot("mysdk", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +java_import { + name: "mysdk_myjavalib@current", + sdk_member_name: "myjavalib", + host_supported: true, + target: { + android: { + jars: ["java/android/myjavalib.jar"], + }, + linux_glibc: { + jars: ["java/linux_glibc/myjavalib.jar"], + }, + }, +} + +java_import { + name: "myjavalib", + prefer: false, + host_supported: true, + target: { + android: { + jars: ["java/android/myjavalib.jar"], + }, + linux_glibc: { + jars: ["java/linux_glibc/myjavalib.jar"], + }, + }, +} + +sdk_snapshot { + name: "mysdk@current", + host_supported: true, + java_header_libs: ["mysdk_myjavalib@current"], +} +`), + checkAllCopyRules(` +.intermediates/myjavalib/android_common/turbine-combined/myjavalib.jar -> java/android/myjavalib.jar +.intermediates/myjavalib/linux_glibc_common/javac/myjavalib.jar -> java/linux_glibc/myjavalib.jar +`), + ) +} + +func TestSnapshotWithJavaImplLibrary(t *testing.T) { + result := testSdkWithJava(t, ` + module_exports { + name: "myexports", + java_libs: ["myjavalib"], + } + + java_library { + name: "myjavalib", + srcs: ["Test.java"], + aidl: { + export_include_dirs: ["aidl"], + }, + system_modules: "none", + sdk_version: "none", + compile_dex: true, + host_supported: true, + } + `) + + result.CheckSnapshot("myexports", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +java_import { + name: "myexports_myjavalib@current", + sdk_member_name: "myjavalib", + jars: ["java/myjavalib.jar"], +} + +java_import { + name: "myjavalib", + prefer: false, + jars: ["java/myjavalib.jar"], +} + +module_exports_snapshot { + name: "myexports@current", + java_libs: ["myexports_myjavalib@current"], +} + +`), + checkAllCopyRules(` +.intermediates/myjavalib/android_common/javac/myjavalib.jar -> java/myjavalib.jar +aidl/foo/bar/Test.aidl -> aidl/aidl/foo/bar/Test.aidl +`), + ) +} + +func TestHostSnapshotWithJavaImplLibrary(t *testing.T) { + // b/145598135 - Generating host snapshots for anything other than linux is not supported. + SkipIfNotLinux(t) + + result := testSdkWithJava(t, ` + module_exports { + name: "myexports", + device_supported: false, + host_supported: true, + java_libs: ["myjavalib"], + } + + java_library { + name: "myjavalib", + device_supported: false, + host_supported: true, + srcs: ["Test.java"], + aidl: { + export_include_dirs: ["aidl"], + }, + system_modules: "none", + sdk_version: "none", + compile_dex: true, + } + `) + + result.CheckSnapshot("myexports", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +java_import { + name: "myexports_myjavalib@current", + sdk_member_name: "myjavalib", + device_supported: false, + host_supported: true, + jars: ["java/myjavalib.jar"], +} + +java_import { + name: "myjavalib", + prefer: false, + device_supported: false, + host_supported: true, + jars: ["java/myjavalib.jar"], +} + +module_exports_snapshot { + name: "myexports@current", + device_supported: false, + host_supported: true, + java_libs: ["myexports_myjavalib@current"], +} +`), + checkAllCopyRules(` +.intermediates/myjavalib/linux_glibc_common/javac/myjavalib.jar -> java/myjavalib.jar +aidl/foo/bar/Test.aidl -> aidl/aidl/foo/bar/Test.aidl +`), + ) +} + +func TestSnapshotWithJavaTest(t *testing.T) { + result := testSdkWithJava(t, ` + module_exports { + name: "myexports", + java_tests: ["myjavatests"], + } + + java_test { + name: "myjavatests", + srcs: ["Test.java"], + system_modules: "none", + sdk_version: "none", + compile_dex: true, + host_supported: true, + } + `) + + result.CheckSnapshot("myexports", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +java_test_import { + name: "myexports_myjavatests@current", + sdk_member_name: "myjavatests", + jars: ["java/myjavatests.jar"], + test_config: "java/myjavatests-AndroidTest.xml", +} + +java_test_import { + name: "myjavatests", + prefer: false, + jars: ["java/myjavatests.jar"], + test_config: "java/myjavatests-AndroidTest.xml", +} + +module_exports_snapshot { + name: "myexports@current", + java_tests: ["myexports_myjavatests@current"], +} +`), + checkAllCopyRules(` +.intermediates/myjavatests/android_common/javac/myjavatests.jar -> java/myjavatests.jar +.intermediates/myjavatests/android_common/myjavatests.config -> java/myjavatests-AndroidTest.xml +`), + ) +} + +func TestHostSnapshotWithJavaTest(t *testing.T) { + // b/145598135 - Generating host snapshots for anything other than linux is not supported. + SkipIfNotLinux(t) + + result := testSdkWithJava(t, ` + module_exports { + name: "myexports", + device_supported: false, + host_supported: true, + java_tests: ["myjavatests"], + } + + java_test { + name: "myjavatests", + device_supported: false, + host_supported: true, + srcs: ["Test.java"], + system_modules: "none", + sdk_version: "none", + compile_dex: true, + } + `) + + result.CheckSnapshot("myexports", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +java_test_import { + name: "myexports_myjavatests@current", + sdk_member_name: "myjavatests", + device_supported: false, + host_supported: true, + jars: ["java/myjavatests.jar"], + test_config: "java/myjavatests-AndroidTest.xml", +} + +java_test_import { + name: "myjavatests", + prefer: false, + device_supported: false, + host_supported: true, + jars: ["java/myjavatests.jar"], + test_config: "java/myjavatests-AndroidTest.xml", +} + +module_exports_snapshot { + name: "myexports@current", + device_supported: false, + host_supported: true, + java_tests: ["myexports_myjavatests@current"], +} +`), + checkAllCopyRules(` +.intermediates/myjavatests/linux_glibc_common/javac/myjavatests.jar -> java/myjavatests.jar +.intermediates/myjavatests/linux_glibc_common/myjavatests.config -> java/myjavatests-AndroidTest.xml +`), + ) +} + +func testSdkWithDroidstubs(t *testing.T, bp string) *testSdkResult { + t.Helper() + + fs := map[string][]byte{ + "foo/bar/Foo.java": nil, + "stubs-sources/foo/bar/Foo.java": nil, + } + return testSdkWithFs(t, bp, fs) +} + +// Note: This test does not verify that a droidstubs can be referenced, either +// directly or indirectly from an APEX as droidstubs can never be a part of an +// apex. +func TestBasicSdkWithDroidstubs(t *testing.T) { + testSdkWithDroidstubs(t, ` + sdk { + name: "mysdk", + stubs_sources: ["mystub"], + } + sdk_snapshot { + name: "mysdk@10", + stubs_sources: ["mystub_mysdk@10"], + } + prebuilt_stubs_sources { + name: "mystub_mysdk@10", + sdk_member_name: "mystub", + srcs: ["stubs-sources/foo/bar/Foo.java"], + } + droidstubs { + name: "mystub", + srcs: ["foo/bar/Foo.java"], + sdk_version: "none", + system_modules: "none", + } + java_library { + name: "myjavalib", + srcs: [":mystub"], + sdk_version: "none", + system_modules: "none", + } + `) +} + +func TestSnapshotWithDroidstubs(t *testing.T) { + result := testSdkWithDroidstubs(t, ` + module_exports { + name: "myexports", + stubs_sources: ["myjavaapistubs"], + } + + droidstubs { + name: "myjavaapistubs", + srcs: ["foo/bar/Foo.java"], + system_modules: "none", + sdk_version: "none", + } + `) + + result.CheckSnapshot("myexports", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +prebuilt_stubs_sources { + name: "myexports_myjavaapistubs@current", + sdk_member_name: "myjavaapistubs", + srcs: ["java/myjavaapistubs_stubs_sources"], +} + +prebuilt_stubs_sources { + name: "myjavaapistubs", + prefer: false, + srcs: ["java/myjavaapistubs_stubs_sources"], +} + +module_exports_snapshot { + name: "myexports@current", + stubs_sources: ["myexports_myjavaapistubs@current"], +} + +`), + checkAllCopyRules(""), + checkMergeZips(".intermediates/myexports/common_os/tmp/java/myjavaapistubs_stubs_sources.zip"), + ) +} + +func TestHostSnapshotWithDroidstubs(t *testing.T) { + // b/145598135 - Generating host snapshots for anything other than linux is not supported. + SkipIfNotLinux(t) + + result := testSdkWithDroidstubs(t, ` + module_exports { + name: "myexports", + device_supported: false, + host_supported: true, + stubs_sources: ["myjavaapistubs"], + } + + droidstubs { + name: "myjavaapistubs", + device_supported: false, + host_supported: true, + srcs: ["foo/bar/Foo.java"], + system_modules: "none", + sdk_version: "none", + } + `) + + result.CheckSnapshot("myexports", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +prebuilt_stubs_sources { + name: "myexports_myjavaapistubs@current", + sdk_member_name: "myjavaapistubs", + device_supported: false, + host_supported: true, + srcs: ["java/myjavaapistubs_stubs_sources"], +} + +prebuilt_stubs_sources { + name: "myjavaapistubs", + prefer: false, + device_supported: false, + host_supported: true, + srcs: ["java/myjavaapistubs_stubs_sources"], +} + +module_exports_snapshot { + name: "myexports@current", + device_supported: false, + host_supported: true, + stubs_sources: ["myexports_myjavaapistubs@current"], +} +`), + checkAllCopyRules(""), + checkMergeZips(".intermediates/myexports/common_os/tmp/java/myjavaapistubs_stubs_sources.zip"), + ) +} + +func TestSnapshotWithJavaSystemModules(t *testing.T) { + result := testSdkWithJava(t, ` + sdk { + name: "mysdk", + java_header_libs: ["exported-system-module"], + java_system_modules: ["my-system-modules"], + } + + java_system_modules { + name: "my-system-modules", + libs: ["system-module", "exported-system-module"], + } + + java_library { + name: "system-module", + srcs: ["Test.java"], + sdk_version: "none", + system_modules: "none", + } + + java_library { + name: "exported-system-module", + srcs: ["Test.java"], + sdk_version: "none", + system_modules: "none", + } + `) + + result.CheckSnapshot("mysdk", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +java_import { + name: "mysdk_exported-system-module@current", + sdk_member_name: "exported-system-module", + jars: ["java/exported-system-module.jar"], +} + +java_import { + name: "exported-system-module", + prefer: false, + jars: ["java/exported-system-module.jar"], +} + +java_import { + name: "mysdk_system-module@current", + sdk_member_name: "system-module", + visibility: ["//visibility:private"], + jars: ["java/system-module.jar"], +} + +java_import { + name: "mysdk_system-module", + prefer: false, + visibility: ["//visibility:private"], + jars: ["java/system-module.jar"], +} + +java_system_modules_import { + name: "mysdk_my-system-modules@current", + sdk_member_name: "my-system-modules", + libs: [ + "mysdk_system-module@current", + "mysdk_exported-system-module@current", + ], +} + +java_system_modules_import { + name: "my-system-modules", + prefer: false, + libs: [ + "mysdk_system-module", + "exported-system-module", + ], +} + +sdk_snapshot { + name: "mysdk@current", + java_header_libs: ["mysdk_exported-system-module@current"], + java_system_modules: ["mysdk_my-system-modules@current"], +} +`), + checkAllCopyRules(` +.intermediates/exported-system-module/android_common/turbine-combined/exported-system-module.jar -> java/exported-system-module.jar +.intermediates/system-module/android_common/turbine-combined/system-module.jar -> java/system-module.jar +`), + ) +} + +func TestHostSnapshotWithJavaSystemModules(t *testing.T) { + // b/145598135 - Generating host snapshots for anything other than linux is not supported. + SkipIfNotLinux(t) + + result := testSdkWithJava(t, ` + sdk { + name: "mysdk", + device_supported: false, + host_supported: true, + java_system_modules: ["my-system-modules"], + } + + java_system_modules { + name: "my-system-modules", + device_supported: false, + host_supported: true, + libs: ["system-module"], + } + + java_library { + name: "system-module", + device_supported: false, + host_supported: true, + srcs: ["Test.java"], + sdk_version: "none", + system_modules: "none", + } + `) + + result.CheckSnapshot("mysdk", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +java_import { + name: "mysdk_system-module@current", + sdk_member_name: "system-module", + visibility: ["//visibility:private"], + device_supported: false, + host_supported: true, + jars: ["java/system-module.jar"], +} + +java_import { + name: "mysdk_system-module", + prefer: false, + visibility: ["//visibility:private"], + device_supported: false, + host_supported: true, + jars: ["java/system-module.jar"], +} + +java_system_modules_import { + name: "mysdk_my-system-modules@current", + sdk_member_name: "my-system-modules", + device_supported: false, + host_supported: true, + libs: ["mysdk_system-module@current"], +} + +java_system_modules_import { + name: "my-system-modules", + prefer: false, + device_supported: false, + host_supported: true, + libs: ["mysdk_system-module"], +} + +sdk_snapshot { + name: "mysdk@current", + device_supported: false, + host_supported: true, + java_system_modules: ["mysdk_my-system-modules@current"], +} +`), + checkAllCopyRules(".intermediates/system-module/linux_glibc_common/javac/system-module.jar -> java/system-module.jar"), + ) +} + +func TestDeviceAndHostSnapshotWithOsSpecificMembers(t *testing.T) { + // b/145598135 - Generating host snapshots for anything other than linux is not supported. + SkipIfNotLinux(t) + + result := testSdkWithJava(t, ` + module_exports { + name: "myexports", + host_supported: true, + java_libs: ["myjavalib"], + target: { + android: { + java_header_libs: ["androidjavalib"], + }, + host: { + java_header_libs: ["hostjavalib"], + }, + }, + } + + java_library { + name: "myjavalib", + host_supported: true, + srcs: ["Test.java"], + system_modules: "none", + sdk_version: "none", + } + + java_library { + name: "androidjavalib", + srcs: ["Test.java"], + system_modules: "none", + sdk_version: "none", + } + + java_library_host { + name: "hostjavalib", + srcs: ["Test.java"], + } + `) + + result.CheckSnapshot("myexports", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +java_import { + name: "myexports_hostjavalib@current", + sdk_member_name: "hostjavalib", + device_supported: false, + host_supported: true, + jars: ["java/hostjavalib.jar"], +} + +java_import { + name: "hostjavalib", + prefer: false, + device_supported: false, + host_supported: true, + jars: ["java/hostjavalib.jar"], +} + +java_import { + name: "myexports_androidjavalib@current", + sdk_member_name: "androidjavalib", + jars: ["java/androidjavalib.jar"], +} + +java_import { + name: "androidjavalib", + prefer: false, + jars: ["java/androidjavalib.jar"], +} + +java_import { + name: "myexports_myjavalib@current", + sdk_member_name: "myjavalib", + host_supported: true, + target: { + android: { + jars: ["java/android/myjavalib.jar"], + }, + linux_glibc: { + jars: ["java/linux_glibc/myjavalib.jar"], + }, + }, +} + +java_import { + name: "myjavalib", + prefer: false, + host_supported: true, + target: { + android: { + jars: ["java/android/myjavalib.jar"], + }, + linux_glibc: { + jars: ["java/linux_glibc/myjavalib.jar"], + }, + }, +} + +module_exports_snapshot { + name: "myexports@current", + host_supported: true, + java_libs: ["myexports_myjavalib@current"], + target: { + android: { + java_header_libs: ["myexports_androidjavalib@current"], + }, + linux_glibc: { + java_header_libs: ["myexports_hostjavalib@current"], + }, + }, +} +`), + checkAllCopyRules(` +.intermediates/hostjavalib/linux_glibc_common/javac/hostjavalib.jar -> java/hostjavalib.jar +.intermediates/androidjavalib/android_common/turbine-combined/androidjavalib.jar -> java/androidjavalib.jar +.intermediates/myjavalib/android_common/javac/myjavalib.jar -> java/android/myjavalib.jar +.intermediates/myjavalib/linux_glibc_common/javac/myjavalib.jar -> java/linux_glibc/myjavalib.jar +`), + ) +} + +func TestSnapshotWithJavaSdkLibrary(t *testing.T) { + result := testSdkWithJava(t, ` + sdk { + name: "mysdk", + java_sdk_libs: ["myjavalib"], + } + + java_sdk_library { + name: "myjavalib", + apex_available: ["//apex_available:anyapex"], + srcs: ["Test.java"], + sdk_version: "current", + shared_library: false, + stubs_library_visibility: ["//other"], + stubs_source_visibility: ["//another"], + } + `) + + result.CheckSnapshot("mysdk", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +java_sdk_library_import { + name: "mysdk_myjavalib@current", + sdk_member_name: "myjavalib", + apex_available: ["//apex_available:anyapex"], + shared_library: false, + public: { + jars: ["sdk_library/public/myjavalib-stubs.jar"], + stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], + current_api: "sdk_library/public/myjavalib.txt", + removed_api: "sdk_library/public/myjavalib-removed.txt", + sdk_version: "current", + }, + system: { + jars: ["sdk_library/system/myjavalib-stubs.jar"], + stub_srcs: ["sdk_library/system/myjavalib_stub_sources"], + current_api: "sdk_library/system/myjavalib.txt", + removed_api: "sdk_library/system/myjavalib-removed.txt", + sdk_version: "system_current", + }, + test: { + jars: ["sdk_library/test/myjavalib-stubs.jar"], + stub_srcs: ["sdk_library/test/myjavalib_stub_sources"], + current_api: "sdk_library/test/myjavalib.txt", + removed_api: "sdk_library/test/myjavalib-removed.txt", + sdk_version: "test_current", + }, +} + +java_sdk_library_import { + name: "myjavalib", + prefer: false, + apex_available: ["//apex_available:anyapex"], + shared_library: false, + public: { + jars: ["sdk_library/public/myjavalib-stubs.jar"], + stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], + current_api: "sdk_library/public/myjavalib.txt", + removed_api: "sdk_library/public/myjavalib-removed.txt", + sdk_version: "current", + }, + system: { + jars: ["sdk_library/system/myjavalib-stubs.jar"], + stub_srcs: ["sdk_library/system/myjavalib_stub_sources"], + current_api: "sdk_library/system/myjavalib.txt", + removed_api: "sdk_library/system/myjavalib-removed.txt", + sdk_version: "system_current", + }, + test: { + jars: ["sdk_library/test/myjavalib-stubs.jar"], + stub_srcs: ["sdk_library/test/myjavalib_stub_sources"], + current_api: "sdk_library/test/myjavalib.txt", + removed_api: "sdk_library/test/myjavalib-removed.txt", + sdk_version: "test_current", + }, +} + +sdk_snapshot { + name: "mysdk@current", + java_sdk_libs: ["mysdk_myjavalib@current"], +} +`), + checkAllCopyRules(` +.intermediates/myjavalib.stubs/android_common/javac/myjavalib.stubs.jar -> sdk_library/public/myjavalib-stubs.jar +.intermediates/myjavalib.stubs.source/android_common/myjavalib.stubs.source_api.txt -> sdk_library/public/myjavalib.txt +.intermediates/myjavalib.stubs.source/android_common/myjavalib.stubs.source_removed.txt -> sdk_library/public/myjavalib-removed.txt +.intermediates/myjavalib.stubs.system/android_common/javac/myjavalib.stubs.system.jar -> sdk_library/system/myjavalib-stubs.jar +.intermediates/myjavalib.stubs.source.system/android_common/myjavalib.stubs.source.system_api.txt -> sdk_library/system/myjavalib.txt +.intermediates/myjavalib.stubs.source.system/android_common/myjavalib.stubs.source.system_removed.txt -> sdk_library/system/myjavalib-removed.txt +.intermediates/myjavalib.stubs.test/android_common/javac/myjavalib.stubs.test.jar -> sdk_library/test/myjavalib-stubs.jar +.intermediates/myjavalib.stubs.source.test/android_common/myjavalib.stubs.source.test_api.txt -> sdk_library/test/myjavalib.txt +.intermediates/myjavalib.stubs.source.test/android_common/myjavalib.stubs.source.test_removed.txt -> sdk_library/test/myjavalib-removed.txt +`), + checkMergeZips( + ".intermediates/mysdk/common_os/tmp/sdk_library/public/myjavalib_stub_sources.zip", + ".intermediates/mysdk/common_os/tmp/sdk_library/system/myjavalib_stub_sources.zip", + ".intermediates/mysdk/common_os/tmp/sdk_library/test/myjavalib_stub_sources.zip"), + ) +} + +func TestSnapshotWithJavaSdkLibrary_SdkVersion_None(t *testing.T) { + result := testSdkWithJava(t, ` + sdk { + name: "mysdk", + java_sdk_libs: ["myjavalib"], + } + + java_sdk_library { + name: "myjavalib", + srcs: ["Test.java"], + sdk_version: "none", + system_modules: "none", + } + `) + + result.CheckSnapshot("mysdk", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +java_sdk_library_import { + name: "mysdk_myjavalib@current", + sdk_member_name: "myjavalib", + shared_library: true, + public: { + jars: ["sdk_library/public/myjavalib-stubs.jar"], + stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], + current_api: "sdk_library/public/myjavalib.txt", + removed_api: "sdk_library/public/myjavalib-removed.txt", + sdk_version: "none", + }, +} + +java_sdk_library_import { + name: "myjavalib", + prefer: false, + shared_library: true, + public: { + jars: ["sdk_library/public/myjavalib-stubs.jar"], + stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], + current_api: "sdk_library/public/myjavalib.txt", + removed_api: "sdk_library/public/myjavalib-removed.txt", + sdk_version: "none", + }, +} + +sdk_snapshot { + name: "mysdk@current", + java_sdk_libs: ["mysdk_myjavalib@current"], +} +`), + checkAllCopyRules(` +.intermediates/myjavalib.stubs/android_common/javac/myjavalib.stubs.jar -> sdk_library/public/myjavalib-stubs.jar +.intermediates/myjavalib.stubs.source/android_common/myjavalib.stubs.source_api.txt -> sdk_library/public/myjavalib.txt +.intermediates/myjavalib.stubs.source/android_common/myjavalib.stubs.source_removed.txt -> sdk_library/public/myjavalib-removed.txt +`), + checkMergeZips( + ".intermediates/mysdk/common_os/tmp/sdk_library/public/myjavalib_stub_sources.zip", + ), + ) +} + +func TestSnapshotWithJavaSdkLibrary_SdkVersion_ForScope(t *testing.T) { + result := testSdkWithJava(t, ` + sdk { + name: "mysdk", + java_sdk_libs: ["myjavalib"], + } + + java_sdk_library { + name: "myjavalib", + srcs: ["Test.java"], + sdk_version: "module_current", + public: { + enabled: true, + sdk_version: "module_current", + }, + } + `) + + result.CheckSnapshot("mysdk", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +java_sdk_library_import { + name: "mysdk_myjavalib@current", + sdk_member_name: "myjavalib", + shared_library: true, + public: { + jars: ["sdk_library/public/myjavalib-stubs.jar"], + stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], + current_api: "sdk_library/public/myjavalib.txt", + removed_api: "sdk_library/public/myjavalib-removed.txt", + sdk_version: "module_current", + }, +} + +java_sdk_library_import { + name: "myjavalib", + prefer: false, + shared_library: true, + public: { + jars: ["sdk_library/public/myjavalib-stubs.jar"], + stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], + current_api: "sdk_library/public/myjavalib.txt", + removed_api: "sdk_library/public/myjavalib-removed.txt", + sdk_version: "module_current", + }, +} + +sdk_snapshot { + name: "mysdk@current", + java_sdk_libs: ["mysdk_myjavalib@current"], +} +`), + checkAllCopyRules(` +.intermediates/myjavalib.stubs/android_common/javac/myjavalib.stubs.jar -> sdk_library/public/myjavalib-stubs.jar +.intermediates/myjavalib.stubs.source/android_common/myjavalib.stubs.source_api.txt -> sdk_library/public/myjavalib.txt +.intermediates/myjavalib.stubs.source/android_common/myjavalib.stubs.source_removed.txt -> sdk_library/public/myjavalib-removed.txt +`), + checkMergeZips( + ".intermediates/mysdk/common_os/tmp/sdk_library/public/myjavalib_stub_sources.zip", + ), + ) +} + +func TestSnapshotWithJavaSdkLibrary_ApiScopes(t *testing.T) { + result := testSdkWithJava(t, ` + sdk { + name: "mysdk", + java_sdk_libs: ["myjavalib"], + } + + java_sdk_library { + name: "myjavalib", + apex_available: ["//apex_available:anyapex"], + srcs: ["Test.java"], + sdk_version: "current", + public: { + enabled: true, + }, + system: { + enabled: true, + }, + } + `) + + result.CheckSnapshot("mysdk", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +java_sdk_library_import { + name: "mysdk_myjavalib@current", + sdk_member_name: "myjavalib", + apex_available: ["//apex_available:anyapex"], + shared_library: true, + public: { + jars: ["sdk_library/public/myjavalib-stubs.jar"], + stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], + current_api: "sdk_library/public/myjavalib.txt", + removed_api: "sdk_library/public/myjavalib-removed.txt", + sdk_version: "current", + }, + system: { + jars: ["sdk_library/system/myjavalib-stubs.jar"], + stub_srcs: ["sdk_library/system/myjavalib_stub_sources"], + current_api: "sdk_library/system/myjavalib.txt", + removed_api: "sdk_library/system/myjavalib-removed.txt", + sdk_version: "system_current", + }, +} + +java_sdk_library_import { + name: "myjavalib", + prefer: false, + apex_available: ["//apex_available:anyapex"], + shared_library: true, + public: { + jars: ["sdk_library/public/myjavalib-stubs.jar"], + stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], + current_api: "sdk_library/public/myjavalib.txt", + removed_api: "sdk_library/public/myjavalib-removed.txt", + sdk_version: "current", + }, + system: { + jars: ["sdk_library/system/myjavalib-stubs.jar"], + stub_srcs: ["sdk_library/system/myjavalib_stub_sources"], + current_api: "sdk_library/system/myjavalib.txt", + removed_api: "sdk_library/system/myjavalib-removed.txt", + sdk_version: "system_current", + }, +} + +sdk_snapshot { + name: "mysdk@current", + java_sdk_libs: ["mysdk_myjavalib@current"], +} +`), + checkAllCopyRules(` +.intermediates/myjavalib.stubs/android_common/javac/myjavalib.stubs.jar -> sdk_library/public/myjavalib-stubs.jar +.intermediates/myjavalib.stubs.source/android_common/myjavalib.stubs.source_api.txt -> sdk_library/public/myjavalib.txt +.intermediates/myjavalib.stubs.source/android_common/myjavalib.stubs.source_removed.txt -> sdk_library/public/myjavalib-removed.txt +.intermediates/myjavalib.stubs.system/android_common/javac/myjavalib.stubs.system.jar -> sdk_library/system/myjavalib-stubs.jar +.intermediates/myjavalib.stubs.source.system/android_common/myjavalib.stubs.source.system_api.txt -> sdk_library/system/myjavalib.txt +.intermediates/myjavalib.stubs.source.system/android_common/myjavalib.stubs.source.system_removed.txt -> sdk_library/system/myjavalib-removed.txt +`), + checkMergeZips( + ".intermediates/mysdk/common_os/tmp/sdk_library/public/myjavalib_stub_sources.zip", + ".intermediates/mysdk/common_os/tmp/sdk_library/system/myjavalib_stub_sources.zip", + ), + ) +} + +func TestSnapshotWithJavaSdkLibrary_ModuleLib(t *testing.T) { + result := testSdkWithJava(t, ` + sdk { + name: "mysdk", + java_sdk_libs: ["myjavalib"], + } + + java_sdk_library { + name: "myjavalib", + apex_available: ["//apex_available:anyapex"], + srcs: ["Test.java"], + sdk_version: "current", + public: { + enabled: true, + }, + system: { + enabled: true, + }, + module_lib: { + enabled: true, + }, + } + `) + + result.CheckSnapshot("mysdk", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +java_sdk_library_import { + name: "mysdk_myjavalib@current", + sdk_member_name: "myjavalib", + apex_available: ["//apex_available:anyapex"], + shared_library: true, + public: { + jars: ["sdk_library/public/myjavalib-stubs.jar"], + stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], + current_api: "sdk_library/public/myjavalib.txt", + removed_api: "sdk_library/public/myjavalib-removed.txt", + sdk_version: "current", + }, + system: { + jars: ["sdk_library/system/myjavalib-stubs.jar"], + stub_srcs: ["sdk_library/system/myjavalib_stub_sources"], + current_api: "sdk_library/system/myjavalib.txt", + removed_api: "sdk_library/system/myjavalib-removed.txt", + sdk_version: "system_current", + }, + module_lib: { + jars: ["sdk_library/module-lib/myjavalib-stubs.jar"], + stub_srcs: ["sdk_library/module-lib/myjavalib_stub_sources"], + current_api: "sdk_library/module-lib/myjavalib.txt", + removed_api: "sdk_library/module-lib/myjavalib-removed.txt", + sdk_version: "module_current", + }, +} + +java_sdk_library_import { + name: "myjavalib", + prefer: false, + apex_available: ["//apex_available:anyapex"], + shared_library: true, + public: { + jars: ["sdk_library/public/myjavalib-stubs.jar"], + stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], + current_api: "sdk_library/public/myjavalib.txt", + removed_api: "sdk_library/public/myjavalib-removed.txt", + sdk_version: "current", + }, + system: { + jars: ["sdk_library/system/myjavalib-stubs.jar"], + stub_srcs: ["sdk_library/system/myjavalib_stub_sources"], + current_api: "sdk_library/system/myjavalib.txt", + removed_api: "sdk_library/system/myjavalib-removed.txt", + sdk_version: "system_current", + }, + module_lib: { + jars: ["sdk_library/module-lib/myjavalib-stubs.jar"], + stub_srcs: ["sdk_library/module-lib/myjavalib_stub_sources"], + current_api: "sdk_library/module-lib/myjavalib.txt", + removed_api: "sdk_library/module-lib/myjavalib-removed.txt", + sdk_version: "module_current", + }, +} + +sdk_snapshot { + name: "mysdk@current", + java_sdk_libs: ["mysdk_myjavalib@current"], +} +`), + checkAllCopyRules(` +.intermediates/myjavalib.stubs/android_common/javac/myjavalib.stubs.jar -> sdk_library/public/myjavalib-stubs.jar +.intermediates/myjavalib.stubs.source/android_common/myjavalib.stubs.source_api.txt -> sdk_library/public/myjavalib.txt +.intermediates/myjavalib.stubs.source/android_common/myjavalib.stubs.source_removed.txt -> sdk_library/public/myjavalib-removed.txt +.intermediates/myjavalib.stubs.system/android_common/javac/myjavalib.stubs.system.jar -> sdk_library/system/myjavalib-stubs.jar +.intermediates/myjavalib.stubs.source.system/android_common/myjavalib.stubs.source.system_api.txt -> sdk_library/system/myjavalib.txt +.intermediates/myjavalib.stubs.source.system/android_common/myjavalib.stubs.source.system_removed.txt -> sdk_library/system/myjavalib-removed.txt +.intermediates/myjavalib.stubs.module_lib/android_common/javac/myjavalib.stubs.module_lib.jar -> sdk_library/module-lib/myjavalib-stubs.jar +.intermediates/myjavalib.api.module_lib/android_common/myjavalib.api.module_lib_api.txt -> sdk_library/module-lib/myjavalib.txt +.intermediates/myjavalib.api.module_lib/android_common/myjavalib.api.module_lib_removed.txt -> sdk_library/module-lib/myjavalib-removed.txt +`), + checkMergeZips( + ".intermediates/mysdk/common_os/tmp/sdk_library/public/myjavalib_stub_sources.zip", + ".intermediates/mysdk/common_os/tmp/sdk_library/system/myjavalib_stub_sources.zip", + ".intermediates/mysdk/common_os/tmp/sdk_library/module-lib/myjavalib_stub_sources.zip", + ), + ) +} + +func TestSnapshotWithJavaSdkLibrary_SystemServer(t *testing.T) { + result := testSdkWithJava(t, ` + sdk { + name: "mysdk", + java_sdk_libs: ["myjavalib"], + } + + java_sdk_library { + name: "myjavalib", + apex_available: ["//apex_available:anyapex"], + srcs: ["Test.java"], + sdk_version: "current", + public: { + enabled: true, + }, + system_server: { + enabled: true, + }, + } + `) + + result.CheckSnapshot("mysdk", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +java_sdk_library_import { + name: "mysdk_myjavalib@current", + sdk_member_name: "myjavalib", + apex_available: ["//apex_available:anyapex"], + shared_library: true, + public: { + jars: ["sdk_library/public/myjavalib-stubs.jar"], + stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], + current_api: "sdk_library/public/myjavalib.txt", + removed_api: "sdk_library/public/myjavalib-removed.txt", + sdk_version: "current", + }, + system_server: { + jars: ["sdk_library/system-server/myjavalib-stubs.jar"], + stub_srcs: ["sdk_library/system-server/myjavalib_stub_sources"], + current_api: "sdk_library/system-server/myjavalib.txt", + removed_api: "sdk_library/system-server/myjavalib-removed.txt", + sdk_version: "system_server_current", + }, +} + +java_sdk_library_import { + name: "myjavalib", + prefer: false, + apex_available: ["//apex_available:anyapex"], + shared_library: true, + public: { + jars: ["sdk_library/public/myjavalib-stubs.jar"], + stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], + current_api: "sdk_library/public/myjavalib.txt", + removed_api: "sdk_library/public/myjavalib-removed.txt", + sdk_version: "current", + }, + system_server: { + jars: ["sdk_library/system-server/myjavalib-stubs.jar"], + stub_srcs: ["sdk_library/system-server/myjavalib_stub_sources"], + current_api: "sdk_library/system-server/myjavalib.txt", + removed_api: "sdk_library/system-server/myjavalib-removed.txt", + sdk_version: "system_server_current", + }, +} + +sdk_snapshot { + name: "mysdk@current", + java_sdk_libs: ["mysdk_myjavalib@current"], +} +`), + checkAllCopyRules(` +.intermediates/myjavalib.stubs/android_common/javac/myjavalib.stubs.jar -> sdk_library/public/myjavalib-stubs.jar +.intermediates/myjavalib.stubs.source/android_common/myjavalib.stubs.source_api.txt -> sdk_library/public/myjavalib.txt +.intermediates/myjavalib.stubs.source/android_common/myjavalib.stubs.source_removed.txt -> sdk_library/public/myjavalib-removed.txt +.intermediates/myjavalib.stubs.system_server/android_common/javac/myjavalib.stubs.system_server.jar -> sdk_library/system-server/myjavalib-stubs.jar +.intermediates/myjavalib.stubs.source.system_server/android_common/myjavalib.stubs.source.system_server_api.txt -> sdk_library/system-server/myjavalib.txt +.intermediates/myjavalib.stubs.source.system_server/android_common/myjavalib.stubs.source.system_server_removed.txt -> sdk_library/system-server/myjavalib-removed.txt +`), + checkMergeZips( + ".intermediates/mysdk/common_os/tmp/sdk_library/public/myjavalib_stub_sources.zip", + ".intermediates/mysdk/common_os/tmp/sdk_library/system-server/myjavalib_stub_sources.zip", + ), + ) +} + +func TestSnapshotWithJavaSdkLibrary_NamingScheme(t *testing.T) { + result := testSdkWithJava(t, ` + sdk { + name: "mysdk", + java_sdk_libs: ["myjavalib"], + } + + java_sdk_library { + name: "myjavalib", + apex_available: ["//apex_available:anyapex"], + srcs: ["Test.java"], + sdk_version: "current", + naming_scheme: "framework-modules", + public: { + enabled: true, + }, + } + `) + + result.CheckSnapshot("mysdk", "", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +java_sdk_library_import { + name: "mysdk_myjavalib@current", + sdk_member_name: "myjavalib", + apex_available: ["//apex_available:anyapex"], + naming_scheme: "framework-modules", + shared_library: true, + public: { + jars: ["sdk_library/public/myjavalib-stubs.jar"], + stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], + current_api: "sdk_library/public/myjavalib.txt", + removed_api: "sdk_library/public/myjavalib-removed.txt", + sdk_version: "current", + }, +} + +java_sdk_library_import { + name: "myjavalib", + prefer: false, + apex_available: ["//apex_available:anyapex"], + naming_scheme: "framework-modules", + shared_library: true, + public: { + jars: ["sdk_library/public/myjavalib-stubs.jar"], + stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], + current_api: "sdk_library/public/myjavalib.txt", + removed_api: "sdk_library/public/myjavalib-removed.txt", + sdk_version: "current", + }, +} + +sdk_snapshot { + name: "mysdk@current", + java_sdk_libs: ["mysdk_myjavalib@current"], +} +`), + checkAllCopyRules(` +.intermediates/myjavalib-stubs-publicapi/android_common/javac/myjavalib-stubs-publicapi.jar -> sdk_library/public/myjavalib-stubs.jar +.intermediates/myjavalib-stubs-srcs-publicapi/android_common/myjavalib-stubs-srcs-publicapi_api.txt -> sdk_library/public/myjavalib.txt +.intermediates/myjavalib-stubs-srcs-publicapi/android_common/myjavalib-stubs-srcs-publicapi_removed.txt -> sdk_library/public/myjavalib-removed.txt +`), + checkMergeZips( + ".intermediates/mysdk/common_os/tmp/sdk_library/public/myjavalib_stub_sources.zip", + ), + ) +} diff --git a/sdk/sdk.go b/sdk/sdk.go new file mode 100644 index 000000000..cb5a6053d --- /dev/null +++ b/sdk/sdk.go @@ -0,0 +1,468 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sdk + +import ( + "fmt" + "io" + "reflect" + "strconv" + + "github.com/google/blueprint" + "github.com/google/blueprint/proptools" + + "android/soong/android" + // This package doesn't depend on the apex package, but import it to make its mutators to be + // registered before mutators in this package. See RegisterPostDepsMutators for more details. + _ "android/soong/apex" +) + +func init() { + pctx.Import("android/soong/android") + pctx.Import("android/soong/java/config") + + android.RegisterModuleType("sdk", SdkModuleFactory) + android.RegisterModuleType("sdk_snapshot", SnapshotModuleFactory) + android.PreDepsMutators(RegisterPreDepsMutators) + android.PostDepsMutators(RegisterPostDepsMutators) +} + +type sdk struct { + android.ModuleBase + android.DefaultableModuleBase + + // The dynamically generated information about the registered SdkMemberType + dynamicSdkMemberTypes *dynamicSdkMemberTypes + + // The dynamically created instance of the properties struct containing the sdk member + // list properties, e.g. java_libs. + dynamicMemberTypeListProperties interface{} + + // Information about the OsType specific member variants associated with this variant. + // + // Set by OsType specific variants in the collectMembers() method and used by the + // CommonOS variant when building the snapshot. That work is all done on separate + // calls to the sdk.GenerateAndroidBuildActions method which is guaranteed to be + // called for the OsType specific variants before the CommonOS variant (because + // the latter depends on the former). + memberRefs []sdkMemberRef + + // The multilib variants that are used by this sdk variant. + multilibUsages multilibUsage + + properties sdkProperties + + snapshotFile android.OptionalPath + + // The builder, preserved for testing. + builderForTests *snapshotBuilder +} + +type sdkProperties struct { + Snapshot bool `blueprint:"mutated"` + + // True if this is a module_exports (or module_exports_snapshot) module type. + Module_exports bool `blueprint:"mutated"` +} + +// Contains information about the sdk properties that list sdk members, e.g. +// Java_header_libs. +type sdkMemberListProperty struct { + // getter for the list of member names + getter func(properties interface{}) []string + + // the type of member referenced in the list + memberType android.SdkMemberType + + // the dependency tag used for items in this list that can be used to determine the memberType + // for a resolved dependency. + dependencyTag android.SdkMemberTypeDependencyTag +} + +func (p *sdkMemberListProperty) propertyName() string { + return p.memberType.SdkPropertyName() +} + +// Cache of dynamically generated dynamicSdkMemberTypes objects. The key is the pointer +// to a slice of SdkMemberType instances held in android.SdkMemberTypes. +var dynamicSdkMemberTypesMap android.OncePer + +// A dynamically generated set of member list properties and associated structure type. +type dynamicSdkMemberTypes struct { + // The dynamically generated structure type. + // + // Contains one []string exported field for each android.SdkMemberTypes. The name of the field + // is the exported form of the value returned by SdkMemberType.SdkPropertyName(). + propertiesStructType reflect.Type + + // Information about each of the member type specific list properties. + memberListProperties []*sdkMemberListProperty +} + +func (d *dynamicSdkMemberTypes) createMemberListProperties() interface{} { + return reflect.New(d.propertiesStructType).Interface() +} + +func getDynamicSdkMemberTypes(registry *android.SdkMemberTypesRegistry) *dynamicSdkMemberTypes { + + // Get a key that uniquely identifies the registry contents. + key := registry.UniqueOnceKey() + + // Get the registered types. + registeredTypes := registry.RegisteredTypes() + + // Get the cached value, creating new instance if necessary. + return dynamicSdkMemberTypesMap.Once(key, func() interface{} { + return createDynamicSdkMemberTypes(registeredTypes) + }).(*dynamicSdkMemberTypes) +} + +// Create the dynamicSdkMemberTypes from the list of registered member types. +// +// A struct is created which contains one exported field per member type corresponding to +// the SdkMemberType.SdkPropertyName() value. +// +// A list of sdkMemberListProperty instances is created, one per member type that provides: +// * a reference to the member type. +// * a getter for the corresponding field in the properties struct. +// * a dependency tag that identifies the member type of a resolved dependency. +// +func createDynamicSdkMemberTypes(sdkMemberTypes []android.SdkMemberType) *dynamicSdkMemberTypes { + + var listProperties []*sdkMemberListProperty + var fields []reflect.StructField + + // Iterate over the member types creating StructField and sdkMemberListProperty objects. + for f, memberType := range sdkMemberTypes { + p := memberType.SdkPropertyName() + + // Create a dynamic exported field for the member type's property. + fields = append(fields, reflect.StructField{ + Name: proptools.FieldNameForProperty(p), + Type: reflect.TypeOf([]string{}), + Tag: `android:"arch_variant"`, + }) + + // Copy the field index for use in the getter func as using the loop variable directly will + // cause all funcs to use the last value. + fieldIndex := f + + // Create an sdkMemberListProperty for the member type. + memberListProperty := &sdkMemberListProperty{ + getter: func(properties interface{}) []string { + // The properties is expected to be of the following form (where + // <Module_types> is the name of an SdkMemberType.SdkPropertyName(). + // properties *struct {<Module_types> []string, ....} + // + // Although it accesses the field by index the following reflection code is equivalent to: + // *properties.<Module_types> + // + list := reflect.ValueOf(properties).Elem().Field(fieldIndex).Interface().([]string) + return list + }, + + memberType: memberType, + + dependencyTag: android.DependencyTagForSdkMemberType(memberType), + } + + listProperties = append(listProperties, memberListProperty) + } + + // Create a dynamic struct from the collated fields. + propertiesStructType := reflect.StructOf(fields) + + return &dynamicSdkMemberTypes{ + memberListProperties: listProperties, + propertiesStructType: propertiesStructType, + } +} + +// sdk defines an SDK which is a logical group of modules (e.g. native libs, headers, java libs, etc.) +// which Mainline modules like APEX can choose to build with. +func SdkModuleFactory() android.Module { + return newSdkModule(false) +} + +func newSdkModule(moduleExports bool) *sdk { + s := &sdk{} + s.properties.Module_exports = moduleExports + // Get the dynamic sdk member type data for the currently registered sdk member types. + var registry *android.SdkMemberTypesRegistry + if moduleExports { + registry = android.ModuleExportsMemberTypes + } else { + registry = android.SdkMemberTypes + } + s.dynamicSdkMemberTypes = getDynamicSdkMemberTypes(registry) + // Create an instance of the dynamically created struct that contains all the + // properties for the member type specific list properties. + s.dynamicMemberTypeListProperties = s.dynamicSdkMemberTypes.createMemberListProperties() + s.AddProperties(&s.properties, s.dynamicMemberTypeListProperties) + android.InitCommonOSAndroidMultiTargetsArchModule(s, android.HostAndDeviceSupported, android.MultilibCommon) + android.InitDefaultableModule(s) + android.AddLoadHook(s, func(ctx android.LoadHookContext) { + type props struct { + Compile_multilib *string + } + p := &props{Compile_multilib: proptools.StringPtr("both")} + ctx.AppendProperties(p) + }) + return s +} + +// sdk_snapshot is a versioned snapshot of an SDK. This is an auto-generated module. +func SnapshotModuleFactory() android.Module { + s := newSdkModule(false) + s.properties.Snapshot = true + return s +} + +func (s *sdk) memberListProperties() []*sdkMemberListProperty { + return s.dynamicSdkMemberTypes.memberListProperties +} + +func (s *sdk) getExportedMembers() map[string]struct{} { + // Collect all the exported members. + exportedMembers := make(map[string]struct{}) + + for _, memberListProperty := range s.memberListProperties() { + names := memberListProperty.getter(s.dynamicMemberTypeListProperties) + + // Every member specified explicitly in the properties is exported by the sdk. + for _, name := range names { + exportedMembers[name] = struct{}{} + } + } + + return exportedMembers +} + +func (s *sdk) snapshot() bool { + return s.properties.Snapshot +} + +func (s *sdk) GenerateAndroidBuildActions(ctx android.ModuleContext) { + if s.snapshot() { + // We don't need to create a snapshot out of sdk_snapshot. + // That doesn't make sense. We need a snapshot to create sdk_snapshot. + return + } + + // This method is guaranteed to be called on OsType specific variants before it is called + // on their corresponding CommonOS variant. + if !s.IsCommonOSVariant() { + // Update the OsType specific sdk variant with information about its members. + s.collectMembers(ctx) + } else { + // Get the OsType specific variants on which the CommonOS depends. + osSpecificVariants := android.GetOsSpecificVariantsOfCommonOSVariant(ctx) + var sdkVariants []*sdk + for _, m := range osSpecificVariants { + if sdkVariant, ok := m.(*sdk); ok { + sdkVariants = append(sdkVariants, sdkVariant) + } + } + + // Generate the snapshot from the member info. + p := s.buildSnapshot(ctx, sdkVariants) + s.snapshotFile = android.OptionalPathForPath(p) + ctx.InstallFile(android.PathForMainlineSdksInstall(ctx), s.Name()+"-current.zip", p) + } +} + +func (s *sdk) AndroidMkEntries() []android.AndroidMkEntries { + if !s.snapshotFile.Valid() { + return []android.AndroidMkEntries{} + } + + return []android.AndroidMkEntries{android.AndroidMkEntries{ + Class: "FAKE", + OutputFile: s.snapshotFile, + DistFile: s.snapshotFile, + Include: "$(BUILD_PHONY_PACKAGE)", + ExtraFooters: []android.AndroidMkExtraFootersFunc{ + func(w io.Writer, name, prefix, moduleDir string, entries *android.AndroidMkEntries) { + // Allow the sdk to be built by simply passing its name on the command line. + fmt.Fprintln(w, ".PHONY:", s.Name()) + fmt.Fprintln(w, s.Name()+":", s.snapshotFile.String()) + }, + }, + }} +} + +// RegisterPreDepsMutators registers pre-deps mutators to support modules implementing SdkAware +// interface and the sdk module type. This function has been made public to be called by tests +// outside of the sdk package +func RegisterPreDepsMutators(ctx android.RegisterMutatorsContext) { + ctx.BottomUp("SdkMember", memberMutator).Parallel() + ctx.TopDown("SdkMember_deps", memberDepsMutator).Parallel() + ctx.BottomUp("SdkMemberInterVersion", memberInterVersionMutator).Parallel() +} + +// RegisterPostDepsMutators registers post-deps mutators to support modules implementing SdkAware +// interface and the sdk module type. This function has been made public to be called by tests +// outside of the sdk package +func RegisterPostDepsMutators(ctx android.RegisterMutatorsContext) { + // These must run AFTER apexMutator. Note that the apex package is imported even though there is + // no direct dependency to the package here. sdkDepsMutator sets the SDK requirements from an + // APEX to its dependents. Since different versions of the same SDK can be used by different + // APEXes, the apex and its dependents (which includes the dependencies to the sdk members) + // should have been mutated for the apex before the SDK requirements are set. + ctx.TopDown("SdkDepsMutator", sdkDepsMutator).Parallel() + ctx.BottomUp("SdkDepsReplaceMutator", sdkDepsReplaceMutator).Parallel() + ctx.TopDown("SdkRequirementCheck", sdkRequirementsMutator).Parallel() +} + +type dependencyTag struct { + blueprint.BaseDependencyTag +} + +// For dependencies from an in-development version of an SDK member to frozen versions of the same member +// e.g. libfoo -> libfoo.mysdk.11 and libfoo.mysdk.12 +type sdkMemberVersionedDepTag struct { + dependencyTag + member string + version string +} + +// Mark this tag so dependencies that use it are excluded from visibility enforcement. +func (t sdkMemberVersionedDepTag) ExcludeFromVisibilityEnforcement() {} + +// Step 1: create dependencies from an SDK module to its members. +func memberMutator(mctx android.BottomUpMutatorContext) { + if s, ok := mctx.Module().(*sdk); ok { + // Add dependencies from enabled and non CommonOS variants to the sdk member variants. + if s.Enabled() && !s.IsCommonOSVariant() { + for _, memberListProperty := range s.memberListProperties() { + names := memberListProperty.getter(s.dynamicMemberTypeListProperties) + if len(names) > 0 { + tag := memberListProperty.dependencyTag + memberListProperty.memberType.AddDependencies(mctx, tag, names) + } + } + } + } +} + +// Step 2: record that dependencies of SDK modules are members of the SDK modules +func memberDepsMutator(mctx android.TopDownMutatorContext) { + if s, ok := mctx.Module().(*sdk); ok { + mySdkRef := android.ParseSdkRef(mctx, mctx.ModuleName(), "name") + if s.snapshot() && mySdkRef.Unversioned() { + mctx.PropertyErrorf("name", "sdk_snapshot should be named as <name>@<version>. "+ + "Did you manually modify Android.bp?") + } + if !s.snapshot() && !mySdkRef.Unversioned() { + mctx.PropertyErrorf("name", "sdk shouldn't be named as <name>@<version>.") + } + if mySdkRef.Version != "" && mySdkRef.Version != "current" { + if _, err := strconv.Atoi(mySdkRef.Version); err != nil { + mctx.PropertyErrorf("name", "version %q is neither a number nor \"current\"", mySdkRef.Version) + } + } + + mctx.VisitDirectDeps(func(child android.Module) { + if member, ok := child.(android.SdkAware); ok { + member.MakeMemberOf(mySdkRef) + } + }) + } +} + +// Step 3: create dependencies from the unversioned SDK member to snapshot versions +// of the same member. By having these dependencies, they are mutated for multiple Mainline modules +// (apex and apk), each of which might want different sdks to be built with. For example, if both +// apex A and B are referencing libfoo which is a member of sdk 'mysdk', the two APEXes can be +// built with libfoo.mysdk.11 and libfoo.mysdk.12, respectively depending on which sdk they are +// using. +func memberInterVersionMutator(mctx android.BottomUpMutatorContext) { + if m, ok := mctx.Module().(android.SdkAware); ok && m.IsInAnySdk() { + if !m.ContainingSdk().Unversioned() { + memberName := m.MemberName() + tag := sdkMemberVersionedDepTag{member: memberName, version: m.ContainingSdk().Version} + mctx.AddReverseDependency(mctx.Module(), tag, memberName) + } + } +} + +// Step 4: transitively ripple down the SDK requirements from the root modules like APEX to its +// descendants +func sdkDepsMutator(mctx android.TopDownMutatorContext) { + if m, ok := mctx.Module().(android.SdkAware); ok { + // Module types for Mainline modules (e.g. APEX) are expected to implement RequiredSdks() + // by reading its own properties like `uses_sdks`. + requiredSdks := m.RequiredSdks() + if len(requiredSdks) > 0 { + mctx.VisitDirectDeps(func(m android.Module) { + if dep, ok := m.(android.SdkAware); ok { + dep.BuildWithSdks(requiredSdks) + } + }) + } + } +} + +// Step 5: if libfoo.mysdk.11 is in the context where version 11 of mysdk is requested, the +// versioned module is used instead of the un-versioned (in-development) module libfoo +func sdkDepsReplaceMutator(mctx android.BottomUpMutatorContext) { + if m, ok := mctx.Module().(android.SdkAware); ok && m.IsInAnySdk() { + if sdk := m.ContainingSdk(); !sdk.Unversioned() { + if m.RequiredSdks().Contains(sdk) { + // Note that this replacement is done only for the modules that have the same + // variations as the current module. Since current module is already mutated for + // apex references in other APEXes are not affected by this replacement. + memberName := m.MemberName() + mctx.ReplaceDependencies(memberName) + } + } + } +} + +// Step 6: ensure that the dependencies outside of the APEX are all from the required SDKs +func sdkRequirementsMutator(mctx android.TopDownMutatorContext) { + if m, ok := mctx.Module().(interface { + android.DepIsInSameApex + android.RequiredSdks + }); ok { + requiredSdks := m.RequiredSdks() + if len(requiredSdks) == 0 { + return + } + mctx.VisitDirectDeps(func(dep android.Module) { + tag := mctx.OtherModuleDependencyTag(dep) + if tag == android.DefaultsDepTag { + // dependency to defaults is always okay + return + } + + // Ignore the dependency from the unversioned member to any versioned members as an + // apex that depends on the unversioned member will not also be depending on a versioned + // member. + if _, ok := tag.(sdkMemberVersionedDepTag); ok { + return + } + + // If the dep is outside of the APEX, but is not in any of the + // required SDKs, we know that the dep is a violation. + if sa, ok := dep.(android.SdkAware); ok { + if !m.DepIsInSameApex(mctx, dep) && !requiredSdks.Contains(sa.ContainingSdk()) { + mctx.ModuleErrorf("depends on %q (in SDK %q) that isn't part of the required SDKs: %v", + sa.Name(), sa.ContainingSdk(), requiredSdks) + } + } + }) + } +} diff --git a/sdk/sdk_test.go b/sdk/sdk_test.go new file mode 100644 index 000000000..56be7417b --- /dev/null +++ b/sdk/sdk_test.go @@ -0,0 +1,410 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sdk + +import ( + "testing" + + "github.com/google/blueprint/proptools" +) + +// Needed in an _test.go file in this package to ensure tests run correctly, particularly in IDE. +func TestMain(m *testing.M) { + runTestWithBuildDir(m) +} + +func TestDepNotInRequiredSdks(t *testing.T) { + testSdkError(t, `module "myjavalib".*depends on "otherlib".*that isn't part of the required SDKs:.*`, ` + sdk { + name: "mysdk", + java_header_libs: ["sdkmember"], + } + + sdk_snapshot { + name: "mysdk@1", + java_header_libs: ["sdkmember_mysdk_1"], + } + + java_import { + name: "sdkmember", + prefer: false, + host_supported: true, + } + + java_import { + name: "sdkmember_mysdk_1", + sdk_member_name: "sdkmember", + host_supported: true, + } + + java_library { + name: "myjavalib", + srcs: ["Test.java"], + libs: [ + "sdkmember", + "otherlib", + ], + system_modules: "none", + sdk_version: "none", + compile_dex: true, + host_supported: true, + apex_available: ["myapex"], + } + + // this lib is no in mysdk + java_library { + name: "otherlib", + srcs: ["Test.java"], + system_modules: "none", + sdk_version: "none", + compile_dex: true, + host_supported: true, + } + + apex { + name: "myapex", + java_libs: ["myjavalib"], + uses_sdks: ["mysdk@1"], + key: "myapex.key", + certificate: ":myapex.cert", + } + `) +} + +// Ensure that prebuilt modules have the same effective visibility as the source +// modules. +func TestSnapshotVisibility(t *testing.T) { + packageBp := ` + package { + default_visibility: ["//other/foo"], + } + + sdk { + name: "mysdk", + visibility: [ + "//other/foo", + // This short form will be replaced with //package:__subpackages__ in the + // generated sdk_snapshot. + ":__subpackages__", + ], + java_header_libs: [ + "myjavalib", + "mypublicjavalib", + "mydefaultedjavalib", + "myprivatejavalib", + ], + } + + java_library { + name: "myjavalib", + // Uses package default visibility + srcs: ["Test.java"], + system_modules: "none", + sdk_version: "none", + } + + java_defaults { + name: "java-defaults", + visibility: ["//other/bar"], + } + + java_library { + name: "mypublicjavalib", + defaults: ["java-defaults"], + visibility: ["//visibility:public"], + srcs: ["Test.java"], + system_modules: "none", + sdk_version: "none", + } + + java_defaults { + name: "myjavadefaults", + visibility: ["//other/bar"], + } + + java_library { + name: "mydefaultedjavalib", + defaults: ["myjavadefaults"], + srcs: ["Test.java"], + system_modules: "none", + sdk_version: "none", + } + + java_library { + name: "myprivatejavalib", + srcs: ["Test.java"], + visibility: ["//visibility:private"], + system_modules: "none", + sdk_version: "none", + } + ` + + result := testSdkWithFs(t, ``, + map[string][]byte{ + "package/Test.java": nil, + "package/Android.bp": []byte(packageBp), + }) + + result.CheckSnapshot("mysdk", "package", + checkAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +java_import { + name: "mysdk_myjavalib@current", + sdk_member_name: "myjavalib", + visibility: [ + "//other/foo", + "//package", + ], + jars: ["java/myjavalib.jar"], +} + +java_import { + name: "myjavalib", + prefer: false, + visibility: [ + "//other/foo", + "//package", + ], + jars: ["java/myjavalib.jar"], +} + +java_import { + name: "mysdk_mypublicjavalib@current", + sdk_member_name: "mypublicjavalib", + visibility: ["//visibility:public"], + jars: ["java/mypublicjavalib.jar"], +} + +java_import { + name: "mypublicjavalib", + prefer: false, + visibility: ["//visibility:public"], + jars: ["java/mypublicjavalib.jar"], +} + +java_import { + name: "mysdk_mydefaultedjavalib@current", + sdk_member_name: "mydefaultedjavalib", + visibility: [ + "//other/bar", + "//package", + ], + jars: ["java/mydefaultedjavalib.jar"], +} + +java_import { + name: "mydefaultedjavalib", + prefer: false, + visibility: [ + "//other/bar", + "//package", + ], + jars: ["java/mydefaultedjavalib.jar"], +} + +java_import { + name: "mysdk_myprivatejavalib@current", + sdk_member_name: "myprivatejavalib", + visibility: ["//package"], + jars: ["java/myprivatejavalib.jar"], +} + +java_import { + name: "myprivatejavalib", + prefer: false, + visibility: ["//package"], + jars: ["java/myprivatejavalib.jar"], +} + +sdk_snapshot { + name: "mysdk@current", + visibility: [ + "//other/foo", + "//package:__subpackages__", + ], + java_header_libs: [ + "mysdk_myjavalib@current", + "mysdk_mypublicjavalib@current", + "mysdk_mydefaultedjavalib@current", + "mysdk_myprivatejavalib@current", + ], +} +`)) +} + +func TestSDkInstall(t *testing.T) { + sdk := ` + sdk { + name: "mysdk", + } + ` + result := testSdkWithFs(t, ``, + map[string][]byte{ + "Android.bp": []byte(sdk), + }) + + result.CheckSnapshot("mysdk", "", + checkAllOtherCopyRules(`.intermediates/mysdk/common_os/mysdk-current.zip -> mysdk-current.zip`), + ) +} + +type EmbeddedPropertiesStruct struct { + S_Embedded_Common string `android:"arch_variant"` + S_Embedded_Different string `android:"arch_variant"` +} + +type testPropertiesStruct struct { + name string + private string + Public_Kept string `sdk:"keep"` + S_Common string + S_Different string `android:"arch_variant"` + A_Common []string + A_Different []string `android:"arch_variant"` + F_Common *bool + F_Different *bool `android:"arch_variant"` + EmbeddedPropertiesStruct +} + +func (p *testPropertiesStruct) optimizableProperties() interface{} { + return p +} + +func (p *testPropertiesStruct) String() string { + return p.name +} + +var _ propertiesContainer = (*testPropertiesStruct)(nil) + +func TestCommonValueOptimization(t *testing.T) { + common := &testPropertiesStruct{name: "common"} + structs := []propertiesContainer{ + &testPropertiesStruct{ + name: "struct-0", + private: "common", + Public_Kept: "common", + S_Common: "common", + S_Different: "upper", + A_Common: []string{"first", "second"}, + A_Different: []string{"alpha", "beta"}, + F_Common: proptools.BoolPtr(false), + F_Different: proptools.BoolPtr(false), + EmbeddedPropertiesStruct: EmbeddedPropertiesStruct{ + S_Embedded_Common: "embedded_common", + S_Embedded_Different: "embedded_upper", + }, + }, + &testPropertiesStruct{ + name: "struct-1", + private: "common", + Public_Kept: "common", + S_Common: "common", + S_Different: "lower", + A_Common: []string{"first", "second"}, + A_Different: []string{"alpha", "delta"}, + F_Common: proptools.BoolPtr(false), + F_Different: proptools.BoolPtr(true), + EmbeddedPropertiesStruct: EmbeddedPropertiesStruct{ + S_Embedded_Common: "embedded_common", + S_Embedded_Different: "embedded_lower", + }, + }, + } + + extractor := newCommonValueExtractor(common) + + h := TestHelper{t} + + err := extractor.extractCommonProperties(common, structs) + h.AssertDeepEquals("unexpected error", nil, err) + + h.AssertDeepEquals("common properties not correct", + &testPropertiesStruct{ + name: "common", + private: "", + Public_Kept: "", + S_Common: "common", + S_Different: "", + A_Common: []string{"first", "second"}, + A_Different: []string(nil), + F_Common: proptools.BoolPtr(false), + F_Different: nil, + EmbeddedPropertiesStruct: EmbeddedPropertiesStruct{ + S_Embedded_Common: "embedded_common", + S_Embedded_Different: "", + }, + }, + common) + + h.AssertDeepEquals("updated properties[0] not correct", + &testPropertiesStruct{ + name: "struct-0", + private: "common", + Public_Kept: "common", + S_Common: "", + S_Different: "upper", + A_Common: nil, + A_Different: []string{"alpha", "beta"}, + F_Common: nil, + F_Different: proptools.BoolPtr(false), + EmbeddedPropertiesStruct: EmbeddedPropertiesStruct{ + S_Embedded_Common: "", + S_Embedded_Different: "embedded_upper", + }, + }, + structs[0]) + + h.AssertDeepEquals("updated properties[1] not correct", + &testPropertiesStruct{ + name: "struct-1", + private: "common", + Public_Kept: "common", + S_Common: "", + S_Different: "lower", + A_Common: nil, + A_Different: []string{"alpha", "delta"}, + F_Common: nil, + F_Different: proptools.BoolPtr(true), + EmbeddedPropertiesStruct: EmbeddedPropertiesStruct{ + S_Embedded_Common: "", + S_Embedded_Different: "embedded_lower", + }, + }, + structs[1]) +} + +func TestCommonValueOptimization_InvalidArchSpecificVariants(t *testing.T) { + common := &testPropertiesStruct{name: "common"} + structs := []propertiesContainer{ + &testPropertiesStruct{ + name: "struct-0", + S_Common: "should-be-but-is-not-common0", + }, + &testPropertiesStruct{ + name: "struct-1", + S_Common: "should-be-but-is-not-common1", + }, + } + + extractor := newCommonValueExtractor(common) + + h := TestHelper{t} + + err := extractor.extractCommonProperties(common, structs) + h.AssertErrorMessageEquals("unexpected error", `field "S_Common" is not tagged as "arch_variant" but has arch specific properties: + "struct-0" has value "should-be-but-is-not-common0" + "struct-1" has value "should-be-but-is-not-common1"`, err) +} diff --git a/sdk/testing.go b/sdk/testing.go new file mode 100644 index 000000000..436175419 --- /dev/null +++ b/sdk/testing.go @@ -0,0 +1,429 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sdk + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "android/soong/android" + "android/soong/apex" + "android/soong/cc" + "android/soong/java" +) + +func testSdkContext(bp string, fs map[string][]byte) (*android.TestContext, android.Config) { + bp = bp + ` + apex_key { + name: "myapex.key", + public_key: "myapex.avbpubkey", + private_key: "myapex.pem", + } + + android_app_certificate { + name: "myapex.cert", + certificate: "myapex", + } + ` + cc.GatherRequiredDepsForTest(android.Android, android.Windows) + + mockFS := map[string][]byte{ + "build/make/target/product/security": nil, + "apex_manifest.json": nil, + "system/sepolicy/apex/myapex-file_contexts": nil, + "system/sepolicy/apex/myapex2-file_contexts": nil, + "myapex.avbpubkey": nil, + "myapex.pem": nil, + "myapex.x509.pem": nil, + "myapex.pk8": nil, + } + + cc.GatherRequiredFilesForTest(mockFS) + + for k, v := range fs { + mockFS[k] = v + } + + config := android.TestArchConfig(buildDir, nil, bp, mockFS) + + // Add windows as a default disable OS to test behavior when some OS variants + // are disabled. + config.Targets[android.Windows] = []android.Target{ + {android.Windows, android.Arch{ArchType: android.X86_64}, android.NativeBridgeDisabled, "", ""}, + } + + ctx := android.NewTestArchContext() + + // Enable androidmk support. + // * Register the singleton + // * Configure that we are inside make + // * Add CommonOS to ensure that androidmk processing works. + android.RegisterAndroidMkBuildComponents(ctx) + android.SetInMakeForTests(config) + config.Targets[android.CommonOS] = []android.Target{ + {android.CommonOS, android.Arch{ArchType: android.Common}, android.NativeBridgeDisabled, "", ""}, + } + + // from android package + android.RegisterPackageBuildComponents(ctx) + ctx.PreArchMutators(android.RegisterVisibilityRuleChecker) + ctx.PreArchMutators(android.RegisterDefaultsPreArchMutators) + ctx.PreArchMutators(android.RegisterVisibilityRuleGatherer) + ctx.PostDepsMutators(android.RegisterVisibilityRuleEnforcer) + + // from java package + java.RegisterJavaBuildComponents(ctx) + java.RegisterAppBuildComponents(ctx) + java.RegisterSdkLibraryBuildComponents(ctx) + java.RegisterStubsBuildComponents(ctx) + java.RegisterSystemModulesBuildComponents(ctx) + + // from cc package + cc.RegisterRequiredBuildComponentsForTest(ctx) + + // from apex package + ctx.RegisterModuleType("apex", apex.BundleFactory) + ctx.RegisterModuleType("apex_key", apex.ApexKeyFactory) + ctx.PostDepsMutators(apex.RegisterPostDepsMutators) + + // from this package + ctx.RegisterModuleType("sdk", SdkModuleFactory) + ctx.RegisterModuleType("sdk_snapshot", SnapshotModuleFactory) + ctx.RegisterModuleType("module_exports", ModuleExportsFactory) + ctx.RegisterModuleType("module_exports_snapshot", ModuleExportsSnapshotsFactory) + ctx.PreDepsMutators(RegisterPreDepsMutators) + ctx.PostDepsMutators(RegisterPostDepsMutators) + + ctx.Register(config) + + return ctx, config +} + +func testSdkWithFs(t *testing.T, bp string, fs map[string][]byte) *testSdkResult { + t.Helper() + ctx, config := testSdkContext(bp, fs) + _, errs := ctx.ParseBlueprintsFiles(".") + android.FailIfErrored(t, errs) + _, errs = ctx.PrepareBuildActions(config) + android.FailIfErrored(t, errs) + return &testSdkResult{ + TestHelper: TestHelper{t: t}, + ctx: ctx, + config: config, + } +} + +func testSdkError(t *testing.T, pattern, bp string) { + t.Helper() + ctx, config := testSdkContext(bp, nil) + _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) + if len(errs) > 0 { + android.FailIfNoMatchingErrors(t, pattern, errs) + return + } + _, errs = ctx.PrepareBuildActions(config) + if len(errs) > 0 { + android.FailIfNoMatchingErrors(t, pattern, errs) + return + } + + t.Fatalf("missing expected error %q (0 errors are returned)", pattern) +} + +func ensureListContains(t *testing.T, result []string, expected string) { + t.Helper() + if !android.InList(expected, result) { + t.Errorf("%q is not found in %v", expected, result) + } +} + +func pathsToStrings(paths android.Paths) []string { + var ret []string + for _, p := range paths { + ret = append(ret, p.String()) + } + return ret +} + +// Provides general test support. +type TestHelper struct { + t *testing.T +} + +func (h *TestHelper) AssertStringEquals(message string, expected string, actual string) { + h.t.Helper() + if actual != expected { + h.t.Errorf("%s: expected %s, actual %s", message, expected, actual) + } +} + +func (h *TestHelper) AssertErrorMessageEquals(message string, expected string, actual error) { + h.t.Helper() + if actual == nil { + h.t.Errorf("Expected error but was nil") + } else if actual.Error() != expected { + h.t.Errorf("%s: expected %s, actual %s", message, expected, actual.Error()) + } +} + +func (h *TestHelper) AssertTrimmedStringEquals(message string, expected string, actual string) { + h.t.Helper() + h.AssertStringEquals(message, strings.TrimSpace(expected), strings.TrimSpace(actual)) +} + +func (h *TestHelper) AssertDeepEquals(message string, expected interface{}, actual interface{}) { + h.t.Helper() + if !reflect.DeepEqual(actual, expected) { + h.t.Errorf("%s: expected %#v, actual %#v", message, expected, actual) + } +} + +// Encapsulates result of processing an SDK definition. Provides support for +// checking the state of the build structures. +type testSdkResult struct { + TestHelper + ctx *android.TestContext + config android.Config +} + +// Analyse the sdk build rules to extract information about what it is doing. + +// e.g. find the src/dest pairs from each cp command, the various zip files +// generated, etc. +func (r *testSdkResult) getSdkSnapshotBuildInfo(sdk *sdk) *snapshotBuildInfo { + androidBpContents := sdk.GetAndroidBpContentsForTests() + + info := &snapshotBuildInfo{ + r: r, + androidBpContents: androidBpContents, + } + + buildParams := sdk.BuildParamsForTests() + copyRules := &strings.Builder{} + otherCopyRules := &strings.Builder{} + snapshotDirPrefix := sdk.builderForTests.snapshotDir.String() + "/" + for _, bp := range buildParams { + switch bp.Rule.String() { + case android.Cp.String(): + output := bp.Output + // Get destination relative to the snapshot root + dest := output.Rel() + src := android.NormalizePathForTesting(bp.Input) + // We differentiate between copy rules for the snapshot, and copy rules for the install file. + if strings.HasPrefix(output.String(), snapshotDirPrefix) { + // Get source relative to build directory. + _, _ = fmt.Fprintf(copyRules, "%s -> %s\n", src, dest) + info.snapshotContents = append(info.snapshotContents, dest) + } else { + _, _ = fmt.Fprintf(otherCopyRules, "%s -> %s\n", src, dest) + } + + case repackageZip.String(): + // Add the destdir to the snapshot contents as that is effectively where + // the content of the repackaged zip is copied. + dest := bp.Args["destdir"] + info.snapshotContents = append(info.snapshotContents, dest) + + case zipFiles.String(): + // This could be an intermediate zip file and not the actual output zip. + // In that case this will be overridden when the rule to merge the zips + // is processed. + info.outputZip = android.NormalizePathForTesting(bp.Output) + + case mergeZips.String(): + // Copy the current outputZip to the intermediateZip. + info.intermediateZip = info.outputZip + mergeInput := android.NormalizePathForTesting(bp.Input) + if info.intermediateZip != mergeInput { + r.t.Errorf("Expected intermediate zip %s to be an input to merge zips but found %s instead", + info.intermediateZip, mergeInput) + } + + // Override output zip (which was actually the intermediate zip file) with the actual + // output zip. + info.outputZip = android.NormalizePathForTesting(bp.Output) + + // Save the zips to be merged into the intermediate zip. + info.mergeZips = android.NormalizePathsForTesting(bp.Inputs) + } + } + + info.copyRules = copyRules.String() + info.otherCopyRules = otherCopyRules.String() + + return info +} + +func (r *testSdkResult) Module(name string, variant string) android.Module { + return r.ctx.ModuleForTests(name, variant).Module() +} + +func (r *testSdkResult) ModuleForTests(name string, variant string) android.TestingModule { + return r.ctx.ModuleForTests(name, variant) +} + +// Check the snapshot build rules. +// +// Takes a list of functions which check different facets of the snapshot build rules. +// Allows each test to customize what is checked without duplicating lots of code +// or proliferating check methods of different flavors. +func (r *testSdkResult) CheckSnapshot(name string, dir string, checkers ...snapshotBuildInfoChecker) { + r.t.Helper() + + // The sdk CommonOS variant is always responsible for generating the snapshot. + variant := android.CommonOS.Name + + sdk := r.Module(name, variant).(*sdk) + + snapshotBuildInfo := r.getSdkSnapshotBuildInfo(sdk) + + // Check state of the snapshot build. + for _, checker := range checkers { + checker(snapshotBuildInfo) + } + + // Make sure that the generated zip file is in the correct place. + actual := snapshotBuildInfo.outputZip + if dir != "" { + dir = filepath.Clean(dir) + "/" + } + r.AssertStringEquals("Snapshot zip file in wrong place", + fmt.Sprintf(".intermediates/%s%s/%s/%s-current.zip", dir, name, variant, name), actual) + + // Populate a mock filesystem with the files that would have been copied by + // the rules. + fs := make(map[string][]byte) + for _, dest := range snapshotBuildInfo.snapshotContents { + fs[dest] = nil + } + + // Process the generated bp file to make sure it is valid. + testSdkWithFs(r.t, snapshotBuildInfo.androidBpContents, fs) +} + +type snapshotBuildInfoChecker func(info *snapshotBuildInfo) + +// Check that the snapshot's generated Android.bp is correct. +// +// Both the expected and actual string are both trimmed before comparing. +func checkAndroidBpContents(expected string) snapshotBuildInfoChecker { + return func(info *snapshotBuildInfo) { + info.r.t.Helper() + info.r.AssertTrimmedStringEquals("Android.bp contents do not match", expected, info.androidBpContents) + } +} + +// Check that the snapshot's copy rules are correct. +// +// The copy rules are formatted as <src> -> <dest>, one per line and then compared +// to the supplied expected string. Both the expected and actual string are trimmed +// before comparing. +func checkAllCopyRules(expected string) snapshotBuildInfoChecker { + return func(info *snapshotBuildInfo) { + info.r.t.Helper() + info.r.AssertTrimmedStringEquals("Incorrect copy rules", expected, info.copyRules) + } +} + +func checkAllOtherCopyRules(expected string) snapshotBuildInfoChecker { + return func(info *snapshotBuildInfo) { + info.r.t.Helper() + info.r.AssertTrimmedStringEquals("Incorrect copy rules", expected, info.otherCopyRules) + } +} + +// Check that the specified paths match the list of zips to merge with the intermediate zip. +func checkMergeZips(expected ...string) snapshotBuildInfoChecker { + return func(info *snapshotBuildInfo) { + info.r.t.Helper() + if info.intermediateZip == "" { + info.r.t.Errorf("No intermediate zip file was created") + } + + info.r.AssertDeepEquals("mismatching merge zip files", expected, info.mergeZips) + } +} + +// Encapsulates information about the snapshot build structure in order to insulate tests from +// knowing too much about internal structures. +// +// All source/input paths are relative either the build directory. All dest/output paths are +// relative to the snapshot root directory. +type snapshotBuildInfo struct { + r *testSdkResult + + // The contents of the generated Android.bp file + androidBpContents string + + // The paths, relative to the snapshot root, of all files and directories copied into the + // snapshot. + snapshotContents []string + + // A formatted representation of the src/dest pairs for a snapshot, one pair per line, + // of the format src -> dest + copyRules string + + // A formatted representation of the src/dest pairs for files not in a snapshot, one pair + // per line, of the format src -> dest + otherCopyRules string + + // The path to the intermediate zip, which is a zip created from the source files copied + // into the snapshot directory and which will be merged with other zips to form the final output. + // Is am empty string if there is no intermediate zip because there are no zips to merge in. + intermediateZip string + + // The paths to the zips to merge into the output zip, does not include the intermediate + // zip. + mergeZips []string + + // The final output zip. + outputZip string +} + +var buildDir string + +func setUp() { + var err error + buildDir, err = ioutil.TempDir("", "soong_sdk_test") + if err != nil { + panic(err) + } +} + +func tearDown() { + _ = os.RemoveAll(buildDir) +} + +func runTestWithBuildDir(m *testing.M) { + run := func() int { + setUp() + defer tearDown() + + return m.Run() + } + + os.Exit(run()) +} + +func SkipIfNotLinux(t *testing.T) { + t.Helper() + if android.BuildOs != android.Linux { + t.Skipf("Skipping as sdk snapshot generation is only supported on %s not %s", android.Linux, android.BuildOs) + } +} diff --git a/sdk/update.go b/sdk/update.go new file mode 100644 index 000000000..59a764001 --- /dev/null +++ b/sdk/update.go @@ -0,0 +1,1494 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sdk + +import ( + "fmt" + "reflect" + "sort" + "strings" + + "android/soong/apex" + "android/soong/cc" + + "github.com/google/blueprint" + "github.com/google/blueprint/proptools" + + "android/soong/android" +) + +var pctx = android.NewPackageContext("android/soong/sdk") + +var ( + repackageZip = pctx.AndroidStaticRule("SnapshotRepackageZip", + blueprint.RuleParams{ + Command: `${config.Zip2ZipCmd} -i $in -o $out -x META-INF/**/* "**/*:$destdir"`, + CommandDeps: []string{ + "${config.Zip2ZipCmd}", + }, + }, + "destdir") + + zipFiles = pctx.AndroidStaticRule("SnapshotZipFiles", + blueprint.RuleParams{ + Command: `${config.SoongZipCmd} -C $basedir -l $out.rsp -o $out`, + CommandDeps: []string{ + "${config.SoongZipCmd}", + }, + Rspfile: "$out.rsp", + RspfileContent: "$in", + }, + "basedir") + + mergeZips = pctx.AndroidStaticRule("SnapshotMergeZips", + blueprint.RuleParams{ + Command: `${config.MergeZipsCmd} $out $in`, + CommandDeps: []string{ + "${config.MergeZipsCmd}", + }, + }) +) + +type generatedContents struct { + content strings.Builder + indentLevel int +} + +// generatedFile abstracts operations for writing contents into a file and emit a build rule +// for the file. +type generatedFile struct { + generatedContents + path android.OutputPath +} + +func newGeneratedFile(ctx android.ModuleContext, path ...string) *generatedFile { + return &generatedFile{ + path: android.PathForModuleOut(ctx, path...).OutputPath, + } +} + +func (gc *generatedContents) Indent() { + gc.indentLevel++ +} + +func (gc *generatedContents) Dedent() { + gc.indentLevel-- +} + +func (gc *generatedContents) Printfln(format string, args ...interface{}) { + fmt.Fprintf(&(gc.content), strings.Repeat(" ", gc.indentLevel)+format+"\n", args...) +} + +func (gf *generatedFile) build(pctx android.PackageContext, ctx android.BuilderContext, implicits android.Paths) { + rb := android.NewRuleBuilder() + + content := gf.content.String() + + // ninja consumes newline characters in rspfile_content. Prevent it by + // escaping the backslash in the newline character. The extra backslash + // is removed when the rspfile is written to the actual script file + content = strings.ReplaceAll(content, "\n", "\\n") + + rb.Command(). + Implicits(implicits). + Text("echo").Text(proptools.ShellEscape(content)). + // convert \\n to \n + Text("| sed 's/\\\\n/\\n/g' >").Output(gf.path) + rb.Command(). + Text("chmod a+x").Output(gf.path) + rb.Build(pctx, ctx, gf.path.Base(), "Build "+gf.path.Base()) +} + +// Collect all the members. +// +// Returns a list containing type (extracted from the dependency tag) and the variant +// plus the multilib usages. +func (s *sdk) collectMembers(ctx android.ModuleContext) { + s.multilibUsages = multilibNone + ctx.WalkDeps(func(child android.Module, parent android.Module) bool { + tag := ctx.OtherModuleDependencyTag(child) + if memberTag, ok := tag.(android.SdkMemberTypeDependencyTag); ok { + memberType := memberTag.SdkMemberType() + + // Make sure that the resolved module is allowed in the member list property. + if !memberType.IsInstance(child) { + ctx.ModuleErrorf("module %q is not valid in property %s", ctx.OtherModuleName(child), memberType.SdkPropertyName()) + } + + // Keep track of which multilib variants are used by the sdk. + s.multilibUsages = s.multilibUsages.addArchType(child.Target().Arch.ArchType) + + s.memberRefs = append(s.memberRefs, sdkMemberRef{memberType, child.(android.SdkAware)}) + + // If the member type supports transitive sdk members then recurse down into + // its dependencies, otherwise exit traversal. + return memberType.HasTransitiveSdkMembers() + } + + return false + }) +} + +// Organize the members. +// +// The members are first grouped by type and then grouped by name. The order of +// the types is the order they are referenced in android.SdkMemberTypesRegistry. +// The names are in the order in which the dependencies were added. +// +// Returns the members as well as the multilib setting to use. +func (s *sdk) organizeMembers(ctx android.ModuleContext, memberRefs []sdkMemberRef) []*sdkMember { + byType := make(map[android.SdkMemberType][]*sdkMember) + byName := make(map[string]*sdkMember) + + for _, memberRef := range memberRefs { + memberType := memberRef.memberType + variant := memberRef.variant + + name := ctx.OtherModuleName(variant) + member := byName[name] + if member == nil { + member = &sdkMember{memberType: memberType, name: name} + byName[name] = member + byType[memberType] = append(byType[memberType], member) + } + + // Only append new variants to the list. This is needed because a member can be both + // exported by the sdk and also be a transitive sdk member. + member.variants = appendUniqueVariants(member.variants, variant) + } + + var members []*sdkMember + for _, memberListProperty := range s.memberListProperties() { + membersOfType := byType[memberListProperty.memberType] + members = append(members, membersOfType...) + } + + return members +} + +func appendUniqueVariants(variants []android.SdkAware, newVariant android.SdkAware) []android.SdkAware { + for _, v := range variants { + if v == newVariant { + return variants + } + } + return append(variants, newVariant) +} + +// SDK directory structure +// <sdk_root>/ +// Android.bp : definition of a 'sdk' module is here. This is a hand-made one. +// <api_ver>/ : below this directory are all auto-generated +// Android.bp : definition of 'sdk_snapshot' module is here +// aidl/ +// frameworks/base/core/..../IFoo.aidl : an exported AIDL file +// java/ +// <module_name>.jar : the stub jar for a java library 'module_name' +// include/ +// bionic/libc/include/stdlib.h : an exported header file +// include_gen/ +// <module_name>/com/android/.../IFoo.h : a generated header file +// <arch>/include/ : arch-specific exported headers +// <arch>/include_gen/ : arch-specific generated headers +// <arch>/lib/ +// libFoo.so : a stub library + +// A name that uniquely identifies a prebuilt SDK member for a version of SDK snapshot +// This isn't visible to users, so could be changed in future. +func versionedSdkMemberName(ctx android.ModuleContext, memberName string, version string) string { + return ctx.ModuleName() + "_" + memberName + string(android.SdkVersionSeparator) + version +} + +// buildSnapshot is the main function in this source file. It creates rules to copy +// the contents (header files, stub libraries, etc) into the zip file. +func (s *sdk) buildSnapshot(ctx android.ModuleContext, sdkVariants []*sdk) android.OutputPath { + + allMembersByName := make(map[string]struct{}) + exportedMembersByName := make(map[string]struct{}) + var memberRefs []sdkMemberRef + for _, sdkVariant := range sdkVariants { + memberRefs = append(memberRefs, sdkVariant.memberRefs...) + + // Record the names of all the members, both explicitly specified and implicitly + // included. + for _, memberRef := range sdkVariant.memberRefs { + allMembersByName[memberRef.variant.Name()] = struct{}{} + } + + // Merge the exported member sets from all sdk variants. + for key, _ := range sdkVariant.getExportedMembers() { + exportedMembersByName[key] = struct{}{} + } + } + + snapshotDir := android.PathForModuleOut(ctx, "snapshot") + + bp := newGeneratedFile(ctx, "snapshot", "Android.bp") + + bpFile := &bpFile{ + modules: make(map[string]*bpModule), + } + + builder := &snapshotBuilder{ + ctx: ctx, + sdk: s, + version: "current", + snapshotDir: snapshotDir.OutputPath, + copies: make(map[string]string), + filesToZip: []android.Path{bp.path}, + bpFile: bpFile, + prebuiltModules: make(map[string]*bpModule), + allMembersByName: allMembersByName, + exportedMembersByName: exportedMembersByName, + } + s.builderForTests = builder + + members := s.organizeMembers(ctx, memberRefs) + for _, member := range members { + memberType := member.memberType + + memberCtx := &memberContext{ctx, builder, memberType, member.name} + + prebuiltModule := memberType.AddPrebuiltModule(memberCtx, member) + s.createMemberSnapshot(memberCtx, member, prebuiltModule) + } + + // Create a transformer that will transform an unversioned module into a versioned module. + unversionedToVersionedTransformer := unversionedToVersionedTransformation{builder: builder} + + // Create a transformer that will transform an unversioned module by replacing any references + // to internal members with a unique module name and setting prefer: false. + unversionedTransformer := unversionedTransformation{builder: builder} + + for _, unversioned := range builder.prebuiltOrder { + // Prune any empty property sets. + unversioned = unversioned.transform(pruneEmptySetTransformer{}) + + // Copy the unversioned module so it can be modified to make it versioned. + versioned := unversioned.deepCopy() + + // Transform the unversioned module into a versioned one. + versioned.transform(unversionedToVersionedTransformer) + bpFile.AddModule(versioned) + + // Transform the unversioned module to make it suitable for use in the snapshot. + unversioned.transform(unversionedTransformer) + bpFile.AddModule(unversioned) + } + + // Create the snapshot module. + snapshotName := ctx.ModuleName() + string(android.SdkVersionSeparator) + builder.version + var snapshotModuleType string + if s.properties.Module_exports { + snapshotModuleType = "module_exports_snapshot" + } else { + snapshotModuleType = "sdk_snapshot" + } + snapshotModule := bpFile.newModule(snapshotModuleType) + snapshotModule.AddProperty("name", snapshotName) + + // Make sure that the snapshot has the same visibility as the sdk. + visibility := android.EffectiveVisibilityRules(ctx, s) + if len(visibility) != 0 { + snapshotModule.AddProperty("visibility", visibility) + } + + addHostDeviceSupportedProperties(s.ModuleBase.DeviceSupported(), s.ModuleBase.HostSupported(), snapshotModule) + + var dynamicMemberPropertiesContainers []propertiesContainer + osTypeToMemberProperties := make(map[android.OsType]*sdk) + for _, sdkVariant := range sdkVariants { + properties := sdkVariant.dynamicMemberTypeListProperties + osTypeToMemberProperties[sdkVariant.Target().Os] = sdkVariant + dynamicMemberPropertiesContainers = append(dynamicMemberPropertiesContainers, &dynamicMemberPropertiesContainer{sdkVariant, properties}) + } + + // Extract the common lists of members into a separate struct. + commonDynamicMemberProperties := s.dynamicSdkMemberTypes.createMemberListProperties() + extractor := newCommonValueExtractor(commonDynamicMemberProperties) + extractCommonProperties(ctx, extractor, commonDynamicMemberProperties, dynamicMemberPropertiesContainers) + + // Add properties common to all os types. + s.addMemberPropertiesToPropertySet(builder, snapshotModule, commonDynamicMemberProperties) + + // Iterate over the os types in a fixed order. + targetPropertySet := snapshotModule.AddPropertySet("target") + for _, osType := range s.getPossibleOsTypes() { + if sdkVariant, ok := osTypeToMemberProperties[osType]; ok { + osPropertySet := targetPropertySet.AddPropertySet(sdkVariant.Target().Os.Name) + + // Compile_multilib defaults to both and must always be set to both on the + // device and so only needs to be set when targeted at the host and is neither + // unspecified or both. + multilib := sdkVariant.multilibUsages + if (osType.Class == android.Host || osType.Class == android.HostCross) && + multilib != multilibNone && multilib != multilibBoth { + osPropertySet.AddProperty("compile_multilib", multilib.String()) + } + + s.addMemberPropertiesToPropertySet(builder, osPropertySet, sdkVariant.dynamicMemberTypeListProperties) + } + } + + // Prune any empty property sets. + snapshotModule.transform(pruneEmptySetTransformer{}) + + bpFile.AddModule(snapshotModule) + + // generate Android.bp + bp = newGeneratedFile(ctx, "snapshot", "Android.bp") + generateBpContents(&bp.generatedContents, bpFile) + + bp.build(pctx, ctx, nil) + + filesToZip := builder.filesToZip + + // zip them all + outputZipFile := android.PathForModuleOut(ctx, ctx.ModuleName()+"-current.zip").OutputPath + outputDesc := "Building snapshot for " + ctx.ModuleName() + + // If there are no zips to merge then generate the output zip directly. + // Otherwise, generate an intermediate zip file into which other zips can be + // merged. + var zipFile android.OutputPath + var desc string + if len(builder.zipsToMerge) == 0 { + zipFile = outputZipFile + desc = outputDesc + } else { + zipFile = android.PathForModuleOut(ctx, ctx.ModuleName()+"-current.unmerged.zip").OutputPath + desc = "Building intermediate snapshot for " + ctx.ModuleName() + } + + ctx.Build(pctx, android.BuildParams{ + Description: desc, + Rule: zipFiles, + Inputs: filesToZip, + Output: zipFile, + Args: map[string]string{ + "basedir": builder.snapshotDir.String(), + }, + }) + + if len(builder.zipsToMerge) != 0 { + ctx.Build(pctx, android.BuildParams{ + Description: outputDesc, + Rule: mergeZips, + Input: zipFile, + Inputs: builder.zipsToMerge, + Output: outputZipFile, + }) + } + + return outputZipFile +} + +func extractCommonProperties(ctx android.ModuleContext, extractor *commonValueExtractor, commonProperties interface{}, inputPropertiesSlice interface{}) { + err := extractor.extractCommonProperties(commonProperties, inputPropertiesSlice) + if err != nil { + ctx.ModuleErrorf("error extracting common properties: %s", err) + } +} + +func (s *sdk) addMemberPropertiesToPropertySet(builder *snapshotBuilder, propertySet android.BpPropertySet, dynamicMemberTypeListProperties interface{}) { + for _, memberListProperty := range s.memberListProperties() { + names := memberListProperty.getter(dynamicMemberTypeListProperties) + if len(names) > 0 { + propertySet.AddProperty(memberListProperty.propertyName(), builder.versionedSdkMemberNames(names, false)) + } + } +} + +type propertyTag struct { + name string +} + +// A BpPropertyTag to add to a property that contains references to other sdk members. +// +// This will cause the references to be rewritten to a versioned reference in the version +// specific instance of a snapshot module. +var requiredSdkMemberReferencePropertyTag = propertyTag{"requiredSdkMemberReferencePropertyTag"} +var optionalSdkMemberReferencePropertyTag = propertyTag{"optionalSdkMemberReferencePropertyTag"} + +// A BpPropertyTag that indicates the property should only be present in the versioned +// module. +// +// This will cause the property to be removed from the unversioned instance of a +// snapshot module. +var sdkVersionedOnlyPropertyTag = propertyTag{"sdkVersionedOnlyPropertyTag"} + +type unversionedToVersionedTransformation struct { + identityTransformation + builder *snapshotBuilder +} + +func (t unversionedToVersionedTransformation) transformModule(module *bpModule) *bpModule { + // Use a versioned name for the module but remember the original name for the + // snapshot. + name := module.getValue("name").(string) + module.setProperty("name", t.builder.versionedSdkMemberName(name, true)) + module.insertAfter("name", "sdk_member_name", name) + return module +} + +func (t unversionedToVersionedTransformation) transformProperty(name string, value interface{}, tag android.BpPropertyTag) (interface{}, android.BpPropertyTag) { + if tag == requiredSdkMemberReferencePropertyTag || tag == optionalSdkMemberReferencePropertyTag { + required := tag == requiredSdkMemberReferencePropertyTag + return t.builder.versionedSdkMemberNames(value.([]string), required), tag + } else { + return value, tag + } +} + +type unversionedTransformation struct { + identityTransformation + builder *snapshotBuilder +} + +func (t unversionedTransformation) transformModule(module *bpModule) *bpModule { + // If the module is an internal member then use a unique name for it. + name := module.getValue("name").(string) + module.setProperty("name", t.builder.unversionedSdkMemberName(name, true)) + + // Set prefer: false - this is not strictly required as that is the default. + module.insertAfter("name", "prefer", false) + + return module +} + +func (t unversionedTransformation) transformProperty(name string, value interface{}, tag android.BpPropertyTag) (interface{}, android.BpPropertyTag) { + if tag == requiredSdkMemberReferencePropertyTag || tag == optionalSdkMemberReferencePropertyTag { + required := tag == requiredSdkMemberReferencePropertyTag + return t.builder.unversionedSdkMemberNames(value.([]string), required), tag + } else if tag == sdkVersionedOnlyPropertyTag { + // The property is not allowed in the unversioned module so remove it. + return nil, nil + } else { + return value, tag + } +} + +type pruneEmptySetTransformer struct { + identityTransformation +} + +var _ bpTransformer = (*pruneEmptySetTransformer)(nil) + +func (t pruneEmptySetTransformer) transformPropertySetAfterContents(name string, propertySet *bpPropertySet, tag android.BpPropertyTag) (*bpPropertySet, android.BpPropertyTag) { + if len(propertySet.properties) == 0 { + return nil, nil + } else { + return propertySet, tag + } +} + +func generateBpContents(contents *generatedContents, bpFile *bpFile) { + contents.Printfln("// This is auto-generated. DO NOT EDIT.") + for _, bpModule := range bpFile.order { + contents.Printfln("") + contents.Printfln("%s {", bpModule.moduleType) + outputPropertySet(contents, bpModule.bpPropertySet) + contents.Printfln("}") + } +} + +func outputPropertySet(contents *generatedContents, set *bpPropertySet) { + contents.Indent() + + // Output the properties first, followed by the nested sets. This ensures a + // consistent output irrespective of whether property sets are created before + // or after the properties. This simplifies the creation of the module. + for _, name := range set.order { + value := set.getValue(name) + + switch v := value.(type) { + case []string: + length := len(v) + if length > 1 { + contents.Printfln("%s: [", name) + contents.Indent() + for i := 0; i < length; i = i + 1 { + contents.Printfln("%q,", v[i]) + } + contents.Dedent() + contents.Printfln("],") + } else if length == 0 { + contents.Printfln("%s: [],", name) + } else { + contents.Printfln("%s: [%q],", name, v[0]) + } + + case bool: + contents.Printfln("%s: %t,", name, v) + + case *bpPropertySet: + // Do not write property sets in the properties phase. + + default: + contents.Printfln("%s: %q,", name, value) + } + } + + for _, name := range set.order { + value := set.getValue(name) + + // Only write property sets in the sets phase. + switch v := value.(type) { + case *bpPropertySet: + contents.Printfln("%s: {", name) + outputPropertySet(contents, v) + contents.Printfln("},") + } + } + + contents.Dedent() +} + +func (s *sdk) GetAndroidBpContentsForTests() string { + contents := &generatedContents{} + generateBpContents(contents, s.builderForTests.bpFile) + return contents.content.String() +} + +type snapshotBuilder struct { + ctx android.ModuleContext + sdk *sdk + version string + snapshotDir android.OutputPath + bpFile *bpFile + + // Map from destination to source of each copy - used to eliminate duplicates and + // detect conflicts. + copies map[string]string + + filesToZip android.Paths + zipsToMerge android.Paths + + prebuiltModules map[string]*bpModule + prebuiltOrder []*bpModule + + // The set of all members by name. + allMembersByName map[string]struct{} + + // The set of exported members by name. + exportedMembersByName map[string]struct{} +} + +func (s *snapshotBuilder) CopyToSnapshot(src android.Path, dest string) { + if existing, ok := s.copies[dest]; ok { + if existing != src.String() { + s.ctx.ModuleErrorf("conflicting copy, %s copied from both %s and %s", dest, existing, src) + return + } + } else { + path := s.snapshotDir.Join(s.ctx, dest) + s.ctx.Build(pctx, android.BuildParams{ + Rule: android.Cp, + Input: src, + Output: path, + }) + s.filesToZip = append(s.filesToZip, path) + + s.copies[dest] = src.String() + } +} + +func (s *snapshotBuilder) UnzipToSnapshot(zipPath android.Path, destDir string) { + ctx := s.ctx + + // Repackage the zip file so that the entries are in the destDir directory. + // This will allow the zip file to be merged into the snapshot. + tmpZipPath := android.PathForModuleOut(ctx, "tmp", destDir+".zip").OutputPath + + ctx.Build(pctx, android.BuildParams{ + Description: "Repackaging zip file " + destDir + " for snapshot " + ctx.ModuleName(), + Rule: repackageZip, + Input: zipPath, + Output: tmpZipPath, + Args: map[string]string{ + "destdir": destDir, + }, + }) + + // Add the repackaged zip file to the files to merge. + s.zipsToMerge = append(s.zipsToMerge, tmpZipPath) +} + +func (s *snapshotBuilder) AddPrebuiltModule(member android.SdkMember, moduleType string) android.BpModule { + name := member.Name() + if s.prebuiltModules[name] != nil { + panic(fmt.Sprintf("Duplicate module detected, module %s has already been added", name)) + } + + m := s.bpFile.newModule(moduleType) + m.AddProperty("name", name) + + variant := member.Variants()[0] + + if s.isInternalMember(name) { + // An internal member is only referenced from the sdk snapshot which is in the + // same package so can be marked as private. + m.AddProperty("visibility", []string{"//visibility:private"}) + } else { + // Extract visibility information from a member variant. All variants have the same + // visibility so it doesn't matter which one is used. + visibility := android.EffectiveVisibilityRules(s.ctx, variant) + if len(visibility) != 0 { + m.AddProperty("visibility", visibility) + } + } + + deviceSupported := false + hostSupported := false + + for _, variant := range member.Variants() { + osClass := variant.Target().Os.Class + if osClass == android.Host || osClass == android.HostCross { + hostSupported = true + } else if osClass == android.Device { + deviceSupported = true + } + } + + addHostDeviceSupportedProperties(deviceSupported, hostSupported, m) + + // Where available copy apex_available properties from the member. + if apexAware, ok := variant.(interface{ ApexAvailable() []string }); ok { + apexAvailable := apexAware.ApexAvailable() + + // Add in any baseline apex available settings. + apexAvailable = append(apexAvailable, apex.BaselineApexAvailable(member.Name())...) + + if len(apexAvailable) > 0 { + // Remove duplicates and sort. + apexAvailable = android.FirstUniqueStrings(apexAvailable) + sort.Strings(apexAvailable) + + m.AddProperty("apex_available", apexAvailable) + } + } + + // Disable installation in the versioned module of those modules that are ever installable. + if installable, ok := variant.(interface{ EverInstallable() bool }); ok { + if installable.EverInstallable() { + m.AddPropertyWithTag("installable", false, sdkVersionedOnlyPropertyTag) + } + } + + s.prebuiltModules[name] = m + s.prebuiltOrder = append(s.prebuiltOrder, m) + return m +} + +func addHostDeviceSupportedProperties(deviceSupported bool, hostSupported bool, bpModule *bpModule) { + if !deviceSupported { + bpModule.AddProperty("device_supported", false) + } + if hostSupported { + bpModule.AddProperty("host_supported", true) + } +} + +func (s *snapshotBuilder) SdkMemberReferencePropertyTag(required bool) android.BpPropertyTag { + if required { + return requiredSdkMemberReferencePropertyTag + } else { + return optionalSdkMemberReferencePropertyTag + } +} + +func (s *snapshotBuilder) OptionalSdkMemberReferencePropertyTag() android.BpPropertyTag { + return optionalSdkMemberReferencePropertyTag +} + +// Get a versioned name appropriate for the SDK snapshot version being taken. +func (s *snapshotBuilder) versionedSdkMemberName(unversionedName string, required bool) string { + if _, ok := s.allMembersByName[unversionedName]; !ok { + if required { + s.ctx.ModuleErrorf("Required member reference %s is not a member of the sdk", unversionedName) + } + return unversionedName + } + return versionedSdkMemberName(s.ctx, unversionedName, s.version) +} + +func (s *snapshotBuilder) versionedSdkMemberNames(members []string, required bool) []string { + var references []string = nil + for _, m := range members { + references = append(references, s.versionedSdkMemberName(m, required)) + } + return references +} + +// Get an internal name unique to the sdk. +func (s *snapshotBuilder) unversionedSdkMemberName(unversionedName string, required bool) string { + if _, ok := s.allMembersByName[unversionedName]; !ok { + if required { + s.ctx.ModuleErrorf("Required member reference %s is not a member of the sdk", unversionedName) + } + return unversionedName + } + + if s.isInternalMember(unversionedName) { + return s.ctx.ModuleName() + "_" + unversionedName + } else { + return unversionedName + } +} + +func (s *snapshotBuilder) unversionedSdkMemberNames(members []string, required bool) []string { + var references []string = nil + for _, m := range members { + references = append(references, s.unversionedSdkMemberName(m, required)) + } + return references +} + +func (s *snapshotBuilder) isInternalMember(memberName string) bool { + _, ok := s.exportedMembersByName[memberName] + return !ok +} + +type sdkMemberRef struct { + memberType android.SdkMemberType + variant android.SdkAware +} + +var _ android.SdkMember = (*sdkMember)(nil) + +type sdkMember struct { + memberType android.SdkMemberType + name string + variants []android.SdkAware +} + +func (m *sdkMember) Name() string { + return m.name +} + +func (m *sdkMember) Variants() []android.SdkAware { + return m.variants +} + +// Track usages of multilib variants. +type multilibUsage int + +const ( + multilibNone multilibUsage = 0 + multilib32 multilibUsage = 1 + multilib64 multilibUsage = 2 + multilibBoth = multilib32 | multilib64 +) + +// Add the multilib that is used in the arch type. +func (m multilibUsage) addArchType(archType android.ArchType) multilibUsage { + multilib := archType.Multilib + switch multilib { + case "": + return m + case "lib32": + return m | multilib32 + case "lib64": + return m | multilib64 + default: + panic(fmt.Errorf("Unknown Multilib field in ArchType, expected 'lib32' or 'lib64', found %q", multilib)) + } +} + +func (m multilibUsage) String() string { + switch m { + case multilibNone: + return "" + case multilib32: + return "32" + case multilib64: + return "64" + case multilibBoth: + return "both" + default: + panic(fmt.Errorf("Unknown multilib value, found %b, expected one of %b, %b, %b or %b", + m, multilibNone, multilib32, multilib64, multilibBoth)) + } +} + +type baseInfo struct { + Properties android.SdkMemberProperties +} + +func (b *baseInfo) optimizableProperties() interface{} { + return b.Properties +} + +type osTypeSpecificInfo struct { + baseInfo + + osType android.OsType + + // The list of arch type specific info for this os type. + // + // Nil if there is one variant whose arch type is common + archInfos []*archTypeSpecificInfo +} + +var _ propertiesContainer = (*osTypeSpecificInfo)(nil) + +type variantPropertiesFactoryFunc func() android.SdkMemberProperties + +// Create a new osTypeSpecificInfo for the specified os type and its properties +// structures populated with information from the variants. +func newOsTypeSpecificInfo(ctx android.SdkMemberContext, osType android.OsType, variantPropertiesFactory variantPropertiesFactoryFunc, osTypeVariants []android.Module) *osTypeSpecificInfo { + osInfo := &osTypeSpecificInfo{ + osType: osType, + } + + osSpecificVariantPropertiesFactory := func() android.SdkMemberProperties { + properties := variantPropertiesFactory() + properties.Base().Os = osType + return properties + } + + // Create a structure into which properties common across the architectures in + // this os type will be stored. + osInfo.Properties = osSpecificVariantPropertiesFactory() + + // Group the variants by arch type. + var variantsByArchName = make(map[string][]android.Module) + var archTypes []android.ArchType + for _, variant := range osTypeVariants { + archType := variant.Target().Arch.ArchType + archTypeName := archType.Name + if _, ok := variantsByArchName[archTypeName]; !ok { + archTypes = append(archTypes, archType) + } + + variantsByArchName[archTypeName] = append(variantsByArchName[archTypeName], variant) + } + + if commonVariants, ok := variantsByArchName["common"]; ok { + if len(osTypeVariants) != 1 { + panic("Expected to only have 1 variant when arch type is common but found " + string(len(osTypeVariants))) + } + + // A common arch type only has one variant and its properties should be treated + // as common to the os type. + osInfo.Properties.PopulateFromVariant(ctx, commonVariants[0]) + } else { + // Create an arch specific info for each supported architecture type. + for _, archType := range archTypes { + archTypeName := archType.Name + + archVariants := variantsByArchName[archTypeName] + archInfo := newArchSpecificInfo(ctx, archType, osSpecificVariantPropertiesFactory, archVariants) + + osInfo.archInfos = append(osInfo.archInfos, archInfo) + } + } + + return osInfo +} + +// Optimize the properties by extracting common properties from arch type specific +// properties into os type specific properties. +func (osInfo *osTypeSpecificInfo) optimizeProperties(ctx *memberContext, commonValueExtractor *commonValueExtractor) { + // Nothing to do if there is only a single common architecture. + if len(osInfo.archInfos) == 0 { + return + } + + multilib := multilibNone + for _, archInfo := range osInfo.archInfos { + multilib = multilib.addArchType(archInfo.archType) + + // Optimize the arch properties first. + archInfo.optimizeProperties(ctx, commonValueExtractor) + } + + extractCommonProperties(ctx.sdkMemberContext, commonValueExtractor, osInfo.Properties, osInfo.archInfos) + + // Choose setting for compile_multilib that is appropriate for the arch variants supplied. + osInfo.Properties.Base().Compile_multilib = multilib.String() +} + +// Add the properties for an os to a property set. +// +// Maps the properties related to the os variants through to an appropriate +// module structure that will produce equivalent set of variants when it is +// processed in a build. +func (osInfo *osTypeSpecificInfo) addToPropertySet(ctx *memberContext, bpModule android.BpModule, targetPropertySet android.BpPropertySet) { + + var osPropertySet android.BpPropertySet + var archPropertySet android.BpPropertySet + var archOsPrefix string + if osInfo.Properties.Base().Os_count == 1 { + // There is only one os type present in the variants so don't bother + // with adding target specific properties. + + // Create a structure that looks like: + // module_type { + // name: "...", + // ... + // <common properties> + // ... + // <single os type specific properties> + // + // arch: { + // <arch specific sections> + // } + // + osPropertySet = bpModule + archPropertySet = osPropertySet.AddPropertySet("arch") + + // Arch specific properties need to be added to an arch specific section + // within arch. + archOsPrefix = "" + } else { + // Create a structure that looks like: + // module_type { + // name: "...", + // ... + // <common properties> + // ... + // target: { + // <arch independent os specific sections, e.g. android> + // ... + // <arch and os specific sections, e.g. android_x86> + // } + // + osType := osInfo.osType + osPropertySet = targetPropertySet.AddPropertySet(osType.Name) + archPropertySet = targetPropertySet + + // Arch specific properties need to be added to an os and arch specific + // section prefixed with <os>_. + archOsPrefix = osType.Name + "_" + } + + // Add the os specific but arch independent properties to the module. + osInfo.Properties.AddToPropertySet(ctx, osPropertySet) + + // Add arch (and possibly os) specific sections for each set of arch (and possibly + // os) specific properties. + // + // The archInfos list will be empty if the os contains variants for the common + // architecture. + for _, archInfo := range osInfo.archInfos { + archInfo.addToPropertySet(ctx, archPropertySet, archOsPrefix) + } +} + +func (osInfo *osTypeSpecificInfo) isHostVariant() bool { + osClass := osInfo.osType.Class + return osClass == android.Host || osClass == android.HostCross +} + +var _ isHostVariant = (*osTypeSpecificInfo)(nil) + +func (osInfo *osTypeSpecificInfo) String() string { + return fmt.Sprintf("OsType{%s}", osInfo.osType) +} + +type archTypeSpecificInfo struct { + baseInfo + + archType android.ArchType + + linkInfos []*linkTypeSpecificInfo +} + +var _ propertiesContainer = (*archTypeSpecificInfo)(nil) + +// Create a new archTypeSpecificInfo for the specified arch type and its properties +// structures populated with information from the variants. +func newArchSpecificInfo(ctx android.SdkMemberContext, archType android.ArchType, variantPropertiesFactory variantPropertiesFactoryFunc, archVariants []android.Module) *archTypeSpecificInfo { + + // Create an arch specific info into which the variant properties can be copied. + archInfo := &archTypeSpecificInfo{archType: archType} + + // Create the properties into which the arch type specific properties will be + // added. + archInfo.Properties = variantPropertiesFactory() + + if len(archVariants) == 1 { + archInfo.Properties.PopulateFromVariant(ctx, archVariants[0]) + } else { + // There is more than one variant for this arch type which must be differentiated + // by link type. + for _, linkVariant := range archVariants { + linkType := getLinkType(linkVariant) + if linkType == "" { + panic(fmt.Errorf("expected one arch specific variant as it is not identified by link type but found %d", len(archVariants))) + } else { + linkInfo := newLinkSpecificInfo(ctx, linkType, variantPropertiesFactory, linkVariant) + + archInfo.linkInfos = append(archInfo.linkInfos, linkInfo) + } + } + } + + return archInfo +} + +func (archInfo *archTypeSpecificInfo) optimizableProperties() interface{} { + return archInfo.Properties +} + +// Get the link type of the variant +// +// If the variant is not differentiated by link type then it returns "", +// otherwise it returns one of "static" or "shared". +func getLinkType(variant android.Module) string { + linkType := "" + if linkable, ok := variant.(cc.LinkableInterface); ok { + if linkable.Shared() && linkable.Static() { + panic(fmt.Errorf("expected variant %q to be either static or shared but was both", variant.String())) + } else if linkable.Shared() { + linkType = "shared" + } else if linkable.Static() { + linkType = "static" + } else { + panic(fmt.Errorf("expected variant %q to be either static or shared but was neither", variant.String())) + } + } + return linkType +} + +// Optimize the properties by extracting common properties from link type specific +// properties into arch type specific properties. +func (archInfo *archTypeSpecificInfo) optimizeProperties(ctx *memberContext, commonValueExtractor *commonValueExtractor) { + if len(archInfo.linkInfos) == 0 { + return + } + + extractCommonProperties(ctx.sdkMemberContext, commonValueExtractor, archInfo.Properties, archInfo.linkInfos) +} + +// Add the properties for an arch type to a property set. +func (archInfo *archTypeSpecificInfo) addToPropertySet(ctx *memberContext, archPropertySet android.BpPropertySet, archOsPrefix string) { + archTypeName := archInfo.archType.Name + archTypePropertySet := archPropertySet.AddPropertySet(archOsPrefix + archTypeName) + archInfo.Properties.AddToPropertySet(ctx, archTypePropertySet) + + for _, linkInfo := range archInfo.linkInfos { + linkPropertySet := archTypePropertySet.AddPropertySet(linkInfo.linkType) + linkInfo.Properties.AddToPropertySet(ctx, linkPropertySet) + } +} + +func (archInfo *archTypeSpecificInfo) String() string { + return fmt.Sprintf("ArchType{%s}", archInfo.archType) +} + +type linkTypeSpecificInfo struct { + baseInfo + + linkType string +} + +var _ propertiesContainer = (*linkTypeSpecificInfo)(nil) + +// Create a new linkTypeSpecificInfo for the specified link type and its properties +// structures populated with information from the variant. +func newLinkSpecificInfo(ctx android.SdkMemberContext, linkType string, variantPropertiesFactory variantPropertiesFactoryFunc, linkVariant android.Module) *linkTypeSpecificInfo { + linkInfo := &linkTypeSpecificInfo{ + baseInfo: baseInfo{ + // Create the properties into which the link type specific properties will be + // added. + Properties: variantPropertiesFactory(), + }, + linkType: linkType, + } + linkInfo.Properties.PopulateFromVariant(ctx, linkVariant) + return linkInfo +} + +func (l *linkTypeSpecificInfo) String() string { + return fmt.Sprintf("LinkType{%s}", l.linkType) +} + +type memberContext struct { + sdkMemberContext android.ModuleContext + builder *snapshotBuilder + memberType android.SdkMemberType + name string +} + +func (m *memberContext) SdkModuleContext() android.ModuleContext { + return m.sdkMemberContext +} + +func (m *memberContext) SnapshotBuilder() android.SnapshotBuilder { + return m.builder +} + +func (m *memberContext) MemberType() android.SdkMemberType { + return m.memberType +} + +func (m *memberContext) Name() string { + return m.name +} + +func (s *sdk) createMemberSnapshot(ctx *memberContext, member *sdkMember, bpModule android.BpModule) { + + memberType := member.memberType + + // Group the variants by os type. + variantsByOsType := make(map[android.OsType][]android.Module) + variants := member.Variants() + for _, variant := range variants { + osType := variant.Target().Os + variantsByOsType[osType] = append(variantsByOsType[osType], variant) + } + + osCount := len(variantsByOsType) + variantPropertiesFactory := func() android.SdkMemberProperties { + properties := memberType.CreateVariantPropertiesStruct() + base := properties.Base() + base.Os_count = osCount + return properties + } + + osTypeToInfo := make(map[android.OsType]*osTypeSpecificInfo) + + // The set of properties that are common across all architectures and os types. + commonProperties := variantPropertiesFactory() + commonProperties.Base().Os = android.CommonOS + + // Create common value extractor that can be used to optimize the properties. + commonValueExtractor := newCommonValueExtractor(commonProperties) + + // The list of property structures which are os type specific but common across + // architectures within that os type. + var osSpecificPropertiesContainers []*osTypeSpecificInfo + + for osType, osTypeVariants := range variantsByOsType { + osInfo := newOsTypeSpecificInfo(ctx, osType, variantPropertiesFactory, osTypeVariants) + osTypeToInfo[osType] = osInfo + // Add the os specific properties to a list of os type specific yet architecture + // independent properties structs. + osSpecificPropertiesContainers = append(osSpecificPropertiesContainers, osInfo) + + // Optimize the properties across all the variants for a specific os type. + osInfo.optimizeProperties(ctx, commonValueExtractor) + } + + // Extract properties which are common across all architectures and os types. + extractCommonProperties(ctx.sdkMemberContext, commonValueExtractor, commonProperties, osSpecificPropertiesContainers) + + // Add the common properties to the module. + commonProperties.AddToPropertySet(ctx, bpModule) + + // Create a target property set into which target specific properties can be + // added. + targetPropertySet := bpModule.AddPropertySet("target") + + // Iterate over the os types in a fixed order. + for _, osType := range s.getPossibleOsTypes() { + osInfo := osTypeToInfo[osType] + if osInfo == nil { + continue + } + + osInfo.addToPropertySet(ctx, bpModule, targetPropertySet) + } +} + +// Compute the list of possible os types that this sdk could support. +func (s *sdk) getPossibleOsTypes() []android.OsType { + var osTypes []android.OsType + for _, osType := range android.OsTypeList { + if s.DeviceSupported() { + if osType.Class == android.Device && osType != android.Fuchsia { + osTypes = append(osTypes, osType) + } + } + if s.HostSupported() { + if osType.Class == android.Host || osType.Class == android.HostCross { + osTypes = append(osTypes, osType) + } + } + } + sort.SliceStable(osTypes, func(i, j int) bool { return osTypes[i].Name < osTypes[j].Name }) + return osTypes +} + +// Given a set of properties (struct value), return the value of the field within that +// struct (or one of its embedded structs). +type fieldAccessorFunc func(structValue reflect.Value) reflect.Value + +// Checks the metadata to determine whether the property should be ignored for the +// purposes of common value extraction or not. +type extractorMetadataPredicate func(metadata propertiesContainer) bool + +// Indicates whether optimizable properties are provided by a host variant or +// not. +type isHostVariant interface { + isHostVariant() bool +} + +// A property that can be optimized by the commonValueExtractor. +type extractorProperty struct { + // The name of the field for this property. + name string + + // Filter that can use metadata associated with the properties being optimized + // to determine whether the field should be ignored during common value + // optimization. + filter extractorMetadataPredicate + + // Retrieves the value on which common value optimization will be performed. + getter fieldAccessorFunc + + // The empty value for the field. + emptyValue reflect.Value + + // True if the property can support arch variants false otherwise. + archVariant bool +} + +func (p extractorProperty) String() string { + return p.name +} + +// Supports extracting common values from a number of instances of a properties +// structure into a separate common set of properties. +type commonValueExtractor struct { + // The properties that the extractor can optimize. + properties []extractorProperty +} + +// Create a new common value extractor for the structure type for the supplied +// properties struct. +// +// The returned extractor can be used on any properties structure of the same type +// as the supplied set of properties. +func newCommonValueExtractor(propertiesStruct interface{}) *commonValueExtractor { + structType := getStructValue(reflect.ValueOf(propertiesStruct)).Type() + extractor := &commonValueExtractor{} + extractor.gatherFields(structType, nil) + return extractor +} + +// Gather the fields from the supplied structure type from which common values will +// be extracted. +// +// This is recursive function. If it encounters an embedded field (no field name) +// that is a struct then it will recurse into that struct passing in the accessor +// for the field. That will then be used in the accessors for the fields in the +// embedded struct. +func (e *commonValueExtractor) gatherFields(structType reflect.Type, containingStructAccessor fieldAccessorFunc) { + for f := 0; f < structType.NumField(); f++ { + field := structType.Field(f) + if field.PkgPath != "" { + // Ignore unexported fields. + continue + } + + // Ignore fields whose value should be kept. + if proptools.HasTag(field, "sdk", "keep") { + continue + } + + var filter extractorMetadataPredicate + + // Add a filter + if proptools.HasTag(field, "sdk", "ignored-on-host") { + filter = func(metadata propertiesContainer) bool { + if m, ok := metadata.(isHostVariant); ok { + if m.isHostVariant() { + return false + } + } + return true + } + } + + // Save a copy of the field index for use in the function. + fieldIndex := f + + name := field.Name + + fieldGetter := func(value reflect.Value) reflect.Value { + if containingStructAccessor != nil { + // This is an embedded structure so first access the field for the embedded + // structure. + value = containingStructAccessor(value) + } + + // Skip through interface and pointer values to find the structure. + value = getStructValue(value) + + defer func() { + if r := recover(); r != nil { + panic(fmt.Errorf("%s for fieldIndex %d of field %s of value %#v", r, fieldIndex, name, value.Interface())) + } + }() + + // Return the field. + return value.Field(fieldIndex) + } + + if field.Type.Kind() == reflect.Struct && field.Anonymous { + // Gather fields from the embedded structure. + e.gatherFields(field.Type, fieldGetter) + } else { + property := extractorProperty{ + name, + filter, + fieldGetter, + reflect.Zero(field.Type), + proptools.HasTag(field, "android", "arch_variant"), + } + e.properties = append(e.properties, property) + } + } +} + +func getStructValue(value reflect.Value) reflect.Value { +foundStruct: + for { + kind := value.Kind() + switch kind { + case reflect.Interface, reflect.Ptr: + value = value.Elem() + case reflect.Struct: + break foundStruct + default: + panic(fmt.Errorf("expecting struct, interface or pointer, found %v of kind %s", value, kind)) + } + } + return value +} + +// A container of properties to be optimized. +// +// Allows additional information to be associated with the properties, e.g. for +// filtering. +type propertiesContainer interface { + fmt.Stringer + + // Get the properties that need optimizing. + optimizableProperties() interface{} +} + +// A wrapper for dynamic member properties to allow them to be optimized. +type dynamicMemberPropertiesContainer struct { + sdkVariant *sdk + dynamicMemberProperties interface{} +} + +func (c dynamicMemberPropertiesContainer) optimizableProperties() interface{} { + return c.dynamicMemberProperties +} + +func (c dynamicMemberPropertiesContainer) String() string { + return c.sdkVariant.String() +} + +// Extract common properties from a slice of property structures of the same type. +// +// All the property structures must be of the same type. +// commonProperties - must be a pointer to the structure into which common properties will be added. +// inputPropertiesSlice - must be a slice of propertiesContainer interfaces. +// +// Iterates over each exported field (capitalized name) and checks to see whether they +// have the same value (using DeepEquals) across all the input properties. If it does not then no +// change is made. Otherwise, the common value is stored in the field in the commonProperties +// and the field in each of the input properties structure is set to its default value. +func (e *commonValueExtractor) extractCommonProperties(commonProperties interface{}, inputPropertiesSlice interface{}) error { + commonPropertiesValue := reflect.ValueOf(commonProperties) + commonStructValue := commonPropertiesValue.Elem() + + sliceValue := reflect.ValueOf(inputPropertiesSlice) + + for _, property := range e.properties { + fieldGetter := property.getter + filter := property.filter + if filter == nil { + filter = func(metadata propertiesContainer) bool { + return true + } + } + + // Check to see if all the structures have the same value for the field. The commonValue + // is nil on entry to the loop and if it is nil on exit then there is no common value or + // all the values have been filtered out, otherwise it points to the common value. + var commonValue *reflect.Value + + // Assume that all the values will be the same. + // + // While similar to this is not quite the same as commonValue == nil. If all the values + // have been filtered out then this will be false but commonValue == nil will be true. + valuesDiffer := false + + for i := 0; i < sliceValue.Len(); i++ { + container := sliceValue.Index(i).Interface().(propertiesContainer) + itemValue := reflect.ValueOf(container.optimizableProperties()) + fieldValue := fieldGetter(itemValue) + + if !filter(container) { + expectedValue := property.emptyValue.Interface() + actualValue := fieldValue.Interface() + if !reflect.DeepEqual(expectedValue, actualValue) { + return fmt.Errorf("field %q is supposed to be ignored for %q but is set to %#v instead of %#v", property, container, actualValue, expectedValue) + } + continue + } + + if commonValue == nil { + // Use the first value as the commonProperties value. + commonValue = &fieldValue + } else { + // If the value does not match the current common value then there is + // no value in common so break out. + if !reflect.DeepEqual(fieldValue.Interface(), commonValue.Interface()) { + commonValue = nil + valuesDiffer = true + break + } + } + } + + // If the fields all have common value then store it in the common struct field + // and set the input struct's field to the empty value. + if commonValue != nil { + emptyValue := property.emptyValue + fieldGetter(commonStructValue).Set(*commonValue) + for i := 0; i < sliceValue.Len(); i++ { + container := sliceValue.Index(i).Interface().(propertiesContainer) + itemValue := reflect.ValueOf(container.optimizableProperties()) + fieldValue := fieldGetter(itemValue) + fieldValue.Set(emptyValue) + } + } + + if valuesDiffer && !property.archVariant { + // The values differ but the property does not support arch variants so it + // is an error. + var details strings.Builder + for i := 0; i < sliceValue.Len(); i++ { + container := sliceValue.Index(i).Interface().(propertiesContainer) + itemValue := reflect.ValueOf(container.optimizableProperties()) + fieldValue := fieldGetter(itemValue) + + _, _ = fmt.Fprintf(&details, "\n %q has value %q", container.String(), fieldValue.Interface()) + } + + return fmt.Errorf("field %q is not tagged as \"arch_variant\" but has arch specific properties:%s", property.String(), details.String()) + } + } + + return nil +} diff --git a/sh/Android.bp b/sh/Android.bp new file mode 100644 index 000000000..0f40c5f68 --- /dev/null +++ b/sh/Android.bp @@ -0,0 +1,17 @@ +bootstrap_go_package { + name: "soong-sh", + pkgPath: "android/soong/sh", + deps: [ + "blueprint", + "soong", + "soong-android", + "soong-tradefed", + ], + srcs: [ + "sh_binary.go", + ], + testSrcs: [ + "sh_binary_test.go", + ], + pluginFor: ["soong_build"], +} diff --git a/sh/sh_binary.go b/sh/sh_binary.go new file mode 100644 index 000000000..ab0490ac1 --- /dev/null +++ b/sh/sh_binary.go @@ -0,0 +1,296 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sh + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/google/blueprint/proptools" + + "android/soong/android" + "android/soong/tradefed" +) + +// sh_binary is for shell scripts (and batch files) that are installed as +// executable files into .../bin/ +// +// Do not use them for prebuilt C/C++/etc files. Use cc_prebuilt_binary +// instead. + +var pctx = android.NewPackageContext("android/soong/sh") + +func init() { + pctx.Import("android/soong/android") + + android.RegisterModuleType("sh_binary", ShBinaryFactory) + android.RegisterModuleType("sh_binary_host", ShBinaryHostFactory) + android.RegisterModuleType("sh_test", ShTestFactory) + android.RegisterModuleType("sh_test_host", ShTestHostFactory) +} + +type shBinaryProperties struct { + // Source file of this prebuilt. + Src *string `android:"path,arch_variant"` + + // optional subdirectory under which this file is installed into + Sub_dir *string `android:"arch_variant"` + + // optional name for the installed file. If unspecified, name of the module is used as the file name + Filename *string `android:"arch_variant"` + + // when set to true, and filename property is not set, the name for the installed file + // is the same as the file name of the source file. + Filename_from_src *bool `android:"arch_variant"` + + // Whether this module is directly installable to one of the partitions. Default: true. + Installable *bool + + // install symlinks to the binary + Symlinks []string `android:"arch_variant"` +} + +type TestProperties struct { + // list of compatibility suites (for example "cts", "vts") that the module should be + // installed into. + Test_suites []string `android:"arch_variant"` + + // the name of the test configuration (for example "AndroidTest.xml") that should be + // installed with the module. + Test_config *string `android:"path,arch_variant"` + + // list of files or filegroup modules that provide data that should be installed alongside + // the test. + Data []string `android:"path,arch_variant"` + + // Add RootTargetPreparer to auto generated test config. This guarantees the test to run + // with root permission. + Require_root *bool + + // the name of the test configuration template (for example "AndroidTestTemplate.xml") that + // should be installed with the module. + Test_config_template *string `android:"path,arch_variant"` + + // Flag to indicate whether or not to create test config automatically. If AndroidTest.xml + // doesn't exist next to the Android.bp, this attribute doesn't need to be set to true + // explicitly. + Auto_gen_config *bool +} + +type ShBinary struct { + android.ModuleBase + + properties shBinaryProperties + + sourceFilePath android.Path + outputFilePath android.OutputPath + installedFile android.InstallPath +} + +var _ android.HostToolProvider = (*ShBinary)(nil) + +type ShTest struct { + ShBinary + + testProperties TestProperties + + data android.Paths + testConfig android.Path +} + +func (s *ShBinary) HostToolPath() android.OptionalPath { + return android.OptionalPathForPath(s.installedFile) +} + +func (s *ShBinary) DepsMutator(ctx android.BottomUpMutatorContext) { + if s.properties.Src == nil { + ctx.PropertyErrorf("src", "missing prebuilt source file") + } +} + +func (s *ShBinary) OutputFile() android.OutputPath { + return s.outputFilePath +} + +func (s *ShBinary) SubDir() string { + return proptools.String(s.properties.Sub_dir) +} + +func (s *ShBinary) Installable() bool { + return s.properties.Installable == nil || proptools.Bool(s.properties.Installable) +} + +func (s *ShBinary) Symlinks() []string { + return s.properties.Symlinks +} + +func (s *ShBinary) generateAndroidBuildActions(ctx android.ModuleContext) { + s.sourceFilePath = android.PathForModuleSrc(ctx, proptools.String(s.properties.Src)) + filename := proptools.String(s.properties.Filename) + filename_from_src := proptools.Bool(s.properties.Filename_from_src) + if filename == "" { + if filename_from_src { + filename = s.sourceFilePath.Base() + } else { + filename = ctx.ModuleName() + } + } else if filename_from_src { + ctx.PropertyErrorf("filename_from_src", "filename is set. filename_from_src can't be true") + return + } + s.outputFilePath = android.PathForModuleOut(ctx, filename).OutputPath + + // This ensures that outputFilePath has the correct name for others to + // use, as the source file may have a different name. + ctx.Build(pctx, android.BuildParams{ + Rule: android.CpExecutable, + Output: s.outputFilePath, + Input: s.sourceFilePath, + }) +} + +func (s *ShBinary) GenerateAndroidBuildActions(ctx android.ModuleContext) { + s.generateAndroidBuildActions(ctx) + installDir := android.PathForModuleInstall(ctx, "bin", proptools.String(s.properties.Sub_dir)) + s.installedFile = ctx.InstallExecutable(installDir, s.outputFilePath.Base(), s.outputFilePath) +} + +func (s *ShBinary) AndroidMkEntries() []android.AndroidMkEntries { + return []android.AndroidMkEntries{android.AndroidMkEntries{ + Class: "EXECUTABLES", + OutputFile: android.OptionalPathForPath(s.outputFilePath), + Include: "$(BUILD_SYSTEM)/soong_cc_prebuilt.mk", + ExtraEntries: []android.AndroidMkExtraEntriesFunc{ + func(entries *android.AndroidMkEntries) { + s.customAndroidMkEntries(entries) + }, + }, + }} +} + +func (s *ShBinary) customAndroidMkEntries(entries *android.AndroidMkEntries) { + entries.SetString("LOCAL_MODULE_RELATIVE_PATH", proptools.String(s.properties.Sub_dir)) + entries.SetString("LOCAL_MODULE_SUFFIX", "") + entries.SetString("LOCAL_MODULE_STEM", s.outputFilePath.Rel()) + if len(s.properties.Symlinks) > 0 { + entries.SetString("LOCAL_MODULE_SYMLINKS", strings.Join(s.properties.Symlinks, " ")) + } +} + +func (s *ShTest) GenerateAndroidBuildActions(ctx android.ModuleContext) { + s.ShBinary.generateAndroidBuildActions(ctx) + testDir := "nativetest" + if ctx.Target().Arch.ArchType.Multilib == "lib64" { + testDir = "nativetest64" + } + if ctx.Target().NativeBridge == android.NativeBridgeEnabled { + testDir = filepath.Join(testDir, ctx.Target().NativeBridgeRelativePath) + } else if !ctx.Host() && ctx.Config().HasMultilibConflict(ctx.Arch().ArchType) { + testDir = filepath.Join(testDir, ctx.Arch().ArchType.String()) + } + installDir := android.PathForModuleInstall(ctx, testDir, proptools.String(s.properties.Sub_dir)) + s.installedFile = ctx.InstallExecutable(installDir, s.outputFilePath.Base(), s.outputFilePath) + + s.data = android.PathsForModuleSrc(ctx, s.testProperties.Data) + + var configs []tradefed.Config + if Bool(s.testProperties.Require_root) { + configs = append(configs, tradefed.Object{"target_preparer", "com.android.tradefed.targetprep.RootTargetPreparer", nil}) + } else { + options := []tradefed.Option{{Name: "force-root", Value: "false"}} + configs = append(configs, tradefed.Object{"target_preparer", "com.android.tradefed.targetprep.RootTargetPreparer", options}) + } + s.testConfig = tradefed.AutoGenShellTestConfig(ctx, s.testProperties.Test_config, + s.testProperties.Test_config_template, s.testProperties.Test_suites, configs, s.testProperties.Auto_gen_config, s.outputFilePath.Base()) +} + +func (s *ShTest) InstallInData() bool { + return true +} + +func (s *ShTest) AndroidMkEntries() []android.AndroidMkEntries { + return []android.AndroidMkEntries{android.AndroidMkEntries{ + Class: "NATIVE_TESTS", + OutputFile: android.OptionalPathForPath(s.outputFilePath), + Include: "$(BUILD_SYSTEM)/soong_cc_prebuilt.mk", + ExtraEntries: []android.AndroidMkExtraEntriesFunc{ + func(entries *android.AndroidMkEntries) { + s.customAndroidMkEntries(entries) + + entries.AddStrings("LOCAL_COMPATIBILITY_SUITE", s.testProperties.Test_suites...) + if s.testConfig != nil { + entries.SetPath("LOCAL_FULL_TEST_CONFIG", s.testConfig) + } + for _, d := range s.data { + rel := d.Rel() + path := d.String() + if !strings.HasSuffix(path, rel) { + panic(fmt.Errorf("path %q does not end with %q", path, rel)) + } + path = strings.TrimSuffix(path, rel) + entries.AddStrings("LOCAL_TEST_DATA", path+":"+rel) + } + }, + }, + }} +} + +func InitShBinaryModule(s *ShBinary) { + s.AddProperties(&s.properties) +} + +// sh_binary is for a shell script or batch file to be installed as an +// executable binary to <partition>/bin. +func ShBinaryFactory() android.Module { + module := &ShBinary{} + module.Prefer32(func(ctx android.BaseModuleContext, base *android.ModuleBase, class android.OsClass) bool { + return class == android.Device && ctx.Config().DevicePrefer32BitExecutables() + }) + InitShBinaryModule(module) + android.InitAndroidArchModule(module, android.HostAndDeviceSupported, android.MultilibFirst) + return module +} + +// sh_binary_host is for a shell script to be installed as an executable binary +// to $(HOST_OUT)/bin. +func ShBinaryHostFactory() android.Module { + module := &ShBinary{} + InitShBinaryModule(module) + android.InitAndroidArchModule(module, android.HostSupported, android.MultilibFirst) + return module +} + +// sh_test defines a shell script based test module. +func ShTestFactory() android.Module { + module := &ShTest{} + InitShBinaryModule(&module.ShBinary) + module.AddProperties(&module.testProperties) + + android.InitAndroidArchModule(module, android.HostAndDeviceSupported, android.MultilibFirst) + return module +} + +// sh_test_host defines a shell script based test module that runs on a host. +func ShTestHostFactory() android.Module { + module := &ShTest{} + InitShBinaryModule(&module.ShBinary) + module.AddProperties(&module.testProperties) + + android.InitAndroidArchModule(module, android.HostSupported, android.MultilibFirst) + return module +} + +var Bool = proptools.Bool diff --git a/sh/sh_binary_test.go b/sh/sh_binary_test.go new file mode 100644 index 000000000..6c0d96abe --- /dev/null +++ b/sh/sh_binary_test.go @@ -0,0 +1,99 @@ +package sh + +import ( + "io/ioutil" + "os" + "reflect" + "testing" + + "android/soong/android" +) + +var buildDir string + +func setUp() { + var err error + buildDir, err = ioutil.TempDir("", "soong_sh_test") + if err != nil { + panic(err) + } +} + +func tearDown() { + os.RemoveAll(buildDir) +} + +func TestMain(m *testing.M) { + run := func() int { + setUp() + defer tearDown() + + return m.Run() + } + + os.Exit(run()) +} + +func testShBinary(t *testing.T, bp string) (*android.TestContext, android.Config) { + fs := map[string][]byte{ + "test.sh": nil, + "testdata/data1": nil, + "testdata/sub/data2": nil, + } + + config := android.TestArchConfig(buildDir, nil, bp, fs) + + ctx := android.NewTestArchContext() + ctx.RegisterModuleType("sh_test", ShTestFactory) + ctx.RegisterModuleType("sh_test_host", ShTestHostFactory) + ctx.Register(config) + _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) + android.FailIfErrored(t, errs) + _, errs = ctx.PrepareBuildActions(config) + android.FailIfErrored(t, errs) + + return ctx, config +} + +func TestShTestTestData(t *testing.T) { + ctx, config := testShBinary(t, ` + sh_test { + name: "foo", + src: "test.sh", + filename: "test.sh", + data: [ + "testdata/data1", + "testdata/sub/data2", + ], + } + `) + + mod := ctx.ModuleForTests("foo", "android_arm64_armv8-a").Module().(*ShTest) + + entries := android.AndroidMkEntriesForTest(t, config, "", mod)[0] + expected := []string{":testdata/data1", ":testdata/sub/data2"} + actual := entries.EntryMap["LOCAL_TEST_DATA"] + if !reflect.DeepEqual(expected, actual) { + t.Errorf("Unexpected test data expected: %q, actual: %q", expected, actual) + } +} + +func TestShTestHost(t *testing.T) { + ctx, _ := testShBinary(t, ` + sh_test_host { + name: "foo", + src: "test.sh", + filename: "test.sh", + data: [ + "testdata/data1", + "testdata/sub/data2", + ], + } + `) + + buildOS := android.BuildOs.String() + mod := ctx.ModuleForTests("foo", buildOS+"_x86_64").Module().(*ShTest) + if !mod.Host() { + t.Errorf("host bit is not set for a sh_test_host module.") + } +} diff --git a/shared/Android.bp b/shared/Android.bp new file mode 100644 index 000000000..07dfe11d3 --- /dev/null +++ b/shared/Android.bp @@ -0,0 +1,7 @@ +bootstrap_go_package { + name: "soong-shared", + pkgPath: "android/soong/shared", + srcs: [ + "paths.go", + ], +} diff --git a/sysprop/Android.bp b/sysprop/Android.bp new file mode 100644 index 000000000..48094f1ef --- /dev/null +++ b/sysprop/Android.bp @@ -0,0 +1,18 @@ +bootstrap_go_package { + name: "soong-sysprop", + pkgPath: "android/soong/sysprop", + deps: [ + "blueprint", + "soong", + "soong-android", + "soong-cc", + "soong-java", + ], + srcs: [ + "sysprop_library.go", + ], + testSrcs: [ + "sysprop_test.go", + ], + pluginFor: ["soong_build"], +} diff --git a/sysprop/sysprop_library.go b/sysprop/sysprop_library.go index 08845b7fe..14fab6884 100644 --- a/sysprop/sysprop_library.go +++ b/sysprop/sysprop_library.go @@ -15,11 +15,16 @@ package sysprop import ( + "fmt" + "io" + "path" + + "github.com/google/blueprint" + "github.com/google/blueprint/proptools" + "android/soong/android" "android/soong/cc" "android/soong/java" - "github.com/google/blueprint" - "github.com/google/blueprint/proptools" ) type dependencyTag struct { @@ -27,11 +32,97 @@ type dependencyTag struct { name string } +type syspropGenProperties struct { + Srcs []string `android:"path"` + Scope string + Name *string +} + +type syspropJavaGenRule struct { + android.ModuleBase + + properties syspropGenProperties + + genSrcjars android.Paths +} + +var _ android.OutputFileProducer = (*syspropJavaGenRule)(nil) + +var ( + syspropJava = pctx.AndroidStaticRule("syspropJava", + blueprint.RuleParams{ + Command: `rm -rf $out.tmp && mkdir -p $out.tmp && ` + + `$syspropJavaCmd --scope $scope --java-output-dir $out.tmp $in && ` + + `$soongZipCmd -jar -o $out -C $out.tmp -D $out.tmp && rm -rf $out.tmp`, + CommandDeps: []string{ + "$syspropJavaCmd", + "$soongZipCmd", + }, + }, "scope") +) + +func init() { + pctx.HostBinToolVariable("soongZipCmd", "soong_zip") + pctx.HostBinToolVariable("syspropJavaCmd", "sysprop_java") + + android.PreArchMutators(func(ctx android.RegisterMutatorsContext) { + ctx.BottomUp("sysprop_deps", syspropDepsMutator).Parallel() + }) +} + +func (g *syspropJavaGenRule) GenerateAndroidBuildActions(ctx android.ModuleContext) { + var checkApiFileTimeStamp android.WritablePath + + ctx.VisitDirectDeps(func(dep android.Module) { + if m, ok := dep.(*syspropLibrary); ok { + checkApiFileTimeStamp = m.checkApiFileTimeStamp + } + }) + + for _, syspropFile := range android.PathsForModuleSrc(ctx, g.properties.Srcs) { + srcJarFile := android.GenPathWithExt(ctx, "sysprop", syspropFile, "srcjar") + + ctx.Build(pctx, android.BuildParams{ + Rule: syspropJava, + Description: "sysprop_java " + syspropFile.Rel(), + Output: srcJarFile, + Input: syspropFile, + Implicit: checkApiFileTimeStamp, + Args: map[string]string{ + "scope": g.properties.Scope, + }, + }) + + g.genSrcjars = append(g.genSrcjars, srcJarFile) + } +} + +func (g *syspropJavaGenRule) OutputFiles(tag string) (android.Paths, error) { + switch tag { + case "": + return g.genSrcjars, nil + default: + return nil, fmt.Errorf("unsupported module reference tag %q", tag) + } +} + +func syspropJavaGenFactory() android.Module { + g := &syspropJavaGenRule{} + g.AddProperties(&g.properties) + android.InitAndroidModule(g) + return g +} + type syspropLibrary struct { - java.SdkLibrary + android.ModuleBase + android.ApexModuleBase + + properties syspropLibraryProperties - commonProperties commonProperties - syspropLibraryProperties syspropLibraryProperties + checkApiFileTimeStamp android.WritablePath + latestApiFile android.Path + currentApiFile android.Path + dumpedApiFile android.WritablePath } type syspropLibraryProperties struct { @@ -41,17 +132,34 @@ type syspropLibraryProperties struct { // list of package names that will be documented and publicized as API Api_packages []string -} -type commonProperties struct { - Srcs []string - Recovery *bool + // If set to true, allow this module to be dexed and installed on devices. + Installable *bool + + // Make this module available when building for recovery Recovery_available *bool - Vendor_available *bool + + // Make this module available when building for vendor + Vendor_available *bool + + // list of .sysprop files which defines the properties. + Srcs []string `android:"path"` + + // If set to true, build a variant of the module for the host. Defaults to false. + Host_supported *bool + + // Whether public stub exists or not. + Public_stub *bool `blueprint:"mutated"` + + Cpp struct { + // Minimum sdk version that the artifact should support when it runs as part of mainline modules(APEX). + // Forwarded to cc_library.min_sdk_version + Min_sdk_version *string + } } var ( - Bool = proptools.Bool + pctx = android.NewPackageContext("android/soong/sysprop") syspropCcTag = dependencyTag{name: "syspropCc"} ) @@ -59,80 +167,322 @@ func init() { android.RegisterModuleType("sysprop_library", syspropLibraryFactory) } +func (m *syspropLibrary) Name() string { + return m.BaseModuleName() + "_sysprop_library" +} + +func (m *syspropLibrary) Owner() string { + return m.properties.Property_owner +} + func (m *syspropLibrary) CcModuleName() string { - return "lib" + m.Name() + return "lib" + m.BaseModuleName() +} + +func (m *syspropLibrary) JavaPublicStubName() string { + if proptools.Bool(m.properties.Public_stub) { + return m.BaseModuleName() + "_public" + } + return "" +} + +func (m *syspropLibrary) javaGenModuleName() string { + return m.BaseModuleName() + "_java_gen" +} + +func (m *syspropLibrary) javaGenPublicStubName() string { + return m.BaseModuleName() + "_java_gen_public" } -func (m *syspropLibrary) SyspropJavaModule() *java.SdkLibrary { - return &m.SdkLibrary +func (m *syspropLibrary) BaseModuleName() string { + return m.ModuleBase.Name() } +func (m *syspropLibrary) HasPublicStub() bool { + return proptools.Bool(m.properties.Public_stub) +} + +func (m *syspropLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) { + baseModuleName := m.BaseModuleName() + + for _, syspropFile := range android.PathsForModuleSrc(ctx, m.properties.Srcs) { + if syspropFile.Ext() != ".sysprop" { + ctx.PropertyErrorf("srcs", "srcs contains non-sysprop file %q", syspropFile.String()) + } + } + + if ctx.Failed() { + return + } + + m.currentApiFile = android.PathForSource(ctx, ctx.ModuleDir(), "api", baseModuleName+"-current.txt") + m.latestApiFile = android.PathForSource(ctx, ctx.ModuleDir(), "api", baseModuleName+"-latest.txt") + + // dump API rule + rule := android.NewRuleBuilder() + m.dumpedApiFile = android.PathForModuleOut(ctx, "api-dump.txt") + rule.Command(). + BuiltTool(ctx, "sysprop_api_dump"). + Output(m.dumpedApiFile). + Inputs(android.PathsForModuleSrc(ctx, m.properties.Srcs)) + rule.Build(pctx, ctx, baseModuleName+"_api_dump", baseModuleName+" api dump") + + // check API rule + rule = android.NewRuleBuilder() + + // 1. current.txt <-> api_dump.txt + msg := fmt.Sprintf(`\n******************************\n`+ + `API of sysprop_library %s doesn't match with current.txt\n`+ + `Please update current.txt by:\n`+ + `m %s-dump-api && rm -rf %q && cp -f %q %q\n`+ + `******************************\n`, baseModuleName, baseModuleName, + m.currentApiFile.String(), m.dumpedApiFile.String(), m.currentApiFile.String()) + + rule.Command(). + Text("( cmp").Flag("-s"). + Input(m.dumpedApiFile). + Input(m.currentApiFile). + Text("|| ( echo").Flag("-e"). + Flag(`"` + msg + `"`). + Text("; exit 38) )") + + // 2. current.txt <-> latest.txt + msg = fmt.Sprintf(`\n******************************\n`+ + `API of sysprop_library %s doesn't match with latest version\n`+ + `Please fix the breakage and rebuild.\n`+ + `******************************\n`, baseModuleName) + + rule.Command(). + Text("( "). + BuiltTool(ctx, "sysprop_api_checker"). + Input(m.latestApiFile). + Input(m.currentApiFile). + Text(" || ( echo").Flag("-e"). + Flag(`"` + msg + `"`). + Text("; exit 38) )") + + m.checkApiFileTimeStamp = android.PathForModuleOut(ctx, "check_api.timestamp") + + rule.Command(). + Text("touch"). + Output(m.checkApiFileTimeStamp) + + rule.Build(pctx, ctx, baseModuleName+"_check_api", baseModuleName+" check api") +} + +func (m *syspropLibrary) AndroidMk() android.AndroidMkData { + return android.AndroidMkData{ + Custom: func(w io.Writer, name, prefix, moduleDir string, data android.AndroidMkData) { + // sysprop_library module itself is defined as a FAKE module to perform API check. + // Actual implementation libraries are created on LoadHookMutator + fmt.Fprintln(w, "\ninclude $(CLEAR_VARS)") + fmt.Fprintf(w, "LOCAL_MODULE := %s\n", m.Name()) + fmt.Fprintf(w, "LOCAL_MODULE_CLASS := FAKE\n") + fmt.Fprintf(w, "LOCAL_MODULE_TAGS := optional\n") + fmt.Fprintf(w, "include $(BUILD_SYSTEM)/base_rules.mk\n\n") + fmt.Fprintf(w, "$(LOCAL_BUILT_MODULE): %s\n", m.checkApiFileTimeStamp.String()) + fmt.Fprintf(w, "\ttouch $@\n\n") + fmt.Fprintf(w, ".PHONY: %s-check-api %s-dump-api\n\n", name, name) + + // dump API rule + fmt.Fprintf(w, "%s-dump-api: %s\n\n", name, m.dumpedApiFile.String()) + + // check API rule + fmt.Fprintf(w, "%s-check-api: %s\n\n", name, m.checkApiFileTimeStamp.String()) + }} +} + +// sysprop_library creates schematized APIs from sysprop description files (.sysprop). +// Both Java and C++ modules can link against sysprop_library, and API stability check +// against latest APIs (see build/soong/scripts/freeze-sysprop-api-files.sh) +// is performed. func syspropLibraryFactory() android.Module { m := &syspropLibrary{} m.AddProperties( - &m.commonProperties, - &m.syspropLibraryProperties, + &m.properties, ) - m.InitSdkLibraryProperties() - m.SetNoDist() - android.InitAndroidMultiTargetsArchModule(m, android.DeviceSupported, "common") + android.InitAndroidModule(m) + android.InitApexModule(m) android.AddLoadHook(m, func(ctx android.LoadHookContext) { syspropLibraryHook(ctx, m) }) - return m } +type ccLibraryProperties struct { + Name *string + Srcs []string + Soc_specific *bool + Device_specific *bool + Product_specific *bool + Sysprop struct { + Platform *bool + } + Target struct { + Android struct { + Header_libs []string + Shared_libs []string + } + Host struct { + Static_libs []string + } + } + Required []string + Recovery *bool + Recovery_available *bool + Vendor_available *bool + Host_supported *bool + Apex_available []string + Min_sdk_version *string +} + +type javaLibraryProperties struct { + Name *string + Srcs []string + Soc_specific *bool + Device_specific *bool + Product_specific *bool + Required []string + Sdk_version *string + Installable *bool + Libs []string + Stem *string +} + func syspropLibraryHook(ctx android.LoadHookContext, m *syspropLibrary) { - if len(m.commonProperties.Srcs) == 0 { + if len(m.properties.Srcs) == 0 { ctx.PropertyErrorf("srcs", "sysprop_library must specify srcs") } - if len(m.syspropLibraryProperties.Api_packages) == 0 { - ctx.PropertyErrorf("api_packages", "sysprop_library must specify api_packages") + missing_api := false + + for _, txt := range []string{"-current.txt", "-latest.txt"} { + path := path.Join(ctx.ModuleDir(), "api", m.BaseModuleName()+txt) + file := android.ExistentPathForSource(ctx, path) + if !file.Valid() { + ctx.ModuleErrorf("API file %#v doesn't exist", path) + missing_api = true + } } - socSpecific := ctx.SocSpecific() - deviceSpecific := ctx.DeviceSpecific() - productSpecific := ctx.ProductSpecific() + if missing_api { + script := "build/soong/scripts/gen-sysprop-api-files.sh" + p := android.ExistentPathForSource(ctx, script) - owner := m.syspropLibraryProperties.Property_owner + if !p.Valid() { + panic(fmt.Sprintf("script file %s doesn't exist", script)) + } - switch owner { + ctx.ModuleErrorf("One or more api files are missing. "+ + "You can create them by:\n"+ + "%s %q %q", script, ctx.ModuleDir(), m.BaseModuleName()) + return + } + + // ctx's Platform or Specific functions represent where this sysprop_library installed. + installedInSystem := ctx.Platform() || ctx.SystemExtSpecific() + installedInVendorOrOdm := ctx.SocSpecific() || ctx.DeviceSpecific() + isOwnerPlatform := false + stub := "sysprop-library-stub-" + + switch m.Owner() { case "Platform": // Every partition can access platform-defined properties - break + stub += "platform" + isOwnerPlatform = true case "Vendor": // System can't access vendor's properties - if !socSpecific && !deviceSpecific && !productSpecific { + if installedInSystem { ctx.ModuleErrorf("None of soc_specific, device_specific, product_specific is true. " + "System can't access sysprop_library owned by Vendor") } + stub += "vendor" case "Odm": // Only vendor can access Odm-defined properties - if !socSpecific && !deviceSpecific { + if !installedInVendorOrOdm { ctx.ModuleErrorf("Neither soc_speicifc nor device_specific is true. " + "Odm-defined properties should be accessed only in Vendor or Odm") } + stub += "vendor" default: ctx.PropertyErrorf("property_owner", - "Unknown value %s: must be one of Platform, Vendor or Odm", owner) + "Unknown value %s: must be one of Platform, Vendor or Odm", m.Owner()) } - ccProps := struct { - Name *string - Soc_specific *bool - Device_specific *bool - Product_specific *bool - Sysprop struct { - Platform *bool - } - }{} - + ccProps := ccLibraryProperties{} ccProps.Name = proptools.StringPtr(m.CcModuleName()) - ccProps.Soc_specific = proptools.BoolPtr(socSpecific) - ccProps.Device_specific = proptools.BoolPtr(deviceSpecific) - ccProps.Product_specific = proptools.BoolPtr(productSpecific) - ccProps.Sysprop.Platform = proptools.BoolPtr(owner == "Platform") + ccProps.Srcs = m.properties.Srcs + ccProps.Soc_specific = proptools.BoolPtr(ctx.SocSpecific()) + ccProps.Device_specific = proptools.BoolPtr(ctx.DeviceSpecific()) + ccProps.Product_specific = proptools.BoolPtr(ctx.ProductSpecific()) + ccProps.Sysprop.Platform = proptools.BoolPtr(isOwnerPlatform) + ccProps.Target.Android.Header_libs = []string{"libbase_headers"} + ccProps.Target.Android.Shared_libs = []string{"liblog"} + ccProps.Target.Host.Static_libs = []string{"libbase", "liblog"} + ccProps.Recovery_available = m.properties.Recovery_available + ccProps.Vendor_available = m.properties.Vendor_available + ccProps.Host_supported = m.properties.Host_supported + ccProps.Apex_available = m.ApexProperties.Apex_available + ccProps.Min_sdk_version = m.properties.Cpp.Min_sdk_version + ctx.CreateModule(cc.LibraryFactory, &ccProps) + + scope := "internal" + + // We need to only use public version, if the partition where sysprop_library will be installed + // is different from owner. + + if ctx.ProductSpecific() { + // Currently product partition can't own any sysprop_library. + scope = "public" + } else if isOwnerPlatform && installedInVendorOrOdm { + // Vendor or Odm should use public version of Platform's sysprop_library. + scope = "public" + } + + ctx.CreateModule(syspropJavaGenFactory, &syspropGenProperties{ + Srcs: m.properties.Srcs, + Scope: scope, + Name: proptools.StringPtr(m.javaGenModuleName()), + }) + + ctx.CreateModule(java.LibraryFactory, &javaLibraryProperties{ + Name: proptools.StringPtr(m.BaseModuleName()), + Srcs: []string{":" + m.javaGenModuleName()}, + Soc_specific: proptools.BoolPtr(ctx.SocSpecific()), + Device_specific: proptools.BoolPtr(ctx.DeviceSpecific()), + Product_specific: proptools.BoolPtr(ctx.ProductSpecific()), + Installable: m.properties.Installable, + Sdk_version: proptools.StringPtr("core_current"), + Libs: []string{stub}, + }) - ctx.CreateModule(android.ModuleFactoryAdaptor(cc.LibraryFactory), &m.commonProperties, &ccProps) + // if platform sysprop_library is installed in /system or /system-ext, we regard it as an API + // and allow any modules (even from different partition) to link against the sysprop_library. + // To do that, we create a public stub and expose it to modules with sdk_version: system_*. + if isOwnerPlatform && installedInSystem { + m.properties.Public_stub = proptools.BoolPtr(true) + ctx.CreateModule(syspropJavaGenFactory, &syspropGenProperties{ + Srcs: m.properties.Srcs, + Scope: "public", + Name: proptools.StringPtr(m.javaGenPublicStubName()), + }) + + ctx.CreateModule(java.LibraryFactory, &javaLibraryProperties{ + Name: proptools.StringPtr(m.JavaPublicStubName()), + Srcs: []string{":" + m.javaGenPublicStubName()}, + Installable: proptools.BoolPtr(false), + Sdk_version: proptools.StringPtr("core_current"), + Libs: []string{stub}, + Stem: proptools.StringPtr(m.BaseModuleName()), + }) + } +} + +func syspropDepsMutator(ctx android.BottomUpMutatorContext) { + if m, ok := ctx.Module().(*syspropLibrary); ok { + ctx.AddReverseDependency(m, nil, m.javaGenModuleName()) + + if proptools.Bool(m.properties.Public_stub) { + ctx.AddReverseDependency(m, nil, m.javaGenPublicStubName()) + } + } } diff --git a/sysprop/sysprop_test.go b/sysprop/sysprop_test.go index 07406b3c8..850338675 100644 --- a/sysprop/sysprop_test.go +++ b/sysprop/sysprop_test.go @@ -15,6 +15,8 @@ package sysprop import ( + "reflect" + "android/soong/android" "android/soong/cc" "android/soong/java" @@ -24,6 +26,7 @@ import ( "strings" "testing" + "github.com/google/blueprint" "github.com/google/blueprint/proptools" ) @@ -52,92 +55,59 @@ func TestMain(m *testing.M) { os.Exit(run()) } -func testContext(config android.Config, bp string, - fs map[string][]byte) *android.TestContext { +func testContext(config android.Config) *android.TestContext { ctx := android.NewTestArchContext() - ctx.RegisterModuleType("android_app", android.ModuleFactoryAdaptor(java.AndroidAppFactory)) - ctx.RegisterModuleType("droiddoc_template", android.ModuleFactoryAdaptor(java.ExportedDroiddocDirFactory)) - ctx.RegisterModuleType("java_library", android.ModuleFactoryAdaptor(java.LibraryFactory)) - ctx.RegisterModuleType("java_system_modules", android.ModuleFactoryAdaptor(java.SystemModulesFactory)) - ctx.RegisterModuleType("prebuilt_apis", android.ModuleFactoryAdaptor(java.PrebuiltApisFactory)) - ctx.PreArchMutators(func(ctx android.RegisterMutatorsContext) { - ctx.TopDown("load_hooks", android.LoadHookMutator).Parallel() - }) - ctx.PreArchMutators(android.RegisterPrebuiltsPreArchMutators) - ctx.PreArchMutators(android.RegisterPrebuiltsPostDepsMutators) + java.RegisterJavaBuildComponents(ctx) + java.RegisterAppBuildComponents(ctx) + java.RegisterSystemModulesBuildComponents(ctx) + ctx.PreArchMutators(android.RegisterDefaultsPreArchMutators) ctx.PreArchMutators(func(ctx android.RegisterMutatorsContext) { - ctx.TopDown("prebuilt_apis", java.PrebuiltApisMutator).Parallel() - ctx.TopDown("java_sdk_library", java.SdkLibraryMutator).Parallel() + ctx.BottomUp("sysprop_deps", syspropDepsMutator).Parallel() }) - ctx.RegisterModuleType("cc_library", android.ModuleFactoryAdaptor(cc.LibraryFactory)) - ctx.RegisterModuleType("cc_library_headers", android.ModuleFactoryAdaptor(cc.LibraryHeaderFactory)) - ctx.RegisterModuleType("cc_library_static", android.ModuleFactoryAdaptor(cc.LibraryFactory)) - ctx.RegisterModuleType("cc_object", android.ModuleFactoryAdaptor(cc.ObjectFactory)) - ctx.RegisterModuleType("llndk_library", android.ModuleFactoryAdaptor(cc.LlndkLibraryFactory)) - ctx.RegisterModuleType("toolchain_library", android.ModuleFactoryAdaptor(cc.ToolchainLibraryFactory)) + cc.RegisterRequiredBuildComponentsForTest(ctx) ctx.PreDepsMutators(func(ctx android.RegisterMutatorsContext) { - ctx.BottomUp("image", cc.ImageMutator).Parallel() - ctx.BottomUp("link", cc.LinkageMutator).Parallel() - ctx.BottomUp("vndk", cc.VndkMutator).Parallel() - ctx.BottomUp("version", cc.VersionMutator).Parallel() - ctx.BottomUp("begin", cc.BeginMutator).Parallel() - ctx.BottomUp("sysprop", cc.SyspropMutator).Parallel() + ctx.BottomUp("sysprop_java", java.SyspropMutator).Parallel() }) - ctx.RegisterModuleType("sysprop_library", android.ModuleFactoryAdaptor(syspropLibraryFactory)) + ctx.RegisterModuleType("sysprop_library", syspropLibraryFactory) - ctx.Register() + ctx.Register(config) - bp += java.GatherRequiredDepsForTest() + return ctx +} + +func run(t *testing.T, ctx *android.TestContext, config android.Config) { + t.Helper() + _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) + android.FailIfErrored(t, errs) + _, errs = ctx.PrepareBuildActions(config) + android.FailIfErrored(t, errs) +} + +func testConfig(env map[string]string, bp string, fs map[string][]byte) android.Config { bp += cc.GatherRequiredDepsForTest(android.Android) mockFS := map[string][]byte{ - "Android.bp": []byte(bp), - "a.java": nil, - "b.java": nil, - "c.java": nil, - "d.cpp": nil, - "api/current.txt": nil, - "api/removed.txt": nil, - "api/system-current.txt": nil, - "api/system-removed.txt": nil, - "api/test-current.txt": nil, - "api/test-removed.txt": nil, - "framework/aidl/a.aidl": nil, - - "prebuilts/sdk/current/core/android.jar": nil, - "prebuilts/sdk/current/public/android.jar": nil, - "prebuilts/sdk/current/public/framework.aidl": nil, - "prebuilts/sdk/current/public/core.jar": nil, - "prebuilts/sdk/current/system/android.jar": nil, - "prebuilts/sdk/current/test/android.jar": nil, - "prebuilts/sdk/28/public/api/sysprop-platform.txt": nil, - "prebuilts/sdk/28/system/api/sysprop-platform.txt": nil, - "prebuilts/sdk/28/test/api/sysprop-platform.txt": nil, - "prebuilts/sdk/28/public/api/sysprop-platform-removed.txt": nil, - "prebuilts/sdk/28/system/api/sysprop-platform-removed.txt": nil, - "prebuilts/sdk/28/test/api/sysprop-platform-removed.txt": nil, - "prebuilts/sdk/28/public/api/sysprop-platform-on-product.txt": nil, - "prebuilts/sdk/28/system/api/sysprop-platform-on-product.txt": nil, - "prebuilts/sdk/28/test/api/sysprop-platform-on-product.txt": nil, - "prebuilts/sdk/28/public/api/sysprop-platform-on-product-removed.txt": nil, - "prebuilts/sdk/28/system/api/sysprop-platform-on-product-removed.txt": nil, - "prebuilts/sdk/28/test/api/sysprop-platform-on-product-removed.txt": nil, - "prebuilts/sdk/28/public/api/sysprop-vendor.txt": nil, - "prebuilts/sdk/28/system/api/sysprop-vendor.txt": nil, - "prebuilts/sdk/28/test/api/sysprop-vendor.txt": nil, - "prebuilts/sdk/28/public/api/sysprop-vendor-removed.txt": nil, - "prebuilts/sdk/28/system/api/sysprop-vendor-removed.txt": nil, - "prebuilts/sdk/28/test/api/sysprop-vendor-removed.txt": nil, - "prebuilts/sdk/tools/core-lambda-stubs.jar": nil, - "prebuilts/sdk/Android.bp": []byte(`prebuilt_apis { name: "sdk", api_dirs: ["28", "current"],}`), + "a.java": nil, + "b.java": nil, + "c.java": nil, + "d.cpp": nil, + "api/sysprop-platform-current.txt": nil, + "api/sysprop-platform-latest.txt": nil, + "api/sysprop-platform-on-product-current.txt": nil, + "api/sysprop-platform-on-product-latest.txt": nil, + "api/sysprop-vendor-current.txt": nil, + "api/sysprop-vendor-latest.txt": nil, + "api/sysprop-odm-current.txt": nil, + "api/sysprop-odm-latest.txt": nil, + "framework/aidl/a.aidl": nil, // For framework-res, which is an implicit dependency for framework - "AndroidManifest.xml": nil, - "build/target/product/security/testkey": nil, + "AndroidManifest.xml": nil, + "build/make/target/product/security/testkey": nil, "build/soong/scripts/jar-wrapper.sh": nil, @@ -159,27 +129,14 @@ func testContext(config android.Config, bp string, "android/sysprop/PlatformProperties.sysprop": nil, "com/android/VendorProperties.sysprop": nil, + "com/android2/OdmProperties.sysprop": nil, } for k, v := range fs { mockFS[k] = v } - ctx.MockFileSystem(mockFS) - - return ctx -} - -func run(t *testing.T, ctx *android.TestContext, config android.Config) { - t.Helper() - _, errs := ctx.ParseFileList(".", []string{"Android.bp", "prebuilts/sdk/Android.bp"}) - android.FailIfErrored(t, errs) - _, errs = ctx.PrepareBuildActions(config) - android.FailIfErrored(t, errs) -} - -func testConfig(env map[string]string) android.Config { - config := java.TestConfig(buildDir, env) + config := java.TestConfig(buildDir, env, bp, mockFS) config.TestProductVariables.DeviceSystemSdkVersions = []string{"28"} config.TestProductVariables.DeviceVndkVersion = proptools.StringPtr("current") @@ -191,8 +148,8 @@ func testConfig(env map[string]string) android.Config { func test(t *testing.T, bp string) *android.TestContext { t.Helper() - config := testConfig(nil) - ctx := testContext(config, bp, nil) + config := testConfig(nil, bp, nil) + ctx := testContext(config) run(t, ctx, config) return ctx @@ -202,10 +159,12 @@ func TestSyspropLibrary(t *testing.T) { ctx := test(t, ` sysprop_library { name: "sysprop-platform", + apex_available: ["//apex_available:platform"], srcs: ["android/sysprop/PlatformProperties.sysprop"], api_packages: ["android.sysprop"], property_owner: "Platform", vendor_available: true, + host_supported: true, } sysprop_library { @@ -225,6 +184,14 @@ func TestSyspropLibrary(t *testing.T) { vendor_available: true, } + sysprop_library { + name: "sysprop-odm", + srcs: ["com/android2/OdmProperties.sysprop"], + api_packages: ["com.android2"], + property_owner: "Odm", + device_specific: true, + } + java_library { name: "java-platform", srcs: ["c.java"], @@ -233,6 +200,13 @@ func TestSyspropLibrary(t *testing.T) { } java_library { + name: "java-platform-private", + srcs: ["c.java"], + platform_apis: true, + libs: ["sysprop-platform"], + } + + java_library { name: "java-product", srcs: ["c.java"], sdk_version: "system_current", @@ -274,6 +248,11 @@ func TestSyspropLibrary(t *testing.T) { static_libs: ["sysprop-platform", "sysprop-vendor"], } + cc_library { + name: "libbase", + host_supported: true, + } + cc_library_headers { name: "libbase_headers", vendor_available: true, @@ -282,48 +261,80 @@ func TestSyspropLibrary(t *testing.T) { cc_library { name: "liblog", - no_libgcc: true, + no_libcrt: true, nocrt: true, system_shared_libs: [], recovery_available: true, + host_supported: true, + } + + cc_binary_host { + name: "hostbin", + static_libs: ["sysprop-platform"], } llndk_library { name: "liblog", symbol_file: "", } + + java_library { + name: "sysprop-library-stub-platform", + sdk_version: "core_current", + } + + java_library { + name: "sysprop-library-stub-vendor", + soc_specific: true, + sdk_version: "core_current", + } `) + // Check for generated cc_library for _, variant := range []string{ - "android_arm_armv7-a-neon_core_shared", - "android_arm_armv7-a-neon_core_static", - "android_arm_armv7-a-neon_vendor_shared", - "android_arm_armv7-a-neon_vendor_static", - "android_arm64_armv8-a_core_shared", - "android_arm64_armv8-a_core_static", - "android_arm64_armv8-a_vendor_shared", - "android_arm64_armv8-a_vendor_static", + "android_vendor.VER_arm_armv7-a-neon_shared", + "android_vendor.VER_arm_armv7-a-neon_static", + "android_vendor.VER_arm64_armv8-a_shared", + "android_vendor.VER_arm64_armv8-a_static", } { - // Check for generated cc_library ctx.ModuleForTests("libsysprop-platform", variant) ctx.ModuleForTests("libsysprop-vendor", variant) + ctx.ModuleForTests("libsysprop-odm", variant) + } + + for _, variant := range []string{ + "android_arm_armv7-a-neon_shared", + "android_arm_armv7-a-neon_static", + "android_arm64_armv8-a_shared", + "android_arm64_armv8-a_static", + } { + library := ctx.ModuleForTests("libsysprop-platform", variant).Module().(*cc.Module) + expectedApexAvailableOnLibrary := []string{"//apex_available:platform"} + if !reflect.DeepEqual(library.ApexProperties.Apex_available, expectedApexAvailableOnLibrary) { + t.Errorf("apex available property on libsysprop-platform must be %#v, but was %#v.", + expectedApexAvailableOnLibrary, library.ApexProperties.Apex_available) + } + + // core variant of vendor-owned sysprop_library is for product + ctx.ModuleForTests("libsysprop-vendor", variant) } ctx.ModuleForTests("sysprop-platform", "android_common") + ctx.ModuleForTests("sysprop-platform_public", "android_common") ctx.ModuleForTests("sysprop-vendor", "android_common") // Check for exported includes - coreVariant := "android_arm64_armv8-a_core_static" - vendorVariant := "android_arm64_armv8-a_vendor_static" + coreVariant := "android_arm64_armv8-a_static" + vendorVariant := "android_vendor.VER_arm64_armv8-a_static" - platformInternalPath := "libsysprop-platform/android_arm64_armv8-a_core_static/gen/sysprop/include" - platformSystemCorePath := "libsysprop-platform/android_arm64_armv8-a_core_static/gen/sysprop/system/include" - platformSystemVendorPath := "libsysprop-platform/android_arm64_armv8-a_vendor_static/gen/sysprop/system/include" + platformInternalPath := "libsysprop-platform/android_arm64_armv8-a_static/gen/sysprop/include" + platformPublicCorePath := "libsysprop-platform/android_arm64_armv8-a_static/gen/sysprop/public/include" + platformPublicVendorPath := "libsysprop-platform/android_vendor.VER_arm64_armv8-a_static/gen/sysprop/public/include" - platformOnProductPath := "libsysprop-platform-on-product/android_arm64_armv8-a_core_static/gen/sysprop/system/include" + platformOnProductPath := "libsysprop-platform-on-product/android_arm64_armv8-a_static/gen/sysprop/public/include" - vendorInternalPath := "libsysprop-vendor/android_arm64_armv8-a_vendor_static/gen/sysprop/include" - vendorSystemPath := "libsysprop-vendor/android_arm64_armv8-a_core_static/gen/sysprop/system/include" + vendorInternalPath := "libsysprop-vendor/android_vendor.VER_arm64_armv8-a_static/gen/sysprop/include" + vendorPublicPath := "libsysprop-vendor/android_arm64_armv8-a_static/gen/sysprop/public/include" platformClient := ctx.ModuleForTests("cc-client-platform", coreVariant) platformFlags := platformClient.Rule("cc").Args["cFlags"] @@ -346,20 +357,33 @@ func TestSyspropLibrary(t *testing.T) { productClient := ctx.ModuleForTests("cc-client-product", coreVariant) productFlags := productClient.Rule("cc").Args["cFlags"] - // Product should use platform's and vendor's system headers + // Product should use platform's and vendor's public headers if !strings.Contains(productFlags, platformOnProductPath) || - !strings.Contains(productFlags, vendorSystemPath) { + !strings.Contains(productFlags, vendorPublicPath) { t.Errorf("flags for product must contain %#v and %#v, but was %#v.", - platformSystemCorePath, vendorSystemPath, productFlags) + platformPublicCorePath, vendorPublicPath, productFlags) } vendorClient := ctx.ModuleForTests("cc-client-vendor", vendorVariant) vendorFlags := vendorClient.Rule("cc").Args["cFlags"] - // Vendor should use platform's system header and vendor's internal header - if !strings.Contains(vendorFlags, platformSystemVendorPath) || + // Vendor should use platform's public header and vendor's internal header + if !strings.Contains(vendorFlags, platformPublicVendorPath) || !strings.Contains(vendorFlags, vendorInternalPath) { t.Errorf("flags for vendor must contain %#v and %#v, but was %#v.", - platformSystemVendorPath, vendorInternalPath, vendorFlags) + platformPublicVendorPath, vendorInternalPath, vendorFlags) } + + // Java modules linking against system API should use public stub + javaSystemApiClient := ctx.ModuleForTests("java-platform", "android_common") + publicStubFound := false + ctx.VisitDirectDeps(javaSystemApiClient.Module(), func(dep blueprint.Module) { + if dep.Name() == "sysprop-platform_public" { + publicStubFound = true + } + }) + if !publicStubFound { + t.Errorf("system api client should use public stub") + } + } diff --git a/tradefed/Android.bp b/tradefed/Android.bp new file mode 100644 index 000000000..6e5e5330e --- /dev/null +++ b/tradefed/Android.bp @@ -0,0 +1,14 @@ +bootstrap_go_package { + name: "soong-tradefed", + pkgPath: "android/soong/tradefed", + deps: [ + "blueprint", + "soong-android", + ], + srcs: [ + "autogen.go", + "config.go", + "makevars.go", + ], + pluginFor: ["soong_build"], +} diff --git a/tradefed/autogen.go b/tradefed/autogen.go index 952b02236..28291465b 100644 --- a/tradefed/autogen.go +++ b/tradefed/autogen.go @@ -24,6 +24,8 @@ import ( "android/soong/android" ) +const test_xml_indent = " " + func getTestConfigTemplate(ctx android.ModuleContext, prop *string) android.OptionalPath { return ctx.ExpandOptionalSource(prop, "test_config_template") } @@ -38,20 +40,20 @@ func getTestConfig(ctx android.ModuleContext, prop *string) android.Path { } var autogenTestConfig = pctx.StaticRule("autogenTestConfig", blueprint.RuleParams{ - Command: "sed 's&{MODULE}&${name}&g;s&{EXTRA_CONFIGS}&'${extraConfigs}'&g' $template > $out", + Command: "sed 's&{MODULE}&${name}&g;s&{EXTRA_CONFIGS}&'${extraConfigs}'&g;s&{OUTPUT_FILENAME}&'${outputFileName}'&g' $template > $out", CommandDeps: []string{"$template"}, -}, "name", "template", "extraConfigs") +}, "name", "template", "extraConfigs", "outputFileName") -func testConfigPath(ctx android.ModuleContext, prop *string, testSuites []string) (path android.Path, autogenPath android.WritablePath) { - if p := getTestConfig(ctx, prop); p != nil { +func testConfigPath(ctx android.ModuleContext, prop *string, testSuites []string, autoGenConfig *bool, testConfigTemplateProp *string) (path android.Path, autogenPath android.WritablePath) { + p := getTestConfig(ctx, prop) + if !Bool(autoGenConfig) && p != nil { return p, nil - } else if !android.InList("cts", testSuites) { + } else if BoolDefault(autoGenConfig, true) && (!android.InList("cts", testSuites) || testConfigTemplateProp != nil) { outputFile := android.PathForModuleOut(ctx, ctx.ModuleName()+".config") return nil, outputFile } else { // CTS modules can be used for test data, so test config files must be - // explicitly created using AndroidTest.xml - // TODO(b/112602712): remove the path check + // explicitly created using AndroidTest.xml or test_config_template. return nil, nil } } @@ -62,31 +64,63 @@ type Config interface { type Option struct { Name string + Key string Value string } var _ Config = Option{} func (o Option) Config() string { + if o.Key != "" { + return fmt.Sprintf(`<option name="%s" key="%s" value="%s" />`, o.Name, o.Key, o.Value) + } return fmt.Sprintf(`<option name="%s" value="%s" />`, o.Name, o.Value) } -type Preparer struct { - Class string +// It can be a template of object or target_preparer. +type Object struct { + // Set it as a target_preparer if object type == "target_preparer". + Type string + Class string + Options []Option } -var _ Config = Preparer{} +var _ Config = Object{} + +func (ob Object) Config() string { + var optionStrings []string + for _, option := range ob.Options { + optionStrings = append(optionStrings, option.Config()) + } + var options string + if len(ob.Options) == 0 { + options = "" + } else { + optionDelimiter := fmt.Sprintf("\\n%s%s", test_xml_indent, test_xml_indent) + options = optionDelimiter + strings.Join(optionStrings, optionDelimiter) + } + if ob.Type == "target_preparer" { + return fmt.Sprintf(`<target_preparer class="%s">%s\n%s</target_preparer>`, ob.Class, options, test_xml_indent) + } else { + return fmt.Sprintf(`<object type="%s" class="%s">%s\n%s</object>`, ob.Type, ob.Class, options, test_xml_indent) + } -func (p Preparer) Config() string { - return fmt.Sprintf(`<target_preparer class="%s" />`, p.Class) } func autogenTemplate(ctx android.ModuleContext, output android.WritablePath, template string, configs []Config) { + autogenTemplateWithNameAndOutputFile(ctx, ctx.ModuleName(), output, template, configs, "") +} + +func autogenTemplateWithName(ctx android.ModuleContext, name string, output android.WritablePath, template string, configs []Config) { + autogenTemplateWithNameAndOutputFile(ctx, ctx.ModuleName(), output, template, configs, "") +} + +func autogenTemplateWithNameAndOutputFile(ctx android.ModuleContext, name string, output android.WritablePath, template string, configs []Config, outputFileName string) { var configStrings []string for _, config := range configs { configStrings = append(configStrings, config.Config()) } - extraConfigs := strings.Join(configStrings, "\n ") + extraConfigs := strings.Join(configStrings, fmt.Sprintf("\\n%s", test_xml_indent)) extraConfigs = proptools.NinjaAndShellEscape(extraConfigs) ctx.Build(pctx, android.BuildParams{ @@ -94,16 +128,17 @@ func autogenTemplate(ctx android.ModuleContext, output android.WritablePath, tem Description: "test config", Output: output, Args: map[string]string{ - "name": ctx.ModuleName(), - "template": template, - "extraConfigs": extraConfigs, + "name": name, + "template": template, + "extraConfigs": extraConfigs, + "outputFileName": outputFileName, }, }) } func AutoGenNativeTestConfig(ctx android.ModuleContext, testConfigProp *string, - testConfigTemplateProp *string, testSuites []string, config []Config) android.Path { - path, autogenPath := testConfigPath(ctx, testConfigProp, testSuites) + testConfigTemplateProp *string, testSuites []string, config []Config, autoGenConfig *bool) android.Path { + path, autogenPath := testConfigPath(ctx, testConfigProp, testSuites, autoGenConfig, testConfigTemplateProp) if autogenPath != nil { templatePath := getTestConfigTemplate(ctx, testConfigTemplateProp) if templatePath.Valid() { @@ -120,9 +155,24 @@ func AutoGenNativeTestConfig(ctx android.ModuleContext, testConfigProp *string, return path } +func AutoGenShellTestConfig(ctx android.ModuleContext, testConfigProp *string, + testConfigTemplateProp *string, testSuites []string, config []Config, autoGenConfig *bool, outputFileName string) android.Path { + path, autogenPath := testConfigPath(ctx, testConfigProp, testSuites, autoGenConfig, testConfigTemplateProp) + if autogenPath != nil { + templatePath := getTestConfigTemplate(ctx, testConfigTemplateProp) + if templatePath.Valid() { + autogenTemplateWithNameAndOutputFile(ctx, ctx.ModuleName(), autogenPath, templatePath.String(), config, outputFileName) + } else { + autogenTemplateWithNameAndOutputFile(ctx, ctx.ModuleName(), autogenPath, "${ShellTestConfigTemplate}", config, outputFileName) + } + return autogenPath + } + return path +} + func AutoGenNativeBenchmarkTestConfig(ctx android.ModuleContext, testConfigProp *string, - testConfigTemplateProp *string, testSuites []string, configs []Config) android.Path { - path, autogenPath := testConfigPath(ctx, testConfigProp, testSuites) + testConfigTemplateProp *string, testSuites []string, configs []Config, autoGenConfig *bool) android.Path { + path, autogenPath := testConfigPath(ctx, testConfigProp, testSuites, autoGenConfig, testConfigTemplateProp) if autogenPath != nil { templatePath := getTestConfigTemplate(ctx, testConfigTemplateProp) if templatePath.Valid() { @@ -135,8 +185,9 @@ func AutoGenNativeBenchmarkTestConfig(ctx android.ModuleContext, testConfigProp return path } -func AutoGenJavaTestConfig(ctx android.ModuleContext, testConfigProp *string, testConfigTemplateProp *string, testSuites []string) android.Path { - path, autogenPath := testConfigPath(ctx, testConfigProp, testSuites) +func AutoGenJavaTestConfig(ctx android.ModuleContext, testConfigProp *string, testConfigTemplateProp *string, + testSuites []string, autoGenConfig *bool) android.Path { + path, autogenPath := testConfigPath(ctx, testConfigProp, testSuites, autoGenConfig, testConfigTemplateProp) if autogenPath != nil { templatePath := getTestConfigTemplate(ctx, testConfigTemplateProp) if templatePath.Valid() { @@ -154,9 +205,9 @@ func AutoGenJavaTestConfig(ctx android.ModuleContext, testConfigProp *string, te } func AutoGenPythonBinaryHostTestConfig(ctx android.ModuleContext, testConfigProp *string, - testConfigTemplateProp *string, testSuites []string) android.Path { + testConfigTemplateProp *string, testSuites []string, autoGenConfig *bool) android.Path { - path, autogenPath := testConfigPath(ctx, testConfigProp, testSuites) + path, autogenPath := testConfigPath(ctx, testConfigProp, testSuites, autoGenConfig, testConfigTemplateProp) if autogenPath != nil { templatePath := getTestConfigTemplate(ctx, testConfigTemplateProp) if templatePath.Valid() { @@ -169,34 +220,64 @@ func AutoGenPythonBinaryHostTestConfig(ctx android.ModuleContext, testConfigProp return path } +func AutoGenRustTestConfig(ctx android.ModuleContext, name string, testConfigProp *string, + testConfigTemplateProp *string, testSuites []string, autoGenConfig *bool) android.Path { + path, autogenPath := testConfigPath(ctx, testConfigProp, testSuites, autoGenConfig, testConfigTemplateProp) + if autogenPath != nil { + templatePathString := "${RustHostTestConfigTemplate}" + if ctx.Device() { + templatePathString = "${RustDeviceTestConfigTemplate}" + } + templatePath := getTestConfigTemplate(ctx, testConfigTemplateProp) + if templatePath.Valid() { + templatePathString = templatePath.String() + } + autogenTemplateWithName(ctx, name, autogenPath, templatePathString, nil) + return autogenPath + } + return path +} + var autogenInstrumentationTest = pctx.StaticRule("autogenInstrumentationTest", blueprint.RuleParams{ - Command: "${AutoGenTestConfigScript} $out $in ${EmptyTestConfig} $template", + Command: "${AutoGenTestConfigScript} $out $in ${EmptyTestConfig} $template ${extraConfigs}", CommandDeps: []string{ "${AutoGenTestConfigScript}", "${EmptyTestConfig}", "$template", }, -}, "name", "template") +}, "name", "template", "extraConfigs") -func AutoGenInstrumentationTestConfig(ctx android.ModuleContext, testConfigProp *string, testConfigTemplateProp *string, manifest android.Path, testSuites []string) android.Path { - path, autogenPath := testConfigPath(ctx, testConfigProp, testSuites) +func AutoGenInstrumentationTestConfig(ctx android.ModuleContext, testConfigProp *string, + testConfigTemplateProp *string, manifest android.Path, testSuites []string, autoGenConfig *bool, configs []Config) android.Path { + path, autogenPath := testConfigPath(ctx, testConfigProp, testSuites, autoGenConfig, testConfigTemplateProp) + var configStrings []string if autogenPath != nil { template := "${InstrumentationTestConfigTemplate}" moduleTemplate := getTestConfigTemplate(ctx, testConfigTemplateProp) if moduleTemplate.Valid() { template = moduleTemplate.String() } + for _, config := range configs { + configStrings = append(configStrings, config.Config()) + } + extraConfigs := strings.Join(configStrings, fmt.Sprintf("\\n%s", test_xml_indent)) + extraConfigs = fmt.Sprintf("--extra-configs '%s'", extraConfigs) + ctx.Build(pctx, android.BuildParams{ Rule: autogenInstrumentationTest, Description: "test config", Input: manifest, Output: autogenPath, Args: map[string]string{ - "name": ctx.ModuleName(), - "template": template, + "name": ctx.ModuleName(), + "template": template, + "extraConfigs": extraConfigs, }, }) return autogenPath } return path } + +var Bool = proptools.Bool +var BoolDefault = proptools.BoolDefault diff --git a/tradefed/config.go b/tradefed/config.go index 141e0c570..34195c364 100644 --- a/tradefed/config.go +++ b/tradefed/config.go @@ -31,6 +31,9 @@ func init() { pctx.SourcePathVariable("NativeHostTestConfigTemplate", "build/make/core/native_host_test_config_template.xml") pctx.SourcePathVariable("NativeTestConfigTemplate", "build/make/core/native_test_config_template.xml") pctx.SourcePathVariable("PythonBinaryHostTestConfigTemplate", "build/make/core/python_binary_host_test_config_template.xml") + pctx.SourcePathVariable("RustDeviceTestConfigTemplate", "build/make/core/rust_device_test_config_template.xml") + pctx.SourcePathVariable("RustHostTestConfigTemplate", "build/make/core/rust_host_test_config_template.xml") + pctx.SourcePathVariable("ShellTestConfigTemplate", "build/make/core/shell_test_config_template.xml") pctx.SourcePathVariable("EmptyTestConfig", "build/make/core/empty_test_config.xml") } diff --git a/tradefed/makevars.go b/tradefed/makevars.go index aad7273ec..f9682e448 100644 --- a/tradefed/makevars.go +++ b/tradefed/makevars.go @@ -31,6 +31,9 @@ func makeVarsProvider(ctx android.MakeVarsContext) { ctx.Strict("NATIVE_HOST_TEST_CONFIG_TEMPLATE", "${NativeHostTestConfigTemplate}") ctx.Strict("NATIVE_TEST_CONFIG_TEMPLATE", "${NativeTestConfigTemplate}") ctx.Strict("PYTHON_BINARY_HOST_TEST_CONFIG_TEMPLATE", "${PythonBinaryHostTestConfigTemplate}") + ctx.Strict("RUST_DEVICE_TEST_CONFIG_TEMPLATE", "${RustDeviceTestConfigTemplate}") + ctx.Strict("RUST_HOST_TEST_CONFIG_TEMPLATE", "${RustHostTestConfigTemplate}") + ctx.Strict("SHELL_TEST_CONFIG_TEMPLATE", "${ShellTestConfigTemplate}") ctx.Strict("EMPTY_TEST_CONFIG", "${EmptyTestConfig}") } diff --git a/ui/build/Android.bp b/ui/build/Android.bp index cb2b76c2e..4ef27212a 100644 --- a/ui/build/Android.bp +++ b/ui/build/Android.bp @@ -60,6 +60,7 @@ bootstrap_go_package { "util.go", ], testSrcs: [ + "cleanbuild_test.go", "config_test.go", "environment_test.go", "rbe_test.go", @@ -69,11 +70,13 @@ bootstrap_go_package { ], darwin: { srcs: [ + "config_darwin.go", "sandbox_darwin.go", ], }, linux: { srcs: [ + "config_linux.go", "sandbox_linux.go", ], }, diff --git a/ui/build/build.go b/ui/build/build.go index 58b6da94e..349a7de5e 100644 --- a/ui/build/build.go +++ b/ui/build/build.go @@ -19,6 +19,8 @@ import ( "os" "path/filepath" "text/template" + + "android/soong/ui/metrics" ) // Ensures the out directory exists, and has the proper files to prevent kati @@ -33,12 +35,24 @@ func SetupOutDir(ctx Context, config Config) { // can be parsed as ninja output. ensureEmptyFileExists(ctx, filepath.Join(config.OutDir(), "ninja_build")) ensureEmptyFileExists(ctx, filepath.Join(config.OutDir(), ".out-dir")) + + if buildDateTimeFile, ok := config.environ.Get("BUILD_DATETIME_FILE"); ok { + err := ioutil.WriteFile(buildDateTimeFile, []byte(config.buildDateTime), 0777) + if err != nil { + ctx.Fatalln("Failed to write BUILD_DATETIME to file:", err) + } + } else { + ctx.Fatalln("Missing BUILD_DATETIME_FILE") + } } var combinedBuildNinjaTemplate = template.Must(template.New("combined").Parse(` builddir = {{.OutDir}} -pool local_pool +{{if .UseRemoteBuild }}pool local_pool depth = {{.Parallel}} +{{end -}} +pool highmem_pool + depth = {{.HighmemParallel}} build _kati_always_build_: phony {{if .HasKatiSuffix}}subninja {{.KatiBuildNinjaFile}} subninja {{.KatiPackageNinjaFile}} @@ -129,6 +143,29 @@ func Build(ctx Context, config Config, what int) { ctx.Verboseln("Starting build with args:", config.Arguments()) ctx.Verboseln("Environment:", config.Environment().Environ()) + if totalRAM := config.TotalRAM(); totalRAM != 0 { + ram := float32(totalRAM) / (1024 * 1024 * 1024) + ctx.Verbosef("Total RAM: %.3vGB", ram) + + if ram <= 16 { + ctx.Println("************************************************************") + ctx.Printf("You are building on a machine with %.3vGB of RAM\n", ram) + ctx.Println("") + ctx.Println("The minimum required amount of free memory is around 16GB,") + ctx.Println("and even with that, some configurations may not work.") + ctx.Println("") + ctx.Println("If you run into segfaults or other errors, try reducing your") + ctx.Println("-j value.") + ctx.Println("************************************************************") + } else if ram <= float32(config.Parallel()) { + ctx.Printf("Warning: high -j%d count compared to %.3vGB of RAM", config.Parallel(), ram) + ctx.Println("If you run into segfaults or other errors, try a lower -j value") + } + } + + ctx.BeginTrace(metrics.Total, "total") + defer ctx.EndTrace() + if config.SkipMake() { ctx.Verboseln("Skipping Make/Kati as requested") what = what & (BuildSoong | BuildNinja) @@ -171,11 +208,13 @@ func Build(ctx Context, config Config, what int) { runMakeProductConfig(ctx, config) } - if inList("installclean", config.Arguments()) { + if inList("installclean", config.Arguments()) || + inList("install-clean", config.Arguments()) { installClean(ctx, config, what) ctx.Println("Deleted images and staging directories.") return - } else if inList("dataclean", config.Arguments()) { + } else if inList("dataclean", config.Arguments()) || + inList("data-clean", config.Arguments()) { dataClean(ctx, config, what) ctx.Println("Deleted data files.") return diff --git a/ui/build/cleanbuild.go b/ui/build/cleanbuild.go index c47f614d6..0bcdccb7b 100644 --- a/ui/build/cleanbuild.go +++ b/ui/build/cleanbuild.go @@ -15,10 +15,12 @@ package build import ( + "bytes" "fmt" "io/ioutil" "os" "path/filepath" + "sort" "strings" "android/soong/ui/metrics" @@ -87,6 +89,7 @@ func installClean(ctx Context, config Config, what int) { // otherwise we'd have to rebuild any generated files created with // those tools. removeGlobs(ctx, + hostOut("apex"), hostOut("obj/NOTICE_FILES"), hostOut("obj/PACKAGING"), hostOut("coverage"), @@ -96,22 +99,30 @@ func installClean(ctx Context, config Config, what int) { hostOut("sdk_addon"), hostOut("testcases"), hostOut("vts"), + hostOut("vts10"), + hostOut("vts-core"), productOut("*.img"), productOut("*.zip"), productOut("android-info.txt"), + productOut("apex"), productOut("kernel"), productOut("data"), productOut("skin"), productOut("obj/NOTICE_FILES"), productOut("obj/PACKAGING"), productOut("ramdisk"), + productOut("debug_ramdisk"), + productOut("vendor-ramdisk"), + productOut("vendor-ramdisk-debug.cpio.gz"), + productOut("vendor_debug_ramdisk"), + productOut("test_harness_ramdisk"), productOut("recovery"), productOut("root"), productOut("system"), productOut("system_other"), productOut("vendor"), productOut("product"), - productOut("product_services"), + productOut("system_ext"), productOut("oem"), productOut("obj/FAKE"), productOut("breakpad"), @@ -172,3 +183,93 @@ func installCleanIfNecessary(ctx Context, config Config) { writeConfig() } + +// cleanOldFiles takes an input file (with all paths relative to basePath), and removes files from +// the filesystem if they were removed from the input file since the last execution. +func cleanOldFiles(ctx Context, basePath, file string) { + file = filepath.Join(basePath, file) + oldFile := file + ".previous" + + if _, err := os.Stat(file); err != nil { + ctx.Fatalf("Expected %q to be readable", file) + } + + if _, err := os.Stat(oldFile); os.IsNotExist(err) { + if err := os.Rename(file, oldFile); err != nil { + ctx.Fatalf("Failed to rename file list (%q->%q): %v", file, oldFile, err) + } + return + } + + var newPaths, oldPaths []string + if newData, err := ioutil.ReadFile(file); err == nil { + if oldData, err := ioutil.ReadFile(oldFile); err == nil { + // Common case: nothing has changed + if bytes.Equal(newData, oldData) { + return + } + newPaths = strings.Fields(string(newData)) + oldPaths = strings.Fields(string(oldData)) + } else { + ctx.Fatalf("Failed to read list of installable files (%q): %v", oldFile, err) + } + } else { + ctx.Fatalf("Failed to read list of installable files (%q): %v", file, err) + } + + // These should be mostly sorted by make already, but better make sure Go concurs + sort.Strings(newPaths) + sort.Strings(oldPaths) + + for len(oldPaths) > 0 { + if len(newPaths) > 0 { + if oldPaths[0] == newPaths[0] { + // Same file; continue + newPaths = newPaths[1:] + oldPaths = oldPaths[1:] + continue + } else if oldPaths[0] > newPaths[0] { + // New file; ignore + newPaths = newPaths[1:] + continue + } + } + // File only exists in the old list; remove if it exists + old := filepath.Join(basePath, oldPaths[0]) + oldPaths = oldPaths[1:] + if fi, err := os.Stat(old); err == nil { + if fi.IsDir() { + if err := os.Remove(old); err == nil { + ctx.Println("Removed directory that is no longer installed: ", old) + cleanEmptyDirs(ctx, filepath.Dir(old)) + } else { + ctx.Println("Failed to remove directory that is no longer installed (%q): %v", old, err) + ctx.Println("It's recommended to run `m installclean`") + } + } else { + if err := os.Remove(old); err == nil { + ctx.Println("Removed file that is no longer installed: ", old) + cleanEmptyDirs(ctx, filepath.Dir(old)) + } else if !os.IsNotExist(err) { + ctx.Fatalf("Failed to remove file that is no longer installed (%q): %v", old, err) + } + } + } + } + + // Use the new list as the base for the next build + os.Rename(file, oldFile) +} + +func cleanEmptyDirs(ctx Context, dir string) { + files, err := ioutil.ReadDir(dir) + if err != nil || len(files) > 0 { + return + } + if err := os.Remove(dir); err == nil { + ctx.Println("Removed directory that is no longer installed: ", dir) + } else { + ctx.Fatalf("Failed to remove directory that is no longer installed (%q): %v", dir, err) + } + cleanEmptyDirs(ctx, filepath.Dir(dir)) +} diff --git a/ui/build/cleanbuild_test.go b/ui/build/cleanbuild_test.go new file mode 100644 index 000000000..89f4ad9e1 --- /dev/null +++ b/ui/build/cleanbuild_test.go @@ -0,0 +1,100 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package build + +import ( + "android/soong/ui/logger" + "bytes" + "io/ioutil" + "os" + "path/filepath" + "reflect" + "sort" + "strings" + "testing" +) + +func TestCleanOldFiles(t *testing.T) { + dir, err := ioutil.TempDir("", "testcleanoldfiles") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + ctx := testContext() + logBuf := &bytes.Buffer{} + ctx.Logger = logger.New(logBuf) + + touch := func(names ...string) { + for _, name := range names { + if f, err := os.Create(filepath.Join(dir, name)); err != nil { + t.Fatal(err) + } else { + f.Close() + } + } + } + runCleanOldFiles := func(names ...string) { + data := []byte(strings.Join(names, " ")) + if err := ioutil.WriteFile(filepath.Join(dir, ".installed"), data, 0666); err != nil { + t.Fatal(err) + } + + cleanOldFiles(ctx, dir, ".installed") + } + + assertFileList := func(names ...string) { + t.Helper() + + sort.Strings(names) + + var foundNames []string + if foundFiles, err := ioutil.ReadDir(dir); err == nil { + for _, fi := range foundFiles { + foundNames = append(foundNames, fi.Name()) + } + } else { + t.Fatal(err) + } + + if !reflect.DeepEqual(names, foundNames) { + t.Errorf("Expected a different list of files:\nwant: %v\n got: %v", names, foundNames) + t.Error("Log: ", logBuf.String()) + logBuf.Reset() + } + } + + // Initial list of potential files + runCleanOldFiles("foo", "bar") + touch("foo", "bar", "baz") + assertFileList("foo", "bar", "baz", ".installed.previous") + + // This should be a no-op, as the list hasn't changed + runCleanOldFiles("foo", "bar") + assertFileList("foo", "bar", "baz", ".installed", ".installed.previous") + + // This should be a no-op, as only a file was added + runCleanOldFiles("foo", "bar", "foo2") + assertFileList("foo", "bar", "baz", ".installed.previous") + + // "bar" should be removed, foo2 should be ignored as it was never there + runCleanOldFiles("foo") + assertFileList("foo", "baz", ".installed.previous") + + // Recreate bar, and create foo2. Ensure that they aren't removed + touch("bar", "foo2") + runCleanOldFiles("foo", "baz") + assertFileList("foo", "bar", "baz", "foo2", ".installed.previous") +} diff --git a/ui/build/config.go b/ui/build/config.go index 28ce57d04..7f28d3af5 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -16,8 +16,6 @@ package build import ( "fmt" - "io/ioutil" - "log" "os" "path/filepath" "runtime" @@ -36,10 +34,11 @@ type Config struct{ *configImpl } type configImpl struct { // From the environment - arguments []string - goma bool - environ *Environment - distDir string + arguments []string + goma bool + environ *Environment + distDir string + buildDateTime string // From the arguments parallel int @@ -56,17 +55,47 @@ type configImpl struct { targetDevice string targetDeviceDir string + // Autodetected + totalRAM uint64 + pdkBuild bool brokenDupRules bool - brokenPhonyTargets bool brokenUsesNetwork bool + brokenNinjaEnvVars []string pathReplaced bool } const srcDirFileCheck = "build/soong/root.bp" +var buildFiles = []string{"Android.mk", "Android.bp"} + +type BuildAction uint + +const ( + // Builds all of the modules and their dependencies of a specified directory, relative to the root + // directory of the source tree. + BUILD_MODULES_IN_A_DIRECTORY BuildAction = iota + + // Builds all of the modules and their dependencies of a list of specified directories. All specified + // directories are relative to the root directory of the source tree. + BUILD_MODULES_IN_DIRECTORIES + + // Build a list of specified modules. If none was specified, simply build the whole source tree. + BUILD_MODULES +) + +// checkTopDir validates that the current directory is at the root directory of the source tree. +func checkTopDir(ctx Context) { + if _, err := os.Stat(srcDirFileCheck); err != nil { + if os.IsNotExist(err) { + ctx.Fatalf("Current working directory must be the source tree. %q not found.", srcDirFileCheck) + } + ctx.Fatalln("Error verifying tree state:", err) + } +} + func NewConfig(ctx Context, args ...string) Config { ret := &configImpl{ environ: OsEnvironment(), @@ -76,6 +105,8 @@ func NewConfig(ctx Context, args ...string) Config { ret.parallel = runtime.NumCPU() + 2 ret.keepGoing = 1 + ret.totalRAM = detectTotalRAM(ctx) + ret.parseArgs(ctx, args) // Make sure OUT_DIR is set appropriately @@ -124,6 +155,7 @@ func NewConfig(ctx Context, args ...string) Config { "DIST_DIR", // Variables that have caused problems in the past + "BASH_ENV", "CDPATH", "DISPLAY", "GREP_OPTIONS", @@ -165,46 +197,50 @@ func NewConfig(ctx Context, args ...string) Config { tmpDir := absPath(ctx, ret.TempDir()) ret.environ.Set("TMPDIR", tmpDir) + // Always set ASAN_SYMBOLIZER_PATH so that ASAN-based tools can symbolize any crashes + symbolizerPath := filepath.Join("prebuilts/clang/host", ret.HostPrebuiltTag(), + "llvm-binutils-stable/llvm-symbolizer") + ret.environ.Set("ASAN_SYMBOLIZER_PATH", absPath(ctx, symbolizerPath)) + // Precondition: the current directory is the top of the source tree - if _, err := os.Stat(srcDirFileCheck); err != nil { - if os.IsNotExist(err) { - log.Fatalf("Current working directory must be the source tree. %q not found", srcDirFileCheck) - } - log.Fatalln("Error verifying tree state:", err) - } + checkTopDir(ctx) if srcDir := absPath(ctx, "."); strings.ContainsRune(srcDir, ' ') { - log.Println("You are building in a directory whose absolute path contains a space character:") - log.Println() - log.Printf("%q\n", srcDir) - log.Println() - log.Fatalln("Directory names containing spaces are not supported") + ctx.Println("You are building in a directory whose absolute path contains a space character:") + ctx.Println() + ctx.Printf("%q\n", srcDir) + ctx.Println() + ctx.Fatalln("Directory names containing spaces are not supported") } if outDir := ret.OutDir(); strings.ContainsRune(outDir, ' ') { - log.Println("The absolute path of your output directory ($OUT_DIR) contains a space character:") - log.Println() - log.Printf("%q\n", outDir) - log.Println() - log.Fatalln("Directory names containing spaces are not supported") + ctx.Println("The absolute path of your output directory ($OUT_DIR) contains a space character:") + ctx.Println() + ctx.Printf("%q\n", outDir) + ctx.Println() + ctx.Fatalln("Directory names containing spaces are not supported") } if distDir := ret.DistDir(); strings.ContainsRune(distDir, ' ') { - log.Println("The absolute path of your dist directory ($DIST_DIR) contains a space character:") - log.Println() - log.Printf("%q\n", distDir) - log.Println() - log.Fatalln("Directory names containing spaces are not supported") + ctx.Println("The absolute path of your dist directory ($DIST_DIR) contains a space character:") + ctx.Println() + ctx.Printf("%q\n", distDir) + ctx.Println() + ctx.Fatalln("Directory names containing spaces are not supported") } // Configure Java-related variables, including adding it to $PATH java8Home := filepath.Join("prebuilts/jdk/jdk8", ret.HostPrebuiltTag()) java9Home := filepath.Join("prebuilts/jdk/jdk9", ret.HostPrebuiltTag()) + java11Home := filepath.Join("prebuilts/jdk/jdk11", ret.HostPrebuiltTag()) javaHome := func() string { if override, ok := ret.environ.Get("OVERRIDE_ANDROID_JAVA_HOME"); ok { return override } - return java9Home + if toolchain11, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN"); ok && toolchain11 != "true" { + ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN is no longer supported. An OpenJDK 11 toolchain is now the global default.") + } + return java11Home }() absJavaHome := absPath(ctx, javaHome) @@ -214,26 +250,23 @@ func NewConfig(ctx Context, args ...string) Config { if path, ok := ret.environ.Get("PATH"); ok && path != "" { newPath = append(newPath, path) } + ret.environ.Unset("OVERRIDE_ANDROID_JAVA_HOME") ret.environ.Set("JAVA_HOME", absJavaHome) ret.environ.Set("ANDROID_JAVA_HOME", javaHome) ret.environ.Set("ANDROID_JAVA8_HOME", java8Home) ret.environ.Set("ANDROID_JAVA9_HOME", java9Home) + ret.environ.Set("ANDROID_JAVA11_HOME", java11Home) ret.environ.Set("PATH", strings.Join(newPath, string(filepath.ListSeparator))) outDir := ret.OutDir() buildDateTimeFile := filepath.Join(outDir, "build_date.txt") - var content string if buildDateTime, ok := ret.environ.Get("BUILD_DATETIME"); ok && buildDateTime != "" { - content = buildDateTime + ret.buildDateTime = buildDateTime } else { - content = strconv.FormatInt(time.Now().Unix(), 10) + ret.buildDateTime = strconv.FormatInt(time.Now().Unix(), 10) } - err := ioutil.WriteFile(buildDateTimeFile, []byte(content), 0777) - if err != nil { - ctx.Fatalln("Failed to write BUILD_DATETIME to file:", err) - } ret.environ.Set("BUILD_DATETIME_FILE", buildDateTimeFile) if ret.UseRBE() { @@ -247,6 +280,12 @@ func NewConfig(ctx Context, args ...string) Config { return c } +// NewBuildActionConfig returns a build configuration based on the build action. The arguments are +// processed based on the build action and extracts any arguments that belongs to the build action. +func NewBuildActionConfig(action BuildAction, dir string, ctx Context, args ...string) Config { + return NewConfig(ctx, getConfigArgs(action, dir, ctx, args)...) +} + // storeConfigMetrics selects a set of configuration information and store in // the metrics system for further analysis. func storeConfigMetrics(ctx Context, config Config) { @@ -262,6 +301,219 @@ func storeConfigMetrics(ctx Context, config Config) { ctx.Metrics.BuildConfig(b) } +// getConfigArgs processes the command arguments based on the build action and creates a set of new +// arguments to be accepted by Config. +func getConfigArgs(action BuildAction, dir string, ctx Context, args []string) []string { + // The next block of code verifies that the current directory is the root directory of the source + // tree. It then finds the relative path of dir based on the root directory of the source tree + // and verify that dir is inside of the source tree. + checkTopDir(ctx) + topDir, err := os.Getwd() + if err != nil { + ctx.Fatalf("Error retrieving top directory: %v", err) + } + dir, err = filepath.EvalSymlinks(dir) + if err != nil { + ctx.Fatalf("Unable to evaluate symlink of %s: %v", dir, err) + } + dir, err = filepath.Abs(dir) + if err != nil { + ctx.Fatalf("Unable to find absolute path %s: %v", dir, err) + } + relDir, err := filepath.Rel(topDir, dir) + if err != nil { + ctx.Fatalf("Unable to find relative path %s of %s: %v", relDir, topDir, err) + } + // If there are ".." in the path, it's not in the source tree. + if strings.Contains(relDir, "..") { + ctx.Fatalf("Directory %s is not under the source tree %s", dir, topDir) + } + + configArgs := args[:] + + // If the arguments contains GET-INSTALL-PATH, change the target name prefix from MODULES-IN- to + // GET-INSTALL-PATH-IN- to extract the installation path instead of building the modules. + targetNamePrefix := "MODULES-IN-" + if inList("GET-INSTALL-PATH", configArgs) { + targetNamePrefix = "GET-INSTALL-PATH-IN-" + configArgs = removeFromList("GET-INSTALL-PATH", configArgs) + } + + var targets []string + + switch action { + case BUILD_MODULES: + // No additional processing is required when building a list of specific modules or all modules. + case BUILD_MODULES_IN_A_DIRECTORY: + // If dir is the root source tree, all the modules are built of the source tree are built so + // no need to find the build file. + if topDir == dir { + break + } + + buildFile := findBuildFile(ctx, relDir) + if buildFile == "" { + ctx.Fatalf("Build file not found for %s directory", relDir) + } + targets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)} + case BUILD_MODULES_IN_DIRECTORIES: + newConfigArgs, dirs := splitArgs(configArgs) + configArgs = newConfigArgs + targets = getTargetsFromDirs(ctx, relDir, dirs, targetNamePrefix) + } + + // Tidy only override all other specified targets. + tidyOnly := os.Getenv("WITH_TIDY_ONLY") + if tidyOnly == "true" || tidyOnly == "1" { + configArgs = append(configArgs, "tidy_only") + } else { + configArgs = append(configArgs, targets...) + } + + return configArgs +} + +// convertToTarget replaces "/" to "-" in dir and pre-append the targetNamePrefix to the target name. +func convertToTarget(dir string, targetNamePrefix string) string { + return targetNamePrefix + strings.ReplaceAll(dir, "/", "-") +} + +// hasBuildFile returns true if dir contains an Android build file. +func hasBuildFile(ctx Context, dir string) bool { + for _, buildFile := range buildFiles { + _, err := os.Stat(filepath.Join(dir, buildFile)) + if err == nil { + return true + } + if !os.IsNotExist(err) { + ctx.Fatalf("Error retrieving the build file stats: %v", err) + } + } + return false +} + +// findBuildFile finds a build file (makefile or blueprint file) by looking if there is a build file +// in the current and any sub directory of dir. If a build file is not found, traverse the path +// up by one directory and repeat again until either a build file is found or reached to the root +// source tree. The returned filename of build file is "Android.mk". If one was not found, a blank +// string is returned. +func findBuildFile(ctx Context, dir string) string { + // If the string is empty or ".", assume it is top directory of the source tree. + if dir == "" || dir == "." { + return "" + } + + found := false + for buildDir := dir; buildDir != "."; buildDir = filepath.Dir(buildDir) { + err := filepath.Walk(buildDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if found { + return filepath.SkipDir + } + if info.IsDir() { + return nil + } + for _, buildFile := range buildFiles { + if info.Name() == buildFile { + found = true + return filepath.SkipDir + } + } + return nil + }) + if err != nil { + ctx.Fatalf("Error finding Android build file: %v", err) + } + + if found { + return filepath.Join(buildDir, "Android.mk") + } + } + + return "" +} + +// splitArgs iterates over the arguments list and splits into two lists: arguments and directories. +func splitArgs(args []string) (newArgs []string, dirs []string) { + specialArgs := map[string]bool{ + "showcommands": true, + "snod": true, + "dist": true, + "checkbuild": true, + } + + newArgs = []string{} + dirs = []string{} + + for _, arg := range args { + // It's a dash argument if it starts with "-" or it's a key=value pair, it's not a directory. + if strings.IndexRune(arg, '-') == 0 || strings.IndexRune(arg, '=') != -1 { + newArgs = append(newArgs, arg) + continue + } + + if _, ok := specialArgs[arg]; ok { + newArgs = append(newArgs, arg) + continue + } + + dirs = append(dirs, arg) + } + + return newArgs, dirs +} + +// getTargetsFromDirs iterates over the dirs list and creates a list of targets to build. If a +// directory from the dirs list does not exist, a fatal error is raised. relDir is related to the +// source root tree where the build action command was invoked. Each directory is validated if the +// build file can be found and follows the format "dir1:target1,target2,...". Target is optional. +func getTargetsFromDirs(ctx Context, relDir string, dirs []string, targetNamePrefix string) (targets []string) { + for _, dir := range dirs { + // The directory may have specified specific modules to build. ":" is the separator to separate + // the directory and the list of modules. + s := strings.Split(dir, ":") + l := len(s) + if l > 2 { // more than one ":" was specified. + ctx.Fatalf("%s not in proper directory:target1,target2,... format (\":\" was specified more than once)", dir) + } + + dir = filepath.Join(relDir, s[0]) + if _, err := os.Stat(dir); err != nil { + ctx.Fatalf("couldn't find directory %s", dir) + } + + // Verify that if there are any targets specified after ":". Each target is separated by ",". + var newTargets []string + if l == 2 && s[1] != "" { + newTargets = strings.Split(s[1], ",") + if inList("", newTargets) { + ctx.Fatalf("%s not in proper directory:target1,target2,... format", dir) + } + } + + // If there are specified targets to build in dir, an android build file must exist for the one + // shot build. For the non-targets case, find the appropriate build file and build all the + // modules in dir (or the closest one in the dir path). + if len(newTargets) > 0 { + if !hasBuildFile(ctx, dir) { + ctx.Fatalf("Couldn't locate a build file from %s directory", dir) + } + } else { + buildFile := findBuildFile(ctx, dir) + if buildFile == "" { + ctx.Fatalf("Build file not found for %s directory", dir) + } + newTargets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)} + } + + targets = append(targets, newTargets...) + } + + return targets +} + func (c *configImpl) parseArgs(ctx Context, args []string) { for i := 0; i < len(args); i++ { arg := strings.TrimSpace(args[i]) @@ -418,7 +670,7 @@ func (c *configImpl) Arguments() []string { func (c *configImpl) OutDir() string { if outDir, ok := c.environ.Get("OUT_DIR"); ok { - return filepath.Clean(outDir) + return outDir } return "out" } @@ -501,6 +753,40 @@ func (c *configImpl) Parallel() int { return c.parallel } +func (c *configImpl) HighmemParallel() int { + if i, ok := c.environ.GetInt("NINJA_HIGHMEM_NUM_JOBS"); ok { + return i + } + + const minMemPerHighmemProcess = 8 * 1024 * 1024 * 1024 + parallel := c.Parallel() + if c.UseRemoteBuild() { + // Ninja doesn't support nested pools, and when remote builds are enabled the total ninja parallelism + // is set very high (i.e. 500). Using a large value here would cause the total number of running jobs + // to be the sum of the sizes of the local and highmem pools, which will cause extra CPU contention. + // Return 1/16th of the size of the local pool, rounding up. + return (parallel + 15) / 16 + } else if c.totalRAM == 0 { + // Couldn't detect the total RAM, don't restrict highmem processes. + return parallel + } else if c.totalRAM <= 16*1024*1024*1024 { + // Less than 16GB of ram, restrict to 1 highmem processes + return 1 + } else if c.totalRAM <= 32*1024*1024*1024 { + // Less than 32GB of ram, restrict to 2 highmem processes + return 2 + } else if p := int(c.totalRAM / minMemPerHighmemProcess); p < parallel { + // If less than 8GB total RAM per process, reduce the number of highmem processes + return p + } + // No restriction on highmem processes + return parallel +} + +func (c *configImpl) TotalRAM() uint64 { + return c.totalRAM +} + // ForceUseGoma determines whether we should override Goma deprecation // and use Goma for the current build or not. func (c *configImpl) ForceUseGoma() bool { @@ -638,10 +924,11 @@ func (c *configImpl) UseRemoteBuild() bool { // gomacc) are run in parallel. Note the parallelism of all other jobs is // still limited by Parallel() func (c *configImpl) RemoteParallel() int { - if v, ok := c.environ.Get("NINJA_REMOTE_NUM_JOBS"); ok { - if i, err := strconv.Atoi(v); err == nil { - return i - } + if !c.UseRemoteBuild() { + return 0 + } + if i, ok := c.environ.GetInt("NINJA_REMOTE_NUM_JOBS"); ok { + return i } return 500 } @@ -756,14 +1043,6 @@ func (c *configImpl) BuildBrokenDupRules() bool { return c.brokenDupRules } -func (c *configImpl) SetBuildBrokenPhonyTargets(val bool) { - c.brokenPhonyTargets = val -} - -func (c *configImpl) BuildBrokenPhonyTargets() bool { - return c.brokenPhonyTargets -} - func (c *configImpl) SetBuildBrokenUsesNetwork(val bool) { c.brokenUsesNetwork = val } @@ -772,6 +1051,14 @@ func (c *configImpl) BuildBrokenUsesNetwork() bool { return c.brokenUsesNetwork } +func (c *configImpl) SetBuildBrokenNinjaUsesEnvVars(val []string) { + c.brokenNinjaEnvVars = val +} + +func (c *configImpl) BuildBrokenNinjaUsesEnvVars() []string { + return c.brokenNinjaEnvVars +} + func (c *configImpl) SetTargetDeviceDir(dir string) { c.targetDeviceDir = dir } @@ -788,6 +1075,10 @@ func (c *configImpl) IsPdkBuild() bool { return c.pdkBuild } +func (c *configImpl) BuildDateTime() string { + return c.buildDateTime +} + func (c *configImpl) MetricsUploaderApp() string { if p, ok := c.environ.Get("ANDROID_ENABLE_METRICS_UPLOAD"); ok { return p diff --git a/ui/build/config_darwin.go b/ui/build/config_darwin.go new file mode 100644 index 000000000..fe74e31da --- /dev/null +++ b/ui/build/config_darwin.go @@ -0,0 +1,40 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package build + +import ( + "encoding/binary" + "syscall" +) + +func detectTotalRAM(ctx Context) uint64 { + s, err := syscall.Sysctl("hw.memsize") + if err != nil { + ctx.Printf("Failed to get system memory size: %v", err) + return 0 + } + + // syscall.Sysctl assumes that the return value is a string and trims the last byte if it is 0. + if len(s) == 7 { + s += "\x00" + } + + if len(s) != 8 { + ctx.Printf("Failed to get system memory size, returned %d bytes, expecting 8 bytes", len(s)) + return 0 + } + + return binary.LittleEndian.Uint64([]byte(s)) +} diff --git a/ui/build/config_linux.go b/ui/build/config_linux.go new file mode 100644 index 000000000..162d372d0 --- /dev/null +++ b/ui/build/config_linux.go @@ -0,0 +1,27 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package build + +import "syscall" + +func detectTotalRAM(ctx Context) uint64 { + var info syscall.Sysinfo_t + err := syscall.Sysinfo(&info) + if err != nil { + ctx.Printf("Failed to get system memory size: %v", err) + return 0 + } + return uint64(info.Totalram) * uint64(info.Unit) +} diff --git a/ui/build/config_test.go b/ui/build/config_test.go index e1fb27107..7b14c4703 100644 --- a/ui/build/config_test.go +++ b/ui/build/config_test.go @@ -17,20 +17,23 @@ package build import ( "bytes" "context" + "fmt" + "io/ioutil" + "os" + "path/filepath" "reflect" "strings" "testing" "android/soong/ui/logger" "android/soong/ui/status" - "android/soong/ui/terminal" ) func testContext() Context { return Context{&ContextImpl{ Context: context.Background(), Logger: logger.New(&bytes.Buffer{}), - Writer: terminal.NewWriter(terminal.NewCustomStdio(&bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{})), + Writer: &bytes.Buffer{}, Status: &status.Status{}, }} } @@ -175,3 +178,820 @@ func TestConfigParseArgsVars(t *testing.T) { }) } } + +func TestConfigCheckTopDir(t *testing.T) { + ctx := testContext() + buildRootDir := filepath.Dir(srcDirFileCheck) + expectedErrStr := fmt.Sprintf("Current working directory must be the source tree. %q not found.", srcDirFileCheck) + + tests := []struct { + // ********* Setup ********* + // Test description. + description string + + // ********* Action ********* + // If set to true, the build root file is created. + rootBuildFile bool + + // The current path where Soong is being executed. + path string + + // ********* Validation ********* + // Expecting error and validate the error string against expectedErrStr. + wantErr bool + }{{ + description: "current directory is the root source tree", + rootBuildFile: true, + path: ".", + wantErr: false, + }, { + description: "one level deep in the source tree", + rootBuildFile: true, + path: "1", + wantErr: true, + }, { + description: "very deep in the source tree", + rootBuildFile: true, + path: "1/2/3/4/5/6/7/8/9/1/2/3/4/5/6/7/8/9/1/2/3/4/5/6/7/8/9/1/2/3/4/5/6/7", + wantErr: true, + }, { + description: "outside of source tree", + rootBuildFile: false, + path: "1/2/3/4/5", + wantErr: true, + }} + + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + defer logger.Recover(func(err error) { + if !tt.wantErr { + t.Fatalf("Got unexpected error: %v", err) + } + if expectedErrStr != err.Error() { + t.Fatalf("expected %s, got %s", expectedErrStr, err.Error()) + } + }) + + // Create the root source tree. + rootDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(rootDir) + + // Create the build root file. This is to test if topDir returns an error if the build root + // file does not exist. + if tt.rootBuildFile { + dir := filepath.Join(rootDir, buildRootDir) + if err := os.MkdirAll(dir, 0755); err != nil { + t.Errorf("failed to create %s directory: %v", dir, err) + } + f := filepath.Join(rootDir, srcDirFileCheck) + if err := ioutil.WriteFile(f, []byte{}, 0644); err != nil { + t.Errorf("failed to create file %s: %v", f, err) + } + } + + // Next block of code is to set the current directory. + dir := rootDir + if tt.path != "" { + dir = filepath.Join(dir, tt.path) + if err := os.MkdirAll(dir, 0755); err != nil { + t.Errorf("failed to create %s directory: %v", dir, err) + } + } + curDir, err := os.Getwd() + if err != nil { + t.Fatalf("failed to get the current directory: %v", err) + } + defer func() { os.Chdir(curDir) }() + + if err := os.Chdir(dir); err != nil { + t.Fatalf("failed to change directory to %s: %v", dir, err) + } + + checkTopDir(ctx) + }) + } +} + +func TestConfigConvertToTarget(t *testing.T) { + tests := []struct { + // ********* Setup ********* + // Test description. + description string + + // ********* Action ********* + // The current directory where Soong is being executed. + dir string + + // The current prefix string to be pre-appended to the target. + prefix string + + // ********* Validation ********* + // The expected target to be invoked in ninja. + expectedTarget string + }{{ + description: "one level directory in source tree", + dir: "test1", + prefix: "MODULES-IN-", + expectedTarget: "MODULES-IN-test1", + }, { + description: "multiple level directories in source tree", + dir: "test1/test2/test3/test4", + prefix: "GET-INSTALL-PATH-IN-", + expectedTarget: "GET-INSTALL-PATH-IN-test1-test2-test3-test4", + }} + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + target := convertToTarget(tt.dir, tt.prefix) + if target != tt.expectedTarget { + t.Errorf("expected %s, got %s for target", tt.expectedTarget, target) + } + }) + } +} + +func setTop(t *testing.T, dir string) func() { + curDir, err := os.Getwd() + if err != nil { + t.Fatalf("failed to get current directory: %v", err) + } + if err := os.Chdir(dir); err != nil { + t.Fatalf("failed to change directory to top dir %s: %v", dir, err) + } + return func() { os.Chdir(curDir) } +} + +func createBuildFiles(t *testing.T, topDir string, buildFiles []string) { + for _, buildFile := range buildFiles { + buildFile = filepath.Join(topDir, buildFile) + if err := ioutil.WriteFile(buildFile, []byte{}, 0644); err != nil { + t.Errorf("failed to create file %s: %v", buildFile, err) + } + } +} + +func createDirectories(t *testing.T, topDir string, dirs []string) { + for _, dir := range dirs { + dir = filepath.Join(topDir, dir) + if err := os.MkdirAll(dir, 0755); err != nil { + t.Errorf("failed to create %s directory: %v", dir, err) + } + } +} + +func TestConfigGetTargets(t *testing.T) { + ctx := testContext() + tests := []struct { + // ********* Setup ********* + // Test description. + description string + + // Directories that exist in the source tree. + dirsInTrees []string + + // Build files that exists in the source tree. + buildFiles []string + + // ********* Action ********* + // Directories passed in to soong_ui. + dirs []string + + // Current directory that the user executed the build action command. + curDir string + + // ********* Validation ********* + // Expected targets from the function. + expectedTargets []string + + // Expecting error from running test case. + errStr string + }{{ + description: "one target dir specified", + dirsInTrees: []string{"0/1/2/3"}, + buildFiles: []string{"0/1/2/3/Android.bp"}, + dirs: []string{"1/2/3"}, + curDir: "0", + expectedTargets: []string{"MODULES-IN-0-1-2-3"}, + }, { + description: "one target dir specified, build file does not exist", + dirsInTrees: []string{"0/1/2/3"}, + buildFiles: []string{}, + dirs: []string{"1/2/3"}, + curDir: "0", + errStr: "Build file not found for 0/1/2/3 directory", + }, { + description: "one target dir specified, invalid targets specified", + dirsInTrees: []string{"0/1/2/3"}, + buildFiles: []string{}, + dirs: []string{"1/2/3:t1:t2"}, + curDir: "0", + errStr: "1/2/3:t1:t2 not in proper directory:target1,target2,... format (\":\" was specified more than once)", + }, { + description: "one target dir specified, no targets specified but has colon", + dirsInTrees: []string{"0/1/2/3"}, + buildFiles: []string{"0/1/2/3/Android.bp"}, + dirs: []string{"1/2/3:"}, + curDir: "0", + expectedTargets: []string{"MODULES-IN-0-1-2-3"}, + }, { + description: "one target dir specified, two targets specified", + dirsInTrees: []string{"0/1/2/3"}, + buildFiles: []string{"0/1/2/3/Android.bp"}, + dirs: []string{"1/2/3:t1,t2"}, + curDir: "0", + expectedTargets: []string{"t1", "t2"}, + }, { + description: "one target dir specified, no targets and has a comma", + dirsInTrees: []string{"0/1/2/3"}, + buildFiles: []string{"0/1/2/3/Android.bp"}, + dirs: []string{"1/2/3:,"}, + curDir: "0", + errStr: "0/1/2/3 not in proper directory:target1,target2,... format", + }, { + description: "one target dir specified, improper targets defined", + dirsInTrees: []string{"0/1/2/3"}, + buildFiles: []string{"0/1/2/3/Android.bp"}, + dirs: []string{"1/2/3:,t1"}, + curDir: "0", + errStr: "0/1/2/3 not in proper directory:target1,target2,... format", + }, { + description: "one target dir specified, blank target", + dirsInTrees: []string{"0/1/2/3"}, + buildFiles: []string{"0/1/2/3/Android.bp"}, + dirs: []string{"1/2/3:t1,"}, + curDir: "0", + errStr: "0/1/2/3 not in proper directory:target1,target2,... format", + }, { + description: "one target dir specified, many targets specified", + dirsInTrees: []string{"0/1/2/3"}, + buildFiles: []string{"0/1/2/3/Android.bp"}, + dirs: []string{"1/2/3:t1,t2,t3,t4,t5,t6,t7,t8,t9,t10"}, + curDir: "0", + expectedTargets: []string{"t1", "t2", "t3", "t4", "t5", "t6", "t7", "t8", "t9", "t10"}, + }, { + description: "one target dir specified, one target specified, build file does not exist", + dirsInTrees: []string{"0/1/2/3"}, + buildFiles: []string{}, + dirs: []string{"1/2/3:t1"}, + curDir: "0", + errStr: "Couldn't locate a build file from 0/1/2/3 directory", + }, { + description: "one target dir specified, one target specified, build file not in target dir", + dirsInTrees: []string{"0/1/2/3"}, + buildFiles: []string{"0/1/2/Android.mk"}, + dirs: []string{"1/2/3:t1"}, + curDir: "0", + errStr: "Couldn't locate a build file from 0/1/2/3 directory", + }, { + description: "one target dir specified, build file not in target dir", + dirsInTrees: []string{"0/1/2/3"}, + buildFiles: []string{"0/1/2/Android.mk"}, + dirs: []string{"1/2/3"}, + curDir: "0", + expectedTargets: []string{"MODULES-IN-0-1-2"}, + }, { + description: "multiple targets dir specified, targets specified", + dirsInTrees: []string{"0/1/2/3", "0/3/4"}, + buildFiles: []string{"0/1/2/3/Android.bp", "0/3/4/Android.mk"}, + dirs: []string{"1/2/3:t1,t2", "3/4:t3,t4,t5"}, + curDir: "0", + expectedTargets: []string{"t1", "t2", "t3", "t4", "t5"}, + }, { + description: "multiple targets dir specified, one directory has targets specified", + dirsInTrees: []string{"0/1/2/3", "0/3/4"}, + buildFiles: []string{"0/1/2/3/Android.bp", "0/3/4/Android.mk"}, + dirs: []string{"1/2/3:t1,t2", "3/4"}, + curDir: "0", + expectedTargets: []string{"t1", "t2", "MODULES-IN-0-3-4"}, + }, { + description: "two dirs specified, only one dir exist", + dirsInTrees: []string{"0/1/2/3"}, + buildFiles: []string{"0/1/2/3/Android.mk"}, + dirs: []string{"1/2/3:t1", "3/4"}, + curDir: "0", + errStr: "couldn't find directory 0/3/4", + }, { + description: "multiple targets dirs specified at root source tree", + dirsInTrees: []string{"0/1/2/3", "0/3/4"}, + buildFiles: []string{"0/1/2/3/Android.bp", "0/3/4/Android.mk"}, + dirs: []string{"0/1/2/3:t1,t2", "0/3/4"}, + curDir: ".", + expectedTargets: []string{"t1", "t2", "MODULES-IN-0-3-4"}, + }, { + description: "no directories specified", + dirsInTrees: []string{"0/1/2/3", "0/3/4"}, + buildFiles: []string{"0/1/2/3/Android.bp", "0/3/4/Android.mk"}, + dirs: []string{}, + curDir: ".", + }} + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + defer logger.Recover(func(err error) { + if tt.errStr == "" { + t.Fatalf("Got unexpected error: %v", err) + } + if tt.errStr != err.Error() { + t.Errorf("expected %s, got %s", tt.errStr, err.Error()) + } + }) + + // Create the root source tree. + topDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + defer os.RemoveAll(topDir) + + createDirectories(t, topDir, tt.dirsInTrees) + createBuildFiles(t, topDir, tt.buildFiles) + r := setTop(t, topDir) + defer r() + + targets := getTargetsFromDirs(ctx, tt.curDir, tt.dirs, "MODULES-IN-") + if !reflect.DeepEqual(targets, tt.expectedTargets) { + t.Errorf("expected %v, got %v for targets", tt.expectedTargets, targets) + } + + // If the execution reached here and there was an expected error code, the unit test case failed. + if tt.errStr != "" { + t.Errorf("expecting error %s", tt.errStr) + } + }) + } +} + +func TestConfigFindBuildFile(t *testing.T) { + ctx := testContext() + + tests := []struct { + // ********* Setup ********* + // Test description. + description string + + // Array of build files to create in dir. + buildFiles []string + + // Directories that exist in the source tree. + dirsInTrees []string + + // ********* Action ********* + // The base directory is where findBuildFile is invoked. + dir string + + // ********* Validation ********* + // Expected build file path to find. + expectedBuildFile string + }{{ + description: "build file exists at leaf directory", + buildFiles: []string{"1/2/3/Android.bp"}, + dirsInTrees: []string{"1/2/3"}, + dir: "1/2/3", + expectedBuildFile: "1/2/3/Android.mk", + }, { + description: "build file exists in all directory paths", + buildFiles: []string{"1/Android.mk", "1/2/Android.mk", "1/2/3/Android.mk"}, + dirsInTrees: []string{"1/2/3"}, + dir: "1/2/3", + expectedBuildFile: "1/2/3/Android.mk", + }, { + description: "build file does not exist in all directory paths", + buildFiles: []string{}, + dirsInTrees: []string{"1/2/3"}, + dir: "1/2/3", + expectedBuildFile: "", + }, { + description: "build file exists only at top directory", + buildFiles: []string{"Android.bp"}, + dirsInTrees: []string{"1/2/3"}, + dir: "1/2/3", + expectedBuildFile: "", + }, { + description: "build file exist in a subdirectory", + buildFiles: []string{"1/2/Android.bp"}, + dirsInTrees: []string{"1/2/3"}, + dir: "1/2/3", + expectedBuildFile: "1/2/Android.mk", + }, { + description: "build file exists in a subdirectory", + buildFiles: []string{"1/Android.mk"}, + dirsInTrees: []string{"1/2/3"}, + dir: "1/2/3", + expectedBuildFile: "1/Android.mk", + }, { + description: "top directory", + buildFiles: []string{"Android.bp"}, + dirsInTrees: []string{}, + dir: ".", + expectedBuildFile: "", + }, { + description: "build file exists in subdirectory", + buildFiles: []string{"1/2/3/Android.bp", "1/2/4/Android.bp"}, + dirsInTrees: []string{"1/2/3", "1/2/4"}, + dir: "1/2", + expectedBuildFile: "1/2/Android.mk", + }, { + description: "build file exists in parent subdirectory", + buildFiles: []string{"1/5/Android.bp"}, + dirsInTrees: []string{"1/2/3", "1/2/4", "1/5"}, + dir: "1/2", + expectedBuildFile: "1/Android.mk", + }, { + description: "build file exists in deep parent's subdirectory.", + buildFiles: []string{"1/5/6/Android.bp"}, + dirsInTrees: []string{"1/2/3", "1/2/4", "1/5/6", "1/5/7"}, + dir: "1/2", + expectedBuildFile: "1/Android.mk", + }} + + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + defer logger.Recover(func(err error) { + t.Fatalf("Got unexpected error: %v", err) + }) + + topDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + defer os.RemoveAll(topDir) + + createDirectories(t, topDir, tt.dirsInTrees) + createBuildFiles(t, topDir, tt.buildFiles) + + curDir, err := os.Getwd() + if err != nil { + t.Fatalf("Could not get working directory: %v", err) + } + defer func() { os.Chdir(curDir) }() + if err := os.Chdir(topDir); err != nil { + t.Fatalf("Could not change top dir to %s: %v", topDir, err) + } + + buildFile := findBuildFile(ctx, tt.dir) + if buildFile != tt.expectedBuildFile { + t.Errorf("expected %q, got %q for build file", tt.expectedBuildFile, buildFile) + } + }) + } +} + +func TestConfigSplitArgs(t *testing.T) { + tests := []struct { + // ********* Setup ********* + // Test description. + description string + + // ********* Action ********* + // Arguments passed in to soong_ui. + args []string + + // ********* Validation ********* + // Expected newArgs list after extracting the directories. + expectedNewArgs []string + + // Expected directories + expectedDirs []string + }{{ + description: "flags but no directories specified", + args: []string{"showcommands", "-j", "-k"}, + expectedNewArgs: []string{"showcommands", "-j", "-k"}, + expectedDirs: []string{}, + }, { + description: "flags and one directory specified", + args: []string{"snod", "-j", "dir:target1,target2"}, + expectedNewArgs: []string{"snod", "-j"}, + expectedDirs: []string{"dir:target1,target2"}, + }, { + description: "flags and directories specified", + args: []string{"dist", "-k", "dir1", "dir2:target1,target2"}, + expectedNewArgs: []string{"dist", "-k"}, + expectedDirs: []string{"dir1", "dir2:target1,target2"}, + }, { + description: "only directories specified", + args: []string{"dir1", "dir2", "dir3:target1,target2"}, + expectedNewArgs: []string{}, + expectedDirs: []string{"dir1", "dir2", "dir3:target1,target2"}, + }} + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + args, dirs := splitArgs(tt.args) + if !reflect.DeepEqual(tt.expectedNewArgs, args) { + t.Errorf("expected %v, got %v for arguments", tt.expectedNewArgs, args) + } + if !reflect.DeepEqual(tt.expectedDirs, dirs) { + t.Errorf("expected %v, got %v for directories", tt.expectedDirs, dirs) + } + }) + } +} + +type envVar struct { + name string + value string +} + +type buildActionTestCase struct { + // ********* Setup ********* + // Test description. + description string + + // Directories that exist in the source tree. + dirsInTrees []string + + // Build files that exists in the source tree. + buildFiles []string + + // Create root symlink that points to topDir. + rootSymlink bool + + // ********* Action ********* + // Arguments passed in to soong_ui. + args []string + + // Directory where the build action was invoked. + curDir string + + // WITH_TIDY_ONLY environment variable specified. + tidyOnly string + + // ********* Validation ********* + // Expected arguments to be in Config instance. + expectedArgs []string + + // Expecting error from running test case. + expectedErrStr string +} + +func testGetConfigArgs(t *testing.T, tt buildActionTestCase, action BuildAction) { + ctx := testContext() + + defer logger.Recover(func(err error) { + if tt.expectedErrStr == "" { + t.Fatalf("Got unexpected error: %v", err) + } + if tt.expectedErrStr != err.Error() { + t.Errorf("expected %s, got %s", tt.expectedErrStr, err.Error()) + } + }) + + // Environment variables to set it to blank on every test case run. + resetEnvVars := []string{ + "WITH_TIDY_ONLY", + } + + for _, name := range resetEnvVars { + if err := os.Unsetenv(name); err != nil { + t.Fatalf("failed to unset environment variable %s: %v", name, err) + } + } + if tt.tidyOnly != "" { + if err := os.Setenv("WITH_TIDY_ONLY", tt.tidyOnly); err != nil { + t.Errorf("failed to set WITH_TIDY_ONLY to %s: %v", tt.tidyOnly, err) + } + } + + // Create the root source tree. + topDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + defer os.RemoveAll(topDir) + + createDirectories(t, topDir, tt.dirsInTrees) + createBuildFiles(t, topDir, tt.buildFiles) + + if tt.rootSymlink { + // Create a secondary root source tree which points to the true root source tree. + symlinkTopDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("failed to create symlink temp dir: %v", err) + } + defer os.RemoveAll(symlinkTopDir) + + symlinkTopDir = filepath.Join(symlinkTopDir, "root") + err = os.Symlink(topDir, symlinkTopDir) + if err != nil { + t.Fatalf("failed to create symlink: %v", err) + } + topDir = symlinkTopDir + } + + r := setTop(t, topDir) + defer r() + + // The next block is to create the root build file. + rootBuildFileDir := filepath.Dir(srcDirFileCheck) + if err := os.MkdirAll(rootBuildFileDir, 0755); err != nil { + t.Fatalf("Failed to create %s directory: %v", rootBuildFileDir, err) + } + + if err := ioutil.WriteFile(srcDirFileCheck, []byte{}, 0644); err != nil { + t.Fatalf("failed to create %s file: %v", srcDirFileCheck, err) + } + + args := getConfigArgs(action, tt.curDir, ctx, tt.args) + if !reflect.DeepEqual(tt.expectedArgs, args) { + t.Fatalf("expected %v, got %v for config arguments", tt.expectedArgs, args) + } + + // If the execution reached here and there was an expected error code, the unit test case failed. + if tt.expectedErrStr != "" { + t.Errorf("expecting error %s", tt.expectedErrStr) + } +} + +func TestGetConfigArgsBuildModules(t *testing.T) { + tests := []buildActionTestCase{{ + description: "normal execution from the root source tree directory", + dirsInTrees: []string{"0/1/2", "0/2", "0/3"}, + buildFiles: []string{"0/1/2/Android.mk", "0/2/Android.bp", "0/3/Android.mk"}, + args: []string{"-j", "fake_module", "fake_module2"}, + curDir: ".", + tidyOnly: "", + expectedArgs: []string{"-j", "fake_module", "fake_module2"}, + }, { + description: "normal execution in deep directory", + dirsInTrees: []string{"0/1/2", "0/2", "0/3", "1/2/3/4/5/6/7/8/9/1/2/3/4/5/6"}, + buildFiles: []string{"0/1/2/Android.mk", "0/2/Android.bp", "1/2/3/4/5/6/7/8/9/1/2/3/4/5/6/Android.mk"}, + args: []string{"-j", "fake_module", "fake_module2", "-k"}, + curDir: "1/2/3/4/5/6/7/8/9", + tidyOnly: "", + expectedArgs: []string{"-j", "fake_module", "fake_module2", "-k"}, + }, { + description: "normal execution in deep directory, no targets", + dirsInTrees: []string{"0/1/2", "0/2", "0/3", "1/2/3/4/5/6/7/8/9/1/2/3/4/5/6"}, + buildFiles: []string{"0/1/2/Android.mk", "0/2/Android.bp", "1/2/3/4/5/6/7/8/9/1/2/3/4/5/6/Android.mk"}, + args: []string{"-j", "-k"}, + curDir: "1/2/3/4/5/6/7/8/9", + tidyOnly: "", + expectedArgs: []string{"-j", "-k"}, + }, { + description: "normal execution in root source tree, no args", + dirsInTrees: []string{"0/1/2", "0/2", "0/3"}, + buildFiles: []string{"0/1/2/Android.mk", "0/2/Android.bp"}, + args: []string{}, + curDir: "0/2", + tidyOnly: "", + expectedArgs: []string{}, + }, { + description: "normal execution in symlink root source tree, no args", + dirsInTrees: []string{"0/1/2", "0/2", "0/3"}, + buildFiles: []string{"0/1/2/Android.mk", "0/2/Android.bp"}, + rootSymlink: true, + args: []string{}, + curDir: "0/2", + tidyOnly: "", + expectedArgs: []string{}, + }} + for _, tt := range tests { + t.Run("build action BUILD_MODULES with dependencies, "+tt.description, func(t *testing.T) { + testGetConfigArgs(t, tt, BUILD_MODULES) + }) + } +} + +func TestGetConfigArgsBuildModulesInDirectory(t *testing.T) { + tests := []buildActionTestCase{{ + description: "normal execution in a directory", + dirsInTrees: []string{"0/1/2"}, + buildFiles: []string{"0/1/2/Android.mk"}, + args: []string{"fake-module"}, + curDir: "0/1/2", + tidyOnly: "", + expectedArgs: []string{"fake-module", "MODULES-IN-0-1-2"}, + }, { + description: "build file in parent directory", + dirsInTrees: []string{"0/1/2"}, + buildFiles: []string{"0/1/Android.mk"}, + args: []string{}, + curDir: "0/1/2", + tidyOnly: "", + expectedArgs: []string{"MODULES-IN-0-1"}, + }, + { + description: "build file in parent directory, multiple module names passed in", + dirsInTrees: []string{"0/1/2"}, + buildFiles: []string{"0/1/Android.mk"}, + args: []string{"fake-module1", "fake-module2", "fake-module3"}, + curDir: "0/1/2", + tidyOnly: "", + expectedArgs: []string{"fake-module1", "fake-module2", "fake-module3", "MODULES-IN-0-1"}, + }, { + description: "build file in 2nd level parent directory", + dirsInTrees: []string{"0/1/2"}, + buildFiles: []string{"0/Android.bp"}, + args: []string{}, + curDir: "0/1/2", + tidyOnly: "", + expectedArgs: []string{"MODULES-IN-0"}, + }, { + description: "build action executed at root directory", + dirsInTrees: []string{}, + buildFiles: []string{}, + rootSymlink: false, + args: []string{}, + curDir: ".", + tidyOnly: "", + expectedArgs: []string{}, + }, { + description: "build action executed at root directory in symlink", + dirsInTrees: []string{}, + buildFiles: []string{}, + rootSymlink: true, + args: []string{}, + curDir: ".", + tidyOnly: "", + expectedArgs: []string{}, + }, { + description: "build file not found", + dirsInTrees: []string{"0/1/2"}, + buildFiles: []string{}, + args: []string{}, + curDir: "0/1/2", + tidyOnly: "", + expectedArgs: []string{"MODULES-IN-0-1-2"}, + expectedErrStr: "Build file not found for 0/1/2 directory", + }, { + description: "GET-INSTALL-PATH specified,", + dirsInTrees: []string{"0/1/2"}, + buildFiles: []string{"0/1/Android.mk"}, + args: []string{"GET-INSTALL-PATH", "-j", "-k", "GET-INSTALL-PATH"}, + curDir: "0/1/2", + tidyOnly: "", + expectedArgs: []string{"-j", "-k", "GET-INSTALL-PATH-IN-0-1"}, + }, { + description: "tidy only environment variable specified,", + dirsInTrees: []string{"0/1/2"}, + buildFiles: []string{"0/1/Android.mk"}, + args: []string{"GET-INSTALL-PATH"}, + curDir: "0/1/2", + tidyOnly: "true", + expectedArgs: []string{"tidy_only"}, + }, { + description: "normal execution in root directory with args", + dirsInTrees: []string{}, + buildFiles: []string{}, + args: []string{"-j", "-k", "fake_module"}, + curDir: "", + tidyOnly: "", + expectedArgs: []string{"-j", "-k", "fake_module"}, + }} + for _, tt := range tests { + t.Run("build action BUILD_MODULES_IN_DIR, "+tt.description, func(t *testing.T) { + testGetConfigArgs(t, tt, BUILD_MODULES_IN_A_DIRECTORY) + }) + } +} + +func TestGetConfigArgsBuildModulesInDirectories(t *testing.T) { + tests := []buildActionTestCase{{ + description: "normal execution in a directory", + dirsInTrees: []string{"0/1/2/3.1", "0/1/2/3.2", "0/1/2/3.3"}, + buildFiles: []string{"0/1/2/3.1/Android.bp", "0/1/2/3.2/Android.bp", "0/1/2/3.3/Android.bp"}, + args: []string{"3.1/", "3.2/", "3.3/"}, + curDir: "0/1/2", + tidyOnly: "", + expectedArgs: []string{"MODULES-IN-0-1-2-3.1", "MODULES-IN-0-1-2-3.2", "MODULES-IN-0-1-2-3.3"}, + }, { + description: "GET-INSTALL-PATH specified", + dirsInTrees: []string{"0/1/2/3.1", "0/1/2/3.2", "0/1/3"}, + buildFiles: []string{"0/1/2/3.1/Android.bp", "0/1/2/3.2/Android.bp", "0/1/Android.bp"}, + args: []string{"GET-INSTALL-PATH", "2/3.1/", "2/3.2", "3"}, + curDir: "0/1", + tidyOnly: "", + expectedArgs: []string{"GET-INSTALL-PATH-IN-0-1-2-3.1", "GET-INSTALL-PATH-IN-0-1-2-3.2", "GET-INSTALL-PATH-IN-0-1"}, + }, { + description: "tidy only environment variable specified", + dirsInTrees: []string{"0/1/2/3.1", "0/1/2/3.2", "0/1/2/3.3"}, + buildFiles: []string{"0/1/2/3.1/Android.bp", "0/1/2/3.2/Android.bp", "0/1/2/3.3/Android.bp"}, + args: []string{"GET-INSTALL-PATH", "3.1/", "3.2/", "3.3"}, + curDir: "0/1/2", + tidyOnly: "1", + expectedArgs: []string{"tidy_only"}, + }, { + description: "normal execution from top dir directory", + dirsInTrees: []string{"0/1/2/3.1", "0/1/2/3.2", "0/1/3", "0/2"}, + buildFiles: []string{"0/1/2/3.1/Android.bp", "0/1/2/3.2/Android.bp", "0/1/3/Android.bp", "0/2/Android.bp"}, + rootSymlink: false, + args: []string{"0/1/2/3.1", "0/1/2/3.2", "0/1/3", "0/2"}, + curDir: ".", + tidyOnly: "", + expectedArgs: []string{"MODULES-IN-0-1-2-3.1", "MODULES-IN-0-1-2-3.2", "MODULES-IN-0-1-3", "MODULES-IN-0-2"}, + }, { + description: "normal execution from top dir directory in symlink", + dirsInTrees: []string{"0/1/2/3.1", "0/1/2/3.2", "0/1/3", "0/2"}, + buildFiles: []string{"0/1/2/3.1/Android.bp", "0/1/2/3.2/Android.bp", "0/1/3/Android.bp", "0/2/Android.bp"}, + rootSymlink: true, + args: []string{"0/1/2/3.1", "0/1/2/3.2", "0/1/3", "0/2"}, + curDir: ".", + tidyOnly: "", + expectedArgs: []string{"MODULES-IN-0-1-2-3.1", "MODULES-IN-0-1-2-3.2", "MODULES-IN-0-1-3", "MODULES-IN-0-2"}, + }} + for _, tt := range tests { + t.Run("build action BUILD_MODULES_IN_DIRS, "+tt.description, func(t *testing.T) { + testGetConfigArgs(t, tt, BUILD_MODULES_IN_DIRECTORIES) + }) + } +} diff --git a/ui/build/context.go b/ui/build/context.go index 249e89822..3945ce007 100644 --- a/ui/build/context.go +++ b/ui/build/context.go @@ -16,12 +16,12 @@ package build import ( "context" + "io" "android/soong/ui/logger" "android/soong/ui/metrics" "android/soong/ui/metrics/metrics_proto" "android/soong/ui/status" - "android/soong/ui/terminal" "android/soong/ui/tracer" ) @@ -35,7 +35,7 @@ type ContextImpl struct { Metrics *metrics.Metrics - Writer terminal.Writer + Writer io.Writer Status *status.Status Thread tracer.Thread @@ -70,7 +70,7 @@ func (c ContextImpl) CompleteTrace(name, desc string, begin, end uint64) { if c.Metrics != nil { realTime := end - begin c.Metrics.SetTimeMetrics( - metrics_proto.PerfInfo{ + soong_metrics_proto.PerfInfo{ Desc: &desc, Name: &name, StartTime: &begin, diff --git a/ui/build/dumpvars.go b/ui/build/dumpvars.go index 2524e274b..bd073e53f 100644 --- a/ui/build/dumpvars.go +++ b/ui/build/dumpvars.go @@ -17,6 +17,8 @@ package build import ( "bytes" "fmt" + "io/ioutil" + "os" "strings" "android/soong/ui/metrics" @@ -40,6 +42,7 @@ func DumpMakeVars(ctx Context, config Config, goals, vars []string) (map[string] soongUiVars := map[string]func() string{ "OUT_DIR": func() string { return config.OutDir() }, "DIST_DIR": func() string { return config.DistDir() }, + "TMPDIR": func() string { return absPath(ctx, config.TempDir()) }, } makeVars := make([]string, 0, len(vars)) @@ -51,8 +54,16 @@ func DumpMakeVars(ctx Context, config Config, goals, vars []string) (map[string] var ret map[string]string if len(makeVars) > 0 { - var err error - ret, err = dumpMakeVars(ctx, config, goals, makeVars, false) + // It's not safe to use the same TMPDIR as the build, as that can be removed. + tmpDir, err := ioutil.TempDir("", "dumpvars") + if err != nil { + return nil, err + } + defer os.RemoveAll(tmpDir) + + SetupLitePath(ctx, config, tmpDir) + + ret, err = dumpMakeVars(ctx, config, goals, makeVars, false, tmpDir) if err != nil { return ret, err } @@ -69,7 +80,7 @@ func DumpMakeVars(ctx Context, config Config, goals, vars []string) (map[string] return ret, nil } -func dumpMakeVars(ctx Context, config Config, goals, vars []string, write_soong_vars bool) (map[string]string, error) { +func dumpMakeVars(ctx Context, config Config, goals, vars []string, write_soong_vars bool, tmpDir string) (map[string]string, error) { ctx.BeginTrace(metrics.RunKati, "dumpvars") defer ctx.EndTrace() @@ -85,6 +96,9 @@ func dumpMakeVars(ctx Context, config Config, goals, vars []string, write_soong_ cmd.Environment.Set("WRITE_SOONG_VARIABLES", "true") } cmd.Environment.Set("DUMP_MANY_VARS", strings.Join(vars, " ")) + if tmpDir != "" { + cmd.Environment.Set("TMPDIR", tmpDir) + } cmd.Sandbox = dumpvarsSandbox output := bytes.Buffer{} cmd.Stdout = &output @@ -169,7 +183,7 @@ func runMakeProductConfig(ctx Context, config Config) { // Variables to export into the environment of Kati/Ninja exportEnvVars := []string{ // So that we can use the correct TARGET_PRODUCT if it's been - // modified by PRODUCT-*/APP-* arguments + // modified by a buildspec.mk "TARGET_PRODUCT", "TARGET_BUILD_VARIANT", "TARGET_BUILD_APPS", @@ -203,20 +217,53 @@ func runMakeProductConfig(ctx Context, config Config) { // Whether --werror_overriding_commands will work "BUILD_BROKEN_DUP_RULES", - // Used to turn on --werror_ options in Kati - "BUILD_BROKEN_PHONY_TARGETS", - // Whether to enable the network during the build "BUILD_BROKEN_USES_NETWORK", + // Extra environment variables to be exported to ninja + "BUILD_BROKEN_NINJA_USES_ENV_VARS", + // Not used, but useful to be in the soong.log "BOARD_VNDK_VERSION", - "BUILD_BROKEN_ANDROIDMK_EXPORTS", - "BUILD_BROKEN_DUP_COPY_HEADERS", - "BUILD_BROKEN_ENG_DEBUG_TAGS", + + "DEFAULT_WARNING_BUILD_MODULE_TYPES", + "DEFAULT_ERROR_BUILD_MODULE_TYPES", + "BUILD_BROKEN_PREBUILT_ELF_FILES", + "BUILD_BROKEN_TREBLE_SYSPROP_NEVERALLOW", + "BUILD_BROKEN_USES_BUILD_AUX_EXECUTABLE", + "BUILD_BROKEN_USES_BUILD_AUX_STATIC_LIBRARY", + "BUILD_BROKEN_USES_BUILD_COPY_HEADERS", + "BUILD_BROKEN_USES_BUILD_EXECUTABLE", + "BUILD_BROKEN_USES_BUILD_FUZZ_TEST", + "BUILD_BROKEN_USES_BUILD_HEADER_LIBRARY", + "BUILD_BROKEN_USES_BUILD_HOST_DALVIK_JAVA_LIBRARY", + "BUILD_BROKEN_USES_BUILD_HOST_DALVIK_STATIC_JAVA_LIBRARY", + "BUILD_BROKEN_USES_BUILD_HOST_EXECUTABLE", + "BUILD_BROKEN_USES_BUILD_HOST_FUZZ_TEST", + "BUILD_BROKEN_USES_BUILD_HOST_JAVA_LIBRARY", + "BUILD_BROKEN_USES_BUILD_HOST_NATIVE_TEST", + "BUILD_BROKEN_USES_BUILD_HOST_PREBUILT", + "BUILD_BROKEN_USES_BUILD_HOST_SHARED_LIBRARY", + "BUILD_BROKEN_USES_BUILD_HOST_STATIC_LIBRARY", + "BUILD_BROKEN_USES_BUILD_HOST_STATIC_TEST_LIBRARY", + "BUILD_BROKEN_USES_BUILD_HOST_TEST_CONFIG", + "BUILD_BROKEN_USES_BUILD_JAVA_LIBRARY", + "BUILD_BROKEN_USES_BUILD_MULTI_PREBUILT", + "BUILD_BROKEN_USES_BUILD_NATIVE_BENCHMARK", + "BUILD_BROKEN_USES_BUILD_NATIVE_TEST", + "BUILD_BROKEN_USES_BUILD_NOTICE_FILE", + "BUILD_BROKEN_USES_BUILD_PACKAGE", + "BUILD_BROKEN_USES_BUILD_PHONY_PACKAGE", + "BUILD_BROKEN_USES_BUILD_PREBUILT", + "BUILD_BROKEN_USES_BUILD_RRO_PACKAGE", + "BUILD_BROKEN_USES_BUILD_SHARED_LIBRARY", + "BUILD_BROKEN_USES_BUILD_STATIC_JAVA_LIBRARY", + "BUILD_BROKEN_USES_BUILD_STATIC_LIBRARY", + "BUILD_BROKEN_USES_BUILD_STATIC_TEST_LIBRARY", + "BUILD_BROKEN_USES_BUILD_TARGET_TEST_CONFIG", }, exportEnvVars...), BannerVars...) - make_vars, err := dumpMakeVars(ctx, config, config.Arguments(), allVars, true) + make_vars, err := dumpMakeVars(ctx, config, config.Arguments(), allVars, true, "") if err != nil { ctx.Fatalln("Error dumping make vars:", err) } @@ -224,7 +271,7 @@ func runMakeProductConfig(ctx Context, config Config) { env := config.Environment() // Print the banner like make does if !env.IsEnvTrue("ANDROID_QUIET_BUILD") { - ctx.Writer.Print(Banner(make_vars)) + fmt.Fprintln(ctx.Writer, Banner(make_vars)) } // Populate the environment @@ -243,6 +290,6 @@ func runMakeProductConfig(ctx Context, config Config) { config.SetPdkBuild(make_vars["TARGET_BUILD_PDK"] == "true") config.SetBuildBrokenDupRules(make_vars["BUILD_BROKEN_DUP_RULES"] == "true") - config.SetBuildBrokenPhonyTargets(make_vars["BUILD_BROKEN_PHONY_TARGETS"] == "true") config.SetBuildBrokenUsesNetwork(make_vars["BUILD_BROKEN_USES_NETWORK"] == "true") + config.SetBuildBrokenNinjaUsesEnvVars(strings.Fields(make_vars["BUILD_BROKEN_NINJA_USES_ENV_VARS"])) } diff --git a/ui/build/environment.go b/ui/build/environment.go index d8ff7f207..9bca7c06d 100644 --- a/ui/build/environment.go +++ b/ui/build/environment.go @@ -19,6 +19,7 @@ import ( "fmt" "io" "os" + "strconv" "strings" ) @@ -44,6 +45,17 @@ func (e *Environment) Get(key string) (string, bool) { return "", false } +// Get returns the int value associated with the key, and whether it exists +// and is a valid int. +func (e *Environment) GetInt(key string) (int, bool) { + if v, ok := e.Get(key); ok { + if i, err := strconv.Atoi(v); err == nil { + return i, true + } + } + return 0, false +} + // Set sets the value associated with the key, overwriting the current value // if it exists. func (e *Environment) Set(key, value string) { diff --git a/ui/build/exec.go b/ui/build/exec.go index 5c312bcd9..e435c53be 100644 --- a/ui/build/exec.go +++ b/ui/build/exec.go @@ -15,7 +15,10 @@ package build import ( + "bufio" + "io" "os/exec" + "strings" ) // Cmd is a wrapper of os/exec.Cmd that integrates with the build context for @@ -139,3 +142,34 @@ func (c *Cmd) RunAndPrintOrFatal() { st.Finish() c.reportError(err) } + +// RunAndStreamOrFatal will run the command, while running print +// any output, then handle any errors with a call to ctx.Fatal +func (c *Cmd) RunAndStreamOrFatal() { + out, err := c.StdoutPipe() + if err != nil { + c.ctx.Fatal(err) + } + c.Stderr = c.Stdout + + st := c.ctx.Status.StartTool() + + c.StartOrFatal() + + buf := bufio.NewReaderSize(out, 2*1024*1024) + for { + // Attempt to read whole lines, but write partial lines that are too long to fit in the buffer or hit EOF + line, err := buf.ReadString('\n') + if line != "" { + st.Print(strings.TrimSuffix(line, "\n")) + } else if err == io.EOF { + break + } else if err != nil { + c.ctx.Fatal(err) + } + } + + err = c.Wait() + st.Finish() + c.reportError(err) +} diff --git a/ui/build/goma.go b/ui/build/goma.go index ff0b40e6d..ae9b78498 100644 --- a/ui/build/goma.go +++ b/ui/build/goma.go @@ -72,6 +72,7 @@ func startGoma(ctx Context, config Config) { } cmd := Command(ctx, config, "goma_ctl.py ensure_start", gomaCtl, "ensure_start") + cmd.Environment.Set("DIST_DIR", config.DistDir()) if output, err := cmd.CombinedOutput(); err != nil { ctx.Fatalf("goma_ctl.py ensure_start failed with: %v\n%s\n", err, output) diff --git a/ui/build/kati.go b/ui/build/kati.go index 959d0bdf4..a845c5be2 100644 --- a/ui/build/kati.go +++ b/ui/build/kati.go @@ -42,9 +42,6 @@ func genKatiSuffix(ctx Context, config Config) { if args := config.KatiArgs(); len(args) > 0 { katiSuffix += "-" + spaceSlashReplacer.Replace(strings.Join(args, "_")) } - if oneShot, ok := config.Environment().Get("ONE_SHOT_MAKEFILE"); ok { - katiSuffix += "-" + spaceSlashReplacer.Replace(oneShot) - } // If the suffix is too long, replace it with a md5 hash and write a // file that contains the original suffix. @@ -81,6 +78,9 @@ func runKati(ctx Context, config Config, extraSuffix string, args []string, envF "--werror_suffix_rules", "--warn_real_to_phony", "--warn_phony_looks_real", + "--werror_real_to_phony", + "--werror_phony_looks_real", + "--werror_writable", "--top_level_phony", "--kati_stats", }, args...) @@ -89,6 +89,10 @@ func runKati(ctx Context, config Config, extraSuffix string, args []string, envF args = append(args, "--empty_ninja_file") } + if config.UseRemoteBuild() { + args = append(args, "--default_pool=local_pool") + } + cmd := Command(ctx, config, "ckati", executable, args...) cmd.Sandbox = katiSandbox pipe, err := cmd.StdoutPipe() @@ -138,13 +142,6 @@ func runKatiBuild(ctx Context, config Config) { args = append(args, "--werror_overriding_commands") } - if !config.BuildBrokenPhonyTargets() { - args = append(args, - "--werror_real_to_phony", - "--werror_phony_looks_real", - "--werror_writable") - } - args = append(args, config.KatiArgs()...) args = append(args, @@ -154,6 +151,63 @@ func runKatiBuild(ctx Context, config Config) { "KATI_PACKAGE_MK_DIR="+config.KatiPackageMkDir()) runKati(ctx, config, katiBuildSuffix, args, func(env *Environment) {}) + + cleanCopyHeaders(ctx, config) + cleanOldInstalledFiles(ctx, config) +} + +func cleanCopyHeaders(ctx Context, config Config) { + ctx.BeginTrace("clean", "clean copy headers") + defer ctx.EndTrace() + + data, err := ioutil.ReadFile(filepath.Join(config.ProductOut(), ".copied_headers_list")) + if err != nil { + if os.IsNotExist(err) { + return + } + ctx.Fatalf("Failed to read copied headers list: %v", err) + } + + headers := strings.Fields(string(data)) + if len(headers) < 1 { + ctx.Fatal("Failed to parse copied headers list: %q", string(data)) + } + headerDir := headers[0] + headers = headers[1:] + + filepath.Walk(headerDir, + func(path string, info os.FileInfo, err error) error { + if err != nil { + return nil + } + if info.IsDir() { + return nil + } + if !inList(path, headers) { + ctx.Printf("Removing obsolete header %q", path) + if err := os.Remove(path); err != nil { + ctx.Fatalf("Failed to remove obsolete header %q: %v", path, err) + } + } + return nil + }) +} + +func cleanOldInstalledFiles(ctx Context, config Config) { + ctx.BeginTrace("clean", "clean old installed files") + defer ctx.EndTrace() + + // We shouldn't be removing files from one side of the two-step asan builds + var suffix string + if v, ok := config.Environment().Get("SANITIZE_TARGET"); ok { + if sanitize := strings.Fields(v); inList("address", sanitize) { + suffix = "_asan" + } + } + + cleanOldFiles(ctx, config.ProductOut(), ".installable_files"+suffix) + + cleanOldFiles(ctx, config.HostOut(), ".installable_test_files") } func runKatiPackage(ctx Context, config Config) { @@ -162,11 +216,8 @@ func runKatiPackage(ctx Context, config Config) { args := []string{ "--writable", config.DistDir() + "/", - "--werror_writable", "--werror_implicit_rules", "--werror_overriding_commands", - "--werror_real_to_phony", - "--werror_phony_looks_real", "-f", "build/make/packaging/main.mk", "KATI_PACKAGE_MK_DIR=" + config.KatiPackageMkDir(), } @@ -181,6 +232,7 @@ func runKatiPackage(ctx Context, config Config) { "TMPDIR", // Tool configs + "ASAN_SYMBOLIZER_PATH", "JAVA_HOME", "PYTHONDONTWRITEBYTECODE", @@ -202,11 +254,8 @@ func runKatiCleanSpec(ctx Context, config Config) { defer ctx.EndTrace() runKati(ctx, config, katiCleanspecSuffix, []string{ - "--werror_writable", "--werror_implicit_rules", "--werror_overriding_commands", - "--werror_real_to_phony", - "--werror_phony_looks_real", "-f", "build/make/core/cleanbuild.mk", "SOONG_MAKEVARS_MK=" + config.SoongMakeVarsMk(), "TARGET_DEVICE_DIR=" + config.TargetDeviceDir(), diff --git a/ui/build/ninja.go b/ui/build/ninja.go index 7497c94bd..dfc3be1f3 100644 --- a/ui/build/ninja.go +++ b/ui/build/ninja.go @@ -18,6 +18,7 @@ import ( "fmt" "os" "path/filepath" + "sort" "strconv" "strings" "time" @@ -37,13 +38,14 @@ func runNinja(ctx Context, config Config) { executable := config.PrebuiltBuildTool("ninja") args := []string{ "-d", "keepdepfile", + "-d", "keeprsp", "--frontend_file", fifo, } args = append(args, config.NinjaArgs()...) var parallel int - if config.UseGoma() || config.UseRBE() { + if config.UseRemoteBuild() { parallel = config.RemoteParallel() } else { parallel = config.Parallel() @@ -65,8 +67,6 @@ func runNinja(ctx Context, config Config) { cmd.Environment.AppendFromKati(config.KatiEnvFile()) } - cmd.Environment.Set("DIST_DIR", config.DistDir()) - // Allow both NINJA_ARGS and NINJA_EXTRA_ARGS, since both have been // used in the past to specify extra ninja arguments. if extra, ok := cmd.Environment.Get("NINJA_ARGS"); ok { @@ -85,6 +85,88 @@ func runNinja(ctx Context, config Config) { ninjaHeartbeatDuration = overrideDuration } } + + // Filter the environment, as ninja does not rebuild files when environment variables change. + // + // Anything listed here must not change the output of rules/actions when the value changes, + // otherwise incremental builds may be unsafe. Vars explicitly set to stable values + // elsewhere in soong_ui are fine. + // + // For the majority of cases, either Soong or the makefiles should be replicating any + // necessary environment variables in the command line of each action that needs it. + if cmd.Environment.IsEnvTrue("ALLOW_NINJA_ENV") { + ctx.Println("Allowing all environment variables during ninja; incremental builds may be unsafe.") + } else { + cmd.Environment.Allow(append([]string{ + "ASAN_SYMBOLIZER_PATH", + "HOME", + "JAVA_HOME", + "LANG", + "LC_MESSAGES", + "OUT_DIR", + "PATH", + "PWD", + "PYTHONDONTWRITEBYTECODE", + "TMPDIR", + "USER", + + // TODO: remove these carefully + "ASAN_OPTIONS", + "TARGET_BUILD_APPS", + "TARGET_BUILD_VARIANT", + "TARGET_PRODUCT", + // b/147197813 - used by art-check-debug-apex-gen + "EMMA_INSTRUMENT_FRAMEWORK", + + // Goma -- gomacc may not need all of these + "GOMA_DIR", + "GOMA_DISABLED", + "GOMA_FAIL_FAST", + "GOMA_FALLBACK", + "GOMA_GCE_SERVICE_ACCOUNT", + "GOMA_TMP_DIR", + "GOMA_USE_LOCAL", + + // RBE client + "RBE_compare", + "RBE_exec_root", + "RBE_exec_strategy", + "RBE_invocation_id", + "RBE_log_dir", + "RBE_platform", + "RBE_remote_accept_cache", + "RBE_remote_update_cache", + "RBE_server_address", + // TODO: remove old FLAG_ variables. + "FLAG_compare", + "FLAG_exec_root", + "FLAG_exec_strategy", + "FLAG_invocation_id", + "FLAG_log_dir", + "FLAG_platform", + "FLAG_remote_accept_cache", + "FLAG_remote_update_cache", + "FLAG_server_address", + + // ccache settings + "CCACHE_COMPILERCHECK", + "CCACHE_SLOPPINESS", + "CCACHE_BASEDIR", + "CCACHE_CPP2", + "CCACHE_DIR", + }, config.BuildBrokenNinjaUsesEnvVars()...)...) + } + + cmd.Environment.Set("DIST_DIR", config.DistDir()) + cmd.Environment.Set("SHELL", "/bin/bash") + + ctx.Verboseln("Ninja environment: ") + envVars := cmd.Environment.Environ() + sort.Strings(envVars) + for _, envVar := range envVars { + ctx.Verbosef(" %s", envVar) + } + // Poll the ninja log for updates; if it isn't updated enough, then we want to show some diagnostics done := make(chan struct{}) defer close(done) @@ -103,7 +185,7 @@ func runNinja(ctx Context, config Config) { }() ctx.Status.Status("Starting ninja...") - cmd.RunAndPrintOrFatal() + cmd.RunAndStreamOrFatal() } type statusChecker struct { diff --git a/ui/build/path.go b/ui/build/path.go index 0e1c02cac..f515775e3 100644 --- a/ui/build/path.go +++ b/ui/build/path.go @@ -18,6 +18,7 @@ import ( "fmt" "io/ioutil" "os" + "os/exec" "path/filepath" "runtime" "strings" @@ -53,6 +54,55 @@ func parsePathDir(dir string) []string { return ret } +// A "lite" version of SetupPath used for dumpvars, or other places that need +// minimal overhead (but at the expense of logging). If tmpDir is empty, the +// default TMPDIR is used from config. +func SetupLitePath(ctx Context, config Config, tmpDir string) { + if config.pathReplaced { + return + } + + ctx.BeginTrace(metrics.RunSetupTool, "litepath") + defer ctx.EndTrace() + + origPath, _ := config.Environment().Get("PATH") + + if tmpDir == "" { + tmpDir, _ = config.Environment().Get("TMPDIR") + } + myPath := filepath.Join(tmpDir, "path") + ensureEmptyDirectoriesExist(ctx, myPath) + + os.Setenv("PATH", origPath) + for name, pathConfig := range paths.Configuration { + if !pathConfig.Symlink { + continue + } + + origExec, err := exec.LookPath(name) + if err != nil { + continue + } + origExec, err = filepath.Abs(origExec) + if err != nil { + continue + } + + err = os.Symlink(origExec, filepath.Join(myPath, name)) + if err != nil { + ctx.Fatalln("Failed to create symlink:", err) + } + } + + myPath, _ = filepath.Abs(myPath) + + prebuiltsPath, _ := filepath.Abs("prebuilts/build-tools/path/" + runtime.GOOS + "-x86") + myPath = prebuiltsPath + string(os.PathListSeparator) + myPath + + config.Environment().Set("PATH", myPath) + config.pathReplaced = true +} + func SetupPath(ctx Context, config Config) { if config.pathReplaced { return diff --git a/ui/build/paths/config.go b/ui/build/paths/config.go index f4bb89fbb..571740133 100644 --- a/ui/build/paths/config.go +++ b/ui/build/paths/config.go @@ -74,41 +74,27 @@ func GetConfig(name string) PathConfig { } var Configuration = map[string]PathConfig{ - "bash": Allowed, - "bc": Allowed, - "bzip2": Allowed, - "date": Allowed, - "dd": Allowed, - "diff": Allowed, - "egrep": Allowed, - "expr": Allowed, - "find": Allowed, - "fuser": Allowed, - "getopt": Allowed, - "git": Allowed, - "grep": Allowed, - "gzip": Allowed, - "hexdump": Allowed, - "jar": Allowed, - "java": Allowed, - "javap": Allowed, - "lsof": Allowed, - "m4": Allowed, - "openssl": Allowed, - "patch": Allowed, - "pstree": Allowed, - "python3": Allowed, - "realpath": Allowed, - "rsync": Allowed, - "sed": Allowed, - "sh": Allowed, - "tar": Allowed, - "timeout": Allowed, - "tr": Allowed, - "unzip": Allowed, - "xz": Allowed, - "zip": Allowed, - "zipinfo": Allowed, + "bash": Allowed, + "dd": Allowed, + "diff": Allowed, + "dlv": Allowed, + "expr": Allowed, + "fuser": Allowed, + "getopt": Allowed, + "git": Allowed, + "hexdump": Allowed, + "jar": Allowed, + "java": Allowed, + "javap": Allowed, + "lsof": Allowed, + "openssl": Allowed, + "patch": Allowed, + "pstree": Allowed, + "rsync": Allowed, + "sh": Allowed, + "tr": Allowed, + "unzip": Allowed, + "zip": Allowed, // Host toolchain is removed. In-tree toolchain should be used instead. // GCC also can't find cc1 with this implementation. @@ -124,66 +110,18 @@ var Configuration = map[string]PathConfig{ "ld.gold": Forbidden, "pkg-config": Forbidden, - // On Linux we'll use the toybox versions of these instead. - "basename": LinuxOnlyPrebuilt, - "cat": LinuxOnlyPrebuilt, - "chmod": LinuxOnlyPrebuilt, - "cmp": LinuxOnlyPrebuilt, - "cp": LinuxOnlyPrebuilt, - "comm": LinuxOnlyPrebuilt, - "cut": LinuxOnlyPrebuilt, - "dirname": LinuxOnlyPrebuilt, - "du": LinuxOnlyPrebuilt, - "echo": LinuxOnlyPrebuilt, - "env": LinuxOnlyPrebuilt, - "head": LinuxOnlyPrebuilt, - "getconf": LinuxOnlyPrebuilt, - "hostname": LinuxOnlyPrebuilt, - "id": LinuxOnlyPrebuilt, - "ln": LinuxOnlyPrebuilt, - "ls": LinuxOnlyPrebuilt, - "md5sum": LinuxOnlyPrebuilt, - "mkdir": LinuxOnlyPrebuilt, - "mktemp": LinuxOnlyPrebuilt, - "mv": LinuxOnlyPrebuilt, - "od": LinuxOnlyPrebuilt, - "paste": LinuxOnlyPrebuilt, - "pgrep": LinuxOnlyPrebuilt, - "pkill": LinuxOnlyPrebuilt, - "ps": LinuxOnlyPrebuilt, - "pwd": LinuxOnlyPrebuilt, - "readlink": LinuxOnlyPrebuilt, - "rm": LinuxOnlyPrebuilt, - "rmdir": LinuxOnlyPrebuilt, - "seq": LinuxOnlyPrebuilt, - "setsid": LinuxOnlyPrebuilt, - "sha1sum": LinuxOnlyPrebuilt, - "sha256sum": LinuxOnlyPrebuilt, - "sha512sum": LinuxOnlyPrebuilt, - "sleep": LinuxOnlyPrebuilt, - "sort": LinuxOnlyPrebuilt, - "stat": LinuxOnlyPrebuilt, - "tail": LinuxOnlyPrebuilt, - "tee": LinuxOnlyPrebuilt, - "touch": LinuxOnlyPrebuilt, - "true": LinuxOnlyPrebuilt, - "uname": LinuxOnlyPrebuilt, - "uniq": LinuxOnlyPrebuilt, - "unix2dos": LinuxOnlyPrebuilt, - "wc": LinuxOnlyPrebuilt, - "whoami": LinuxOnlyPrebuilt, - "which": LinuxOnlyPrebuilt, - "xargs": LinuxOnlyPrebuilt, - "xxd": LinuxOnlyPrebuilt, + // These are toybox tools that only work on Linux. + "pgrep": LinuxOnlyPrebuilt, + "pkill": LinuxOnlyPrebuilt, + "ps": LinuxOnlyPrebuilt, } func init() { if runtime.GOOS == "darwin" { - Configuration["md5"] = Allowed Configuration["sw_vers"] = Allowed Configuration["xcrun"] = Allowed - // We don't have darwin prebuilts for some tools (like toybox), + // We don't have darwin prebuilts for some tools, // so allow the host versions. for name, config := range Configuration { if config.LinuxOnlyPrebuilt { diff --git a/ui/build/proc_sync.go b/ui/build/proc_sync.go index 857786dab..0cfe79834 100644 --- a/ui/build/proc_sync.go +++ b/ui/build/proc_sync.go @@ -34,6 +34,14 @@ func BecomeSingletonOrFail(ctx Context, config Config) (lock *fileLock) { if err != nil { ctx.Logger.Fatal(err) } + lockfilePollDuration := time.Second + lockfileTimeout := time.Second * 10 + if envTimeout := os.Getenv("SOONG_LOCK_TIMEOUT"); envTimeout != "" { + lockfileTimeout, err = time.ParseDuration(envTimeout) + if err != nil { + ctx.Logger.Fatalf("failure parsing SOONG_LOCK_TIMEOUT %q: %s", envTimeout, err) + } + } err = lockSynchronous(*lockingInfo, newSleepWaiter(lockfilePollDuration, lockfileTimeout), ctx.Logger) if err != nil { ctx.Logger.Fatal(err) @@ -41,9 +49,6 @@ func BecomeSingletonOrFail(ctx Context, config Config) (lock *fileLock) { return lockingInfo } -var lockfileTimeout = time.Second * 10 -var lockfilePollDuration = time.Second - type lockable interface { tryLock() error Unlock() error @@ -80,15 +85,18 @@ func lockSynchronous(lock lockable, waiter waiter, logger logger.Logger) (err er return nil } - waited = true - done, description := waiter.checkDeadline() + if !waited { + logger.Printf("Waiting up to %s to lock %v to ensure no other Soong process is running in the same output directory\n", description, lock.description()) + } + + waited = true + if done { return fmt.Errorf("Tried to lock %s, but timed out %s . Make sure no other Soong process is using it", lock.description(), waiter.summarize()) } else { - logger.Printf("Waiting up to %s to lock %v to ensure no other Soong process is running in the same output directory\n", description, lock.description()) waiter.wait() } } diff --git a/ui/build/rbe_test.go b/ui/build/rbe_test.go index d3a52b7ef..c56a33c0a 100644 --- a/ui/build/rbe_test.go +++ b/ui/build/rbe_test.go @@ -46,9 +46,9 @@ func TestDumpRBEMetrics(t *testing.T) { for _, tt := range tests { t.Run(tt.description, func(t *testing.T) { - tmpDir, err := ioutil.TempDir("", "tmpdir") + tmpDir, err := ioutil.TempDir("", "") if err != nil { - t.Fatalf("failed to make temp dir: %v", err) + t.Fatalf("failed to create a temp directory: %v", err) } defer os.RemoveAll(tmpDir) @@ -60,7 +60,14 @@ func TestDumpRBEMetrics(t *testing.T) { env := Environment(tt.env) env.Set("OUT_DIR", tmpDir) env.Set("RBE_DIR", tmpDir) - env.Set("RBE_output_dir", tmpDir) + + tmpRBEDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("failed to create a temp directory for RBE: %v", err) + } + defer os.RemoveAll(tmpRBEDir) + env.Set("RBE_output_dir", tmpRBEDir) + config := Config{&configImpl{ environ: &env, }} @@ -105,9 +112,9 @@ func TestDumpRBEMetricsErrors(t *testing.T) { } }) - tmpDir, err := ioutil.TempDir("", "tmpdir") + tmpDir, err := ioutil.TempDir("", "") if err != nil { - t.Fatalf("failed to make temp dir: %v", err) + t.Fatalf("failed to create a temp directory: %v", err) } defer os.RemoveAll(tmpDir) diff --git a/ui/build/sandbox_linux.go b/ui/build/sandbox_linux.go index b94db7448..2de772b00 100644 --- a/ui/build/sandbox_linux.go +++ b/ui/build/sandbox_linux.go @@ -90,10 +90,7 @@ func (c *Cmd) sandboxSupported() bool { return } - c.ctx.Println("Build sandboxing disabled due to nsjail error. This may become fatal in the future.") - c.ctx.Println("Please let us know why nsjail doesn't work in your environment at:") - c.ctx.Println(" https://groups.google.com/forum/#!forum/android-building") - c.ctx.Println(" https://issuetracker.google.com/issues/new?component=381517") + c.ctx.Println("Build sandboxing disabled due to nsjail error.") for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") { c.ctx.Verboseln(line) @@ -162,6 +159,10 @@ func (c *Cmd) wrapSandbox() { c.ctx.Printf("AllowBuildBrokenUsesNetwork: %v", c.Sandbox.AllowBuildBrokenUsesNetwork) c.ctx.Printf("BuildBrokenUsesNetwork: %v", c.config.BuildBrokenUsesNetwork()) sandboxArgs = append(sandboxArgs, "-N") + } else if dlv, _ := c.config.Environment().Get("SOONG_DELVE"); dlv != "" { + // The debugger is enabled and soong_build will pause until a remote delve process connects, allow + // network connections. + sandboxArgs = append(sandboxArgs, "-N") } // Stop nsjail from parsing arguments diff --git a/ui/build/soong.go b/ui/build/soong.go index 2ce1ac935..afbc0734b 100644 --- a/ui/build/soong.go +++ b/ui/build/soong.go @@ -119,8 +119,9 @@ func runSoong(ctx Context, config Config) { "-j", strconv.Itoa(config.Parallel()), "--frontend_file", fifo, "-f", filepath.Join(config.SoongOutDir(), file)) + cmd.Environment.Set("SOONG_SANDBOX_SOONG_BUILD", "true") cmd.Sandbox = soongSandbox - cmd.RunAndPrintOrFatal() + cmd.RunAndStreamOrFatal() } ninja("minibootstrap", ".minibootstrap/build.ninja") diff --git a/ui/build/upload.go b/ui/build/upload.go index d7c57b8db..1cc2e940c 100644 --- a/ui/build/upload.go +++ b/ui/build/upload.go @@ -108,6 +108,6 @@ func UploadMetrics(ctx Context, config Config, forceDumbOutput bool, buildStarte if forceDumbOutput { cmd.RunOrFatal() } else { - cmd.RunAndPrintOrFatal() + cmd.RunAndStreamOrFatal() } } diff --git a/ui/build/upload_test.go b/ui/build/upload_test.go index eb2dafa30..dccf156a2 100644 --- a/ui/build/upload_test.go +++ b/ui/build/upload_test.go @@ -19,6 +19,7 @@ import ( "io/ioutil" "os" "path/filepath" + "strconv" "strings" "testing" "time" @@ -93,6 +94,7 @@ func TestUploadMetrics(t *testing.T) { "OUT_DIR=" + outDir, "ANDROID_ENABLE_METRICS_UPLOAD=" + tt.uploader, }, + buildDateTime: strconv.FormatInt(time.Now().UnixNano()/int64(time.Millisecond), 10), }} UploadMetrics(ctx, config, false, time.Now(), metricsFiles...) diff --git a/ui/build/util.go b/ui/build/util.go index 14ab6ef07..d44cd6daf 100644 --- a/ui/build/util.go +++ b/ui/build/util.go @@ -45,6 +45,17 @@ func inList(s string, list []string) bool { return indexList(s, list) != -1 } +// removeFromlist removes all occurrences of the string in list. +func removeFromList(s string, list []string) []string { + filteredList := make([]string, 0, len(list)) + for _, ls := range list { + if s != ls { + filteredList = append(filteredList, ls) + } + } + return filteredList +} + // ensureDirectoriesExist is a shortcut to os.MkdirAll, sending errors to the ctx logger. func ensureDirectoriesExist(ctx Context, dirs ...string) { for _, dir := range dirs { diff --git a/ui/metrics/metrics.go b/ui/metrics/metrics.go index 4de5a706c..ab57ee59b 100644 --- a/ui/metrics/metrics.go +++ b/ui/metrics/metrics.go @@ -21,9 +21,9 @@ import ( "strings" "time" - "android/soong/ui/metrics/metrics_proto" - "github.com/golang/protobuf/proto" + + soong_metrics_proto "android/soong/ui/metrics/metrics_proto" ) const ( @@ -37,19 +37,19 @@ const ( ) type Metrics struct { - metrics metrics_proto.MetricsBase + metrics soong_metrics_proto.MetricsBase TimeTracer TimeTracer } func New() (metrics *Metrics) { m := &Metrics{ - metrics: metrics_proto.MetricsBase{}, + metrics: soong_metrics_proto.MetricsBase{}, TimeTracer: &timeTracerImpl{}, } return m } -func (m *Metrics) SetTimeMetrics(perf metrics_proto.PerfInfo) { +func (m *Metrics) SetTimeMetrics(perf soong_metrics_proto.PerfInfo) { switch perf.GetName() { case RunKati: m.metrics.KatiRuns = append(m.metrics.KatiRuns, &perf) @@ -60,12 +60,14 @@ func (m *Metrics) SetTimeMetrics(perf metrics_proto.PerfInfo) { case PrimaryNinja: m.metrics.NinjaRuns = append(m.metrics.NinjaRuns, &perf) break + case Total: + m.metrics.Total = &perf default: // ignored } } -func (m *Metrics) BuildConfig(b *metrics_proto.BuildConfig) { +func (m *Metrics) BuildConfig(b *soong_metrics_proto.BuildConfig) { m.metrics.BuildConfig = b } @@ -84,11 +86,11 @@ func (m *Metrics) SetMetadataMetrics(metadata map[string]string) { case "TARGET_BUILD_VARIANT": switch v { case "user": - m.metrics.TargetBuildVariant = metrics_proto.MetricsBase_USER.Enum() + m.metrics.TargetBuildVariant = soong_metrics_proto.MetricsBase_USER.Enum() case "userdebug": - m.metrics.TargetBuildVariant = metrics_proto.MetricsBase_USERDEBUG.Enum() + m.metrics.TargetBuildVariant = soong_metrics_proto.MetricsBase_USERDEBUG.Enum() case "eng": - m.metrics.TargetBuildVariant = metrics_proto.MetricsBase_ENG.Enum() + m.metrics.TargetBuildVariant = soong_metrics_proto.MetricsBase_ENG.Enum() default: // ignored } @@ -118,18 +120,18 @@ func (m *Metrics) SetMetadataMetrics(metadata map[string]string) { } } -func (m *Metrics) getArch(arch string) *metrics_proto.MetricsBase_ARCH { +func (m *Metrics) getArch(arch string) *soong_metrics_proto.MetricsBase_Arch { switch arch { case "arm": - return metrics_proto.MetricsBase_ARM.Enum() + return soong_metrics_proto.MetricsBase_ARM.Enum() case "arm64": - return metrics_proto.MetricsBase_ARM64.Enum() + return soong_metrics_proto.MetricsBase_ARM64.Enum() case "x86": - return metrics_proto.MetricsBase_X86.Enum() + return soong_metrics_proto.MetricsBase_X86.Enum() case "x86_64": - return metrics_proto.MetricsBase_X86_64.Enum() + return soong_metrics_proto.MetricsBase_X86_64.Enum() default: - return metrics_proto.MetricsBase_UNKNOWN.Enum() + return soong_metrics_proto.MetricsBase_UNKNOWN.Enum() } } @@ -137,10 +139,6 @@ func (m *Metrics) SetBuildDateTime(buildTimestamp time.Time) { m.metrics.BuildDateTimestamp = proto.Int64(buildTimestamp.UnixNano() / int64(time.Second)) } -func (m *Metrics) Serialize() (data []byte, err error) { - return proto.Marshal(&m.metrics) -} - func (m *Metrics) SetBuildCommand(cmd []string) { m.metrics.BuildCommand = proto.String(strings.Join(cmd, " ")) } @@ -153,12 +151,35 @@ func (m *Metrics) Dump(outputPath string) error { m.metrics.Hostname = proto.String(hostname) } m.metrics.HostOs = proto.String(runtime.GOOS) - data, err := m.Serialize() + return writeMessageToFile(&m.metrics, outputPath) +} + +type CriticalUserJourneysMetrics struct { + cujs soong_metrics_proto.CriticalUserJourneysMetrics +} + +func NewCriticalUserJourneysMetrics() *CriticalUserJourneysMetrics { + return &CriticalUserJourneysMetrics{} +} + +func (c *CriticalUserJourneysMetrics) Add(name string, metrics *Metrics) { + c.cujs.Cujs = append(c.cujs.Cujs, &soong_metrics_proto.CriticalUserJourneyMetrics{ + Name: proto.String(name), + Metrics: &metrics.metrics, + }) +} + +func (c *CriticalUserJourneysMetrics) Dump(outputPath string) (err error) { + return writeMessageToFile(&c.cujs, outputPath) +} + +func writeMessageToFile(pb proto.Message, outputPath string) (err error) { + data, err := proto.Marshal(pb) if err != nil { return err } tempPath := outputPath + ".tmp" - err = ioutil.WriteFile(tempPath, []byte(data), 0777) + err = ioutil.WriteFile(tempPath, []byte(data), 0644) if err != nil { return err } diff --git a/ui/metrics/metrics_proto/metrics.pb.go b/ui/metrics/metrics_proto/metrics.pb.go index 338a922b3..a2cfe5d60 100644 --- a/ui/metrics/metrics_proto/metrics.pb.go +++ b/ui/metrics/metrics_proto/metrics.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // source: metrics.proto -package metrics_proto +package soong_metrics_proto import ( fmt "fmt" @@ -20,60 +20,60 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package -type MetricsBase_BUILDVARIANT int32 +type MetricsBase_BuildVariant int32 const ( - MetricsBase_USER MetricsBase_BUILDVARIANT = 0 - MetricsBase_USERDEBUG MetricsBase_BUILDVARIANT = 1 - MetricsBase_ENG MetricsBase_BUILDVARIANT = 2 + MetricsBase_USER MetricsBase_BuildVariant = 0 + MetricsBase_USERDEBUG MetricsBase_BuildVariant = 1 + MetricsBase_ENG MetricsBase_BuildVariant = 2 ) -var MetricsBase_BUILDVARIANT_name = map[int32]string{ +var MetricsBase_BuildVariant_name = map[int32]string{ 0: "USER", 1: "USERDEBUG", 2: "ENG", } -var MetricsBase_BUILDVARIANT_value = map[string]int32{ +var MetricsBase_BuildVariant_value = map[string]int32{ "USER": 0, "USERDEBUG": 1, "ENG": 2, } -func (x MetricsBase_BUILDVARIANT) Enum() *MetricsBase_BUILDVARIANT { - p := new(MetricsBase_BUILDVARIANT) +func (x MetricsBase_BuildVariant) Enum() *MetricsBase_BuildVariant { + p := new(MetricsBase_BuildVariant) *p = x return p } -func (x MetricsBase_BUILDVARIANT) String() string { - return proto.EnumName(MetricsBase_BUILDVARIANT_name, int32(x)) +func (x MetricsBase_BuildVariant) String() string { + return proto.EnumName(MetricsBase_BuildVariant_name, int32(x)) } -func (x *MetricsBase_BUILDVARIANT) UnmarshalJSON(data []byte) error { - value, err := proto.UnmarshalJSONEnum(MetricsBase_BUILDVARIANT_value, data, "MetricsBase_BUILDVARIANT") +func (x *MetricsBase_BuildVariant) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(MetricsBase_BuildVariant_value, data, "MetricsBase_BuildVariant") if err != nil { return err } - *x = MetricsBase_BUILDVARIANT(value) + *x = MetricsBase_BuildVariant(value) return nil } -func (MetricsBase_BUILDVARIANT) EnumDescriptor() ([]byte, []int) { +func (MetricsBase_BuildVariant) EnumDescriptor() ([]byte, []int) { return fileDescriptor_6039342a2ba47b72, []int{0, 0} } -type MetricsBase_ARCH int32 +type MetricsBase_Arch int32 const ( - MetricsBase_UNKNOWN MetricsBase_ARCH = 0 - MetricsBase_ARM MetricsBase_ARCH = 1 - MetricsBase_ARM64 MetricsBase_ARCH = 2 - MetricsBase_X86 MetricsBase_ARCH = 3 - MetricsBase_X86_64 MetricsBase_ARCH = 4 + MetricsBase_UNKNOWN MetricsBase_Arch = 0 + MetricsBase_ARM MetricsBase_Arch = 1 + MetricsBase_ARM64 MetricsBase_Arch = 2 + MetricsBase_X86 MetricsBase_Arch = 3 + MetricsBase_X86_64 MetricsBase_Arch = 4 ) -var MetricsBase_ARCH_name = map[int32]string{ +var MetricsBase_Arch_name = map[int32]string{ 0: "UNKNOWN", 1: "ARM", 2: "ARM64", @@ -81,7 +81,7 @@ var MetricsBase_ARCH_name = map[int32]string{ 4: "X86_64", } -var MetricsBase_ARCH_value = map[string]int32{ +var MetricsBase_Arch_value = map[string]int32{ "UNKNOWN": 0, "ARM": 1, "ARM64": 2, @@ -89,69 +89,69 @@ var MetricsBase_ARCH_value = map[string]int32{ "X86_64": 4, } -func (x MetricsBase_ARCH) Enum() *MetricsBase_ARCH { - p := new(MetricsBase_ARCH) +func (x MetricsBase_Arch) Enum() *MetricsBase_Arch { + p := new(MetricsBase_Arch) *p = x return p } -func (x MetricsBase_ARCH) String() string { - return proto.EnumName(MetricsBase_ARCH_name, int32(x)) +func (x MetricsBase_Arch) String() string { + return proto.EnumName(MetricsBase_Arch_name, int32(x)) } -func (x *MetricsBase_ARCH) UnmarshalJSON(data []byte) error { - value, err := proto.UnmarshalJSONEnum(MetricsBase_ARCH_value, data, "MetricsBase_ARCH") +func (x *MetricsBase_Arch) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(MetricsBase_Arch_value, data, "MetricsBase_Arch") if err != nil { return err } - *x = MetricsBase_ARCH(value) + *x = MetricsBase_Arch(value) return nil } -func (MetricsBase_ARCH) EnumDescriptor() ([]byte, []int) { +func (MetricsBase_Arch) EnumDescriptor() ([]byte, []int) { return fileDescriptor_6039342a2ba47b72, []int{0, 1} } -type ModuleTypeInfo_BUILDSYSTEM int32 +type ModuleTypeInfo_BuildSystem int32 const ( - ModuleTypeInfo_UNKNOWN ModuleTypeInfo_BUILDSYSTEM = 0 - ModuleTypeInfo_SOONG ModuleTypeInfo_BUILDSYSTEM = 1 - ModuleTypeInfo_MAKE ModuleTypeInfo_BUILDSYSTEM = 2 + ModuleTypeInfo_UNKNOWN ModuleTypeInfo_BuildSystem = 0 + ModuleTypeInfo_SOONG ModuleTypeInfo_BuildSystem = 1 + ModuleTypeInfo_MAKE ModuleTypeInfo_BuildSystem = 2 ) -var ModuleTypeInfo_BUILDSYSTEM_name = map[int32]string{ +var ModuleTypeInfo_BuildSystem_name = map[int32]string{ 0: "UNKNOWN", 1: "SOONG", 2: "MAKE", } -var ModuleTypeInfo_BUILDSYSTEM_value = map[string]int32{ +var ModuleTypeInfo_BuildSystem_value = map[string]int32{ "UNKNOWN": 0, "SOONG": 1, "MAKE": 2, } -func (x ModuleTypeInfo_BUILDSYSTEM) Enum() *ModuleTypeInfo_BUILDSYSTEM { - p := new(ModuleTypeInfo_BUILDSYSTEM) +func (x ModuleTypeInfo_BuildSystem) Enum() *ModuleTypeInfo_BuildSystem { + p := new(ModuleTypeInfo_BuildSystem) *p = x return p } -func (x ModuleTypeInfo_BUILDSYSTEM) String() string { - return proto.EnumName(ModuleTypeInfo_BUILDSYSTEM_name, int32(x)) +func (x ModuleTypeInfo_BuildSystem) String() string { + return proto.EnumName(ModuleTypeInfo_BuildSystem_name, int32(x)) } -func (x *ModuleTypeInfo_BUILDSYSTEM) UnmarshalJSON(data []byte) error { - value, err := proto.UnmarshalJSONEnum(ModuleTypeInfo_BUILDSYSTEM_value, data, "ModuleTypeInfo_BUILDSYSTEM") +func (x *ModuleTypeInfo_BuildSystem) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(ModuleTypeInfo_BuildSystem_value, data, "ModuleTypeInfo_BuildSystem") if err != nil { return err } - *x = ModuleTypeInfo_BUILDSYSTEM(value) + *x = ModuleTypeInfo_BuildSystem(value) return nil } -func (ModuleTypeInfo_BUILDSYSTEM) EnumDescriptor() ([]byte, []int) { +func (ModuleTypeInfo_BuildSystem) EnumDescriptor() ([]byte, []int) { return fileDescriptor_6039342a2ba47b72, []int{3, 0} } @@ -165,17 +165,17 @@ type MetricsBase struct { // The target product information, eg. aosp_arm. TargetProduct *string `protobuf:"bytes,4,opt,name=target_product,json=targetProduct" json:"target_product,omitempty"` // The target build variant information, eg. eng. - TargetBuildVariant *MetricsBase_BUILDVARIANT `protobuf:"varint,5,opt,name=target_build_variant,json=targetBuildVariant,enum=build_metrics.MetricsBase_BUILDVARIANT,def=2" json:"target_build_variant,omitempty"` + TargetBuildVariant *MetricsBase_BuildVariant `protobuf:"varint,5,opt,name=target_build_variant,json=targetBuildVariant,enum=soong_build_metrics.MetricsBase_BuildVariant,def=2" json:"target_build_variant,omitempty"` // The target arch information, eg. arm. - TargetArch *MetricsBase_ARCH `protobuf:"varint,6,opt,name=target_arch,json=targetArch,enum=build_metrics.MetricsBase_ARCH,def=0" json:"target_arch,omitempty"` + TargetArch *MetricsBase_Arch `protobuf:"varint,6,opt,name=target_arch,json=targetArch,enum=soong_build_metrics.MetricsBase_Arch,def=0" json:"target_arch,omitempty"` // The target arch variant information, eg. armv7-a-neon. TargetArchVariant *string `protobuf:"bytes,7,opt,name=target_arch_variant,json=targetArchVariant" json:"target_arch_variant,omitempty"` // The target cpu variant information, eg. generic. TargetCpuVariant *string `protobuf:"bytes,8,opt,name=target_cpu_variant,json=targetCpuVariant" json:"target_cpu_variant,omitempty"` // The host arch information, eg. x86_64. - HostArch *MetricsBase_ARCH `protobuf:"varint,9,opt,name=host_arch,json=hostArch,enum=build_metrics.MetricsBase_ARCH,def=0" json:"host_arch,omitempty"` + HostArch *MetricsBase_Arch `protobuf:"varint,9,opt,name=host_arch,json=hostArch,enum=soong_build_metrics.MetricsBase_Arch,def=0" json:"host_arch,omitempty"` // The host 2nd arch information, eg. x86. - Host_2NdArch *MetricsBase_ARCH `protobuf:"varint,10,opt,name=host_2nd_arch,json=host2ndArch,enum=build_metrics.MetricsBase_ARCH,def=0" json:"host_2nd_arch,omitempty"` + Host_2NdArch *MetricsBase_Arch `protobuf:"varint,10,opt,name=host_2nd_arch,json=host2ndArch,enum=soong_build_metrics.MetricsBase_Arch,def=0" json:"host_2nd_arch,omitempty"` // The host os information, eg. linux. HostOs *string `protobuf:"bytes,11,opt,name=host_os,json=hostOs" json:"host_os,omitempty"` // The host os extra information, eg. Linux-4.17.0-3rodete2-amd64-x86_64-Debian-GNU. @@ -195,7 +195,9 @@ type MetricsBase struct { // The metrics for calling Soong. SoongRuns []*PerfInfo `protobuf:"bytes,19,rep,name=soong_runs,json=soongRuns" json:"soong_runs,omitempty"` // The metrics for calling Ninja. - NinjaRuns []*PerfInfo `protobuf:"bytes,20,rep,name=ninja_runs,json=ninjaRuns" json:"ninja_runs,omitempty"` + NinjaRuns []*PerfInfo `protobuf:"bytes,20,rep,name=ninja_runs,json=ninjaRuns" json:"ninja_runs,omitempty"` + // The metrics for the whole build + Total *PerfInfo `protobuf:"bytes,21,opt,name=total" json:"total,omitempty"` BuildConfig *BuildConfig `protobuf:"bytes,23,opt,name=build_config,json=buildConfig" json:"build_config,omitempty"` // The hostname of the machine. Hostname *string `protobuf:"bytes,24,opt,name=hostname" json:"hostname,omitempty"` @@ -231,10 +233,10 @@ func (m *MetricsBase) XXX_DiscardUnknown() { var xxx_messageInfo_MetricsBase proto.InternalMessageInfo -const Default_MetricsBase_TargetBuildVariant MetricsBase_BUILDVARIANT = MetricsBase_ENG -const Default_MetricsBase_TargetArch MetricsBase_ARCH = MetricsBase_UNKNOWN -const Default_MetricsBase_HostArch MetricsBase_ARCH = MetricsBase_UNKNOWN -const Default_MetricsBase_Host_2NdArch MetricsBase_ARCH = MetricsBase_UNKNOWN +const Default_MetricsBase_TargetBuildVariant MetricsBase_BuildVariant = MetricsBase_ENG +const Default_MetricsBase_TargetArch MetricsBase_Arch = MetricsBase_UNKNOWN +const Default_MetricsBase_HostArch MetricsBase_Arch = MetricsBase_UNKNOWN +const Default_MetricsBase_Host_2NdArch MetricsBase_Arch = MetricsBase_UNKNOWN func (m *MetricsBase) GetBuildDateTimestamp() int64 { if m != nil && m.BuildDateTimestamp != nil { @@ -264,14 +266,14 @@ func (m *MetricsBase) GetTargetProduct() string { return "" } -func (m *MetricsBase) GetTargetBuildVariant() MetricsBase_BUILDVARIANT { +func (m *MetricsBase) GetTargetBuildVariant() MetricsBase_BuildVariant { if m != nil && m.TargetBuildVariant != nil { return *m.TargetBuildVariant } return Default_MetricsBase_TargetBuildVariant } -func (m *MetricsBase) GetTargetArch() MetricsBase_ARCH { +func (m *MetricsBase) GetTargetArch() MetricsBase_Arch { if m != nil && m.TargetArch != nil { return *m.TargetArch } @@ -292,14 +294,14 @@ func (m *MetricsBase) GetTargetCpuVariant() string { return "" } -func (m *MetricsBase) GetHostArch() MetricsBase_ARCH { +func (m *MetricsBase) GetHostArch() MetricsBase_Arch { if m != nil && m.HostArch != nil { return *m.HostArch } return Default_MetricsBase_HostArch } -func (m *MetricsBase) GetHost_2NdArch() MetricsBase_ARCH { +func (m *MetricsBase) GetHost_2NdArch() MetricsBase_Arch { if m != nil && m.Host_2NdArch != nil { return *m.Host_2NdArch } @@ -376,6 +378,13 @@ func (m *MetricsBase) GetNinjaRuns() []*PerfInfo { return nil } +func (m *MetricsBase) GetTotal() *PerfInfo { + if m != nil { + return m.Total + } + return nil +} + func (m *MetricsBase) GetBuildConfig() *BuildConfig { if m != nil { return m.BuildConfig @@ -532,7 +541,7 @@ func (m *PerfInfo) GetMemoryUse() uint64 { type ModuleTypeInfo struct { // The build system, eg. Soong or Make. - BuildSystem *ModuleTypeInfo_BUILDSYSTEM `protobuf:"varint,1,opt,name=build_system,json=buildSystem,enum=build_metrics.ModuleTypeInfo_BUILDSYSTEM,def=0" json:"build_system,omitempty"` + BuildSystem *ModuleTypeInfo_BuildSystem `protobuf:"varint,1,opt,name=build_system,json=buildSystem,enum=soong_build_metrics.ModuleTypeInfo_BuildSystem,def=0" json:"build_system,omitempty"` // The module type, eg. java_library, cc_binary, and etc. ModuleType *string `protobuf:"bytes,2,opt,name=module_type,json=moduleType" json:"module_type,omitempty"` // The number of logical modules. @@ -567,9 +576,9 @@ func (m *ModuleTypeInfo) XXX_DiscardUnknown() { var xxx_messageInfo_ModuleTypeInfo proto.InternalMessageInfo -const Default_ModuleTypeInfo_BuildSystem ModuleTypeInfo_BUILDSYSTEM = ModuleTypeInfo_UNKNOWN +const Default_ModuleTypeInfo_BuildSystem ModuleTypeInfo_BuildSystem = ModuleTypeInfo_UNKNOWN -func (m *ModuleTypeInfo) GetBuildSystem() ModuleTypeInfo_BUILDSYSTEM { +func (m *ModuleTypeInfo) GetBuildSystem() ModuleTypeInfo_BuildSystem { if m != nil && m.BuildSystem != nil { return *m.BuildSystem } @@ -590,74 +599,169 @@ func (m *ModuleTypeInfo) GetNumOfModules() uint32 { return 0 } +type CriticalUserJourneyMetrics struct { + // The name of a critical user journey test. + Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + // The metrics produced when running the critical user journey test. + Metrics *MetricsBase `protobuf:"bytes,2,opt,name=metrics" json:"metrics,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *CriticalUserJourneyMetrics) Reset() { *m = CriticalUserJourneyMetrics{} } +func (m *CriticalUserJourneyMetrics) String() string { return proto.CompactTextString(m) } +func (*CriticalUserJourneyMetrics) ProtoMessage() {} +func (*CriticalUserJourneyMetrics) Descriptor() ([]byte, []int) { + return fileDescriptor_6039342a2ba47b72, []int{4} +} + +func (m *CriticalUserJourneyMetrics) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_CriticalUserJourneyMetrics.Unmarshal(m, b) +} +func (m *CriticalUserJourneyMetrics) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_CriticalUserJourneyMetrics.Marshal(b, m, deterministic) +} +func (m *CriticalUserJourneyMetrics) XXX_Merge(src proto.Message) { + xxx_messageInfo_CriticalUserJourneyMetrics.Merge(m, src) +} +func (m *CriticalUserJourneyMetrics) XXX_Size() int { + return xxx_messageInfo_CriticalUserJourneyMetrics.Size(m) +} +func (m *CriticalUserJourneyMetrics) XXX_DiscardUnknown() { + xxx_messageInfo_CriticalUserJourneyMetrics.DiscardUnknown(m) +} + +var xxx_messageInfo_CriticalUserJourneyMetrics proto.InternalMessageInfo + +func (m *CriticalUserJourneyMetrics) GetName() string { + if m != nil && m.Name != nil { + return *m.Name + } + return "" +} + +func (m *CriticalUserJourneyMetrics) GetMetrics() *MetricsBase { + if m != nil { + return m.Metrics + } + return nil +} + +type CriticalUserJourneysMetrics struct { + // A set of metrics from a run of the critical user journey tests. + Cujs []*CriticalUserJourneyMetrics `protobuf:"bytes,1,rep,name=cujs" json:"cujs,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *CriticalUserJourneysMetrics) Reset() { *m = CriticalUserJourneysMetrics{} } +func (m *CriticalUserJourneysMetrics) String() string { return proto.CompactTextString(m) } +func (*CriticalUserJourneysMetrics) ProtoMessage() {} +func (*CriticalUserJourneysMetrics) Descriptor() ([]byte, []int) { + return fileDescriptor_6039342a2ba47b72, []int{5} +} + +func (m *CriticalUserJourneysMetrics) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_CriticalUserJourneysMetrics.Unmarshal(m, b) +} +func (m *CriticalUserJourneysMetrics) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_CriticalUserJourneysMetrics.Marshal(b, m, deterministic) +} +func (m *CriticalUserJourneysMetrics) XXX_Merge(src proto.Message) { + xxx_messageInfo_CriticalUserJourneysMetrics.Merge(m, src) +} +func (m *CriticalUserJourneysMetrics) XXX_Size() int { + return xxx_messageInfo_CriticalUserJourneysMetrics.Size(m) +} +func (m *CriticalUserJourneysMetrics) XXX_DiscardUnknown() { + xxx_messageInfo_CriticalUserJourneysMetrics.DiscardUnknown(m) +} + +var xxx_messageInfo_CriticalUserJourneysMetrics proto.InternalMessageInfo + +func (m *CriticalUserJourneysMetrics) GetCujs() []*CriticalUserJourneyMetrics { + if m != nil { + return m.Cujs + } + return nil +} + func init() { - proto.RegisterEnum("build_metrics.MetricsBase_BUILDVARIANT", MetricsBase_BUILDVARIANT_name, MetricsBase_BUILDVARIANT_value) - proto.RegisterEnum("build_metrics.MetricsBase_ARCH", MetricsBase_ARCH_name, MetricsBase_ARCH_value) - proto.RegisterEnum("build_metrics.ModuleTypeInfo_BUILDSYSTEM", ModuleTypeInfo_BUILDSYSTEM_name, ModuleTypeInfo_BUILDSYSTEM_value) - proto.RegisterType((*MetricsBase)(nil), "build_metrics.MetricsBase") - proto.RegisterType((*BuildConfig)(nil), "build_metrics.BuildConfig") - proto.RegisterType((*PerfInfo)(nil), "build_metrics.PerfInfo") - proto.RegisterType((*ModuleTypeInfo)(nil), "build_metrics.ModuleTypeInfo") + proto.RegisterEnum("soong_build_metrics.MetricsBase_BuildVariant", MetricsBase_BuildVariant_name, MetricsBase_BuildVariant_value) + proto.RegisterEnum("soong_build_metrics.MetricsBase_Arch", MetricsBase_Arch_name, MetricsBase_Arch_value) + proto.RegisterEnum("soong_build_metrics.ModuleTypeInfo_BuildSystem", ModuleTypeInfo_BuildSystem_name, ModuleTypeInfo_BuildSystem_value) + proto.RegisterType((*MetricsBase)(nil), "soong_build_metrics.MetricsBase") + proto.RegisterType((*BuildConfig)(nil), "soong_build_metrics.BuildConfig") + proto.RegisterType((*PerfInfo)(nil), "soong_build_metrics.PerfInfo") + proto.RegisterType((*ModuleTypeInfo)(nil), "soong_build_metrics.ModuleTypeInfo") + proto.RegisterType((*CriticalUserJourneyMetrics)(nil), "soong_build_metrics.CriticalUserJourneyMetrics") + proto.RegisterType((*CriticalUserJourneysMetrics)(nil), "soong_build_metrics.CriticalUserJourneysMetrics") } func init() { proto.RegisterFile("metrics.proto", fileDescriptor_6039342a2ba47b72) } var fileDescriptor_6039342a2ba47b72 = []byte{ - // 887 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x55, 0x6f, 0x6f, 0xe2, 0x36, - 0x18, 0x3f, 0x0a, 0x2d, 0xc9, 0x13, 0xe0, 0x52, 0xb7, 0x52, 0x73, 0x9d, 0x4e, 0x87, 0xd8, 0x3f, - 0x26, 0x6d, 0xec, 0x84, 0x2a, 0x54, 0x55, 0xdb, 0x0b, 0xa0, 0xa8, 0x87, 0xee, 0x80, 0x93, 0x81, - 0xee, 0xb6, 0x17, 0x8b, 0xd2, 0xc4, 0xd0, 0x6c, 0x24, 0x46, 0xb6, 0x73, 0x5a, 0x3f, 0xc4, 0x3e, - 0xe3, 0xbe, 0xc8, 0x5e, 0x4c, 0x7e, 0x4c, 0x28, 0xed, 0x8b, 0x4e, 0x7d, 0x67, 0x3f, 0xbf, 0x3f, - 0xfe, 0xd9, 0xb1, 0x9f, 0x40, 0x35, 0x61, 0x4a, 0xc4, 0xa1, 0x6c, 0xad, 0x05, 0x57, 0x9c, 0x54, - 0x6f, 0xb2, 0x78, 0x15, 0xf9, 0x9b, 0x62, 0xe3, 0x5f, 0x1b, 0x9c, 0x91, 0x19, 0xf7, 0x02, 0xc9, - 0xc8, 0x5b, 0x38, 0x36, 0x84, 0x28, 0x50, 0xcc, 0x57, 0x71, 0xc2, 0xa4, 0x0a, 0x92, 0xb5, 0x57, - 0xa8, 0x17, 0x9a, 0x45, 0x4a, 0x10, 0xbb, 0x0c, 0x14, 0x9b, 0xe5, 0x08, 0x79, 0x05, 0x96, 0x51, - 0xc4, 0x91, 0xb7, 0x57, 0x2f, 0x34, 0x6d, 0x5a, 0xc6, 0xf9, 0x30, 0x22, 0x17, 0xf0, 0x6a, 0xbd, - 0x0a, 0xd4, 0x82, 0x8b, 0xc4, 0xff, 0xcc, 0x84, 0x8c, 0x79, 0xea, 0x87, 0x3c, 0x62, 0x69, 0x90, - 0x30, 0xaf, 0x88, 0xdc, 0x93, 0x9c, 0x70, 0x6d, 0xf0, 0xfe, 0x06, 0x26, 0x5f, 0x43, 0x4d, 0x05, - 0x62, 0xc9, 0x94, 0xbf, 0x16, 0x3c, 0xca, 0x42, 0xe5, 0x95, 0x50, 0x50, 0x35, 0xd5, 0x8f, 0xa6, - 0x48, 0x7e, 0x87, 0xe3, 0x0d, 0xcd, 0x84, 0xf8, 0x1c, 0x88, 0x38, 0x48, 0x95, 0xb7, 0x5f, 0x2f, - 0x34, 0x6b, 0xed, 0x6f, 0x5b, 0x0f, 0x76, 0xdb, 0xda, 0xd9, 0x69, 0xab, 0x37, 0x1f, 0x7e, 0xb8, - 0xbc, 0xee, 0xd2, 0x61, 0x77, 0x3c, 0xbb, 0x28, 0x0e, 0xc6, 0x57, 0x94, 0x18, 0xa7, 0x9e, 0x96, - 0x5c, 0x1b, 0x1f, 0x32, 0x04, 0x67, 0xe3, 0x1f, 0x88, 0xf0, 0xd6, 0x3b, 0x40, 0xdb, 0x37, 0x4f, - 0xd8, 0x76, 0x69, 0xff, 0xdd, 0x45, 0x79, 0x3e, 0x7e, 0x3f, 0x9e, 0xfc, 0x32, 0xa6, 0x60, 0xc4, - 0x5d, 0x11, 0xde, 0x92, 0x16, 0x1c, 0xed, 0x58, 0x6d, 0x93, 0x96, 0x71, 0x5b, 0x87, 0xf7, 0xc4, - 0x7c, 0xe9, 0xef, 0x61, 0x13, 0xc8, 0x0f, 0xd7, 0xd9, 0x96, 0x6e, 0x21, 0xdd, 0x35, 0x48, 0x7f, - 0x9d, 0xe5, 0xec, 0x01, 0xd8, 0xb7, 0x5c, 0x6e, 0x62, 0xda, 0xcf, 0x8c, 0x69, 0x69, 0x29, 0x86, - 0xfc, 0x00, 0x55, 0xb4, 0x69, 0xa7, 0x91, 0xb1, 0x82, 0x67, 0x5a, 0x39, 0x5a, 0xde, 0x4e, 0x23, - 0x74, 0x3b, 0x81, 0x32, 0xba, 0x71, 0xe9, 0x39, 0x98, 0xfb, 0x40, 0x4f, 0x27, 0x92, 0x34, 0x36, - 0xcb, 0x70, 0xe9, 0xb3, 0xbf, 0x94, 0x08, 0xbc, 0x0a, 0xc2, 0x8e, 0x81, 0x07, 0xba, 0xb4, 0xe5, - 0x84, 0x82, 0x4b, 0xa9, 0x2d, 0xaa, 0xf7, 0x9c, 0xbe, 0xae, 0x4d, 0x24, 0xf9, 0x06, 0x5e, 0xee, - 0x70, 0x30, 0x70, 0xcd, 0x5c, 0x93, 0x2d, 0x0b, 0x83, 0xfc, 0x00, 0x47, 0x3b, 0xbc, 0xed, 0xe6, - 0x5e, 0x9a, 0xc3, 0xdc, 0x72, 0x77, 0x72, 0xf3, 0x4c, 0xf9, 0x51, 0x2c, 0x3c, 0xd7, 0xe4, 0xe6, - 0x99, 0xba, 0x8c, 0x05, 0x39, 0x07, 0x47, 0x32, 0x95, 0xad, 0x7d, 0xc5, 0xf9, 0x4a, 0x7a, 0x87, - 0xf5, 0x62, 0xd3, 0x69, 0x9f, 0x3c, 0x3a, 0x9c, 0x8f, 0x4c, 0x2c, 0x86, 0xe9, 0x82, 0x53, 0x40, - 0xee, 0x4c, 0x53, 0xc9, 0x19, 0xd8, 0x7f, 0x06, 0x2a, 0xf6, 0x45, 0x96, 0x4a, 0x8f, 0x3c, 0xad, - 0xb3, 0x34, 0x93, 0x66, 0xa9, 0x24, 0x1d, 0x00, 0xc9, 0x79, 0xba, 0x34, 0xb2, 0xa3, 0xa7, 0x65, - 0x36, 0x52, 0x73, 0x5d, 0x1a, 0xa7, 0x7f, 0x04, 0x46, 0x77, 0xfc, 0x3f, 0x3a, 0xa4, 0xa2, 0xee, - 0x67, 0xa8, 0x18, 0x52, 0xc8, 0xd3, 0x45, 0xbc, 0xf4, 0x4e, 0xea, 0x85, 0xa6, 0xd3, 0x3e, 0x7d, - 0xa4, 0xc4, 0x17, 0xd2, 0x47, 0x06, 0x75, 0x6e, 0xee, 0x27, 0xe4, 0x14, 0xf0, 0x26, 0xe1, 0xfb, - 0xf6, 0xf0, 0xe0, 0xb6, 0x73, 0xf2, 0x25, 0x54, 0x73, 0xeb, 0x24, 0x09, 0xd2, 0xc8, 0x3b, 0x45, - 0x42, 0x65, 0xa3, 0xc7, 0x5a, 0xe3, 0x2d, 0x54, 0x76, 0xdf, 0x25, 0xb1, 0xa0, 0x34, 0x9f, 0x0e, - 0xa8, 0xfb, 0x82, 0x54, 0xc1, 0xd6, 0xa3, 0xcb, 0x41, 0x6f, 0x7e, 0xe5, 0x16, 0x48, 0x19, 0xf4, - 0x93, 0x75, 0xf7, 0x1a, 0x3f, 0x41, 0x49, 0x5f, 0x40, 0xe2, 0x40, 0x7e, 0x05, 0xdd, 0x17, 0x1a, - 0xed, 0xd2, 0x91, 0x5b, 0x20, 0x36, 0xec, 0x77, 0xe9, 0xa8, 0x73, 0xe6, 0xee, 0xe9, 0xda, 0xa7, - 0xf3, 0x8e, 0x5b, 0x24, 0x00, 0x07, 0x9f, 0xce, 0x3b, 0x7e, 0xe7, 0xcc, 0x2d, 0x35, 0x96, 0xe0, - 0xec, 0x6c, 0x46, 0xf7, 0xb2, 0x4c, 0x32, 0x7f, 0xc9, 0x93, 0x00, 0x3b, 0x9e, 0x45, 0xcb, 0x99, - 0x64, 0x57, 0x3c, 0x09, 0xf4, 0x95, 0xd0, 0x90, 0xb8, 0x61, 0xd8, 0xe5, 0x2c, 0x7a, 0x90, 0x49, - 0x46, 0x6f, 0x18, 0xf9, 0x0a, 0x6a, 0x0b, 0x2e, 0x42, 0xe6, 0x6f, 0x95, 0x45, 0xc4, 0x2b, 0x58, - 0x9d, 0x1b, 0x79, 0xe3, 0xef, 0x02, 0x58, 0xf9, 0x81, 0x13, 0x02, 0xa5, 0x88, 0xc9, 0x10, 0x97, - 0xb0, 0x29, 0x8e, 0x75, 0x0d, 0x8f, 0xcd, 0xb4, 0x50, 0x1c, 0x93, 0xd7, 0x00, 0x52, 0x05, 0x42, - 0x61, 0x1f, 0x46, 0xdb, 0x12, 0xb5, 0xb1, 0xa2, 0xdb, 0x2f, 0xf9, 0x02, 0x6c, 0xc1, 0x82, 0x95, - 0x41, 0x4b, 0x88, 0x5a, 0xba, 0x80, 0xe0, 0x6b, 0x80, 0x84, 0x25, 0x5c, 0xdc, 0xe9, 0x5c, 0xd8, - 0x0e, 0x4b, 0xd4, 0x36, 0x95, 0xb9, 0x64, 0x8d, 0x7f, 0x0a, 0x50, 0x1b, 0xf1, 0x28, 0x5b, 0xb1, - 0xd9, 0xdd, 0x9a, 0x61, 0xaa, 0x79, 0xfe, 0xed, 0xe5, 0x9d, 0x54, 0x2c, 0xc1, 0x74, 0xb5, 0xf6, - 0x77, 0x8f, 0x5f, 0xfe, 0x03, 0x91, 0xe9, 0xa2, 0xd3, 0x5f, 0xa7, 0xb3, 0xc1, 0x68, 0xa7, 0x07, - 0xa0, 0x64, 0x8a, 0x36, 0xe4, 0x0d, 0x38, 0x09, 0x6a, 0x7c, 0x75, 0xb7, 0xce, 0xf7, 0x07, 0xc9, - 0xd6, 0x46, 0x1f, 0x60, 0x9a, 0x25, 0x3e, 0x5f, 0xf8, 0xa6, 0x28, 0x71, 0xa7, 0x55, 0x5a, 0x49, - 0xb3, 0x64, 0xb2, 0x30, 0xeb, 0xc9, 0xc6, 0x8f, 0xe0, 0xec, 0xac, 0xf5, 0xf0, 0x73, 0xdb, 0xb0, - 0x3f, 0x9d, 0x4c, 0xc6, 0xfa, 0x5e, 0x58, 0x50, 0x1a, 0x75, 0xdf, 0x0f, 0xdc, 0xbd, 0xde, 0xe1, - 0xbb, 0xe2, 0x6f, 0xf9, 0xbf, 0xcf, 0xc7, 0x7f, 0xdf, 0x7f, 0x01, 0x00, 0x00, 0xff, 0xff, 0xf1, - 0x41, 0x61, 0x5e, 0x0b, 0x07, 0x00, 0x00, + // 951 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x56, 0xdf, 0x4e, 0xdb, 0x48, + 0x17, 0xaf, 0x49, 0x20, 0xf1, 0x71, 0x92, 0xba, 0x03, 0x15, 0x2e, 0x15, 0xfa, 0x22, 0x7f, 0xdb, + 0x15, 0x17, 0x5b, 0x5a, 0xb1, 0x15, 0xaa, 0x50, 0xb5, 0x12, 0x04, 0x84, 0xba, 0x08, 0x52, 0x0d, + 0xa4, 0x5b, 0xed, 0x5e, 0x58, 0x13, 0x7b, 0x02, 0xee, 0xc6, 0x9e, 0x68, 0x66, 0x5c, 0x2d, 0x0f, + 0xb1, 0x8f, 0xb9, 0xda, 0xd7, 0x58, 0xcd, 0x19, 0xdb, 0x18, 0x29, 0x55, 0x51, 0xef, 0xec, 0xf3, + 0xfb, 0x33, 0xbf, 0x33, 0x9e, 0x39, 0x32, 0xf4, 0x33, 0xae, 0x65, 0x1a, 0xab, 0xdd, 0x85, 0x14, + 0x5a, 0x90, 0x75, 0x25, 0x44, 0x7e, 0x1d, 0x4d, 0x8b, 0x74, 0x9e, 0x44, 0x25, 0x14, 0xfe, 0x0b, + 0xe0, 0x9d, 0xdb, 0xe7, 0x23, 0xa6, 0x38, 0x79, 0x0d, 0x1b, 0x96, 0x90, 0x30, 0xcd, 0x23, 0x9d, + 0x66, 0x5c, 0x69, 0x96, 0x2d, 0x02, 0x67, 0xe8, 0xec, 0xb4, 0x28, 0x41, 0xec, 0x98, 0x69, 0x7e, + 0x55, 0x21, 0xe4, 0x19, 0x74, 0xad, 0x22, 0x4d, 0x82, 0x95, 0xa1, 0xb3, 0xe3, 0xd2, 0x0e, 0xbe, + 0xbf, 0x4f, 0xc8, 0x01, 0x3c, 0x5b, 0xcc, 0x99, 0x9e, 0x09, 0x99, 0x45, 0x5f, 0xb8, 0x54, 0xa9, + 0xc8, 0xa3, 0x58, 0x24, 0x3c, 0x67, 0x19, 0x0f, 0x5a, 0xc8, 0xdd, 0xac, 0x08, 0x1f, 0x2d, 0x3e, + 0x2a, 0x61, 0xf2, 0x02, 0x06, 0x9a, 0xc9, 0x6b, 0xae, 0xa3, 0x85, 0x14, 0x49, 0x11, 0xeb, 0xa0, + 0x8d, 0x82, 0xbe, 0xad, 0x7e, 0xb0, 0x45, 0x92, 0xc0, 0x46, 0x49, 0xb3, 0x21, 0xbe, 0x30, 0x99, + 0xb2, 0x5c, 0x07, 0xab, 0x43, 0x67, 0x67, 0xb0, 0xf7, 0x72, 0x77, 0x49, 0xcf, 0xbb, 0x8d, 0x7e, + 0x77, 0x8f, 0x0c, 0xf2, 0xd1, 0x8a, 0x0e, 0x5a, 0x27, 0x17, 0xa7, 0x94, 0x58, 0xbf, 0x26, 0x40, + 0xc6, 0xe0, 0x95, 0xab, 0x30, 0x19, 0xdf, 0x04, 0x6b, 0x68, 0xfe, 0xe2, 0x9b, 0xe6, 0x87, 0x32, + 0xbe, 0x39, 0xe8, 0x4c, 0x2e, 0xce, 0x2e, 0xc6, 0xbf, 0x5d, 0x50, 0xb0, 0x16, 0xa6, 0x48, 0x76, + 0x61, 0xbd, 0x61, 0x58, 0xa7, 0xee, 0x60, 0x8b, 0x4f, 0xee, 0x88, 0x55, 0x80, 0x9f, 0xa0, 0x8c, + 0x15, 0xc5, 0x8b, 0xa2, 0xa6, 0x77, 0x91, 0xee, 0x5b, 0x64, 0xb4, 0x28, 0x2a, 0xf6, 0x19, 0xb8, + 0x37, 0x42, 0x95, 0x61, 0xdd, 0xef, 0x0a, 0xdb, 0x35, 0x06, 0x18, 0x95, 0x42, 0x1f, 0xcd, 0xf6, + 0xf2, 0xc4, 0x1a, 0xc2, 0x77, 0x19, 0x7a, 0xc6, 0x64, 0x2f, 0x4f, 0xd0, 0x73, 0x13, 0x3a, 0xe8, + 0x29, 0x54, 0xe0, 0x61, 0x0f, 0x6b, 0xe6, 0x75, 0xac, 0x48, 0x58, 0x2e, 0x26, 0x54, 0xc4, 0xff, + 0xd2, 0x92, 0x05, 0x3d, 0x84, 0x3d, 0x0b, 0x9f, 0x98, 0x52, 0xcd, 0x89, 0xa5, 0x50, 0xca, 0x58, + 0xf4, 0xef, 0x38, 0x23, 0x53, 0x1b, 0x2b, 0xf2, 0x23, 0x3c, 0x6e, 0x70, 0x30, 0xf6, 0xc0, 0x1e, + 0x9f, 0x9a, 0x85, 0x41, 0x5e, 0xc2, 0x7a, 0x83, 0x57, 0xb7, 0xf8, 0xd8, 0x6e, 0x6c, 0xcd, 0x6d, + 0xe4, 0x16, 0x85, 0x8e, 0x92, 0x54, 0x06, 0xbe, 0xcd, 0x2d, 0x0a, 0x7d, 0x9c, 0x4a, 0xf2, 0x0b, + 0x78, 0x8a, 0xeb, 0x62, 0x11, 0x69, 0x21, 0xe6, 0x2a, 0x78, 0x32, 0x6c, 0xed, 0x78, 0x7b, 0xdb, + 0x4b, 0xb7, 0xe8, 0x03, 0x97, 0xb3, 0xf7, 0xf9, 0x4c, 0x50, 0x40, 0xc5, 0x95, 0x11, 0x90, 0x03, + 0x70, 0xff, 0x64, 0x3a, 0x8d, 0x64, 0x91, 0xab, 0x80, 0x3c, 0x44, 0xdd, 0x35, 0x7c, 0x5a, 0xe4, + 0x8a, 0xbc, 0x03, 0xb0, 0x4c, 0x14, 0xaf, 0x3f, 0x44, 0xec, 0x22, 0x5a, 0xa9, 0xf3, 0x34, 0xff, + 0xcc, 0xac, 0x7a, 0xe3, 0x41, 0x6a, 0x14, 0xa0, 0xfa, 0x67, 0x58, 0xd5, 0x42, 0xb3, 0x79, 0xf0, + 0x74, 0xe8, 0x7c, 0x5b, 0x68, 0xb9, 0x64, 0x04, 0x3d, 0x4b, 0x88, 0x45, 0x3e, 0x4b, 0xaf, 0x83, + 0x4d, 0xd4, 0x0e, 0x97, 0x6a, 0xf1, 0x1a, 0x8e, 0x90, 0x47, 0xbd, 0xe9, 0xdd, 0x0b, 0xd9, 0x02, + 0x3c, 0xa2, 0x38, 0x4a, 0x02, 0xfc, 0x16, 0xf5, 0x3b, 0xf9, 0x3f, 0xf4, 0xab, 0x05, 0xb2, 0x8c, + 0xe5, 0x49, 0xb0, 0x85, 0x84, 0x5e, 0xa9, 0xc7, 0x5a, 0xf8, 0x1a, 0x7a, 0xf7, 0xee, 0x78, 0x17, + 0xda, 0x93, 0xcb, 0x13, 0xea, 0x3f, 0x22, 0x7d, 0x70, 0xcd, 0xd3, 0xf1, 0xc9, 0xd1, 0xe4, 0xd4, + 0x77, 0x48, 0x07, 0xcc, 0x5c, 0xf0, 0x57, 0xc2, 0x77, 0xd0, 0xc6, 0x53, 0xe0, 0x41, 0x75, 0xaa, + 0xfd, 0x47, 0x06, 0x3d, 0xa4, 0xe7, 0xbe, 0x43, 0x5c, 0x58, 0x3d, 0xa4, 0xe7, 0xfb, 0x6f, 0xfc, + 0x15, 0x53, 0xfb, 0xf4, 0x76, 0xdf, 0x6f, 0x11, 0x80, 0xb5, 0x4f, 0x6f, 0xf7, 0xa3, 0xfd, 0x37, + 0x7e, 0x3b, 0xbc, 0x06, 0xaf, 0xd1, 0x8c, 0x19, 0x9b, 0x85, 0xe2, 0xd1, 0xb5, 0xc8, 0x18, 0x0e, + 0xd7, 0x2e, 0xed, 0x14, 0x8a, 0x9f, 0x8a, 0x8c, 0x99, 0x53, 0x66, 0x20, 0x39, 0xe5, 0x38, 0x50, + 0xbb, 0x74, 0xad, 0x50, 0x9c, 0x4e, 0x39, 0xf9, 0x01, 0x06, 0x33, 0x21, 0x63, 0x1e, 0xd5, 0xca, + 0x16, 0xe2, 0x3d, 0xac, 0x4e, 0xac, 0x3c, 0xfc, 0xdb, 0x81, 0x6e, 0xb5, 0xe5, 0x84, 0x40, 0x3b, + 0xe1, 0x2a, 0xc6, 0x25, 0x5c, 0x8a, 0xcf, 0xa6, 0x86, 0xdb, 0x66, 0xa7, 0x35, 0x3e, 0x93, 0x6d, + 0x00, 0xa5, 0x99, 0xd4, 0x38, 0xf2, 0xd1, 0xb6, 0x4d, 0x5d, 0xac, 0x98, 0x49, 0x4f, 0x9e, 0x83, + 0x2b, 0x39, 0x9b, 0x5b, 0xb4, 0x8d, 0x68, 0xd7, 0x14, 0x10, 0xdc, 0x06, 0xc8, 0x78, 0x26, 0xe4, + 0xad, 0xc9, 0x85, 0x93, 0xb7, 0x4d, 0x5d, 0x5b, 0x99, 0x28, 0x1e, 0xfe, 0xe3, 0xc0, 0xe0, 0x5c, + 0x24, 0xc5, 0x9c, 0x5f, 0xdd, 0x2e, 0x38, 0xa6, 0xfa, 0xa3, 0x3a, 0x01, 0xea, 0x56, 0x69, 0x9e, + 0x61, 0xba, 0xc1, 0xde, 0xab, 0xe5, 0x23, 0xe5, 0x9e, 0xd4, 0x1e, 0x88, 0x4b, 0x94, 0x35, 0x86, + 0xcb, 0xf4, 0xae, 0x4a, 0xfe, 0x07, 0x5e, 0x86, 0x9a, 0x48, 0xdf, 0x2e, 0xaa, 0x2e, 0x21, 0xab, + 0x6d, 0xcc, 0x36, 0xe6, 0x45, 0x16, 0x89, 0x59, 0x64, 0x8b, 0x0a, 0xfb, 0xed, 0xd3, 0x5e, 0x5e, + 0x64, 0xe3, 0x99, 0x5d, 0x4f, 0x85, 0xaf, 0xca, 0xef, 0x55, 0xba, 0xde, 0xfb, 0xe8, 0x2e, 0xac, + 0x5e, 0x8e, 0xc7, 0x17, 0xe6, 0x74, 0x74, 0xa1, 0x7d, 0x7e, 0x78, 0x76, 0xe2, 0xaf, 0x84, 0x73, + 0xd8, 0x1a, 0xc9, 0x54, 0xa7, 0x31, 0x9b, 0x4f, 0x14, 0x97, 0xbf, 0x8a, 0x42, 0xe6, 0xfc, 0xb6, + 0x9c, 0x88, 0xf5, 0xa6, 0x3b, 0x8d, 0x4d, 0x3f, 0x80, 0x4e, 0xd9, 0x25, 0xa6, 0xfc, 0xda, 0x1d, + 0x68, 0x0c, 0x55, 0x5a, 0x09, 0xc2, 0x29, 0x3c, 0x5f, 0xb2, 0x9a, 0xaa, 0x96, 0x1b, 0x41, 0x3b, + 0x2e, 0x3e, 0xab, 0xc0, 0xc1, 0x0b, 0xbd, 0x7c, 0x67, 0xbf, 0x9e, 0x96, 0xa2, 0xf8, 0xe8, 0xe9, + 0xef, 0xe5, 0x3f, 0x43, 0xa9, 0x88, 0xf0, 0x47, 0xe2, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x94, + 0x52, 0xed, 0xc2, 0x58, 0x08, 0x00, 0x00, } diff --git a/ui/metrics/metrics_proto/metrics.proto b/ui/metrics/metrics_proto/metrics.proto index c44e9f812..bbb3021b8 100644 --- a/ui/metrics/metrics_proto/metrics.proto +++ b/ui/metrics/metrics_proto/metrics.proto @@ -14,10 +14,8 @@ syntax = "proto2"; -option optimize_for = LITE_RUNTIME; - -package build_metrics; -option go_package = "metrics_proto"; +package soong_build_metrics; +option go_package = "soong_metrics_proto"; message MetricsBase { // Timestamp generated when the build starts. @@ -32,15 +30,15 @@ message MetricsBase { // The target product information, eg. aosp_arm. optional string target_product = 4; - enum BUILDVARIANT { + enum BuildVariant { USER = 0; USERDEBUG = 1; ENG = 2; } // The target build variant information, eg. eng. - optional BUILDVARIANT target_build_variant = 5 [default = ENG]; + optional BuildVariant target_build_variant = 5 [default = ENG]; - enum ARCH { + enum Arch { UNKNOWN = 0; ARM = 1; ARM64 = 2; @@ -48,7 +46,7 @@ message MetricsBase { X86_64 = 4; } // The target arch information, eg. arm. - optional ARCH target_arch = 6 [default = UNKNOWN]; + optional Arch target_arch = 6 [default = UNKNOWN]; // The target arch variant information, eg. armv7-a-neon. optional string target_arch_variant = 7; @@ -57,10 +55,10 @@ message MetricsBase { optional string target_cpu_variant = 8; // The host arch information, eg. x86_64. - optional ARCH host_arch = 9 [default = UNKNOWN]; + optional Arch host_arch = 9 [default = UNKNOWN]; // The host 2nd arch information, eg. x86. - optional ARCH host_2nd_arch = 10 [default = UNKNOWN]; + optional Arch host_2nd_arch = 10 [default = UNKNOWN]; // The host os information, eg. linux. optional string host_os = 11; @@ -92,6 +90,9 @@ message MetricsBase { // The metrics for calling Ninja. repeated PerfInfo ninja_runs = 20; + // The metrics for the whole build + optional PerfInfo total = 21; + optional BuildConfig build_config = 23; // The hostname of the machine. @@ -129,13 +130,13 @@ message PerfInfo { } message ModuleTypeInfo { - enum BUILDSYSTEM { + enum BuildSystem { UNKNOWN = 0; SOONG = 1; MAKE = 2; } // The build system, eg. Soong or Make. - optional BUILDSYSTEM build_system = 1 [default = UNKNOWN]; + optional BuildSystem build_system = 1 [default = UNKNOWN]; // The module type, eg. java_library, cc_binary, and etc. optional string module_type = 2; @@ -143,3 +144,16 @@ message ModuleTypeInfo { // The number of logical modules. optional uint32 num_of_modules = 3; } + +message CriticalUserJourneyMetrics { + // The name of a critical user journey test. + optional string name = 1; + + // The metrics produced when running the critical user journey test. + optional MetricsBase metrics = 2; +} + +message CriticalUserJourneysMetrics { + // A set of metrics from a run of the critical user journey tests. + repeated CriticalUserJourneyMetrics cujs = 1; +} diff --git a/ui/metrics/time.go b/ui/metrics/time.go index c0de35765..401656337 100644 --- a/ui/metrics/time.go +++ b/ui/metrics/time.go @@ -35,7 +35,7 @@ type timeEvent struct { type TimeTracer interface { Begin(name, desc string, thread tracer.Thread) - End(thread tracer.Thread) metrics_proto.PerfInfo + End(thread tracer.Thread) soong_metrics_proto.PerfInfo } type timeTracerImpl struct { @@ -52,7 +52,7 @@ func (t *timeTracerImpl) Begin(name, desc string, _ tracer.Thread) { t.activeEvents = append(t.activeEvents, timeEvent{name: name, desc: desc, start: _now()}) } -func (t *timeTracerImpl) End(tracer.Thread) metrics_proto.PerfInfo { +func (t *timeTracerImpl) End(tracer.Thread) soong_metrics_proto.PerfInfo { if len(t.activeEvents) < 1 { panic("Internal error: No pending events for endAt to end!") } @@ -60,7 +60,7 @@ func (t *timeTracerImpl) End(tracer.Thread) metrics_proto.PerfInfo { t.activeEvents = t.activeEvents[:len(t.activeEvents)-1] realTime := uint64(_now().Sub(lastEvent.start).Nanoseconds()) - return metrics_proto.PerfInfo{ + return soong_metrics_proto.PerfInfo{ Desc: proto.String(lastEvent.desc), Name: proto.String(lastEvent.name), StartTime: proto.Uint64(uint64(lastEvent.start.UnixNano())), diff --git a/ui/status/Android.bp b/ui/status/Android.bp index 901a71309..ec929b342 100644 --- a/ui/status/Android.bp +++ b/ui/status/Android.bp @@ -19,14 +19,17 @@ bootstrap_go_package { "golang-protobuf-proto", "soong-ui-logger", "soong-ui-status-ninja_frontend", + "soong-ui-status-build_error_proto", ], srcs: [ + "critical_path.go", "kati.go", "log.go", "ninja.go", "status.go", ], testSrcs: [ + "critical_path_test.go", "kati_test.go", "ninja_test.go", "status_test.go", @@ -41,3 +44,12 @@ bootstrap_go_package { "ninja_frontend/frontend.pb.go", ], } + +bootstrap_go_package { + name: "soong-ui-status-build_error_proto", + pkgPath: "android/soong/ui/status/build_error_proto", + deps: ["golang-protobuf-proto"], + srcs: [ + "build_error_proto/build_error.pb.go", + ], +} diff --git a/ui/status/build_error_proto/build_error.pb.go b/ui/status/build_error_proto/build_error.pb.go new file mode 100644 index 000000000..d4d0a6ea9 --- /dev/null +++ b/ui/status/build_error_proto/build_error.pb.go @@ -0,0 +1,175 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: build_error.proto + +package soong_build_error_proto + +import ( + fmt "fmt" + proto "github.com/golang/protobuf/proto" + math "math" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package + +type BuildError struct { + // List of error messages of the overall build. The error messages + // are not associated with a build action. + ErrorMessages []string `protobuf:"bytes,1,rep,name=error_messages,json=errorMessages" json:"error_messages,omitempty"` + // List of build action errors. + ActionErrors []*BuildActionError `protobuf:"bytes,2,rep,name=action_errors,json=actionErrors" json:"action_errors,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *BuildError) Reset() { *m = BuildError{} } +func (m *BuildError) String() string { return proto.CompactTextString(m) } +func (*BuildError) ProtoMessage() {} +func (*BuildError) Descriptor() ([]byte, []int) { + return fileDescriptor_a2e15b05802a5501, []int{0} +} + +func (m *BuildError) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_BuildError.Unmarshal(m, b) +} +func (m *BuildError) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_BuildError.Marshal(b, m, deterministic) +} +func (m *BuildError) XXX_Merge(src proto.Message) { + xxx_messageInfo_BuildError.Merge(m, src) +} +func (m *BuildError) XXX_Size() int { + return xxx_messageInfo_BuildError.Size(m) +} +func (m *BuildError) XXX_DiscardUnknown() { + xxx_messageInfo_BuildError.DiscardUnknown(m) +} + +var xxx_messageInfo_BuildError proto.InternalMessageInfo + +func (m *BuildError) GetErrorMessages() []string { + if m != nil { + return m.ErrorMessages + } + return nil +} + +func (m *BuildError) GetActionErrors() []*BuildActionError { + if m != nil { + return m.ActionErrors + } + return nil +} + +// Build is composed of a list of build action. There can be a set of build +// actions that can failed. +type BuildActionError struct { + // Description of the command. + Description *string `protobuf:"bytes,1,opt,name=description" json:"description,omitempty"` + // The command name that raised the error. + Command *string `protobuf:"bytes,2,opt,name=command" json:"command,omitempty"` + // The command output stream. + Output *string `protobuf:"bytes,3,opt,name=output" json:"output,omitempty"` + // List of artifacts (i.e. files) that was produced by the command. + Artifacts []string `protobuf:"bytes,4,rep,name=artifacts" json:"artifacts,omitempty"` + // The error string produced by the build action. + Error *string `protobuf:"bytes,5,opt,name=error" json:"error,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *BuildActionError) Reset() { *m = BuildActionError{} } +func (m *BuildActionError) String() string { return proto.CompactTextString(m) } +func (*BuildActionError) ProtoMessage() {} +func (*BuildActionError) Descriptor() ([]byte, []int) { + return fileDescriptor_a2e15b05802a5501, []int{1} +} + +func (m *BuildActionError) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_BuildActionError.Unmarshal(m, b) +} +func (m *BuildActionError) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_BuildActionError.Marshal(b, m, deterministic) +} +func (m *BuildActionError) XXX_Merge(src proto.Message) { + xxx_messageInfo_BuildActionError.Merge(m, src) +} +func (m *BuildActionError) XXX_Size() int { + return xxx_messageInfo_BuildActionError.Size(m) +} +func (m *BuildActionError) XXX_DiscardUnknown() { + xxx_messageInfo_BuildActionError.DiscardUnknown(m) +} + +var xxx_messageInfo_BuildActionError proto.InternalMessageInfo + +func (m *BuildActionError) GetDescription() string { + if m != nil && m.Description != nil { + return *m.Description + } + return "" +} + +func (m *BuildActionError) GetCommand() string { + if m != nil && m.Command != nil { + return *m.Command + } + return "" +} + +func (m *BuildActionError) GetOutput() string { + if m != nil && m.Output != nil { + return *m.Output + } + return "" +} + +func (m *BuildActionError) GetArtifacts() []string { + if m != nil { + return m.Artifacts + } + return nil +} + +func (m *BuildActionError) GetError() string { + if m != nil && m.Error != nil { + return *m.Error + } + return "" +} + +func init() { + proto.RegisterType((*BuildError)(nil), "soong_build_error.BuildError") + proto.RegisterType((*BuildActionError)(nil), "soong_build_error.BuildActionError") +} + +func init() { proto.RegisterFile("build_error.proto", fileDescriptor_a2e15b05802a5501) } + +var fileDescriptor_a2e15b05802a5501 = []byte{ + // 229 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x64, 0x90, 0xc1, 0x4a, 0xc3, 0x40, + 0x10, 0x86, 0x49, 0x63, 0x95, 0x4c, 0xad, 0xd8, 0x41, 0x74, 0x04, 0x0f, 0xa1, 0x22, 0xe4, 0x94, + 0x83, 0x6f, 0x60, 0x41, 0xf0, 0xe2, 0x25, 0x47, 0x2f, 0x61, 0xdd, 0xac, 0x65, 0xc1, 0x64, 0xc2, + 0xce, 0xe6, 0xe8, 0x8b, 0xf8, 0xb4, 0x92, 0x69, 0xa5, 0xa5, 0x39, 0x7e, 0xdf, 0x3f, 0xfb, 0xef, + 0xce, 0xc2, 0xea, 0x73, 0xf0, 0xdf, 0x4d, 0xed, 0x42, 0xe0, 0x50, 0xf6, 0x81, 0x23, 0xe3, 0x4a, + 0x98, 0xbb, 0x6d, 0x7d, 0x14, 0xac, 0x7f, 0x00, 0x36, 0x23, 0xbe, 0x8e, 0x84, 0x4f, 0x70, 0xa5, + 0xba, 0x6e, 0x9d, 0x88, 0xd9, 0x3a, 0xa1, 0x24, 0x4f, 0x8b, 0xac, 0x5a, 0xaa, 0x7d, 0xdf, 0x4b, + 0x7c, 0x83, 0xa5, 0xb1, 0xd1, 0x73, 0xb7, 0x2b, 0x11, 0x9a, 0xe5, 0x69, 0xb1, 0x78, 0x7e, 0x2c, + 0x27, 0xfd, 0xa5, 0x96, 0xbf, 0xe8, 0xb0, 0x5e, 0x51, 0x5d, 0x9a, 0x03, 0xc8, 0xfa, 0x37, 0x81, + 0xeb, 0xd3, 0x11, 0xcc, 0x61, 0xd1, 0x38, 0xb1, 0xc1, 0xf7, 0xa3, 0xa3, 0x24, 0x4f, 0x8a, 0xac, + 0x3a, 0x56, 0x48, 0x70, 0x61, 0xb9, 0x6d, 0x4d, 0xd7, 0xd0, 0x4c, 0xd3, 0x7f, 0xc4, 0x5b, 0x38, + 0xe7, 0x21, 0xf6, 0x43, 0xa4, 0x54, 0x83, 0x3d, 0xe1, 0x03, 0x64, 0x26, 0x44, 0xff, 0x65, 0x6c, + 0x14, 0x3a, 0xd3, 0xa5, 0x0e, 0x02, 0x6f, 0x60, 0xae, 0xcf, 0xa5, 0xb9, 0x1e, 0xda, 0xc1, 0xe6, + 0xfe, 0xe3, 0x6e, 0xb2, 0x50, 0xad, 0x3f, 0xf9, 0x17, 0x00, 0x00, 0xff, 0xff, 0xb6, 0x18, 0x9e, + 0x17, 0x5d, 0x01, 0x00, 0x00, +} diff --git a/ui/status/build_error_proto/build_error.proto b/ui/status/build_error_proto/build_error.proto new file mode 100644 index 000000000..9c8470d8c --- /dev/null +++ b/ui/status/build_error_proto/build_error.proto @@ -0,0 +1,46 @@ +// Copyright 2019 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +package soong_build_error; +option go_package = "soong_build_error_proto"; + +message BuildError { + // List of error messages of the overall build. The error messages + // are not associated with a build action. + repeated string error_messages = 1; + + // List of build action errors. + repeated BuildActionError action_errors = 2; +} + +// Build is composed of a list of build action. There can be a set of build +// actions that can failed. +message BuildActionError { + // Description of the command. + optional string description = 1; + + // The command name that raised the error. + optional string command = 2; + + // The command output stream. + optional string output = 3; + + // List of artifacts (i.e. files) that was produced by the command. + repeated string artifacts = 4; + + // The error string produced by the build action. + optional string error = 5; +} diff --git a/ui/status/build_error_proto/regen.sh b/ui/status/build_error_proto/regen.sh new file mode 100755 index 000000000..7c3ec8faf --- /dev/null +++ b/ui/status/build_error_proto/regen.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +aprotoc --go_out=paths=source_relative:. build_error.proto diff --git a/ui/status/critical_path.go b/ui/status/critical_path.go new file mode 100644 index 000000000..8065c60f0 --- /dev/null +++ b/ui/status/critical_path.go @@ -0,0 +1,154 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package status + +import ( + "time" + + "android/soong/ui/logger" +) + +func NewCriticalPath(log logger.Logger) StatusOutput { + return &criticalPath{ + log: log, + running: make(map[*Action]time.Time), + nodes: make(map[string]*node), + clock: osClock{}, + } +} + +type criticalPath struct { + log logger.Logger + + nodes map[string]*node + running map[*Action]time.Time + + start, end time.Time + + clock clock +} + +type clock interface { + Now() time.Time +} + +type osClock struct{} + +func (osClock) Now() time.Time { return time.Now() } + +// A critical path node stores the critical path (the minimum time to build the node and all of its dependencies given +// perfect parallelism) for an node. +type node struct { + action *Action + cumulativeDuration time.Duration + duration time.Duration + input *node +} + +func (cp *criticalPath) StartAction(action *Action, counts Counts) { + start := cp.clock.Now() + if cp.start.IsZero() { + cp.start = start + } + cp.running[action] = start +} + +func (cp *criticalPath) FinishAction(result ActionResult, counts Counts) { + if start, ok := cp.running[result.Action]; ok { + delete(cp.running, result.Action) + + // Determine the input to this edge with the longest cumulative duration + var criticalPathInput *node + for _, input := range result.Action.Inputs { + if x := cp.nodes[input]; x != nil { + if criticalPathInput == nil || x.cumulativeDuration > criticalPathInput.cumulativeDuration { + criticalPathInput = x + } + } + } + + end := cp.clock.Now() + duration := end.Sub(start) + + cumulativeDuration := duration + if criticalPathInput != nil { + cumulativeDuration += criticalPathInput.cumulativeDuration + } + + node := &node{ + action: result.Action, + cumulativeDuration: cumulativeDuration, + duration: duration, + input: criticalPathInput, + } + + for _, output := range result.Action.Outputs { + cp.nodes[output] = node + } + + cp.end = end + } +} + +func (cp *criticalPath) Flush() { + criticalPath := cp.criticalPath() + + if len(criticalPath) > 0 { + // Log the critical path to the verbose log + criticalTime := criticalPath[0].cumulativeDuration.Round(time.Second) + cp.log.Verbosef("critical path took %s", criticalTime.String()) + if !cp.start.IsZero() { + elapsedTime := cp.end.Sub(cp.start).Round(time.Second) + cp.log.Verbosef("elapsed time %s", elapsedTime.String()) + if elapsedTime > 0 { + cp.log.Verbosef("perfect parallelism ratio %d%%", + int(float64(criticalTime)/float64(elapsedTime)*100)) + } + } + cp.log.Verbose("critical path:") + for i := len(criticalPath) - 1; i >= 0; i-- { + duration := criticalPath[i].duration + duration = duration.Round(time.Second) + seconds := int(duration.Seconds()) + cp.log.Verbosef(" %2d:%02d %s", + seconds/60, seconds%60, criticalPath[i].action.Description) + } + } +} + +func (cp *criticalPath) Message(level MsgLevel, msg string) {} + +func (cp *criticalPath) Write(p []byte) (n int, err error) { return len(p), nil } + +func (cp *criticalPath) criticalPath() []*node { + var max *node + + // Find the node with the longest critical path + for _, node := range cp.nodes { + if max == nil || node.cumulativeDuration > max.cumulativeDuration { + max = node + } + } + + // Follow the critical path back to the leaf node + var criticalPath []*node + node := max + for node != nil { + criticalPath = append(criticalPath, node) + node = node.input + } + + return criticalPath +} diff --git a/ui/status/critical_path_test.go b/ui/status/critical_path_test.go new file mode 100644 index 000000000..965e0ad13 --- /dev/null +++ b/ui/status/critical_path_test.go @@ -0,0 +1,166 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package status + +import ( + "reflect" + "testing" + "time" +) + +type testCriticalPath struct { + *criticalPath + Counts + + actions map[int]*Action +} + +type testClock time.Time + +func (t testClock) Now() time.Time { return time.Time(t) } + +func (t *testCriticalPath) start(id int, startTime time.Duration, outputs, inputs []string) { + t.clock = testClock(time.Unix(0, 0).Add(startTime)) + action := &Action{ + Description: outputs[0], + Outputs: outputs, + Inputs: inputs, + } + + t.actions[id] = action + t.StartAction(action, t.Counts) +} + +func (t *testCriticalPath) finish(id int, endTime time.Duration) { + t.clock = testClock(time.Unix(0, 0).Add(endTime)) + t.FinishAction(ActionResult{ + Action: t.actions[id], + }, t.Counts) +} + +func TestCriticalPath(t *testing.T) { + tests := []struct { + name string + msgs func(*testCriticalPath) + want []string + wantTime time.Duration + }{ + { + name: "empty", + msgs: func(cp *testCriticalPath) {}, + }, + { + name: "duplicate", + msgs: func(cp *testCriticalPath) { + cp.start(0, 0, []string{"a"}, nil) + cp.start(1, 0, []string{"a"}, nil) + cp.finish(0, 1000) + cp.finish(0, 2000) + }, + want: []string{"a"}, + wantTime: 1000, + }, + { + name: "linear", + // a + // | + // b + // | + // c + msgs: func(cp *testCriticalPath) { + cp.start(0, 0, []string{"a"}, nil) + cp.finish(0, 1000) + cp.start(1, 1000, []string{"b"}, []string{"a"}) + cp.finish(1, 2000) + cp.start(2, 3000, []string{"c"}, []string{"b"}) + cp.finish(2, 4000) + }, + want: []string{"c", "b", "a"}, + wantTime: 3000, + }, + { + name: "diamond", + // a + // |\ + // b c + // |/ + // d + msgs: func(cp *testCriticalPath) { + cp.start(0, 0, []string{"a"}, nil) + cp.finish(0, 1000) + cp.start(1, 1000, []string{"b"}, []string{"a"}) + cp.start(2, 1000, []string{"c"}, []string{"a"}) + cp.finish(1, 2000) + cp.finish(2, 3000) + cp.start(3, 3000, []string{"d"}, []string{"b", "c"}) + cp.finish(3, 4000) + }, + want: []string{"d", "c", "a"}, + wantTime: 4000, + }, + { + name: "multiple", + // a d + // | | + // b e + // | + // c + msgs: func(cp *testCriticalPath) { + cp.start(0, 0, []string{"a"}, nil) + cp.start(3, 0, []string{"d"}, nil) + cp.finish(0, 1000) + cp.finish(3, 1000) + cp.start(1, 1000, []string{"b"}, []string{"a"}) + cp.start(4, 1000, []string{"e"}, []string{"d"}) + cp.finish(1, 2000) + cp.start(2, 2000, []string{"c"}, []string{"b"}) + cp.finish(2, 3000) + cp.finish(4, 4000) + + }, + want: []string{"e", "d"}, + wantTime: 4000, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cp := &testCriticalPath{ + criticalPath: NewCriticalPath(nil).(*criticalPath), + actions: make(map[int]*Action), + } + + tt.msgs(cp) + + criticalPath := cp.criticalPath.criticalPath() + + var descs []string + for _, x := range criticalPath { + descs = append(descs, x.action.Description) + } + + if !reflect.DeepEqual(descs, tt.want) { + t.Errorf("criticalPath.criticalPath() = %v, want %v", descs, tt.want) + } + + var gotTime time.Duration + if len(criticalPath) > 0 { + gotTime = criticalPath[0].cumulativeDuration + } + if gotTime != tt.wantTime { + t.Errorf("cumulativeDuration[0].cumulativeDuration = %v, want %v", gotTime, tt.wantTime) + } + }) + } +} diff --git a/ui/status/log.go b/ui/status/log.go index 921aa4401..d40724809 100644 --- a/ui/status/log.go +++ b/ui/status/log.go @@ -15,11 +15,17 @@ package status import ( - "android/soong/ui/logger" "compress/gzip" + "errors" "fmt" "io" + "io/ioutil" "strings" + + "github.com/golang/protobuf/proto" + + "android/soong/ui/logger" + "android/soong/ui/status/build_error_proto" ) type verboseLog struct { @@ -71,9 +77,13 @@ func (v *verboseLog) Message(level MsgLevel, message string) { fmt.Fprintf(v.w, "%s%s\n", level.Prefix(), message) } -type errorLog struct { - w io.WriteCloser +func (v *verboseLog) Write(p []byte) (int, error) { + fmt.Fprint(v.w, string(p)) + return len(p), nil +} +type errorLog struct { + w io.WriteCloser empty bool } @@ -97,20 +107,17 @@ func (e *errorLog) FinishAction(result ActionResult, counts Counts) { return } - cmd := result.Command - if cmd == "" { - cmd = result.Description - } - if !e.empty { fmt.Fprintf(e.w, "\n\n") } e.empty = false fmt.Fprintf(e.w, "FAILED: %s\n", result.Description) + if len(result.Outputs) > 0 { fmt.Fprintf(e.w, "Outputs: %s\n", strings.Join(result.Outputs, " ")) } + fmt.Fprintf(e.w, "Error: %s\n", result.Error) if result.Command != "" { fmt.Fprintf(e.w, "Command: %s\n", result.Command) @@ -134,3 +141,60 @@ func (e *errorLog) Message(level MsgLevel, message string) { fmt.Fprintf(e.w, "error: %s\n", message) } + +func (e *errorLog) Write(p []byte) (int, error) { + fmt.Fprint(e.w, string(p)) + return len(p), nil +} + +type errorProtoLog struct { + errorProto soong_build_error_proto.BuildError + filename string + log logger.Logger +} + +func NewProtoErrorLog(log logger.Logger, filename string) StatusOutput { + return &errorProtoLog{ + errorProto: soong_build_error_proto.BuildError{}, + filename: filename, + log: log, + } +} + +func (e *errorProtoLog) StartAction(action *Action, counts Counts) {} + +func (e *errorProtoLog) FinishAction(result ActionResult, counts Counts) { + if result.Error == nil { + return + } + + e.errorProto.ActionErrors = append(e.errorProto.ActionErrors, &soong_build_error_proto.BuildActionError{ + Description: proto.String(result.Description), + Command: proto.String(result.Command), + Output: proto.String(result.Output), + Artifacts: result.Outputs, + Error: proto.String(result.Error.Error()), + }) +} + +func (e *errorProtoLog) Flush() { + data, err := proto.Marshal(&e.errorProto) + if err != nil { + e.log.Printf("Failed to marshal build status proto: %v\n", err) + return + } + err = ioutil.WriteFile(e.filename, []byte(data), 0644) + if err != nil { + e.log.Printf("Failed to write file %s: %v\n", e.filename, err) + } +} + +func (e *errorProtoLog) Message(level MsgLevel, message string) { + if level > ErrorLvl { + e.errorProto.ErrorMessages = append(e.errorProto.ErrorMessages, message) + } +} + +func (e *errorProtoLog) Write(p []byte) (int, error) { + return 0, errors.New("not supported") +} diff --git a/ui/status/ninja.go b/ui/status/ninja.go index ee2a2daaa..9cf2f6a81 100644 --- a/ui/status/ninja.go +++ b/ui/status/ninja.go @@ -142,6 +142,7 @@ func (n *NinjaReader) run() { action := &Action{ Description: msg.EdgeStarted.GetDesc(), Outputs: msg.EdgeStarted.Outputs, + Inputs: msg.EdgeStarted.Inputs, Command: msg.EdgeStarted.GetCommand(), } n.status.StartAction(action) diff --git a/ui/status/status.go b/ui/status/status.go index 46ec72e80..df33baa8b 100644 --- a/ui/status/status.go +++ b/ui/status/status.go @@ -32,6 +32,10 @@ type Action struct { // but they can be any string. Outputs []string + // Inputs is the (optional) list of inputs. Usually these are files, + // but they can be any string. + Inputs []string + // Command is the actual command line executed to perform the action. // It's optional, but one of either Description or Command should be // set. @@ -173,6 +177,9 @@ type StatusOutput interface { // Flush is called when your outputs should be flushed / closed. No // output is expected after this call. Flush() + + // Write lets StatusOutput implement io.Writer + Write(p []byte) (n int, err error) } // Status is the multiplexer / accumulator between ToolStatus instances (via diff --git a/ui/status/status_test.go b/ui/status/status_test.go index e62785f43..949458222 100644 --- a/ui/status/status_test.go +++ b/ui/status/status_test.go @@ -27,6 +27,11 @@ func (c *counterOutput) FinishAction(result ActionResult, counts Counts) { func (c counterOutput) Message(level MsgLevel, msg string) {} func (c counterOutput) Flush() {} +func (c counterOutput) Write(p []byte) (int, error) { + // Discard writes + return len(p), nil +} + func (c counterOutput) Expect(t *testing.T, counts Counts) { if Counts(c) == counts { return diff --git a/ui/terminal/Android.bp b/ui/terminal/Android.bp index 7104a5047..b533b0d30 100644 --- a/ui/terminal/Android.bp +++ b/ui/terminal/Android.bp @@ -17,11 +17,15 @@ bootstrap_go_package { pkgPath: "android/soong/ui/terminal", deps: ["soong-ui-status"], srcs: [ + "dumb_status.go", + "format.go", + "smart_status.go", "status.go", - "writer.go", + "stdio.go", "util.go", ], testSrcs: [ + "status_test.go", "util_test.go", ], darwin: { diff --git a/ui/terminal/dumb_status.go b/ui/terminal/dumb_status.go new file mode 100644 index 000000000..201770fac --- /dev/null +++ b/ui/terminal/dumb_status.go @@ -0,0 +1,71 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package terminal + +import ( + "fmt" + "io" + + "android/soong/ui/status" +) + +type dumbStatusOutput struct { + writer io.Writer + formatter formatter +} + +// NewDumbStatusOutput returns a StatusOutput that represents the +// current build status similarly to Ninja's built-in terminal +// output. +func NewDumbStatusOutput(w io.Writer, formatter formatter) status.StatusOutput { + return &dumbStatusOutput{ + writer: w, + formatter: formatter, + } +} + +func (s *dumbStatusOutput) Message(level status.MsgLevel, message string) { + if level >= status.StatusLvl { + fmt.Fprintln(s.writer, s.formatter.message(level, message)) + } +} + +func (s *dumbStatusOutput) StartAction(action *status.Action, counts status.Counts) { +} + +func (s *dumbStatusOutput) FinishAction(result status.ActionResult, counts status.Counts) { + str := result.Description + if str == "" { + str = result.Command + } + + progress := s.formatter.progress(counts) + str + + output := s.formatter.result(result) + output = string(stripAnsiEscapes([]byte(output))) + + if output != "" { + fmt.Fprint(s.writer, progress, "\n", output) + } else { + fmt.Fprintln(s.writer, progress) + } +} + +func (s *dumbStatusOutput) Flush() {} + +func (s *dumbStatusOutput) Write(p []byte) (int, error) { + fmt.Fprint(s.writer, string(p)) + return len(p), nil +} diff --git a/ui/terminal/format.go b/ui/terminal/format.go new file mode 100644 index 000000000..4205bdc22 --- /dev/null +++ b/ui/terminal/format.go @@ -0,0 +1,123 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package terminal + +import ( + "fmt" + "strings" + "time" + + "android/soong/ui/status" +) + +type formatter struct { + format string + quiet bool + start time.Time +} + +// newFormatter returns a formatter for formatting output to +// the terminal in a format similar to Ninja. +// format takes nearly all the same options as NINJA_STATUS. +// %c is currently unsupported. +func newFormatter(format string, quiet bool) formatter { + return formatter{ + format: format, + quiet: quiet, + start: time.Now(), + } +} + +func (s formatter) message(level status.MsgLevel, message string) string { + if level >= status.ErrorLvl { + return fmt.Sprintf("FAILED: %s", message) + } else if level > status.StatusLvl { + return fmt.Sprintf("%s%s", level.Prefix(), message) + } else if level == status.StatusLvl { + return message + } + return "" +} + +func (s formatter) progress(counts status.Counts) string { + if s.format == "" { + return fmt.Sprintf("[%3d%% %d/%d] ", 100*counts.FinishedActions/counts.TotalActions, counts.FinishedActions, counts.TotalActions) + } + + buf := &strings.Builder{} + for i := 0; i < len(s.format); i++ { + c := s.format[i] + if c != '%' { + buf.WriteByte(c) + continue + } + + i = i + 1 + if i == len(s.format) { + buf.WriteByte(c) + break + } + + c = s.format[i] + switch c { + case '%': + buf.WriteByte(c) + case 's': + fmt.Fprintf(buf, "%d", counts.StartedActions) + case 't': + fmt.Fprintf(buf, "%d", counts.TotalActions) + case 'r': + fmt.Fprintf(buf, "%d", counts.RunningActions) + case 'u': + fmt.Fprintf(buf, "%d", counts.TotalActions-counts.StartedActions) + case 'f': + fmt.Fprintf(buf, "%d", counts.FinishedActions) + case 'o': + fmt.Fprintf(buf, "%.1f", float64(counts.FinishedActions)/time.Since(s.start).Seconds()) + case 'c': + // TODO: implement? + buf.WriteRune('?') + case 'p': + fmt.Fprintf(buf, "%3d%%", 100*counts.FinishedActions/counts.TotalActions) + case 'e': + fmt.Fprintf(buf, "%.3f", time.Since(s.start).Seconds()) + default: + buf.WriteString("unknown placeholder '") + buf.WriteByte(c) + buf.WriteString("'") + } + } + return buf.String() +} + +func (s formatter) result(result status.ActionResult) string { + var ret string + if result.Error != nil { + targets := strings.Join(result.Outputs, " ") + if s.quiet || result.Command == "" { + ret = fmt.Sprintf("FAILED: %s\n%s", targets, result.Output) + } else { + ret = fmt.Sprintf("FAILED: %s\n%s\n%s", targets, result.Command, result.Output) + } + } else if result.Output != "" { + ret = result.Output + } + + if len(ret) > 0 && ret[len(ret)-1] != '\n' { + ret += "\n" + } + + return ret +} diff --git a/ui/terminal/smart_status.go b/ui/terminal/smart_status.go new file mode 100644 index 000000000..6bdf14074 --- /dev/null +++ b/ui/terminal/smart_status.go @@ -0,0 +1,435 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package terminal + +import ( + "fmt" + "io" + "os" + "os/signal" + "strconv" + "strings" + "sync" + "syscall" + "time" + + "android/soong/ui/status" +) + +const tableHeightEnVar = "SOONG_UI_TABLE_HEIGHT" + +type actionTableEntry struct { + action *status.Action + startTime time.Time +} + +type smartStatusOutput struct { + writer io.Writer + formatter formatter + + lock sync.Mutex + + haveBlankLine bool + + tableMode bool + tableHeight int + requestedTableHeight int + termWidth, termHeight int + + runningActions []actionTableEntry + ticker *time.Ticker + done chan bool + sigwinch chan os.Signal + sigwinchHandled chan bool +} + +// NewSmartStatusOutput returns a StatusOutput that represents the +// current build status similarly to Ninja's built-in terminal +// output. +func NewSmartStatusOutput(w io.Writer, formatter formatter) status.StatusOutput { + s := &smartStatusOutput{ + writer: w, + formatter: formatter, + + haveBlankLine: true, + + tableMode: true, + + done: make(chan bool), + sigwinch: make(chan os.Signal), + } + + if env, ok := os.LookupEnv(tableHeightEnVar); ok { + h, _ := strconv.Atoi(env) + s.tableMode = h > 0 + s.requestedTableHeight = h + } + + s.updateTermSize() + + if s.tableMode { + // Add empty lines at the bottom of the screen to scroll back the existing history + // and make room for the action table. + // TODO: read the cursor position to see if the empty lines are necessary? + for i := 0; i < s.tableHeight; i++ { + fmt.Fprintln(w) + } + + // Hide the cursor to prevent seeing it bouncing around + fmt.Fprintf(s.writer, ansi.hideCursor()) + + // Configure the empty action table + s.actionTable() + + // Start a tick to update the action table periodically + s.startActionTableTick() + } + + s.startSigwinch() + + return s +} + +func (s *smartStatusOutput) Message(level status.MsgLevel, message string) { + if level < status.StatusLvl { + return + } + + str := s.formatter.message(level, message) + + s.lock.Lock() + defer s.lock.Unlock() + + if level > status.StatusLvl { + s.print(str) + } else { + s.statusLine(str) + } +} + +func (s *smartStatusOutput) StartAction(action *status.Action, counts status.Counts) { + startTime := time.Now() + + str := action.Description + if str == "" { + str = action.Command + } + + progress := s.formatter.progress(counts) + + s.lock.Lock() + defer s.lock.Unlock() + + s.runningActions = append(s.runningActions, actionTableEntry{ + action: action, + startTime: startTime, + }) + + s.statusLine(progress + str) +} + +func (s *smartStatusOutput) FinishAction(result status.ActionResult, counts status.Counts) { + str := result.Description + if str == "" { + str = result.Command + } + + progress := s.formatter.progress(counts) + str + + output := s.formatter.result(result) + + s.lock.Lock() + defer s.lock.Unlock() + + for i, runningAction := range s.runningActions { + if runningAction.action == result.Action { + s.runningActions = append(s.runningActions[:i], s.runningActions[i+1:]...) + break + } + } + + if output != "" { + s.statusLine(progress) + s.requestLine() + s.print(output) + } else { + s.statusLine(progress) + } +} + +func (s *smartStatusOutput) Flush() { + if s.tableMode { + // Stop the action table tick outside of the lock to avoid lock ordering issues between s.done and + // s.lock, the goroutine in startActionTableTick can get blocked on the lock and be unable to read + // from the channel. + s.stopActionTableTick() + } + + s.lock.Lock() + defer s.lock.Unlock() + + s.stopSigwinch() + + s.requestLine() + + s.runningActions = nil + + if s.tableMode { + // Update the table after clearing runningActions to clear it + s.actionTable() + + // Reset the scrolling region to the whole terminal + fmt.Fprintf(s.writer, ansi.resetScrollingMargins()) + _, height, _ := termSize(s.writer) + // Move the cursor to the top of the now-blank, previously non-scrolling region + fmt.Fprintf(s.writer, ansi.setCursor(height-s.tableHeight, 1)) + // Turn the cursor back on + fmt.Fprintf(s.writer, ansi.showCursor()) + } +} + +func (s *smartStatusOutput) Write(p []byte) (int, error) { + s.lock.Lock() + defer s.lock.Unlock() + s.print(string(p)) + return len(p), nil +} + +func (s *smartStatusOutput) requestLine() { + if !s.haveBlankLine { + fmt.Fprintln(s.writer) + s.haveBlankLine = true + } +} + +func (s *smartStatusOutput) print(str string) { + if !s.haveBlankLine { + fmt.Fprint(s.writer, "\r", ansi.clearToEndOfLine()) + s.haveBlankLine = true + } + fmt.Fprint(s.writer, str) + if len(str) == 0 || str[len(str)-1] != '\n' { + fmt.Fprint(s.writer, "\n") + } +} + +func (s *smartStatusOutput) statusLine(str string) { + idx := strings.IndexRune(str, '\n') + if idx != -1 { + str = str[0:idx] + } + + // Limit line width to the terminal width, otherwise we'll wrap onto + // another line and we won't delete the previous line. + str = elide(str, s.termWidth) + + // Move to the beginning on the line, turn on bold, print the output, + // turn off bold, then clear the rest of the line. + start := "\r" + ansi.bold() + end := ansi.regular() + ansi.clearToEndOfLine() + fmt.Fprint(s.writer, start, str, end) + s.haveBlankLine = false +} + +func elide(str string, width int) string { + if width > 0 && len(str) > width { + // TODO: Just do a max. Ninja elides the middle, but that's + // more complicated and these lines aren't that important. + str = str[:width] + } + + return str +} + +func (s *smartStatusOutput) startActionTableTick() { + s.ticker = time.NewTicker(time.Second) + go func() { + for { + select { + case <-s.ticker.C: + s.lock.Lock() + s.actionTable() + s.lock.Unlock() + case <-s.done: + return + } + } + }() +} + +func (s *smartStatusOutput) stopActionTableTick() { + s.ticker.Stop() + s.done <- true +} + +func (s *smartStatusOutput) startSigwinch() { + signal.Notify(s.sigwinch, syscall.SIGWINCH) + go func() { + for _ = range s.sigwinch { + s.lock.Lock() + s.updateTermSize() + if s.tableMode { + s.actionTable() + } + s.lock.Unlock() + if s.sigwinchHandled != nil { + s.sigwinchHandled <- true + } + } + }() +} + +func (s *smartStatusOutput) stopSigwinch() { + signal.Stop(s.sigwinch) + close(s.sigwinch) +} + +func (s *smartStatusOutput) updateTermSize() { + if w, h, ok := termSize(s.writer); ok { + firstUpdate := s.termHeight == 0 && s.termWidth == 0 + oldScrollingHeight := s.termHeight - s.tableHeight + + s.termWidth, s.termHeight = w, h + + if s.tableMode { + tableHeight := s.requestedTableHeight + if tableHeight == 0 { + tableHeight = s.termHeight / 4 + if tableHeight < 1 { + tableHeight = 1 + } else if tableHeight > 10 { + tableHeight = 10 + } + } + if tableHeight > s.termHeight-1 { + tableHeight = s.termHeight - 1 + } + s.tableHeight = tableHeight + + scrollingHeight := s.termHeight - s.tableHeight + + if !firstUpdate { + // If the scrolling region has changed, attempt to pan the existing text so that it is + // not overwritten by the table. + if scrollingHeight < oldScrollingHeight { + pan := oldScrollingHeight - scrollingHeight + if pan > s.tableHeight { + pan = s.tableHeight + } + fmt.Fprint(s.writer, ansi.panDown(pan)) + } + } + } + } +} + +func (s *smartStatusOutput) actionTable() { + scrollingHeight := s.termHeight - s.tableHeight + + // Update the scrolling region in case the height of the terminal changed + + fmt.Fprint(s.writer, ansi.setScrollingMargins(1, scrollingHeight)) + + // Write as many status lines as fit in the table + for tableLine := 0; tableLine < s.tableHeight; tableLine++ { + if tableLine >= s.tableHeight { + break + } + // Move the cursor to the correct line of the non-scrolling region + fmt.Fprint(s.writer, ansi.setCursor(scrollingHeight+1+tableLine, 1)) + + if tableLine < len(s.runningActions) { + runningAction := s.runningActions[tableLine] + + seconds := int(time.Since(runningAction.startTime).Round(time.Second).Seconds()) + + desc := runningAction.action.Description + if desc == "" { + desc = runningAction.action.Command + } + + color := "" + if seconds >= 60 { + color = ansi.red() + ansi.bold() + } else if seconds >= 30 { + color = ansi.yellow() + ansi.bold() + } + + durationStr := fmt.Sprintf(" %2d:%02d ", seconds/60, seconds%60) + desc = elide(desc, s.termWidth-len(durationStr)) + durationStr = color + durationStr + ansi.regular() + fmt.Fprint(s.writer, durationStr, desc) + } + fmt.Fprint(s.writer, ansi.clearToEndOfLine()) + } + + // Move the cursor back to the last line of the scrolling region + fmt.Fprint(s.writer, ansi.setCursor(scrollingHeight, 1)) +} + +var ansi = ansiImpl{} + +type ansiImpl struct{} + +func (ansiImpl) clearToEndOfLine() string { + return "\x1b[K" +} + +func (ansiImpl) setCursor(row, column int) string { + // Direct cursor address + return fmt.Sprintf("\x1b[%d;%dH", row, column) +} + +func (ansiImpl) setScrollingMargins(top, bottom int) string { + // Set Top and Bottom Margins DECSTBM + return fmt.Sprintf("\x1b[%d;%dr", top, bottom) +} + +func (ansiImpl) resetScrollingMargins() string { + // Set Top and Bottom Margins DECSTBM + return fmt.Sprintf("\x1b[r") +} + +func (ansiImpl) red() string { + return "\x1b[31m" +} + +func (ansiImpl) yellow() string { + return "\x1b[33m" +} + +func (ansiImpl) bold() string { + return "\x1b[1m" +} + +func (ansiImpl) regular() string { + return "\x1b[0m" +} + +func (ansiImpl) showCursor() string { + return "\x1b[?25h" +} + +func (ansiImpl) hideCursor() string { + return "\x1b[?25l" +} + +func (ansiImpl) panDown(lines int) string { + return fmt.Sprintf("\x1b[%dS", lines) +} + +func (ansiImpl) panUp(lines int) string { + return fmt.Sprintf("\x1b[%dT", lines) +} diff --git a/ui/terminal/status.go b/ui/terminal/status.go index 2445c5b2c..60dfc7025 100644 --- a/ui/terminal/status.go +++ b/ui/terminal/status.go @@ -15,131 +15,23 @@ package terminal import ( - "fmt" - "strings" - "time" + "io" "android/soong/ui/status" ) -type statusOutput struct { - writer Writer - format string - - start time.Time - quiet bool -} - // NewStatusOutput returns a StatusOutput that represents the // current build status similarly to Ninja's built-in terminal // output. // // statusFormat takes nearly all the same options as NINJA_STATUS. // %c is currently unsupported. -func NewStatusOutput(w Writer, statusFormat string, quietBuild bool) status.StatusOutput { - return &statusOutput{ - writer: w, - format: statusFormat, - - start: time.Now(), - quiet: quietBuild, - } -} - -func (s *statusOutput) Message(level status.MsgLevel, message string) { - if level >= status.ErrorLvl { - s.writer.Print(fmt.Sprintf("FAILED: %s", message)) - } else if level > status.StatusLvl { - s.writer.Print(fmt.Sprintf("%s%s", level.Prefix(), message)) - } else if level == status.StatusLvl { - s.writer.StatusLine(message) - } -} - -func (s *statusOutput) StartAction(action *status.Action, counts status.Counts) { - if !s.writer.isSmartTerminal() { - return - } - - str := action.Description - if str == "" { - str = action.Command - } +func NewStatusOutput(w io.Writer, statusFormat string, forceDumbOutput, quietBuild bool) status.StatusOutput { + formatter := newFormatter(statusFormat, quietBuild) - s.writer.StatusLine(s.progress(counts) + str) -} - -func (s *statusOutput) FinishAction(result status.ActionResult, counts status.Counts) { - str := result.Description - if str == "" { - str = result.Command - } - - progress := s.progress(counts) + str - - if result.Error != nil { - targets := strings.Join(result.Outputs, " ") - if s.quiet || result.Command == "" { - s.writer.StatusAndMessage(progress, fmt.Sprintf("FAILED: %s\n%s", targets, result.Output)) - } else { - s.writer.StatusAndMessage(progress, fmt.Sprintf("FAILED: %s\n%s\n%s", targets, result.Command, result.Output)) - } - } else if result.Output != "" { - s.writer.StatusAndMessage(progress, result.Output) + if !forceDumbOutput && isSmartTerminal(w) { + return NewSmartStatusOutput(w, formatter) } else { - s.writer.StatusLine(progress) - } -} - -func (s *statusOutput) Flush() {} - -func (s *statusOutput) progress(counts status.Counts) string { - if s.format == "" { - return fmt.Sprintf("[%3d%% %d/%d] ", 100*counts.FinishedActions/counts.TotalActions, counts.FinishedActions, counts.TotalActions) - } - - buf := &strings.Builder{} - for i := 0; i < len(s.format); i++ { - c := s.format[i] - if c != '%' { - buf.WriteByte(c) - continue - } - - i = i + 1 - if i == len(s.format) { - buf.WriteByte(c) - break - } - - c = s.format[i] - switch c { - case '%': - buf.WriteByte(c) - case 's': - fmt.Fprintf(buf, "%d", counts.StartedActions) - case 't': - fmt.Fprintf(buf, "%d", counts.TotalActions) - case 'r': - fmt.Fprintf(buf, "%d", counts.RunningActions) - case 'u': - fmt.Fprintf(buf, "%d", counts.TotalActions-counts.StartedActions) - case 'f': - fmt.Fprintf(buf, "%d", counts.FinishedActions) - case 'o': - fmt.Fprintf(buf, "%.1f", float64(counts.FinishedActions)/time.Since(s.start).Seconds()) - case 'c': - // TODO: implement? - buf.WriteRune('?') - case 'p': - fmt.Fprintf(buf, "%3d%%", 100*counts.FinishedActions/counts.TotalActions) - case 'e': - fmt.Fprintf(buf, "%.3f", time.Since(s.start).Seconds()) - default: - buf.WriteString("unknown placeholder '") - buf.WriteByte(c) - buf.WriteString("'") - } + return NewDumbStatusOutput(w, formatter) } - return buf.String() } diff --git a/ui/terminal/status_test.go b/ui/terminal/status_test.go new file mode 100644 index 000000000..9f6082988 --- /dev/null +++ b/ui/terminal/status_test.go @@ -0,0 +1,295 @@ +// Copyright 2018 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package terminal + +import ( + "bytes" + "fmt" + "os" + "syscall" + "testing" + + "android/soong/ui/status" +) + +func TestStatusOutput(t *testing.T) { + tests := []struct { + name string + calls func(stat status.StatusOutput) + smart string + dumb string + }{ + { + name: "two actions", + calls: twoActions, + smart: "\r\x1b[1m[ 0% 0/2] action1\x1b[0m\x1b[K\r\x1b[1m[ 50% 1/2] action1\x1b[0m\x1b[K\r\x1b[1m[ 50% 1/2] action2\x1b[0m\x1b[K\r\x1b[1m[100% 2/2] action2\x1b[0m\x1b[K\n", + dumb: "[ 50% 1/2] action1\n[100% 2/2] action2\n", + }, + { + name: "two parallel actions", + calls: twoParallelActions, + smart: "\r\x1b[1m[ 0% 0/2] action1\x1b[0m\x1b[K\r\x1b[1m[ 0% 0/2] action2\x1b[0m\x1b[K\r\x1b[1m[ 50% 1/2] action1\x1b[0m\x1b[K\r\x1b[1m[100% 2/2] action2\x1b[0m\x1b[K\n", + dumb: "[ 50% 1/2] action1\n[100% 2/2] action2\n", + }, + { + name: "action with output", + calls: actionsWithOutput, + smart: "\r\x1b[1m[ 0% 0/3] action1\x1b[0m\x1b[K\r\x1b[1m[ 33% 1/3] action1\x1b[0m\x1b[K\r\x1b[1m[ 33% 1/3] action2\x1b[0m\x1b[K\r\x1b[1m[ 66% 2/3] action2\x1b[0m\x1b[K\noutput1\noutput2\n\r\x1b[1m[ 66% 2/3] action3\x1b[0m\x1b[K\r\x1b[1m[100% 3/3] action3\x1b[0m\x1b[K\n", + dumb: "[ 33% 1/3] action1\n[ 66% 2/3] action2\noutput1\noutput2\n[100% 3/3] action3\n", + }, + { + name: "action with output without newline", + calls: actionsWithOutputWithoutNewline, + smart: "\r\x1b[1m[ 0% 0/3] action1\x1b[0m\x1b[K\r\x1b[1m[ 33% 1/3] action1\x1b[0m\x1b[K\r\x1b[1m[ 33% 1/3] action2\x1b[0m\x1b[K\r\x1b[1m[ 66% 2/3] action2\x1b[0m\x1b[K\noutput1\noutput2\n\r\x1b[1m[ 66% 2/3] action3\x1b[0m\x1b[K\r\x1b[1m[100% 3/3] action3\x1b[0m\x1b[K\n", + dumb: "[ 33% 1/3] action1\n[ 66% 2/3] action2\noutput1\noutput2\n[100% 3/3] action3\n", + }, + { + name: "action with error", + calls: actionsWithError, + smart: "\r\x1b[1m[ 0% 0/3] action1\x1b[0m\x1b[K\r\x1b[1m[ 33% 1/3] action1\x1b[0m\x1b[K\r\x1b[1m[ 33% 1/3] action2\x1b[0m\x1b[K\r\x1b[1m[ 66% 2/3] action2\x1b[0m\x1b[K\nFAILED: f1 f2\ntouch f1 f2\nerror1\nerror2\n\r\x1b[1m[ 66% 2/3] action3\x1b[0m\x1b[K\r\x1b[1m[100% 3/3] action3\x1b[0m\x1b[K\n", + dumb: "[ 33% 1/3] action1\n[ 66% 2/3] action2\nFAILED: f1 f2\ntouch f1 f2\nerror1\nerror2\n[100% 3/3] action3\n", + }, + { + name: "action with empty description", + calls: actionWithEmptyDescription, + smart: "\r\x1b[1m[ 0% 0/1] command1\x1b[0m\x1b[K\r\x1b[1m[100% 1/1] command1\x1b[0m\x1b[K\n", + dumb: "[100% 1/1] command1\n", + }, + { + name: "messages", + calls: actionsWithMessages, + smart: "\r\x1b[1m[ 0% 0/2] action1\x1b[0m\x1b[K\r\x1b[1m[ 50% 1/2] action1\x1b[0m\x1b[K\r\x1b[1mstatus\x1b[0m\x1b[K\r\x1b[Kprint\nFAILED: error\n\r\x1b[1m[ 50% 1/2] action2\x1b[0m\x1b[K\r\x1b[1m[100% 2/2] action2\x1b[0m\x1b[K\n", + dumb: "[ 50% 1/2] action1\nstatus\nprint\nFAILED: error\n[100% 2/2] action2\n", + }, + { + name: "action with long description", + calls: actionWithLongDescription, + smart: "\r\x1b[1m[ 0% 0/2] action with very long descrip\x1b[0m\x1b[K\r\x1b[1m[ 50% 1/2] action with very long descrip\x1b[0m\x1b[K\n", + dumb: "[ 50% 1/2] action with very long description to test eliding\n", + }, + { + name: "action with output with ansi codes", + calls: actionWithOuptutWithAnsiCodes, + smart: "\r\x1b[1m[ 0% 0/1] action1\x1b[0m\x1b[K\r\x1b[1m[100% 1/1] action1\x1b[0m\x1b[K\n\x1b[31mcolor\x1b[0m\n", + dumb: "[100% 1/1] action1\ncolor\n", + }, + } + + os.Setenv(tableHeightEnVar, "") + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + t.Run("smart", func(t *testing.T) { + smart := &fakeSmartTerminal{termWidth: 40} + stat := NewStatusOutput(smart, "", false, false) + tt.calls(stat) + stat.Flush() + + if g, w := smart.String(), tt.smart; g != w { + t.Errorf("want:\n%q\ngot:\n%q", w, g) + } + }) + + t.Run("dumb", func(t *testing.T) { + dumb := &bytes.Buffer{} + stat := NewStatusOutput(dumb, "", false, false) + tt.calls(stat) + stat.Flush() + + if g, w := dumb.String(), tt.dumb; g != w { + t.Errorf("want:\n%q\ngot:\n%q", w, g) + } + }) + + t.Run("force dumb", func(t *testing.T) { + smart := &fakeSmartTerminal{termWidth: 40} + stat := NewStatusOutput(smart, "", true, false) + tt.calls(stat) + stat.Flush() + + if g, w := smart.String(), tt.dumb; g != w { + t.Errorf("want:\n%q\ngot:\n%q", w, g) + } + }) + }) + } +} + +type runner struct { + counts status.Counts + stat status.StatusOutput +} + +func newRunner(stat status.StatusOutput, totalActions int) *runner { + return &runner{ + counts: status.Counts{TotalActions: totalActions}, + stat: stat, + } +} + +func (r *runner) startAction(action *status.Action) { + r.counts.StartedActions++ + r.counts.RunningActions++ + r.stat.StartAction(action, r.counts) +} + +func (r *runner) finishAction(result status.ActionResult) { + r.counts.FinishedActions++ + r.counts.RunningActions-- + r.stat.FinishAction(result, r.counts) +} + +func (r *runner) finishAndStartAction(result status.ActionResult, action *status.Action) { + r.counts.FinishedActions++ + r.stat.FinishAction(result, r.counts) + + r.counts.StartedActions++ + r.stat.StartAction(action, r.counts) +} + +var ( + action1 = &status.Action{Description: "action1"} + result1 = status.ActionResult{Action: action1} + action2 = &status.Action{Description: "action2"} + result2 = status.ActionResult{Action: action2} + action3 = &status.Action{Description: "action3"} + result3 = status.ActionResult{Action: action3} +) + +func twoActions(stat status.StatusOutput) { + runner := newRunner(stat, 2) + runner.startAction(action1) + runner.finishAction(result1) + runner.startAction(action2) + runner.finishAction(result2) +} + +func twoParallelActions(stat status.StatusOutput) { + runner := newRunner(stat, 2) + runner.startAction(action1) + runner.startAction(action2) + runner.finishAction(result1) + runner.finishAction(result2) +} + +func actionsWithOutput(stat status.StatusOutput) { + result2WithOutput := status.ActionResult{Action: action2, Output: "output1\noutput2\n"} + + runner := newRunner(stat, 3) + runner.startAction(action1) + runner.finishAction(result1) + runner.startAction(action2) + runner.finishAction(result2WithOutput) + runner.startAction(action3) + runner.finishAction(result3) +} + +func actionsWithOutputWithoutNewline(stat status.StatusOutput) { + result2WithOutputWithoutNewline := status.ActionResult{Action: action2, Output: "output1\noutput2"} + + runner := newRunner(stat, 3) + runner.startAction(action1) + runner.finishAction(result1) + runner.startAction(action2) + runner.finishAction(result2WithOutputWithoutNewline) + runner.startAction(action3) + runner.finishAction(result3) +} + +func actionsWithError(stat status.StatusOutput) { + action2WithError := &status.Action{Description: "action2", Outputs: []string{"f1", "f2"}, Command: "touch f1 f2"} + result2WithError := status.ActionResult{Action: action2WithError, Output: "error1\nerror2\n", Error: fmt.Errorf("error1")} + + runner := newRunner(stat, 3) + runner.startAction(action1) + runner.finishAction(result1) + runner.startAction(action2WithError) + runner.finishAction(result2WithError) + runner.startAction(action3) + runner.finishAction(result3) +} + +func actionWithEmptyDescription(stat status.StatusOutput) { + action1 := &status.Action{Command: "command1"} + result1 := status.ActionResult{Action: action1} + + runner := newRunner(stat, 1) + runner.startAction(action1) + runner.finishAction(result1) +} + +func actionsWithMessages(stat status.StatusOutput) { + runner := newRunner(stat, 2) + + runner.startAction(action1) + runner.finishAction(result1) + + stat.Message(status.VerboseLvl, "verbose") + stat.Message(status.StatusLvl, "status") + stat.Message(status.PrintLvl, "print") + stat.Message(status.ErrorLvl, "error") + + runner.startAction(action2) + runner.finishAction(result2) +} + +func actionWithLongDescription(stat status.StatusOutput) { + action1 := &status.Action{Description: "action with very long description to test eliding"} + result1 := status.ActionResult{Action: action1} + + runner := newRunner(stat, 2) + + runner.startAction(action1) + + runner.finishAction(result1) +} + +func actionWithOuptutWithAnsiCodes(stat status.StatusOutput) { + result1WithOutputWithAnsiCodes := status.ActionResult{Action: action1, Output: "\x1b[31mcolor\x1b[0m"} + + runner := newRunner(stat, 1) + runner.startAction(action1) + runner.finishAction(result1WithOutputWithAnsiCodes) +} + +func TestSmartStatusOutputWidthChange(t *testing.T) { + os.Setenv(tableHeightEnVar, "") + + smart := &fakeSmartTerminal{termWidth: 40} + stat := NewStatusOutput(smart, "", false, false) + smartStat := stat.(*smartStatusOutput) + smartStat.sigwinchHandled = make(chan bool) + + runner := newRunner(stat, 2) + + action := &status.Action{Description: "action with very long description to test eliding"} + result := status.ActionResult{Action: action} + + runner.startAction(action) + smart.termWidth = 30 + // Fake a SIGWINCH + smartStat.sigwinch <- syscall.SIGWINCH + <-smartStat.sigwinchHandled + runner.finishAction(result) + + stat.Flush() + + w := "\r\x1b[1m[ 0% 0/2] action with very long descrip\x1b[0m\x1b[K\r\x1b[1m[ 50% 1/2] action with very lo\x1b[0m\x1b[K\n" + + if g := smart.String(); g != w { + t.Errorf("want:\n%q\ngot:\n%q", w, g) + } +} diff --git a/ui/terminal/stdio.go b/ui/terminal/stdio.go new file mode 100644 index 000000000..dec296312 --- /dev/null +++ b/ui/terminal/stdio.go @@ -0,0 +1,55 @@ +// Copyright 2018 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package terminal provides a set of interfaces that can be used to interact +// with the terminal (including falling back when the terminal is detected to +// be a redirect or other dumb terminal) +package terminal + +import ( + "io" + "os" +) + +// StdioInterface represents a set of stdin/stdout/stderr Reader/Writers +type StdioInterface interface { + Stdin() io.Reader + Stdout() io.Writer + Stderr() io.Writer +} + +// StdioImpl uses the OS stdin/stdout/stderr to implement StdioInterface +type StdioImpl struct{} + +func (StdioImpl) Stdin() io.Reader { return os.Stdin } +func (StdioImpl) Stdout() io.Writer { return os.Stdout } +func (StdioImpl) Stderr() io.Writer { return os.Stderr } + +var _ StdioInterface = StdioImpl{} + +type customStdio struct { + stdin io.Reader + stdout io.Writer + stderr io.Writer +} + +func NewCustomStdio(stdin io.Reader, stdout, stderr io.Writer) StdioInterface { + return customStdio{stdin, stdout, stderr} +} + +func (c customStdio) Stdin() io.Reader { return c.stdin } +func (c customStdio) Stdout() io.Writer { return c.stdout } +func (c customStdio) Stderr() io.Writer { return c.stderr } + +var _ StdioInterface = customStdio{} diff --git a/ui/terminal/util.go b/ui/terminal/util.go index a85a517b9..7a603d7f7 100644 --- a/ui/terminal/util.go +++ b/ui/terminal/util.go @@ -22,18 +22,23 @@ import ( "unsafe" ) -func isTerminal(w io.Writer) bool { +func isSmartTerminal(w io.Writer) bool { if f, ok := w.(*os.File); ok { + if term, ok := os.LookupEnv("TERM"); ok && term == "dumb" { + return false + } var termios syscall.Termios _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, f.Fd(), ioctlGetTermios, uintptr(unsafe.Pointer(&termios)), 0, 0, 0) return err == 0 + } else if _, ok := w.(*fakeSmartTerminal); ok { + return true } return false } -func termWidth(w io.Writer) (int, bool) { +func termSize(w io.Writer) (width int, height int, ok bool) { if f, ok := w.(*os.File); ok { var winsize struct { ws_row, ws_column uint16 @@ -42,9 +47,11 @@ func termWidth(w io.Writer) (int, bool) { _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, f.Fd(), syscall.TIOCGWINSZ, uintptr(unsafe.Pointer(&winsize)), 0, 0, 0) - return int(winsize.ws_column), err == 0 + return int(winsize.ws_column), int(winsize.ws_row), err == 0 + } else if f, ok := w.(*fakeSmartTerminal); ok { + return f.termWidth, f.termHeight, true } - return 0, false + return 0, 0, false } // stripAnsiEscapes strips ANSI control codes from a byte array in place. @@ -99,3 +106,8 @@ func stripAnsiEscapes(input []byte) []byte { return input } + +type fakeSmartTerminal struct { + bytes.Buffer + termWidth, termHeight int +} diff --git a/ui/terminal/writer.go b/ui/terminal/writer.go deleted file mode 100644 index ebe4b2aad..000000000 --- a/ui/terminal/writer.go +++ /dev/null @@ -1,229 +0,0 @@ -// Copyright 2018 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package terminal provides a set of interfaces that can be used to interact -// with the terminal (including falling back when the terminal is detected to -// be a redirect or other dumb terminal) -package terminal - -import ( - "fmt" - "io" - "os" - "strings" - "sync" -) - -// Writer provides an interface to write temporary and permanent messages to -// the terminal. -// -// The terminal is considered to be a dumb terminal if TERM==dumb, or if a -// terminal isn't detected on stdout/stderr (generally because it's a pipe or -// file). Dumb terminals will strip out all ANSI escape sequences, including -// colors. -type Writer interface { - // Print prints the string to the terminal, overwriting any current - // status being displayed. - // - // On a dumb terminal, the status messages will be kept. - Print(str string) - - // Status prints the first line of the string to the terminal, - // overwriting any previous status line. Strings longer than the width - // of the terminal will be cut off. - // - // On a dumb terminal, previous status messages will remain, and the - // entire first line of the string will be printed. - StatusLine(str string) - - // StatusAndMessage prints the first line of status to the terminal, - // similarly to StatusLine(), then prints the full msg below that. The - // status line is retained. - // - // There is guaranteed to be no other output in between the status and - // message. - StatusAndMessage(status, msg string) - - // Finish ensures that the output ends with a newline (preserving any - // current status line that is current displayed). - // - // This does nothing on dumb terminals. - Finish() - - // Write implements the io.Writer interface. This is primarily so that - // the logger can use this interface to print to stderr without - // breaking the other semantics of this interface. - // - // Try to use any of the other functions if possible. - Write(p []byte) (n int, err error) - - isSmartTerminal() bool -} - -// NewWriter creates a new Writer based on the stdio and the TERM -// environment variable. -func NewWriter(stdio StdioInterface) Writer { - w := &writerImpl{ - stdio: stdio, - - haveBlankLine: true, - } - - if term, ok := os.LookupEnv("TERM"); ok && term != "dumb" { - w.smartTerminal = isTerminal(stdio.Stdout()) - } - w.stripEscapes = !w.smartTerminal - - return w -} - -type writerImpl struct { - stdio StdioInterface - - haveBlankLine bool - - // Protecting the above, we assume that smartTerminal and stripEscapes - // does not change after initial setup. - lock sync.Mutex - - smartTerminal bool - stripEscapes bool -} - -func (w *writerImpl) isSmartTerminal() bool { - return w.smartTerminal -} - -func (w *writerImpl) requestLine() { - if !w.haveBlankLine { - fmt.Fprintln(w.stdio.Stdout()) - w.haveBlankLine = true - } -} - -func (w *writerImpl) Print(str string) { - if w.stripEscapes { - str = string(stripAnsiEscapes([]byte(str))) - } - - w.lock.Lock() - defer w.lock.Unlock() - w.print(str) -} - -func (w *writerImpl) print(str string) { - if !w.haveBlankLine { - fmt.Fprint(w.stdio.Stdout(), "\r", "\x1b[K") - w.haveBlankLine = true - } - fmt.Fprint(w.stdio.Stdout(), str) - if len(str) == 0 || str[len(str)-1] != '\n' { - fmt.Fprint(w.stdio.Stdout(), "\n") - } -} - -func (w *writerImpl) StatusLine(str string) { - w.lock.Lock() - defer w.lock.Unlock() - - w.statusLine(str) -} - -func (w *writerImpl) statusLine(str string) { - if !w.smartTerminal { - fmt.Fprintln(w.stdio.Stdout(), str) - return - } - - idx := strings.IndexRune(str, '\n') - if idx != -1 { - str = str[0:idx] - } - - // Limit line width to the terminal width, otherwise we'll wrap onto - // another line and we won't delete the previous line. - // - // Run this on every line in case the window has been resized while - // we're printing. This could be optimized to only re-run when we get - // SIGWINCH if it ever becomes too time consuming. - if max, ok := termWidth(w.stdio.Stdout()); ok { - if len(str) > max { - // TODO: Just do a max. Ninja elides the middle, but that's - // more complicated and these lines aren't that important. - str = str[:max] - } - } - - // Move to the beginning on the line, print the output, then clear - // the rest of the line. - fmt.Fprint(w.stdio.Stdout(), "\r", str, "\x1b[K") - w.haveBlankLine = false -} - -func (w *writerImpl) StatusAndMessage(status, msg string) { - if w.stripEscapes { - msg = string(stripAnsiEscapes([]byte(msg))) - } - - w.lock.Lock() - defer w.lock.Unlock() - - w.statusLine(status) - w.requestLine() - w.print(msg) -} - -func (w *writerImpl) Finish() { - w.lock.Lock() - defer w.lock.Unlock() - - w.requestLine() -} - -func (w *writerImpl) Write(p []byte) (n int, err error) { - w.Print(string(p)) - return len(p), nil -} - -// StdioInterface represents a set of stdin/stdout/stderr Reader/Writers -type StdioInterface interface { - Stdin() io.Reader - Stdout() io.Writer - Stderr() io.Writer -} - -// StdioImpl uses the OS stdin/stdout/stderr to implement StdioInterface -type StdioImpl struct{} - -func (StdioImpl) Stdin() io.Reader { return os.Stdin } -func (StdioImpl) Stdout() io.Writer { return os.Stdout } -func (StdioImpl) Stderr() io.Writer { return os.Stderr } - -var _ StdioInterface = StdioImpl{} - -type customStdio struct { - stdin io.Reader - stdout io.Writer - stderr io.Writer -} - -func NewCustomStdio(stdin io.Reader, stdout, stderr io.Writer) StdioInterface { - return customStdio{stdin, stdout, stderr} -} - -func (c customStdio) Stdin() io.Reader { return c.stdin } -func (c customStdio) Stdout() io.Writer { return c.stdout } -func (c customStdio) Stderr() io.Writer { return c.stderr } - -var _ StdioInterface = customStdio{} diff --git a/ui/tracer/status.go b/ui/tracer/status.go index af50e2d4d..c83125514 100644 --- a/ui/tracer/status.go +++ b/ui/tracer/status.go @@ -85,3 +85,8 @@ func (s *statusOutput) FinishAction(result status.ActionResult, counts status.Co func (s *statusOutput) Flush() {} func (s *statusOutput) Message(level status.MsgLevel, message string) {} + +func (s *statusOutput) Write(p []byte) (int, error) { + // Discard writes + return len(p), nil +} diff --git a/vnames.go.json b/vnames.go.json new file mode 100644 index 000000000..584209735 --- /dev/null +++ b/vnames.go.json @@ -0,0 +1,9 @@ +[ + { + "pattern": "(.*)", + "vname": { + "corpus": "android.googlesource.com/platform/superproject", + "path": "build/soong/@1@" + } + } +] diff --git a/vnames.json b/vnames.json new file mode 100644 index 000000000..f9d3adc8b --- /dev/null +++ b/vnames.json @@ -0,0 +1,18 @@ +[ + { + "pattern": "out/(.*)", + "vname": { + "corpus": "android.googlesource.com/platform/superproject", + "root": "out", + "path": "@1@" + } + }, + { + "pattern": "(.*)", + "vname": { + "corpus": "android.googlesource.com/platform/superproject", + "path": "@1@" + } + } +] + diff --git a/xml/Android.bp b/xml/Android.bp new file mode 100644 index 000000000..cd25cff7b --- /dev/null +++ b/xml/Android.bp @@ -0,0 +1,18 @@ +bootstrap_go_package { + name: "soong-xml", + pkgPath: "android/soong/xml", + deps: [ + "blueprint", + "blueprint-pathtools", + "soong", + "soong-android", + "soong-etc", + ], + srcs: [ + "xml.go", + ], + testSrcs: [ + "xml_test.go", + ], + pluginFor: ["soong_build"], +} diff --git a/xml/xml.go b/xml/xml.go index 7c670fb8f..8810ae4d5 100644 --- a/xml/xml.go +++ b/xml/xml.go @@ -16,6 +16,7 @@ package xml import ( "android/soong/android" + "android/soong/etc" "github.com/google/blueprint" "github.com/google/blueprint/proptools" @@ -62,7 +63,7 @@ type prebuiltEtcXmlProperties struct { } type prebuiltEtcXml struct { - android.PrebuiltEtc + etc.PrebuiltEtc properties prebuiltEtcXmlProperties } @@ -71,10 +72,6 @@ func (p *prebuiltEtcXml) timestampFilePath(ctx android.ModuleContext) android.Wr return android.PathForModuleOut(ctx, p.PrebuiltEtc.SourceFilePath(ctx).Base()+"-timestamp") } -func (p *prebuiltEtcXml) DepsMutator(ctx android.BottomUpMutatorContext) { - p.PrebuiltEtc.DepsMutator(ctx) -} - func (p *prebuiltEtcXml) GenerateAndroidBuildActions(ctx android.ModuleContext) { p.PrebuiltEtc.GenerateAndroidBuildActions(ctx) @@ -125,9 +122,8 @@ func (p *prebuiltEtcXml) GenerateAndroidBuildActions(ctx android.ModuleContext) func PrebuiltEtcXmlFactory() android.Module { module := &prebuiltEtcXml{} module.AddProperties(&module.properties) - - android.InitPrebuiltEtcModule(&module.PrebuiltEtc) + etc.InitPrebuiltEtcModule(&module.PrebuiltEtc, "etc") // This module is device-only - android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibCommon) + android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibFirst) return module } diff --git a/xml/xml_test.go b/xml/xml_test.go index e8fa49c2b..abcb1085f 100644 --- a/xml/xml_test.go +++ b/xml/xml_test.go @@ -15,27 +15,52 @@ package xml import ( - "android/soong/android" "io/ioutil" "os" "testing" + + "android/soong/android" + "android/soong/etc" ) +var buildDir string + +func setUp() { + var err error + buildDir, err = ioutil.TempDir("", "soong_xml_test") + if err != nil { + panic(err) + } +} + +func tearDown() { + os.RemoveAll(buildDir) +} + +func TestMain(m *testing.M) { + run := func() int { + setUp() + defer tearDown() + + return m.Run() + } + + os.Exit(run()) +} + func testXml(t *testing.T, bp string) *android.TestContext { - config, buildDir := setup(t) - defer teardown(buildDir) - ctx := android.NewTestArchContext() - ctx.RegisterModuleType("prebuilt_etc", android.ModuleFactoryAdaptor(android.PrebuiltEtcFactory)) - ctx.RegisterModuleType("prebuilt_etc_xml", android.ModuleFactoryAdaptor(PrebuiltEtcXmlFactory)) - ctx.Register() - mockFiles := map[string][]byte{ - "Android.bp": []byte(bp), - "foo.xml": nil, - "foo.dtd": nil, - "bar.xml": nil, - "bar.xsd": nil, + fs := map[string][]byte{ + "foo.xml": nil, + "foo.dtd": nil, + "bar.xml": nil, + "bar.xsd": nil, + "baz.xml": nil, } - ctx.MockFileSystem(mockFiles) + config := android.TestArchConfig(buildDir, nil, bp, fs) + ctx := android.NewTestArchContext() + ctx.RegisterModuleType("prebuilt_etc", etc.PrebuiltEtcFactory) + ctx.RegisterModuleType("prebuilt_etc_xml", PrebuiltEtcXmlFactory) + ctx.Register(config) _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) android.FailIfErrored(t, errs) _, errs = ctx.PrepareBuildActions(config) @@ -44,19 +69,11 @@ func testXml(t *testing.T, bp string) *android.TestContext { return ctx } -func setup(t *testing.T) (config android.Config, buildDir string) { - buildDir, err := ioutil.TempDir("", "soong_xml_test") - if err != nil { - t.Fatal(err) +func assertEqual(t *testing.T, name, expected, actual string) { + t.Helper() + if expected != actual { + t.Errorf(name+" expected %q != got %q", expected, actual) } - - config = android.TestArchConfig(buildDir, nil) - - return -} - -func teardown(buildDir string) { - os.RemoveAll(buildDir) } // Minimal test @@ -72,15 +89,28 @@ func TestPrebuiltEtcXml(t *testing.T) { src: "bar.xml", schema: "bar.xsd", } + prebuilt_etc_xml { + name: "baz.xml", + src: "baz.xml", + } `) - xmllint := ctx.ModuleForTests("foo.xml", "android_common").Rule("xmllint") - input := xmllint.Input.String() - if input != "foo.xml" { - t.Errorf("input expected %q != got %q", "foo.xml", input) - } - schema := xmllint.Args["dtd"] - if schema != "foo.dtd" { - t.Errorf("dtd expected %q != got %q", "foo.dtdl", schema) + for _, tc := range []struct { + rule, input, schemaType, schema string + }{ + {rule: "xmllint-dtd", input: "foo.xml", schemaType: "dtd", schema: "foo.dtd"}, + {rule: "xmllint-xsd", input: "bar.xml", schemaType: "xsd", schema: "bar.xsd"}, + {rule: "xmllint-minimal", input: "baz.xml"}, + } { + t.Run(tc.schemaType, func(t *testing.T) { + rule := ctx.ModuleForTests(tc.input, "android_arm64_armv8-a").Rule(tc.rule) + assertEqual(t, "input", tc.input, rule.Input.String()) + if tc.schemaType != "" { + assertEqual(t, "schema", tc.schema, rule.Args[tc.schemaType]) + } + }) } + + m := ctx.ModuleForTests("foo.xml", "android_arm64_armv8-a").Module().(*prebuiltEtcXml) + assertEqual(t, "installDir", buildDir+"/target/product/test_device/system/etc", m.InstallDirPath().String()) } diff --git a/zip/cmd/main.go b/zip/cmd/main.go index b4f75f7af..fba2e4b1e 100644 --- a/zip/cmd/main.go +++ b/zip/cmd/main.go @@ -105,12 +105,6 @@ var ( nonDeflatedFiles = make(uniqueSet) ) -func usage() { - fmt.Fprintf(os.Stderr, "usage: soong_zip -o zipfile [-m manifest] [-C dir] [-f|-l file] [-D dir]...\n") - flag.PrintDefaults() - os.Exit(2) -} - func main() { var expandedArgs []string for _, arg := range os.Args { @@ -128,7 +122,11 @@ func main() { } flags := flag.NewFlagSet("flags", flag.ExitOnError) - flags.Usage = usage + flags.Usage = func() { + fmt.Fprintf(os.Stderr, "usage: soong_zip -o zipfile [-m manifest] [-C dir] [-f|-l file] [-D dir]...\n") + flags.PrintDefaults() + os.Exit(2) + } out := flags.String("o", "", "file to write zip file to") manifest := flags.String("m", "", "input jar manifest file name") @@ -138,6 +136,7 @@ func main() { writeIfChanged := flags.Bool("write_if_changed", false, "only update resultant .zip if it has changed") ignoreMissingFiles := flags.Bool("ignore_missing_files", false, "continue if a requested file does not exist") symlinks := flags.Bool("symlinks", true, "store symbolic links in zip instead of following them") + srcJar := flags.Bool("srcjar", false, "move .java files to locations that match their package statement") parallelJobs := flags.Int("parallel", runtime.NumCPU(), "number of parallel threads to use") cpuProfile := flags.String("cpuprofile", "", "write cpu profile to file") @@ -193,6 +192,7 @@ func main() { FileArgs: fileArgsBuilder.FileArgs(), OutputFilePath: *out, EmulateJar: *emulateJar, + SrcJar: *srcJar, AddDirectoryEntriesToZip: *directories, CompressionLevel: *compLevel, ManifestSourcePath: *manifest, diff --git a/zip/zip.go b/zip/zip.go index 1f5fe4335..3c710a782 100644 --- a/zip/zip.go +++ b/zip/zip.go @@ -145,7 +145,7 @@ func (b *FileArgsBuilder) List(name string) *FileArgsBuilder { } arg := b.state - arg.SourceFiles = strings.Split(string(list), "\n") + arg.SourceFiles = strings.Fields(string(list)) b.fileArgs = append(b.fileArgs, arg) return b } @@ -210,6 +210,7 @@ type ZipArgs struct { FileArgs []FileArg OutputFilePath string EmulateJar bool + SrcJar bool AddDirectoryEntriesToZip bool CompressionLevel int ManifestSourcePath string @@ -364,7 +365,7 @@ func ZipTo(args ZipArgs, w io.Writer) error { } } - return z.write(w, pathMappings, args.ManifestSourcePath, args.EmulateJar, args.NumParallelJobs) + return z.write(w, pathMappings, args.ManifestSourcePath, args.EmulateJar, args.SrcJar, args.NumParallelJobs) } func Zip(args ZipArgs) error { @@ -446,7 +447,9 @@ func jarSort(mappings []pathMapping) { sort.SliceStable(mappings, less) } -func (z *ZipWriter) write(f io.Writer, pathMappings []pathMapping, manifest string, emulateJar bool, parallelJobs int) error { +func (z *ZipWriter) write(f io.Writer, pathMappings []pathMapping, manifest string, emulateJar, srcJar bool, + parallelJobs int) error { + z.errors = make(chan error) defer close(z.errors) @@ -489,7 +492,7 @@ func (z *ZipWriter) write(f io.Writer, pathMappings []pathMapping, manifest stri if emulateJar && ele.dest == jar.ManifestFile { err = z.addManifest(ele.dest, ele.src, ele.zipMethod) } else { - err = z.addFile(ele.dest, ele.src, ele.zipMethod, emulateJar) + err = z.addFile(ele.dest, ele.src, ele.zipMethod, emulateJar, srcJar) } if err != nil { z.errors <- err @@ -588,7 +591,7 @@ func (z *ZipWriter) write(f io.Writer, pathMappings []pathMapping, manifest stri } // imports (possibly with compression) <src> into the zip at sub-path <dest> -func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar bool) error { +func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar, srcJar bool) error { var fileSize int64 var executable bool @@ -606,12 +609,9 @@ func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar bool) er return nil } return err - } else if s.IsDir() { - if z.directories { - return z.writeDirectory(dest, src, emulateJar) - } - return nil - } else { + } + + createParentDirs := func(dest, src string) error { if err := z.writeDirectory(filepath.Dir(dest), src, emulateJar); err != nil { return err } @@ -625,32 +625,64 @@ func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar bool) er z.createdFiles[dest] = src - if s.Mode()&os.ModeSymlink != 0 { - return z.writeSymlink(dest, src) - } else if !s.Mode().IsRegular() { - return fmt.Errorf("%s is not a file, directory, or symlink", src) + return nil + } + + if s.IsDir() { + if z.directories { + return z.writeDirectory(dest, src, emulateJar) + } + return nil + } else if s.Mode()&os.ModeSymlink != 0 { + err = createParentDirs(dest, src) + if err != nil { + return err + } + + return z.writeSymlink(dest, src) + } else if s.Mode().IsRegular() { + r, err := z.fs.Open(src) + if err != nil { + return err + } + + if srcJar && filepath.Ext(src) == ".java" { + // rewrite the destination using the package path if it can be determined + pkg, err := jar.JavaPackage(r, src) + if err != nil { + // ignore errors for now, leaving the file at in its original location in the zip + } else { + dest = filepath.Join(filepath.Join(strings.Split(pkg, ".")...), filepath.Base(src)) + } + + _, err = r.Seek(0, io.SeekStart) + if err != nil { + return err + } } fileSize = s.Size() executable = s.Mode()&0100 != 0 - } - r, err := z.fs.Open(src) - if err != nil { - return err - } + header := &zip.FileHeader{ + Name: dest, + Method: method, + UncompressedSize64: uint64(fileSize), + } - header := &zip.FileHeader{ - Name: dest, - Method: method, - UncompressedSize64: uint64(fileSize), - } + if executable { + header.SetMode(0700) + } - if executable { - header.SetMode(0700) - } + err = createParentDirs(dest, src) + if err != nil { + return err + } - return z.writeFileContents(header, r) + return z.writeFileContents(header, r) + } else { + return fmt.Errorf("%s is not a file, directory, or symlink", src) + } } func (z *ZipWriter) addManifest(dest string, src string, method uint16) error { diff --git a/zip/zip_test.go b/zip/zip_test.go index 93c5f3dee..9705d6c49 100644 --- a/zip/zip_test.go +++ b/zip/zip_test.go @@ -40,14 +40,16 @@ var ( ) var mockFs = pathtools.MockFs(map[string][]byte{ - "a/a/a": fileA, - "a/a/b": fileB, - "a/a/c -> ../../c": nil, - "a/a/d -> b": nil, - "c": fileC, - "l": []byte("a/a/a\na/a/b\nc\n"), - "l2": []byte("missing\n"), - "manifest.txt": fileCustomManifest, + "a/a/a": fileA, + "a/a/b": fileB, + "a/a/c -> ../../c": nil, + "dangling -> missing": nil, + "a/a/d -> b": nil, + "c": fileC, + "l_nl": []byte("a/a/a\na/a/b\nc\n"), + "l_sp": []byte("a/a/a a/a/b c"), + "l2": []byte("missing\n"), + "manifest.txt": fileCustomManifest, }) func fh(name string, contents []byte, method uint16) zip.FileHeader { @@ -210,9 +212,32 @@ func TestZip(t *testing.T) { }, }, { + name: "dangling symlinks", + args: fileArgsBuilder(). + File("dangling"), + compressionLevel: 9, + storeSymlinks: true, + + files: []zip.FileHeader{ + fhLink("dangling", "missing"), + }, + }, + { + name: "list", + args: fileArgsBuilder(). + List("l_nl"), + compressionLevel: 9, + + files: []zip.FileHeader{ + fh("a/a/a", fileA, zip.Deflate), + fh("a/a/b", fileB, zip.Deflate), + fh("c", fileC, zip.Deflate), + }, + }, + { name: "list", args: fileArgsBuilder(). - List("l"), + List("l_sp"), compressionLevel: 9, files: []zip.FileHeader{ @@ -554,3 +579,70 @@ func TestReadRespFile(t *testing.T) { }) } } + +func TestSrcJar(t *testing.T) { + mockFs := pathtools.MockFs(map[string][]byte{ + "wrong_package.java": []byte("package foo;"), + "foo/correct_package.java": []byte("package foo;"), + "src/no_package.java": nil, + "src2/parse_error.java": []byte("error"), + }) + + want := []string{ + "foo/", + "foo/wrong_package.java", + "foo/correct_package.java", + "no_package.java", + "src2/", + "src2/parse_error.java", + } + + args := ZipArgs{} + args.FileArgs = NewFileArgsBuilder().File("**/*.java").FileArgs() + + args.SrcJar = true + args.AddDirectoryEntriesToZip = true + args.Filesystem = mockFs + args.Stderr = &bytes.Buffer{} + + buf := &bytes.Buffer{} + err := ZipTo(args, buf) + if err != nil { + t.Fatalf("got error %v", err) + } + + br := bytes.NewReader(buf.Bytes()) + zr, err := zip.NewReader(br, int64(br.Len())) + if err != nil { + t.Fatal(err) + } + + var got []string + for _, f := range zr.File { + r, err := f.Open() + if err != nil { + t.Fatalf("error when opening %s: %s", f.Name, err) + } + + crc := crc32.NewIEEE() + len, err := io.Copy(crc, r) + r.Close() + if err != nil { + t.Fatalf("error when reading %s: %s", f.Name, err) + } + + if uint64(len) != f.UncompressedSize64 { + t.Errorf("incorrect length for %s, want %d got %d", f.Name, f.UncompressedSize64, len) + } + + if crc.Sum32() != f.CRC32 { + t.Errorf("incorrect crc for %s, want %x got %x", f.Name, f.CRC32, crc) + } + + got = append(got, f.Name) + } + + if !reflect.DeepEqual(want, got) { + t.Errorf("want files %q, got %q", want, got) + } +} |